Merge branch 'simp/state' into simp/integration
This commit is contained in:
+193
-289
@@ -107,11 +107,8 @@ def active_session_limit_message(
|
||||
max_sessions: int,
|
||||
entries: Optional[list[dict[str, Any]]] = None,
|
||||
) -> str:
|
||||
# Name the holders: the slots are shared across CLI, desktop/TUI and the
|
||||
# messaging gateway, so the surface that gets rejected is usually NOT the
|
||||
# one squatting on them (idle desktop chats starving a Discord bot, say).
|
||||
# Without this the message is unactionable and the only way to find out is
|
||||
# reading runtime/active_sessions.json by hand.
|
||||
# Name the holders: slots are shared across CLI, desktop/TUI and gateway,
|
||||
# so the rejected surface is usually NOT the one squatting on them.
|
||||
held = summarize_holders(entries or [])
|
||||
detail = f" Held by: {held}." if held else ""
|
||||
return (
|
||||
@@ -124,42 +121,25 @@ def _registry_home(registry_home: str | Path | None = None) -> Path:
|
||||
return Path(registry_home) if registry_home is not None else Path(get_hermes_home())
|
||||
|
||||
|
||||
# WHY A REFUSAL IS REFUSED, in a form a caller can branch on.
|
||||
#
|
||||
# The two refusals mean opposite things to an automated client. Capacity is
|
||||
# "the machine is busy, come back later". Ownership is "this specific session
|
||||
# has a live owner, and writing to it would interleave with theirs".
|
||||
#
|
||||
# Callers used to have only the human-readable message, so anything that needed
|
||||
# to DECIDE had to match prose -- which silently changes meaning whenever the
|
||||
# wording is improved. The reason is the contract; the message is for people.
|
||||
# Machine-readable refusal reasons: the reason is the contract, the message is
|
||||
# for people. Capacity = "busy, come back later"; ownership = "this session
|
||||
# has a live owner and writing would interleave with theirs".
|
||||
SESSION_NOT_OWNED = "SESSION_NOT_OWNED"
|
||||
MAX_CONCURRENT_SESSIONS = "MAX_CONCURRENT_SESSIONS"
|
||||
# Ownership could not be PROVEN either way: the registry was unreadable or
|
||||
# corrupt. Distinct from SESSION_NOT_OWNED on purpose -- "someone else owns
|
||||
# this" and "I cannot tell who owns this" call for different operator action,
|
||||
# and collapsing the second into a silent go-ahead is exactly the fail-open
|
||||
# hole that let two writers share one session (#94595 review, blocker 2).
|
||||
# Ownership could not be PROVEN (registry unreadable/corrupt). Deliberately
|
||||
# distinct from SESSION_NOT_OWNED: collapsing "can't tell" into a silent
|
||||
# go-ahead is the fail-open hole that let two writers share one session.
|
||||
SESSION_COORDINATION_UNAVAILABLE = "SESSION_COORDINATION_UNAVAILABLE"
|
||||
|
||||
# Advertised through the gateway so a client can tell a build that enforces
|
||||
# per-session exclusivity from one that does not.
|
||||
#
|
||||
# A module constant rather than a config flag, deliberately: it is true because
|
||||
# try_acquire_active_session below performs the check atomically, so it cannot
|
||||
# be turned on by an operator who has not got the enforcement, and cannot drift
|
||||
# out of step with it without this file changing.
|
||||
# Advertised through the gateway. A module constant, not a config flag: it is
|
||||
# true because try_acquire_active_session checks atomically, so it cannot drift
|
||||
# out of step with the enforcement without this file changing.
|
||||
PER_SESSION_EXCLUSIVE_SUBMIT = True
|
||||
|
||||
|
||||
class ActiveSessionRefusal(str):
|
||||
"""A refusal message that also carries a machine-readable ``reason``.
|
||||
|
||||
A ``str`` subclass so every existing caller keeps working untouched -- they
|
||||
format it, hand it back as a JSON-RPC message, or just test it for None --
|
||||
while a caller that must act on WHICH refusal happened reads ``.reason``
|
||||
instead of matching the wording.
|
||||
"""
|
||||
"""A refusal message (``str`` subclass, so existing callers are untouched)
|
||||
that also carries a machine-readable ``reason``."""
|
||||
|
||||
reason: str
|
||||
|
||||
@@ -170,13 +150,10 @@ class ActiveSessionRefusal(str):
|
||||
|
||||
|
||||
def _is_same_writer(entry: dict[str, Any], metadata: Optional[dict[str, Any]]) -> bool:
|
||||
"""True when an existing lease belongs to the very caller now re-acquiring it.
|
||||
|
||||
Both halves are required. A pid alone would let two live sessions in one
|
||||
process steal each other's lease -- which is a real hazard, since each holds
|
||||
its own snapshot of the transcript. A live session id alone would let another
|
||||
process with a coincidentally equal id do the same.
|
||||
"""
|
||||
"""True when an existing lease belongs to the very caller re-acquiring it.
|
||||
Identity is (pid, live_session_id): pid alone would let two live sessions in
|
||||
one process steal each other's lease; the live id alone would let another
|
||||
process with an equal id do the same."""
|
||||
try:
|
||||
if int(entry.get("pid") or -1) != os.getpid():
|
||||
return False
|
||||
@@ -223,6 +200,19 @@ def _lease_paths(
|
||||
return _state_path(home), _lock_path(home)
|
||||
|
||||
|
||||
def _flock(fh, *, lock: bool) -> None:
|
||||
"""Exclusive whole-file lock/unlock on ``fh`` (fcntl on POSIX, msvcrt on Windows)."""
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
fh.seek(0)
|
||||
msvcrt.locking(fh.fileno(), msvcrt.LK_LOCK if lock else msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(fh.fileno(), fcntl.LOCK_EX if lock else fcntl.LOCK_UN)
|
||||
|
||||
|
||||
class _FileLock:
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
@@ -231,45 +221,21 @@ class _FileLock:
|
||||
def __enter__(self):
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._fh = open(self.path, "a+b")
|
||||
if os.name == "nt":
|
||||
try:
|
||||
import msvcrt
|
||||
|
||||
self._fh.seek(0)
|
||||
msvcrt.locking(self._fh.fileno(), msvcrt.LK_LOCK, 1)
|
||||
except Exception as exc:
|
||||
self._fh.close()
|
||||
self._fh = None
|
||||
raise RuntimeError("active session file lock unavailable") from exc
|
||||
else:
|
||||
try:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(self._fh.fileno(), fcntl.LOCK_EX)
|
||||
except Exception as exc:
|
||||
self._fh.close()
|
||||
self._fh = None
|
||||
raise RuntimeError("active session file lock unavailable") from exc
|
||||
try:
|
||||
_flock(self._fh, lock=True)
|
||||
except Exception as exc:
|
||||
self._fh.close()
|
||||
self._fh = None
|
||||
raise RuntimeError("active session file lock unavailable") from exc
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
if self._fh is None:
|
||||
return
|
||||
if os.name == "nt":
|
||||
try:
|
||||
import msvcrt
|
||||
|
||||
self._fh.seek(0)
|
||||
msvcrt.locking(self._fh.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(self._fh.fileno(), fcntl.LOCK_UN)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
_flock(self._fh, lock=False)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self._fh.close()
|
||||
finally:
|
||||
@@ -306,58 +272,51 @@ def _read_entries(path: Path, *, strict: bool = False) -> list[dict[str, Any]]:
|
||||
seen_leases: set[str] = set()
|
||||
for entry in valid:
|
||||
lease_id = entry.get("lease_id")
|
||||
session_id = entry.get("session_id")
|
||||
pid = entry.get("pid")
|
||||
if not isinstance(lease_id, str) or not lease_id.strip():
|
||||
raise ActiveSessionRegistryError(
|
||||
f"active session registry contains an invalid lease id: {path}"
|
||||
)
|
||||
if lease_id in seen_leases:
|
||||
raise ActiveSessionRegistryError(
|
||||
f"active session registry contains a duplicate lease id: {path}"
|
||||
)
|
||||
seen_leases.add(lease_id)
|
||||
if not isinstance(session_id, str) or not session_id.strip():
|
||||
raise ActiveSessionRegistryError(
|
||||
f"active session registry contains an invalid session id: {path}"
|
||||
)
|
||||
if isinstance(pid, bool) or not isinstance(pid, (int, str)):
|
||||
pid_int = 0
|
||||
else:
|
||||
try:
|
||||
pid_int = int(pid)
|
||||
except (TypeError, ValueError):
|
||||
pid_int = 0
|
||||
if pid_int <= 0:
|
||||
raise ActiveSessionRegistryError(
|
||||
f"active session registry contains an invalid pid: {path}"
|
||||
)
|
||||
surface = entry.get("surface")
|
||||
if surface is not None and not isinstance(surface, str):
|
||||
raise ActiveSessionRegistryError(
|
||||
f"active session registry contains an invalid surface: {path}"
|
||||
)
|
||||
tracked = entry.get("track_liveness")
|
||||
if tracked is not None and not isinstance(tracked, bool):
|
||||
raise ActiveSessionRegistryError(
|
||||
f"active session registry contains an invalid liveness marker: {path}"
|
||||
)
|
||||
metadata = entry.get("metadata")
|
||||
if metadata is not None and not isinstance(metadata, dict):
|
||||
raise ActiveSessionRegistryError(
|
||||
f"active session registry contains invalid metadata: {path}"
|
||||
)
|
||||
process_start = entry.get("process_start_time")
|
||||
parsed_process_start = _optional_float(process_start)
|
||||
if process_start not in (None, "") and (
|
||||
parsed_process_start is None or not math.isfinite(parsed_process_start)
|
||||
# (problem-if-True predicate, message fragment) — checked lazily, in
|
||||
# this order, so an unhashable lease id is reported before the dup check.
|
||||
for bad, what in (
|
||||
(lambda: not _nonblank_str(lease_id), "an invalid lease id"),
|
||||
(lambda: lease_id in seen_leases, "a duplicate lease id"),
|
||||
(lambda: not _nonblank_str(entry.get("session_id")), "an invalid session id"),
|
||||
(lambda: _registry_pid(entry.get("pid")) <= 0, "an invalid pid"),
|
||||
(lambda: not _optional_isinstance(entry.get("surface"), str), "an invalid surface"),
|
||||
(lambda: not _optional_isinstance(entry.get("track_liveness"), bool), "an invalid liveness marker"),
|
||||
(lambda: not _optional_isinstance(entry.get("metadata"), dict), "invalid metadata"),
|
||||
(lambda: not _valid_process_start(entry.get("process_start_time")), "an invalid process start time"),
|
||||
):
|
||||
raise ActiveSessionRegistryError(
|
||||
f"active session registry contains an invalid process start time: {path}"
|
||||
)
|
||||
if bad():
|
||||
raise ActiveSessionRegistryError(
|
||||
f"active session registry contains {what}: {path}"
|
||||
)
|
||||
seen_leases.add(lease_id)
|
||||
return valid
|
||||
|
||||
|
||||
def _nonblank_str(v: Any) -> bool:
|
||||
return isinstance(v, str) and bool(v.strip())
|
||||
|
||||
|
||||
def _optional_isinstance(v: Any, typ) -> bool:
|
||||
return v is None or isinstance(v, typ)
|
||||
|
||||
|
||||
def _registry_pid(pid: Any) -> int:
|
||||
"""Registry pid as int; 0 for bools, non-int/str, or unparseable values."""
|
||||
if isinstance(pid, bool) or not isinstance(pid, (int, str)):
|
||||
return 0
|
||||
try:
|
||||
return int(pid)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _valid_process_start(v: Any) -> bool:
|
||||
if v in (None, ""):
|
||||
return True
|
||||
parsed = _optional_float(v)
|
||||
return parsed is not None and math.isfinite(parsed)
|
||||
|
||||
|
||||
def _write_entries(path: Path, entries: list[dict[str, Any]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_name(f"{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
|
||||
@@ -392,20 +351,24 @@ def _optional_float(value: Any) -> Optional[float]:
|
||||
return None
|
||||
|
||||
|
||||
def _pid_liveness(pid: Any, process_start_time: Any = None) -> Optional[bool]:
|
||||
"""Return True/False for live/dead, or None when liveness is unknowable."""
|
||||
def _pid_liveness(pid: Any, process_start_time: Any = None, *, lenient: bool = False) -> Optional[bool]:
|
||||
"""Return True/False for live/dead, or None when liveness is unknowable.
|
||||
|
||||
``lenient`` never returns None: an unparseable pid or a failed existence
|
||||
probe counts as dead, an unreadable current start time as alive.
|
||||
"""
|
||||
try:
|
||||
pid_int = int(pid)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return False if lenient else None
|
||||
if pid_int <= 0:
|
||||
return None
|
||||
return False if lenient else None
|
||||
try:
|
||||
from gateway.status import _pid_exists
|
||||
|
||||
exists = bool(_pid_exists(pid_int))
|
||||
except Exception:
|
||||
return None
|
||||
return False if lenient else None
|
||||
if not exists:
|
||||
return False
|
||||
expected_start = _optional_float(process_start_time)
|
||||
@@ -413,32 +376,12 @@ def _pid_liveness(pid: Any, process_start_time: Any = None) -> Optional[bool]:
|
||||
return True
|
||||
current_start = _process_start_time(pid_int)
|
||||
if current_start is None:
|
||||
return None
|
||||
return True if lenient else None
|
||||
return abs(current_start - expected_start) < 0.001
|
||||
|
||||
|
||||
def _pid_alive(pid: Any, process_start_time: Any = None) -> bool:
|
||||
try:
|
||||
pid_int = int(pid)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if pid_int <= 0:
|
||||
return False
|
||||
try:
|
||||
from gateway.status import _pid_exists
|
||||
|
||||
exists = bool(_pid_exists(pid_int))
|
||||
except Exception:
|
||||
return False
|
||||
if not exists:
|
||||
return False
|
||||
expected_start = _optional_float(process_start_time)
|
||||
if expected_start is None:
|
||||
return True
|
||||
current_start = _process_start_time(pid_int)
|
||||
if current_start is None:
|
||||
return True
|
||||
return abs(current_start - expected_start) < 0.001
|
||||
return bool(_pid_liveness(pid, process_start_time, lenient=True))
|
||||
|
||||
|
||||
def _prune_dead(
|
||||
@@ -468,12 +411,9 @@ class ActiveSessionLease:
|
||||
surface: str
|
||||
enabled: bool = True
|
||||
released: bool = False
|
||||
# Registry paths pinned at acquisition time. A lease acquired under the
|
||||
# root ``HERMES_HOME`` must release against the same registry even when
|
||||
# ``release()`` runs inside a profile home override (native multiplex
|
||||
# routes turns under ``_profile_runtime_scope``), otherwise the root
|
||||
# entry survives until process exit and the session cap fills with
|
||||
# phantom leases (#85431).
|
||||
# Pinned at acquisition: a lease acquired under the root HERMES_HOME must
|
||||
# release against the same registry even when release() runs inside a
|
||||
# profile-home override, or phantom leases fill the session cap.
|
||||
state_path: Optional[Path] = None
|
||||
lock_path: Optional[Path] = None
|
||||
track_liveness: bool = False
|
||||
@@ -484,6 +424,33 @@ class ActiveSessionLease:
|
||||
release_active_session(self)
|
||||
|
||||
|
||||
def _clean_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
|
||||
return {str(k): v for k, v in metadata.items() if isinstance(k, str)}
|
||||
|
||||
|
||||
def _without_lease(entries: list[dict[str, Any]], lease_id: str) -> list[dict[str, Any]]:
|
||||
return [e for e in entries if str(e.get("lease_id") or "") != lease_id]
|
||||
|
||||
|
||||
def _read_live_entries(
|
||||
state_path: Path, *, track_liveness: bool, warn: str,
|
||||
) -> Optional[tuple[list[dict[str, Any]], list[dict[str, Any]]]]:
|
||||
"""``(raw, pruned)`` from the registry, or None when it is unreadable.
|
||||
|
||||
Liveness-tracked callers re-raise instead: they must not proceed on an
|
||||
unprovable registry. Untracked callers get ``warn`` logged and decide
|
||||
how to degrade themselves.
|
||||
"""
|
||||
try:
|
||||
raw_entries = _read_entries(state_path, strict=True)
|
||||
return raw_entries, _prune_dead(raw_entries, strict=track_liveness)
|
||||
except ActiveSessionRegistryError:
|
||||
if track_liveness:
|
||||
raise
|
||||
logger.warning(warn)
|
||||
return None
|
||||
|
||||
|
||||
def _lease_entry(
|
||||
*,
|
||||
lease_id: str,
|
||||
@@ -505,9 +472,7 @@ def _lease_entry(
|
||||
if track_liveness:
|
||||
entry["track_liveness"] = True
|
||||
if metadata:
|
||||
entry["metadata"] = {
|
||||
str(k): v for k, v in metadata.items() if isinstance(k, str)
|
||||
}
|
||||
entry["metadata"] = _clean_metadata(metadata)
|
||||
return entry
|
||||
|
||||
|
||||
@@ -522,27 +487,20 @@ def try_acquire_active_session(
|
||||
) -> tuple[Optional[ActiveSessionLease], Optional[str]]:
|
||||
"""Acquire an active-session slot.
|
||||
|
||||
Per-session exclusivity is CORRECTNESS and is enforced unconditionally:
|
||||
at most one live owner may run a given stored session, whether or not an
|
||||
operator configured ``max_concurrent_sessions`` (#94595). The concurrency
|
||||
cap remains a resource POLICY and applies only when configured. Liveness
|
||||
tracking keeps richer desktop lifecycle semantics; ``registry_home`` lets
|
||||
profile-scoped backends share the owning profile's registry even when
|
||||
launched from another home.
|
||||
Per-session exclusivity is CORRECTNESS, enforced unconditionally: at most
|
||||
one live owner per stored session. ``max_concurrent_sessions`` is resource
|
||||
POLICY and applies only when configured. ``registry_home`` lets
|
||||
profile-scoped backends share the owning profile's registry.
|
||||
|
||||
Returns ``(lease, None)`` on success and ``(None, refusal)`` otherwise,
|
||||
where ``refusal`` is an :class:`ActiveSessionRefusal` carrying a
|
||||
machine-readable ``reason``. Ownership uncertainty fails CLOSED: when the
|
||||
registry cannot be read, the caller gets ``SESSION_COORDINATION_UNAVAILABLE``
|
||||
rather than a silent go-ahead that could reopen the double-writer state.
|
||||
Returns ``(lease, None)`` or ``(None, ActiveSessionRefusal)``. Ownership
|
||||
uncertainty fails CLOSED with ``SESSION_COORDINATION_UNAVAILABLE``.
|
||||
"""
|
||||
max_sessions = resolve_max_concurrent_sessions(config)
|
||||
lease_id = uuid.uuid4().hex
|
||||
key = str(session_id or "")
|
||||
|
||||
# A session with no stored id yet cannot collide with another writer, and
|
||||
# the strict registry schema (rightly) refuses entries with empty session
|
||||
# ids. Nothing to fence, nothing to record: hand back a no-op lease.
|
||||
# No stored id yet => nothing to fence or record (and the strict schema
|
||||
# refuses empty session ids): hand back a no-op lease.
|
||||
if not key and not track_liveness:
|
||||
return ActiveSessionLease(
|
||||
lease_id=lease_id,
|
||||
@@ -560,22 +518,23 @@ def try_acquire_active_session(
|
||||
)
|
||||
|
||||
state_path, lock_path = _lease_paths(registry_home=registry_home)
|
||||
lease = ActiveSessionLease(
|
||||
lease_id=lease_id,
|
||||
session_id=key,
|
||||
surface=str(surface),
|
||||
state_path=state_path,
|
||||
lock_path=lock_path,
|
||||
track_liveness=track_liveness,
|
||||
)
|
||||
with _FileLock(lock_path):
|
||||
try:
|
||||
raw_entries = _read_entries(state_path, strict=True)
|
||||
entries = _prune_dead(raw_entries, strict=track_liveness)
|
||||
except ActiveSessionRegistryError:
|
||||
if track_liveness:
|
||||
raise
|
||||
# A capacity cap could afford to degrade open -- worst case, more
|
||||
# sessions than the operator wanted. Exclusivity cannot: "could not
|
||||
# prove ownership" must never be collapsed into "no owner exists",
|
||||
# because that silently reopens the exact double-writer state this
|
||||
# fence guarantees against. Refuse, and say which file to fix.
|
||||
logger.warning(
|
||||
"Active-session registry is unavailable; refusing the session "
|
||||
"rather than risking a concurrent writer"
|
||||
)
|
||||
# A capacity cap could degrade open; exclusivity cannot: "could not
|
||||
# prove ownership" must never become "no owner exists".
|
||||
loaded = _read_live_entries(
|
||||
state_path, track_liveness=track_liveness,
|
||||
warn="Active-session registry is unavailable; refusing the session "
|
||||
"rather than risking a concurrent writer",
|
||||
)
|
||||
if loaded is None:
|
||||
return None, ActiveSessionRefusal(
|
||||
(
|
||||
"Hermes could not read the active-session registry at "
|
||||
@@ -584,47 +543,27 @@ def try_acquire_active_session(
|
||||
),
|
||||
SESSION_COORDINATION_UNAVAILABLE,
|
||||
)
|
||||
raw_entries, entries = loaded
|
||||
pruned = len(raw_entries) - len(entries)
|
||||
if pruned:
|
||||
logger.info("Pruned %d stale active session lease(s)", pruned)
|
||||
|
||||
# Correctness first, and under the same lock that just pruned the dead
|
||||
# owners -- so an owner that died is never mistaken for one that is
|
||||
# running, and a live one is never overlooked.
|
||||
#
|
||||
# An empty key is exempt: a session with no stored id yet cannot collide
|
||||
# with another, and treating "" as an identity would make every unsaved
|
||||
# draft exclude every other one.
|
||||
# Correctness first, under the same lock that just pruned dead owners.
|
||||
# An empty key is exempt: treating "" as an identity would make every
|
||||
# unsaved draft exclude every other one.
|
||||
if key:
|
||||
for index, existing in enumerate(entries):
|
||||
if str(existing.get("session_id") or "") != key:
|
||||
continue
|
||||
|
||||
# THE SAME WRITER IS NOT A SECOND WRITER.
|
||||
#
|
||||
# A live session that lost its lease reference -- its record was
|
||||
# rebuilt in place, so the object holding the lease is unreachable
|
||||
# while the session itself is still the one being driven -- would
|
||||
# otherwise be fenced out of its own session by its own leak, and
|
||||
# permanently: pruning only removes entries whose PROCESS is dead,
|
||||
# and this process is very much alive.
|
||||
#
|
||||
# Identity here is (pid, live session id). Two processes never
|
||||
# match, because their pids differ. Two live sessions inside one
|
||||
# process never match, because their live ids differ. Only the
|
||||
# exact same writer re-acquiring its own session matches, and that
|
||||
# is re-entrancy rather than a concurrent writer.
|
||||
# The same writer is not a second writer: a live session that
|
||||
# leaked its lease reference would otherwise be fenced out of
|
||||
# its own session permanently (pruning only removes entries
|
||||
# whose PROCESS is dead). Re-entrancy, not concurrency.
|
||||
if _is_same_writer(existing, metadata):
|
||||
entries[index] = entry
|
||||
_write_entries(state_path, entries)
|
||||
return ActiveSessionLease(
|
||||
lease_id=lease_id,
|
||||
session_id=key,
|
||||
surface=str(surface),
|
||||
state_path=state_path,
|
||||
lock_path=lock_path,
|
||||
track_liveness=track_liveness,
|
||||
), None
|
||||
return lease, None
|
||||
|
||||
_write_entries(state_path, entries)
|
||||
logger.info(
|
||||
@@ -656,14 +595,7 @@ def try_acquire_active_session(
|
||||
entries.append(entry)
|
||||
_write_entries(state_path, entries)
|
||||
|
||||
return ActiveSessionLease(
|
||||
lease_id=lease_id,
|
||||
session_id=key,
|
||||
surface=str(surface),
|
||||
state_path=state_path,
|
||||
lock_path=lock_path,
|
||||
track_liveness=track_liveness,
|
||||
), None
|
||||
return lease, None
|
||||
|
||||
|
||||
def release_active_session(lease: ActiveSessionLease) -> None:
|
||||
@@ -673,25 +605,16 @@ def release_active_session(lease: ActiveSessionLease) -> None:
|
||||
with _FileLock(lock_path):
|
||||
if lease.released:
|
||||
return
|
||||
try:
|
||||
raw_entries = _read_entries(state_path, strict=True)
|
||||
entries = _prune_dead(raw_entries, strict=lease.track_liveness)
|
||||
except ActiveSessionRegistryError:
|
||||
if lease.track_liveness:
|
||||
raise
|
||||
logger.warning(
|
||||
"Active-session registry is unavailable; preserving it while "
|
||||
"releasing an untracked lease"
|
||||
)
|
||||
lease.released = True
|
||||
return
|
||||
kept = [
|
||||
entry
|
||||
for entry in entries
|
||||
if str(entry.get("lease_id") or "") != lease.lease_id
|
||||
]
|
||||
if len(kept) != len(entries):
|
||||
_write_entries(state_path, kept)
|
||||
loaded = _read_live_entries(
|
||||
state_path, track_liveness=lease.track_liveness,
|
||||
warn="Active-session registry is unavailable; preserving it while "
|
||||
"releasing an untracked lease",
|
||||
)
|
||||
if loaded is not None:
|
||||
entries = loaded[1]
|
||||
kept = _without_lease(entries, lease.lease_id)
|
||||
if len(kept) != len(entries):
|
||||
_write_entries(state_path, kept)
|
||||
lease.released = True
|
||||
|
||||
|
||||
@@ -717,17 +640,14 @@ def transfer_active_session(
|
||||
# thread acquired the file lock. Never resurrect a durably removed lease.
|
||||
if lease.released:
|
||||
return False
|
||||
try:
|
||||
raw_entries = _read_entries(state_path, strict=True)
|
||||
entries = _prune_dead(raw_entries, strict=lease.track_liveness)
|
||||
except ActiveSessionRegistryError:
|
||||
if lease.track_liveness:
|
||||
raise
|
||||
logger.warning(
|
||||
"Active-session registry is unavailable; refusing to overwrite "
|
||||
"it during lease transfer"
|
||||
)
|
||||
loaded = _read_live_entries(
|
||||
state_path, track_liveness=lease.track_liveness,
|
||||
warn="Active-session registry is unavailable; refusing to overwrite "
|
||||
"it during lease transfer",
|
||||
)
|
||||
if loaded is None:
|
||||
return False
|
||||
entries = loaded[1]
|
||||
updated = False
|
||||
for entry in entries:
|
||||
if str(entry.get("lease_id") or "") != lease.lease_id:
|
||||
@@ -735,9 +655,7 @@ def transfer_active_session(
|
||||
entry["session_id"] = new_session_id
|
||||
entry["updated_at"] = time.time()
|
||||
if metadata:
|
||||
entry["metadata"] = {
|
||||
str(k): v for k, v in metadata.items() if isinstance(k, str)
|
||||
}
|
||||
entry["metadata"] = _clean_metadata(metadata)
|
||||
updated = True
|
||||
break
|
||||
if not updated and lease.track_liveness:
|
||||
@@ -760,12 +678,10 @@ def transfer_active_session(
|
||||
def release_orphaned_leases(live_lease_ids: set[str]) -> int:
|
||||
"""Drop this process's registry entries that no live session owns.
|
||||
|
||||
``_prune_dead`` only reclaims leases whose owning process died. A server
|
||||
that runs for days (``hermes dashboard`` / ``serve``) never trips that
|
||||
check, so a lease whose session skipped teardown is held until restart.
|
||||
The owning process is the only authority on which of its own leases are
|
||||
real, so it drops the rest itself — exact, with no heartbeat write on the
|
||||
turn path and no staleness threshold to tune.
|
||||
``_prune_dead`` only reclaims leases of dead processes; a days-long server
|
||||
never trips it, so a lease whose session skipped teardown is held until
|
||||
restart. The owning process is the only authority on its own leases —
|
||||
exact, no heartbeat on the turn path, no staleness threshold.
|
||||
"""
|
||||
pid = os.getpid()
|
||||
state_path = _state_path()
|
||||
@@ -774,14 +690,13 @@ def release_orphaned_leases(live_lease_ids: set[str]) -> int:
|
||||
if not state_path.exists():
|
||||
return 0
|
||||
with _FileLock(_lock_path()):
|
||||
try:
|
||||
raw_entries = _read_entries(state_path, strict=True)
|
||||
entries = _prune_dead(raw_entries)
|
||||
except ActiveSessionRegistryError:
|
||||
logger.warning(
|
||||
"Active-session registry is unavailable; skipping orphaned-lease sweep"
|
||||
)
|
||||
loaded = _read_live_entries(
|
||||
state_path, track_liveness=False,
|
||||
warn="Active-session registry is unavailable; skipping orphaned-lease sweep",
|
||||
)
|
||||
if loaded is None:
|
||||
return 0
|
||||
entries = loaded[1]
|
||||
kept = [
|
||||
entry
|
||||
for entry in entries
|
||||
@@ -813,12 +728,9 @@ def active_session_liveness_guard(
|
||||
*,
|
||||
registry_home: str | Path | None = None,
|
||||
) -> Iterator[bool]:
|
||||
"""Hold the registry lock while reporting whether ``session_id`` is leased.
|
||||
|
||||
Keeping the lock across the caller's lifecycle mutation prevents a new
|
||||
backend from acquiring a lease and reopening the row between the liveness
|
||||
check and the corresponding ``end_session`` write.
|
||||
"""
|
||||
"""Hold the registry lock while reporting whether ``session_id`` is leased,
|
||||
so no new backend can acquire a lease between the check and the caller's
|
||||
``end_session`` write."""
|
||||
target = str(session_id or "")
|
||||
state_path, lock_path = _lease_paths(registry_home=registry_home)
|
||||
with _FileLock(lock_path):
|
||||
@@ -834,12 +746,9 @@ def release_active_session_liveness_guard(
|
||||
lease: ActiveSessionLease,
|
||||
session_id: str,
|
||||
) -> Iterator[bool]:
|
||||
"""Remove ``lease`` and hold its registry lock through a lifecycle write.
|
||||
|
||||
This makes automatic cleanup one atomic ownership decision: the local
|
||||
runtime disappears, sibling liveness is checked, and the caller may end the
|
||||
durable row before any new backend can acquire/reopen it.
|
||||
"""
|
||||
"""Remove ``lease`` and hold its registry lock through a lifecycle write,
|
||||
making cleanup one atomic ownership decision (release, check siblings,
|
||||
end the durable row) before any new backend can reopen it."""
|
||||
if not lease.enabled or lease.released:
|
||||
with active_session_liveness_guard(
|
||||
session_id, registry_home=_registry_home_for_lease(lease)
|
||||
@@ -850,13 +759,8 @@ def release_active_session_liveness_guard(
|
||||
target = str(session_id or "")
|
||||
state_path, lock_path = _lease_paths(lease)
|
||||
with _FileLock(lock_path):
|
||||
raw_entries = _read_entries(state_path, strict=True)
|
||||
entries = _prune_dead(raw_entries, strict=True)
|
||||
kept = [
|
||||
entry
|
||||
for entry in entries
|
||||
if str(entry.get("lease_id") or "") != lease.lease_id
|
||||
]
|
||||
entries = _prune_dead(_read_entries(state_path, strict=True), strict=True)
|
||||
kept = _without_lease(entries, lease.lease_id)
|
||||
if len(kept) != len(entries):
|
||||
_write_entries(state_path, kept)
|
||||
lease.released = True
|
||||
|
||||
+767
-1420
File diff suppressed because it is too large
Load Diff
+1258
-2254
File diff suppressed because it is too large
Load Diff
+30
-105
@@ -11,40 +11,26 @@ so when the whole graph completes the root wakes back up — its
|
||||
assignee (the orchestrator profile) gets a chance to judge completion
|
||||
and add more tasks if the work isn't done yet.
|
||||
|
||||
Design notes
|
||||
------------
|
||||
|
||||
* Mirrors the shape of ``hermes_cli/kanban_specify.py``: lazy aux
|
||||
client import inside the function, lenient response parse, never
|
||||
raises on expected failure modes.
|
||||
|
||||
* The system prompt sees the *configured* profile roster — names plus
|
||||
descriptions plus the default fallback. Profiles without a
|
||||
description are still listed (with a note) so the decomposer can
|
||||
match on name as a fallback, but the user has an obvious incentive
|
||||
to describe them.
|
||||
|
||||
* ``fanout=false`` collapses to the same effect as ``kanban specify``:
|
||||
we tighten the body and flip ``triage -> todo`` as a single task,
|
||||
no children created. This makes ``decompose`` a strict superset of
|
||||
``specify`` from the user's perspective.
|
||||
|
||||
* If the LLM picks an assignee that doesn't exist as a profile, we
|
||||
rewrite it to the configured ``default_assignee`` (or the default
|
||||
profile if unset). A child task NEVER ends up with ``assignee=None``.
|
||||
Design notes: mirrors ``kanban_specify`` (lazy aux import, lenient parse,
|
||||
never raises on expected failures). The prompt sees the configured profile
|
||||
roster; undescribed profiles are listed with a note so name-matching still
|
||||
works. ``fanout=false`` collapses to the ``specify`` behaviour (tighten +
|
||||
promote, no children), making ``decompose`` a strict superset. Unknown
|
||||
assignees are rewritten to ``default_assignee`` — a child NEVER ends up with
|
||||
``assignee=None``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
from hermes_cli import kanban_db as kb
|
||||
from hermes_cli import profiles as profiles_mod
|
||||
from hermes_cli.kanban_specify import _extract_json_blob, _title_body, _truncate
|
||||
from hermes_cli.kanban_specify import _profile_author as _specify_author
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -136,37 +122,9 @@ class DecomposeOutcome:
|
||||
new_title: Optional[str] = None
|
||||
|
||||
|
||||
def _truncate(text: str, limit: int) -> str:
|
||||
if len(text) <= limit:
|
||||
return text
|
||||
return text[: limit - 1] + "…"
|
||||
|
||||
|
||||
def _extract_json_blob(raw: str) -> Optional[dict]:
|
||||
if not raw:
|
||||
return None
|
||||
stripped = _FENCE_RE.sub("", raw.strip())
|
||||
first = stripped.find("{")
|
||||
last = stripped.rfind("}")
|
||||
if first == -1 or last == -1 or last <= first:
|
||||
return None
|
||||
candidate = stripped[first : last + 1]
|
||||
try:
|
||||
val = json.loads(candidate)
|
||||
except (ValueError, json.JSONDecodeError):
|
||||
return None
|
||||
if not isinstance(val, dict):
|
||||
return None
|
||||
return val
|
||||
|
||||
|
||||
def _profile_author() -> str:
|
||||
"""Mirror of ``hermes_cli.kanban._profile_author``."""
|
||||
return (
|
||||
os.environ.get("HERMES_PROFILE")
|
||||
or os.environ.get("USER")
|
||||
or "decomposer"
|
||||
)
|
||||
return _specify_author("decomposer")
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
@@ -177,31 +135,15 @@ def _load_config() -> dict:
|
||||
return {}
|
||||
|
||||
|
||||
def _resolve_orchestrator_profile(cfg: dict) -> str:
|
||||
"""Resolve which profile owns the root/orchestration task after fan-out.
|
||||
def _resolve_profile_from_cfg(cfg: dict, key: str) -> str:
|
||||
"""``kanban.<key>`` if it names an existing profile, else the active
|
||||
default profile — so a task is never stranded for lack of an owner.
|
||||
|
||||
Falls back to the active default profile when ``kanban.orchestrator_profile``
|
||||
is unset, so a task is never stranded for lack of an orchestrator.
|
||||
``orchestrator_profile`` owns the root after fan-out; ``default_assignee``
|
||||
catches children the decomposer can't route.
|
||||
"""
|
||||
kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {}
|
||||
explicit = (kanban_cfg.get("orchestrator_profile") or "").strip()
|
||||
if explicit:
|
||||
try:
|
||||
if profiles_mod.profile_exists(explicit):
|
||||
return explicit
|
||||
except Exception:
|
||||
pass
|
||||
# Fall back to the active default profile.
|
||||
try:
|
||||
return profiles_mod.get_active_profile_name() or "default"
|
||||
except Exception:
|
||||
return "default"
|
||||
|
||||
|
||||
def _resolve_default_assignee(cfg: dict) -> str:
|
||||
"""Resolve which profile catches child tasks the orchestrator can't route."""
|
||||
kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {}
|
||||
explicit = (kanban_cfg.get("default_assignee") or "").strip()
|
||||
explicit = (kanban_cfg.get(key) or "").strip()
|
||||
if explicit:
|
||||
try:
|
||||
if profiles_mod.profile_exists(explicit):
|
||||
@@ -215,12 +157,8 @@ def _resolve_default_assignee(cfg: dict) -> str:
|
||||
|
||||
|
||||
def _build_roster() -> tuple[list[dict], set[str]]:
|
||||
"""Return (roster_for_prompt, valid_assignee_names).
|
||||
|
||||
Each roster entry is ``{name, description, has_description}``. The
|
||||
valid-set is used after the LLM responds to rewrite invalid
|
||||
assignees to the default fallback.
|
||||
"""
|
||||
"""``(roster_for_prompt, valid_assignee_names)``; entries are
|
||||
``{name, description, has_description}``."""
|
||||
roster: list[dict] = []
|
||||
valid: set[str] = set()
|
||||
try:
|
||||
@@ -255,11 +193,8 @@ def _normalize_assignee_choice(
|
||||
default_assignee: str,
|
||||
valid_names: set[str],
|
||||
) -> str:
|
||||
"""Return a valid assignee, falling back to ``default_assignee``.
|
||||
|
||||
Fan-out children and the single-task fallback should share the same
|
||||
routing guarantee: promoted work must not be left unassigned.
|
||||
"""
|
||||
"""A valid assignee, else ``default_assignee`` — promoted work is never
|
||||
left unassigned."""
|
||||
if not isinstance(assignee, str) or not assignee.strip():
|
||||
return default_assignee
|
||||
chosen = assignee.strip()
|
||||
@@ -274,13 +209,9 @@ def decompose_task(
|
||||
author: Optional[str] = None,
|
||||
timeout: Optional[int] = None,
|
||||
) -> DecomposeOutcome:
|
||||
"""Decompose a triage task into a graph of child tasks.
|
||||
|
||||
Returns an outcome describing what happened. Never raises for
|
||||
expected failure modes (task not in triage, no aux client
|
||||
configured, API error, malformed response, decomposer returned
|
||||
fanout=true with empty task list) — those surface via ``ok=False``.
|
||||
"""
|
||||
"""Decompose a triage task into a graph of child tasks. Expected failures
|
||||
(not in triage, no aux client, API error, malformed/empty reply) surface
|
||||
as ``ok=False``."""
|
||||
with kb.connect_closing() as conn:
|
||||
task = kb.get_task(conn, task_id)
|
||||
if task is None:
|
||||
@@ -291,8 +222,8 @@ def decompose_task(
|
||||
)
|
||||
|
||||
cfg = _load_config()
|
||||
orchestrator = _resolve_orchestrator_profile(cfg)
|
||||
default_assignee = _resolve_default_assignee(cfg)
|
||||
orchestrator = _resolve_profile_from_cfg(cfg, "orchestrator_profile")
|
||||
default_assignee = _resolve_profile_from_cfg(cfg, "default_assignee")
|
||||
kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {}
|
||||
auto_promote = bool(kanban_cfg.get("auto_promote_children", True))
|
||||
roster, valid_names = _build_roster()
|
||||
@@ -312,10 +243,8 @@ def decompose_task(
|
||||
)
|
||||
|
||||
try:
|
||||
# Route through call_llm so auxiliary.kanban_decomposer.* config
|
||||
# (provider/model/base_url, extra_body, reasoning_effort, retries)
|
||||
# all apply — the previous direct client.chat.completions.create()
|
||||
# path dropped auxiliary.<task>.extra_body entirely (#35566).
|
||||
# call_llm applies all auxiliary.kanban_decomposer.* config
|
||||
# (provider/model/base_url, extra_body, reasoning_effort, retries).
|
||||
resp = call_llm(
|
||||
task="kanban_decomposer",
|
||||
messages=[
|
||||
@@ -337,7 +266,7 @@ def decompose_task(
|
||||
except Exception:
|
||||
raw = ""
|
||||
|
||||
parsed = _extract_json_blob(raw)
|
||||
parsed = _extract_json_blob(raw, _FENCE_RE)
|
||||
if parsed is None:
|
||||
return DecomposeOutcome(task_id, False, "LLM returned malformed JSON")
|
||||
|
||||
@@ -346,10 +275,7 @@ def decompose_task(
|
||||
|
||||
if not fanout:
|
||||
# Fall back to single-task spec promotion (same effect as specify).
|
||||
new_title = parsed.get("title")
|
||||
new_body = parsed.get("body")
|
||||
title_val = new_title.strip() if isinstance(new_title, str) and new_title.strip() else None
|
||||
body_val = new_body if isinstance(new_body, str) and new_body.strip() else None
|
||||
title_val, body_val = _title_body(parsed)
|
||||
assignee_val = None
|
||||
if not task.assignee:
|
||||
assignee_val = _normalize_assignee_choice(
|
||||
@@ -385,8 +311,7 @@ def decompose_task(
|
||||
task_id, False, "decomposer returned fanout=true with empty tasks list",
|
||||
)
|
||||
|
||||
# Rewrite invalid assignees to the default fallback. Never leave a
|
||||
# task with assignee=None — the user explicitly does not want that.
|
||||
# Unknown assignees route to the default; never assignee=None.
|
||||
children: list[dict] = []
|
||||
for idx, entry in enumerate(raw_tasks):
|
||||
if not isinstance(entry, dict):
|
||||
|
||||
+133
-353
@@ -1,30 +1,13 @@
|
||||
"""Kanban diagnostics — structured, actionable distress signals for tasks.
|
||||
|
||||
A ``Diagnostic`` is a machine-readable description of something that's wrong
|
||||
with a kanban task: a hallucinated card id, a spawn crash-loop, a task
|
||||
stuck blocked for too long, etc. Each one carries:
|
||||
A ``Diagnostic`` carries a **kind** (canonical code the UI/tests match on), a
|
||||
**severity**, title/detail text, and **actions** the dashboard renders as
|
||||
buttons and the CLI as hints. Rules are stateless and read-only over
|
||||
(task, events, runs, optional graph); callers compute on demand.
|
||||
|
||||
* A **kind** (canonical code; UI/tests match on this).
|
||||
* A **severity** (``warning`` / ``error`` / ``critical``).
|
||||
* A **title** (one-line human description) and **detail** (longer text).
|
||||
* A list of **suggested actions** — structured entries the dashboard
|
||||
turns into buttons and the CLI turns into hints.
|
||||
|
||||
Rules run over (task, recent events, recent runs, optional graph context) and
|
||||
emit diagnostics. They are stateless and read-only — no DB writes. Callers compute
|
||||
diagnostics on demand (on ``/board`` load, ``/tasks/:id`` fetch, or
|
||||
``hermes kanban diagnostics``).
|
||||
|
||||
Design goals:
|
||||
|
||||
* Fixable-on-the-operator's-side signals only (missing config, phantom
|
||||
ids, crash loop). Not "the provider returned 502 once" — that's a
|
||||
transient runtime blip, not a diagnostic.
|
||||
* Recoverable: every diagnostic comes with at least one suggested
|
||||
recovery action the operator can actually take from the UI.
|
||||
* Auto-clearing: when the underlying failure mode resolves (a clean
|
||||
``completed`` event arrives, a spawn succeeds, the task gets
|
||||
unblocked), the diagnostic stops firing. The audit event trail stays.
|
||||
Design goals: operator-fixable signals only (not a one-off provider 502);
|
||||
every diagnostic has at least one recovery action; diagnostics auto-clear
|
||||
when the failure mode resolves (the audit event trail stays).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -35,9 +18,7 @@ import json
|
||||
import time
|
||||
|
||||
|
||||
# Severity rungs, ordered least → most urgent. The UI colors them
|
||||
# amber (warning), orange (error), red (critical). Sorted outputs put
|
||||
# critical first so operators see the worst fires at the top.
|
||||
# Least → most urgent; sorted outputs put critical first.
|
||||
SEVERITY_ORDER = ("warning", "error", "critical")
|
||||
|
||||
|
||||
@@ -52,24 +33,10 @@ def severity_at_or_above(severity: Optional[str], threshold: Optional[str]) -> b
|
||||
|
||||
@dataclass
|
||||
class DiagnosticAction:
|
||||
"""A single recovery action attached to a diagnostic.
|
||||
|
||||
The ``kind`` determines how both the UI and CLI render it:
|
||||
|
||||
* ``reclaim`` / ``reassign`` — POST to the matching /tasks/:id/*
|
||||
endpoint; dashboard wires into the existing recovery popover.
|
||||
* ``unblock`` — PATCH status back to ``ready`` (for stuck-blocked
|
||||
diagnostics).
|
||||
* ``cli_hint`` — print/copy a shell command (e.g.
|
||||
``hermes -p <profile> auth``). No HTTP side effect.
|
||||
* ``open_docs`` — deep-link to the docs URL named in ``payload.url``.
|
||||
* ``comment`` — nudge the operator to add a comment (for
|
||||
stuck-blocked tasks that need human input).
|
||||
|
||||
``suggested=True`` marks the action as the recommended first step;
|
||||
the UI highlights it. Multiple actions can be suggested if they're
|
||||
equally valid.
|
||||
"""
|
||||
"""A recovery action. ``kind`` drives rendering: ``reclaim``/``reassign``
|
||||
POST to /tasks/:id/*; ``unblock`` PATCHes status to ready; ``cli_hint``
|
||||
shows ``payload.command``; ``open_docs`` links ``payload.url``; ``comment``
|
||||
nudges the operator. ``suggested=True`` = recommended first step."""
|
||||
|
||||
kind: str
|
||||
label: str
|
||||
@@ -122,20 +89,10 @@ class Diagnostic:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _task_field(task, name, default=None):
|
||||
"""Read a field from a task regardless of representation.
|
||||
|
||||
Callers pass sqlite3.Row (dict-like with [] but no attribute
|
||||
access), kanban_db.Task dataclasses (attribute access), or plain
|
||||
dicts (both). This normalises them so rule functions don't have
|
||||
to branch on type each time.
|
||||
"""
|
||||
"""Read a field from a sqlite3.Row, a kanban_db.Task dataclass, or a dict."""
|
||||
if task is None:
|
||||
return default
|
||||
# sqlite Row + plain dicts both support mapping access; Row also
|
||||
# supports .keys().
|
||||
try:
|
||||
# Row raises IndexError if the key isn't a column in the query;
|
||||
# dicts return default via .get. Handle both.
|
||||
if hasattr(task, "keys") and name in task.keys():
|
||||
return task[name]
|
||||
except Exception:
|
||||
@@ -169,20 +126,42 @@ def _event_ts(ev) -> int:
|
||||
return int(t or 0)
|
||||
|
||||
|
||||
def _first_field(task, primary: str, legacy: str, default=None):
|
||||
"""``task[primary]`` unless it is None, else ``task[legacy]`` (old DB rows)."""
|
||||
v = _task_field(task, primary, None)
|
||||
return v if v is not None else _task_field(task, legacy, default)
|
||||
|
||||
|
||||
def _latest_event_ts(events: Iterable[Any], kinds: set[str]) -> int:
|
||||
"""Max ``created_at`` over events whose kind is in ``kinds`` (0 if none)."""
|
||||
latest = 0
|
||||
for ev in events:
|
||||
if _event_kind(ev) in kinds:
|
||||
latest = max(latest, _event_ts(ev))
|
||||
return latest
|
||||
|
||||
|
||||
def _log_hint_action(task_id: str) -> DiagnosticAction:
|
||||
return DiagnosticAction(
|
||||
kind="cli_hint",
|
||||
label=f"Check logs: hermes kanban log {task_id}",
|
||||
payload={"command": f"hermes kanban log {task_id}"},
|
||||
suggested=True,
|
||||
)
|
||||
|
||||
|
||||
def _error_snippet(last_err) -> str:
|
||||
"""First 500 chars of the error (with ellipsis), or "" when absent."""
|
||||
err_text = (last_err or "").strip() if last_err else ""
|
||||
return err_text[:500] + ("…" if len(err_text) > 500 else "") if err_text else ""
|
||||
|
||||
|
||||
def _active_hallucination_events(
|
||||
events: Iterable[Any],
|
||||
kind: str,
|
||||
) -> list[Any]:
|
||||
"""Return events of ``kind`` that have no ``completed``/``edited``
|
||||
event *strictly after* them. Walks chronologically: each clean
|
||||
event resets the accumulator; each matching event gets appended.
|
||||
|
||||
Events must be sorted by id (i.e. arrival order); callers pass the
|
||||
task's full event list which the DB already returns in that order.
|
||||
"""
|
||||
# Events arrive sorted by id asc (chronological). Walk once, track
|
||||
# which hallucination events are still "active" (no clean event
|
||||
# supersedes them).
|
||||
"""Events of ``kind`` with no ``completed``/``edited`` event strictly after
|
||||
them. Requires id-sorted (arrival-order) input, which the DB provides."""
|
||||
active: list[Any] = []
|
||||
for ev in events:
|
||||
k = _event_kind(ev)
|
||||
@@ -191,9 +170,9 @@ def _active_hallucination_events(
|
||||
elif k == kind:
|
||||
active.append(ev)
|
||||
return active
|
||||
# Standard always-available actions. Every diagnostic can offer these as
|
||||
# fallbacks regardless of kind — they're the two baseline recovery
|
||||
# primitives the kernel supports.
|
||||
|
||||
|
||||
# Baseline recovery primitives every diagnostic can fall back on.
|
||||
def _generic_recovery_actions(task: Any, *, running: bool) -> list[DiagnosticAction]:
|
||||
out: list[DiagnosticAction] = []
|
||||
if running:
|
||||
@@ -214,22 +193,16 @@ def _generic_recovery_actions(task: Any, *, running: bool) -> list[DiagnosticAct
|
||||
# Rule implementations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Each rule takes (task, events, runs, now_ts, config) and returns
|
||||
# zero or more Diagnostic instances. ``events`` / ``runs`` are lists of
|
||||
# kanban_db.Event / kanban_db.Run (or plain dicts matching the same
|
||||
# shape — for test convenience).
|
||||
# Each rule: (task, events, runs, now_ts, config) -> list[Diagnostic].
|
||||
# ``events``/``runs`` are kanban_db rows/dataclasses or same-shaped dicts.
|
||||
|
||||
RuleFn = Callable[[Any, list[Any], list[Any], int, dict], list[Diagnostic]]
|
||||
|
||||
|
||||
def _aux_slot_explicit(slot: Any) -> bool:
|
||||
"""Return True if the auxiliary slot has user-supplied non-default fields.
|
||||
|
||||
Defaults from ``DEFAULT_CONFIG`` use ``provider: "auto"`` with empty
|
||||
model/base_url/api_key — that path falls through to the main model. An
|
||||
"explicit" config is one where the user actively set a provider (not
|
||||
"auto"), or supplied a model / base_url / api_key.
|
||||
"""
|
||||
"""True if the aux slot was user-configured: provider other than "auto",
|
||||
or any of model/base_url/api_key set (the default falls through to the
|
||||
main model)."""
|
||||
if not isinstance(slot, dict):
|
||||
return False
|
||||
provider = str(slot.get("provider") or "").strip().lower()
|
||||
@@ -242,12 +215,9 @@ def _aux_slot_explicit(slot: Any) -> bool:
|
||||
|
||||
|
||||
def _main_model_visible(raw_config: Any) -> bool:
|
||||
"""Best-effort check that a main model is configured.
|
||||
|
||||
Diagnostics runs in the dashboard process which may not share the CLI's
|
||||
runtime state, so we read the raw config dict. If we cannot prove the
|
||||
main model is set, we err on the side of NOT firing the diagnostic.
|
||||
"""
|
||||
"""Best-effort "a main model is configured" from the raw config dict (the
|
||||
dashboard process may not share CLI runtime state). Unprovable => False,
|
||||
which errs toward NOT firing the diagnostic."""
|
||||
if not isinstance(raw_config, dict):
|
||||
return False
|
||||
model_cfg = raw_config.get("model")
|
||||
@@ -264,17 +234,9 @@ def _main_model_visible(raw_config: Any) -> bool:
|
||||
|
||||
|
||||
def triage_aux_status(config: Optional[dict]) -> Optional[dict]:
|
||||
"""Inspect raw config and report whether triage paths look configured.
|
||||
|
||||
Returns ``None`` when config context is unavailable (suppress diagnostic
|
||||
to avoid noisy false positives in tests / low-level callers). Otherwise
|
||||
returns a dict with:
|
||||
|
||||
- ``auto_decompose``: bool — whether the dispatcher auto-runs decompose
|
||||
- ``decomposer_explicit``: bool — user-supplied decomposer slot
|
||||
- ``specifier_explicit``: bool — user-supplied specifier slot
|
||||
- ``main_model_visible``: bool — main model can serve as auto fallback
|
||||
"""
|
||||
"""Report whether the triage aux paths look configured: ``{auto_decompose,
|
||||
decomposer_explicit, specifier_explicit, main_model_visible}``. ``None``
|
||||
when no config context is present (keeps low-level callers/tests silent)."""
|
||||
if not isinstance(config, dict):
|
||||
return None
|
||||
|
||||
@@ -285,9 +247,7 @@ def triage_aux_status(config: Optional[dict]) -> Optional[dict]:
|
||||
aux = config.get("auxiliary")
|
||||
kanban_cfg = config.get("kanban") if isinstance(config.get("kanban"), dict) else {}
|
||||
|
||||
# Have we been handed any config context at all? When neither auxiliary
|
||||
# nor kanban nor model keys are present, the caller is a low-level test
|
||||
# passing {} — stay silent.
|
||||
# No auxiliary/kanban/model keys at all => a low-level caller passing {}.
|
||||
if (
|
||||
not isinstance(aux, dict)
|
||||
and not kanban_cfg
|
||||
@@ -323,14 +283,8 @@ def _positive_int(value: Any, default: int) -> int:
|
||||
|
||||
|
||||
def _rule_hallucinated_cards(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
"""Blocked-hallucination gate fires: a worker called kanban_complete
|
||||
with created_cards that didn't exist or weren't created by the
|
||||
completing profile. Task stayed in its prior state; the operator
|
||||
needs to decide how to proceed.
|
||||
|
||||
Auto-clears when a successful completion (or edit) follows the
|
||||
blocked event.
|
||||
"""
|
||||
"""A worker's kanban_complete named created_cards that don't exist / weren't
|
||||
its own; the completion was blocked. Clears on a later completion/edit."""
|
||||
hits = _active_hallucination_events(events, "completion_blocked_hallucination")
|
||||
if not hits:
|
||||
return []
|
||||
@@ -343,13 +297,11 @@ def _rule_hallucinated_cards(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
if pid not in phantom_ids:
|
||||
phantom_ids.append(pid)
|
||||
running = _task_field(task, "status") == "running"
|
||||
actions: list[DiagnosticAction] = []
|
||||
actions.append(DiagnosticAction(
|
||||
actions = [DiagnosticAction(
|
||||
kind="comment",
|
||||
label="Add a comment explaining what to do",
|
||||
suggested=False,
|
||||
))
|
||||
actions.extend(_generic_recovery_actions(task, running=running))
|
||||
)] + _generic_recovery_actions(task, running=running)
|
||||
return [Diagnostic(
|
||||
kind="hallucinated_cards",
|
||||
severity="error",
|
||||
@@ -370,22 +322,12 @@ def _rule_hallucinated_cards(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
|
||||
|
||||
def _rule_triage_aux_unavailable(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
"""A triage task cannot leave triage without an auxiliary helper.
|
||||
|
||||
With the auto-decompose dispatcher (kanban.auto_decompose, default True),
|
||||
triage tasks fan out via ``auxiliary.kanban_decomposer`` and fall back to
|
||||
``auxiliary.triage_specifier`` when the decomposer returns ``fanout=false``.
|
||||
With auto-decompose off, the user must run ``hermes kanban specify``,
|
||||
which only needs ``auxiliary.triage_specifier``.
|
||||
|
||||
The default slot is ``provider: auto`` → auto-falls back to the main model,
|
||||
so this rule only fires when:
|
||||
|
||||
- the relevant slot is explicitly set to something broken, OR
|
||||
- the auto fallback has no main model to fall back to.
|
||||
|
||||
Config context is required; pass {} from tests to keep the rule silent.
|
||||
"""
|
||||
"""A triage task can't leave triage without a usable aux model. With
|
||||
auto-decompose on the primary slot is ``auxiliary.kanban_decomposer``
|
||||
(specifier as fallback); off, it is ``auxiliary.triage_specifier``. The
|
||||
default ``provider: auto`` falls back to the main model, so this fires only
|
||||
when the slot isn't explicit AND no main model is visible. Requires config
|
||||
context ({} keeps it silent)."""
|
||||
if _task_field(task, "status") != "triage":
|
||||
return []
|
||||
|
||||
@@ -421,9 +363,6 @@ def _rule_triage_aux_unavailable(task, events, runs, now, cfg) -> list[Diagnosti
|
||||
"`hermes kanban specify`, which uses auxiliary.triage_specifier."
|
||||
)
|
||||
|
||||
# The primary slot is usable when either: it was explicitly configured by
|
||||
# the user, OR the default `provider: auto` can fall back to the main
|
||||
# model. If both fail, we have a real configuration gap.
|
||||
if primary_explicit or main_visible:
|
||||
return []
|
||||
|
||||
@@ -482,12 +421,8 @@ def _rule_triage_aux_unavailable(task, events, runs, now, cfg) -> list[Diagnosti
|
||||
|
||||
|
||||
def _rule_prose_phantom_refs(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
"""Advisory prose-scan: the completion summary mentions ``t_<hex>``
|
||||
ids that don't resolve. Non-blocking; surfaced as a warning only.
|
||||
|
||||
Auto-clears when a fresh clean completion arrives AFTER the
|
||||
suspected event.
|
||||
"""
|
||||
"""Advisory: the completion summary mentions ``t_<hex>`` ids that don't
|
||||
resolve. Warning only; clears on a later clean completion."""
|
||||
hits = _active_hallucination_events(events, "suspected_hallucinated_references")
|
||||
if not hits:
|
||||
return []
|
||||
@@ -516,32 +451,14 @@ def _rule_prose_phantom_refs(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
|
||||
|
||||
def _rule_repeated_failures(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
"""Task's unified ``consecutive_failures`` counter is climbing —
|
||||
something about this task+profile combo is broken and each retry
|
||||
fails the same way. Triggers regardless of the specific failure
|
||||
mode (spawn error, timeout, crash) because operationally they
|
||||
all look the same: the kernel keeps retrying and the operator
|
||||
needs to intervene.
|
||||
"""``consecutive_failures`` >= cfg["failure_threshold"] (legacy key
|
||||
``spawn_failure_threshold``), regardless of failure mode — the kernel keeps
|
||||
retrying and the operator must intervene. Runtime callers derive the
|
||||
threshold from ``kanban.failure_limit`` so it doesn't lag the breaker.
|
||||
|
||||
Threshold: cfg["failure_threshold"]. Runtime callers should derive
|
||||
this from ``kanban.failure_limit`` unless the user explicitly set a
|
||||
diagnostics threshold, so the signal does not lag behind the
|
||||
dispatcher's circuit breaker.
|
||||
|
||||
Accepts the legacy ``spawn_failure_threshold`` config key for
|
||||
back-compat.
|
||||
|
||||
Terminal statuses are exempt: a done/archived card has nothing left
|
||||
to retry, so a lingering failure streak is history, not a signal.
|
||||
(``complete_task`` resets the counter, but a manual done — e.g. a
|
||||
dashboard drag — ends no run and used to leave the flag stuck.)
|
||||
|
||||
A fresh attempt in flight (``running``) is also exempt: retrying a
|
||||
task should clear the stale failure banner until this attempt also
|
||||
resolves. Otherwise a card that's actively trying again still shows
|
||||
"failed Nx", which reads as a current failure. It re-fires if the new
|
||||
run fails too (status leaves ``running`` with a recorded outcome).
|
||||
"""
|
||||
Exempt: done/archived (a manual done ends no run, so the streak is history)
|
||||
and running (a retry in flight must not read as a current failure; re-fires
|
||||
if it fails too)."""
|
||||
if _task_field(task, "status") in ("done", "archived", "running"):
|
||||
return []
|
||||
threshold = _positive_int(cfg.get(
|
||||
@@ -549,26 +466,13 @@ def _rule_repeated_failures(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
cfg.get("spawn_failure_threshold", 3),
|
||||
), 3)
|
||||
failure_limit = _positive_int(cfg.get("failure_limit"), threshold)
|
||||
# Read the new unified counter name, with a fallback to the legacy
|
||||
# column name so this rule keeps working against old DB rows the
|
||||
# caller somehow materialised without running the migration.
|
||||
failures = (
|
||||
_task_field(task, "consecutive_failures", None)
|
||||
if _task_field(task, "consecutive_failures", None) is not None
|
||||
else _task_field(task, "spawn_failures", 0)
|
||||
)
|
||||
failures = _first_field(task, "consecutive_failures", "spawn_failures", 0)
|
||||
if failures is None or failures < threshold:
|
||||
return []
|
||||
last_err = (
|
||||
_task_field(task, "last_failure_error", None)
|
||||
if _task_field(task, "last_failure_error", None) is not None
|
||||
else _task_field(task, "last_spawn_error", None)
|
||||
)
|
||||
last_err = _first_field(task, "last_failure_error", "last_spawn_error")
|
||||
assignee = _task_field(task, "assignee")
|
||||
|
||||
# Classify the most recent failure by peeking at run outcomes so
|
||||
# the title + suggested action can be specific without a separate
|
||||
# per-outcome rule.
|
||||
# Most recent failure outcome makes the title/action specific.
|
||||
ordered_runs = sorted(runs, key=lambda r: _task_field(r, "id", 0))
|
||||
most_recent_outcome = None
|
||||
for r in reversed(ordered_runs):
|
||||
@@ -596,19 +500,13 @@ def _rule_repeated_failures(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
# to diagnose; reclaim/reassign are the recovery levers.
|
||||
task_id = _task_field(task, "id")
|
||||
if task_id:
|
||||
actions.append(DiagnosticAction(
|
||||
kind="cli_hint",
|
||||
label=f"Check logs: hermes kanban log {task_id}",
|
||||
payload={"command": f"hermes kanban log {task_id}"},
|
||||
suggested=True,
|
||||
))
|
||||
actions.append(_log_hint_action(task_id))
|
||||
actions.extend(_generic_recovery_actions(
|
||||
task, running=_task_field(task, "status") == "running",
|
||||
))
|
||||
|
||||
severity = "critical" if failures >= threshold * 2 else "error"
|
||||
err_text = (last_err or "").strip() if last_err else ""
|
||||
err_snippet = err_text[:500] + ("…" if len(err_text) > 500 else "") if err_text else ""
|
||||
err_snippet = _error_snippet(last_err)
|
||||
outcome_label = {
|
||||
"spawn_failed": "spawn",
|
||||
"timed_out": "timeout",
|
||||
@@ -651,29 +549,14 @@ def _rule_repeated_failures(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
|
||||
|
||||
def _rule_repeated_crashes(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
"""The worker spawns fine but keeps crashing mid-run. Check the last
|
||||
N runs' outcomes; N consecutive ``crashed`` without a successful
|
||||
``completed`` means something about the task + profile combo is
|
||||
broken (OOM, missing dependency, tool it needs is down).
|
||||
"""Trailing run outcomes show >= cfg["crash_threshold"] (default 2)
|
||||
consecutive ``crashed`` with no ``completed``/``reclaimed`` between. Fires
|
||||
earlier than ``repeated_failures`` for a crash-specific heads-up and
|
||||
suppresses itself when the unified rule is about to fire.
|
||||
|
||||
Threshold: cfg["crash_threshold"] (default 2).
|
||||
|
||||
Narrower than ``repeated_failures`` — fires earlier (2 crashes vs 3
|
||||
total failures) so the operator gets a crash-specific heads-up
|
||||
before the unified rule kicks in. Suppresses itself when the
|
||||
unified rule is also about to fire, to avoid double-flagging.
|
||||
|
||||
Terminal statuses are exempt for the same reason as
|
||||
``repeated_failures`` — with one extra wrinkle: this rule reads run
|
||||
history, and a manual done (dashboard drag) appends no ``completed``
|
||||
run to break the crash streak, so the flag was permanent (#kanban
|
||||
desktop dogfood). Done means done.
|
||||
|
||||
``running`` is exempt too: a fresh attempt is in flight, and its
|
||||
in-flight run (no outcome yet) doesn't break the trailing crash scan,
|
||||
so a retried card kept showing "crashed Nx" over an active run. The
|
||||
banner re-fires if the new attempt also crashes.
|
||||
"""
|
||||
Exempt: done/archived (a manual done appends no completed run, so the
|
||||
streak would be permanent) and running (an in-flight run has no outcome
|
||||
and wouldn't break the scan)."""
|
||||
if _task_field(task, "status") in ("done", "archived", "running"):
|
||||
return []
|
||||
failure_threshold = int(cfg.get(
|
||||
@@ -702,29 +585,19 @@ def _rule_repeated_crashes(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
# A success (or manual reclaim) breaks the streak.
|
||||
break
|
||||
else:
|
||||
# Other outcomes (timed_out, blocked, spawn_failed, gave_up)
|
||||
# aren't crash signals — don't count them, but they also
|
||||
# don't break the crash streak.
|
||||
# Other outcomes neither count as crashes nor break the streak.
|
||||
continue
|
||||
if consecutive < threshold:
|
||||
return []
|
||||
task_id = _task_field(task, "id")
|
||||
actions: list[DiagnosticAction] = []
|
||||
if task_id:
|
||||
actions.append(DiagnosticAction(
|
||||
kind="cli_hint",
|
||||
label=f"Check logs: hermes kanban log {task_id}",
|
||||
payload={"command": f"hermes kanban log {task_id}"},
|
||||
suggested=True,
|
||||
))
|
||||
actions.append(_log_hint_action(task_id))
|
||||
running = _task_field(task, "status") == "running"
|
||||
actions.extend(_generic_recovery_actions(task, running=running))
|
||||
severity = "critical" if consecutive >= threshold * 2 else "error"
|
||||
# Put the actual error up-front so operators see WHAT broke without
|
||||
# having to open the logs. Truncate defensively — these can be huge
|
||||
# (full tracebacks).
|
||||
err_text = (last_err or "").strip() if last_err else ""
|
||||
err_snippet = err_text[:500] + ("…" if len(err_text) > 500 else "") if err_text else ""
|
||||
# Error up-front so operators see WHAT broke without opening the logs.
|
||||
err_snippet = _error_snippet(last_err)
|
||||
if err_snippet:
|
||||
title = f"Agent crashed {consecutive}x: {err_snippet.splitlines()[0][:160]}"
|
||||
detail = (
|
||||
@@ -751,14 +624,9 @@ def _rule_repeated_crashes(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
|
||||
|
||||
def _rule_review_dependency_deadlock(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
"""Detect a legacy review handoff that starves downstream children.
|
||||
|
||||
Older workers were instructed to sticky-block an implementation with a
|
||||
``review-required:`` reason. A separately modelled reviewer child cannot
|
||||
promote until that parent is terminal, so the lane has no autonomous next
|
||||
step. This compatibility diagnostic is graph-aware but deliberately leaves
|
||||
both the dependency graph and the user's sticky block unchanged.
|
||||
"""
|
||||
"""Legacy review handoff starving children: the implementation is
|
||||
sticky-blocked with a ``review-required:`` reason while todo children wait
|
||||
for it to be terminal. Graph-aware; deliberately mutates nothing."""
|
||||
if _task_field(task, "status") != "blocked":
|
||||
return []
|
||||
|
||||
@@ -829,21 +697,13 @@ def _rule_review_dependency_deadlock(task, events, runs, now, cfg) -> list[Diagn
|
||||
|
||||
|
||||
def _rule_stuck_in_blocked(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
"""Task has been in ``blocked`` status for too long without a comment.
|
||||
|
||||
Threshold: cfg["blocked_stale_hours"] (default 24).
|
||||
Surfaced as a warning so humans know there's a pending unblock.
|
||||
"""
|
||||
"""Blocked for >= cfg["blocked_stale_hours"] (default 24) with no comment
|
||||
or unblock since the last ``blocked`` event."""
|
||||
hours = float(cfg.get("blocked_stale_hours", 24))
|
||||
status = _task_field(task, "status")
|
||||
if status != "blocked":
|
||||
return []
|
||||
# Find the most recent ``blocked`` event.
|
||||
last_blocked_ts = 0
|
||||
for ev in events:
|
||||
if _event_kind(ev) == "blocked":
|
||||
t = _event_ts(ev)
|
||||
last_blocked_ts = max(last_blocked_ts, t)
|
||||
last_blocked_ts = _latest_event_ts(events, {"blocked"})
|
||||
if last_blocked_ts == 0:
|
||||
return []
|
||||
age_hours = (now - last_blocked_ts) / 3600.0
|
||||
@@ -879,29 +739,17 @@ def _rule_stuck_in_blocked(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
|
||||
|
||||
def _rule_block_unblock_cycling(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
"""Task has cycled through blocked → unblocked many times — the
|
||||
``unblock`` is not fixing the underlying problem and the worker
|
||||
keeps re-blocking for substantially the same reason.
|
||||
|
||||
``_rule_stuck_in_blocked`` resets its timer on any ``commented`` /
|
||||
``unblocked`` event, so a task that cycles every few minutes is
|
||||
invisible to it regardless of how many times it cycles (#29747
|
||||
gap 1). This rule complements that one by counting block→unblock
|
||||
cycles in a sliding window.
|
||||
|
||||
Threshold: cfg["block_cycle_threshold"] (default 3) cycles within
|
||||
cfg["block_cycle_window_seconds"] (default 24h).
|
||||
"""
|
||||
""">= cfg["block_cycle_threshold"] (default 3) blocked-after-unblocked
|
||||
cycles within cfg["block_cycle_window_seconds"] (default 24h). Complements
|
||||
``_rule_stuck_in_blocked``, whose timer any unblock resets, so fast cyclers
|
||||
are invisible to it."""
|
||||
threshold = _positive_int(cfg.get("block_cycle_threshold"), 3)
|
||||
window_seconds = float(cfg.get("block_cycle_window_seconds", 24 * 3600))
|
||||
cycle_cutoff = now - window_seconds
|
||||
|
||||
# Walk events chronologically (arrival order — callers pre-sort by
|
||||
# id, which is the canonical chronological order; ``created_at``
|
||||
# alone is insufficient because multiple events can share the same
|
||||
# second). Count "blocked after unblocked" transitions: every time
|
||||
# a blocked event follows at least one unblocked event since the
|
||||
# last cycle was counted, that's a new cycle.
|
||||
# Walk in id (arrival) order — created_at alone can't order events that
|
||||
# share a second. A blocked event after >= 1 unblocked since the last
|
||||
# counted cycle is a new cycle.
|
||||
cycles = 0
|
||||
seen_unblock_since_last_cycle = False
|
||||
initial_blocked_ts = 0
|
||||
@@ -956,66 +804,31 @@ def _rule_block_unblock_cycling(task, events, runs, now, cfg) -> list[Diagnostic
|
||||
|
||||
|
||||
def _rule_stranded_in_ready(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
"""Task has been in ``ready`` status for too long without any worker
|
||||
claiming it.
|
||||
|
||||
Threshold: cfg["stranded_threshold_seconds"] (default 1800 = 30 min).
|
||||
|
||||
Catches every "task waiting for a worker that never comes" case
|
||||
without caring WHY:
|
||||
|
||||
* Operator typo'd the assignee — no profile or external worker matches.
|
||||
* Profile was deleted, leaving its tasks stranded.
|
||||
* External worker pool (Codex CLI, Claude Code lane, custom daemon)
|
||||
is down, hung, or wasn't started.
|
||||
* Dispatcher is misconfigured (wrong board, wrong HERMES_HOME).
|
||||
|
||||
Pre-rule, all of these silently rotted in ``skipped_nonspawnable`` —
|
||||
the dispatcher correctly skipped them (good — no respawn loop) but
|
||||
nobody surfaced the fact that operator-actionable work was
|
||||
accumulating. The rule fires when a ready task's promoted-to-ready
|
||||
timestamp is older than the threshold AND the assignee is non-empty
|
||||
(truly unassigned tasks have their own ``skipped_unassigned`` signal
|
||||
on the dispatcher and a different operator response).
|
||||
|
||||
The signal is age-based on purpose: it's identity-agnostic, so it
|
||||
works for Hermes profiles, registered lanes, external workers, and
|
||||
typos uniformly. No registry to curate, no per-board allowlist.
|
||||
"""
|
||||
"""Assigned, unclaimed, ``ready`` for >= cfg["stranded_threshold_seconds"]
|
||||
(default 30 min). Deliberately age-based and identity-agnostic so it
|
||||
catches typo'd assignees, deleted profiles, and down external worker
|
||||
pools alike without a registry to curate. Unassigned tasks are excluded —
|
||||
the dispatcher's ``skipped_unassigned`` already covers them."""
|
||||
threshold_seconds = float(
|
||||
cfg.get("stranded_threshold_seconds", 30 * 60)
|
||||
)
|
||||
status = _task_field(task, "status")
|
||||
if status != "ready":
|
||||
return []
|
||||
# Skip tasks with a live claim — they're being worked on, even if
|
||||
# the worker hasn't reported progress yet (run-level liveness
|
||||
# extends the claim TTL; we don't want to second-guess that here).
|
||||
# A live claim means it's being worked on even without progress yet.
|
||||
if _task_field(task, "claim_lock"):
|
||||
return []
|
||||
assignee = _task_field(task, "assignee") or ""
|
||||
if not assignee.strip():
|
||||
# Unassigned tasks: the dispatcher's ``skipped_unassigned`` is
|
||||
# already the right signal. A separate diagnostic here would
|
||||
# double-flag the same condition.
|
||||
return []
|
||||
|
||||
# Find the most recent event that put this task into ready.
|
||||
# ``created`` covers tasks born ready; ``promoted`` covers parent-
|
||||
# done auto-promotion; ``reclaimed`` covers TTL/crash recovery;
|
||||
# ``unblocked`` covers human-driven resumes.
|
||||
READY_TRANSITION_KINDS = {
|
||||
"created", "promoted", "reclaimed", "unblocked",
|
||||
}
|
||||
last_ready_ts = 0
|
||||
for ev in events:
|
||||
if _event_kind(ev) in READY_TRANSITION_KINDS:
|
||||
t = _event_ts(ev)
|
||||
last_ready_ts = max(last_ready_ts, t)
|
||||
# Most recent event that put the task into ready.
|
||||
last_ready_ts = _latest_event_ts(
|
||||
events, {"created", "promoted", "reclaimed", "unblocked"},
|
||||
)
|
||||
|
||||
# Fallback: if no qualifying event exists (very old task or events
|
||||
# truncated), fall back to ``created_at`` on the task row. Better
|
||||
# to occasionally over-flag an ancient task than miss a stranded one.
|
||||
# No qualifying event (old task / truncated events): fall back to
|
||||
# created_at — over-flagging an ancient task beats missing a stranded one.
|
||||
if last_ready_ts == 0:
|
||||
last_ready_ts = int(_task_field(task, "created_at", default=0) or 0)
|
||||
if last_ready_ts == 0:
|
||||
@@ -1031,9 +844,7 @@ def _rule_stranded_in_ready(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
else:
|
||||
age_str = f"{int(age_seconds / 60)}m"
|
||||
|
||||
# Severity escalates with age. Below 2x threshold = warning;
|
||||
# 2x – 6x = error; beyond 6x = critical (something is clearly
|
||||
# broken, not just slow).
|
||||
# Escalate with age: <2x threshold warning, 2x-6x error, >6x critical.
|
||||
if age_seconds >= threshold_seconds * 6:
|
||||
severity = "critical"
|
||||
elif age_seconds >= threshold_seconds * 2:
|
||||
@@ -1078,8 +889,7 @@ def _rule_stranded_in_ready(task, events, runs, now, cfg) -> list[Diagnostic]:
|
||||
)]
|
||||
|
||||
|
||||
# Registry — order matters: rules higher on the list render first when
|
||||
# severity ties. Add new rules here.
|
||||
# Order matters: earlier rules render first on severity ties.
|
||||
_RULES: list[RuleFn] = [
|
||||
_rule_hallucinated_cards,
|
||||
_rule_triage_aux_unavailable,
|
||||
@@ -1093,21 +903,6 @@ _RULES: list[RuleFn] = [
|
||||
]
|
||||
|
||||
|
||||
# Known kinds (for the UI's filter / legend / i18n keys). Update when
|
||||
# rules are added.
|
||||
DIAGNOSTIC_KINDS = (
|
||||
"hallucinated_cards",
|
||||
"triage_aux_unavailable",
|
||||
"prose_phantom_refs",
|
||||
"repeated_failures",
|
||||
"repeated_crashes",
|
||||
"review_dependency_deadlock",
|
||||
"stuck_in_blocked",
|
||||
"block_unblock_cycling",
|
||||
"stranded_in_ready",
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_CONFIG = {
|
||||
# Match the dispatcher default (kanban.failure_limit) so repeated-failure
|
||||
# diagnostics do not lag behind the default auto-block threshold.
|
||||
@@ -1116,21 +911,16 @@ DEFAULT_CONFIG = {
|
||||
"spawn_failure_threshold": 2,
|
||||
"crash_threshold": 2,
|
||||
"blocked_stale_hours": 24,
|
||||
# Stranded-task threshold. 30 min by default — below that, the
|
||||
# signal is dominated by tasks that are about to be claimed on the
|
||||
# next dispatcher tick (default 60s) and would just be noise.
|
||||
# Below 30 min the signal is dominated by tasks about to be claimed on
|
||||
# the next dispatcher tick.
|
||||
"stranded_threshold_seconds": 30 * 60,
|
||||
}
|
||||
|
||||
|
||||
def config_from_kanban_config(kanban_cfg: Optional[dict]) -> dict:
|
||||
"""Build diagnostics config from the runtime ``kanban`` config section.
|
||||
|
||||
``kanban.diagnostics.failure_threshold`` remains an explicit override.
|
||||
Otherwise, derive the repeated-failure threshold from
|
||||
``kanban.failure_limit`` so CLI/dashboard diagnostics match the
|
||||
dispatcher's actual circuit-breaker threshold.
|
||||
"""
|
||||
"""Diagnostics config from the ``kanban`` section. ``kanban.diagnostics.
|
||||
failure_threshold`` is an explicit override; otherwise the threshold is
|
||||
``kanban.failure_limit`` so diagnostics match the dispatcher's breaker."""
|
||||
kanban_cfg = kanban_cfg or {}
|
||||
diag_cfg = dict(kanban_cfg.get("diagnostics") or {})
|
||||
diag_cfg.setdefault(
|
||||
@@ -1146,13 +936,9 @@ def config_from_kanban_config(kanban_cfg: Optional[dict]) -> dict:
|
||||
|
||||
|
||||
def config_from_runtime_config(raw_config: Optional[dict]) -> dict:
|
||||
"""Build diagnostics config from the full Hermes runtime config.
|
||||
|
||||
Carries through ``kanban``, ``auxiliary``, and ``model`` keys so triage-
|
||||
aware rules can inspect the active aux-helper and main-model state.
|
||||
Folds the ``kanban`` block through ``config_from_kanban_config`` so the
|
||||
repeated-failure threshold derivation still applies.
|
||||
"""
|
||||
"""Diagnostics config from the full runtime config: folds ``kanban`` through
|
||||
``config_from_kanban_config`` and carries ``kanban``/``auxiliary``/``model``
|
||||
through for the triage-aware rules."""
|
||||
raw_config = raw_config or {}
|
||||
if not isinstance(raw_config, dict):
|
||||
return {}
|
||||
@@ -1177,12 +963,8 @@ def compute_task_diagnostics(
|
||||
config: Optional[dict] = None,
|
||||
graph: Optional[dict] = None,
|
||||
) -> list[Diagnostic]:
|
||||
"""Run every rule against a single task's state and return a
|
||||
severity-sorted list of active diagnostics.
|
||||
|
||||
Sorting: critical first, then error, then warning; ties broken by
|
||||
most-recent ``last_seen_at``.
|
||||
"""
|
||||
"""Run every rule for one task; critical first, then error, warning; ties
|
||||
broken by most-recent ``last_seen_at``."""
|
||||
now_ts = int(now if now is not None else time.time())
|
||||
config = config or {}
|
||||
cfg = {**DEFAULT_CONFIG, **config}
|
||||
@@ -1202,9 +984,7 @@ def compute_task_diagnostics(
|
||||
try:
|
||||
out.extend(rule(task, events, runs, now_ts, cfg))
|
||||
except Exception:
|
||||
# A broken rule must never crash the dashboard. Rule bugs
|
||||
# get caught in tests; in production we'd rather drop the
|
||||
# diagnostic than 500 a whole /board request.
|
||||
# A broken rule must never 500 a whole /board request.
|
||||
continue
|
||||
severity_idx = {s: i for i, s in enumerate(SEVERITY_ORDER)}
|
||||
out.sort(
|
||||
|
||||
@@ -1,32 +1,13 @@
|
||||
"""Kanban triage specifier — flesh out a one-liner into a real spec.
|
||||
|
||||
Used by ``hermes kanban specify [task_id | --all]``. Takes a task that
|
||||
lives in the Triage column (a rough idea, typically only a title), calls
|
||||
the auxiliary LLM to produce:
|
||||
``hermes kanban specify [task_id | --all]`` asks the auxiliary LLM for a
|
||||
tightened title + concrete body for a Triage task, then flips it
|
||||
``triage -> todo`` via ``kanban_db.specify_triage_task``.
|
||||
|
||||
* A tightened title (optional — only replaces if the model proposes a
|
||||
materially different one)
|
||||
* A concrete body: goal, proposed approach, acceptance criteria
|
||||
|
||||
and then flips the task ``triage -> todo`` via
|
||||
``kanban_db.specify_triage_task``. The dispatcher promotes it to
|
||||
``ready`` on its next tick (or immediately if there are no open parents).
|
||||
|
||||
Design notes
|
||||
------------
|
||||
|
||||
* This module intentionally mirrors ``hermes_cli/goals.py`` — same aux
|
||||
client pattern, same "empty config => skip, don't crash" tolerance.
|
||||
Keeps the surface area tiny and the failure modes predictable.
|
||||
|
||||
* The prompt is a short system + user pair. We ask for JSON with
|
||||
``{title, body}``; if parsing fails, we fall back to treating the
|
||||
whole response as the body and leave the title untouched. No
|
||||
retry loop — one shot, keep cost bounded.
|
||||
|
||||
* Structured output / JSON mode is not requested explicitly so the
|
||||
specifier works on providers that don't implement it. The parse
|
||||
is lenient (tolerates markdown code fences around the JSON).
|
||||
Mirrors ``hermes_cli/goals.py``: same aux-client pattern, same "empty config
|
||||
=> skip, don't crash" tolerance. One shot, no retry loop. JSON mode is not
|
||||
requested (works on providers without it); the parse is lenient and falls
|
||||
back to "whole reply is the body" so a malformed reply never strands a task.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -108,13 +89,12 @@ def _truncate(text: str, limit: int) -> str:
|
||||
_FENCE_RE = re.compile(r"^\s*```(?:json)?\s*|\s*```\s*$", re.IGNORECASE)
|
||||
|
||||
|
||||
def _extract_json_blob(raw: str) -> Optional[dict]:
|
||||
"""Lenient JSON extraction — tolerates fenced code blocks and
|
||||
leading/trailing whitespace. Returns None if nothing parses."""
|
||||
def _extract_json_blob(raw: str, fence_re: re.Pattern = _FENCE_RE) -> Optional[dict]:
|
||||
"""Lenient JSON object extraction: strip code fences, take the first ``{``
|
||||
to the last ``}``. None if nothing parses to a dict."""
|
||||
if not raw:
|
||||
return None
|
||||
stripped = _FENCE_RE.sub("", raw.strip())
|
||||
# Greedy: find the first `{` and last `}` and try that slice.
|
||||
stripped = fence_re.sub("", raw.strip())
|
||||
first = stripped.find("{")
|
||||
last = stripped.rfind("}")
|
||||
if first == -1 or last == -1 or last <= first:
|
||||
@@ -124,19 +104,24 @@ def _extract_json_blob(raw: str) -> Optional[dict]:
|
||||
val = json.loads(candidate)
|
||||
except (ValueError, json.JSONDecodeError):
|
||||
return None
|
||||
if not isinstance(val, dict):
|
||||
return None
|
||||
return val
|
||||
return val if isinstance(val, dict) else None
|
||||
|
||||
|
||||
def _profile_author() -> str:
|
||||
def _nonblank(v) -> Optional[str]:
|
||||
return v if isinstance(v, str) and v.strip() else None
|
||||
|
||||
|
||||
def _title_body(parsed: dict) -> tuple[Optional[str], Optional[str]]:
|
||||
"""``(title, body)`` from an LLM reply: title stripped, body verbatim,
|
||||
either None when missing/blank."""
|
||||
title = _nonblank(parsed.get("title"))
|
||||
return (title.strip() if title else None), _nonblank(parsed.get("body"))
|
||||
|
||||
|
||||
def _profile_author(default: str = "specifier") -> str:
|
||||
"""Mirror of ``hermes_cli.kanban._profile_author``. Kept local to
|
||||
avoid a circular import when kanban.py imports this module."""
|
||||
return (
|
||||
os.environ.get("HERMES_PROFILE")
|
||||
or os.environ.get("USER")
|
||||
or "specifier"
|
||||
)
|
||||
return os.environ.get("HERMES_PROFILE") or os.environ.get("USER") or default
|
||||
|
||||
|
||||
def specify_task(
|
||||
@@ -145,13 +130,9 @@ def specify_task(
|
||||
author: Optional[str] = None,
|
||||
timeout: Optional[int] = None,
|
||||
) -> SpecifyOutcome:
|
||||
"""Specify a single triage task and promote it to ``todo``.
|
||||
|
||||
Returns an outcome describing what happened. Never raises for expected
|
||||
failure modes (task not in triage, no aux client configured, API
|
||||
error, malformed response) — those surface via ``ok=False`` so the
|
||||
``--all`` sweep can continue past individual failures.
|
||||
"""
|
||||
"""Specify one triage task and promote it to ``todo``. Expected failures
|
||||
(not in triage, no aux client, API error, malformed reply) surface as
|
||||
``ok=False`` so an ``--all`` sweep continues."""
|
||||
with kb.connect_closing() as conn:
|
||||
task = kb.get_task(conn, task_id)
|
||||
if task is None:
|
||||
@@ -174,9 +155,8 @@ def specify_task(
|
||||
)
|
||||
|
||||
try:
|
||||
# Route through call_llm so auxiliary.triage_specifier.* config
|
||||
# (provider/model/base_url, extra_body, reasoning_effort, retries)
|
||||
# all apply — the direct-create path dropped extra_body (#35566).
|
||||
# call_llm applies all auxiliary.triage_specifier.* config
|
||||
# (provider/model/base_url, extra_body, reasoning_effort, retries).
|
||||
resp = call_llm(
|
||||
task="triage_specifier",
|
||||
messages=[
|
||||
@@ -206,9 +186,7 @@ def specify_task(
|
||||
new_title: Optional[str]
|
||||
new_body: Optional[str]
|
||||
if parsed is None:
|
||||
# Fall back: treat the whole reply as the body, leave title as-is.
|
||||
# Worst case the user edits afterward — still better than stranding
|
||||
# the task in triage on a malformed LLM reply.
|
||||
# Whole reply becomes the body; the user can edit afterward.
|
||||
stripped_raw = raw.strip()
|
||||
if not stripped_raw:
|
||||
return SpecifyOutcome(
|
||||
@@ -217,16 +195,7 @@ def specify_task(
|
||||
new_title = None
|
||||
new_body = stripped_raw
|
||||
else:
|
||||
title_val = parsed.get("title")
|
||||
body_val = parsed.get("body")
|
||||
new_title = (
|
||||
title_val.strip()
|
||||
if isinstance(title_val, str) and title_val.strip()
|
||||
else None
|
||||
)
|
||||
new_body = (
|
||||
body_val if isinstance(body_val, str) and body_val.strip() else None
|
||||
)
|
||||
new_title, new_body = _title_body(parsed)
|
||||
if new_body is None and new_title is None:
|
||||
return SpecifyOutcome(
|
||||
task_id, False, "LLM response missing title and body"
|
||||
@@ -241,8 +210,7 @@ def specify_task(
|
||||
author=author or _profile_author(),
|
||||
)
|
||||
if not ok:
|
||||
# Race: someone else promoted / archived the task between our
|
||||
# read above and the write. Report, don't crash.
|
||||
# Race: promoted/archived between our read and the write.
|
||||
return SpecifyOutcome(
|
||||
task_id, False, "task moved out of triage before promotion"
|
||||
)
|
||||
@@ -250,10 +218,7 @@ def specify_task(
|
||||
|
||||
|
||||
def list_triage_ids(*, tenant: Optional[str] = None) -> list[str]:
|
||||
"""Return task ids currently in the triage column.
|
||||
|
||||
``tenant`` narrows the sweep; ``None`` returns every triage task.
|
||||
"""
|
||||
"""Task ids in the triage column; ``tenant`` narrows the sweep."""
|
||||
with kb.connect_closing() as conn:
|
||||
tasks = kb.list_tasks(
|
||||
conn,
|
||||
|
||||
+32
-44
@@ -19,6 +19,7 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
import json
|
||||
import sqlite3
|
||||
import time
|
||||
from typing import Any, Iterable, Optional
|
||||
|
||||
from hermes_cli import kanban_db as kb
|
||||
@@ -83,16 +84,12 @@ def _activate_root_inline(
|
||||
) -> bool:
|
||||
"""Inline blocked→done CAS flip + event insert for the swarm root.
|
||||
|
||||
Runs INSIDE create_swarm's outer write_txn, so it must not call
|
||||
``kb.complete_task`` — that helper opens its own transaction and fires
|
||||
post-commit side effects (workspace cleanup, failure-counter clear,
|
||||
``recompute_ready``) that would execute while the outer transaction can
|
||||
still roll back. Instead we do the minimal durable writes here and let
|
||||
the caller run ``recompute_ready`` after the outer commit.
|
||||
Runs INSIDE create_swarm's write_txn, so it must not call
|
||||
``kb.complete_task`` (own transaction + post-commit side effects that
|
||||
would run while the outer txn can still roll back). The caller runs
|
||||
``recompute_ready`` after the outer commit.
|
||||
"""
|
||||
import time as _time
|
||||
|
||||
now = int(_time.time())
|
||||
now = int(time.time())
|
||||
cur = conn.execute(
|
||||
"""
|
||||
UPDATE tasks
|
||||
@@ -179,9 +176,8 @@ def create_swarm(
|
||||
raise RuntimeError("could not activate the completed swarm topology")
|
||||
activated = True
|
||||
if activated:
|
||||
# Outside the outer transaction: promote the root's children now
|
||||
# that its 'done' flip is durable (recompute_ready opens its own
|
||||
# txn and must never run under an open write_txn).
|
||||
# After commit: recompute_ready opens its own txn and must never run
|
||||
# under an open write_txn.
|
||||
kb.recompute_ready(conn)
|
||||
root = kb.get_task(conn, created.root_id)
|
||||
run = kb.latest_run(conn, created.root_id)
|
||||
@@ -249,9 +245,8 @@ def _create_swarm_uncommitted(
|
||||
workspace_path=workspace_path,
|
||||
)
|
||||
|
||||
# If idempotency returned an existing non-archived root, do not duplicate the
|
||||
# swarm graph. Recover the topology from the root's latest blackboard, if it
|
||||
# was created by this helper previously.
|
||||
# Idempotency may return an existing root: recover its topology from the
|
||||
# blackboard instead of duplicating the graph.
|
||||
existing = latest_blackboard(conn, root).get("topology")
|
||||
if isinstance(existing, dict):
|
||||
worker_ids = [str(x) for x in existing.get("worker_ids", []) if x]
|
||||
@@ -266,61 +261,54 @@ def _create_swarm_uncommitted(
|
||||
)
|
||||
|
||||
context_suffix = _swarm_context(root, goal)
|
||||
worker_ids: list[str] = []
|
||||
for spec in worker_specs:
|
||||
worker_id = kb.create_task(
|
||||
common = dict(
|
||||
created_by=created_by, tenant=tenant,
|
||||
workspace_kind=workspace_kind, workspace_path=workspace_path,
|
||||
)
|
||||
worker_ids = [
|
||||
kb.create_task(
|
||||
conn,
|
||||
title=spec.title,
|
||||
body=(spec.body or "") + context_suffix,
|
||||
assignee=spec.profile,
|
||||
created_by=created_by,
|
||||
parents=[root],
|
||||
tenant=tenant,
|
||||
priority=spec.priority or priority,
|
||||
workspace_kind=workspace_kind,
|
||||
workspace_path=workspace_path,
|
||||
skills=spec.skills or None,
|
||||
max_runtime_seconds=spec.max_runtime_seconds,
|
||||
**common,
|
||||
)
|
||||
worker_ids.append(worker_id)
|
||||
for spec in worker_specs
|
||||
]
|
||||
|
||||
verifier_body = (
|
||||
"Review every worker handoff and blackboard update. Gate the swarm: "
|
||||
"complete only with metadata {\"gate\": \"pass\"} when evidence is "
|
||||
"sufficient; otherwise block with exact missing work."
|
||||
+ context_suffix
|
||||
)
|
||||
verifier = kb.create_task(
|
||||
conn,
|
||||
title=verifier_title,
|
||||
body=verifier_body,
|
||||
body=(
|
||||
"Review every worker handoff and blackboard update. Gate the swarm: "
|
||||
"complete only with metadata {\"gate\": \"pass\"} when evidence is "
|
||||
"sufficient; otherwise block with exact missing work."
|
||||
+ context_suffix
|
||||
),
|
||||
assignee=verifier_assignee,
|
||||
created_by=created_by,
|
||||
parents=worker_ids,
|
||||
tenant=tenant,
|
||||
priority=priority,
|
||||
workspace_kind=workspace_kind,
|
||||
workspace_path=workspace_path,
|
||||
skills=["requesting-code-review"],
|
||||
**common,
|
||||
)
|
||||
|
||||
synthesizer_body = (
|
||||
"Synthesize the verified worker outputs into the final deliverable. "
|
||||
"Do not start until the verifier has passed the gate."
|
||||
+ context_suffix
|
||||
)
|
||||
synthesizer = kb.create_task(
|
||||
conn,
|
||||
title=synthesizer_title,
|
||||
body=synthesizer_body,
|
||||
body=(
|
||||
"Synthesize the verified worker outputs into the final deliverable. "
|
||||
"Do not start until the verifier has passed the gate."
|
||||
+ context_suffix
|
||||
),
|
||||
assignee=synthesizer_assignee,
|
||||
created_by=created_by,
|
||||
parents=[verifier],
|
||||
tenant=tenant,
|
||||
priority=priority,
|
||||
workspace_kind=workspace_kind,
|
||||
workspace_path=workspace_path,
|
||||
skills=["humanizer"],
|
||||
**common,
|
||||
)
|
||||
|
||||
created = SwarmCreated(root, worker_ids, verifier, synthesizer)
|
||||
|
||||
@@ -14,30 +14,21 @@ source board's slug)::
|
||||
attachments/<task>/… attachment blobs (unless --no-attachments)
|
||||
logs/<task>.log worker logs (only with --include-logs)
|
||||
|
||||
Two things make this more than a ``tar czf`` of the board directory.
|
||||
Two things make this more than ``tar czf`` of the board directory:
|
||||
|
||||
**The database is live.** Kanban runs in WAL mode and a dispatcher may be
|
||||
mid-write, so copying ``kanban.db`` off the filesystem yields a torn
|
||||
snapshot that is missing whatever still sits in the ``-wal`` file. Export
|
||||
goes through SQLite's online-backup API instead, which produces a
|
||||
consistent single-file image of a database that is being written to.
|
||||
* **The database is live** (WAL mode, dispatcher may be mid-write), so the
|
||||
export uses SQLite's online-backup API for a consistent image instead of a
|
||||
file copy that would miss the ``-wal`` sidecar.
|
||||
* **Rows carry machine-local state** — claims, PIDs, heartbeats, absolute
|
||||
paths, gateway chat subscriptions, session ids. Shipping them verbatim
|
||||
would import claims owned by a stranger's process or push events into a
|
||||
stranger's Telegram thread. Scrubbed on export and re-scrubbed on import
|
||||
(an archive is untrusted); see :func:`_scrub_local_state` and
|
||||
:func:`_relocate_imported_rows`.
|
||||
|
||||
**Rows carry machine-local state.** Claims, PIDs, heartbeats, absolute
|
||||
workspace and attachment paths, gateway chat subscriptions, and session
|
||||
ids are all meaningful only on the machine that wrote them. Shipping them
|
||||
verbatim is how an imported board arrives holding claims owned by a
|
||||
process on somebody else's laptop, or starts pushing task events into a
|
||||
stranger's Telegram thread. Everything machine-local is scrubbed on the
|
||||
export side (so the archive itself never carries it) and defensively
|
||||
re-scrubbed on import; see :func:`_scrub_local_state` and
|
||||
:func:`_relocate_imported_rows`.
|
||||
|
||||
Imports always land as a **new** board — the slug auto-suffixes on
|
||||
collision — so an import can never mutate a board that is already there.
|
||||
That also means an imported board is never ``default``, which is what
|
||||
lets the import side ignore the default board's split on-disk layout
|
||||
(``<root>/kanban.db`` beside ``<root>/kanban/attachments/``) and put
|
||||
everything inside one ``boards/<slug>/`` directory.
|
||||
Imports always land as a **new** board (slug auto-suffixes on collision), so
|
||||
an import never mutates an existing board and is never ``default`` — which
|
||||
lets the importer ignore the default board's split on-disk layout.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -74,13 +65,8 @@ _DISPATCHABLE_STATUSES = ("ready", "running", "todo", "scheduled")
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _snapshot_db(source: Path, target: Path) -> None:
|
||||
"""Write a consistent copy of ``source`` to ``target``.
|
||||
|
||||
Uses SQLite's online-backup API rather than a file copy: in WAL mode
|
||||
a just-committed page can still live in the ``-wal`` sidecar, so
|
||||
copying only ``kanban.db`` loses recent writes and can produce a
|
||||
torn image if the dispatcher commits mid-copy.
|
||||
"""
|
||||
"""Consistent copy of ``source`` via the online-backup API (a file copy
|
||||
would miss pages still in the ``-wal`` sidecar and could tear)."""
|
||||
src = sqlite3.connect(str(source))
|
||||
try:
|
||||
dst = sqlite3.connect(str(target))
|
||||
@@ -93,14 +79,9 @@ def _snapshot_db(source: Path, target: Path) -> None:
|
||||
|
||||
|
||||
def _scrub_local_state(conn: sqlite3.Connection) -> None:
|
||||
"""Strip machine-local runtime state. Caller owns the transaction.
|
||||
|
||||
Runs on the export side so the archive itself never carries another
|
||||
machine's claims, PIDs, or — the one that actually matters for a
|
||||
board shared with someone else — the gateway chat ids subscribed to
|
||||
its task events. Repeated on import because an archive is untrusted
|
||||
input.
|
||||
"""
|
||||
"""Strip machine-local runtime state (claims, PIDs, and above all the
|
||||
gateway chat ids subscribed to task events). Caller owns the transaction.
|
||||
Run on export and again on import (an archive is untrusted input)."""
|
||||
conn.execute("DELETE FROM kanban_notify_subs")
|
||||
conn.execute(
|
||||
"""
|
||||
@@ -157,12 +138,9 @@ def export_board(
|
||||
include_attachments: bool = True,
|
||||
include_logs: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Export ``board`` to a ``tar.gz`` archive. Returns a summary dict.
|
||||
|
||||
``output_path`` may be given with or without the ``.tar.gz`` suffix.
|
||||
Workspaces are never included: they are git worktrees and scratch
|
||||
trees that are large, machine-local, and rebuilt on demand.
|
||||
"""
|
||||
"""Export ``board`` to a ``tar.gz`` (suffix optional on ``output_path``);
|
||||
returns a summary dict. Workspaces are never included — large,
|
||||
machine-local, rebuilt on demand."""
|
||||
slug = kb._normalize_board_slug(board) or kb.get_current_board()
|
||||
if not kb.board_exists(slug):
|
||||
raise ValueError(f"board {slug!r} does not exist")
|
||||
@@ -241,12 +219,8 @@ def export_board(
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _available_slug(preferred: str) -> str:
|
||||
"""Return ``preferred``, or the first free ``<preferred>-N`` variant.
|
||||
|
||||
``default`` always reports as existing, so an archive exported from a
|
||||
default board naturally lands as ``default-2`` instead of colliding
|
||||
with the importer's own default board.
|
||||
"""
|
||||
"""``preferred`` or the first free ``<preferred>-N``. ``default`` always
|
||||
exists, so a default-board export lands as ``default-2``."""
|
||||
if not kb.board_exists(preferred):
|
||||
return preferred
|
||||
# Leave headroom for the suffix inside the 64-char slug limit.
|
||||
@@ -295,22 +269,17 @@ def _read_board_metadata(path: Path) -> dict[str, Any]:
|
||||
def _relocate_imported_rows(
|
||||
conn: sqlite3.Connection, slug: str
|
||||
) -> tuple[dict[str, int], list[str]]:
|
||||
"""Re-anchor an imported board's rows to this machine.
|
||||
"""Re-anchor an imported board's rows to this machine; returns
|
||||
``(stats, warnings)``.
|
||||
|
||||
Returns ``(stats, warnings)``. Three things move:
|
||||
|
||||
* Attachment rows are repointed at this board's attachments tree.
|
||||
Rows whose blob did not travel (an export made with
|
||||
``--no-attachments``) are dropped, because a row pointing at a file
|
||||
that does not exist breaks download in every UI that lists it.
|
||||
* Workspace paths are cleared. ``scratch`` tasks regenerate one under
|
||||
this board on the next claim, so they are simply reset. ``dir`` and
|
||||
``worktree`` tasks cannot be resolved without a path that means
|
||||
something here, so any that are still dispatchable are parked in
|
||||
``triage`` — otherwise the dispatcher claims them, fails to build a
|
||||
workspace, and burns them straight into the failure breaker.
|
||||
* Runtime state is scrubbed again. Export already did this, but an
|
||||
archive is an untrusted input and the cost is one UPDATE.
|
||||
* Attachment rows are repointed at this board's tree; rows whose blob
|
||||
did not travel (``--no-attachments``) are dropped, since a dangling row
|
||||
breaks download in every UI.
|
||||
* Workspace paths are cleared. ``scratch`` regenerates on next claim;
|
||||
dispatchable ``dir``/``worktree`` tasks are parked in ``triage``,
|
||||
otherwise the dispatcher claims them, fails to build a workspace, and
|
||||
burns them into the failure breaker.
|
||||
* Runtime state is scrubbed again (untrusted input, one UPDATE).
|
||||
"""
|
||||
warnings: list[str] = []
|
||||
now = int(time.time())
|
||||
@@ -389,12 +358,8 @@ def import_board(
|
||||
*,
|
||||
activate: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Import a board archive as a new board. Returns a summary dict.
|
||||
|
||||
``slug`` overrides the name from the archive. Either way the final
|
||||
slug auto-suffixes if it is taken, so an import never merges into or
|
||||
overwrites an existing board.
|
||||
"""
|
||||
"""Import an archive as a NEW board (``slug`` overrides the archive's;
|
||||
either way it auto-suffixes if taken). Returns a summary dict."""
|
||||
archive = Path(archive_path).expanduser()
|
||||
if not archive.exists():
|
||||
raise FileNotFoundError(f"archive not found: {archive}")
|
||||
|
||||
+367
-11202
File diff suppressed because it is too large
Load Diff
+179
-282
@@ -1,9 +1,7 @@
|
||||
"""Shared module-level constants for the SessionDB family of modules.
|
||||
"""Shared constants and helpers for the SessionDB family of modules.
|
||||
|
||||
Extracted verbatim from hermes_state.py so the SessionDB mixin modules
|
||||
(hermes_state_search / hermes_state_schema / hermes_state_portability) can
|
||||
reference them without importing hermes_state (which would be a cycle).
|
||||
hermes_state re-imports every name here for backward compatibility.
|
||||
Lives outside hermes_state so the mixin modules can import it without a
|
||||
cycle; hermes_state re-exports every name for backward compatibility.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
@@ -29,16 +27,11 @@ from agent.context_compressor import (
|
||||
)
|
||||
|
||||
|
||||
# Session preview = the head of the first user message, shown wherever a
|
||||
# session has no title (sidebar rows, pickers, exports, the desktop's
|
||||
# `sessionTitle` fallback).
|
||||
#
|
||||
# A /skill invocation expands into a message that embeds the whole skill body,
|
||||
# so the plain head of it previews the SKILL's opening prose as if the user had
|
||||
# written it. Scaffolded rows therefore carry a wider excerpt so
|
||||
# ``_shape_preview`` can hand it to ``describe_skill_invocation`` and recover
|
||||
# ``/work — fix the title leak``: the whole message while it stays under the
|
||||
# budget, and head + tail (where the typed instruction lands) once it doesn't.
|
||||
# Session preview = head of the first user message, shown wherever a session
|
||||
# has no title. A /skill invocation embeds the whole skill body, so its plain
|
||||
# head would preview the SKILL's prose; scaffolded rows carry a wider excerpt
|
||||
# (whole message under budget, else head + tail where the typed instruction
|
||||
# lands) so ``_shape_preview`` can recover ``/work — fix the title leak``.
|
||||
_PREVIEW_HEAD_CHARS = 63
|
||||
|
||||
|
||||
@@ -49,13 +42,9 @@ _PREVIEW_MAX_CHARS = 60
|
||||
|
||||
|
||||
def escape_like(text: str) -> str:
|
||||
"""Escape SQL LIKE wildcards so operator/session-derived text matches
|
||||
literally. Pair with ``ESCAPE '\\'`` in the clause.
|
||||
|
||||
``%`` and ``_`` are wildcards to LIKE, and ``_`` in particular is common
|
||||
in the values these patterns run against (branch names, session titles,
|
||||
filesystem paths). A match documented as substring/prefix must not
|
||||
silently widen.
|
||||
"""Escape LIKE wildcards (``%``, ``_``) so derived text matches literally;
|
||||
pair with ``ESCAPE '\\'``. ``_`` is common in branch names, titles and
|
||||
paths, and a documented substring/prefix match must not silently widen.
|
||||
"""
|
||||
return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
@@ -90,10 +79,9 @@ def _sql_starts_with(expression: str, prefixes: tuple[str, ...]) -> str:
|
||||
return "(" + " OR ".join(checks) + ")"
|
||||
|
||||
|
||||
# Current and historical long-form prefixes share this complete introduction;
|
||||
# their stale-item guidance diverges only after it. Matching the whole intro
|
||||
# avoids treating an ordinary user message that merely starts with the short
|
||||
# bracketed label as a compaction carrier.
|
||||
# Current and legacy long-form prefixes share this whole introduction; matching
|
||||
# all of it keeps an ordinary message that merely starts with the bracketed
|
||||
# label from counting as a compaction carrier.
|
||||
_PREVIEW_LONG_FORM_PREFIX = SUMMARY_PREFIX.split("Do NOT answer", 1)[0]
|
||||
_PREVIEW_SUMMARY_PREFIXES = (
|
||||
_PREVIEW_LONG_FORM_PREFIX,
|
||||
@@ -127,9 +115,8 @@ _PREVIEW_FORCE_USER_REMAINDER_SQL = (
|
||||
f" + {len(_SUMMARY_END_MARKER)})"
|
||||
)
|
||||
|
||||
# Session preview subqueries select their first eligible user-authored content.
|
||||
# Pure compaction rows are ineligible; force-user-leading and merged carriers
|
||||
# remain eligible only when authentic content survives the wire boundary.
|
||||
# Pure compaction rows are ineligible for previews; force-user-leading and
|
||||
# merged carriers are eligible only when authentic content survives.
|
||||
_PREVIEW_ELIGIBLE_SQL = (
|
||||
f"((NOT {_PREVIEW_STANDALONE_SUMMARY_SQL} AND NOT {_PREVIEW_MERGED_SUMMARY_SQL})"
|
||||
f" OR ({_PREVIEW_STANDALONE_SUMMARY_SQL}"
|
||||
@@ -140,10 +127,8 @@ _PREVIEW_ELIGIBLE_SQL = (
|
||||
)
|
||||
|
||||
|
||||
# The shared ``_preview_raw`` SELECT expression, interpolated by every listing
|
||||
# query. A scaffolded row gets a wider excerpt: the whole message while it fits
|
||||
# the budget, else head + tail (where the typed instruction lands) spliced
|
||||
# around SKILL_EXCERPT_JOINT.
|
||||
# Shared ``_preview_raw`` SELECT expression for every listing query (scaffolded
|
||||
# rows: head + tail spliced around SKILL_EXCERPT_JOINT when over budget).
|
||||
_PREVIEW_RAW_SELECT = (
|
||||
f"CASE WHEN {_PREVIEW_STANDALONE_SUMMARY_SQL}"
|
||||
f" THEN {_PREVIEW_FORCE_USER_REMAINDER_SQL}"
|
||||
@@ -173,8 +158,17 @@ def _shape_preview(raw: Any) -> str:
|
||||
return text
|
||||
|
||||
|
||||
# A child session counts as a /branch (kept visible, never cascade-deleted) if
|
||||
# it carries the stable marker OR the legacy end_reason heuristic holds.
|
||||
# Correlated ``_preview_raw`` column for a ``sessions s`` row.
|
||||
_PREVIEW_RAW_SUBQUERY_SQL = (
|
||||
f"COALESCE((SELECT {_PREVIEW_RAW_SELECT} FROM messages m"
|
||||
f" WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL"
|
||||
f" AND {_PREVIEW_ELIGIBLE_SQL}"
|
||||
f" ORDER BY m.timestamp, m.id LIMIT 1), '') AS _preview_raw"
|
||||
)
|
||||
|
||||
|
||||
# A /branch child (kept visible, never cascade-deleted): stable marker OR the
|
||||
# legacy end_reason heuristic.
|
||||
_BRANCH_CHILD_SQL = (
|
||||
"json_extract(COALESCE({a}.model_config, '{{}}'), '$._branched_from') IS NOT NULL"
|
||||
" OR EXISTS (SELECT 1 FROM sessions p"
|
||||
@@ -193,12 +187,10 @@ _COMPRESSION_CHILD_SQL = (
|
||||
|
||||
_RESET_END_REASONS = (
|
||||
"session_reset",
|
||||
# switch_session() never creates a child row, but pre-marker DBs can hold
|
||||
# legacy reset children whose parent later ended with 'session_switch'
|
||||
# (resumed then switched away before reopen-time stamping existed). Also
|
||||
# keeps this set identical to the recovery fence in
|
||||
# find_latest_gateway_session_for_peer, which interpolates
|
||||
# _RESET_END_REASONS_SQL so the two cannot drift.
|
||||
# switch_session() creates no child row, but pre-marker DBs hold legacy
|
||||
# reset children whose parent later ended 'session_switch'. Also keeps
|
||||
# this set identical to the recovery fence in
|
||||
# find_latest_gateway_session_for_peer (which interpolates the SQL form).
|
||||
"session_switch",
|
||||
"idle",
|
||||
"daily",
|
||||
@@ -207,34 +199,25 @@ _RESET_END_REASONS = (
|
||||
)
|
||||
_RESET_END_REASONS_SQL = ", ".join(f"'{reason}'" for reason in _RESET_END_REASONS)
|
||||
|
||||
# Accidental end reasons that recovery treats as resumable (see
|
||||
# docs/session-lifecycle.md "recoverable accidental reasons"). Interpolated
|
||||
# into the recovery SQL below AND exposed as SessionDB.RECOVERABLE_END_REASONS
|
||||
# so the tuple is the single source of truth — literals cannot drift.
|
||||
# Accidental end reasons recovery treats as resumable (docs/session-lifecycle.md).
|
||||
# Single source of truth: interpolated into recovery SQL AND exposed as
|
||||
# SessionDB.RECOVERABLE_END_REASONS.
|
||||
_RECOVERABLE_END_REASONS = (
|
||||
"agent_close",
|
||||
"ws_orphan_reap",
|
||||
# A stale sentinel-parked runtime quietly superseded by a fresh
|
||||
# session.resume of the same stored session (no reclaimed broadcast);
|
||||
# the stored session stays resumable like any accidental end.
|
||||
# Stale sentinel-parked runtime superseded by a fresh session.resume.
|
||||
"superseded_by_resume",
|
||||
# Startup sweep of rows orphaned by a dead gateway process (#65194):
|
||||
# the in-process ws-orphan grace timer died with the process, so the
|
||||
# row was closed at the next boot instead. Same accident class as
|
||||
# ws_orphan_reap — kept distinct for forensics — and equally resumable.
|
||||
# Startup sweep of rows orphaned by a dead gateway process: same accident
|
||||
# class as ws_orphan_reap, kept distinct for forensics.
|
||||
"startup_orphan_reap",
|
||||
)
|
||||
_RECOVERABLE_END_REASONS_SQL = ", ".join(f"'{reason}'" for reason in _RECOVERABLE_END_REASONS)
|
||||
|
||||
# End reasons written by AUTOMATIC infrastructure cleanup (server shutdown,
|
||||
# orphan reapers, idle/LRU eviction) rather than by a deliberate conversation
|
||||
# boundary (compression, session_reset, session_switch, explicit user close).
|
||||
# An automatic stamp records "some runtime went away", NOT "this conversation
|
||||
# ended" — so a writer that can prove the conversation is still live (e.g. an
|
||||
# active compression rotation holding the lease, #88197) may treat the stamp
|
||||
# as stale and clear it. Superset of the recoverable set: those are already
|
||||
# resumable accidents; the extra TUI reasons are the same accident class but
|
||||
# were historically only known to tui_gateway's _AUTOMATIC_SESSION_END_REASONS.
|
||||
# End reasons written by AUTOMATIC cleanup (shutdown, orphan reapers, idle/LRU
|
||||
# eviction), not by a deliberate conversation boundary. Such a stamp means
|
||||
# "some runtime went away", NOT "this conversation ended", so a writer that can
|
||||
# prove liveness (e.g. a compression rotation holding the lease) may clear it.
|
||||
# Superset of the recoverable set plus the TUI gateway's automatic reasons.
|
||||
_AUTOMATIC_END_REASONS = frozenset(_RECOVERABLE_END_REASONS) | {
|
||||
"tui_shutdown",
|
||||
"ws_disconnect",
|
||||
@@ -244,23 +227,17 @@ _AUTOMATIC_END_REASONS = frozenset(_RECOVERABLE_END_REASONS) | {
|
||||
|
||||
|
||||
def is_automatic_end_reason(reason) -> bool:
|
||||
"""True when *reason* is an automatic-cleanup end stamp (see above).
|
||||
|
||||
Single owner of the "accidental vs deliberate end" predicate — every
|
||||
compression-liveness site must call this instead of re-implementing the
|
||||
reason taxonomy (#88197, never-patch-predicates).
|
||||
"""
|
||||
"""True when *reason* is an automatic-cleanup end stamp (see above). Single
|
||||
owner of the accidental-vs-deliberate predicate; compression-liveness
|
||||
sites must call this rather than re-implement the taxonomy."""
|
||||
return isinstance(reason, str) and reason in _AUTOMATIC_END_REASONS
|
||||
|
||||
|
||||
def _legacy_reset_child_sql(alias: str, reasons_sql: str) -> str:
|
||||
"""Pre-marker reset-continuation heuristic.
|
||||
|
||||
A child is a legacy reset continuation when it rides its parent's exact
|
||||
non-empty routing key and the parent ended at a reset boundary. Shared by
|
||||
the listing predicate (``_RESET_CHILD_SQL``) and ``reopen_session()``'s
|
||||
marker-stamping UPDATE so the two sites cannot drift; ``reasons_sql`` is
|
||||
either the literal ``_RESET_END_REASONS_SQL`` or a bound-placeholder list.
|
||||
"""Pre-marker reset-continuation heuristic: child rides its parent's exact
|
||||
non-empty routing key and the parent ended at a reset boundary. Shared by
|
||||
``_RESET_CHILD_SQL`` and ``reopen_session()``'s marker-stamping UPDATE so
|
||||
the two cannot drift; ``reasons_sql`` is a literal or placeholder list.
|
||||
"""
|
||||
return (
|
||||
f"EXISTS (SELECT 1 FROM sessions p"
|
||||
@@ -272,19 +249,17 @@ def _legacy_reset_child_sql(alias: str, reasons_sql: str) -> str:
|
||||
)
|
||||
|
||||
|
||||
# A reset starts a separate user-visible conversation even though gateway rows
|
||||
# retain parent_session_id for durable lineage. New rows carry the stable
|
||||
# marker; the same-key fallback recovers rows written before the marker existed.
|
||||
# Requiring the exact non-empty routing key keeps ordinary child/subagent rows
|
||||
# out even when their parent is later reset.
|
||||
# A reset starts a separate user-visible conversation though rows keep
|
||||
# parent_session_id for lineage. Stable marker, or the same-key fallback for
|
||||
# pre-marker rows (the exact-key requirement keeps subagent children out).
|
||||
_RESET_CHILD_SQL = (
|
||||
"json_extract(COALESCE({a}.model_config, '{{}}'), '$._reset_from') IS NOT NULL"
|
||||
" OR " + _legacy_reset_child_sql("{a}", _RESET_END_REASONS_SQL)
|
||||
)
|
||||
|
||||
|
||||
# Rows that surface in pickers: roots + branch/reset children. Subagent runs
|
||||
# and compression continuations stay hidden.
|
||||
# Picker-visible rows: roots + branch/reset children (not subagent runs or
|
||||
# compression continuations).
|
||||
_LISTABLE_CHILD_SQL = (
|
||||
f"(s.parent_session_id IS NULL OR {_BRANCH_CHILD_SQL.format(a='s')}"
|
||||
f" OR {_RESET_CHILD_SQL.format(a='s')})"
|
||||
@@ -305,14 +280,9 @@ def _ephemeral_child_sql(alias: str = "s") -> str:
|
||||
|
||||
|
||||
def _sql_session_last_active(alias: str = "s") -> str:
|
||||
"""SQL expression for session recency used by list/status surfaces.
|
||||
|
||||
Freshest of ``last_activity_at`` (mid-turn agent activity heartbeat) and
|
||||
the latest message timestamp, then fall back to ``started_at``.
|
||||
|
||||
Must not prefer a stale heartbeat over a newer message: durable
|
||||
heartbeats are rate-limited (~60s), so after a turn writes messages
|
||||
``last_activity_at`` can lag ``MAX(messages.timestamp)``.
|
||||
"""Session recency: freshest of ``last_activity_at`` and the latest message
|
||||
timestamp, else ``started_at``. Heartbeats are rate-limited (~60s) so
|
||||
``last_activity_at`` can lag a newer message; never prefer it alone.
|
||||
"""
|
||||
msg_max = (
|
||||
f"(SELECT MAX(_act_m.timestamp) FROM messages _act_m "
|
||||
@@ -356,24 +326,38 @@ def _sql_session_last_active_by_id(session_id_expr: str) -> str:
|
||||
|
||||
SCHEMA_VERSION = 28
|
||||
|
||||
# Auto-maintenance VACUUMs only when at least this fraction of pages is on the
|
||||
# freelist; below it a full rewrite costs more I/O than it returns.
|
||||
AUTO_VACUUM_MIN_FREELIST_RATIO = 0.25
|
||||
|
||||
# FTS storage-layout version, tracked INDEPENDENTLY of SCHEMA_VERSION in the
|
||||
# state_meta key ``fts_storage_version``. The main schema version advances
|
||||
# freely on open (so future migrations always land); the FTS *layout* only
|
||||
# reaches the current version when a DB is either born fresh or explicitly
|
||||
# optimized via ``hermes sessions optimize-storage``. A legacy DB sits at
|
||||
# layout 0 (marker absent) with a working inline index until the user opts in.
|
||||
# 1 = v23 external-content layout (content/tool_name/tool_calls,
|
||||
# tool-row-excluded trigram)
|
||||
|
||||
# FTS storage layout, tracked INDEPENDENTLY of SCHEMA_VERSION (state_meta
|
||||
# ``fts_storage_version``): the schema version advances freely on open, but
|
||||
# the FTS layout only changes when a DB is born fresh or explicitly optimized
|
||||
# via ``hermes sessions optimize-storage``. Legacy DBs sit at 0 (marker
|
||||
# absent) with a working inline index. 1 = v23 external-content layout.
|
||||
FTS_STORAGE_VERSION = 1
|
||||
|
||||
|
||||
# Cap on user-controlled FTS5 query input before regex/sanitizer processing.
|
||||
# Search queries do not need to be arbitrarily large, and bounding them keeps
|
||||
# sanitizer/runtime behavior predictable under adversarial input.
|
||||
# Cap on user-controlled FTS5 query input before sanitizer processing.
|
||||
MAX_FTS5_QUERY_CHARS = 2_048
|
||||
|
||||
|
||||
# ── Helpers shared by SessionDB, its mixins and the registry ──────────────
|
||||
|
||||
def stat_db_file_identity(path) -> "tuple[int, int] | None":
|
||||
"""``(st_dev, st_ino)`` for *path*, or None. st_ino=0 (Windows, some
|
||||
network FS) would false-positive every replaced-file check, so it counts
|
||||
as unknown."""
|
||||
try:
|
||||
st = os.stat(path)
|
||||
except OSError:
|
||||
return None
|
||||
if not st.st_dev or not st.st_ino:
|
||||
return None
|
||||
return (st.st_dev, st.st_ino)
|
||||
|
||||
|
||||
_FTS_TRIGGERS = (
|
||||
"messages_fts_insert",
|
||||
"messages_fts_delete",
|
||||
@@ -628,10 +612,8 @@ CREATE INDEX IF NOT EXISTS idx_async_delegations_delivery
|
||||
"""
|
||||
|
||||
|
||||
# Indexes that reference columns added in later schema versions must be
|
||||
# created AFTER _reconcile_columns() has had a chance to ADD them on
|
||||
# existing databases. SCHEMA_SQL above is run by sqlite executescript
|
||||
# which would otherwise fail on legacy DBs ("no such column: active").
|
||||
# Indexes on columns added in later schema versions must run AFTER
|
||||
# _reconcile_columns() adds them, or executescript fails on legacy DBs.
|
||||
DEFERRED_INDEX_SQL = """
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_session_active
|
||||
ON messages(session_id, active, timestamp);
|
||||
@@ -648,25 +630,17 @@ CREATE INDEX IF NOT EXISTS idx_sessions_system_prompt_hash
|
||||
"""
|
||||
|
||||
|
||||
# ── Deferred FTS rebuild bookkeeping (schema v23) ──
|
||||
# While a background index rebuild is pending, two state_meta keys define
|
||||
# which message rows are currently IN the FTS indexes:
|
||||
# ── Deferred FTS rebuild bookkeeping ──
|
||||
# While a background rebuild is pending, two state_meta keys define which rows
|
||||
# are IN the FTS indexes: H = fts_rebuild_high_water (MAX(messages.id) when the
|
||||
# old indexes were dropped), P = fts_rebuild_progress (highest backfilled id).
|
||||
# A row is indexed iff id <= P OR id > H (AUTOINCREMENT ids, so post-drop rows
|
||||
# are indexed live by the insert triggers); rows in (P, H] are not.
|
||||
#
|
||||
# fts_rebuild_high_water H — MAX(messages.id) at the moment the old
|
||||
# indexes were dropped
|
||||
# fts_rebuild_progress P — highest id the chunked backfill has indexed
|
||||
#
|
||||
# A row is indexed iff id <= P (backfilled) OR id > H (inserted after
|
||||
# the drop; ids are AUTOINCREMENT so new rows are always > H and the insert
|
||||
# triggers index them live). Rows in (P, H] are not yet indexed.
|
||||
#
|
||||
# Every trigger below gates on that same predicate: firing an FTS5
|
||||
# external-content 'delete' for a row that is NOT in the index corrupts the
|
||||
# index, and skipping it for a row that IS indexed leaves a stale entry.
|
||||
# When no rebuild is pending both keys are absent and COALESCE turns the
|
||||
# predicate into a tautology (id > -1 OR id <= -1), i.e. normal operation.
|
||||
# The two state_meta PK probes per write are negligible next to the FTS
|
||||
# insert itself.
|
||||
# Every trigger gates on that predicate: an FTS5 external-content 'delete' for
|
||||
# a row NOT in the index corrupts it, and skipping one for an indexed row
|
||||
# leaves a stale entry. With no rebuild pending both keys are absent and
|
||||
# COALESCE makes the predicate a tautology.
|
||||
FTS_SQL = """
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
|
||||
content,
|
||||
@@ -717,20 +691,12 @@ END;
|
||||
"""
|
||||
|
||||
|
||||
# Trigram FTS5 table for CJK substring search. The default unicode61
|
||||
# tokenizer splits CJK characters into individual tokens, breaking phrase
|
||||
# matching. The trigram tokenizer creates overlapping 3-byte sequences so
|
||||
# substring queries work natively for any script (CJK, Thai, etc.).
|
||||
#
|
||||
# The trigram index is the most expensive index in state.db (~2.6x the size
|
||||
# of the text it covers), and ``role='tool'`` rows are ~90% of message bytes
|
||||
# while being almost entirely machine noise (base64 payloads, file dumps,
|
||||
# delegation transcripts). The index therefore reads through
|
||||
# ``messages_fts_trigram_src``, a view that excludes tool rows — they stay
|
||||
# fully stored in ``messages`` and fully searchable via the standard
|
||||
# ``messages_fts`` index; they just don't get trigram (CJK substring)
|
||||
# treatment. ``search_messages`` routes CJK queries that filter on
|
||||
# ``role='tool'`` to the LIKE fallback for the same reason.
|
||||
# Trigram FTS5 table for CJK substring search (unicode61 splits CJK into single
|
||||
# tokens, breaking phrase matching). The trigram index is ~2.6x the text it
|
||||
# covers and ``role='tool'`` rows are ~90% of message bytes of machine noise,
|
||||
# so it reads through the ``messages_fts_trigram_src`` view, which excludes
|
||||
# tool rows; those remain searchable via ``messages_fts``, and
|
||||
# ``search_messages`` routes CJK queries filtered on role='tool' to LIKE.
|
||||
FTS_TRIGRAM_SQL = """
|
||||
CREATE VIEW IF NOT EXISTS messages_fts_trigram_src AS
|
||||
SELECT id, role, content, tool_name, tool_calls
|
||||
@@ -796,18 +762,15 @@ _FTS_CJK_TRIGGERS = (
|
||||
)
|
||||
|
||||
|
||||
# state_meta breadcrumb set when a tokenizer-less process had to drop the
|
||||
# cjk triggers to keep message writes alive: rows written from that moment
|
||||
# on are missing from the cjk index, so it must not serve reads until
|
||||
# Set when a tokenizer-less process dropped the cjk triggers to keep writes
|
||||
# alive: the cjk index is missing rows and must not serve reads until
|
||||
# `hermes sessions optimize-storage` rebuilds it on a capable host.
|
||||
FTS_CJK_STALE_KEY = "fts_cjk_stale"
|
||||
|
||||
|
||||
# Durable breadcrumb for a base/trigram FTS index that was detached from the
|
||||
# canonical messages table after runtime corruption. While present, startup
|
||||
# must rebuild the complete index before reinstalling sync triggers: rows may
|
||||
# have been written while those triggers were absent, so merely recreating
|
||||
# them would preserve an unknown index gap.
|
||||
# Set when a base/trigram FTS index was detached after runtime corruption.
|
||||
# While present, startup must rebuild the complete index before reinstalling
|
||||
# sync triggers: rows written while they were absent leave an unknown gap.
|
||||
FTS_STALE_KEY = "fts_stale"
|
||||
|
||||
# Durable diagnostic for stale FTS recovery blocked across process restarts.
|
||||
@@ -815,16 +778,11 @@ FTS_REBUILD_DEFERRAL_KEY = "fts_rebuild_deferral"
|
||||
|
||||
|
||||
# ── Legacy (v22 / inline-content) FTS DDL ──────────────────────────────
|
||||
# Used ONLY to keep an existing pre-v23 install's search working and its
|
||||
# triggers repairable UNTIL the user opts into `hermes db optimize`. This is
|
||||
# the exact inline shape v11..v22 shipped: each virtual table stores its own
|
||||
# copy of ``content || tool_name || tool_calls`` and the trigram table indexes
|
||||
# every row (including role='tool'). We never CREATE these on a fresh install —
|
||||
# fresh installs are born on the v23 external-content schema above. These
|
||||
# constants exist so a legacy DB is never accidentally handed the v23 DDL
|
||||
# (which would create the external-content trigram source VIEW and leave the
|
||||
# DB in a mixed, broken state). `optimize_fts_storage()` is what migrates a
|
||||
# legacy DB to the v23 shape.
|
||||
# Used ONLY to keep a pre-v23 install's search working and its triggers
|
||||
# repairable until `optimize_fts_storage()` migrates it: inline copies of
|
||||
# content || tool_name || tool_calls, trigram over every row. Never created
|
||||
# on a fresh install. Handing a legacy DB the v23 DDL would create the
|
||||
# external-content trigram VIEW and leave it in a mixed, broken state.
|
||||
LEGACY_FTS_SQL = """
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
|
||||
content
|
||||
@@ -882,39 +840,25 @@ END;
|
||||
|
||||
# ── Cross-process full-FTS-rebuild admission (single authority) ──────────────
|
||||
#
|
||||
# Several independent Hermes processes routinely share one state.db (gateway
|
||||
# service, the Desktop app's `hermes serve` backend, interactive CLI sessions,
|
||||
# the TUI slash worker). A full structural FTS rebuild — the FTS5 'rebuild'
|
||||
# command or the drop/recreate script in `_recover_stale_fts` — must only ever
|
||||
# run in ONE of them at a time: two concurrent rebuilds collide on write and
|
||||
# have structurally corrupted state.db in production (PR #93200; the
|
||||
# 2026-08-15 / 2026-08-23 incidents and issues #89293 / #90950).
|
||||
# Several Hermes processes share one state.db. A full structural FTS rebuild
|
||||
# (FTS5 'rebuild' or the drop/recreate in `_recover_stale_fts`) must run in
|
||||
# ONE of them at a time: concurrent rebuilds have structurally corrupted
|
||||
# state.db in production. This is the single admission authority for
|
||||
# `rebuild_fts()`, `_rebuild_fts_indexes()` and `_recover_stale_fts()`. The
|
||||
# chunked backfill (`fts_rebuild_step`) is deliberately NOT routed through it:
|
||||
# it claims progress under SQLite transaction authority and is multi-process.
|
||||
#
|
||||
# This is the single admission authority for every full structural rebuild
|
||||
# entry point: `SessionSearchMixin.rebuild_fts()`,
|
||||
# `SessionSchemaMixin._rebuild_fts_indexes()` (via `_init_schema`), and
|
||||
# `SessionSchemaMixin._recover_stale_fts()`. The chunked deferred backfill
|
||||
# (`fts_rebuild_step`) is deliberately NOT routed through it — it claims
|
||||
# progress under `_execute_write`'s SQLite transaction authority and is
|
||||
# intentionally multi-process.
|
||||
# Semantics mirror `hermes_state._cross_process_repair_lock`: portable (msvcrt
|
||||
# on Windows, flock elsewhere), bounded wait, FAIL CLOSED. The kernel drops
|
||||
# the lock when the holder dies UNLESS a forked child inherited the fd (flock
|
||||
# rides the open file description), which holds it forever; so the holder's
|
||||
# pid + start time are recorded under the lock and a provably-dead holder's
|
||||
# lock is broken by unlinking and retaking on a fresh inode. Indeterminate
|
||||
# liveness still defers. Lives here because the mixins cannot import
|
||||
# hermes_state (cycle).
|
||||
#
|
||||
# Semantics mirror `hermes_state._cross_process_repair_lock` (the schema-
|
||||
# surgery authority): portable (msvcrt on Windows, flock elsewhere), bounded
|
||||
# wait, and FAIL CLOSED — a caller that cannot acquire the lock must NOT
|
||||
# rebuild. The kernel drops both lock types when the holder dies — UNLESS a
|
||||
# forked child inherited the lock fd (flock rides the open file description,
|
||||
# which fork() duplicates), in which case the orphaned descriptor holds the
|
||||
# lock forever (issue #100108). `_acquire_db_flock` therefore records the
|
||||
# holder's pid + start time under the lock and, when the recorded holder is
|
||||
# provably dead, breaks the orphaned lock by unlinking and retaking it on a
|
||||
# fresh inode; indeterminate liveness still defers. It lives here (not
|
||||
# hermes_state) because the search/schema mixins cannot import hermes_state
|
||||
# (cycle).
|
||||
#
|
||||
# The lock file is `<db>.fts_rebuild.lock`, distinct from `<db>.repair.lock`:
|
||||
# schema surgery runs on an EXCLUSIVE offline connection and can legitimately
|
||||
# take minutes in VACUUM, while runtime rebuilds run on live connections. The
|
||||
# timeout is sized for a full 'rebuild' of both indexes on a large DB.
|
||||
# `<db>.fts_rebuild.lock` is distinct from `<db>.repair.lock`: schema surgery
|
||||
# runs on an EXCLUSIVE offline connection and may take minutes in VACUUM.
|
||||
|
||||
logger = logging.getLogger("hermes_state")
|
||||
|
||||
@@ -922,31 +866,23 @@ _FTS_REBUILD_LOCK_TIMEOUT_SECONDS = 120.0
|
||||
_FTS_REBUILD_LOCK_POLL_SECONDS = 0.1
|
||||
_IS_WINDOWS = sys.platform == "win32"
|
||||
|
||||
# Post-break re-acquire budget: once a provably-orphaned lock has been broken
|
||||
# the fresh inode is uncontended (or contended only by live processes), so a
|
||||
# short bounded wait suffices — never re-enter the full timeout.
|
||||
# Post-break re-acquire budget: the fresh inode is contended only by live
|
||||
# processes, so a short wait suffices — never re-enter the full timeout.
|
||||
_LOCK_BREAK_REACQUIRE_SECONDS = 5.0
|
||||
|
||||
# errno set for "another process holds this advisory lock". flock() reports
|
||||
# contention as EWOULDBLOCK/EAGAIN; msvcrt.locking() as EACCES (and EDEADLK
|
||||
# when its internal retry gives up). Anything else — ESTALE on a dropped NFS
|
||||
# handle, ENOTSUP/ENOLCK on a filesystem without advisory locks, EIO — is a
|
||||
# persistent environment failure that no amount of polling turns into an
|
||||
# acquire. Treating every OSError as contention made such a failure look
|
||||
# like a live holder and burned the full 120s admission timeout on every
|
||||
# attempt (#100108, PR #100130).
|
||||
# "Another process holds the lock": flock → EWOULDBLOCK/EAGAIN, msvcrt.locking
|
||||
# → EACCES (EDEADLK when its retry gives up). Anything else (ESTALE, ENOTSUP,
|
||||
# ENOLCK, EIO) is a persistent environment failure that polling cannot fix;
|
||||
# treating it as contention burned the full timeout on every attempt.
|
||||
_LOCK_CONTENTION_ERRNOS = {errno.EAGAIN, errno.EACCES, errno.EWOULDBLOCK}
|
||||
if hasattr(errno, "EDEADLK"):
|
||||
_LOCK_CONTENTION_ERRNOS.add(errno.EDEADLK)
|
||||
|
||||
|
||||
def is_advisory_lock_contention(exc: BaseException) -> bool:
|
||||
"""True when *exc* means another process holds the advisory lock.
|
||||
|
||||
False for every other ``OSError`` (ESTALE, ENOTSUP, ENOLCK, EIO, ...):
|
||||
callers must fail closed IMMEDIATELY rather than poll to the deadline,
|
||||
because retrying cannot succeed and the wait only stalls the caller.
|
||||
"""
|
||||
"""True when *exc* means another process holds the advisory lock. For
|
||||
any other ``OSError`` callers must fail closed IMMEDIATELY: retrying
|
||||
cannot succeed and polling only stalls the caller."""
|
||||
if isinstance(exc, BlockingIOError):
|
||||
return True
|
||||
if not isinstance(exc, OSError):
|
||||
@@ -955,13 +891,9 @@ def is_advisory_lock_contention(exc: BaseException) -> bool:
|
||||
|
||||
|
||||
def _proc_start_ticks(pid: int):
|
||||
"""Kernel start time of *pid* in clock ticks, or None when unknowable.
|
||||
|
||||
Field 22 of ``/proc/<pid>/stat`` (``starttime``) uniquely identifies a
|
||||
process together with its PID: a recycled PID gets a different start
|
||||
time. Returns None off Linux or on any read/parse failure — callers must
|
||||
treat None as "unknowable" and FAIL CLOSED.
|
||||
"""
|
||||
"""Kernel start time of *pid* (field 22 of ``/proc/<pid>/stat``), which
|
||||
with the PID uniquely identifies a process; None off Linux or on any
|
||||
failure — callers must treat None as unknowable and FAIL CLOSED."""
|
||||
try:
|
||||
with open(f"/proc/{pid}/stat", "rb") as fh:
|
||||
stat = fh.read()
|
||||
@@ -988,12 +920,8 @@ def _read_lock_holder_record(handle):
|
||||
|
||||
|
||||
def _write_lock_holder_record(handle) -> None:
|
||||
"""Record this process as the lock holder (advisory, best effort).
|
||||
|
||||
Written under the flock so contenders that time out can tell an
|
||||
orphaned-fd holder (recorded process dead, flock inherited by a forked
|
||||
child — issue #100108) from a live wedged holder.
|
||||
"""
|
||||
"""Record this process as holder (best effort), written under the flock so
|
||||
timed-out contenders can tell an orphaned-fd holder from a live wedged one."""
|
||||
try:
|
||||
record = {
|
||||
"pid": os.getpid(),
|
||||
@@ -1009,12 +937,8 @@ def _write_lock_holder_record(handle) -> None:
|
||||
|
||||
|
||||
def _clear_lock_holder_record(handle) -> None:
|
||||
"""Erase holder metadata before a normal release.
|
||||
|
||||
Guarantees that a surviving record always describes an ABNORMAL exit
|
||||
(holder died without releasing), which is the only condition under which
|
||||
a contender may break the lock.
|
||||
"""
|
||||
"""Erase holder metadata before a normal release, so a surviving record
|
||||
always means an ABNORMAL exit — the only condition allowing a break."""
|
||||
try:
|
||||
handle.seek(0)
|
||||
handle.truncate()
|
||||
@@ -1025,12 +949,8 @@ def _clear_lock_holder_record(handle) -> None:
|
||||
|
||||
def _lock_holder_provably_dead(record) -> bool:
|
||||
"""True ONLY when the recorded holder is provably dead or PID-recycled.
|
||||
|
||||
Any indeterminate state (no record, malformed record, PID owned by
|
||||
another user, /proc unavailable, start-time unknowable) returns False —
|
||||
the caller must FAIL CLOSED and defer, never break a possibly-live
|
||||
holder's lock.
|
||||
"""
|
||||
Anything indeterminate (no/malformed record, PID owned by another user,
|
||||
/proc unavailable) is False — the caller must FAIL CLOSED and defer."""
|
||||
if not isinstance(record, dict):
|
||||
return False
|
||||
try:
|
||||
@@ -1052,34 +972,27 @@ def _lock_holder_provably_dead(record) -> bool:
|
||||
current_ticks = _proc_start_ticks(pid)
|
||||
if current_ticks is None:
|
||||
return False
|
||||
# Same PID, different kernel start time: the recorded holder is dead and
|
||||
# its PID was recycled by an unrelated process.
|
||||
# Same PID, different start time: recycled by an unrelated process.
|
||||
return current_ticks != recorded_ticks
|
||||
|
||||
|
||||
def _acquire_db_flock(lock_path, handle, timeout_seconds, poll_seconds, description):
|
||||
"""Bounded POSIX flock acquire with orphaned-holder staleness break.
|
||||
|
||||
Returns ``(acquired, handle)``; *handle* may have been re-opened (the
|
||||
caller owns closing whichever handle comes back). *acquired* is True on
|
||||
success, False when a holder kept the lock past the deadline, and None
|
||||
when a non-contention ``OSError`` (ESTALE/ENOTSUP/EIO) made acquisition
|
||||
impossible — already logged here; callers treat None as "not acquired"
|
||||
without emitting the held-by-another-process warning.
|
||||
Returns ``(acquired, handle)``; *handle* may have been re-opened and the
|
||||
caller closes whichever comes back. *acquired* is True, False (a holder
|
||||
kept the lock past the deadline), or None (non-contention ``OSError``,
|
||||
already logged; callers treat it as not acquired without the
|
||||
held-by-another-process warning).
|
||||
|
||||
Why breaking exists at all (issue #100108): ``flock`` belongs to the open
|
||||
file DESCRIPTION, which ``fork()`` duplicates into every child. A holder
|
||||
that forks (multiprocessing worker, daemonized helper) and then dies
|
||||
leaves the flock held by a child that will never release it — the
|
||||
kernel's holder-death release never triggers, and every contender defers
|
||||
forever. The recorded-holder liveness check distinguishes exactly that
|
||||
case: the process that ACQUIRED is provably dead (so its critical section
|
||||
died with it), yet the flock is still held. Only then is the lock file
|
||||
unlinked and retaken on a fresh inode; the orphan's flock stays on the
|
||||
old unlinked inode where it blocks nobody. Every successful acquire
|
||||
verifies its inode still names *lock_path*, so a racer that locked a dead
|
||||
inode retries instead of running concurrently with the breaker.
|
||||
Indeterminate liveness always defers (fail closed).
|
||||
``flock`` belongs to the open file DESCRIPTION, which ``fork()``
|
||||
duplicates: a holder that forks then dies leaves the lock held forever by
|
||||
a child that never releases. When the process that ACQUIRED is provably
|
||||
dead (its critical section died with it) yet the flock is held, the file
|
||||
is unlinked and retaken on a fresh inode; the orphan's flock stays on the
|
||||
old inode blocking nobody. Every successful acquire verifies its inode
|
||||
still names *lock_path*, so a racer that locked a dead inode retries
|
||||
instead of running alongside the breaker. Indeterminate liveness defers.
|
||||
"""
|
||||
import fcntl
|
||||
|
||||
@@ -1090,9 +1003,7 @@ def _acquire_db_flock(lock_path, handle, timeout_seconds, poll_seconds, descript
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except (BlockingIOError, OSError) as exc:
|
||||
if not is_advisory_lock_contention(exc):
|
||||
# ESTALE / ENOTSUP / EIO: not a holder, and polling cannot
|
||||
# fix it. Defer NOW instead of pretending a live process
|
||||
# held the lock for the whole timeout (#100108).
|
||||
# Not a holder and polling cannot fix it: defer NOW.
|
||||
logger.warning(
|
||||
"Could not acquire %s %s (%s) — deferring rather than "
|
||||
"waiting out the %.0fs holder timeout on a "
|
||||
@@ -1135,9 +1046,8 @@ def _acquire_db_flock(lock_path, handle, timeout_seconds, poll_seconds, descript
|
||||
broke_lock = True
|
||||
deadline = time.monotonic() + _LOCK_BREAK_REACQUIRE_SECONDS
|
||||
continue
|
||||
# flock acquired — verify the path still names our inode: a breaker
|
||||
# may have unlinked/replaced the file while we waited, and a lock on
|
||||
# a dead inode excludes nobody.
|
||||
# Verify the path still names our inode: a breaker may have replaced
|
||||
# the file while we waited, and a lock on a dead inode excludes nobody.
|
||||
try:
|
||||
fd_stat = os.fstat(handle.fileno())
|
||||
path_stat = os.stat(lock_path)
|
||||
@@ -1178,21 +1088,13 @@ def _describe_lock_holder(record) -> str:
|
||||
def fts_rebuild_admission(db_path, *, timeout_seconds=None):
|
||||
"""Serialize full structural FTS rebuilds on *db_path* across processes.
|
||||
|
||||
Yields True when this process holds the rebuild authority, False when the
|
||||
bounded acquire timed out or the lock file could not be opened at all. A
|
||||
caller that gets False must NOT perform a full rebuild — proceeding is
|
||||
exactly the concurrent-rebuild interleaving this lock exists to prevent
|
||||
(fail closed). The deferred/stale breadcrumb machinery already guarantees
|
||||
a skipped rebuild is retried later.
|
||||
Yields True when this process holds the authority, False when the bounded
|
||||
acquire timed out or the lock file could not be opened. On False the
|
||||
caller must NOT rebuild (fail closed); the stale breadcrumb guarantees a
|
||||
retry. ``db_path`` None (in-memory DB) yields True.
|
||||
|
||||
``db_path`` may be a str or Path; None (in-memory DB / tests without a
|
||||
file path) yields True — a private in-memory DB has no cross-process
|
||||
surface.
|
||||
|
||||
*timeout_seconds* defaults to ``_FTS_REBUILD_LOCK_TIMEOUT_SECONDS``.
|
||||
Opportunistic in-process retries (``retry_deferred_fts_recovery``) pass
|
||||
``0`` so a live holder never stalls a long-lived writer for two minutes;
|
||||
the orphaned-holder break still applies on the single attempt.
|
||||
Opportunistic in-process retries pass ``timeout_seconds=0`` so a live
|
||||
holder never stalls a long-lived writer; the orphan break still applies.
|
||||
"""
|
||||
if db_path is None:
|
||||
yield True
|
||||
@@ -1206,16 +1108,12 @@ def fts_rebuild_admission(db_path, *, timeout_seconds=None):
|
||||
try:
|
||||
handle = open(lock_path, "a+b")
|
||||
except OSError as exc:
|
||||
# Fail closed, exactly as a timed-out acquire does. A lock file we
|
||||
# cannot even open means the filesystem is out of space, inodes or
|
||||
# descriptors — and a sibling process that opened ITS handle before
|
||||
# the disk filled is still holding the authority and rebuilding.
|
||||
# Yielding True here handed every process on a full disk a concurrent
|
||||
# structural rebuild of the same live state.db with no cross-process
|
||||
# authority at all: the disk-full trigger and the re-corruption on
|
||||
# every multi-writer boot in #100368. Deferring costs nothing that
|
||||
# was reachable anyway — the breadcrumb retries, and on a read-only
|
||||
# directory the rebuild's own writes could not have committed either.
|
||||
# Fail closed like a timed-out acquire. An unopenable lock file means
|
||||
# the FS is out of space/inodes/descriptors, and a sibling that opened
|
||||
# its handle earlier may still be rebuilding; yielding True here gave
|
||||
# every process on a full disk a concurrent rebuild of the same DB.
|
||||
# Deferring costs nothing: the breadcrumb retries, and the rebuild's
|
||||
# own writes could not have committed either.
|
||||
logger.warning(
|
||||
"Could not open FTS rebuild lock %s (%s) — deferring this rebuild "
|
||||
"rather than running it without cross-process authority.",
|
||||
@@ -1257,14 +1155,13 @@ def fts_rebuild_admission(db_path, *, timeout_seconds=None):
|
||||
"FTS rebuild lock",
|
||||
)
|
||||
if acquired is None:
|
||||
# Non-contention failure: already logged with the real errno;
|
||||
# a "held by another process" line here would be a lie.
|
||||
# Already logged with the real errno; "held by another process"
|
||||
# would be a lie.
|
||||
acquired = False
|
||||
elif not acquired:
|
||||
record = None if _IS_WINDOWS else _read_lock_holder_record(handle)
|
||||
if timeout <= 0:
|
||||
# Non-blocking probe from an in-process retry: a busy lock
|
||||
# is expected and will be tried again, so keep it quiet.
|
||||
# Non-blocking probe from an in-process retry: keep it quiet.
|
||||
logger.info(
|
||||
"FTS rebuild lock %s is busy — deferring this retry "
|
||||
"(the stale-FTS breadcrumb keeps it retryable). "
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,674 @@
|
||||
"""state.db file-level health helpers.
|
||||
|
||||
Split out of ``hermes_state.py``: header probes (application_id / zeroed-file
|
||||
detection), deleted-WAL-sidecar holder scans, quarantine of zeroed or
|
||||
lock-poisoned databases, ``collect_state_db_stats`` and holder-process
|
||||
classification. Every name is re-imported into ``hermes_state`` so
|
||||
``hermes_state.<name>`` keeps resolving — and tests that monkeypatch it keep
|
||||
intercepting, because intra-module calls to patched helpers go through a
|
||||
lazy ``from hermes_state import ...`` at call time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||
|
||||
from hermes_state_common import (
|
||||
FTS_REBUILD_DEFERRAL_KEY,
|
||||
stat_db_file_identity as _stat_db_file_identity,
|
||||
)
|
||||
|
||||
# Log-record parity with the origin module (caplog tests pin "hermes_state").
|
||||
logger = logging.getLogger("hermes_state")
|
||||
|
||||
|
||||
# _read_sqlite_application_id runs on EVERY write via _raise_if_db_replaced,
|
||||
# against the LIVE state.db. A bare open()/read()/close() there is the
|
||||
# howtocorrupt §2.2 bug: close() cancels every POSIX advisory lock this
|
||||
# process holds on the file — measured on Linux/SQLite 3.53.1, one probe call
|
||||
# drops the WAL-mode DMS shared lock the writer connection holds on state.db
|
||||
# (see hermes_cli/sqlite_safe_read.py for the module built around this rule).
|
||||
# With the DMS lock gone, a fresh opener in another process can treat this
|
||||
# writer as dead and rerun WAL-index recovery underneath it.
|
||||
#
|
||||
# The probe therefore reads through a per-path fd cached for the life of the
|
||||
# process: opening an fd never cancels locks (only close() does), and
|
||||
# os.pread takes no shared file position. When the path is re-pointed at a
|
||||
# new inode (the very replacement this probe exists to detect), the stale fd
|
||||
# is RETIRED, never closed — closing it would cancel the live connection's
|
||||
# locks on the old file, the exact bug being avoided. Replacement events are
|
||||
# rare and halt writes anyway, so the leak is bounded.
|
||||
_HEADER_PROBE_LOCK = threading.Lock()
|
||||
|
||||
|
||||
_HEADER_PROBE_FDS: "dict[str, tuple[int, int, int]]" = {} # key -> (fd, dev, ino)
|
||||
|
||||
|
||||
_RETIRED_HEADER_PROBE_FDS: "list[int]" = [] # intentionally never closed
|
||||
|
||||
|
||||
def _pread_db_header(db_path: Path, length: int) -> "Optional[bytes]":
|
||||
"""Lock-safe raw header read of a possibly-live SQLite database.
|
||||
|
||||
POSIX: pread from a cached, never-closed fd (rebound when the path names
|
||||
a new inode). Windows: plain read — advisory-lock cancellation is a
|
||||
POSIX-only hazard and msvcrt locks do not share the failure mode.
|
||||
"""
|
||||
from hermes_state import _IS_WINDOWS
|
||||
if _IS_WINDOWS:
|
||||
try:
|
||||
with db_path.open("rb") as handle:
|
||||
return handle.read(length)
|
||||
except OSError:
|
||||
return None
|
||||
key = str(db_path)
|
||||
try:
|
||||
st = os.stat(db_path)
|
||||
except OSError:
|
||||
return None
|
||||
with _HEADER_PROBE_LOCK:
|
||||
cached = _HEADER_PROBE_FDS.get(key)
|
||||
if cached is not None and (cached[1], cached[2]) != (st.st_dev, st.st_ino):
|
||||
# Path re-pointed at a new file. Retire (never close) the old fd.
|
||||
_RETIRED_HEADER_PROBE_FDS.append(cached[0])
|
||||
cached = None
|
||||
del _HEADER_PROBE_FDS[key]
|
||||
if cached is None:
|
||||
try:
|
||||
fd = os.open(db_path, os.O_RDONLY)
|
||||
except OSError:
|
||||
return None
|
||||
try:
|
||||
fst = os.fstat(fd)
|
||||
except OSError:
|
||||
_RETIRED_HEADER_PROBE_FDS.append(fd)
|
||||
return None
|
||||
cached = (fd, fst.st_dev, fst.st_ino)
|
||||
_HEADER_PROBE_FDS[key] = cached
|
||||
try:
|
||||
return os.pread(cached[0], length, 0)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _read_sqlite_application_id(db_path: Path) -> "Optional[int]":
|
||||
"""Read application_id from the SQLite header without opening a connection.
|
||||
|
||||
Safe against live databases: routed through :func:`_pread_db_header`,
|
||||
which never issues a ``close()`` that would cancel this process's POSIX
|
||||
locks on the file (howtocorrupt §2.2).
|
||||
"""
|
||||
from hermes_state import _STATE_DB_APPLICATION_ID_OFFSET
|
||||
header = _pread_db_header(db_path, _STATE_DB_APPLICATION_ID_OFFSET + 4)
|
||||
if header is None:
|
||||
return None
|
||||
if len(header) < _STATE_DB_APPLICATION_ID_OFFSET + 4:
|
||||
return None
|
||||
if header[:16] != b"SQLite format 3\x00":
|
||||
return None
|
||||
return int(
|
||||
struct.unpack(
|
||||
">I",
|
||||
header[_STATE_DB_APPLICATION_ID_OFFSET:_STATE_DB_APPLICATION_ID_OFFSET + 4],
|
||||
)[0]
|
||||
)
|
||||
|
||||
|
||||
def _stat_sqlite_sidecar_identity(db_path: Path) -> Dict[str, tuple]:
|
||||
"""Snapshot ``(st_dev, st_ino)`` for existing WAL/SHM sidecars."""
|
||||
identities: Dict[str, tuple] = {}
|
||||
base = os.fspath(db_path)
|
||||
for suffix in ("-wal", "-shm"):
|
||||
ident = _stat_db_file_identity(Path(base + suffix))
|
||||
if ident is not None:
|
||||
identities[suffix] = ident
|
||||
return identities
|
||||
|
||||
|
||||
def _canonical_sqlite_path(path: str) -> str:
|
||||
"""Normalize a /proc fd target, stripping the Linux `` (deleted)`` suffix."""
|
||||
return os.path.normcase(os.path.abspath(path.removesuffix(" (deleted)")))
|
||||
|
||||
|
||||
def _watched_sqlite_sidecar_paths(db_path) -> Set[str]:
|
||||
base = os.path.abspath(os.fspath(db_path))
|
||||
return {
|
||||
_canonical_sqlite_path(base + "-wal"),
|
||||
_canonical_sqlite_path(base + "-shm"),
|
||||
}
|
||||
|
||||
|
||||
def iter_deleted_sqlite_sidecar_holders(db_path) -> List[Tuple[int, str]]:
|
||||
"""Return processes holding an unlinked ``state.db-wal`` / ``-shm``.
|
||||
|
||||
Linux-only (``/proc/<pid>/fd`` readlink). Windows and other hosts
|
||||
return ``[]`` — Windows cannot unlink a sidecar another process still
|
||||
holds, and macOS does not use the `` (deleted)`` suffix.
|
||||
|
||||
The scan includes this process: on the SessionDB open/write refuse
|
||||
path, the in-process writer that still holds the orphan inode is the
|
||||
one that must not mint a replacement WAL (and must stop committing).
|
||||
``_foreign_state_db_holders`` keeps skipping this PID for FTS
|
||||
maintenance so a process does not block its own optional repair.
|
||||
"""
|
||||
if not sys.platform.startswith("linux"):
|
||||
return []
|
||||
|
||||
holders: List[Tuple[int, str]] = []
|
||||
watched = _watched_sqlite_sidecar_paths(db_path)
|
||||
try:
|
||||
for pid_str in os.listdir("/proc"):
|
||||
if not pid_str.isdigit():
|
||||
continue
|
||||
pid = int(pid_str)
|
||||
fd_dir = f"/proc/{pid}/fd"
|
||||
try:
|
||||
fds = os.listdir(fd_dir)
|
||||
except OSError:
|
||||
continue
|
||||
for fd in fds:
|
||||
try:
|
||||
target = os.readlink(f"{fd_dir}/{fd}")
|
||||
except OSError:
|
||||
continue
|
||||
if " (deleted)" not in target:
|
||||
continue
|
||||
if _canonical_sqlite_path(target) in watched:
|
||||
holders.append((pid, target))
|
||||
except Exception as exc:
|
||||
logger.debug("deleted-WAL holder scan failed for %s: %s", db_path, exc)
|
||||
return holders
|
||||
return holders
|
||||
|
||||
|
||||
def refuse_deleted_wal_generation(db_path) -> None:
|
||||
"""Raise if any process holds a deleted WAL/SHM generation for *db_path*.
|
||||
|
||||
Called *before* ``sqlite3.connect`` so a second opener cannot mint a
|
||||
replacement WAL inode while a live writer still holds the orphan.
|
||||
"""
|
||||
from hermes_state import DeletedWalGenerationError, _DELETED_WAL_GENERATION_MSG
|
||||
holders = iter_deleted_sqlite_sidecar_holders(db_path)
|
||||
if not holders:
|
||||
return
|
||||
logger.error(_DELETED_WAL_GENERATION_MSG)
|
||||
raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG)
|
||||
|
||||
|
||||
def _connect_tracked_db(path, tracking_path=None, **kwargs):
|
||||
"""``sqlite3.connect`` that registers the open fd for lock-safety.
|
||||
|
||||
While a connection is live, byte-level probes of the same file are
|
||||
refused: an ``open()``/``close()`` cancels every POSIX advisory lock this
|
||||
process holds on it -- including a running VACUUM's EXCLUSIVE lock.
|
||||
Released automatically on ``close()``.
|
||||
|
||||
The ONLY tolerated fallback is the helper being absent entirely
|
||||
(scaffold/embed installs that ship hermes_state without hermes_cli). A
|
||||
real connection failure must propagate: silently retrying an *untracked*
|
||||
connect would disable the guard for the lifetime of that connection,
|
||||
which is precisely the failure mode this module exists to prevent.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.sqlite_safe_read import connect_tracked
|
||||
except ImportError:
|
||||
logger.debug(
|
||||
"hermes_cli.sqlite_safe_read unavailable; opening %s untracked "
|
||||
"(byte-probe guard inactive in this install)",
|
||||
path,
|
||||
)
|
||||
return sqlite3.connect(str(path), **kwargs)
|
||||
|
||||
# Open through THIS module's sqlite3.connect so callers (and tests) that
|
||||
# patch hermes_state.sqlite3.connect keep control of connection creation;
|
||||
# the helper still owns tracking.
|
||||
return connect_tracked(
|
||||
path,
|
||||
tracking_path=tracking_path,
|
||||
connect_fn=sqlite3.connect,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def is_zeroed_state_db(
|
||||
path: Path, *, probe_bytes: int = 100, force: bool = False
|
||||
) -> bool:
|
||||
"""Detect the #68474/#97568 zeroed state.db signature (0-byte or NUL header).
|
||||
|
||||
Byte-level probe, so it is only safe BEFORE any connection to *path*
|
||||
exists in this process: ``close()`` cancels every POSIX advisory lock the
|
||||
process holds on the file, which can pull the EXCLUSIVE lock out from
|
||||
under a running VACUUM and corrupt the database. The read is routed
|
||||
through ``read_header_bytes_preopen``, which refuses (returning False
|
||||
here) once a connection is live. Pass ``force=True`` only for offline
|
||||
files -- quarantined copies, snapshots, archives.
|
||||
|
||||
Prefer ``hermes_cli.backup.is_zeroed_sqlite_file`` when available; this
|
||||
local copy keeps SessionDB openable without importing the CLI package
|
||||
in constrained embed paths.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.backup import is_zeroed_sqlite_file
|
||||
|
||||
return is_zeroed_sqlite_file(path, probe_bytes=probe_bytes, force=force)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if not path.is_file():
|
||||
# Special files (FIFO, device, socket) are never "zeroed", and
|
||||
# probing a FIFO would block until a writer appears.
|
||||
return False
|
||||
size = path.stat().st_size
|
||||
except OSError:
|
||||
return False
|
||||
if size < 0:
|
||||
return False
|
||||
from hermes_cli.sqlite_safe_read import has_live_connection, read_header_bytes_preopen
|
||||
|
||||
if not force and has_live_connection(path):
|
||||
return False
|
||||
|
||||
head = read_header_bytes_preopen(
|
||||
path, length=max(16, probe_bytes), force=force
|
||||
)
|
||||
if head is None:
|
||||
return False
|
||||
if len(head) == 0:
|
||||
return True
|
||||
if head.startswith(b"SQLite format 3"):
|
||||
return False
|
||||
return all(byte == 0 for byte in head)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def quarantine_cross_process_lock(path: Path, timeout: float = 5.0):
|
||||
"""Acquire the cross-process lock for path.quarantine.lock."""
|
||||
import platform
|
||||
|
||||
lock_path = path.with_name(path.name + ".quarantine.lock")
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = lock_path.open("a+b")
|
||||
acquired = False
|
||||
try:
|
||||
deadline = time.monotonic() + timeout
|
||||
if platform.system() == "Windows":
|
||||
import msvcrt
|
||||
|
||||
while True:
|
||||
try:
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
acquired = True
|
||||
break
|
||||
except OSError:
|
||||
if time.monotonic() >= deadline:
|
||||
break
|
||||
time.sleep(0.020)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
while True:
|
||||
try:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
acquired = True
|
||||
break
|
||||
except (BlockingIOError, OSError):
|
||||
if time.monotonic() >= deadline:
|
||||
break
|
||||
time.sleep(0.020)
|
||||
yield acquired
|
||||
finally:
|
||||
try:
|
||||
if acquired:
|
||||
if platform.system() == "Windows":
|
||||
import msvcrt
|
||||
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
except (OSError, AttributeError):
|
||||
pass
|
||||
finally:
|
||||
handle.close()
|
||||
|
||||
|
||||
def quarantine_zeroed_state_db(
|
||||
path: Path, *, already_locked: bool = False
|
||||
) -> Optional[Path]:
|
||||
"""Move a zeroed state.db aside (preserve bytes) and return quarantine path.
|
||||
|
||||
Uses a cross-process lock (``#68805``) so two concurrent startups cannot
|
||||
race: the first process moves the zeroed file and the second re-checks
|
||||
under the lock, finding the file already gone (or a fresh DB in its place)
|
||||
instead of clobbering the quarantine.
|
||||
"""
|
||||
def _do_quarantine():
|
||||
if not path.exists():
|
||||
logger.info(
|
||||
"quarantine_zeroed_state_db: %s already moved by another process",
|
||||
path,
|
||||
)
|
||||
return None
|
||||
if not is_zeroed_state_db(path):
|
||||
logger.info(
|
||||
"quarantine_zeroed_state_db: %s is no longer zeroed (another "
|
||||
"process quarantined it and a fresh DB was created)",
|
||||
path,
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
ts = time.strftime("%Y%m%d-%H%M%S")
|
||||
except Exception:
|
||||
ts = "unknown"
|
||||
dest = path.with_name(
|
||||
f"{path.name}.zeroed-{ts}-{os.getpid()}.bak"
|
||||
)
|
||||
n = 0
|
||||
while dest.exists():
|
||||
n += 1
|
||||
dest = path.with_name(
|
||||
f"{path.name}.zeroed-{ts}-{os.getpid()}-{n}.bak"
|
||||
)
|
||||
try:
|
||||
path.rename(dest)
|
||||
except OSError as exc:
|
||||
logger.error("Failed to quarantine zeroed %s: %s", path, exc)
|
||||
return None
|
||||
for suffix in ("-wal", "-shm"):
|
||||
side = Path(str(path) + suffix)
|
||||
if side.exists():
|
||||
try:
|
||||
side.rename(Path(str(dest) + suffix))
|
||||
except OSError:
|
||||
pass
|
||||
return dest
|
||||
|
||||
if already_locked:
|
||||
return _do_quarantine()
|
||||
|
||||
with quarantine_cross_process_lock(path) as acquired:
|
||||
if not acquired:
|
||||
logger.error(
|
||||
"quarantine lock for %s not acquired within 5s — refusing to "
|
||||
"quarantine without the cross-process lock. The zeroed file "
|
||||
"is left in place. If sessions fail to load, restore from "
|
||||
"state-snapshots via `hermes snapshot list` / "
|
||||
"`hermes snapshot restore <id>`.",
|
||||
path,
|
||||
)
|
||||
return None
|
||||
return _do_quarantine()
|
||||
|
||||
|
||||
def collect_state_db_stats(db_path: Path) -> Dict[str, Any]:
|
||||
"""Best-effort, strictly read-only stats snapshot of a state.db file.
|
||||
|
||||
Opens the database with ``mode=ro`` (URI) and a short timeout so it can
|
||||
run against a *live* database held by a gateway without ever taking a
|
||||
write lock or mutating the file. Every field is collected independently:
|
||||
a failed pragma/SELECT yields ``None`` for that field, and the helper
|
||||
itself never raises.
|
||||
|
||||
Deliberately does NOT instantiate :class:`SessionDB` — its constructor
|
||||
runs schema DDL (migrations, FTS table creation), which is exactly the
|
||||
kind of write a diagnostics probe must never perform.
|
||||
|
||||
Returned keys (all present, any may be None on failure):
|
||||
|
||||
- ``page_count``, ``page_size``, ``freelist_count`` — PRAGMA values
|
||||
- ``logical_size_bytes`` — page_count * page_size (post-checkpoint size)
|
||||
- ``wal_size_bytes`` — stat() of ``<db>-wal`` (0 when absent)
|
||||
- ``journal_mode`` — PRAGMA journal_mode string
|
||||
- ``messages`` / ``sessions`` — row counts
|
||||
- ``fts_tables`` — dict of {table_name: bool} presence for
|
||||
messages_fts / messages_fts_trigram / messages_fts_cjk
|
||||
- ``fts_storage_version`` — int from state_meta, None when the marker is
|
||||
absent (legacy pre-v23 inline layout)
|
||||
- ``fts_rebuild_pending`` — True when the deferred v23 backfill has not
|
||||
finished (high_water present and progress < high_water)
|
||||
- ``fts_rebuild_high_water`` / ``fts_rebuild_progress`` — raw ints
|
||||
- ``fts_rebuild_deferral`` — durable blocked-repair diagnostic, when present
|
||||
"""
|
||||
from hermes_state import _connect_tracked_db
|
||||
stats: Dict[str, Any] = {
|
||||
"page_count": None,
|
||||
"page_size": None,
|
||||
"freelist_count": None,
|
||||
"logical_size_bytes": None,
|
||||
"wal_size_bytes": None,
|
||||
"journal_mode": None,
|
||||
"messages": None,
|
||||
"sessions": None,
|
||||
"fts_tables": None,
|
||||
"fts_storage_version": None,
|
||||
"fts_rebuild_pending": None,
|
||||
"fts_rebuild_high_water": None,
|
||||
"fts_rebuild_progress": None,
|
||||
"fts_rebuild_deferral": None,
|
||||
}
|
||||
|
||||
# WAL sidecar size needs no connection at all.
|
||||
try:
|
||||
wal_path = Path(str(db_path) + "-wal")
|
||||
stats["wal_size_bytes"] = wal_path.stat().st_size if wal_path.exists() else 0
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
conn = None
|
||||
try:
|
||||
# mode=ro refuses to create the file and refuses every write; a
|
||||
# short timeout keeps doctor snappy when a writer holds the lock.
|
||||
# Route through the tracked connect so byte-probe helpers
|
||||
# (read_header_bytes_preopen) see this connection and refuse raw
|
||||
# opens that could cancel our POSIX locks mid-read.
|
||||
conn = _connect_tracked_db(
|
||||
f"file:{Path(db_path)}?mode=ro",
|
||||
tracking_path=Path(db_path),
|
||||
uri=True,
|
||||
timeout=2.0,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("collect_state_db_stats: cannot open %s read-only: %s",
|
||||
db_path, exc)
|
||||
return stats
|
||||
|
||||
def _scalar(sql: str) -> Any:
|
||||
try:
|
||||
row = conn.execute(sql).fetchone()
|
||||
return row[0] if row else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
try:
|
||||
pc = _scalar("PRAGMA page_count")
|
||||
ps = _scalar("PRAGMA page_size")
|
||||
stats["page_count"] = int(pc) if pc is not None else None
|
||||
stats["page_size"] = int(ps) if ps is not None else None
|
||||
if stats["page_count"] is not None and stats["page_size"] is not None:
|
||||
stats["logical_size_bytes"] = stats["page_count"] * stats["page_size"]
|
||||
|
||||
fl = _scalar("PRAGMA freelist_count")
|
||||
stats["freelist_count"] = int(fl) if fl is not None else None
|
||||
|
||||
jm = _scalar("PRAGMA journal_mode")
|
||||
stats["journal_mode"] = str(jm) if jm is not None else None
|
||||
|
||||
msgs = _scalar("SELECT COUNT(*) FROM messages")
|
||||
stats["messages"] = int(msgs) if msgs is not None else None
|
||||
sess = _scalar("SELECT COUNT(*) FROM sessions")
|
||||
stats["sessions"] = int(sess) if sess is not None else None
|
||||
|
||||
# FTS table presence via sqlite_master (never SELECTs from the
|
||||
# virtual tables themselves — a corrupt index must not fail stats).
|
||||
try:
|
||||
names = {
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' "
|
||||
"AND name IN (?, ?, ?)",
|
||||
("messages_fts", "messages_fts_trigram", "messages_fts_cjk"),
|
||||
).fetchall()
|
||||
}
|
||||
stats["fts_tables"] = {
|
||||
t: (t in names)
|
||||
for t in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk")
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Raw state_meta reads — cheap, and independent of SessionDB.
|
||||
def _meta_int(key: str) -> Optional[int]:
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT value FROM state_meta WHERE key = ?", (key,)
|
||||
).fetchone()
|
||||
return int(row[0]) if row and row[0] is not None else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
stats["fts_storage_version"] = _meta_int("fts_storage_version")
|
||||
high_water = _meta_int("fts_rebuild_high_water")
|
||||
progress = _meta_int("fts_rebuild_progress")
|
||||
stats["fts_rebuild_high_water"] = high_water
|
||||
stats["fts_rebuild_progress"] = progress
|
||||
if high_water is None:
|
||||
stats["fts_rebuild_pending"] = False
|
||||
else:
|
||||
stats["fts_rebuild_pending"] = (progress or 0) < high_water
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT value FROM state_meta WHERE key = ? LIMIT 1",
|
||||
(FTS_REBUILD_DEFERRAL_KEY,),
|
||||
).fetchone()
|
||||
if row:
|
||||
parsed = json.loads(row[0])
|
||||
if isinstance(parsed, dict):
|
||||
stats["fts_rebuild_deferral"] = parsed
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def count_db_holders(db_path: Path) -> Optional[int]:
|
||||
"""Best-effort count of processes holding ``db_path`` open (Linux only).
|
||||
|
||||
Scans ``/proc/*/fd`` symlinks for the resolved database path. Returns
|
||||
the number of distinct PIDs with the file open, or ``None`` on any
|
||||
error or on non-Linux platforms. Never raises; no lsof dependency.
|
||||
Unreadable per-process fd dirs (other users' processes without root)
|
||||
are silently skipped, so the count is a lower bound.
|
||||
"""
|
||||
try:
|
||||
if not sys.platform.startswith("linux"):
|
||||
return None
|
||||
target = os.path.realpath(str(db_path))
|
||||
holders = 0
|
||||
for pid in os.listdir("/proc"):
|
||||
if not pid.isdigit():
|
||||
continue
|
||||
fd_dir = f"/proc/{pid}/fd"
|
||||
try:
|
||||
fds = os.listdir(fd_dir)
|
||||
except OSError:
|
||||
continue # process gone or not ours
|
||||
for fd in fds:
|
||||
try:
|
||||
if os.readlink(f"{fd_dir}/{fd}") == target:
|
||||
holders += 1
|
||||
break # one hit per PID
|
||||
except OSError:
|
||||
continue
|
||||
return holders
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _is_inactive_orphan_desktop_holder(
|
||||
*,
|
||||
ppid: int,
|
||||
age_seconds: float,
|
||||
min_age_seconds: float,
|
||||
ephemeral_backend: bool,
|
||||
connection_statuses: List[str],
|
||||
) -> bool:
|
||||
"""Pure safety predicate for the narrow Desktop holder reap."""
|
||||
return (
|
||||
ppid in (0, 1)
|
||||
and age_seconds >= min_age_seconds
|
||||
and ephemeral_backend
|
||||
and "ESTABLISHED" not in connection_statuses
|
||||
)
|
||||
|
||||
|
||||
def _concrete_state_db_holder_pids(
|
||||
db_path: Path, holders: List[Tuple[int, str]]
|
||||
) -> List[int]:
|
||||
"""Return unique PIDs proven to hold this DB or one of its sidecars."""
|
||||
canonical_db = os.path.normcase(os.path.abspath(os.fspath(db_path)))
|
||||
watched = {
|
||||
canonical_db,
|
||||
canonical_db + "-wal",
|
||||
canonical_db + "-shm",
|
||||
}
|
||||
pids: List[int] = []
|
||||
seen = set()
|
||||
for pid, path in holders:
|
||||
canonical_path = os.path.normcase(
|
||||
os.path.abspath(path.removesuffix(" (deleted)"))
|
||||
)
|
||||
if pid <= 0 or pid in seen or canonical_path not in watched:
|
||||
continue
|
||||
seen.add(pid)
|
||||
pids.append(pid)
|
||||
return pids
|
||||
|
||||
|
||||
def _read_proc_cmdline(pid: int) -> Optional[str]:
|
||||
"""Read /proc/<pid>/cmdline, world-readable even when fd table is not.
|
||||
|
||||
Returns the cmdline as a space-joined string, or None when unreadable
|
||||
(process exited, or hidepid mount).
|
||||
"""
|
||||
try:
|
||||
with open(f"/proc/{pid}/cmdline", "rb") as f:
|
||||
raw = f.read()
|
||||
if not raw:
|
||||
return None
|
||||
return raw.replace(b"\x00", b" ").decode("utf-8", "replace").strip()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
_HERMES_CMDLINE_MARKERS = ("hermes_cli.main", "hermes_cli/main", "hermes serve",
|
||||
"hermes-agent", "hermes gateway", "hermes chat")
|
||||
|
||||
|
||||
def _looks_like_hermes(cmdline: str) -> bool:
|
||||
"""Heuristic: does this cmdline look like a Hermes process?
|
||||
|
||||
Used to decide whether an uninspectable process (fd table unreadable
|
||||
due to different user) should be treated as a potential state.db holder.
|
||||
We only flag processes that look like Hermes, not every system daemon.
|
||||
"""
|
||||
lower = cmdline.lower()
|
||||
return any(marker in lower for marker in _HERMES_CMDLINE_MARKERS)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,699 @@
|
||||
"""Retention pruning, stale-session archiving and VACUUM policy mixin for
|
||||
SessionDB."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from hermes_state_common import (
|
||||
AUTO_VACUUM_MIN_FREELIST_RATIO,
|
||||
_sql_session_last_active,
|
||||
escape_like as _escape_like,
|
||||
)
|
||||
|
||||
# caplog tests pin the "hermes_state" logger name.
|
||||
logger = logging.getLogger("hermes_state")
|
||||
|
||||
|
||||
class SessionMaintenanceMixin:
|
||||
"""Retention pruning, stale-session archiving and VACUUM policy for SessionDB."""
|
||||
|
||||
def prune_empty_ghost_sessions(self, sessions_dir: "Optional[Path]" = None) -> int:
|
||||
"""Remove empty TUI ghost sessions (no messages, no title, >24hr old)."""
|
||||
cutoff = time.time() - 86400
|
||||
|
||||
def _do(conn):
|
||||
rows = conn.execute("""
|
||||
SELECT id FROM sessions
|
||||
WHERE source = 'tui'
|
||||
AND title IS NULL
|
||||
AND ended_at IS NOT NULL
|
||||
AND started_at < ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM messages WHERE messages.session_id = sessions.id
|
||||
)
|
||||
""", (cutoff,)).fetchall()
|
||||
ids = [r[0] for r in rows]
|
||||
if ids:
|
||||
placeholders = ",".join("?" * len(ids))
|
||||
conn.execute(
|
||||
f"DELETE FROM sessions WHERE id IN ({placeholders})", ids
|
||||
)
|
||||
self._delete_unreferenced_system_prompts(conn)
|
||||
return ids
|
||||
|
||||
removed_ids = self._execute_write(_do) or []
|
||||
if sessions_dir and removed_ids:
|
||||
for sid in removed_ids:
|
||||
self._remove_session_files(sessions_dir, sid)
|
||||
return len(removed_ids)
|
||||
|
||||
def sweep_orphaned_sessions(
|
||||
self,
|
||||
*,
|
||||
max_idle_seconds: float,
|
||||
sources: Tuple[str, ...] = ("tui", "desktop", "subagent"),
|
||||
exclude_ids: Tuple[str, ...] = (),
|
||||
exclude_pinned: bool = False,
|
||||
heartbeat_staleness_seconds: Optional[float] = None,
|
||||
heartbeat_ownership_grace_seconds: Optional[float] = None,
|
||||
respect_gateway_heartbeats: bool = True,
|
||||
) -> List[str]:
|
||||
"""Close session rows orphaned by a dead gateway process.
|
||||
|
||||
The TUI/desktop gateway reaps disconnected sessions with an in-process
|
||||
grace timer; a restart destroys the timer and leaves ``ended_at IS
|
||||
NULL`` forever. This closes rows for ``sources`` whose ``started_at``
|
||||
AND canonical last activity (newest of ``last_activity_at`` and the
|
||||
newest message, else ``started_at``) are both older than
|
||||
``max_idle_seconds``, with ``end_reason='startup_orphan_reap'``. The
|
||||
separate ``started_at`` predicate protects fresh compression/branch
|
||||
children whose copied activity is old.
|
||||
|
||||
Only pass sources whose lifecycle the caller owns — never messaging
|
||||
platforms like ``telegram`` (ending those triggers a routing loop).
|
||||
``exclude_ids`` spares rows this process still holds in memory.
|
||||
Non-destructive: messages are kept and the row stays resumable;
|
||||
first-reason-wins via ``ended_at IS NULL``.
|
||||
|
||||
Cross-backend liveness: with ``respect_gateway_heartbeats``, a row is
|
||||
reaped only when stale AND no live backend (heartbeat within
|
||||
``heartbeat_staleness_seconds``, default ``2 * max_idle_seconds``) could
|
||||
own it, where backend B owns session S if ``B.started_at <= S.started_at
|
||||
+ heartbeat_ownership_grace_seconds`` (default = staleness). The grace
|
||||
covers a migrating backend whose sessions predate its first heartbeat
|
||||
but is bounded so a PID-reuse respawn cannot protect rows forever.
|
||||
Disable the gate only for sources owned by state.db itself.
|
||||
|
||||
SELECT, live-lease validation and UPDATE run in one ``BEGIN IMMEDIATE``
|
||||
transaction. Active turn leases / compression locks spare the row;
|
||||
expired guards are removed so their former owner is fenced.
|
||||
"""
|
||||
from hermes_state import SessionCompressionInProgressError, SessionTurnLeaseLostError
|
||||
srcs = tuple(s for s in sources if s)
|
||||
if max_idle_seconds <= 0 or not srcs:
|
||||
return []
|
||||
hb_staleness = (
|
||||
heartbeat_staleness_seconds
|
||||
if heartbeat_staleness_seconds and heartbeat_staleness_seconds > 0
|
||||
else max_idle_seconds * 2
|
||||
)
|
||||
hb_grace = (
|
||||
heartbeat_ownership_grace_seconds
|
||||
if heartbeat_ownership_grace_seconds is not None
|
||||
and heartbeat_ownership_grace_seconds >= 0
|
||||
else hb_staleness
|
||||
)
|
||||
now = time.time()
|
||||
cutoff = now - max_idle_seconds
|
||||
hb_cutoff = now - hb_staleness
|
||||
placeholders = ",".join("?" for _ in srcs)
|
||||
staleness = (
|
||||
f"started_at < ? AND {_sql_session_last_active('sessions')} < ?"
|
||||
)
|
||||
pin_scope = " AND COALESCE(pinned, 0) = 0" if exclude_pinned else ""
|
||||
heartbeat_params: Tuple[float, ...] = ()
|
||||
orphan_predicate = staleness
|
||||
if respect_gateway_heartbeats:
|
||||
orphan_predicate += (
|
||||
" AND NOT EXISTS ("
|
||||
"SELECT 1 FROM gateway_heartbeats h"
|
||||
" WHERE h.last_heartbeat >= ?"
|
||||
" AND h.started_at <= sessions.started_at + ?"
|
||||
")"
|
||||
)
|
||||
heartbeat_params = (hb_cutoff, hb_grace)
|
||||
|
||||
def _do(conn):
|
||||
rows = conn.execute(
|
||||
f"SELECT id FROM sessions WHERE ended_at IS NULL"
|
||||
f" AND source IN ({placeholders}){pin_scope}"
|
||||
f" AND {orphan_predicate}",
|
||||
(*srcs, cutoff, cutoff, *heartbeat_params),
|
||||
).fetchall()
|
||||
excluded = {str(x) for x in exclude_ids if x}
|
||||
victims = []
|
||||
for row in rows:
|
||||
sid = str(row["id"])
|
||||
if sid in excluded:
|
||||
continue
|
||||
try:
|
||||
self._check_transcript_write_guards(
|
||||
conn,
|
||||
sid,
|
||||
compression_lock_holder=None,
|
||||
turn_lease_holder=None,
|
||||
reject_active_turn_lease=True,
|
||||
reject_active_compression_lock=True,
|
||||
)
|
||||
except (
|
||||
SessionCompressionInProgressError,
|
||||
SessionTurnLeaseLostError,
|
||||
):
|
||||
continue
|
||||
victims.append(sid)
|
||||
if not victims:
|
||||
return []
|
||||
closed_at = time.time()
|
||||
marks = ",".join("?" for _ in victims)
|
||||
# Re-apply every predicate under the write lock.
|
||||
conn.execute(
|
||||
f"UPDATE sessions SET ended_at = ?, end_reason = 'startup_orphan_reap'"
|
||||
f" WHERE id IN ({marks}) AND ended_at IS NULL"
|
||||
f" AND source IN ({placeholders}){pin_scope}"
|
||||
f" AND {orphan_predicate}",
|
||||
(
|
||||
closed_at,
|
||||
*victims,
|
||||
*srcs,
|
||||
cutoff,
|
||||
cutoff,
|
||||
*heartbeat_params,
|
||||
),
|
||||
)
|
||||
return victims
|
||||
|
||||
return self._execute_write(_do) or []
|
||||
|
||||
@staticmethod
|
||||
def _prune_filter_where(
|
||||
*,
|
||||
last_active_before: Optional[float] = None,
|
||||
last_active_after: Optional[float] = None,
|
||||
started_before: Optional[float] = None,
|
||||
started_after: Optional[float] = None,
|
||||
source: Optional[str] = None,
|
||||
title_like: Optional[str] = None,
|
||||
end_reason: Optional[str] = None,
|
||||
cwd_prefix: Optional[str] = None,
|
||||
min_messages: Optional[int] = None,
|
||||
max_messages: Optional[int] = None,
|
||||
archived: Optional[bool] = None,
|
||||
model_like: Optional[str] = None,
|
||||
provider: Optional[str] = None,
|
||||
user_id: Optional[str] = None,
|
||||
chat_id: Optional[str] = None,
|
||||
chat_type: Optional[str] = None,
|
||||
branch_like: Optional[str] = None,
|
||||
min_tokens: Optional[int] = None,
|
||||
max_tokens: Optional[int] = None,
|
||||
min_cost: Optional[float] = None,
|
||||
max_cost: Optional[float] = None,
|
||||
min_tool_calls: Optional[int] = None,
|
||||
max_tool_calls: Optional[int] = None,
|
||||
include_pinned: bool = False,
|
||||
) -> Tuple[str, list]:
|
||||
"""Shared WHERE clause for bulk prune/archive selection (alias ``s``).
|
||||
|
||||
Filters AND together; only ended sessions are ever candidates.
|
||||
``archived`` is tri-state (None = both). ``*_like`` filters are
|
||||
case-insensitive substrings; the rest are exact (provider
|
||||
case-insensitive). Token bounds use input+output; cost bounds use
|
||||
``COALESCE(actual_cost_usd, estimated_cost_usd)``.
|
||||
"""
|
||||
from hermes_state import _cwd_prefix_clause
|
||||
clauses = ["s.ended_at IS NOT NULL"]
|
||||
params: list = []
|
||||
if last_active_before is not None:
|
||||
clauses.append(
|
||||
"""COALESCE(
|
||||
(SELECT MAX(m.timestamp) FROM messages m
|
||||
WHERE m.session_id = s.id),
|
||||
s.started_at
|
||||
) < ?"""
|
||||
)
|
||||
params.append(last_active_before)
|
||||
# Orphan-swept rows age from the sweep, not their old activity, or
|
||||
# the next prune pass deletes them before the user can recover.
|
||||
clauses.append(
|
||||
"(COALESCE(s.end_reason, '') != 'startup_orphan_reap' "
|
||||
"OR s.ended_at < ?)"
|
||||
)
|
||||
params.append(last_active_before)
|
||||
if last_active_after is not None:
|
||||
clauses.append(
|
||||
"""COALESCE(
|
||||
(SELECT MAX(m.timestamp) FROM messages m
|
||||
WHERE m.session_id = s.id),
|
||||
s.started_at
|
||||
) >= ?"""
|
||||
)
|
||||
params.append(last_active_after)
|
||||
if started_before is not None:
|
||||
clauses.append("s.started_at < ?")
|
||||
params.append(started_before)
|
||||
if started_after is not None:
|
||||
clauses.append("s.started_at >= ?")
|
||||
params.append(started_after)
|
||||
if source:
|
||||
clauses.append("s.source = ?")
|
||||
params.append(source)
|
||||
if title_like:
|
||||
clauses.append("LOWER(COALESCE(s.title, '')) LIKE ? ESCAPE '\\'")
|
||||
params.append(f"%{_escape_like(title_like.lower())}%")
|
||||
if end_reason:
|
||||
clauses.append("s.end_reason = ?")
|
||||
params.append(end_reason)
|
||||
if cwd_prefix:
|
||||
clause, clause_params = _cwd_prefix_clause(cwd_prefix)
|
||||
clauses.append(clause)
|
||||
params.extend(clause_params)
|
||||
if min_messages is not None:
|
||||
clauses.append("s.message_count >= ?")
|
||||
params.append(min_messages)
|
||||
if max_messages is not None:
|
||||
clauses.append("s.message_count <= ?")
|
||||
params.append(max_messages)
|
||||
if model_like:
|
||||
clauses.append("LOWER(COALESCE(s.model, '')) LIKE ? ESCAPE '\\'")
|
||||
params.append(f"%{_escape_like(model_like.lower())}%")
|
||||
if provider:
|
||||
clauses.append("LOWER(COALESCE(s.billing_provider, '')) = ?")
|
||||
params.append(provider.lower())
|
||||
if user_id:
|
||||
clauses.append("s.user_id = ?")
|
||||
params.append(user_id)
|
||||
if chat_id:
|
||||
clauses.append("s.chat_id = ?")
|
||||
params.append(chat_id)
|
||||
if chat_type:
|
||||
clauses.append("s.chat_type = ?")
|
||||
params.append(chat_type)
|
||||
if branch_like:
|
||||
clauses.append("LOWER(COALESCE(s.git_branch, '')) LIKE ? ESCAPE '\\'")
|
||||
params.append(f"%{_escape_like(branch_like.lower())}%")
|
||||
if min_tokens is not None:
|
||||
clauses.append(
|
||||
"(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) >= ?"
|
||||
)
|
||||
params.append(min_tokens)
|
||||
if max_tokens is not None:
|
||||
clauses.append(
|
||||
"(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) <= ?"
|
||||
)
|
||||
params.append(max_tokens)
|
||||
if min_cost is not None:
|
||||
clauses.append(
|
||||
"COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) >= ?"
|
||||
)
|
||||
params.append(min_cost)
|
||||
if max_cost is not None:
|
||||
clauses.append(
|
||||
"COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) <= ?"
|
||||
)
|
||||
params.append(max_cost)
|
||||
if min_tool_calls is not None:
|
||||
clauses.append("COALESCE(s.tool_call_count, 0) >= ?")
|
||||
params.append(min_tool_calls)
|
||||
if max_tool_calls is not None:
|
||||
clauses.append("COALESCE(s.tool_call_count, 0) <= ?")
|
||||
params.append(max_tool_calls)
|
||||
if archived is True:
|
||||
clauses.append("s.archived = 1")
|
||||
elif archived is False:
|
||||
clauses.append("s.archived = 0")
|
||||
# Pinned is a durable "keep" flag: bulk prune/delete/archive exclude
|
||||
# pinned rows unless the caller explicitly opts in.
|
||||
if not include_pinned:
|
||||
clauses.append("COALESCE(s.pinned, 0) = 0")
|
||||
return " AND ".join(clauses), params
|
||||
|
||||
@staticmethod
|
||||
def _apply_prune_age_filter(
|
||||
older_than_days: Optional[float], filters: Dict[str, Any]
|
||||
) -> None:
|
||||
"""Translate the legacy age window into the shared activity filter."""
|
||||
if (
|
||||
filters.get("last_active_before") is None
|
||||
and filters.get("started_before") is None
|
||||
and older_than_days is not None
|
||||
):
|
||||
filters["last_active_before"] = time.time() - (
|
||||
older_than_days * 86400
|
||||
)
|
||||
|
||||
def list_prune_candidates(
|
||||
self,
|
||||
older_than_days: Optional[float] = None,
|
||||
source: str = None,
|
||||
**filters,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Sessions a matching prune/archive would touch (dry-run), oldest
|
||||
first. Same filters as :meth:`_prune_filter_where`; ``older_than_days``
|
||||
is an inactivity threshold (latest message, else ``started_at``)."""
|
||||
self._apply_prune_age_filter(older_than_days, filters)
|
||||
where, params = self._prune_filter_where(source=source, **filters)
|
||||
with self._read_ctx() as conn:
|
||||
cursor = conn.execute(
|
||||
f"""SELECT s.id, s.source, s.title, s.model, s.started_at,
|
||||
COALESCE(
|
||||
(SELECT MAX(m.timestamp) FROM messages m
|
||||
WHERE m.session_id = s.id),
|
||||
s.started_at
|
||||
) AS last_active,
|
||||
s.ended_at, s.message_count, s.archived
|
||||
FROM sessions s WHERE {where}
|
||||
ORDER BY last_active ASC, s.started_at ASC""",
|
||||
params,
|
||||
)
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
def count_prune_matches(
|
||||
self,
|
||||
older_than_days: Optional[float] = None,
|
||||
source: str = None,
|
||||
**filters,
|
||||
) -> int:
|
||||
"""Count-only variant of :meth:`list_prune_candidates` (the CLI uses it
|
||||
to report how many pinned sessions are spared)."""
|
||||
self._apply_prune_age_filter(older_than_days, filters)
|
||||
where, params = self._prune_filter_where(source=source, **filters)
|
||||
with self._read_ctx() as conn:
|
||||
cursor = conn.execute(
|
||||
f"SELECT COUNT(*) FROM sessions s WHERE {where}", params
|
||||
)
|
||||
return int(cursor.fetchone()[0])
|
||||
|
||||
def count_open_prune_matches(
|
||||
self,
|
||||
older_than_days: Optional[float] = None,
|
||||
source: str = None,
|
||||
**filters,
|
||||
) -> int:
|
||||
"""Count open sessions a matching prune skips: every normal filter with
|
||||
only the ``ended_at`` guard inverted. Visibility-only; live sessions
|
||||
never become prune-eligible."""
|
||||
self._apply_prune_age_filter(older_than_days, filters)
|
||||
where, params = self._prune_filter_where(source=source, **filters)
|
||||
ended_guard = "s.ended_at IS NOT NULL"
|
||||
if not where.startswith(ended_guard):
|
||||
raise RuntimeError("prune filter lost its ended-session safety guard")
|
||||
open_where = f"s.ended_at IS NULL{where[len(ended_guard):]}"
|
||||
with self._read_ctx() as conn:
|
||||
cursor = conn.execute(
|
||||
f"SELECT COUNT(*) FROM sessions s WHERE {open_where}", params
|
||||
)
|
||||
return int(cursor.fetchone()[0])
|
||||
|
||||
def archive_stale_sessions(
|
||||
self, idle_days: float, *, exclude_pinned: bool = True
|
||||
) -> int:
|
||||
"""Archive every session untouched for ``idle_days`` (real recency:
|
||||
freshest of ``last_activity_at`` / latest message / ``started_at``).
|
||||
Unlike :meth:`archive_sessions`, this can archive unended sessions.
|
||||
|
||||
Guards: ``pinned = 0`` when ``exclude_pinned``; ``archived = 0`` so
|
||||
repeats are no-ops; only lineage tips (``end_reason <> 'compression'``)
|
||||
are candidates — a stale tip archives its chain via
|
||||
:meth:`set_session_archived`, so an old compressed-away root with a
|
||||
recent continuation is never matched. Returns the count archived.
|
||||
"""
|
||||
if idle_days is None or idle_days < 0:
|
||||
return 0
|
||||
cutoff = time.time() - float(idle_days) * 86400.0
|
||||
pin_clause = "AND s.pinned = 0" if exclude_pinned else ""
|
||||
rows = self._read_all(
|
||||
f"""
|
||||
SELECT s.id FROM sessions s
|
||||
WHERE s.archived = 0
|
||||
AND COALESCE(s.end_reason, '') <> 'compression'
|
||||
{pin_clause}
|
||||
AND {_sql_session_last_active("s")} < ?
|
||||
ORDER BY s.started_at ASC
|
||||
""",
|
||||
(cutoff,),
|
||||
)
|
||||
ids = [r[0] for r in rows]
|
||||
for sid in ids:
|
||||
self.set_session_archived(sid, True)
|
||||
return len(ids)
|
||||
|
||||
def prune_sessions(
|
||||
self,
|
||||
older_than_days: Optional[float] = 90,
|
||||
source: str = None,
|
||||
sessions_dir: Optional[Path] = None,
|
||||
exclude_active_write_guards: bool = False,
|
||||
**filters,
|
||||
) -> int:
|
||||
"""Delete ended sessions matching the filters; returns the count.
|
||||
|
||||
Default: inactive for ``older_than_days`` (latest message, else
|
||||
``started_at``), optionally by ``source``. Extra keyword filters are
|
||||
those of :meth:`_prune_filter_where`; an explicit ``started_before`` /
|
||||
``last_active_before`` overrides the ``older_than_days`` cutoff
|
||||
(pass ``older_than_days=None`` for no implicit age bound).
|
||||
|
||||
Children outside the window are orphaned (parent NULLed), not cascade-
|
||||
deleted. With *sessions_dir*, on-disk transcript files are removed
|
||||
outside the DB transaction. ``exclude_active_write_guards`` (automatic
|
||||
maintenance) skips rows under a live turn lease or compression lock,
|
||||
while expired/dead holders are reclaimed and fenced in the same write.
|
||||
"""
|
||||
from hermes_state import SessionCompressionInProgressError, SessionTurnLeaseLostError
|
||||
self._apply_prune_age_filter(older_than_days, filters)
|
||||
where, where_params = self._prune_filter_where(source=source, **filters)
|
||||
removed_ids: list[str] = []
|
||||
|
||||
def _do(conn):
|
||||
cursor = conn.execute(
|
||||
f"SELECT s.id FROM sessions s WHERE {where}", where_params
|
||||
)
|
||||
session_ids = {row["id"] for row in cursor.fetchall()}
|
||||
|
||||
if exclude_active_write_guards:
|
||||
protected = set()
|
||||
for sid in session_ids:
|
||||
try:
|
||||
self._check_transcript_write_guards(
|
||||
conn,
|
||||
sid,
|
||||
compression_lock_holder=None,
|
||||
turn_lease_holder=None,
|
||||
reject_active_turn_lease=True,
|
||||
reject_active_compression_lock=True,
|
||||
allow_closed_compression_parent=True,
|
||||
)
|
||||
except (
|
||||
SessionCompressionInProgressError,
|
||||
SessionTurnLeaseLostError,
|
||||
):
|
||||
protected.add(sid)
|
||||
session_ids.difference_update(protected)
|
||||
|
||||
if not session_ids:
|
||||
return 0
|
||||
|
||||
placeholders = ",".join("?" * len(session_ids))
|
||||
conn.execute(
|
||||
f"UPDATE sessions SET parent_session_id = NULL "
|
||||
f"WHERE parent_session_id IN ({placeholders})",
|
||||
list(session_ids),
|
||||
)
|
||||
|
||||
for sid in session_ids:
|
||||
conn.execute("DELETE FROM messages WHERE session_id = ?", (sid,))
|
||||
conn.execute("DELETE FROM sessions WHERE id = ?", (sid,))
|
||||
removed_ids.append(sid)
|
||||
self._delete_unreferenced_system_prompts(conn)
|
||||
return len(session_ids)
|
||||
|
||||
count = self._execute_write(_do)
|
||||
for sid in removed_ids:
|
||||
self._remove_session_files(sessions_dir, sid)
|
||||
return count
|
||||
|
||||
def logical_size_bytes(self) -> Optional[int]:
|
||||
"""``page_count * page_size``: the main-file size once the WAL is
|
||||
checkpointed back in. Prefer over ``os.path.getsize`` when reporting a
|
||||
VACUUM: in WAL mode the rewrite lands in ``-wal`` and the checkpoint is
|
||||
refused while another connection holds a read-mark, so a stat() delta
|
||||
understates the win and can go negative. None if pragmas fail.
|
||||
"""
|
||||
try:
|
||||
with self._read_ctx() as conn:
|
||||
if self._conn is None:
|
||||
return None
|
||||
page_count = conn.execute("PRAGMA page_count").fetchone()[0]
|
||||
page_size = conn.execute("PRAGMA page_size").fetchone()[0]
|
||||
return int(page_count) * int(page_size)
|
||||
except Exception as exc:
|
||||
logger.debug("Could not read logical DB size: %s", exc)
|
||||
return None
|
||||
|
||||
def _freelist_ratio(self) -> Optional[float]:
|
||||
"""Reclaimable fraction (``freelist_count / page_count``) over the
|
||||
existing connection — never a byte-level probe of the live file. Gates
|
||||
VACUUM in :meth:`maybe_auto_prune_and_vacuum`. None if pragmas fail
|
||||
(callers then fall back to the time throttle alone).
|
||||
"""
|
||||
try:
|
||||
with self._read_ctx() as conn:
|
||||
if self._conn is None:
|
||||
return None
|
||||
page_count = int(conn.execute("PRAGMA page_count").fetchone()[0])
|
||||
freelist = int(conn.execute("PRAGMA freelist_count").fetchone()[0])
|
||||
if page_count <= 0:
|
||||
return 0.0
|
||||
return freelist / page_count
|
||||
except Exception as exc:
|
||||
logger.debug("Could not read freelist ratio: %s", exc)
|
||||
return None
|
||||
|
||||
def vacuum(self) -> int:
|
||||
"""VACUUM to reclaim space after large deletes (SQLite never shrinks
|
||||
the file on its own).
|
||||
|
||||
Rewrites the whole DB, cannot run inside a transaction, and takes an
|
||||
exclusive lock — callers must ensure no other writers are active (safe
|
||||
at startup before serving traffic). FTS5 segments are merged first via
|
||||
:meth:`optimize_fts` so the VACUUM reclaims those pages too. Returns
|
||||
the number of FTS indexes optimized (0 on merge failure / no FTS).
|
||||
"""
|
||||
# optimize_fts() manages its own lock.
|
||||
optimized = 0
|
||||
try:
|
||||
optimized = self.optimize_fts()
|
||||
except Exception as exc:
|
||||
logger.warning("FTS optimize before VACUUM failed: %s", exc)
|
||||
# VACUUM cannot be executed inside a transaction.
|
||||
with self._lock:
|
||||
# PASSIVE, not TRUNCATE: a manual `hermes sessions vacuum` runs in
|
||||
# a transient CLI process, and a TRUNCATE reset here would race a
|
||||
# live gateway writer and tear B-tree pages.
|
||||
try:
|
||||
self._conn.execute("PRAGMA wal_checkpoint(PASSIVE)")
|
||||
except Exception as exc:
|
||||
logger.debug("WAL checkpoint (PASSIVE) before VACUUM failed: %s", exc)
|
||||
self._conn.execute("VACUUM")
|
||||
# VACUUM rewrites every page THROUGH the WAL; without this TRUNCATE
|
||||
# a 3 GB database leaves a 3 GB -wal behind and the command is a
|
||||
# net loss on disk.
|
||||
try:
|
||||
self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
except Exception as exc:
|
||||
logger.debug("WAL checkpoint (TRUNCATE) after VACUUM failed: %s", exc)
|
||||
# TRUNCATE may replace the WAL inode; adopt the new sidecars so the
|
||||
# write-path generation guard does not halt this connection.
|
||||
self._record_db_file_identity()
|
||||
return optimized
|
||||
|
||||
def maybe_auto_prune_and_vacuum(
|
||||
self,
|
||||
retention_days: int = 90,
|
||||
min_interval_hours: int = 24,
|
||||
vacuum: bool = True,
|
||||
sessions_dir: Optional[Path] = None,
|
||||
min_vacuum_interval_days: int = 30,
|
||||
min_vacuum_freelist_ratio: float = AUTO_VACUUM_MIN_FREELIST_RATIO,
|
||||
) -> Dict[str, Any]:
|
||||
"""Idempotent startup auto-maintenance: prune inactive sessions,
|
||||
reap stale open state-owned rows, optional VACUUM. Never raises.
|
||||
|
||||
Runs at most once per ``min_interval_hours`` (state_meta). VACUUM has
|
||||
its own ``min_vacuum_interval_days`` throttle and additionally requires
|
||||
``freelist_count / page_count`` > ``min_vacuum_freelist_ratio`` so a
|
||||
small prune on a dense multi-GB database never triggers a full rewrite.
|
||||
With *sessions_dir*, pruned transcripts are removed from disk too.
|
||||
|
||||
Stale-open reconciliation: cron/kanban/subagent/one-shot CLI rows never
|
||||
set ``ended_at`` when their process dies, and prune only deletes ended
|
||||
rows. After pruning, open rows from :attr:`_AUTO_PRUNE_STALE_OPEN_SOURCES`
|
||||
older than ``retention_days`` are closed (``startup_orphan_reap``); they
|
||||
stay resumable and age from their close, so they get one more full
|
||||
retention window. Messaging and UI sources are never touched.
|
||||
|
||||
Returns ``{"skipped", "pruned", "closed", "vacuumed"}`` plus
|
||||
``"freelist_ratio"`` when a VACUUM was considered and ``"error"`` on
|
||||
failure.
|
||||
"""
|
||||
from hermes_state import _release_auto_maintenance_lock, _try_acquire_auto_maintenance_lock
|
||||
result: Dict[str, Any] = {
|
||||
"skipped": False,
|
||||
"pruned": 0,
|
||||
"closed": 0,
|
||||
"vacuumed": False,
|
||||
}
|
||||
maintenance_lock = _try_acquire_auto_maintenance_lock(self.db_path)
|
||||
if maintenance_lock is None:
|
||||
result["skipped"] = True
|
||||
return result
|
||||
try:
|
||||
last_raw = self.get_meta("last_auto_prune")
|
||||
now = time.time()
|
||||
if last_raw:
|
||||
try:
|
||||
last_ts = float(last_raw)
|
||||
if now - last_ts < min_interval_hours * 3600:
|
||||
result["skipped"] = True
|
||||
return result
|
||||
except (TypeError, ValueError):
|
||||
pass # corrupt meta; treat as no prior run
|
||||
|
||||
# Prune first: orphans closed below get a full retention window.
|
||||
pruned = self.prune_sessions(
|
||||
older_than_days=retention_days,
|
||||
sessions_dir=sessions_dir,
|
||||
exclude_active_write_guards=True,
|
||||
)
|
||||
result["pruned"] = pruned
|
||||
|
||||
closed = self.sweep_orphaned_sessions(
|
||||
max_idle_seconds=float(retention_days) * 86400.0,
|
||||
sources=self._AUTO_PRUNE_STALE_OPEN_SOURCES,
|
||||
exclude_pinned=True,
|
||||
# State-owned lifecycles, not gateway heartbeats.
|
||||
respect_gateway_heartbeats=False,
|
||||
)
|
||||
result["closed"] = len(closed)
|
||||
# VACUUM only if rows were freed, the time throttle passed ("not
|
||||
# too often") AND the freelist ratio passed ("only when it pays
|
||||
# off") — it holds an exclusive lock for a full rewrite.
|
||||
last_vacuum_raw = self.get_meta("last_vacuum")
|
||||
vacuum_due = True
|
||||
if last_vacuum_raw:
|
||||
try:
|
||||
vacuum_due = (now - float(last_vacuum_raw)) >= min_vacuum_interval_days * 86400
|
||||
except (TypeError, ValueError):
|
||||
vacuum_due = True
|
||||
if vacuum and pruned > 0 and vacuum_due:
|
||||
ratio = self._freelist_ratio()
|
||||
result["freelist_ratio"] = ratio
|
||||
if ratio is None or ratio > min_vacuum_freelist_ratio:
|
||||
try:
|
||||
self.vacuum()
|
||||
result["vacuumed"] = True
|
||||
self.set_meta("last_vacuum", str(now))
|
||||
except Exception as exc:
|
||||
logger.warning("state.db VACUUM failed: %s", exc)
|
||||
else:
|
||||
logger.debug(
|
||||
"state.db auto-maintenance: skipping VACUUM, only "
|
||||
"%.1f%% of pages reclaimable (threshold %.0f%%)",
|
||||
ratio * 100.0,
|
||||
min_vacuum_freelist_ratio * 100.0,
|
||||
)
|
||||
|
||||
# Record even when pruned == 0 so the throttle holds.
|
||||
self.set_meta("last_auto_prune", str(now))
|
||||
|
||||
if closed or pruned > 0:
|
||||
logger.info(
|
||||
"state.db auto-maintenance: closed %d stale open session(s), "
|
||||
"pruned %d session(s) inactive for %d days%s",
|
||||
len(closed),
|
||||
pruned,
|
||||
retention_days,
|
||||
" + VACUUM" if result["vacuumed"] else "",
|
||||
)
|
||||
except Exception as exc:
|
||||
# Maintenance must never block startup.
|
||||
logger.warning("state.db auto-maintenance failed: %s", exc)
|
||||
result["error"] = str(exc)
|
||||
finally:
|
||||
_release_auto_maintenance_lock(maintenance_lock)
|
||||
|
||||
return result
|
||||
File diff suppressed because it is too large
Load Diff
+119
-300
@@ -1,11 +1,9 @@
|
||||
"""Session listing/rich rows, export, and import (portability) for SessionDB.
|
||||
|
||||
Mixin contract: this is a plain mixin class consumed by
|
||||
``hermes_state.SessionDB``. It defines no ``__init__`` and no state of its
|
||||
own; methods access the host's attributes (``self._conn``, ``self.db_path``,
|
||||
``self._execute_write`` and other SessionDB methods) established by
|
||||
``SessionDB.__init__``. It must never import hermes_state (cycle) — shared
|
||||
module-level constants live in hermes_state_common.
|
||||
Plain mixin consumed by ``hermes_state.SessionDB``: no ``__init__``, no state
|
||||
of its own; methods use host attributes established by ``SessionDB.__init__``.
|
||||
Must never import hermes_state (cycle) — shared constants live in
|
||||
hermes_state_common.
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -16,14 +14,12 @@ from typing import Any, Dict, List, Optional
|
||||
from agent.skill_commands import SKILL_SCAFFOLD_SQL_LIKE
|
||||
from hermes_state_common import (
|
||||
SCHEMA_SQL,
|
||||
_PREVIEW_ELIGIBLE_SQL,
|
||||
_PREVIEW_RAW_SELECT,
|
||||
_PREVIEW_RAW_SUBQUERY_SQL,
|
||||
_shape_preview,
|
||||
_sql_session_last_active,
|
||||
)
|
||||
|
||||
# Moved methods logged under the "hermes_state" logger before the split;
|
||||
# keep that logger identity so log filtering/capture behavior is unchanged.
|
||||
# Keep the pre-split logger identity so log filtering/capture is unchanged.
|
||||
logger = logging.getLogger("hermes_state")
|
||||
|
||||
|
||||
@@ -32,9 +28,8 @@ class SessionPortabilityMixin:
|
||||
|
||||
@classmethod
|
||||
def _compact_session_cols(cls) -> str:
|
||||
"""SELECT list for compact_rows: every ``sessions`` column declared in
|
||||
SCHEMA_SQL except prompt storage internals, aliased with the ``s``
|
||||
prefix used by list_sessions_rich/_get_session_rich_row queries."""
|
||||
"""``s.``-prefixed SELECT list of every SCHEMA_SQL ``sessions`` column
|
||||
except prompt storage internals (the compact_rows projection)."""
|
||||
if cls._session_compact_cols_sql is None:
|
||||
declared = cls._parse_schema_columns(SCHEMA_SQL)["sessions"]
|
||||
cls._session_compact_cols_sql = ", ".join(
|
||||
@@ -43,13 +38,19 @@ class SessionPortabilityMixin:
|
||||
)
|
||||
return cls._session_compact_cols_sql
|
||||
|
||||
@classmethod
|
||||
def _rich_row(cls, row) -> Dict[str, Any]:
|
||||
"""Session row dict with ``_preview_raw`` shaped into ``preview``."""
|
||||
s = cls._session_row_dict(row)
|
||||
s["preview"] = _shape_preview(s.pop("_preview_raw", ""))
|
||||
return s
|
||||
|
||||
def distinct_session_cwds(self, include_archived: bool = False) -> List[Dict[str, Any]]:
|
||||
"""Distinct non-empty session cwds with usage stats, for repo discovery.
|
||||
|
||||
Aggregates across ALL session history (not a single page), so the desktop
|
||||
can surface every git repo the user has worked in — not just the repos
|
||||
that happen to be in the currently-loaded recents. Children/branches
|
||||
count: a worktree session is still a real workspace signal.
|
||||
Aggregates across ALL history (not one page) so every repo the user
|
||||
worked in surfaces. Children/branches count: a worktree session is a
|
||||
real workspace signal.
|
||||
"""
|
||||
where = "cwd IS NOT NULL AND TRIM(cwd) != ''"
|
||||
if not include_archived:
|
||||
@@ -77,41 +78,24 @@ class SessionPortabilityMixin:
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""List the run sessions produced by a single cron job, newest first.
|
||||
|
||||
Cron runs are flat, independent sessions whose id is
|
||||
``cron_{job_id}_{timestamp}`` (see ``cron/scheduler.run_job``). They are
|
||||
never compression roots and never branch, so this deliberately skips the
|
||||
``list_sessions_rich`` recursive compression-chain CTE / leading-wildcard
|
||||
``id_query`` path — that path seeds from *every* ``source='cron'`` row in
|
||||
the DB and only filters to one job's runs after the scan, so it scales
|
||||
with the whole cron pile (a heavy history makes the desktop run-history
|
||||
endpoint time out before it eventually populates).
|
||||
Cron runs are flat sessions with id ``cron_{job_id}_{timestamp}``; they
|
||||
never compress or branch, so this skips ``list_sessions_rich``'s
|
||||
compression-chain CTE / leading-wildcard ``id_query`` path, which seeds
|
||||
from EVERY ``source='cron'`` row and scales with the whole cron pile.
|
||||
Instead: a ``[prefix, prefix_hi)`` index range scan on id, filtered to
|
||||
``source='cron'``, so work scales with the requested window.
|
||||
|
||||
Instead this binds to one job with a ``[prefix, prefix_hi)`` range over
|
||||
the id (an index range scan, not a ``%...%`` substring), filters
|
||||
``source='cron'``, and orders by ``started_at DESC``. Work scales with
|
||||
the requested window, not the total cron history.
|
||||
|
||||
Returns the same enriched row shape as ``list_sessions_rich`` (adds
|
||||
``preview`` + ``last_active``) so callers can reuse it.
|
||||
Returns the ``list_sessions_rich`` row shape (``preview`` + ``last_active``).
|
||||
"""
|
||||
prefix = f"cron_{job_id}_"
|
||||
# Half-open upper bound for an index range scan: increment the final
|
||||
# byte of the prefix so the range covers exactly the ids that start
|
||||
# with ``prefix`` and nothing else. ``prefix`` always ends in '_', but
|
||||
# compute it generically rather than hardcoding the successor char.
|
||||
# Half-open upper bound: bump the final byte so the range covers exactly
|
||||
# the ids starting with ``prefix``.
|
||||
prefix_hi = prefix[:-1] + chr(ord(prefix[-1]) + 1)
|
||||
|
||||
query = f"""
|
||||
SELECT s.*,
|
||||
COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved,
|
||||
COALESCE(
|
||||
(SELECT {_PREVIEW_RAW_SELECT}
|
||||
FROM messages m
|
||||
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
|
||||
AND {_PREVIEW_ELIGIBLE_SQL}
|
||||
ORDER BY m.timestamp, m.id LIMIT 1),
|
||||
''
|
||||
) AS _preview_raw,
|
||||
{_PREVIEW_RAW_SUBQUERY_SQL},
|
||||
{_sql_session_last_active("s")} AS last_active
|
||||
FROM sessions s
|
||||
LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash
|
||||
@@ -120,27 +104,12 @@ class SessionPortabilityMixin:
|
||||
LIMIT ? OFFSET ?
|
||||
"""
|
||||
with self._lock:
|
||||
cursor = self._conn.execute(query, (prefix, prefix_hi, limit, offset))
|
||||
rows = cursor.fetchall()
|
||||
|
||||
runs: List[Dict[str, Any]] = []
|
||||
for row in rows:
|
||||
s = self._session_row_dict(row)
|
||||
s["preview"] = _shape_preview(s.pop("_preview_raw", ""))
|
||||
runs.append(s)
|
||||
return runs
|
||||
rows = self._conn.execute(query, (prefix, prefix_hi, limit, offset)).fetchall()
|
||||
return [self._rich_row(row) for row in rows]
|
||||
|
||||
def _get_session_rich_row(self, session_id: str, compact_rows: bool = False) -> Optional[Dict[str, Any]]:
|
||||
"""Fetch a single session with the same enriched columns as
|
||||
``list_sessions_rich`` (preview + last_active). Returns None if the
|
||||
session doesn't exist.
|
||||
|
||||
Pass ``compact_rows=True`` to omit the ``system_prompt`` blob (see
|
||||
``list_sessions_rich`` for details).
|
||||
|
||||
Thin wrapper over ``_get_session_rich_rows_batch`` so the enriched
|
||||
SELECT lives in exactly one place.
|
||||
"""
|
||||
"""One session with the ``list_sessions_rich`` enriched columns, or
|
||||
None. ``compact_rows=True`` omits the ``system_prompt`` blob."""
|
||||
return self._get_session_rich_rows_batch(
|
||||
[session_id], compact_rows=compact_rows
|
||||
).get(session_id)
|
||||
@@ -148,23 +117,15 @@ class SessionPortabilityMixin:
|
||||
def _get_session_rich_rows_batch(
|
||||
self, session_ids, compact_rows: bool = False
|
||||
) -> Dict[str, Dict[str, Any]]:
|
||||
"""Fetch multiple sessions with the same enriched columns as
|
||||
``_get_session_rich_row``, in a single query.
|
||||
|
||||
Used by ``list_sessions_rich``'s compression-tip projection to resolve
|
||||
every tip row for a page in one round trip instead of one query per
|
||||
compression-root row. Returns a dict keyed by session id; ids that
|
||||
don't exist are simply absent from the result (same as
|
||||
``_get_session_rich_row`` returning ``None`` for them).
|
||||
"""Enriched rows for many sessions in one query, keyed by id; missing
|
||||
ids are simply absent. Resolves a page of compression tips in one
|
||||
round trip instead of one query per root row.
|
||||
"""
|
||||
ids = [sid for sid in session_ids if sid]
|
||||
if not ids:
|
||||
return {}
|
||||
# Old SQLite builds cap bound variables at 999
|
||||
# (SQLITE_MAX_VARIABLE_NUMBER); large pages (limit=10000 callers
|
||||
# exist) could exceed it. Chunk the IN list so the helper is safe at
|
||||
# any page size — this is the single choke point for the enriched
|
||||
# multi-row fetch, so the bound lives here, not at call sites.
|
||||
# Old SQLite caps bound variables at 999 (SQLITE_MAX_VARIABLE_NUMBER);
|
||||
# limit=10000 callers exist. Chunk here — the single choke point.
|
||||
_CHUNK = 900
|
||||
if len(ids) > _CHUNK:
|
||||
result: Dict[str, Dict[str, Any]] = {}
|
||||
@@ -189,45 +150,26 @@ class SessionPortabilityMixin:
|
||||
)
|
||||
query = f"""
|
||||
SELECT {_sel}{prompt_select},
|
||||
COALESCE(
|
||||
(SELECT {_PREVIEW_RAW_SELECT}
|
||||
FROM messages m
|
||||
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
|
||||
AND {_PREVIEW_ELIGIBLE_SQL}
|
||||
ORDER BY m.timestamp, m.id LIMIT 1),
|
||||
''
|
||||
) AS _preview_raw,
|
||||
{_PREVIEW_RAW_SUBQUERY_SQL},
|
||||
{_sql_session_last_active("s")} AS last_active
|
||||
FROM sessions s
|
||||
{prompt_join}
|
||||
WHERE s.id IN ({placeholders})
|
||||
"""
|
||||
with self._lock:
|
||||
cursor = self._conn.execute(query, ids)
|
||||
rows = cursor.fetchall()
|
||||
result: Dict[str, Dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
s = self._session_row_dict(row)
|
||||
s["preview"] = _shape_preview(s.pop("_preview_raw", ""))
|
||||
result[s["id"]] = s
|
||||
return result
|
||||
rows = self._conn.execute(query, ids).fetchall()
|
||||
return {s["id"]: s for s in map(self._rich_row, rows)}
|
||||
|
||||
def get_session_rich_row(self, session_id: str, compact_rows: bool = False) -> Optional[Dict[str, Any]]:
|
||||
"""Public wrapper for :meth:`_get_session_rich_row`.
|
||||
|
||||
Exposes the single-session enriched row (same columns as
|
||||
``list_sessions_rich``: preview + last_active) for callers outside
|
||||
this module, e.g. the web server's session-search hydration.
|
||||
"""
|
||||
"""Public wrapper for :meth:`_get_session_rich_row` (web server hydration)."""
|
||||
return self._get_session_rich_row(session_id, compact_rows=compact_rows)
|
||||
|
||||
def list_skill_scaffolded_sessions(self, limit: int = 200) -> List[Dict[str, Any]]:
|
||||
"""Titled sessions whose first user turn was a ``/skill`` invocation.
|
||||
|
||||
Those titles were generated from the expanded message, which embeds the
|
||||
whole skill body — so they describe the skill rather than the request.
|
||||
Returns ``id``, ``title``, and the full first-turn ``content`` so a
|
||||
caller can re-derive what the user typed. Newest first.
|
||||
Their titles were generated from the expanded skill body, so they
|
||||
describe the skill, not the request. Returns ``id``, ``title`` and the
|
||||
first-turn ``content`` so callers can re-derive what was typed. Newest first.
|
||||
"""
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
@@ -248,63 +190,36 @@ class SessionPortabilityMixin:
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def get_first_assistant_text(self, session_id: str) -> str:
|
||||
"""The session's first assistant reply as plain text ('' when none).
|
||||
|
||||
Pairs with :meth:`list_skill_scaffolded_sessions` so a re-title can feed
|
||||
the titler the same (request, reply) shape the live path uses.
|
||||
"""
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"SELECT content FROM messages "
|
||||
"WHERE session_id = ? AND role = 'assistant' AND content IS NOT NULL "
|
||||
"ORDER BY timestamp, id LIMIT 1",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return ""
|
||||
decoded = self._decode_content(row["content"])
|
||||
return decoded if isinstance(decoded, str) else ""
|
||||
|
||||
def export_session(self, session_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Export a single session with all its messages as a dict."""
|
||||
session = self.get_session(session_id)
|
||||
if not session:
|
||||
return None
|
||||
messages = self.get_messages(session_id)
|
||||
return {**session, "messages": messages}
|
||||
return {**session, "messages": self.get_messages(session_id)}
|
||||
|
||||
def export_session_lineage(self, session_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Export a compression lineage as one logical session dict."""
|
||||
lineage_ids = self.get_compression_lineage(session_id)
|
||||
if not lineage_ids:
|
||||
return None
|
||||
segments = []
|
||||
for sid in lineage_ids:
|
||||
segment = self.export_session(sid)
|
||||
if segment:
|
||||
segments.append(segment)
|
||||
segments = [seg for seg in map(self.export_session, lineage_ids) if seg]
|
||||
if not segments:
|
||||
return None
|
||||
base = dict(segments[-1])
|
||||
total_messages = sum(len(seg.get("messages") or []) for seg in segments)
|
||||
base["segments"] = segments
|
||||
base["lineage_session_ids"] = [seg["id"] for seg in segments]
|
||||
base["message_count"] = total_messages
|
||||
base["messages"] = [msg for seg in segments for msg in (seg.get("messages") or [])]
|
||||
return base
|
||||
messages = [msg for seg in segments for msg in (seg.get("messages") or [])]
|
||||
return {
|
||||
**segments[-1],
|
||||
"segments": segments,
|
||||
"lineage_session_ids": [seg["id"] for seg in segments],
|
||||
"message_count": len(messages),
|
||||
"messages": messages,
|
||||
}
|
||||
|
||||
def export_all(self, source: str = None) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Export all sessions (with messages) as a list of dicts.
|
||||
Suitable for writing to a JSONL file for backup/analysis.
|
||||
"""
|
||||
sessions = self.search_sessions(source=source, limit=100000)
|
||||
results = []
|
||||
for session in sessions:
|
||||
messages = self.get_messages(session["id"])
|
||||
results.append({**session, "messages": messages})
|
||||
return results
|
||||
"""Export all sessions (with messages) as dicts, e.g. for JSONL backup."""
|
||||
return [
|
||||
{**session, "messages": self.get_messages(session["id"])}
|
||||
for session in self.search_sessions(source=source, limit=100000)
|
||||
]
|
||||
|
||||
def adopt_session_lineage_from(
|
||||
self,
|
||||
@@ -313,52 +228,36 @@ class SessionPortabilityMixin:
|
||||
*,
|
||||
retire_donor: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
"""Adopt *session_id*'s full compression lineage from *donor_db* into
|
||||
this store.
|
||||
"""Adopt *session_id*'s full compression lineage from *donor_db*.
|
||||
|
||||
The stranded-bot-session heal (#93091 follow-up to #93296): before the
|
||||
desktop routed session RPCs by their target session, a profile bot's
|
||||
turns executed on whichever backend held window focus — usually the
|
||||
default one — so the bot's canonical session rows and messages
|
||||
accumulated in the DEFAULT profile's state.db. Once routing was fixed,
|
||||
the profile backend correctly received the RPCs but had no such
|
||||
session, so the same chat 4001'd for the opposite reason. This method
|
||||
moves the conversation to where routing now looks for it.
|
||||
Stranded-bot-session heal: before the desktop routed session RPCs by
|
||||
target session, a profile bot's rows accumulated in the DEFAULT
|
||||
profile's state.db; this moves the conversation to where routing now
|
||||
looks. Pure composition: ``donor_db.export_session_lineage()`` ->
|
||||
``self.import_sessions()`` — routing/handoff/activity fields reset,
|
||||
already-present ids skipped (idempotent re-adoption).
|
||||
|
||||
Composition of existing primitives (no new import/export machinery):
|
||||
``donor_db.export_session_lineage()`` -> ``self.import_sessions()``.
|
||||
Import semantics apply unchanged: gateway routing, handoff, and live
|
||||
activity fields are reset; already-present ids are skipped
|
||||
(idempotent re-adoption after a partial run).
|
||||
With ``retire_donor`` and a complete adoption, donor rows are ARCHIVED
|
||||
(never deleted) with ``end_reason='adopted_by_profile'``. That
|
||||
end_reason is deliberately NOT in the recoverable set
|
||||
(agent_close/ws_orphan_reap): resurrection must not undo an adoption.
|
||||
|
||||
When ``retire_donor`` is True and at least one segment was imported
|
||||
(or every segment already exists here), the donor rows are ARCHIVED —
|
||||
never deleted — with ``end_reason='adopted_by_profile'`` so the
|
||||
default profile's list stops advertising a conversation that now
|
||||
lives elsewhere, while the bytes stay recoverable. The archive is
|
||||
deliberately NOT in the recoverable set (agent_close/ws_orphan_reap):
|
||||
canonical-lookup resurrection must not undo an adoption.
|
||||
|
||||
Returns the ``import_sessions`` result dict, plus ``adopted`` (bool)
|
||||
and ``donor_retired`` (bool — True only when EVERY segment's
|
||||
retirement actually applied).
|
||||
Returns the ``import_sessions`` dict plus ``adopted`` and
|
||||
``donor_retired`` (True only when EVERY segment's retirement applied).
|
||||
"""
|
||||
payload = donor_db.export_session_lineage(session_id)
|
||||
if not payload:
|
||||
return {
|
||||
"ok": False,
|
||||
"adopted": False,
|
||||
"donor_retired": False,
|
||||
"ok": False, "adopted": False, "donor_retired": False,
|
||||
"error": f"session {session_id!r} not found in donor store",
|
||||
}
|
||||
|
||||
segments = payload.get("segments") or [payload]
|
||||
|
||||
# Divergence guard: a segment we are about to SKIP (already present
|
||||
# here) may have kept accumulating messages in the donor store after
|
||||
# a partial earlier adoption. Retiring it would strand those newer
|
||||
# messages behind a non-recoverable archive. Compare counts up front
|
||||
# and refuse to retire (still adopt/import) when the donor is ahead.
|
||||
# Divergence guard: a segment we will SKIP (already here) may have kept
|
||||
# growing in the donor after a partial adoption; retiring it would strand
|
||||
# those messages behind a non-recoverable archive. Still import, but
|
||||
# refuse to retire when the donor is ahead.
|
||||
donor_ahead = False
|
||||
for seg in segments:
|
||||
seg_id = seg.get("id")
|
||||
@@ -394,15 +293,11 @@ class SessionPortabilityMixin:
|
||||
if not seg_id:
|
||||
continue
|
||||
try:
|
||||
# TOCTOU close-out: the guard above compared EXPORT-TIME
|
||||
# counts, but another backend can append donor messages
|
||||
# between export and this loop. Re-read both stores right
|
||||
# before stamping; a donor-ahead signal here skips the
|
||||
# stamp so growth never lands behind a non-recoverable
|
||||
# archive. (Count comparison cannot see equal-count
|
||||
# CONTENT divergence — e.g. a donor rewind+rewrite; that
|
||||
# residual case is accepted: bytes stay in the donor
|
||||
# store either way, only reachability differs.)
|
||||
# TOCTOU close-out: the guard above used EXPORT-TIME counts;
|
||||
# re-read both stores right before stamping so donor growth
|
||||
# never lands behind a non-recoverable archive. (Equal-count
|
||||
# CONTENT divergence is accepted: bytes stay in the donor
|
||||
# either way, only reachability differs.)
|
||||
donor_now = len(donor_db.get_messages(seg_id))
|
||||
local_now = len(self.get_messages(seg_id))
|
||||
if donor_now > local_now:
|
||||
@@ -414,17 +309,15 @@ class SessionPortabilityMixin:
|
||||
seg_id, donor_now, local_now,
|
||||
)
|
||||
continue
|
||||
# First end_reason wins in end_session(); reopen first so
|
||||
# the adoption boundary is stamped even on ended segments
|
||||
# (e.g. 'compression' parents).
|
||||
# First end_reason wins in end_session(); reopen so the
|
||||
# adoption boundary is stamped even on ended segments.
|
||||
donor_db.reopen_session(seg_id)
|
||||
donor_db.end_session(seg_id, "adopted_by_profile")
|
||||
donor_db.set_session_archived(seg_id, True)
|
||||
except Exception:
|
||||
# Best-effort by design: a retirement failure must not
|
||||
# fail the adoption (the profile copy is already whole;
|
||||
# a later resume retries retirement idempotently). But
|
||||
# never claim success we didn't have.
|
||||
# Best-effort: a retirement failure must not fail the adoption
|
||||
# (a later resume retries idempotently) — but never claim
|
||||
# success we didn't have.
|
||||
retire_ok = False
|
||||
logger.warning(
|
||||
"failed to retire donor segment %s after adoption",
|
||||
@@ -507,21 +400,16 @@ class SessionPortabilityMixin:
|
||||
def import_sessions(self, sessions: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""Import sessions exported by :meth:`export_session` or ``export_all``.
|
||||
|
||||
Existing session IDs are skipped. Imported child sessions keep their
|
||||
parent only when that parent already exists or is included in the same
|
||||
import payload; otherwise the child is detached so partial imports don't
|
||||
fail foreign-key validation. Gateway routing, handoff, rewind, and other
|
||||
live runtime state are intentionally reset: this restores conversation
|
||||
history, not ownership of a live channel or process.
|
||||
Existing ids are skipped. A child keeps its parent only when the parent
|
||||
exists or is in the same payload; otherwise it is detached so partial
|
||||
imports pass FK validation. Gateway routing, handoff, rewind and other
|
||||
live runtime state are reset: this restores history, not ownership of
|
||||
a live channel or process.
|
||||
|
||||
Activity contract (#76354 review S4): export INCLUDES the live
|
||||
activity fields (``last_activity_at`` / ``last_activity_description``
|
||||
/ ``last_activity_provenance``) because they are part of the durable
|
||||
row, but import deliberately RESETS them to NULL. Resurrecting a
|
||||
stale "working ..." label on a machine where no agent is running
|
||||
would fabricate activity the watchdog and session listings act on.
|
||||
This asymmetry is intentional and covered by regression
|
||||
(tests/gateway/test_watchdog_review_76354.py::test_s4_export_includes_activity_import_resets_it).
|
||||
Activity contract: export INCLUDES ``last_activity_*`` (durable row
|
||||
fields) but import RESETS them to NULL — resurrecting a stale
|
||||
"working ..." label would fabricate activity the watchdog and listings
|
||||
act on. Intentional asymmetry, pinned by regression test.
|
||||
"""
|
||||
if not isinstance(sessions, list):
|
||||
raise ValueError("sessions must be a list")
|
||||
@@ -536,32 +424,14 @@ class SessionPortabilityMixin:
|
||||
total_messages = 0
|
||||
total_bytes = 0
|
||||
session_text_fields = (
|
||||
"source",
|
||||
"user_id",
|
||||
"model",
|
||||
"system_prompt",
|
||||
"end_reason",
|
||||
"cwd",
|
||||
"git_branch",
|
||||
"git_repo_root",
|
||||
"billing_provider",
|
||||
"billing_base_url",
|
||||
"billing_mode",
|
||||
"cost_status",
|
||||
"cost_source",
|
||||
"pricing_version",
|
||||
"title",
|
||||
"source", "user_id", "model", "system_prompt", "end_reason", "cwd",
|
||||
"git_branch", "git_repo_root", "billing_provider", "billing_base_url",
|
||||
"billing_mode", "cost_status", "cost_source", "pricing_version", "title",
|
||||
)
|
||||
# ``role`` is validated separately below (non-empty string).
|
||||
message_text_fields = (
|
||||
"role",
|
||||
"tool_call_id",
|
||||
"tool_name",
|
||||
"effect_disposition",
|
||||
"finish_reason",
|
||||
"reasoning",
|
||||
"reasoning_content",
|
||||
"platform_message_id",
|
||||
"message_id",
|
||||
"tool_call_id", "tool_name", "effect_disposition", "finish_reason",
|
||||
"reasoning", "reasoning_content", "platform_message_id", "message_id",
|
||||
)
|
||||
|
||||
for index, raw in enumerate(sessions):
|
||||
@@ -580,43 +450,23 @@ class SessionPortabilityMixin:
|
||||
errors.append(self._import_error(index, session_id, "messages must be a list"))
|
||||
continue
|
||||
if len(messages) > self._IMPORT_MAX_MESSAGES_PER_SESSION:
|
||||
errors.append(
|
||||
self._import_error(
|
||||
index,
|
||||
session_id,
|
||||
"messages exceeds the per-session import limit",
|
||||
)
|
||||
)
|
||||
errors.append(self._import_error(index, session_id, "messages exceeds the per-session import limit"))
|
||||
continue
|
||||
if any(not isinstance(msg, dict) for msg in messages):
|
||||
errors.append(
|
||||
self._import_error(
|
||||
index,
|
||||
session_id,
|
||||
"messages must contain only objects",
|
||||
)
|
||||
)
|
||||
errors.append(self._import_error(index, session_id, "messages must contain only objects"))
|
||||
continue
|
||||
|
||||
try:
|
||||
session_bytes = len(
|
||||
json.dumps(raw, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
)
|
||||
session_bytes = len(json.dumps(raw, ensure_ascii=False, separators=(",", ":")).encode("utf-8"))
|
||||
except (TypeError, ValueError):
|
||||
errors.append(
|
||||
self._import_error(index, session_id, "session must be JSON serializable")
|
||||
)
|
||||
errors.append(self._import_error(index, session_id, "session must be JSON serializable"))
|
||||
continue
|
||||
if session_bytes > self._IMPORT_MAX_SESSION_BYTES:
|
||||
errors.append(
|
||||
self._import_error(index, session_id, "session exceeds the import size limit")
|
||||
)
|
||||
errors.append(self._import_error(index, session_id, "session exceeds the import size limit"))
|
||||
continue
|
||||
total_bytes += session_bytes
|
||||
if total_bytes > self._IMPORT_MAX_TOTAL_BYTES:
|
||||
errors.append(
|
||||
self._import_error(index, session_id, "import exceeds the total size limit")
|
||||
)
|
||||
errors.append(self._import_error(index, session_id, "import exceeds the total size limit"))
|
||||
continue
|
||||
|
||||
try:
|
||||
@@ -640,8 +490,6 @@ class SessionPortabilityMixin:
|
||||
if not isinstance(role, str) or not role:
|
||||
raise ValueError(f"messages[{message_index}].role must be a non-empty string")
|
||||
for field in message_text_fields:
|
||||
if field == "role":
|
||||
continue
|
||||
clean_message[field] = self._import_text_or_none(
|
||||
clean_message.get(field), field
|
||||
)
|
||||
@@ -655,13 +503,7 @@ class SessionPortabilityMixin:
|
||||
|
||||
total_messages += len(clean_messages)
|
||||
if total_messages > self._IMPORT_MAX_TOTAL_MESSAGES:
|
||||
errors.append(
|
||||
self._import_error(
|
||||
index,
|
||||
session_id,
|
||||
"messages exceeds the total import limit",
|
||||
)
|
||||
)
|
||||
errors.append(self._import_error(index, session_id, "messages exceeds the total import limit"))
|
||||
continue
|
||||
seen_ids.add(session_id)
|
||||
normalized.append(
|
||||
@@ -669,13 +511,7 @@ class SessionPortabilityMixin:
|
||||
)
|
||||
|
||||
if errors:
|
||||
return {
|
||||
"ok": False,
|
||||
"imported": 0,
|
||||
"skipped": 0,
|
||||
"detached": 0,
|
||||
"errors": errors,
|
||||
}
|
||||
return {"ok": False, "imported": 0, "skipped": 0, "detached": 0, "errors": errors}
|
||||
|
||||
def _do(conn):
|
||||
imported_ids: List[str] = []
|
||||
@@ -738,27 +574,17 @@ class SessionPortabilityMixin:
|
||||
"end_reason": raw.get("end_reason"),
|
||||
"input_tokens": self._int_or_default(raw.get("input_tokens")),
|
||||
"output_tokens": self._int_or_default(raw.get("output_tokens")),
|
||||
"cache_read_tokens": self._int_or_default(
|
||||
raw.get("cache_read_tokens")
|
||||
),
|
||||
"cache_write_tokens": self._int_or_default(
|
||||
raw.get("cache_write_tokens")
|
||||
),
|
||||
"reasoning_tokens": self._int_or_default(
|
||||
raw.get("reasoning_tokens")
|
||||
),
|
||||
"cache_read_tokens": self._int_or_default(raw.get("cache_read_tokens")),
|
||||
"cache_write_tokens": self._int_or_default(raw.get("cache_write_tokens")),
|
||||
"reasoning_tokens": self._int_or_default(raw.get("reasoning_tokens")),
|
||||
"cwd": raw.get("cwd"),
|
||||
"git_branch": raw.get("git_branch"),
|
||||
"git_repo_root": raw.get("git_repo_root"),
|
||||
"billing_provider": raw.get("billing_provider"),
|
||||
"billing_base_url": raw.get("billing_base_url"),
|
||||
"billing_mode": raw.get("billing_mode"),
|
||||
"estimated_cost_usd": self._float_or_none(
|
||||
raw.get("estimated_cost_usd")
|
||||
),
|
||||
"actual_cost_usd": self._float_or_none(
|
||||
raw.get("actual_cost_usd")
|
||||
),
|
||||
"estimated_cost_usd": self._float_or_none(raw.get("estimated_cost_usd")),
|
||||
"actual_cost_usd": self._float_or_none(raw.get("actual_cost_usd")),
|
||||
"cost_status": raw.get("cost_status"),
|
||||
"cost_source": raw.get("cost_source"),
|
||||
"pricing_version": raw.get("pricing_version"),
|
||||
@@ -771,18 +597,12 @@ class SessionPortabilityMixin:
|
||||
sanitized_messages: List[Dict[str, Any]] = []
|
||||
for msg in messages:
|
||||
clean = dict(msg)
|
||||
for key in (
|
||||
"reasoning_details",
|
||||
"codex_reasoning_items",
|
||||
"codex_message_items",
|
||||
):
|
||||
for key in ("reasoning_details", "codex_reasoning_items", "codex_message_items"):
|
||||
clean[key] = self._reasoning_json_value(clean.get(key))
|
||||
sanitized_messages.append(clean)
|
||||
|
||||
total_messages, total_tool_calls = self._insert_message_rows(
|
||||
conn,
|
||||
session_id,
|
||||
sanitized_messages,
|
||||
conn, session_id, sanitized_messages
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?",
|
||||
@@ -826,9 +646,8 @@ class SessionPortabilityMixin:
|
||||
(parent_id, session_id),
|
||||
)
|
||||
else:
|
||||
# Drop only the closing edge. Later entries can still attach
|
||||
# to this now-root session, preserving the acyclic portion
|
||||
# of a malformed imported lineage.
|
||||
# Drop only the closing edge; later entries can still attach
|
||||
# to this now-root session.
|
||||
parent_by_child.pop(session_id, None)
|
||||
detached += 1
|
||||
|
||||
|
||||
+64
-122
@@ -1,40 +1,31 @@
|
||||
"""Process-wide shared SessionDB registry (#90837).
|
||||
"""Process-wide shared SessionDB registry.
|
||||
|
||||
A gateway process opens state.db from many call sites — the runner's
|
||||
``AsyncSessionDB``, the ``SessionStore`` per-path cache, per-agent lazy
|
||||
recall (``run_agent._get_session_db_for_recall``), per-job cron opens,
|
||||
and per-message opens in mirror / channel_directory / slash_commands /
|
||||
shutdown_flush / session_search / react_to_message. Each bare
|
||||
``SessionDB()`` mints its own writer connection, ``self._lock``,
|
||||
close-time WAL checkpoint, and async token-writer thread. With N
|
||||
independent writer connections on one WAL file, mutual exclusion relies
|
||||
only on SQLite's WAL write lock plus each instance's busy_timeout retry
|
||||
ladder — and one connection's close-time checkpoint can race another's
|
||||
growth, producing the lost/reordered-page-write signature reported
|
||||
across 11+ incidents (#90837).
|
||||
|
||||
This module owns that boundary: one shared ``SessionDB`` per resolved
|
||||
path per process, refcounted, with generation-aware retirement when the
|
||||
underlying file is replaced (snapshot restore, recovery swap).
|
||||
A gateway process opens state.db from many call sites (runner, SessionStore,
|
||||
per-agent recall, cron, per-message helpers). Each bare ``SessionDB()`` mints
|
||||
its own writer connection, lock, close-time WAL checkpoint and token-writer
|
||||
thread; N independent writers on one WAL file rely only on SQLite's write lock
|
||||
plus busy_timeout, and one connection's close-time checkpoint can race another's
|
||||
growth (lost/reordered-page-write corruption). This module owns that boundary:
|
||||
one shared ``SessionDB`` per resolved path per process, refcounted, with
|
||||
generation-aware retirement when the file is replaced (snapshot restore,
|
||||
recovery swap).
|
||||
|
||||
Lifecycle rules:
|
||||
|
||||
- ``acquire(path)`` returns the current generation for *path*,
|
||||
incrementing its refcount. Same path ⇒ same instance ⇒ same writer
|
||||
connection.
|
||||
- ``close()`` on a shared instance is a NO-OP. The registry — not any
|
||||
individual caller — owns the connection lifecycle, so one caller's
|
||||
``close()`` can never tear down a writer other callers still hold.
|
||||
- ``release(db)`` decrements the generation *db was acquired from*
|
||||
(object-keyed, not pathname-keyed, so an inode replacement cannot
|
||||
strand a still-owned generation). The final release of a retired
|
||||
generation tears it down.
|
||||
- On inode change, the old generation is RETIRED — never lent again —
|
||||
but stays alive until its existing holders release. If a replacement
|
||||
open fails, the registry is left WITHOUT a path entry (never a closed
|
||||
stale object), so the next acquire retries fresh.
|
||||
- All teardown happens OUTSIDE the registry lock: a final release's
|
||||
WAL checkpoint must never stall acquisition for every state.db.
|
||||
- ``acquire(path)`` returns the current generation for *path* and bumps its
|
||||
refcount. Same path ⇒ same instance ⇒ same writer connection.
|
||||
- ``close()`` on a shared instance is a NO-OP: the registry, not any caller,
|
||||
owns the connection lifecycle, so one caller can never tear down a writer
|
||||
others still hold.
|
||||
- ``release(db)`` decrements the generation *db was acquired from* (object-
|
||||
keyed, not pathname-keyed, so an inode replacement cannot strand a
|
||||
still-owned generation). The final release of a retired generation tears
|
||||
it down.
|
||||
- On inode change the old generation is RETIRED (never lent again) but stays
|
||||
alive until its holders release. If the replacement open fails the registry
|
||||
keeps NO path entry (never a closed stale object) so the next acquire retries.
|
||||
- All teardown happens OUTSIDE the registry lock: a final release's WAL
|
||||
checkpoint must never stall acquisition for every state.db.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -44,34 +35,14 @@ import threading
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
|
||||
|
||||
from hermes_state_common import stat_db_file_identity as _stat_db_file_identity
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover - import cycle guard, typed only
|
||||
from hermes_state import SessionDB
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _stat_db_file_identity(path: Path) -> Optional[Tuple[int, int]]:
|
||||
"""Return ``(st_dev, st_ino)`` for *path*, or None when unavailable.
|
||||
|
||||
Mirrors the hermes_state helper of the same name; kept local so this
|
||||
module has no import-time dependency on hermes_state (which imports
|
||||
this module — the cycle is resolved by deferring SessionDB lookup
|
||||
to call time).
|
||||
"""
|
||||
import os
|
||||
|
||||
try:
|
||||
st = os.stat(path)
|
||||
except OSError:
|
||||
return None
|
||||
# Windows volumes (and some network FS) report st_ino=0; a (0, 0)
|
||||
# identity would false-positive every check. Skip the inode half of
|
||||
# the guard there.
|
||||
if not st.st_dev or not st.st_ino:
|
||||
return None
|
||||
return (st.st_dev, st.st_ino)
|
||||
|
||||
|
||||
class _Generation:
|
||||
"""One shared SessionDB generation: instance, refcount, file identity."""
|
||||
|
||||
@@ -85,16 +56,13 @@ class _Generation:
|
||||
|
||||
|
||||
_lock = threading.Lock()
|
||||
# path → live generation (never retired). A retired generation leaves
|
||||
# this table immediately on retirement and lives on in _retired until
|
||||
# its last holder releases.
|
||||
# path → live generation. Retired generations move to _retired (keyed by
|
||||
# id(db)) until their last holder releases.
|
||||
_generations: Dict[Path, _Generation] = {}
|
||||
# Object-keyed retired generations still draining holders.
|
||||
_retired: Dict[int, _Generation] = {} # id(db) → generation
|
||||
# Paths whose next generation is currently being constructed. Construction
|
||||
# stays outside _lock because schema reconciliation can take seconds, but peers
|
||||
# for the SAME file must wait: otherwise every cold caller opens a writable
|
||||
# SQLite connection before the registry chooses one winner.
|
||||
_retired: Dict[int, _Generation] = {}
|
||||
# Paths whose next generation is being constructed. Construction runs outside
|
||||
# _lock (schema reconciliation can take seconds), but peers for the SAME file
|
||||
# must wait or every cold caller opens its own writer before a winner is chosen.
|
||||
_opening: Dict[Path, threading.Event] = {}
|
||||
|
||||
|
||||
@@ -120,20 +88,13 @@ def _teardown(db: "SessionDB") -> None:
|
||||
def acquire(db_path: Optional[Path] = None) -> "SessionDB":
|
||||
"""Return the shared SessionDB for *db_path*, incrementing its refcount.
|
||||
|
||||
The same resolved path always returns the same ``SessionDB`` instance
|
||||
within one process, so all long-lived in-process callers share one
|
||||
writer connection, one ``self._lock``, and one token-writer thread.
|
||||
If the file was replaced (different inode) since the generation opened —
|
||||
``hermes sessions recover``, snapshot restore — that generation is RETIRED
|
||||
but stays alive for its holders, and a fresh one is opened in its place.
|
||||
|
||||
If the underlying file was replaced (different inode) since the
|
||||
shared generation was opened — e.g. by ``hermes sessions recover`` or
|
||||
a snapshot restore — the current generation is RETIRED (never lent
|
||||
again) but stays alive for its existing holders, and a fresh
|
||||
generation is opened in its place.
|
||||
|
||||
Raises whatever ``SessionDB.__init__`` raises (malformed, locked,
|
||||
etc.). On a replacement-open failure the registry holds NO entry for
|
||||
the path, so the next acquire retries fresh rather than handing out
|
||||
a closed stale object.
|
||||
Raises whatever ``SessionDB.__init__`` raises. On a replacement-open
|
||||
failure the registry holds NO entry for the path, so the next acquire
|
||||
retries fresh instead of receiving a closed stale object.
|
||||
"""
|
||||
from hermes_state import _default_db_path
|
||||
|
||||
@@ -153,9 +114,8 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB":
|
||||
and generation.identity is not None
|
||||
and current != generation.identity
|
||||
):
|
||||
# File replaced: retire the live generation (its
|
||||
# holders keep it until they release) and elect one
|
||||
# caller to construct the replacement below.
|
||||
# File replaced: retire, then elect one caller to open
|
||||
# the replacement below.
|
||||
_retire_generation_locked(path, generation)
|
||||
else:
|
||||
generation.refcount += 1
|
||||
@@ -167,13 +127,12 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB":
|
||||
_opening[path] = opening
|
||||
break
|
||||
|
||||
# Another caller is constructing this path. Do not hold the global
|
||||
# registry lock while waiting: unrelated databases continue opening.
|
||||
# A failed opener signals too, so one waiter can retry as the successor.
|
||||
# Another caller is constructing this path; wait without holding the
|
||||
# global lock. A failed opener signals too, so a waiter can retry.
|
||||
opening.wait()
|
||||
|
||||
# Open a fresh generation OUTSIDE the lock. The per-path opening marker
|
||||
# prevents redundant writer connections without serialising other files.
|
||||
# Open OUTSIDE the lock; the per-path marker prevents redundant writers
|
||||
# without serialising other files.
|
||||
try:
|
||||
db = _open_session_db(path)
|
||||
db._shared_registry_owned = True
|
||||
@@ -188,8 +147,7 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB":
|
||||
with _lock:
|
||||
existing = _generations.get(path)
|
||||
if existing is not None:
|
||||
# Defensive: a generation may have been installed by explicit
|
||||
# registry manipulation while this open was in flight.
|
||||
# Defensive: installed by explicit registry manipulation mid-open.
|
||||
existing.refcount += 1
|
||||
winner = existing.db
|
||||
else:
|
||||
@@ -206,9 +164,8 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB":
|
||||
def _retire_generation_locked(path: Path, generation: _Generation) -> None:
|
||||
"""Retire *generation* so it is never lent again (caller holds _lock).
|
||||
|
||||
The instance stays alive — its holders still own references — and is
|
||||
tracked in ``_retired`` keyed by ``id(db)`` so their releases find
|
||||
the right generation even after the path maps to a new one.
|
||||
It stays alive for its holders, tracked in ``_retired`` by ``id(db)`` so
|
||||
their releases find it even after the path maps to a new generation.
|
||||
"""
|
||||
generation.retired = True
|
||||
if _generations.get(path) is generation:
|
||||
@@ -219,15 +176,11 @@ def _retire_generation_locked(path: Path, generation: _Generation) -> None:
|
||||
def release(db: "SessionDB") -> bool:
|
||||
"""Decrement the refcount of a shared SessionDB.
|
||||
|
||||
Returns ``True`` if *db* was a shared instance and its refcount was
|
||||
decremented; ``False`` if *db* is not registry-managed (caller owns
|
||||
its own close()). The final release of a generation tears it down —
|
||||
OUTSIDE the registry lock, so a close-time WAL checkpoint never
|
||||
stalls acquisition for every state.db in the process.
|
||||
|
||||
Object-keyed lookup means an inode replacement cannot strand a
|
||||
still-owned generation: holders of the old generation release into
|
||||
the retired record, not into whatever the path currently names.
|
||||
Returns ``True`` if *db* was shared; ``False`` if it is not registry-managed
|
||||
(caller owns close()). The final release tears the generation down OUTSIDE
|
||||
the registry lock so a close-time WAL checkpoint never stalls acquisition.
|
||||
Lookup is object-keyed, so holders of an old generation release into its
|
||||
retired record, not into whatever the path currently names.
|
||||
"""
|
||||
if db is None:
|
||||
return False
|
||||
@@ -244,8 +197,7 @@ def release(db: "SessionDB") -> bool:
|
||||
return False
|
||||
generation = _generations.get(path)
|
||||
if generation is None or generation.db is not db:
|
||||
# Not a shared instance (caller used SessionDB()
|
||||
# directly) — nothing to do; the caller owns close().
|
||||
# Not shared (bare SessionDB()); the caller owns close().
|
||||
return False
|
||||
generation.refcount -= 1
|
||||
needs_teardown = generation.refcount <= 0
|
||||
@@ -259,21 +211,17 @@ def release(db: "SessionDB") -> bool:
|
||||
_generations.pop(Path(path), None)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
# Teardown OUTSIDE the lock: it stops the token writer, checkpoints
|
||||
# the WAL, and drains the read pool — none of which may hold up
|
||||
# acquisition for every other state.db in the process.
|
||||
# Teardown OUTSIDE the lock: stopping the token writer, WAL checkpoint and
|
||||
# read-pool drain must not block acquisition for every other state.db.
|
||||
if needs_teardown:
|
||||
_teardown(db)
|
||||
return True
|
||||
|
||||
|
||||
def close_all() -> int:
|
||||
"""Close every shared SessionDB in this process, regardless of refcount.
|
||||
"""Close every shared SessionDB regardless of refcount; returns the count.
|
||||
|
||||
Called at gateway shutdown (after all agents and cron jobs have
|
||||
finished) to release every WAL write lock and drain every
|
||||
token-writer thread cleanly. Returns the number of instances
|
||||
closed. Idempotent.
|
||||
For gateway shutdown, after all agents and cron jobs finished. Idempotent.
|
||||
"""
|
||||
closed = 0
|
||||
with _lock:
|
||||
@@ -282,7 +230,6 @@ def close_all() -> int:
|
||||
_retired.clear()
|
||||
for generation in generations:
|
||||
generation.retired = True
|
||||
# Teardown outside the lock, one generation at a time.
|
||||
for generation in generations:
|
||||
_teardown(generation.db)
|
||||
closed += 1
|
||||
@@ -290,12 +237,11 @@ def close_all() -> int:
|
||||
|
||||
|
||||
def live_shared_session_dbs() -> List["SessionDB"]:
|
||||
"""Snapshot of every live (non-retired) shared SessionDB in this process.
|
||||
"""Snapshot of every live (non-retired) shared SessionDB.
|
||||
|
||||
For periodic in-process maintenance (the gateway housekeeping tick's
|
||||
deferred-FTS retry). Refcounts are NOT touched: the caller only invokes
|
||||
a method on an instance that some holder already keeps alive; a
|
||||
concurrent final release closes it and the callee sees ``_conn is None``.
|
||||
For in-process maintenance (housekeeping deferred-FTS retry). Refcounts
|
||||
are NOT touched: a concurrent final release may close an instance, in
|
||||
which case the callee sees ``_conn is None``.
|
||||
"""
|
||||
with _lock:
|
||||
return [g.db for g in _generations.values() if not g.retired]
|
||||
@@ -314,9 +260,7 @@ def stats() -> Dict[str, int]:
|
||||
}
|
||||
|
||||
|
||||
# ── Backwards-compatible aliases (hermes_state re-exports) ──
|
||||
# Kept so call sites and tests can import either from hermes_state
|
||||
# (the historical path) or from this module directly.
|
||||
# ── Backwards-compatible aliases (hermes_state re-exports them) ──
|
||||
|
||||
def get_shared_session_db(db_path: Optional[Path] = None) -> "SessionDB":
|
||||
return acquire(db_path)
|
||||
@@ -333,10 +277,8 @@ def close_shared_session_dbs() -> int:
|
||||
def release_or_close(db: "SessionDB") -> None:
|
||||
"""Release a shared instance, or close it when it is not registry-managed.
|
||||
|
||||
The one-line cleanup for call sites that previously did a plain
|
||||
``db.close()``: shared instances return their refcount to the
|
||||
registry (the registry owns the lifecycle), anything else — read-only
|
||||
opens, CLI one-shots, test fakes — falls back to a direct close.
|
||||
Drop-in for a plain ``db.close()``: read-only opens, CLI one-shots and
|
||||
test fakes fall back to a direct close.
|
||||
"""
|
||||
if not release(db):
|
||||
try:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+291
-595
File diff suppressed because it is too large
Load Diff
+637
-1285
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,529 @@
|
||||
"""Telegram DM topic-mode mixin for :class:`hermes_state.SessionDB`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from hermes_state_common import (
|
||||
_PREVIEW_ELIGIBLE_SQL,
|
||||
_PREVIEW_RAW_SELECT,
|
||||
_shape_preview,
|
||||
_sql_session_last_active,
|
||||
)
|
||||
|
||||
# caplog tests pin the "hermes_state" logger name.
|
||||
logger = logging.getLogger("hermes_state")
|
||||
|
||||
|
||||
def _normalize_telegram_topic_profile_name(profile_name: Optional[str] = None) -> str:
|
||||
"""Empty/missing → ``"default"`` (single namespace for non-multiplexed
|
||||
gateways). Multiplexed callers must pass the *routed* profile
|
||||
(``source.profile``), never the process-global active profile."""
|
||||
name = str(profile_name or "").strip()
|
||||
return name if name else "default"
|
||||
|
||||
|
||||
class SessionTelegramTopicsMixin:
|
||||
"""Telegram DM topic-mode tables, bindings and lookups."""
|
||||
|
||||
def apply_telegram_topic_migration(self) -> None:
|
||||
"""Create Telegram DM topic-mode tables on explicit /topic opt-in.
|
||||
|
||||
Deliberately NOT part of startup reconciliation: operators can upgrade
|
||||
and keep the old bot behavior until a user runs /topic.
|
||||
|
||||
Schema versions: v1 initial; v2 session_id FK ON DELETE CASCADE (pruning
|
||||
clears bindings); v3 ``profile_name`` on both tables so multiplexed
|
||||
gateways sharing one state.db isolate topic state per profile.
|
||||
"""
|
||||
# (table, column list, DDL body). profile_name leads the PK: a private
|
||||
# chat_id is the user id, identical across bots sharing one state.db.
|
||||
tables = (
|
||||
(
|
||||
"telegram_dm_topic_mode",
|
||||
"profile_name, chat_id, user_id, enabled, activated_at, updated_at, "
|
||||
"has_topics_enabled, allows_users_to_create_topics, "
|
||||
"capability_checked_at, intro_message_id, pinned_message_id",
|
||||
"""
|
||||
profile_name TEXT NOT NULL DEFAULT 'default',
|
||||
chat_id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
activated_at REAL NOT NULL,
|
||||
updated_at REAL NOT NULL,
|
||||
has_topics_enabled INTEGER,
|
||||
allows_users_to_create_topics INTEGER,
|
||||
capability_checked_at REAL,
|
||||
intro_message_id TEXT,
|
||||
pinned_message_id TEXT,
|
||||
PRIMARY KEY (profile_name, chat_id)
|
||||
""",
|
||||
),
|
||||
(
|
||||
"telegram_dm_topic_bindings",
|
||||
"profile_name, chat_id, thread_id, user_id, session_key, "
|
||||
"session_id, managed_mode, linked_at, updated_at",
|
||||
"""
|
||||
profile_name TEXT NOT NULL DEFAULT 'default',
|
||||
chat_id TEXT NOT NULL,
|
||||
thread_id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
session_key TEXT NOT NULL,
|
||||
session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
|
||||
managed_mode TEXT NOT NULL DEFAULT 'auto',
|
||||
linked_at REAL NOT NULL,
|
||||
updated_at REAL NOT NULL,
|
||||
PRIMARY KEY (profile_name, chat_id, thread_id)
|
||||
""",
|
||||
),
|
||||
)
|
||||
|
||||
def _do(conn):
|
||||
for table, columns, ddl in tables:
|
||||
conn.execute(f"CREATE TABLE IF NOT EXISTS {table} ({ddl})")
|
||||
have = {row[1] for row in conn.execute(f"PRAGMA table_info('{table}')")}
|
||||
if "profile_name" in have:
|
||||
continue
|
||||
# v1/v2 → v3. SQLite can't ALTER a PK or FK, so rebuild (this
|
||||
# also supplies v2's ON DELETE CASCADE). Legacy rows land in
|
||||
# "default" only, never replicated across profiles.
|
||||
legacy_columns = columns.replace("profile_name, ", "", 1)
|
||||
conn.executescript(
|
||||
f"""
|
||||
CREATE TABLE {table}_new ({ddl});
|
||||
INSERT INTO {table}_new ({columns})
|
||||
SELECT 'default', {legacy_columns} FROM {table};
|
||||
DROP TABLE {table};
|
||||
ALTER TABLE {table}_new RENAME TO {table};
|
||||
"""
|
||||
)
|
||||
|
||||
# Indexes after any rebuild: the user index needs profile_name.
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_session
|
||||
ON telegram_dm_topic_bindings(session_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_user
|
||||
ON telegram_dm_topic_bindings(profile_name, user_id, chat_id);
|
||||
"""
|
||||
)
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO state_meta (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
("telegram_dm_topic_schema_version", "3"),
|
||||
)
|
||||
self._execute_write(_do)
|
||||
|
||||
def enable_telegram_topic_mode(
|
||||
self,
|
||||
*,
|
||||
chat_id: str,
|
||||
user_id: str,
|
||||
profile_name: str = "default",
|
||||
has_topics_enabled: Optional[bool] = None,
|
||||
allows_users_to_create_topics: Optional[bool] = None,
|
||||
) -> None:
|
||||
"""Enable Telegram DM topic mode for one private chat/user.
|
||||
|
||||
Owns the explicit topic migration; SessionDB startup must not create
|
||||
these tables. Multiplexed callers pass the routed ``source.profile``,
|
||||
not the process-global active profile.
|
||||
"""
|
||||
self.apply_telegram_topic_migration()
|
||||
now = time.time()
|
||||
profile_name = _normalize_telegram_topic_profile_name(profile_name)
|
||||
|
||||
def _to_int(value: Optional[bool]) -> Optional[int]:
|
||||
if value is None:
|
||||
return None
|
||||
return 1 if value else 0
|
||||
|
||||
self._write_sql(
|
||||
"""
|
||||
INSERT INTO telegram_dm_topic_mode (
|
||||
profile_name, chat_id, user_id, enabled, activated_at, updated_at,
|
||||
has_topics_enabled, allows_users_to_create_topics,
|
||||
capability_checked_at
|
||||
) VALUES (?, ?, ?, 1, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(profile_name, chat_id) DO UPDATE SET
|
||||
user_id = excluded.user_id,
|
||||
enabled = 1,
|
||||
updated_at = excluded.updated_at,
|
||||
has_topics_enabled = excluded.has_topics_enabled,
|
||||
allows_users_to_create_topics = excluded.allows_users_to_create_topics,
|
||||
capability_checked_at = excluded.capability_checked_at
|
||||
""",
|
||||
(
|
||||
profile_name,
|
||||
str(chat_id),
|
||||
str(user_id),
|
||||
now,
|
||||
now,
|
||||
_to_int(has_topics_enabled),
|
||||
_to_int(allows_users_to_create_topics),
|
||||
now,
|
||||
),
|
||||
)
|
||||
|
||||
def disable_telegram_topic_mode(
|
||||
self,
|
||||
*,
|
||||
chat_id: str,
|
||||
profile_name: str = "default",
|
||||
clear_bindings: bool = True,
|
||||
) -> None:
|
||||
"""Disable Telegram DM topic mode for one private chat.
|
||||
|
||||
``clear_bindings`` also drops the chat's bindings so a later re-enable
|
||||
starts clean. Never creates the tables; absent tables are a no-op.
|
||||
"""
|
||||
profile_name = _normalize_telegram_topic_profile_name(profile_name)
|
||||
|
||||
def _do(conn):
|
||||
try:
|
||||
conn.execute(
|
||||
"UPDATE telegram_dm_topic_mode SET enabled = 0, updated_at = ? "
|
||||
"WHERE profile_name = ? AND chat_id = ?",
|
||||
(time.time(), profile_name, str(chat_id)),
|
||||
)
|
||||
if clear_bindings:
|
||||
conn.execute(
|
||||
"DELETE FROM telegram_dm_topic_bindings "
|
||||
"WHERE profile_name = ? AND chat_id = ?",
|
||||
(profile_name, str(chat_id)),
|
||||
)
|
||||
except sqlite3.OperationalError:
|
||||
return
|
||||
self._execute_write(_do)
|
||||
|
||||
def is_telegram_topic_mode_enabled(
|
||||
self,
|
||||
*,
|
||||
chat_id: str,
|
||||
user_id: str,
|
||||
profile_name: str = "default",
|
||||
) -> bool:
|
||||
"""Return whether Telegram DM topic mode is enabled for this chat/user."""
|
||||
profile_name = _normalize_telegram_topic_profile_name(profile_name)
|
||||
with self._read_ctx() as conn:
|
||||
try:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT enabled FROM telegram_dm_topic_mode
|
||||
WHERE profile_name = ? AND chat_id = ? AND user_id = ?
|
||||
""",
|
||||
(profile_name, str(chat_id), str(user_id)),
|
||||
).fetchone()
|
||||
except sqlite3.OperationalError:
|
||||
return False
|
||||
if row is None:
|
||||
return False
|
||||
enabled = row[0]
|
||||
return bool(enabled)
|
||||
|
||||
def get_telegram_topic_binding(
|
||||
self,
|
||||
*,
|
||||
chat_id: str,
|
||||
thread_id: str,
|
||||
profile_name: str = "default",
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Return the session binding for a Telegram DM topic, if present."""
|
||||
profile_name = _normalize_telegram_topic_profile_name(profile_name)
|
||||
with self._read_ctx() as conn:
|
||||
try:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT * FROM telegram_dm_topic_bindings
|
||||
WHERE profile_name = ? AND chat_id = ? AND thread_id = ?
|
||||
""",
|
||||
(profile_name, str(chat_id), str(thread_id)),
|
||||
).fetchone()
|
||||
except sqlite3.OperationalError:
|
||||
return None
|
||||
return dict(row) if row else None
|
||||
|
||||
def list_telegram_topic_bindings_for_chat(
|
||||
self,
|
||||
*,
|
||||
chat_id: str,
|
||||
profile_name: str = "default",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""All bindings for one chat, newest first. Read-only: [] when the
|
||||
table is absent (never triggers the migration)."""
|
||||
profile_name = _normalize_telegram_topic_profile_name(profile_name)
|
||||
with self._read_ctx() as conn:
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM telegram_dm_topic_bindings "
|
||||
"WHERE profile_name = ? AND chat_id = ? "
|
||||
"ORDER BY updated_at DESC",
|
||||
(profile_name, str(chat_id)),
|
||||
).fetchall()
|
||||
except sqlite3.OperationalError:
|
||||
return []
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def get_telegram_topic_binding_by_session(
|
||||
self,
|
||||
*,
|
||||
session_id: str,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Reverse lookup via the UNIQUE INDEX on session_id; None when
|
||||
unbound or the table is absent."""
|
||||
with self._read_ctx() as conn:
|
||||
try:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT * FROM telegram_dm_topic_bindings
|
||||
WHERE session_id = ?
|
||||
""",
|
||||
(str(session_id),),
|
||||
).fetchone()
|
||||
except sqlite3.OperationalError:
|
||||
return None
|
||||
return dict(row) if row else None
|
||||
|
||||
def delete_telegram_topic_binding(
|
||||
self,
|
||||
*,
|
||||
chat_id: str,
|
||||
thread_id: str,
|
||||
profile_name: str = "default",
|
||||
) -> int:
|
||||
"""Remove the binding row for one (chat, thread) pair.
|
||||
|
||||
Called when the Bot API confirms a topic was deleted externally
|
||||
(``Thread not found`` after the same-thread retry failed); otherwise
|
||||
``gateway.run._recover_telegram_topic_thread_id`` keeps redirecting
|
||||
inbound messages to the dead topic.
|
||||
|
||||
If this removes the chat's *last* binding, ``telegram_dm_topic_mode``
|
||||
is flipped to ``enabled = 0`` in the same transaction; otherwise the
|
||||
chat stays in topic mode with zero lanes and a user who disabled topics
|
||||
in the Telegram client (not via ``/topic off``) stays stuck.
|
||||
|
||||
Returns the number of rows deleted; absent binding or unmigrated tables
|
||||
are silent no-ops (never raise from a cleanup hot path).
|
||||
"""
|
||||
chat_id = str(chat_id)
|
||||
thread_id = str(thread_id)
|
||||
profile_name = _normalize_telegram_topic_profile_name(profile_name)
|
||||
deleted = {"count": 0}
|
||||
|
||||
def _do(conn):
|
||||
try:
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
DELETE FROM telegram_dm_topic_bindings
|
||||
WHERE profile_name = ? AND chat_id = ? AND thread_id = ?
|
||||
""",
|
||||
(profile_name, chat_id, thread_id),
|
||||
)
|
||||
deleted["count"] = cursor.rowcount or 0
|
||||
except sqlite3.OperationalError:
|
||||
deleted["count"] = 0
|
||||
return
|
||||
if not deleted["count"]:
|
||||
return
|
||||
# Last binding gone → disable topic mode. Same transaction, so no
|
||||
# read-after-prune race.
|
||||
try:
|
||||
remaining = conn.execute(
|
||||
"""
|
||||
SELECT 1 FROM telegram_dm_topic_bindings
|
||||
WHERE profile_name = ? AND chat_id = ? LIMIT 1
|
||||
""",
|
||||
(profile_name, chat_id),
|
||||
).fetchone()
|
||||
if remaining is None:
|
||||
conn.execute(
|
||||
"UPDATE telegram_dm_topic_mode "
|
||||
"SET enabled = 0, updated_at = ? "
|
||||
"WHERE profile_name = ? AND chat_id = ?",
|
||||
(time.time(), profile_name, chat_id),
|
||||
)
|
||||
except sqlite3.OperationalError:
|
||||
# telegram_dm_topic_mode absent — binding prune still stands.
|
||||
pass
|
||||
|
||||
self._execute_write(_do)
|
||||
return deleted["count"]
|
||||
|
||||
def bind_telegram_topic(
|
||||
self,
|
||||
*,
|
||||
chat_id: str,
|
||||
thread_id: str,
|
||||
user_id: str,
|
||||
session_key: str,
|
||||
session_id: str,
|
||||
managed_mode: str = "auto",
|
||||
profile_name: str = "default",
|
||||
) -> None:
|
||||
"""Bind one Telegram DM topic thread to one Hermes session.
|
||||
|
||||
A session may be linked to only one topic: rebinding the same pair is
|
||||
idempotent; linking the session to a different topic raises ValueError.
|
||||
"""
|
||||
self.apply_telegram_topic_migration()
|
||||
now = time.time()
|
||||
chat_id = str(chat_id)
|
||||
thread_id = str(thread_id)
|
||||
user_id = str(user_id)
|
||||
session_key = str(session_key)
|
||||
session_id = str(session_id)
|
||||
profile_name = _normalize_telegram_topic_profile_name(profile_name)
|
||||
|
||||
def _do(conn):
|
||||
existing_session = conn.execute(
|
||||
"""
|
||||
SELECT profile_name, chat_id, thread_id
|
||||
FROM telegram_dm_topic_bindings
|
||||
WHERE session_id = ?
|
||||
""",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
if existing_session is not None:
|
||||
if isinstance(existing_session, sqlite3.Row):
|
||||
linked_profile = existing_session["profile_name"]
|
||||
linked_chat = existing_session["chat_id"]
|
||||
linked_thread = existing_session["thread_id"]
|
||||
else:
|
||||
linked_profile, linked_chat, linked_thread = existing_session
|
||||
if (
|
||||
str(linked_profile) != profile_name
|
||||
or str(linked_chat) != chat_id
|
||||
or str(linked_thread) != thread_id
|
||||
):
|
||||
raise ValueError("session is already linked to another Telegram topic")
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO telegram_dm_topic_bindings (
|
||||
profile_name, chat_id, thread_id, user_id, session_key, session_id,
|
||||
managed_mode, linked_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(profile_name, chat_id, thread_id) DO UPDATE SET
|
||||
user_id = excluded.user_id,
|
||||
session_key = excluded.session_key,
|
||||
session_id = excluded.session_id,
|
||||
managed_mode = excluded.managed_mode,
|
||||
updated_at = excluded.updated_at
|
||||
""",
|
||||
(
|
||||
profile_name,
|
||||
chat_id,
|
||||
thread_id,
|
||||
user_id,
|
||||
session_key,
|
||||
session_id,
|
||||
managed_mode,
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
self._execute_write(_do)
|
||||
|
||||
def is_telegram_session_linked_to_topic(self, *, session_id: str) -> bool:
|
||||
"""True if the session is bound to any Telegram DM topic. Read-only:
|
||||
absent tables (nobody ran ``/topic``) mean unbound → False."""
|
||||
with self._read_ctx() as conn:
|
||||
try:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT 1 FROM telegram_dm_topic_bindings
|
||||
WHERE session_id = ?
|
||||
LIMIT 1
|
||||
""",
|
||||
(str(session_id),),
|
||||
).fetchone()
|
||||
except sqlite3.OperationalError:
|
||||
return False
|
||||
return row is not None
|
||||
|
||||
def list_unlinked_telegram_sessions_for_user(
|
||||
self,
|
||||
*,
|
||||
chat_id: str,
|
||||
user_id: str,
|
||||
profile_name: str = "default",
|
||||
limit: int = 10,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""This user's Telegram sessions not bound to a topic.
|
||||
|
||||
Read-only: if the bindings table is absent, every session is unlinked
|
||||
and a simpler query is used. Scoped by ``profile_name`` so multiplexed
|
||||
profiles do not surface each other's sessions.
|
||||
"""
|
||||
profile_name = _normalize_telegram_topic_profile_name(profile_name)
|
||||
# sessions.profile_name is NULL/empty for legacy rows → treat as default.
|
||||
profile_clause = "AND COALESCE(NULLIF(TRIM(s.profile_name), ''), 'default') = ?"
|
||||
with self._read_ctx() as conn:
|
||||
try:
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT s.*,
|
||||
COALESCE(sp.prompt, s.system_prompt)
|
||||
AS _system_prompt_resolved,
|
||||
COALESCE(
|
||||
(SELECT {_PREVIEW_RAW_SELECT}
|
||||
FROM messages m
|
||||
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
|
||||
AND {_PREVIEW_ELIGIBLE_SQL}
|
||||
ORDER BY m.timestamp, m.id LIMIT 1),
|
||||
''
|
||||
) AS _preview_raw,
|
||||
{_sql_session_last_active("s")} AS last_active
|
||||
FROM sessions s
|
||||
LEFT JOIN system_prompts sp
|
||||
ON sp.hash = s.system_prompt_hash
|
||||
WHERE s.source = 'telegram'
|
||||
AND s.user_id = ?
|
||||
{profile_clause}
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM telegram_dm_topic_bindings b
|
||||
WHERE b.session_id = s.id
|
||||
)
|
||||
ORDER BY last_active DESC, s.started_at DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(str(user_id), profile_name, int(limit)),
|
||||
).fetchall()
|
||||
except sqlite3.OperationalError:
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT s.*,
|
||||
COALESCE(sp.prompt, s.system_prompt)
|
||||
AS _system_prompt_resolved,
|
||||
COALESCE(
|
||||
(SELECT {_PREVIEW_RAW_SELECT}
|
||||
FROM messages m
|
||||
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
|
||||
AND {_PREVIEW_ELIGIBLE_SQL}
|
||||
ORDER BY m.timestamp, m.id LIMIT 1),
|
||||
''
|
||||
) AS _preview_raw,
|
||||
{_sql_session_last_active("s")} AS last_active
|
||||
FROM sessions s
|
||||
LEFT JOIN system_prompts sp
|
||||
ON sp.hash = s.system_prompt_hash
|
||||
WHERE s.source = 'telegram'
|
||||
AND s.user_id = ?
|
||||
ORDER BY last_active DESC, s.started_at DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(str(user_id), int(limit)),
|
||||
).fetchall()
|
||||
|
||||
sessions: List[Dict[str, Any]] = []
|
||||
for row in rows:
|
||||
session = self._session_row_dict(row)
|
||||
session["preview"] = _shape_preview(session.pop("_preview_raw", ""))
|
||||
sessions.append(session)
|
||||
return sessions
|
||||
@@ -0,0 +1,305 @@
|
||||
"""Session title mixin for SessionDB: sanitizing, auto/user provenance
|
||||
ranking, and lineage-aware lookups."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from agent.message_sanitization import _sanitize_surrogates
|
||||
from hermes_state_common import _COMPRESSION_CHILD_SQL, escape_like as _escape_like
|
||||
|
||||
# caplog tests pin the "hermes_state" logger name.
|
||||
logger = logging.getLogger("hermes_state")
|
||||
|
||||
|
||||
class SessionTitlesMixin:
|
||||
"""Sanitizing, ranking auto/user titles, lineage-aware lookups."""
|
||||
|
||||
@classmethod
|
||||
def _title_rank(cls, source: Optional[str]) -> int:
|
||||
"""Rank a stored title_source.
|
||||
|
||||
NULL (pre-provenance rows) is indistinguishable from a manual ``/title``
|
||||
of that era, so it ranks as ``user``: auto-titling only ever fills
|
||||
genuinely empty legacy titles.
|
||||
"""
|
||||
if source is None:
|
||||
return cls._TITLE_SOURCE_RANK[cls.TITLE_SOURCE_USER]
|
||||
return cls._TITLE_SOURCE_RANK.get(str(source), 0)
|
||||
|
||||
@staticmethod
|
||||
def sanitize_title(title: Optional[str]) -> Optional[str]:
|
||||
"""Strip control/zero-width/bidi chars, collapse whitespace, normalize
|
||||
empty to None. Raises ValueError if longer than MAX_TITLE_LENGTH
|
||||
after cleaning."""
|
||||
from hermes_state import SessionDB
|
||||
if not title:
|
||||
return None
|
||||
|
||||
# Lone surrogates cannot be bound by sqlite3 (UnicodeEncodeError).
|
||||
title = _sanitize_surrogates(title)
|
||||
|
||||
# ASCII controls, keeping \t \n \r so the whitespace collapse below
|
||||
# turns them into spaces.
|
||||
cleaned = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', title)
|
||||
|
||||
# Zero-width, bidi override, object-replacement, interlinear annotation.
|
||||
cleaned = re.sub(
|
||||
r'[\u200b-\u200f\u2028-\u202e\u2060-\u2069\ufeff\ufffc\ufff9-\ufffb]',
|
||||
'', cleaned,
|
||||
)
|
||||
|
||||
cleaned = re.sub(r'\s+', ' ', cleaned).strip()
|
||||
|
||||
if not cleaned:
|
||||
return None
|
||||
|
||||
if len(cleaned) > SessionDB.MAX_TITLE_LENGTH:
|
||||
raise ValueError(
|
||||
f"Title too long ({len(cleaned)} chars, max {SessionDB.MAX_TITLE_LENGTH})"
|
||||
)
|
||||
|
||||
return cleaned
|
||||
|
||||
def _is_compression_ancestor(
|
||||
self, conn, *, ancestor_id: str, descendant_id: str
|
||||
) -> bool:
|
||||
"""True if *ancestor_id* is a compression predecessor of *descendant_id*.
|
||||
|
||||
Uses the canonical continuation edge ``_COMPRESSION_CHILD_SQL`` (parent
|
||||
ended with ``end_reason = 'compression'`` and child started at/after its
|
||||
``ended_at``), which excludes delegate/branch children that also carry
|
||||
``parent_session_id``. One recursive CTE so the edge is defined once.
|
||||
"""
|
||||
if not ancestor_id or not descendant_id or ancestor_id == descendant_id:
|
||||
return False
|
||||
edge = _COMPRESSION_CHILD_SQL.format(a="child")
|
||||
row = conn.execute(
|
||||
f"""
|
||||
WITH RECURSIVE ancestors(id) AS (
|
||||
SELECT ?
|
||||
UNION
|
||||
SELECT parent.id
|
||||
FROM ancestors a
|
||||
JOIN sessions child ON child.id = a.id
|
||||
JOIN sessions parent ON parent.id = child.parent_session_id
|
||||
WHERE {edge}
|
||||
)
|
||||
SELECT 1 FROM ancestors WHERE id = ? AND id != ? LIMIT 1
|
||||
""",
|
||||
(descendant_id, ancestor_id, descendant_id),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def _set_session_title(
|
||||
self,
|
||||
session_id: str,
|
||||
title: str,
|
||||
*,
|
||||
source: str,
|
||||
) -> bool:
|
||||
"""Write a title, enforcing provenance precedence.
|
||||
|
||||
A ``user`` write always lands. ``derived``/``llm`` land only when the
|
||||
row is untitled or holds strictly lower authority, so derived upgrades
|
||||
to llm exactly once, nothing overwrites a user name, and re-running the
|
||||
titler on an llm row is a no-op (stops sessions renaming themselves).
|
||||
No writer may move a hidden canonical Bot Chat off its title.
|
||||
|
||||
Read and write are one compare-and-swap in a single transaction, so a
|
||||
manual ``/title`` racing an in-flight generation is not clobbered.
|
||||
"""
|
||||
title = self.sanitize_title(title)
|
||||
is_user = source == self.TITLE_SOURCE_USER
|
||||
new_rank = self._title_rank(source) if not is_user else None
|
||||
|
||||
def _do(conn):
|
||||
current = conn.execute(
|
||||
"SELECT title, title_source, hidden FROM sessions WHERE id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
if current is None:
|
||||
return 0
|
||||
# The canonical Bot Chat's NAME is its identity: Bot Mode resolves it
|
||||
# by exact-title lookup on every open, so a rename orphans the whole
|
||||
# conversation (next open mints an empty replacement and UNIQUE(title)
|
||||
# blocks renaming back). Refuse here, the single write path every
|
||||
# surface funnels through. Hidden is the discriminator: canonical
|
||||
# chats are born hidden; a visible session merely named "Bot Chat"
|
||||
# stays renameable. Provenance-blind so the auto-titler no-ops too.
|
||||
if (
|
||||
(current["title"] or "") == self.CANONICAL_BOT_CHAT_TITLE
|
||||
and bool(current["hidden"])
|
||||
and title != self.CANONICAL_BOT_CHAT_TITLE
|
||||
):
|
||||
if is_user:
|
||||
raise ValueError(
|
||||
"This is the bot's canonical Bot Chat — its name is its "
|
||||
"identity, and renaming it would orphan the conversation. "
|
||||
"To start fresh, create a new bot instead."
|
||||
)
|
||||
return 0
|
||||
if not is_user and current["title"] is not None:
|
||||
if self._title_rank(current["title_source"]) >= new_rank:
|
||||
return 0
|
||||
|
||||
if title:
|
||||
cursor = conn.execute(
|
||||
"SELECT id FROM sessions WHERE title = ? AND id != ?",
|
||||
(title, session_id),
|
||||
)
|
||||
conflict = cursor.fetchone()
|
||||
if conflict:
|
||||
conflict_id = conflict["id"]
|
||||
# If the conflicting holder is a hidden compressed ancestor
|
||||
# of this continuation, the user cannot free the title, so
|
||||
# transfer it onto the tip. Uniqueness and lineage are kept.
|
||||
if self._is_compression_ancestor(
|
||||
conn, ancestor_id=conflict_id, descendant_id=session_id
|
||||
):
|
||||
conn.execute(
|
||||
"UPDATE sessions SET title = NULL WHERE id = ?",
|
||||
(conflict_id,),
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Title '{title}' is already in use by session {conflict_id}"
|
||||
)
|
||||
# CAS on the values just read (``IS`` is NULL-safe): a concurrent
|
||||
# write between the SELECT and here loses instead of being overwritten.
|
||||
cursor = conn.execute(
|
||||
"UPDATE sessions SET title = ?, title_source = ? "
|
||||
"WHERE id = ? AND title IS ? AND title_source IS ?",
|
||||
(
|
||||
title,
|
||||
source if title else None,
|
||||
session_id,
|
||||
current["title"],
|
||||
current["title_source"],
|
||||
),
|
||||
)
|
||||
return cursor.rowcount
|
||||
|
||||
rowcount = self._execute_write(_do)
|
||||
return rowcount > 0
|
||||
|
||||
def set_session_title(self, session_id: str, title: str) -> bool:
|
||||
"""Set a title on the user's behalf (``user`` provenance; auto-titling
|
||||
never replaces it). Empty clears the title. Raises ValueError on a
|
||||
title conflict or validation failure. Automatic callers must use
|
||||
:meth:`set_auto_title`."""
|
||||
return self._set_session_title(
|
||||
session_id, title, source=self.TITLE_SOURCE_USER
|
||||
)
|
||||
|
||||
def set_auto_title(self, session_id: str, title: str, *, source: str) -> bool:
|
||||
"""Set an automatic title; False (untouched) when a higher-authority
|
||||
title already holds the row."""
|
||||
if source not in (self.TITLE_SOURCE_DERIVED, self.TITLE_SOURCE_LLM):
|
||||
raise ValueError(f"invalid automatic title source: {source!r}")
|
||||
return self._set_session_title(session_id, title, source=source)
|
||||
|
||||
def set_auto_title_if_empty(self, session_id: str, title: str) -> bool:
|
||||
"""Back-compat shim (third-party plugins reference it by name); new
|
||||
code calls :meth:`set_auto_title` with an explicit source."""
|
||||
return self.set_auto_title(
|
||||
session_id, title, source=self.TITLE_SOURCE_LLM
|
||||
)
|
||||
|
||||
def get_session_title(self, session_id: str) -> Optional[str]:
|
||||
"""Get the title for a session, or None."""
|
||||
with self._read_ctx() as conn:
|
||||
cursor = conn.execute(
|
||||
"SELECT title FROM sessions WHERE id = ?", (session_id,)
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
return row["title"] if row else None
|
||||
|
||||
def get_session_title_source(self, session_id: str) -> Optional[str]:
|
||||
"""Get the provenance of a session's title, or None when untitled."""
|
||||
with self._read_ctx() as conn:
|
||||
cursor = conn.execute(
|
||||
"SELECT title, title_source FROM sessions WHERE id = ?",
|
||||
(session_id,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if not row or row["title"] is None:
|
||||
return None
|
||||
return row["title_source"]
|
||||
|
||||
def set_session_title_source(self, session_id: str, source: str) -> bool:
|
||||
"""Overwrite a title's provenance without touching the text: a title
|
||||
copied across a compression rotation keeps the original's authority."""
|
||||
if source not in self._TITLE_SOURCE_RANK:
|
||||
raise ValueError(f"invalid title source: {source!r}")
|
||||
|
||||
return self._write_rowcount(
|
||||
"UPDATE sessions SET title_source = ? "
|
||||
"WHERE id = ? AND title IS NOT NULL",
|
||||
(source, session_id),
|
||||
) > 0
|
||||
|
||||
def get_session_by_title(self, title: str) -> Optional[Dict[str, Any]]:
|
||||
"""Look up a session by exact title. Returns session dict or None."""
|
||||
with self._read_ctx() as conn:
|
||||
cursor = conn.execute(
|
||||
"SELECT s.*, "
|
||||
"COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved "
|
||||
"FROM sessions s "
|
||||
"LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash "
|
||||
"WHERE s.title = ?",
|
||||
(title,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
return self._session_row_dict(row) if row else None
|
||||
|
||||
def resolve_session_by_title(self, title: str) -> Optional[str]:
|
||||
"""Resolve a title to a session ID, preferring the latest "title #N"
|
||||
continuation over the exact match."""
|
||||
exact = self.get_session_by_title(title)
|
||||
|
||||
# Escape LIKE wildcards so "%"/"_" in titles cannot false-match.
|
||||
escaped = _escape_like(title)
|
||||
with self._read_ctx() as conn:
|
||||
cursor = conn.execute(
|
||||
"SELECT id, title, started_at FROM sessions "
|
||||
"WHERE title LIKE ? ESCAPE '\\' ORDER BY started_at DESC",
|
||||
(f"{escaped} #%",),
|
||||
)
|
||||
numbered = cursor.fetchall()
|
||||
|
||||
if numbered:
|
||||
return numbered[0]["id"]
|
||||
elif exact:
|
||||
return exact["id"]
|
||||
return None
|
||||
|
||||
def get_next_title_in_lineage(self, base_title: str) -> str:
|
||||
"""Next title in a lineage ("my session" → "my session #2"): strip any
|
||||
" #N" suffix, then increment the highest existing number."""
|
||||
match = re.match(r'^(.*?) #(\d+)$', base_title)
|
||||
if match:
|
||||
base = match.group(1)
|
||||
else:
|
||||
base = base_title
|
||||
|
||||
escaped = _escape_like(base)
|
||||
with self._read_ctx() as conn:
|
||||
cursor = conn.execute(
|
||||
"SELECT title FROM sessions WHERE title = ? OR title LIKE ? ESCAPE '\\'",
|
||||
(base, f"{escaped} #%"),
|
||||
)
|
||||
existing = [row["title"] for row in cursor.fetchall()]
|
||||
|
||||
if not existing:
|
||||
return base
|
||||
|
||||
max_num = 1 # The unnumbered original counts as #1
|
||||
for t in existing:
|
||||
m = re.match(r'^.* #(\d+)$', t)
|
||||
if m:
|
||||
max_num = max(max_num, int(m.group(1)))
|
||||
|
||||
return f"{base} #{max_num + 1}"
|
||||
@@ -0,0 +1,597 @@
|
||||
"""Token/usage accounting mixin for SessionDB: the coalescing background
|
||||
token writer, per-model usage rows, and billing-route columns. Writer thread
|
||||
state lives on the SessionDB instance."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
# caplog tests pin the "hermes_state" logger name.
|
||||
logger = logging.getLogger("hermes_state")
|
||||
|
||||
|
||||
class SessionUsageMixin:
|
||||
"""Coalesced token writer, per-model usage rows, billing route."""
|
||||
|
||||
def update_session_billing_route(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
provider: str,
|
||||
base_url: str,
|
||||
billing_mode: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Unconditionally set the billing route (``update_token_counts`` only
|
||||
COALESCE-fills NULLs) so the dashboard reflects the latest /model switch.
|
||||
|
||||
Also nulls ``system_prompt`` so the cached snapshot (stale ``Model:`` /
|
||||
``Provider:`` header) is rebuilt, like ``update_session_model``.
|
||||
"""
|
||||
# Barrier against queued token deltas — see update_session_model.
|
||||
self.flush_token_counts()
|
||||
|
||||
def _do(conn):
|
||||
conn.execute(
|
||||
"""UPDATE sessions SET
|
||||
billing_provider = ?,
|
||||
billing_base_url = ?,
|
||||
billing_mode = COALESCE(?, billing_mode),
|
||||
system_prompt = NULL,
|
||||
system_prompt_hash = NULL
|
||||
WHERE id = ?""",
|
||||
(provider, base_url, billing_mode, session_id),
|
||||
)
|
||||
self._delete_unreferenced_system_prompts(conn)
|
||||
self._execute_write(_do)
|
||||
|
||||
def queue_token_counts(self, session_id: str, **kwargs) -> None:
|
||||
"""Enqueue a token/cost delta for the background writer.
|
||||
|
||||
Same kwargs and semantics as :meth:`update_token_counts`, applied
|
||||
asynchronously; cheap enough for the turn thread. After close() has
|
||||
stopped the writer, falls back to the synchronous path and may raise.
|
||||
"""
|
||||
with self._token_queue_cond:
|
||||
thread = self._token_writer_thread
|
||||
writer_stopped = self._token_writer_stop and (
|
||||
thread is None or not thread.is_alive()
|
||||
)
|
||||
if not writer_stopped:
|
||||
self._token_queue.append((session_id, kwargs))
|
||||
if thread is None or not thread.is_alive():
|
||||
# Daemon so exit never hangs on accounting; the atexit hook
|
||||
# (registered once per instance) drains leftovers. Checking
|
||||
# ``not is_alive()`` rather than ``is None`` respawns a writer
|
||||
# that died from an unexpected escape, otherwise deltas
|
||||
# would pile up until a reader's flush drained them.
|
||||
thread = threading.Thread(
|
||||
target=self._token_writer_loop,
|
||||
name="session-db-token-writer",
|
||||
daemon=True,
|
||||
)
|
||||
self._token_writer_thread = thread
|
||||
thread.start()
|
||||
if self._token_atexit_hook is None:
|
||||
self_ref = weakref.ref(self)
|
||||
|
||||
def _drain_at_exit() -> None:
|
||||
db = self_ref()
|
||||
if db is not None:
|
||||
db._drain_token_queue_at_exit()
|
||||
|
||||
self._token_atexit_hook = _drain_at_exit
|
||||
atexit.register(_drain_at_exit)
|
||||
self._token_queue_cond.notify_all()
|
||||
if writer_stopped:
|
||||
# close() ran (a stop-flagged but live writer still accepts; its
|
||||
# loop drains before exiting). Enqueueing now would drop the delta
|
||||
# silently — no writer, atexit hook gone — so apply inline and let a
|
||||
# closed-connection failure raise at the call site.
|
||||
self.update_token_counts(session_id, **kwargs)
|
||||
|
||||
def flush_token_counts(self, timeout: float = 5.0) -> bool:
|
||||
"""Block until every queued token delta has been applied.
|
||||
|
||||
False on timeout (callers then read totals stale by the queued deltas).
|
||||
Never raises: apply failures are logged by the writer.
|
||||
"""
|
||||
# Lock-free fast path: reads queue-then-busy (see ordering notes below).
|
||||
if not self._token_queue and not self._token_writer_busy:
|
||||
return True
|
||||
batch = None
|
||||
with self._token_queue_cond:
|
||||
deadline = time.monotonic() + timeout
|
||||
while self._token_queue or self._token_writer_busy:
|
||||
# A live writer is authoritative even when stop-flagged: draining
|
||||
# here would race its in-flight batch, and newer deltas committing
|
||||
# before older ones breaks last-non-None-wins / first-accounted-
|
||||
# route / COALESCE-backfill fields. Only a dead writer lets the
|
||||
# caller take leftovers; re-checked each wakeup because the writer
|
||||
# can exit mid-wait with deltas enqueued after its final check.
|
||||
# busy is claimed while draining so a concurrent flush cannot
|
||||
# report drained or pop a newer delta while this batch is
|
||||
# unapplied: a claimed busy means "wait", never "drain alongside".
|
||||
thread = self._token_writer_thread
|
||||
if (
|
||||
(thread is None or not thread.is_alive())
|
||||
and not self._token_writer_busy
|
||||
):
|
||||
self._token_writer_busy = True
|
||||
batch = list(self._token_queue)
|
||||
self._token_queue.clear()
|
||||
break
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return False
|
||||
self._token_queue_cond.wait(remaining)
|
||||
if batch:
|
||||
try:
|
||||
self._apply_token_batch(batch)
|
||||
finally:
|
||||
with self._token_queue_cond:
|
||||
self._token_writer_busy = False
|
||||
self._token_queue_cond.notify_all()
|
||||
return True
|
||||
|
||||
def _token_writer_loop(self) -> None:
|
||||
while True:
|
||||
with self._token_queue_cond:
|
||||
idle_deadline = time.monotonic() + self._TOKEN_WRITER_IDLE_SECONDS
|
||||
while not self._token_queue and not self._token_writer_stop:
|
||||
remaining = idle_deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
# Retire under the same lock queue_token_counts() uses to
|
||||
# decide to spawn, so no delta strands behind an exiting worker.
|
||||
self._token_writer_thread = None
|
||||
return
|
||||
self._token_queue_cond.wait(remaining)
|
||||
if not self._token_queue:
|
||||
self._token_writer_thread = None
|
||||
return # stop requested and fully drained
|
||||
# busy BEFORE clearing the queue: flush's lock-free fast path
|
||||
# reads queue-then-busy and must never see "empty and idle"
|
||||
# while a popped batch is unapplied.
|
||||
self._token_writer_busy = True
|
||||
batch = list(self._token_queue)
|
||||
self._token_queue.clear()
|
||||
try:
|
||||
self._apply_token_batch(batch)
|
||||
finally:
|
||||
with self._token_queue_cond:
|
||||
self._token_writer_busy = False
|
||||
self._token_queue_cond.notify_all()
|
||||
|
||||
def _apply_token_batch(self, batch: List[Tuple[str, Dict[str, Any]]]) -> None:
|
||||
"""Apply queued deltas in order, coalescing where safe. Never raises."""
|
||||
try:
|
||||
coalesced = self._coalesce_token_deltas(batch)
|
||||
except Exception as exc:
|
||||
# Coalescing must never kill the writer (callers cannot observe a
|
||||
# dead one); the merge is only an optimization.
|
||||
logger.warning(
|
||||
"async token accounting: coalesce failed, applying raw "
|
||||
"batch: %s", exc,
|
||||
)
|
||||
coalesced = batch
|
||||
for session_id, kwargs in coalesced:
|
||||
try:
|
||||
self.update_token_counts(session_id, **kwargs)
|
||||
except Exception as exc:
|
||||
# Accounting loss is logged, never raised into a turn.
|
||||
logger.warning(
|
||||
"async token accounting: apply failed (session=%s): %s",
|
||||
session_id, exc,
|
||||
)
|
||||
|
||||
def _coalesce_token_deltas(
|
||||
self, batch: List[Tuple[str, Dict[str, Any]]]
|
||||
) -> List[Tuple[str, Dict[str, Any]]]:
|
||||
"""Merge adjacent incremental deltas with an identical route, so
|
||||
ordering across sessions and /model switches is preserved exactly.
|
||||
absolute=True deltas never merge."""
|
||||
groups: List[Tuple[Optional[tuple], str, Dict[str, Any]]] = []
|
||||
for session_id, kwargs in batch:
|
||||
key = None
|
||||
if not kwargs.get("absolute"):
|
||||
key = (session_id,) + tuple(
|
||||
kwargs.get(f) for f in self._TOKEN_DELTA_ROUTE_FIELDS
|
||||
)
|
||||
if groups and key is not None and groups[-1][0] == key:
|
||||
merged = groups[-1][2]
|
||||
for f in self._TOKEN_DELTA_SUM_FIELDS:
|
||||
merged[f] = merged.get(f, 0) + kwargs.get(f, 0)
|
||||
for f in self._TOKEN_DELTA_COST_FIELDS:
|
||||
value = kwargs.get(f)
|
||||
if value is not None:
|
||||
# All-None runs stay None so COALESCE keeps the stored value.
|
||||
merged[f] = (merged.get(f) or 0.0) + value
|
||||
else:
|
||||
groups.append((key, session_id, dict(kwargs)))
|
||||
return [(sid, kw) for _, sid, kw in groups]
|
||||
|
||||
def _stop_token_writer(self, join_timeout: float = 10.0) -> None:
|
||||
"""Stop the writer thread and drain remaining deltas. Never raises."""
|
||||
with self._token_queue_cond:
|
||||
self._token_writer_stop = True
|
||||
self._token_queue_cond.notify_all()
|
||||
thread = self._token_writer_thread
|
||||
if thread is not None and thread.is_alive():
|
||||
thread.join(timeout=join_timeout)
|
||||
if thread.is_alive():
|
||||
# Writer stuck mid-apply: leave deltas unapplied rather than
|
||||
# race it and misorder/double-count.
|
||||
logger.warning(
|
||||
"async token accounting: writer did not stop within %.0fs; "
|
||||
"%d queued delta(s) not persisted",
|
||||
join_timeout, len(self._token_queue),
|
||||
)
|
||||
return
|
||||
# Writer gone: apply leftovers synchronously under the same busy
|
||||
# protocol. Wait out a flush caller-drain that already claimed busy —
|
||||
# close() nulls the connection right after this returns and must not
|
||||
# yank it mid-batch.
|
||||
with self._token_queue_cond:
|
||||
deadline = time.monotonic() + join_timeout
|
||||
while self._token_writer_busy:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
logger.warning(
|
||||
"async token accounting: concurrent drain did not "
|
||||
"finish within %.0fs; %d queued delta(s) not persisted",
|
||||
join_timeout, len(self._token_queue),
|
||||
)
|
||||
return
|
||||
self._token_queue_cond.wait(remaining)
|
||||
# busy BEFORE clearing the queue (same ordering as the writer loop),
|
||||
# or flush's lock-free fast path could see "empty and idle".
|
||||
batch = list(self._token_queue)
|
||||
if batch:
|
||||
self._token_writer_busy = True
|
||||
self._token_queue.clear()
|
||||
if batch:
|
||||
try:
|
||||
self._apply_token_batch(batch)
|
||||
finally:
|
||||
with self._token_queue_cond:
|
||||
self._token_writer_busy = False
|
||||
self._token_queue_cond.notify_all()
|
||||
|
||||
def _drain_token_queue_at_exit(self) -> None:
|
||||
try:
|
||||
self._stop_token_writer()
|
||||
except Exception:
|
||||
pass # never fatal at interpreter shutdown
|
||||
|
||||
def update_token_counts(
|
||||
self,
|
||||
session_id: str,
|
||||
input_tokens: int = 0,
|
||||
output_tokens: int = 0,
|
||||
model: str = None,
|
||||
cache_read_tokens: int = 0,
|
||||
cache_write_tokens: int = 0,
|
||||
reasoning_tokens: int = 0,
|
||||
estimated_cost_usd: Optional[float] = None,
|
||||
actual_cost_usd: Optional[float] = None,
|
||||
cost_status: Optional[str] = None,
|
||||
cost_source: Optional[str] = None,
|
||||
pricing_version: Optional[str] = None,
|
||||
billing_provider: Optional[str] = None,
|
||||
billing_base_url: Optional[str] = None,
|
||||
billing_mode: Optional[str] = None,
|
||||
api_call_count: int = 0,
|
||||
absolute: bool = False,
|
||||
) -> None:
|
||||
"""Update token counters and backfill model if unset.
|
||||
|
||||
*absolute*=False increments (per-API-call deltas, CLI path);
|
||||
*absolute*=True sets directly (gateway path, where the cached agent
|
||||
holds cumulative totals).
|
||||
"""
|
||||
# Ensure the row exists: under concurrent load the initial
|
||||
# create_session() may have failed on SQLite locking, and the UPDATE
|
||||
# would silently affect 0 rows.
|
||||
self._insert_session_row(session_id, "unknown", model=model)
|
||||
if absolute:
|
||||
sql = """UPDATE sessions SET
|
||||
input_tokens = ?,
|
||||
output_tokens = ?,
|
||||
cache_read_tokens = ?,
|
||||
cache_write_tokens = ?,
|
||||
reasoning_tokens = ?,
|
||||
estimated_cost_usd = COALESCE(?, 0),
|
||||
actual_cost_usd = CASE
|
||||
WHEN ? IS NULL THEN actual_cost_usd
|
||||
ELSE ?
|
||||
END,
|
||||
cost_status = COALESCE(?, cost_status),
|
||||
cost_source = COALESCE(?, cost_source),
|
||||
pricing_version = COALESCE(?, pricing_version),
|
||||
billing_provider = COALESCE(billing_provider, ?),
|
||||
billing_base_url = COALESCE(billing_base_url, ?),
|
||||
billing_mode = COALESCE(billing_mode, ?),
|
||||
model = COALESCE(model, ?),
|
||||
api_call_count = ?
|
||||
WHERE id = ?"""
|
||||
else:
|
||||
sql = """UPDATE sessions SET
|
||||
input_tokens = input_tokens + ?,
|
||||
output_tokens = output_tokens + ?,
|
||||
cache_read_tokens = cache_read_tokens + ?,
|
||||
cache_write_tokens = cache_write_tokens + ?,
|
||||
reasoning_tokens = reasoning_tokens + ?,
|
||||
estimated_cost_usd = COALESCE(estimated_cost_usd, 0) + COALESCE(?, 0),
|
||||
actual_cost_usd = CASE
|
||||
WHEN ? IS NULL THEN actual_cost_usd
|
||||
ELSE COALESCE(actual_cost_usd, 0) + ?
|
||||
END,
|
||||
cost_status = COALESCE(?, cost_status),
|
||||
cost_source = COALESCE(?, cost_source),
|
||||
pricing_version = COALESCE(?, pricing_version),
|
||||
billing_provider = COALESCE(billing_provider, ?),
|
||||
billing_base_url = COALESCE(billing_base_url, ?),
|
||||
billing_mode = COALESCE(billing_mode, ?),
|
||||
model = COALESCE(model, ?),
|
||||
api_call_count = COALESCE(api_call_count, 0) + ?
|
||||
WHERE id = ?"""
|
||||
has_accounted_usage = bool(
|
||||
input_tokens or output_tokens or cache_read_tokens
|
||||
or cache_write_tokens or reasoning_tokens or api_call_count
|
||||
or estimated_cost_usd or actual_cost_usd
|
||||
)
|
||||
params = (
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
cache_read_tokens,
|
||||
cache_write_tokens,
|
||||
reasoning_tokens,
|
||||
estimated_cost_usd,
|
||||
actual_cost_usd,
|
||||
actual_cost_usd,
|
||||
cost_status,
|
||||
cost_source,
|
||||
pricing_version,
|
||||
billing_provider if has_accounted_usage else None,
|
||||
billing_base_url if has_accounted_usage else None,
|
||||
billing_mode if has_accounted_usage else None,
|
||||
model if has_accounted_usage else None,
|
||||
api_call_count,
|
||||
session_id,
|
||||
)
|
||||
# Per-model attribution: the sessions row keeps one (model, provider)
|
||||
# pair, so a mid-session /model switch would attribute every token to
|
||||
# the initial model. Each delta carries the route active at call time
|
||||
# and is recorded into session_model_usage keyed by it. Only the
|
||||
# incremental path records here: absolute cumulative updates cannot be
|
||||
# split back into routes; Insights reconciles the residual instead.
|
||||
record_model_usage = (not absolute) and (
|
||||
input_tokens or output_tokens or cache_read_tokens
|
||||
or cache_write_tokens or reasoning_tokens or api_call_count
|
||||
or estimated_cost_usd
|
||||
)
|
||||
|
||||
def _do(conn):
|
||||
row = conn.execute(
|
||||
"SELECT model, billing_provider, api_call_count FROM sessions WHERE id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
existing_model = row["model"] if row is not None else None
|
||||
existing_provider = row["billing_provider"] if row is not None else None
|
||||
existing_api_calls = int((row["api_call_count"] if row is not None else 0) or 0)
|
||||
|
||||
# create_session records the requested route before any API call.
|
||||
# If that fails and fallback succeeds, the first accounted usage is
|
||||
# the authoritative route; after that keep the row as is (one row
|
||||
# cannot represent mixed-provider usage).
|
||||
first_accounted_route = (
|
||||
existing_api_calls == 0
|
||||
and has_accounted_usage
|
||||
and bool(model)
|
||||
and bool(billing_provider)
|
||||
and (existing_model != model or existing_provider != billing_provider)
|
||||
)
|
||||
if first_accounted_route:
|
||||
conn.execute(
|
||||
"""UPDATE sessions
|
||||
SET model = ?, billing_provider = ?,
|
||||
billing_base_url = ?, billing_mode = ?
|
||||
WHERE id = ?""",
|
||||
(model, billing_provider, billing_base_url, billing_mode, session_id),
|
||||
)
|
||||
conn.execute(sql, params)
|
||||
if record_model_usage:
|
||||
self._record_model_usage(
|
||||
conn,
|
||||
session_id,
|
||||
model=model,
|
||||
billing_provider=billing_provider,
|
||||
billing_base_url=billing_base_url,
|
||||
billing_mode=billing_mode,
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
cache_read_tokens=cache_read_tokens,
|
||||
cache_write_tokens=cache_write_tokens,
|
||||
reasoning_tokens=reasoning_tokens,
|
||||
estimated_cost_usd=estimated_cost_usd,
|
||||
actual_cost_usd=actual_cost_usd,
|
||||
cost_status=cost_status,
|
||||
cost_source=cost_source,
|
||||
api_call_count=api_call_count,
|
||||
)
|
||||
self._execute_write(_do)
|
||||
|
||||
def _record_model_usage(
|
||||
self,
|
||||
conn,
|
||||
session_id: str,
|
||||
*,
|
||||
model: Optional[str],
|
||||
billing_provider: Optional[str],
|
||||
billing_base_url: Optional[str],
|
||||
billing_mode: Optional[str],
|
||||
input_tokens: int,
|
||||
output_tokens: int,
|
||||
cache_read_tokens: int,
|
||||
cache_write_tokens: int,
|
||||
reasoning_tokens: int,
|
||||
estimated_cost_usd: Optional[float],
|
||||
actual_cost_usd: Optional[float],
|
||||
cost_status: Optional[str],
|
||||
cost_source: Optional[str],
|
||||
api_call_count: int,
|
||||
task: str = "",
|
||||
) -> None:
|
||||
"""Accumulate a per-API-call usage delta into session_model_usage.
|
||||
|
||||
Runs inside the caller's write transaction, after the ``sessions``
|
||||
UPDATE, so per-model rows stay consistent with the summary row. A
|
||||
missing model/provider falls back to the session row (same COALESCE
|
||||
behaviour as the summary update). ``task`` is ``''`` for the main loop;
|
||||
auxiliary calls record their task name via :meth:`record_auxiliary_usage`.
|
||||
"""
|
||||
row = conn.execute(
|
||||
"SELECT model, billing_provider, billing_base_url, billing_mode "
|
||||
"FROM sessions WHERE id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
sess_model = row["model"] if row is not None else None
|
||||
sess_provider = row["billing_provider"] if row is not None else None
|
||||
sess_base_url = row["billing_base_url"] if row is not None else None
|
||||
sess_billing_mode = row["billing_mode"] if row is not None else None
|
||||
|
||||
# Aux rows must NOT inherit the main-loop route (vision on gemini while
|
||||
# the main loop runs anthropic); missing info stays 'unknown'/empty.
|
||||
if task:
|
||||
eff_model = model or "unknown"
|
||||
eff_provider = billing_provider or ""
|
||||
eff_base_url = billing_base_url or ""
|
||||
eff_billing_mode = billing_mode or ""
|
||||
else:
|
||||
eff_model = model or sess_model or "unknown"
|
||||
eff_provider = billing_provider or sess_provider or ""
|
||||
eff_base_url = billing_base_url or sess_base_url or ""
|
||||
eff_billing_mode = billing_mode or sess_billing_mode or ""
|
||||
now = time.time()
|
||||
conn.execute(
|
||||
"""INSERT INTO session_model_usage (
|
||||
session_id, model, billing_provider, billing_base_url, billing_mode,
|
||||
task, api_call_count, input_tokens, output_tokens,
|
||||
cache_read_tokens, cache_write_tokens, reasoning_tokens,
|
||||
estimated_cost_usd, actual_cost_usd, cost_status, cost_source,
|
||||
first_seen, last_seen
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(session_id, model, billing_provider, billing_base_url, billing_mode, task)
|
||||
DO UPDATE SET
|
||||
api_call_count = api_call_count + excluded.api_call_count,
|
||||
input_tokens = input_tokens + excluded.input_tokens,
|
||||
output_tokens = output_tokens + excluded.output_tokens,
|
||||
cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens,
|
||||
cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
|
||||
reasoning_tokens = reasoning_tokens + excluded.reasoning_tokens,
|
||||
estimated_cost_usd = estimated_cost_usd + excluded.estimated_cost_usd,
|
||||
actual_cost_usd = actual_cost_usd + excluded.actual_cost_usd,
|
||||
cost_status = COALESCE(excluded.cost_status, cost_status),
|
||||
cost_source = COALESCE(excluded.cost_source, cost_source),
|
||||
last_seen = excluded.last_seen""",
|
||||
(
|
||||
session_id,
|
||||
eff_model,
|
||||
eff_provider,
|
||||
eff_base_url,
|
||||
eff_billing_mode,
|
||||
task or "",
|
||||
api_call_count or 0,
|
||||
input_tokens or 0,
|
||||
output_tokens or 0,
|
||||
cache_read_tokens or 0,
|
||||
cache_write_tokens or 0,
|
||||
reasoning_tokens or 0,
|
||||
float(estimated_cost_usd or 0.0),
|
||||
float(actual_cost_usd or 0.0),
|
||||
cost_status,
|
||||
cost_source,
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
|
||||
def record_auxiliary_usage(
|
||||
self,
|
||||
session_id: str,
|
||||
task: str,
|
||||
*,
|
||||
model: Optional[str] = None,
|
||||
billing_provider: Optional[str] = None,
|
||||
billing_base_url: Optional[str] = None,
|
||||
input_tokens: int = 0,
|
||||
output_tokens: int = 0,
|
||||
cache_read_tokens: int = 0,
|
||||
cache_write_tokens: int = 0,
|
||||
reasoning_tokens: int = 0,
|
||||
estimated_cost_usd: Optional[float] = None,
|
||||
api_call_count: int = 1,
|
||||
) -> None:
|
||||
"""Record an auxiliary LLM call's usage (vision, compression, title
|
||||
generation, ...) against *session_id*.
|
||||
|
||||
Writes a per-(model, provider, task) delta into ``session_model_usage``
|
||||
WITHOUT touching the ``sessions`` summary row: the gateway overwrites
|
||||
session counters with absolute main-loop totals, so aux tokens there
|
||||
would be clobbered or double-counted. Insights read the union.
|
||||
``api_call_count`` may aggregate N calls (background-review forks).
|
||||
Best-effort: callers must never fail an aux call over accounting.
|
||||
"""
|
||||
if not session_id or not task:
|
||||
return
|
||||
# FK to sessions.id: same INSERT OR IGNORE guard as update_token_counts.
|
||||
self._insert_session_row(session_id, "unknown")
|
||||
|
||||
def _do(conn):
|
||||
self._record_model_usage(
|
||||
conn,
|
||||
session_id,
|
||||
model=model,
|
||||
billing_provider=billing_provider,
|
||||
billing_base_url=billing_base_url,
|
||||
billing_mode=None,
|
||||
input_tokens=input_tokens or 0,
|
||||
output_tokens=output_tokens or 0,
|
||||
cache_read_tokens=cache_read_tokens or 0,
|
||||
cache_write_tokens=cache_write_tokens or 0,
|
||||
reasoning_tokens=reasoning_tokens or 0,
|
||||
estimated_cost_usd=estimated_cost_usd,
|
||||
actual_cost_usd=None,
|
||||
cost_status=None,
|
||||
cost_source=None,
|
||||
api_call_count=(
|
||||
1 if api_call_count is None else int(api_call_count)
|
||||
),
|
||||
task=task,
|
||||
)
|
||||
self._execute_write(_do)
|
||||
|
||||
def usage_totals(self, *, min_message_count: int = 1, include_archived: bool = False) -> Dict[str, float]:
|
||||
"""Tokens and spend across the whole store (one scan), so the sidebar
|
||||
total does not shrink with paging. Spend prefers the billed figure over
|
||||
the estimate, the same precedence a single row renders."""
|
||||
where = ["parent_session_id IS NULL", "message_count >= ?"]
|
||||
params: List[Any] = [min_message_count]
|
||||
if not include_archived:
|
||||
where.append("COALESCE(archived, 0) = 0")
|
||||
|
||||
row = self._read_one(
|
||||
f"""
|
||||
SELECT COALESCE(SUM(COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0)), 0),
|
||||
COALESCE(SUM(COALESCE(actual_cost_usd, estimated_cost_usd, 0)), 0)
|
||||
FROM sessions
|
||||
WHERE {' AND '.join(where)}
|
||||
""",
|
||||
params,
|
||||
)
|
||||
|
||||
return {"tokens": int(row[0] or 0), "cost_usd": float(row[1] or 0.0)}
|
||||
@@ -0,0 +1,821 @@
|
||||
"""SQLite journal-mode and PRAGMA policy for state.db.
|
||||
|
||||
Split out of ``hermes_state.py``. Every name is re-imported there so
|
||||
``hermes_state.<name>`` keeps resolving, and tests that monkeypatch it keep
|
||||
intercepting because intra-module calls to patched helpers go through a lazy
|
||||
``from hermes_state import ...`` at call time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from hermes_cli.sqlite_runtime import (
|
||||
is_sqlite_wal_reset_vulnerable as _is_sqlite_wal_reset_vulnerable,
|
||||
)
|
||||
|
||||
# Log-record parity with the origin module (caplog tests pin "hermes_state").
|
||||
logger = logging.getLogger("hermes_state")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# WAL-compatibility fallback
|
||||
# ---------------------------------------------------------------------------
|
||||
# WAL needs mmap shared memory and fcntl byte-range locks, which network
|
||||
# filesystems (NFS, SMB/CIFS, some FUSE, WSL1) don't provide reliably — there
|
||||
# ``PRAGMA journal_mode=WAL`` raises ``locking protocol`` (SQLITE_PROTOCOL).
|
||||
# ZFS instead corrupts the -shm file under concurrent connection bursts (COW +
|
||||
# mmap), presenting as ``disk I/O error``. Propagating either would silently
|
||||
# break everything backed by state.db/kanban.db, so we fall back to
|
||||
# ``journal_mode=DELETE`` (pre-WAL default, works on NFS/ZFS): readers block
|
||||
# during a write, but it works. The WAL-reset-bug gate and the
|
||||
# never-live-downgrade invariant are documented on apply_wal_with_fallback.
|
||||
_WAL_INCOMPAT_MARKERS = (
|
||||
"locking protocol", # SQLITE_PROTOCOL on NFS/SMB
|
||||
"not authorized", # Some FUSE mounts block WAL pragma outright
|
||||
"disk i/o error", # ZFS SHM corruption under concurrent connections
|
||||
)
|
||||
|
||||
|
||||
# SQLite's default is -1 (unlimited), so state.db-wal would keep the high-water
|
||||
# mark of the largest-ever transaction forever. See _apply_wal_size_limit().
|
||||
_WAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024 # 64 MiB
|
||||
|
||||
|
||||
# Dedup sets: kanban_db.connect() runs on every kanban operation, so an
|
||||
# undeduped fallback log line would repeat per connection and fill errors.log.
|
||||
_wal_fallback_warned_paths: set[str] = set()
|
||||
|
||||
|
||||
_wal_fallback_warned_lock = threading.Lock()
|
||||
|
||||
|
||||
_wal_reset_bug_warned_paths: set[str] = set()
|
||||
|
||||
|
||||
_wal_reset_bug_warned_lock = threading.Lock()
|
||||
|
||||
|
||||
# "configured delete overridden by on-disk WAL" ERROR.
|
||||
_delete_overridden_warned_paths: set[str] = set()
|
||||
|
||||
|
||||
_delete_overridden_warned_lock = threading.Lock()
|
||||
|
||||
|
||||
def _on_disk_journal_mode(conn: sqlite3.Connection) -> Optional[str]:
|
||||
"""Read the journal mode from the DB header; ``None`` if undeterminable.
|
||||
|
||||
``None`` (new DB, or PRAGMA failed) sends callers down their fail-closed
|
||||
"unknown → refuse to downgrade" branch. ``disk i/o error`` can be transient
|
||||
on virtualized block devices (XFS on cloud hosts), so it is retried a few
|
||||
times first: transient EIO clears, deterministic filesystem errors do not.
|
||||
"""
|
||||
last_exc: Optional[Exception] = None
|
||||
for _ in range(4):
|
||||
try:
|
||||
row = conn.execute("PRAGMA journal_mode").fetchone()
|
||||
except sqlite3.OperationalError as exc:
|
||||
last_exc = exc
|
||||
if "disk i/o error" not in str(exc).lower():
|
||||
return None
|
||||
time.sleep(0.05)
|
||||
continue
|
||||
if row is None:
|
||||
return None
|
||||
mode = row[0]
|
||||
if isinstance(mode, bytes): # defensive: sqlite3 occasionally returns bytes
|
||||
try:
|
||||
mode = mode.decode("ascii")
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
return str(mode).strip().lower() if mode is not None else None
|
||||
if last_exc is not None:
|
||||
logger.debug(
|
||||
"_on_disk_journal_mode: retries exhausted on disk read (%s)", last_exc
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _apply_wal_size_limit(conn: sqlite3.Connection) -> None:
|
||||
"""Bound the WAL so it returns space to the OS after big transactions.
|
||||
|
||||
SQLite's default ``journal_size_limit`` is -1: a checkpointed WAL is reused
|
||||
in place, never truncated, so ``state.db-wal`` keeps the high-water mark
|
||||
of the largest transaction ever run. One bulk op strands gigabytes —
|
||||
``hermes sessions optimize`` on a 3 GB state.db left a 3 GB WAL and filled
|
||||
the disk, so the maintenance command was self-defeating on the largest
|
||||
DBs. With a limit, each checkpoint truncates the WAL back to it; 64 MiB is
|
||||
above normal transaction sizes (steady-state commits never pay a truncate)
|
||||
while capping slack predictably. kanban_db uses ``wal_autocheckpoint=100``.
|
||||
|
||||
Best-effort: never raises — failure only costs disk slack and must not
|
||||
prevent the database from opening.
|
||||
"""
|
||||
try:
|
||||
conn.execute(f"PRAGMA journal_size_limit={_WAL_SIZE_LIMIT_BYTES}")
|
||||
except sqlite3.OperationalError as exc: # pragma: no cover - defensive
|
||||
logger.debug("journal_size_limit not applied: %s", exc)
|
||||
|
||||
|
||||
def _apply_macos_checkpoint_barrier(conn: sqlite3.Connection) -> None:
|
||||
"""Enable ``PRAGMA checkpoint_fullfsync`` on macOS (no-op elsewhere).
|
||||
|
||||
Apple's ``fsync(2)`` guarantees neither data-on-platter nor write ordering,
|
||||
so WAL's corruption-safety assumption fails on Darwin without ``F_FULLFSYNC``.
|
||||
A launchd shutdown drops the page cache (power-loss for in-flight pages), so
|
||||
a checkpoint that "reported" durable can leave a malformed ``state.db``;
|
||||
a plain in-session kill survives via the page cache. The barrier applies
|
||||
only at checkpoint boundaries (~+0.1 ms/commit vs ~+4 ms for
|
||||
``fullfsync=1``). Best-effort: never raises.
|
||||
"""
|
||||
if sys.platform != "darwin":
|
||||
return
|
||||
try:
|
||||
conn.execute("PRAGMA checkpoint_fullfsync=1")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
|
||||
|
||||
def _enforce_macos_synchronous_full(conn: sqlite3.Connection) -> None:
|
||||
"""Enforce ``PRAGMA synchronous=FULL`` on macOS to prevent btree corruption.
|
||||
|
||||
With NORMAL, a WAL checkpoint racing process termination (launchd shutdown)
|
||||
can leave half-written btree pages (``btreeInitPage error 11``) because
|
||||
Darwin's ``fsync()`` guarantees neither ordering nor durability. Called
|
||||
after every successful WAL activation so a prior connection's NORMAL never
|
||||
sticks. Best-effort: never raises.
|
||||
"""
|
||||
if sys.platform != "darwin":
|
||||
return
|
||||
try:
|
||||
conn.execute("PRAGMA synchronous=FULL")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
|
||||
|
||||
def is_sqlite_wal_reset_vulnerable(
|
||||
version_info: Optional[tuple] = None,
|
||||
) -> bool:
|
||||
"""True when the linked SQLite has the WAL-reset bug (3.7.0–3.51.2;
|
||||
fixed 3.51.3+, backports 3.50.7 / 3.44.6). Pre-WAL libraries are safe.
|
||||
https://sqlite.org/wal.html#walresetbug
|
||||
"""
|
||||
info = version_info if version_info is not None else sqlite3.sqlite_version_info
|
||||
return _is_sqlite_wal_reset_vulnerable(info)
|
||||
|
||||
|
||||
def sqlite_source_id() -> str:
|
||||
"""Return ``sqlite_source_id()``, or an empty string when unavailable."""
|
||||
try:
|
||||
conn = sqlite3.connect(":memory:")
|
||||
try:
|
||||
row = conn.execute("SELECT sqlite_source_id()").fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
except sqlite3.Error:
|
||||
return ""
|
||||
if not row or row[0] is None:
|
||||
return ""
|
||||
return str(row[0])
|
||||
|
||||
|
||||
def _database_has_content(conn: sqlite3.Connection) -> bool:
|
||||
"""Whether the file already holds pages (existing vs brand-new DB).
|
||||
|
||||
``PRAGMA page_count`` is a lock-free header read. Fail-quiet: any error
|
||||
answers False, because the only caller gates a warning on this and an
|
||||
unknown-answer warning would fire on every fresh database — exactly where
|
||||
there is provably no operator choice being overwritten.
|
||||
"""
|
||||
try:
|
||||
row = conn.execute("PRAGMA page_count").fetchone()
|
||||
except sqlite3.Error:
|
||||
return False
|
||||
if not row or row[0] is None:
|
||||
return False
|
||||
try:
|
||||
return int(row[0]) > 0
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def resolve_journal_mode() -> str:
|
||||
"""Return the configured journal mode (``wal`` or ``delete``).
|
||||
|
||||
``database.journal_mode`` in config.yaml is the canonical operator setting;
|
||||
``wal`` is the default, ``delete`` is for filesystems without WAL-safe
|
||||
durability (macOS virtiofs, NFS, SMB). Invalid values fail safe to ``wal``.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.config import load_config_readonly
|
||||
|
||||
config = load_config_readonly() or {}
|
||||
database = config.get("database", {})
|
||||
if not isinstance(database, dict):
|
||||
return "wal"
|
||||
raw = database.get("journal_mode", "wal")
|
||||
except Exception:
|
||||
return "wal"
|
||||
|
||||
if not isinstance(raw, str):
|
||||
return "wal"
|
||||
mode = raw.strip().lower()
|
||||
return mode if mode in ("wal", "delete") else "wal"
|
||||
|
||||
|
||||
class WalUnsupportedError(sqlite3.OperationalError):
|
||||
"""Raised by :func:`apply_wal_with_fallback` when ``require_wal=True`` and
|
||||
the filesystem cannot provide WAL — whether SQLite *raised*
|
||||
``SQLITE_PROTOCOL`` or (macOS NFS) silently returned the still-effective
|
||||
mode. Subclasses ``OperationalError`` so existing DB-init handlers still
|
||||
catch it while WAL-mandating callers can catch the narrower type.
|
||||
"""
|
||||
|
||||
|
||||
def apply_wal_with_fallback(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
db_label: str = "state.db",
|
||||
require_wal: bool = False,
|
||||
) -> str:
|
||||
"""Set ``journal_mode=WAL`` on ``conn``, falling back to DELETE on failure.
|
||||
|
||||
Returns the mode actually set (``"wal"`` or ``"delete"``). Shared by
|
||||
:class:`SessionDB` and ``hermes_cli.kanban_db.connect`` for identical
|
||||
fallback behavior.
|
||||
|
||||
On WAL-incompatible filesystems (NFS, SMB, some FUSE, ZFS) SQLite either
|
||||
raises ``OperationalError`` ("locking protocol" / "disk I/O error") or —
|
||||
macOS NFS / SMB / AgentFS NFS overlay — silently refuses and leaves the DB
|
||||
in DELETE. Either way we log at ERROR (a write now blocks readers — a real
|
||||
concurrency loss) and fall back to DELETE so the feature keeps working.
|
||||
``require_wal=True`` raises :class:`WalUnsupportedError` instead; all
|
||||
current callers keep the default so NFS-homed installs work.
|
||||
|
||||
On SQLite builds with the WAL-reset bug (https://sqlite.org/wal.html#walresetbug,
|
||||
fixed 3.51.3+, backports 3.50.7 / 3.44.6), refuse to enable WAL on
|
||||
fresh / non-WAL databases; an already-WAL DB keeps WAL with a warning.
|
||||
This gate is deliberately RETAINED: an attempt to revert it (theory: DELETE
|
||||
was "the mode that corrupts") was confounded — its clean WAL result came
|
||||
from SQLite 3.53.1, which also carries 3.51.0's close()-broken-POSIX-lock
|
||||
defenses. Re-measured on the bundled 3.50.4 with the lock fix, WAL and
|
||||
DELETE are both clean, so there is no evidence WAL is safer; keep new
|
||||
databases out of WAL until a fixed runtime ships.
|
||||
|
||||
Invariant on every path (NFS and WAL-reset alike): never downgrade to
|
||||
DELETE if the on-disk header reports WAL or the mode cannot be read (see
|
||||
_on_disk_journal_mode). Other gateway/cron/worker connections may hold the
|
||||
DB open, and a live downgrade destroys their committed-but-uncheckpointed
|
||||
transactions.
|
||||
|
||||
The ERROR is deduplicated per ``db_label``: once per process per DB, so
|
||||
state.db and kanban.db on one NFS mount each log once.
|
||||
"""
|
||||
from hermes_state import is_sqlite_wal_reset_vulnerable, resolve_journal_mode
|
||||
configured = resolve_journal_mode()
|
||||
|
||||
# Vulnerable SQLite: never enable WAL on new/non-WAL files. Resolve the
|
||||
# operator setting first so an explicit DELETE request still verifies SQLite
|
||||
# accepted DELETE rather than silently returning MEMORY or another mode.
|
||||
if is_sqlite_wal_reset_vulnerable():
|
||||
return _apply_delete_for_wal_reset_bug(
|
||||
conn,
|
||||
db_label=db_label,
|
||||
require_delete=configured == "delete",
|
||||
)
|
||||
|
||||
# Read-only probe — no flock, no checkpoint, no WAL/SHM unlink — so
|
||||
# WAL-init cannot unlink files other connections hold open.
|
||||
current_mode = _on_disk_journal_mode(conn)
|
||||
if current_mode == "wal":
|
||||
if configured == "delete":
|
||||
# Never-live-downgrade keeps WAL; tell the operator their delete did not apply.
|
||||
_log_configured_delete_overridden_once(db_label)
|
||||
_apply_wal_size_limit(conn)
|
||||
_apply_macos_checkpoint_barrier(conn)
|
||||
_enforce_macos_synchronous_full(conn)
|
||||
return "wal"
|
||||
|
||||
# Honor the canonical database.journal_mode setting (on-disk WAL DBs were
|
||||
# returned above and are never live-downgraded).
|
||||
if configured == "delete":
|
||||
if current_mode is None:
|
||||
# Probe failed (locked/busy): another process may hold this DB open
|
||||
# in WAL, so ownership is not provably exclusive and flipping modes
|
||||
# could destroy a concurrent writer's committed-but-uncheckpointed
|
||||
# transactions. Fail loudly — the operator asked for DELETE and we
|
||||
# cannot verify it.
|
||||
raise sqlite3.OperationalError(
|
||||
"could not verify journal mode before applying configured "
|
||||
"journal_mode=delete (database is locked — possible "
|
||||
"concurrent openers); refusing to downgrade a database "
|
||||
"this process does not exclusively own"
|
||||
)
|
||||
actual = _set_journal_mode_no_wait(conn, "DELETE")
|
||||
if actual != "delete":
|
||||
raise sqlite3.OperationalError(
|
||||
f"could not set configured journal_mode=delete (got {actual or 'no result'})"
|
||||
)
|
||||
return actual
|
||||
|
||||
# Decide BEFORE the flip whether it would overwrite a mode somebody chose:
|
||||
# the probe and page_count are only readable while the file is untouched.
|
||||
# A 0-page DB has no prior choice, and every caller reaches this before
|
||||
# creating schema, so brand-new databases stay quiet.
|
||||
_upgrading_existing_db = (
|
||||
current_mode is not None
|
||||
and current_mode != "wal"
|
||||
and _database_has_content(conn)
|
||||
)
|
||||
|
||||
try:
|
||||
# ``PRAGMA journal_mode=WAL`` RETURNS the resulting mode. Filesystems
|
||||
# that refuse by *raising* SQLITE_PROTOCOL hit the except branch, but
|
||||
# macOS NFS, SMB/CIFS and the AgentFS NFS overlay refuse WITHOUT raising
|
||||
# and just return the still-effective mode. Trust the row, not the
|
||||
# absence of an exception, or we report a false "wal", skip the
|
||||
# fallback ERROR, and leave the DB silently in DELETE.
|
||||
row = conn.execute("PRAGMA journal_mode=WAL").fetchone()
|
||||
mode = str(row[0]).strip().lower() if row and row[0] is not None else ""
|
||||
if mode == "wal":
|
||||
if _upgrading_existing_db:
|
||||
_log_journal_mode_upgrade_once(db_label, current_mode)
|
||||
_apply_wal_size_limit(conn)
|
||||
_apply_macos_checkpoint_barrier(conn)
|
||||
_enforce_macos_synchronous_full(conn)
|
||||
return "wal"
|
||||
# Silent refusal: WAL was not honored, but nothing raised.
|
||||
silent_exc = WalUnsupportedError(
|
||||
f"journal_mode=WAL refused without raising (still {mode!r})"
|
||||
)
|
||||
if require_wal:
|
||||
raise silent_exc
|
||||
_log_wal_fallback_once(db_label, silent_exc)
|
||||
return mode or "delete"
|
||||
except sqlite3.OperationalError as exc:
|
||||
# The require_wal silent-refusal raise above lands here (subclass of
|
||||
# OperationalError) — propagate unchanged, skip the marker logic.
|
||||
if isinstance(exc, WalUnsupportedError):
|
||||
raise
|
||||
msg = str(exc).lower()
|
||||
if not any(marker in msg for marker in _WAL_INCOMPAT_MARKERS):
|
||||
# Unrelated OperationalError — don't silently swallow.
|
||||
raise
|
||||
# ``disk i/o error`` is ambiguous: deterministic WAL-incompatibility on
|
||||
# ZFS / APFS-CoW (SHM corruption under connection bursts), or a one-shot
|
||||
# transient EIO (page-cache pressure, brief lock contention). Treating
|
||||
# a transient EIO as a permanent downgrade signal produced mixed-mode
|
||||
# corruption (process A downgrades to DELETE while siblings set WAL),
|
||||
# so retry the pragma: transient EIO clears and we return "wal";
|
||||
# deterministic cases keep failing into the guarded DELETE fallback.
|
||||
if "disk i/o error" in msg:
|
||||
for _ in range(2):
|
||||
time.sleep(0.05)
|
||||
try:
|
||||
row = conn.execute("PRAGMA journal_mode=WAL").fetchone()
|
||||
except sqlite3.OperationalError as retry_exc:
|
||||
if "disk i/o error" not in str(retry_exc).lower():
|
||||
raise
|
||||
exc = retry_exc
|
||||
continue
|
||||
mode = (
|
||||
str(row[0]).strip().lower()
|
||||
if row and row[0] is not None
|
||||
else ""
|
||||
)
|
||||
if mode == "wal":
|
||||
# Transient EIO cleared and the switch went through; same
|
||||
# header rewrite, so same upgrade signal.
|
||||
if _upgrading_existing_db:
|
||||
_log_journal_mode_upgrade_once(db_label, current_mode)
|
||||
_apply_wal_size_limit(conn)
|
||||
_apply_macos_checkpoint_barrier(conn)
|
||||
_enforce_macos_synchronous_full(conn)
|
||||
return "wal"
|
||||
break
|
||||
# Don't downgrade if another process already set WAL on disk, or if the
|
||||
# mode cannot be read (probe blocked by a concurrent opener's locks) —
|
||||
# ownership is not provably exclusive either way.
|
||||
existing = _on_disk_journal_mode(conn)
|
||||
if existing == "wal" or existing is None:
|
||||
raise
|
||||
if require_wal:
|
||||
raise WalUnsupportedError(str(exc)) from exc
|
||||
_log_wal_fallback_once(db_label, exc)
|
||||
_set_journal_mode_no_wait(conn, "DELETE")
|
||||
return "delete"
|
||||
|
||||
|
||||
def _set_journal_mode_no_wait(conn: sqlite3.Connection, mode: str) -> str:
|
||||
"""Execute ``PRAGMA journal_mode=<mode>`` without waiting on other openers.
|
||||
|
||||
The ONLY place a journal-mode switch may be issued for a non-WAL target.
|
||||
Forces ``busy_timeout=0`` so SQLite's exclusivity requirement becomes a
|
||||
concurrent-opener detector: leaving WAL needs exclusive access, so if ANY
|
||||
other connection (this process or another) holds the DB the pragma fails
|
||||
immediately with ``database is locked`` instead of waiting out a busy
|
||||
timeout and sneaking the flip between a concurrent writer's transactions —
|
||||
exactly how committed-but-uncheckpointed WAL transactions get destroyed.
|
||||
|
||||
Callers must treat a raised ``OperationalError`` as "not exclusively
|
||||
owned: leave the journal mode alone", never as retryable. Returns SQLite's
|
||||
reported mode (lowercase), or ``""`` if no row.
|
||||
"""
|
||||
previous_timeout = 0
|
||||
try:
|
||||
row = conn.execute("PRAGMA busy_timeout").fetchone()
|
||||
if row and row[0] is not None:
|
||||
previous_timeout = int(row[0])
|
||||
except (sqlite3.OperationalError, TypeError, ValueError):
|
||||
previous_timeout = 0
|
||||
conn.execute("PRAGMA busy_timeout=0")
|
||||
try:
|
||||
row = conn.execute(f"PRAGMA journal_mode={mode}").fetchone()
|
||||
return str(row[0]).strip().lower() if row and row[0] is not None else ""
|
||||
finally:
|
||||
try:
|
||||
conn.execute(f"PRAGMA busy_timeout={previous_timeout}")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
|
||||
|
||||
def _apply_delete_for_wal_reset_bug(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
db_label: str,
|
||||
require_delete: bool = False,
|
||||
) -> str:
|
||||
"""Avoid enabling WAL when the linked SQLite has the WAL-reset bug.
|
||||
|
||||
- Already-WAL on disk: leave WAL alone (no live downgrade) and warn.
|
||||
- Mode unreadable (probe blocked by a concurrent opener's locks): not
|
||||
provably exclusive — leave the mode alone and warn. Never treat "could
|
||||
not read the mode" as "not WAL": that confusion once flipped a live WAL
|
||||
state.db to DELETE under a concurrent writer, destroying its
|
||||
committed-but-uncheckpointed transactions.
|
||||
- Otherwise: set DELETE (refusing to wait out concurrent openers) and warn.
|
||||
- For an explicit operator request, verify SQLite accepted DELETE.
|
||||
"""
|
||||
current = _on_disk_journal_mode(conn)
|
||||
|
||||
if current == "wal":
|
||||
_log_wal_reset_bug_once(db_label, kept_wal=True)
|
||||
if require_delete:
|
||||
# Upgrading SQLite (the warning above) doesn't help on a
|
||||
# WAL-incompatible filesystem; emit the actionable message last.
|
||||
_log_configured_delete_overridden_once(db_label)
|
||||
# No TRUNCATE / journal_mode=DELETE while other processes may still
|
||||
# hold this WAL DB open; same safety rule as the NFS path.
|
||||
_apply_wal_size_limit(conn)
|
||||
_apply_macos_checkpoint_barrier(conn)
|
||||
_enforce_macos_synchronous_full(conn)
|
||||
return "wal"
|
||||
|
||||
if current is None:
|
||||
# Probe failed — likely another opener's locks, and the DB may be in
|
||||
# WAL under a live writer. Never flip a mode we cannot even read.
|
||||
if require_delete:
|
||||
raise sqlite3.OperationalError(
|
||||
"could not verify journal mode before applying configured "
|
||||
"journal_mode=delete (database is locked — possible "
|
||||
"concurrent openers); refusing to downgrade a database "
|
||||
"this process does not exclusively own"
|
||||
)
|
||||
_log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True)
|
||||
return "wal"
|
||||
|
||||
actual = ""
|
||||
try:
|
||||
actual = _set_journal_mode_no_wait(conn, "DELETE")
|
||||
except sqlite3.OperationalError as exc:
|
||||
if require_delete:
|
||||
raise
|
||||
lowered = str(exc).lower()
|
||||
if "locked" in lowered or "busy" in lowered:
|
||||
# A concurrent opener appeared between probe and flip (or already
|
||||
# held the DB): SQLite refused the exclusive lock. Leave the mode as is.
|
||||
_log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True)
|
||||
return current or "delete"
|
||||
# Best-effort for the automatic fallback: DELETE is normally already
|
||||
# the default for new file-backed databases.
|
||||
if require_delete and actual != "delete":
|
||||
raise sqlite3.OperationalError(
|
||||
"could not set configured journal_mode=delete "
|
||||
f"(got {actual or 'no result'})"
|
||||
)
|
||||
_log_wal_reset_bug_once(db_label, kept_wal=False)
|
||||
return "delete"
|
||||
|
||||
|
||||
def _wal_reset_repair_hint() -> str:
|
||||
"""Repair hint matching what ``hermes update`` can actually do for this
|
||||
install type (uv-managed venv vs git/pip/docker/nix)."""
|
||||
try:
|
||||
from hermes_cli.config import (
|
||||
detect_install_method,
|
||||
recommended_update_command_for_method,
|
||||
get_project_root,
|
||||
)
|
||||
method = detect_install_method(get_project_root())
|
||||
cmd = recommended_update_command_for_method(method)
|
||||
if method in {"git", "unknown"}:
|
||||
return f"Hermes-managed installs can repair the embedded runtime with `{cmd}`"
|
||||
if method == "docker":
|
||||
return f"update the container image with `{cmd}`"
|
||||
# nix/nixos
|
||||
return cmd
|
||||
except Exception:
|
||||
pass
|
||||
return (
|
||||
"install a Python build bundled with SQLite 3.51.3+ "
|
||||
"(or backports 3.50.7 / 3.44.6) and restart Hermes"
|
||||
)
|
||||
|
||||
|
||||
# Dedup state for _log_journal_mode_upgrade_once.
|
||||
_journal_upgrade_warned_paths: set = set()
|
||||
|
||||
|
||||
_journal_upgrade_warned_lock = threading.Lock()
|
||||
|
||||
|
||||
def _log_wal_reset_bug_once(
|
||||
db_label: str,
|
||||
*,
|
||||
kept_wal: bool,
|
||||
indeterminate: bool = False,
|
||||
) -> None:
|
||||
"""Log once per (process, db_label) about the WAL-reset vulnerability path."""
|
||||
from hermes_state import _wal_reset_bug_warned_paths
|
||||
with _wal_reset_bug_warned_lock:
|
||||
if db_label in _wal_reset_bug_warned_paths:
|
||||
return
|
||||
_wal_reset_bug_warned_paths.add(db_label)
|
||||
if indeterminate:
|
||||
action = (
|
||||
"journal mode could not be verified or exclusively switched "
|
||||
"(database is locked — possible concurrent openers); leaving the "
|
||||
"journal mode untouched (no live downgrade under concurrent "
|
||||
"openers)"
|
||||
)
|
||||
elif kept_wal:
|
||||
action = (
|
||||
"is already in WAL mode — leaving WAL in place (no live "
|
||||
"downgrade under concurrent openers)"
|
||||
)
|
||||
else:
|
||||
action = "using journal_mode=DELETE instead of enabling WAL"
|
||||
# Install-type-aware so the warning never promises a repair path that
|
||||
# doesn't exist for git/pip/system Python installs.
|
||||
repair_hint = _wal_reset_repair_hint()
|
||||
logger.warning(
|
||||
"%s: linked SQLite %s (interpreter %s) is vulnerable to the WAL-reset "
|
||||
"corruption bug (https://sqlite.org/wal.html#walresetbug) — %s. "
|
||||
"Upgrade to SQLite 3.51.3+ (or backports 3.50.7 / 3.44.6); "
|
||||
"%s. See `hermes doctor`. This warning fires once per "
|
||||
"process per database.",
|
||||
db_label,
|
||||
sqlite3.sqlite_version,
|
||||
sys.executable,
|
||||
action,
|
||||
repair_hint,
|
||||
)
|
||||
|
||||
|
||||
def _log_journal_mode_upgrade_once(db_label: str, previous_mode: str) -> None:
|
||||
"""Log a single WARNING per (process, db_label) about a non-WAL -> WAL flip.
|
||||
|
||||
``PRAGMA journal_mode`` is a property of the FILE: switching an existing DB
|
||||
to WAL rewrites its header and outlives the process. Operators do set
|
||||
DELETE on the file directly (the documented WAL-reset-bug mitigation), and
|
||||
nothing told them the next open would silently put WAL back.
|
||||
|
||||
WARNING, not ERROR: the reverse move is ERROR in ``_log_wal_fallback_once``
|
||||
because dropping to DELETE loses concurrency, whereas this direction is
|
||||
normally desirable (managed_uv repairs DELETE-stuck DBs on update). The
|
||||
only problem was invisibility, so this names the durable setting without
|
||||
claiming a degradation. Deduped per process per ``db_label`` because
|
||||
kanban opens a fresh connection per operation.
|
||||
"""
|
||||
from hermes_state import _journal_upgrade_warned_paths
|
||||
with _journal_upgrade_warned_lock:
|
||||
if db_label in _journal_upgrade_warned_paths:
|
||||
return
|
||||
_journal_upgrade_warned_paths.add(db_label)
|
||||
logger.warning(
|
||||
"%s: on-disk journal_mode was %s and has been switched to WAL. This "
|
||||
"rewrites the database header and persists after this process exits. "
|
||||
"If %s was a deliberate choice (for example the mitigation for the "
|
||||
"SQLite WAL-reset bug, or a WAL-unsafe filesystem), setting it with "
|
||||
"PRAGMA on the file will not survive -- every open re-applies the "
|
||||
"configured mode. Set `database.journal_mode: delete` in config.yaml "
|
||||
"to make it stick. This message fires once per process per database.",
|
||||
db_label,
|
||||
previous_mode,
|
||||
previous_mode,
|
||||
)
|
||||
|
||||
|
||||
def _log_wal_fallback_once(db_label: str, exc: Exception) -> None:
|
||||
"""Log a single ERROR per (process, db_label) about WAL fallback.
|
||||
|
||||
ERROR, not WARNING: silently dropping to DELETE is a real concurrency loss
|
||||
(under kanban dispatcher + workers a write blocks readers as SQLITE_BUSY).
|
||||
Deduped because kanban opens a fresh connection per operation.
|
||||
"""
|
||||
from hermes_state import _wal_fallback_warned_paths
|
||||
with _wal_fallback_warned_lock:
|
||||
if db_label in _wal_fallback_warned_paths:
|
||||
return
|
||||
_wal_fallback_warned_paths.add(db_label)
|
||||
logger.error(
|
||||
"%s: WAL journal_mode unsupported on this filesystem (%s) — "
|
||||
"falling back to journal_mode=DELETE (slower rollback-journal "
|
||||
"mode; reduces concurrency but works on NFS/SMB/FUSE/ZFS). See "
|
||||
"https://www.sqlite.org/wal.html for details. This message "
|
||||
"fires once per process per database.",
|
||||
db_label,
|
||||
exc,
|
||||
)
|
||||
|
||||
|
||||
def _log_configured_delete_overridden_once(db_label: str) -> None:
|
||||
"""Log a single ERROR per (process, db_label) when the operator configured
|
||||
``journal_mode=delete`` but the on-disk DB is already WAL.
|
||||
|
||||
Never-live-downgrade keeps WAL (a live downgrade causes mixed-mode
|
||||
corruption); without this the operator would never learn that
|
||||
``database.journal_mode: delete`` had no effect and that a one-time
|
||||
offline ``PRAGMA journal_mode=DELETE`` (no open connections) is required.
|
||||
"""
|
||||
from hermes_state import _delete_overridden_warned_paths
|
||||
with _delete_overridden_warned_lock:
|
||||
if db_label in _delete_overridden_warned_paths:
|
||||
return
|
||||
_delete_overridden_warned_paths.add(db_label)
|
||||
logger.error(
|
||||
"%s: database.journal_mode=delete is configured but the on-disk "
|
||||
"database is already WAL; keeping WAL (a live downgrade under open "
|
||||
"connections can corrupt the DB). To apply journal_mode=DELETE, stop "
|
||||
"all connections to this DB and run a one-time offline "
|
||||
"'PRAGMA journal_mode=DELETE' on the file. This message fires once "
|
||||
"per process per database.",
|
||||
db_label,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config-driven database pragmas
|
||||
# ---------------------------------------------------------------------------
|
||||
# Operators write synchronous as a name; mapped here rather than passed through
|
||||
# so a typo becomes a warning instead of a silently different durability level.
|
||||
_SYNCHRONOUS_LEVELS: Dict[str, int] = {
|
||||
"OFF": 0,
|
||||
"NORMAL": 1,
|
||||
"FULL": 2,
|
||||
"EXTRA": 3,
|
||||
}
|
||||
|
||||
|
||||
_SYNCHRONOUS_NAMES: Dict[int, str] = {v: k for k, v in _SYNCHRONOUS_LEVELS.items()}
|
||||
|
||||
|
||||
_SYNCHRONOUS_FULL = 2
|
||||
|
||||
|
||||
def resolve_synchronous_level(raw_value: Any) -> Optional[int]:
|
||||
"""Map a configured ``database.synchronous`` value to its PRAGMA integer.
|
||||
|
||||
Accepts SQLite's names (``OFF``/``NORMAL``/``FULL``/``EXTRA``, any case) or
|
||||
``0``-``3``. Anything else returns None so the caller warns and leaves the
|
||||
level untouched — guessing at a malformed durability setting is worse.
|
||||
"""
|
||||
if isinstance(raw_value, bool):
|
||||
# bool is an int subclass and YAML turns bare `on`/`off` into one.
|
||||
# "off" is a real durability choice; True is meaningless.
|
||||
return 0 if raw_value is False else None
|
||||
if isinstance(raw_value, int):
|
||||
return raw_value if raw_value in _SYNCHRONOUS_NAMES else None
|
||||
text = str(raw_value).strip()
|
||||
if not text:
|
||||
return None
|
||||
upper = text.upper()
|
||||
if upper in _SYNCHRONOUS_LEVELS:
|
||||
return _SYNCHRONOUS_LEVELS[upper]
|
||||
try:
|
||||
value = int(text)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return value if value in _SYNCHRONOUS_NAMES else None
|
||||
|
||||
|
||||
def _apply_synchronous_pragma(
|
||||
conn: sqlite3.Connection,
|
||||
raw_value: Any,
|
||||
*,
|
||||
db_label: str,
|
||||
) -> None:
|
||||
"""Set ``PRAGMA synchronous`` from config, never below FULL on macOS.
|
||||
|
||||
Kept out of the integer loop in :func:`apply_database_pragmas`: this PRAGMA
|
||||
decides whether a commit is on the platter, so an unrecognised value must
|
||||
not fall through to "SQLite default" the way a bad ``cache_size`` can.
|
||||
|
||||
Darwin floor: :func:`_enforce_macos_synchronous_full` runs during
|
||||
``apply_wal_with_fallback()`` and this runs after it, so a configured
|
||||
``NORMAL`` would otherwise silently undo the macOS btree protection.
|
||||
Raising the level on macOS is allowed; lowering it is refused out loud.
|
||||
"""
|
||||
level = resolve_synchronous_level(raw_value)
|
||||
if level is None:
|
||||
logger.warning(
|
||||
"%s: ignoring unrecognized database.synchronous=%r "
|
||||
"(expected OFF, NORMAL, FULL, EXTRA, or 0-3)",
|
||||
db_label,
|
||||
raw_value,
|
||||
)
|
||||
return
|
||||
if sys.platform == "darwin" and level < _SYNCHRONOUS_FULL:
|
||||
logger.warning(
|
||||
"%s: refusing database.synchronous=%s on macOS; keeping FULL. "
|
||||
"Darwin's fsync() does not guarantee write ordering, so a lower "
|
||||
"level readmits the half-written btree pages FULL exists to "
|
||||
"prevent.",
|
||||
db_label,
|
||||
_SYNCHRONOUS_NAMES[level],
|
||||
)
|
||||
return
|
||||
try:
|
||||
conn.execute(f"PRAGMA synchronous={level}")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
|
||||
|
||||
def apply_database_pragmas(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
db_label: str = "state.db",
|
||||
) -> None:
|
||||
"""Apply optional performance and WAL-sizing PRAGMAs from ``config.yaml``.
|
||||
|
||||
Journal mode is NOT handled here — ``database.journal_mode`` is owned by
|
||||
:func:`resolve_journal_mode` inside :func:`apply_wal_with_fallback`, under
|
||||
all the safety guards.
|
||||
|
||||
Keys under ``database:``: ``cache_size`` (negative = KiB, positive =
|
||||
pages), ``mmap_size`` (bytes, 0 = disabled), ``temp_store`` (0-3),
|
||||
``wal_autocheckpoint`` (pages), ``journal_size_limit`` (bytes), and
|
||||
``synchronous`` (``OFF``/``NORMAL``/``FULL``/``EXTRA`` or ``0``-``3``).
|
||||
Unset ``synchronous`` leaves SQLite's default, a *compile-time* constant
|
||||
(``SQLITE_DEFAULT_WAL_SYNCHRONOUS``) that differs between bundled, distro
|
||||
and Homebrew builds; setting it explicitly is the only way to know.
|
||||
|
||||
Best-effort: config load or pragma failures are ignored so DB init never
|
||||
breaks on a malformed ``database:`` section.
|
||||
"""
|
||||
try:
|
||||
# Local import avoids a circular import with hermes_cli.config.
|
||||
from hermes_cli.config import cfg_get, load_config_readonly
|
||||
|
||||
cfg = load_config_readonly()
|
||||
except Exception:
|
||||
return
|
||||
|
||||
# Applied to ALL connection types: writer, read_only, WAL per-thread readers.
|
||||
for pragma_name in (
|
||||
"cache_size",
|
||||
"mmap_size",
|
||||
"temp_store",
|
||||
"wal_autocheckpoint",
|
||||
"journal_size_limit",
|
||||
):
|
||||
raw_value = cfg_get(cfg, "database", pragma_name, default=None)
|
||||
if raw_value is None:
|
||||
continue
|
||||
try:
|
||||
value = int(str(raw_value).strip())
|
||||
except (TypeError, ValueError):
|
||||
logger.warning(
|
||||
"%s: ignoring non-integer database.%s=%r",
|
||||
db_label,
|
||||
pragma_name,
|
||||
raw_value,
|
||||
)
|
||||
continue
|
||||
try:
|
||||
conn.execute(f"PRAGMA {pragma_name}={value}")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
|
||||
# Last: the sizing pragmas above cannot change durability, and the macOS
|
||||
# enforcement ran earlier during WAL activation (see _apply_synchronous_pragma
|
||||
# for why that ordering needs an explicit floor rather than an override).
|
||||
raw_synchronous = cfg_get(cfg, "database", "synchronous", default=None)
|
||||
if raw_synchronous is not None:
|
||||
_apply_synchronous_pragma(conn, raw_synchronous, db_label=db_label)
|
||||
@@ -102,8 +102,12 @@ def test_repair_path_has_no_bare_connects() -> None:
|
||||
Source-level guard: the bare form is exactly what regressed, and a unit
|
||||
test on the helper alone would not notice a sixth site being added.
|
||||
"""
|
||||
source = Path(hermes_state.__file__).read_text(encoding="utf-8")
|
||||
tree = ast.parse(source, filename=str(hermes_state.__file__))
|
||||
# The repair/probe helpers live in hermes_state_repair; hermes_state only
|
||||
# re-imports them.
|
||||
import hermes_state_repair
|
||||
|
||||
source = Path(hermes_state_repair.__file__).read_text(encoding="utf-8")
|
||||
tree = ast.parse(source, filename=str(hermes_state_repair.__file__))
|
||||
|
||||
def is_db_path_connect(node: ast.AST) -> bool:
|
||||
if not isinstance(node, ast.Call):
|
||||
|
||||
Reference in New Issue
Block a user