Merge branch 'simp/state' into simp/integration

This commit is contained in:
Teknium
2026-09-02 14:19:40 -07:00
25 changed files with 14086 additions and 18392 deletions
+193 -289
View File
@@ -107,11 +107,8 @@ def active_session_limit_message(
max_sessions: int,
entries: Optional[list[dict[str, Any]]] = None,
) -> str:
# Name the holders: the slots are shared across CLI, desktop/TUI and the
# messaging gateway, so the surface that gets rejected is usually NOT the
# one squatting on them (idle desktop chats starving a Discord bot, say).
# Without this the message is unactionable and the only way to find out is
# reading runtime/active_sessions.json by hand.
# Name the holders: slots are shared across CLI, desktop/TUI and gateway,
# so the rejected surface is usually NOT the one squatting on them.
held = summarize_holders(entries or [])
detail = f" Held by: {held}." if held else ""
return (
@@ -124,42 +121,25 @@ def _registry_home(registry_home: str | Path | None = None) -> Path:
return Path(registry_home) if registry_home is not None else Path(get_hermes_home())
# WHY A REFUSAL IS REFUSED, in a form a caller can branch on.
#
# The two refusals mean opposite things to an automated client. Capacity is
# "the machine is busy, come back later". Ownership is "this specific session
# has a live owner, and writing to it would interleave with theirs".
#
# Callers used to have only the human-readable message, so anything that needed
# to DECIDE had to match prose -- which silently changes meaning whenever the
# wording is improved. The reason is the contract; the message is for people.
# Machine-readable refusal reasons: the reason is the contract, the message is
# for people. Capacity = "busy, come back later"; ownership = "this session
# has a live owner and writing would interleave with theirs".
SESSION_NOT_OWNED = "SESSION_NOT_OWNED"
MAX_CONCURRENT_SESSIONS = "MAX_CONCURRENT_SESSIONS"
# Ownership could not be PROVEN either way: the registry was unreadable or
# corrupt. Distinct from SESSION_NOT_OWNED on purpose -- "someone else owns
# this" and "I cannot tell who owns this" call for different operator action,
# and collapsing the second into a silent go-ahead is exactly the fail-open
# hole that let two writers share one session (#94595 review, blocker 2).
# Ownership could not be PROVEN (registry unreadable/corrupt). Deliberately
# distinct from SESSION_NOT_OWNED: collapsing "can't tell" into a silent
# go-ahead is the fail-open hole that let two writers share one session.
SESSION_COORDINATION_UNAVAILABLE = "SESSION_COORDINATION_UNAVAILABLE"
# Advertised through the gateway so a client can tell a build that enforces
# per-session exclusivity from one that does not.
#
# A module constant rather than a config flag, deliberately: it is true because
# try_acquire_active_session below performs the check atomically, so it cannot
# be turned on by an operator who has not got the enforcement, and cannot drift
# out of step with it without this file changing.
# Advertised through the gateway. A module constant, not a config flag: it is
# true because try_acquire_active_session checks atomically, so it cannot drift
# out of step with the enforcement without this file changing.
PER_SESSION_EXCLUSIVE_SUBMIT = True
class ActiveSessionRefusal(str):
"""A refusal message that also carries a machine-readable ``reason``.
A ``str`` subclass so every existing caller keeps working untouched -- they
format it, hand it back as a JSON-RPC message, or just test it for None --
while a caller that must act on WHICH refusal happened reads ``.reason``
instead of matching the wording.
"""
"""A refusal message (``str`` subclass, so existing callers are untouched)
that also carries a machine-readable ``reason``."""
reason: str
@@ -170,13 +150,10 @@ class ActiveSessionRefusal(str):
def _is_same_writer(entry: dict[str, Any], metadata: Optional[dict[str, Any]]) -> bool:
"""True when an existing lease belongs to the very caller now re-acquiring it.
Both halves are required. A pid alone would let two live sessions in one
process steal each other's lease -- which is a real hazard, since each holds
its own snapshot of the transcript. A live session id alone would let another
process with a coincidentally equal id do the same.
"""
"""True when an existing lease belongs to the very caller re-acquiring it.
Identity is (pid, live_session_id): pid alone would let two live sessions in
one process steal each other's lease; the live id alone would let another
process with an equal id do the same."""
try:
if int(entry.get("pid") or -1) != os.getpid():
return False
@@ -223,6 +200,19 @@ def _lease_paths(
return _state_path(home), _lock_path(home)
def _flock(fh, *, lock: bool) -> None:
"""Exclusive whole-file lock/unlock on ``fh`` (fcntl on POSIX, msvcrt on Windows)."""
if os.name == "nt":
import msvcrt
fh.seek(0)
msvcrt.locking(fh.fileno(), msvcrt.LK_LOCK if lock else msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(fh.fileno(), fcntl.LOCK_EX if lock else fcntl.LOCK_UN)
class _FileLock:
def __init__(self, path: Path):
self.path = path
@@ -231,45 +221,21 @@ class _FileLock:
def __enter__(self):
self.path.parent.mkdir(parents=True, exist_ok=True)
self._fh = open(self.path, "a+b")
if os.name == "nt":
try:
import msvcrt
self._fh.seek(0)
msvcrt.locking(self._fh.fileno(), msvcrt.LK_LOCK, 1)
except Exception as exc:
self._fh.close()
self._fh = None
raise RuntimeError("active session file lock unavailable") from exc
else:
try:
import fcntl
fcntl.flock(self._fh.fileno(), fcntl.LOCK_EX)
except Exception as exc:
self._fh.close()
self._fh = None
raise RuntimeError("active session file lock unavailable") from exc
try:
_flock(self._fh, lock=True)
except Exception as exc:
self._fh.close()
self._fh = None
raise RuntimeError("active session file lock unavailable") from exc
return self
def __exit__(self, exc_type, exc, tb):
if self._fh is None:
return
if os.name == "nt":
try:
import msvcrt
self._fh.seek(0)
msvcrt.locking(self._fh.fileno(), msvcrt.LK_UNLCK, 1)
except Exception:
pass
else:
try:
import fcntl
fcntl.flock(self._fh.fileno(), fcntl.LOCK_UN)
except Exception:
pass
try:
_flock(self._fh, lock=False)
except Exception:
pass
try:
self._fh.close()
finally:
@@ -306,58 +272,51 @@ def _read_entries(path: Path, *, strict: bool = False) -> list[dict[str, Any]]:
seen_leases: set[str] = set()
for entry in valid:
lease_id = entry.get("lease_id")
session_id = entry.get("session_id")
pid = entry.get("pid")
if not isinstance(lease_id, str) or not lease_id.strip():
raise ActiveSessionRegistryError(
f"active session registry contains an invalid lease id: {path}"
)
if lease_id in seen_leases:
raise ActiveSessionRegistryError(
f"active session registry contains a duplicate lease id: {path}"
)
seen_leases.add(lease_id)
if not isinstance(session_id, str) or not session_id.strip():
raise ActiveSessionRegistryError(
f"active session registry contains an invalid session id: {path}"
)
if isinstance(pid, bool) or not isinstance(pid, (int, str)):
pid_int = 0
else:
try:
pid_int = int(pid)
except (TypeError, ValueError):
pid_int = 0
if pid_int <= 0:
raise ActiveSessionRegistryError(
f"active session registry contains an invalid pid: {path}"
)
surface = entry.get("surface")
if surface is not None and not isinstance(surface, str):
raise ActiveSessionRegistryError(
f"active session registry contains an invalid surface: {path}"
)
tracked = entry.get("track_liveness")
if tracked is not None and not isinstance(tracked, bool):
raise ActiveSessionRegistryError(
f"active session registry contains an invalid liveness marker: {path}"
)
metadata = entry.get("metadata")
if metadata is not None and not isinstance(metadata, dict):
raise ActiveSessionRegistryError(
f"active session registry contains invalid metadata: {path}"
)
process_start = entry.get("process_start_time")
parsed_process_start = _optional_float(process_start)
if process_start not in (None, "") and (
parsed_process_start is None or not math.isfinite(parsed_process_start)
# (problem-if-True predicate, message fragment) — checked lazily, in
# this order, so an unhashable lease id is reported before the dup check.
for bad, what in (
(lambda: not _nonblank_str(lease_id), "an invalid lease id"),
(lambda: lease_id in seen_leases, "a duplicate lease id"),
(lambda: not _nonblank_str(entry.get("session_id")), "an invalid session id"),
(lambda: _registry_pid(entry.get("pid")) <= 0, "an invalid pid"),
(lambda: not _optional_isinstance(entry.get("surface"), str), "an invalid surface"),
(lambda: not _optional_isinstance(entry.get("track_liveness"), bool), "an invalid liveness marker"),
(lambda: not _optional_isinstance(entry.get("metadata"), dict), "invalid metadata"),
(lambda: not _valid_process_start(entry.get("process_start_time")), "an invalid process start time"),
):
raise ActiveSessionRegistryError(
f"active session registry contains an invalid process start time: {path}"
)
if bad():
raise ActiveSessionRegistryError(
f"active session registry contains {what}: {path}"
)
seen_leases.add(lease_id)
return valid
def _nonblank_str(v: Any) -> bool:
return isinstance(v, str) and bool(v.strip())
def _optional_isinstance(v: Any, typ) -> bool:
return v is None or isinstance(v, typ)
def _registry_pid(pid: Any) -> int:
"""Registry pid as int; 0 for bools, non-int/str, or unparseable values."""
if isinstance(pid, bool) or not isinstance(pid, (int, str)):
return 0
try:
return int(pid)
except (TypeError, ValueError):
return 0
def _valid_process_start(v: Any) -> bool:
if v in (None, ""):
return True
parsed = _optional_float(v)
return parsed is not None and math.isfinite(parsed)
def _write_entries(path: Path, entries: list[dict[str, Any]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(f"{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
@@ -392,20 +351,24 @@ def _optional_float(value: Any) -> Optional[float]:
return None
def _pid_liveness(pid: Any, process_start_time: Any = None) -> Optional[bool]:
"""Return True/False for live/dead, or None when liveness is unknowable."""
def _pid_liveness(pid: Any, process_start_time: Any = None, *, lenient: bool = False) -> Optional[bool]:
"""Return True/False for live/dead, or None when liveness is unknowable.
``lenient`` never returns None: an unparseable pid or a failed existence
probe counts as dead, an unreadable current start time as alive.
"""
try:
pid_int = int(pid)
except (TypeError, ValueError):
return None
return False if lenient else None
if pid_int <= 0:
return None
return False if lenient else None
try:
from gateway.status import _pid_exists
exists = bool(_pid_exists(pid_int))
except Exception:
return None
return False if lenient else None
if not exists:
return False
expected_start = _optional_float(process_start_time)
@@ -413,32 +376,12 @@ def _pid_liveness(pid: Any, process_start_time: Any = None) -> Optional[bool]:
return True
current_start = _process_start_time(pid_int)
if current_start is None:
return None
return True if lenient else None
return abs(current_start - expected_start) < 0.001
def _pid_alive(pid: Any, process_start_time: Any = None) -> bool:
try:
pid_int = int(pid)
except (TypeError, ValueError):
return False
if pid_int <= 0:
return False
try:
from gateway.status import _pid_exists
exists = bool(_pid_exists(pid_int))
except Exception:
return False
if not exists:
return False
expected_start = _optional_float(process_start_time)
if expected_start is None:
return True
current_start = _process_start_time(pid_int)
if current_start is None:
return True
return abs(current_start - expected_start) < 0.001
return bool(_pid_liveness(pid, process_start_time, lenient=True))
def _prune_dead(
@@ -468,12 +411,9 @@ class ActiveSessionLease:
surface: str
enabled: bool = True
released: bool = False
# Registry paths pinned at acquisition time. A lease acquired under the
# root ``HERMES_HOME`` must release against the same registry even when
# ``release()`` runs inside a profile home override (native multiplex
# routes turns under ``_profile_runtime_scope``), otherwise the root
# entry survives until process exit and the session cap fills with
# phantom leases (#85431).
# Pinned at acquisition: a lease acquired under the root HERMES_HOME must
# release against the same registry even when release() runs inside a
# profile-home override, or phantom leases fill the session cap.
state_path: Optional[Path] = None
lock_path: Optional[Path] = None
track_liveness: bool = False
@@ -484,6 +424,33 @@ class ActiveSessionLease:
release_active_session(self)
def _clean_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
return {str(k): v for k, v in metadata.items() if isinstance(k, str)}
def _without_lease(entries: list[dict[str, Any]], lease_id: str) -> list[dict[str, Any]]:
return [e for e in entries if str(e.get("lease_id") or "") != lease_id]
def _read_live_entries(
state_path: Path, *, track_liveness: bool, warn: str,
) -> Optional[tuple[list[dict[str, Any]], list[dict[str, Any]]]]:
"""``(raw, pruned)`` from the registry, or None when it is unreadable.
Liveness-tracked callers re-raise instead: they must not proceed on an
unprovable registry. Untracked callers get ``warn`` logged and decide
how to degrade themselves.
"""
try:
raw_entries = _read_entries(state_path, strict=True)
return raw_entries, _prune_dead(raw_entries, strict=track_liveness)
except ActiveSessionRegistryError:
if track_liveness:
raise
logger.warning(warn)
return None
def _lease_entry(
*,
lease_id: str,
@@ -505,9 +472,7 @@ def _lease_entry(
if track_liveness:
entry["track_liveness"] = True
if metadata:
entry["metadata"] = {
str(k): v for k, v in metadata.items() if isinstance(k, str)
}
entry["metadata"] = _clean_metadata(metadata)
return entry
@@ -522,27 +487,20 @@ def try_acquire_active_session(
) -> tuple[Optional[ActiveSessionLease], Optional[str]]:
"""Acquire an active-session slot.
Per-session exclusivity is CORRECTNESS and is enforced unconditionally:
at most one live owner may run a given stored session, whether or not an
operator configured ``max_concurrent_sessions`` (#94595). The concurrency
cap remains a resource POLICY and applies only when configured. Liveness
tracking keeps richer desktop lifecycle semantics; ``registry_home`` lets
profile-scoped backends share the owning profile's registry even when
launched from another home.
Per-session exclusivity is CORRECTNESS, enforced unconditionally: at most
one live owner per stored session. ``max_concurrent_sessions`` is resource
POLICY and applies only when configured. ``registry_home`` lets
profile-scoped backends share the owning profile's registry.
Returns ``(lease, None)`` on success and ``(None, refusal)`` otherwise,
where ``refusal`` is an :class:`ActiveSessionRefusal` carrying a
machine-readable ``reason``. Ownership uncertainty fails CLOSED: when the
registry cannot be read, the caller gets ``SESSION_COORDINATION_UNAVAILABLE``
rather than a silent go-ahead that could reopen the double-writer state.
Returns ``(lease, None)`` or ``(None, ActiveSessionRefusal)``. Ownership
uncertainty fails CLOSED with ``SESSION_COORDINATION_UNAVAILABLE``.
"""
max_sessions = resolve_max_concurrent_sessions(config)
lease_id = uuid.uuid4().hex
key = str(session_id or "")
# A session with no stored id yet cannot collide with another writer, and
# the strict registry schema (rightly) refuses entries with empty session
# ids. Nothing to fence, nothing to record: hand back a no-op lease.
# No stored id yet => nothing to fence or record (and the strict schema
# refuses empty session ids): hand back a no-op lease.
if not key and not track_liveness:
return ActiveSessionLease(
lease_id=lease_id,
@@ -560,22 +518,23 @@ def try_acquire_active_session(
)
state_path, lock_path = _lease_paths(registry_home=registry_home)
lease = ActiveSessionLease(
lease_id=lease_id,
session_id=key,
surface=str(surface),
state_path=state_path,
lock_path=lock_path,
track_liveness=track_liveness,
)
with _FileLock(lock_path):
try:
raw_entries = _read_entries(state_path, strict=True)
entries = _prune_dead(raw_entries, strict=track_liveness)
except ActiveSessionRegistryError:
if track_liveness:
raise
# A capacity cap could afford to degrade open -- worst case, more
# sessions than the operator wanted. Exclusivity cannot: "could not
# prove ownership" must never be collapsed into "no owner exists",
# because that silently reopens the exact double-writer state this
# fence guarantees against. Refuse, and say which file to fix.
logger.warning(
"Active-session registry is unavailable; refusing the session "
"rather than risking a concurrent writer"
)
# A capacity cap could degrade open; exclusivity cannot: "could not
# prove ownership" must never become "no owner exists".
loaded = _read_live_entries(
state_path, track_liveness=track_liveness,
warn="Active-session registry is unavailable; refusing the session "
"rather than risking a concurrent writer",
)
if loaded is None:
return None, ActiveSessionRefusal(
(
"Hermes could not read the active-session registry at "
@@ -584,47 +543,27 @@ def try_acquire_active_session(
),
SESSION_COORDINATION_UNAVAILABLE,
)
raw_entries, entries = loaded
pruned = len(raw_entries) - len(entries)
if pruned:
logger.info("Pruned %d stale active session lease(s)", pruned)
# Correctness first, and under the same lock that just pruned the dead
# owners -- so an owner that died is never mistaken for one that is
# running, and a live one is never overlooked.
#
# An empty key is exempt: a session with no stored id yet cannot collide
# with another, and treating "" as an identity would make every unsaved
# draft exclude every other one.
# Correctness first, under the same lock that just pruned dead owners.
# An empty key is exempt: treating "" as an identity would make every
# unsaved draft exclude every other one.
if key:
for index, existing in enumerate(entries):
if str(existing.get("session_id") or "") != key:
continue
# THE SAME WRITER IS NOT A SECOND WRITER.
#
# A live session that lost its lease reference -- its record was
# rebuilt in place, so the object holding the lease is unreachable
# while the session itself is still the one being driven -- would
# otherwise be fenced out of its own session by its own leak, and
# permanently: pruning only removes entries whose PROCESS is dead,
# and this process is very much alive.
#
# Identity here is (pid, live session id). Two processes never
# match, because their pids differ. Two live sessions inside one
# process never match, because their live ids differ. Only the
# exact same writer re-acquiring its own session matches, and that
# is re-entrancy rather than a concurrent writer.
# The same writer is not a second writer: a live session that
# leaked its lease reference would otherwise be fenced out of
# its own session permanently (pruning only removes entries
# whose PROCESS is dead). Re-entrancy, not concurrency.
if _is_same_writer(existing, metadata):
entries[index] = entry
_write_entries(state_path, entries)
return ActiveSessionLease(
lease_id=lease_id,
session_id=key,
surface=str(surface),
state_path=state_path,
lock_path=lock_path,
track_liveness=track_liveness,
), None
return lease, None
_write_entries(state_path, entries)
logger.info(
@@ -656,14 +595,7 @@ def try_acquire_active_session(
entries.append(entry)
_write_entries(state_path, entries)
return ActiveSessionLease(
lease_id=lease_id,
session_id=key,
surface=str(surface),
state_path=state_path,
lock_path=lock_path,
track_liveness=track_liveness,
), None
return lease, None
def release_active_session(lease: ActiveSessionLease) -> None:
@@ -673,25 +605,16 @@ def release_active_session(lease: ActiveSessionLease) -> None:
with _FileLock(lock_path):
if lease.released:
return
try:
raw_entries = _read_entries(state_path, strict=True)
entries = _prune_dead(raw_entries, strict=lease.track_liveness)
except ActiveSessionRegistryError:
if lease.track_liveness:
raise
logger.warning(
"Active-session registry is unavailable; preserving it while "
"releasing an untracked lease"
)
lease.released = True
return
kept = [
entry
for entry in entries
if str(entry.get("lease_id") or "") != lease.lease_id
]
if len(kept) != len(entries):
_write_entries(state_path, kept)
loaded = _read_live_entries(
state_path, track_liveness=lease.track_liveness,
warn="Active-session registry is unavailable; preserving it while "
"releasing an untracked lease",
)
if loaded is not None:
entries = loaded[1]
kept = _without_lease(entries, lease.lease_id)
if len(kept) != len(entries):
_write_entries(state_path, kept)
lease.released = True
@@ -717,17 +640,14 @@ def transfer_active_session(
# thread acquired the file lock. Never resurrect a durably removed lease.
if lease.released:
return False
try:
raw_entries = _read_entries(state_path, strict=True)
entries = _prune_dead(raw_entries, strict=lease.track_liveness)
except ActiveSessionRegistryError:
if lease.track_liveness:
raise
logger.warning(
"Active-session registry is unavailable; refusing to overwrite "
"it during lease transfer"
)
loaded = _read_live_entries(
state_path, track_liveness=lease.track_liveness,
warn="Active-session registry is unavailable; refusing to overwrite "
"it during lease transfer",
)
if loaded is None:
return False
entries = loaded[1]
updated = False
for entry in entries:
if str(entry.get("lease_id") or "") != lease.lease_id:
@@ -735,9 +655,7 @@ def transfer_active_session(
entry["session_id"] = new_session_id
entry["updated_at"] = time.time()
if metadata:
entry["metadata"] = {
str(k): v for k, v in metadata.items() if isinstance(k, str)
}
entry["metadata"] = _clean_metadata(metadata)
updated = True
break
if not updated and lease.track_liveness:
@@ -760,12 +678,10 @@ def transfer_active_session(
def release_orphaned_leases(live_lease_ids: set[str]) -> int:
"""Drop this process's registry entries that no live session owns.
``_prune_dead`` only reclaims leases whose owning process died. A server
that runs for days (``hermes dashboard`` / ``serve``) never trips that
check, so a lease whose session skipped teardown is held until restart.
The owning process is the only authority on which of its own leases are
real, so it drops the rest itself — exact, with no heartbeat write on the
turn path and no staleness threshold to tune.
``_prune_dead`` only reclaims leases of dead processes; a days-long server
never trips it, so a lease whose session skipped teardown is held until
restart. The owning process is the only authority on its own leases —
exact, no heartbeat on the turn path, no staleness threshold.
"""
pid = os.getpid()
state_path = _state_path()
@@ -774,14 +690,13 @@ def release_orphaned_leases(live_lease_ids: set[str]) -> int:
if not state_path.exists():
return 0
with _FileLock(_lock_path()):
try:
raw_entries = _read_entries(state_path, strict=True)
entries = _prune_dead(raw_entries)
except ActiveSessionRegistryError:
logger.warning(
"Active-session registry is unavailable; skipping orphaned-lease sweep"
)
loaded = _read_live_entries(
state_path, track_liveness=False,
warn="Active-session registry is unavailable; skipping orphaned-lease sweep",
)
if loaded is None:
return 0
entries = loaded[1]
kept = [
entry
for entry in entries
@@ -813,12 +728,9 @@ def active_session_liveness_guard(
*,
registry_home: str | Path | None = None,
) -> Iterator[bool]:
"""Hold the registry lock while reporting whether ``session_id`` is leased.
Keeping the lock across the caller's lifecycle mutation prevents a new
backend from acquiring a lease and reopening the row between the liveness
check and the corresponding ``end_session`` write.
"""
"""Hold the registry lock while reporting whether ``session_id`` is leased,
so no new backend can acquire a lease between the check and the caller's
``end_session`` write."""
target = str(session_id or "")
state_path, lock_path = _lease_paths(registry_home=registry_home)
with _FileLock(lock_path):
@@ -834,12 +746,9 @@ def release_active_session_liveness_guard(
lease: ActiveSessionLease,
session_id: str,
) -> Iterator[bool]:
"""Remove ``lease`` and hold its registry lock through a lifecycle write.
This makes automatic cleanup one atomic ownership decision: the local
runtime disappears, sibling liveness is checked, and the caller may end the
durable row before any new backend can acquire/reopen it.
"""
"""Remove ``lease`` and hold its registry lock through a lifecycle write,
making cleanup one atomic ownership decision (release, check siblings,
end the durable row) before any new backend can reopen it."""
if not lease.enabled or lease.released:
with active_session_liveness_guard(
session_id, registry_home=_registry_home_for_lease(lease)
@@ -850,13 +759,8 @@ def release_active_session_liveness_guard(
target = str(session_id or "")
state_path, lock_path = _lease_paths(lease)
with _FileLock(lock_path):
raw_entries = _read_entries(state_path, strict=True)
entries = _prune_dead(raw_entries, strict=True)
kept = [
entry
for entry in entries
if str(entry.get("lease_id") or "") != lease.lease_id
]
entries = _prune_dead(_read_entries(state_path, strict=True), strict=True)
kept = _without_lease(entries, lease.lease_id)
if len(kept) != len(entries):
_write_entries(state_path, kept)
lease.released = True
+767 -1420
View File
File diff suppressed because it is too large Load Diff
+1258 -2254
View File
File diff suppressed because it is too large Load Diff
+30 -105
View File
@@ -11,40 +11,26 @@ so when the whole graph completes the root wakes back up — its
assignee (the orchestrator profile) gets a chance to judge completion
and add more tasks if the work isn't done yet.
Design notes
------------
* Mirrors the shape of ``hermes_cli/kanban_specify.py``: lazy aux
client import inside the function, lenient response parse, never
raises on expected failure modes.
* The system prompt sees the *configured* profile roster — names plus
descriptions plus the default fallback. Profiles without a
description are still listed (with a note) so the decomposer can
match on name as a fallback, but the user has an obvious incentive
to describe them.
* ``fanout=false`` collapses to the same effect as ``kanban specify``:
we tighten the body and flip ``triage -> todo`` as a single task,
no children created. This makes ``decompose`` a strict superset of
``specify`` from the user's perspective.
* If the LLM picks an assignee that doesn't exist as a profile, we
rewrite it to the configured ``default_assignee`` (or the default
profile if unset). A child task NEVER ends up with ``assignee=None``.
Design notes: mirrors ``kanban_specify`` (lazy aux import, lenient parse,
never raises on expected failures). The prompt sees the configured profile
roster; undescribed profiles are listed with a note so name-matching still
works. ``fanout=false`` collapses to the ``specify`` behaviour (tighten +
promote, no children), making ``decompose`` a strict superset. Unknown
assignees are rewritten to ``default_assignee`` — a child NEVER ends up with
``assignee=None``.
"""
from __future__ import annotations
import json
import logging
import os
import re
from dataclasses import dataclass
from typing import Optional
from hermes_cli import kanban_db as kb
from hermes_cli import profiles as profiles_mod
from hermes_cli.kanban_specify import _extract_json_blob, _title_body, _truncate
from hermes_cli.kanban_specify import _profile_author as _specify_author
logger = logging.getLogger(__name__)
@@ -136,37 +122,9 @@ class DecomposeOutcome:
new_title: Optional[str] = None
def _truncate(text: str, limit: int) -> str:
if len(text) <= limit:
return text
return text[: limit - 1] + "…"
def _extract_json_blob(raw: str) -> Optional[dict]:
if not raw:
return None
stripped = _FENCE_RE.sub("", raw.strip())
first = stripped.find("{")
last = stripped.rfind("}")
if first == -1 or last == -1 or last <= first:
return None
candidate = stripped[first : last + 1]
try:
val = json.loads(candidate)
except (ValueError, json.JSONDecodeError):
return None
if not isinstance(val, dict):
return None
return val
def _profile_author() -> str:
"""Mirror of ``hermes_cli.kanban._profile_author``."""
return (
os.environ.get("HERMES_PROFILE")
or os.environ.get("USER")
or "decomposer"
)
return _specify_author("decomposer")
def _load_config() -> dict:
@@ -177,31 +135,15 @@ def _load_config() -> dict:
return {}
def _resolve_orchestrator_profile(cfg: dict) -> str:
"""Resolve which profile owns the root/orchestration task after fan-out.
def _resolve_profile_from_cfg(cfg: dict, key: str) -> str:
"""``kanban.<key>`` if it names an existing profile, else the active
default profile — so a task is never stranded for lack of an owner.
Falls back to the active default profile when ``kanban.orchestrator_profile``
is unset, so a task is never stranded for lack of an orchestrator.
``orchestrator_profile`` owns the root after fan-out; ``default_assignee``
catches children the decomposer can't route.
"""
kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {}
explicit = (kanban_cfg.get("orchestrator_profile") or "").strip()
if explicit:
try:
if profiles_mod.profile_exists(explicit):
return explicit
except Exception:
pass
# Fall back to the active default profile.
try:
return profiles_mod.get_active_profile_name() or "default"
except Exception:
return "default"
def _resolve_default_assignee(cfg: dict) -> str:
"""Resolve which profile catches child tasks the orchestrator can't route."""
kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {}
explicit = (kanban_cfg.get("default_assignee") or "").strip()
explicit = (kanban_cfg.get(key) or "").strip()
if explicit:
try:
if profiles_mod.profile_exists(explicit):
@@ -215,12 +157,8 @@ def _resolve_default_assignee(cfg: dict) -> str:
def _build_roster() -> tuple[list[dict], set[str]]:
"""Return (roster_for_prompt, valid_assignee_names).
Each roster entry is ``{name, description, has_description}``. The
valid-set is used after the LLM responds to rewrite invalid
assignees to the default fallback.
"""
"""``(roster_for_prompt, valid_assignee_names)``; entries are
``{name, description, has_description}``."""
roster: list[dict] = []
valid: set[str] = set()
try:
@@ -255,11 +193,8 @@ def _normalize_assignee_choice(
default_assignee: str,
valid_names: set[str],
) -> str:
"""Return a valid assignee, falling back to ``default_assignee``.
Fan-out children and the single-task fallback should share the same
routing guarantee: promoted work must not be left unassigned.
"""
"""A valid assignee, else ``default_assignee`` — promoted work is never
left unassigned."""
if not isinstance(assignee, str) or not assignee.strip():
return default_assignee
chosen = assignee.strip()
@@ -274,13 +209,9 @@ def decompose_task(
author: Optional[str] = None,
timeout: Optional[int] = None,
) -> DecomposeOutcome:
"""Decompose a triage task into a graph of child tasks.
Returns an outcome describing what happened. Never raises for
expected failure modes (task not in triage, no aux client
configured, API error, malformed response, decomposer returned
fanout=true with empty task list) — those surface via ``ok=False``.
"""
"""Decompose a triage task into a graph of child tasks. Expected failures
(not in triage, no aux client, API error, malformed/empty reply) surface
as ``ok=False``."""
with kb.connect_closing() as conn:
task = kb.get_task(conn, task_id)
if task is None:
@@ -291,8 +222,8 @@ def decompose_task(
)
cfg = _load_config()
orchestrator = _resolve_orchestrator_profile(cfg)
default_assignee = _resolve_default_assignee(cfg)
orchestrator = _resolve_profile_from_cfg(cfg, "orchestrator_profile")
default_assignee = _resolve_profile_from_cfg(cfg, "default_assignee")
kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {}
auto_promote = bool(kanban_cfg.get("auto_promote_children", True))
roster, valid_names = _build_roster()
@@ -312,10 +243,8 @@ def decompose_task(
)
try:
# Route through call_llm so auxiliary.kanban_decomposer.* config
# (provider/model/base_url, extra_body, reasoning_effort, retries)
# all apply — the previous direct client.chat.completions.create()
# path dropped auxiliary.<task>.extra_body entirely (#35566).
# call_llm applies all auxiliary.kanban_decomposer.* config
# (provider/model/base_url, extra_body, reasoning_effort, retries).
resp = call_llm(
task="kanban_decomposer",
messages=[
@@ -337,7 +266,7 @@ def decompose_task(
except Exception:
raw = ""
parsed = _extract_json_blob(raw)
parsed = _extract_json_blob(raw, _FENCE_RE)
if parsed is None:
return DecomposeOutcome(task_id, False, "LLM returned malformed JSON")
@@ -346,10 +275,7 @@ def decompose_task(
if not fanout:
# Fall back to single-task spec promotion (same effect as specify).
new_title = parsed.get("title")
new_body = parsed.get("body")
title_val = new_title.strip() if isinstance(new_title, str) and new_title.strip() else None
body_val = new_body if isinstance(new_body, str) and new_body.strip() else None
title_val, body_val = _title_body(parsed)
assignee_val = None
if not task.assignee:
assignee_val = _normalize_assignee_choice(
@@ -385,8 +311,7 @@ def decompose_task(
task_id, False, "decomposer returned fanout=true with empty tasks list",
)
# Rewrite invalid assignees to the default fallback. Never leave a
# task with assignee=None — the user explicitly does not want that.
# Unknown assignees route to the default; never assignee=None.
children: list[dict] = []
for idx, entry in enumerate(raw_tasks):
if not isinstance(entry, dict):
+133 -353
View File
@@ -1,30 +1,13 @@
"""Kanban diagnostics — structured, actionable distress signals for tasks.
A ``Diagnostic`` is a machine-readable description of something that's wrong
with a kanban task: a hallucinated card id, a spawn crash-loop, a task
stuck blocked for too long, etc. Each one carries:
A ``Diagnostic`` carries a **kind** (canonical code the UI/tests match on), a
**severity**, title/detail text, and **actions** the dashboard renders as
buttons and the CLI as hints. Rules are stateless and read-only over
(task, events, runs, optional graph); callers compute on demand.
* A **kind** (canonical code; UI/tests match on this).
* A **severity** (``warning`` / ``error`` / ``critical``).
* A **title** (one-line human description) and **detail** (longer text).
* A list of **suggested actions** — structured entries the dashboard
turns into buttons and the CLI turns into hints.
Rules run over (task, recent events, recent runs, optional graph context) and
emit diagnostics. They are stateless and read-only — no DB writes. Callers compute
diagnostics on demand (on ``/board`` load, ``/tasks/:id`` fetch, or
``hermes kanban diagnostics``).
Design goals:
* Fixable-on-the-operator's-side signals only (missing config, phantom
ids, crash loop). Not "the provider returned 502 once" — that's a
transient runtime blip, not a diagnostic.
* Recoverable: every diagnostic comes with at least one suggested
recovery action the operator can actually take from the UI.
* Auto-clearing: when the underlying failure mode resolves (a clean
``completed`` event arrives, a spawn succeeds, the task gets
unblocked), the diagnostic stops firing. The audit event trail stays.
Design goals: operator-fixable signals only (not a one-off provider 502);
every diagnostic has at least one recovery action; diagnostics auto-clear
when the failure mode resolves (the audit event trail stays).
"""
from __future__ import annotations
@@ -35,9 +18,7 @@ import json
import time
# Severity rungs, ordered least → most urgent. The UI colors them
# amber (warning), orange (error), red (critical). Sorted outputs put
# critical first so operators see the worst fires at the top.
# Least → most urgent; sorted outputs put critical first.
SEVERITY_ORDER = ("warning", "error", "critical")
@@ -52,24 +33,10 @@ def severity_at_or_above(severity: Optional[str], threshold: Optional[str]) -> b
@dataclass
class DiagnosticAction:
"""A single recovery action attached to a diagnostic.
The ``kind`` determines how both the UI and CLI render it:
* ``reclaim`` / ``reassign`` — POST to the matching /tasks/:id/*
endpoint; dashboard wires into the existing recovery popover.
* ``unblock`` — PATCH status back to ``ready`` (for stuck-blocked
diagnostics).
* ``cli_hint`` — print/copy a shell command (e.g.
``hermes -p <profile> auth``). No HTTP side effect.
* ``open_docs`` — deep-link to the docs URL named in ``payload.url``.
* ``comment`` — nudge the operator to add a comment (for
stuck-blocked tasks that need human input).
``suggested=True`` marks the action as the recommended first step;
the UI highlights it. Multiple actions can be suggested if they're
equally valid.
"""
"""A recovery action. ``kind`` drives rendering: ``reclaim``/``reassign``
POST to /tasks/:id/*; ``unblock`` PATCHes status to ready; ``cli_hint``
shows ``payload.command``; ``open_docs`` links ``payload.url``; ``comment``
nudges the operator. ``suggested=True`` = recommended first step."""
kind: str
label: str
@@ -122,20 +89,10 @@ class Diagnostic:
# ---------------------------------------------------------------------------
def _task_field(task, name, default=None):
"""Read a field from a task regardless of representation.
Callers pass sqlite3.Row (dict-like with [] but no attribute
access), kanban_db.Task dataclasses (attribute access), or plain
dicts (both). This normalises them so rule functions don't have
to branch on type each time.
"""
"""Read a field from a sqlite3.Row, a kanban_db.Task dataclass, or a dict."""
if task is None:
return default
# sqlite Row + plain dicts both support mapping access; Row also
# supports .keys().
try:
# Row raises IndexError if the key isn't a column in the query;
# dicts return default via .get. Handle both.
if hasattr(task, "keys") and name in task.keys():
return task[name]
except Exception:
@@ -169,20 +126,42 @@ def _event_ts(ev) -> int:
return int(t or 0)
def _first_field(task, primary: str, legacy: str, default=None):
"""``task[primary]`` unless it is None, else ``task[legacy]`` (old DB rows)."""
v = _task_field(task, primary, None)
return v if v is not None else _task_field(task, legacy, default)
def _latest_event_ts(events: Iterable[Any], kinds: set[str]) -> int:
"""Max ``created_at`` over events whose kind is in ``kinds`` (0 if none)."""
latest = 0
for ev in events:
if _event_kind(ev) in kinds:
latest = max(latest, _event_ts(ev))
return latest
def _log_hint_action(task_id: str) -> DiagnosticAction:
return DiagnosticAction(
kind="cli_hint",
label=f"Check logs: hermes kanban log {task_id}",
payload={"command": f"hermes kanban log {task_id}"},
suggested=True,
)
def _error_snippet(last_err) -> str:
"""First 500 chars of the error (with ellipsis), or "" when absent."""
err_text = (last_err or "").strip() if last_err else ""
return err_text[:500] + ("…" if len(err_text) > 500 else "") if err_text else ""
def _active_hallucination_events(
events: Iterable[Any],
kind: str,
) -> list[Any]:
"""Return events of ``kind`` that have no ``completed``/``edited``
event *strictly after* them. Walks chronologically: each clean
event resets the accumulator; each matching event gets appended.
Events must be sorted by id (i.e. arrival order); callers pass the
task's full event list which the DB already returns in that order.
"""
# Events arrive sorted by id asc (chronological). Walk once, track
# which hallucination events are still "active" (no clean event
# supersedes them).
"""Events of ``kind`` with no ``completed``/``edited`` event strictly after
them. Requires id-sorted (arrival-order) input, which the DB provides."""
active: list[Any] = []
for ev in events:
k = _event_kind(ev)
@@ -191,9 +170,9 @@ def _active_hallucination_events(
elif k == kind:
active.append(ev)
return active
# Standard always-available actions. Every diagnostic can offer these as
# fallbacks regardless of kind — they're the two baseline recovery
# primitives the kernel supports.
# Baseline recovery primitives every diagnostic can fall back on.
def _generic_recovery_actions(task: Any, *, running: bool) -> list[DiagnosticAction]:
out: list[DiagnosticAction] = []
if running:
@@ -214,22 +193,16 @@ def _generic_recovery_actions(task: Any, *, running: bool) -> list[DiagnosticAct
# Rule implementations
# ---------------------------------------------------------------------------
# Each rule takes (task, events, runs, now_ts, config) and returns
# zero or more Diagnostic instances. ``events`` / ``runs`` are lists of
# kanban_db.Event / kanban_db.Run (or plain dicts matching the same
# shape — for test convenience).
# Each rule: (task, events, runs, now_ts, config) -> list[Diagnostic].
# ``events``/``runs`` are kanban_db rows/dataclasses or same-shaped dicts.
RuleFn = Callable[[Any, list[Any], list[Any], int, dict], list[Diagnostic]]
def _aux_slot_explicit(slot: Any) -> bool:
"""Return True if the auxiliary slot has user-supplied non-default fields.
Defaults from ``DEFAULT_CONFIG`` use ``provider: "auto"`` with empty
model/base_url/api_key — that path falls through to the main model. An
"explicit" config is one where the user actively set a provider (not
"auto"), or supplied a model / base_url / api_key.
"""
"""True if the aux slot was user-configured: provider other than "auto",
or any of model/base_url/api_key set (the default falls through to the
main model)."""
if not isinstance(slot, dict):
return False
provider = str(slot.get("provider") or "").strip().lower()
@@ -242,12 +215,9 @@ def _aux_slot_explicit(slot: Any) -> bool:
def _main_model_visible(raw_config: Any) -> bool:
"""Best-effort check that a main model is configured.
Diagnostics runs in the dashboard process which may not share the CLI's
runtime state, so we read the raw config dict. If we cannot prove the
main model is set, we err on the side of NOT firing the diagnostic.
"""
"""Best-effort "a main model is configured" from the raw config dict (the
dashboard process may not share CLI runtime state). Unprovable => False,
which errs toward NOT firing the diagnostic."""
if not isinstance(raw_config, dict):
return False
model_cfg = raw_config.get("model")
@@ -264,17 +234,9 @@ def _main_model_visible(raw_config: Any) -> bool:
def triage_aux_status(config: Optional[dict]) -> Optional[dict]:
"""Inspect raw config and report whether triage paths look configured.
Returns ``None`` when config context is unavailable (suppress diagnostic
to avoid noisy false positives in tests / low-level callers). Otherwise
returns a dict with:
- ``auto_decompose``: bool — whether the dispatcher auto-runs decompose
- ``decomposer_explicit``: bool — user-supplied decomposer slot
- ``specifier_explicit``: bool — user-supplied specifier slot
- ``main_model_visible``: bool — main model can serve as auto fallback
"""
"""Report whether the triage aux paths look configured: ``{auto_decompose,
decomposer_explicit, specifier_explicit, main_model_visible}``. ``None``
when no config context is present (keeps low-level callers/tests silent)."""
if not isinstance(config, dict):
return None
@@ -285,9 +247,7 @@ def triage_aux_status(config: Optional[dict]) -> Optional[dict]:
aux = config.get("auxiliary")
kanban_cfg = config.get("kanban") if isinstance(config.get("kanban"), dict) else {}
# Have we been handed any config context at all? When neither auxiliary
# nor kanban nor model keys are present, the caller is a low-level test
# passing {} — stay silent.
# No auxiliary/kanban/model keys at all => a low-level caller passing {}.
if (
not isinstance(aux, dict)
and not kanban_cfg
@@ -323,14 +283,8 @@ def _positive_int(value: Any, default: int) -> int:
def _rule_hallucinated_cards(task, events, runs, now, cfg) -> list[Diagnostic]:
"""Blocked-hallucination gate fires: a worker called kanban_complete
with created_cards that didn't exist or weren't created by the
completing profile. Task stayed in its prior state; the operator
needs to decide how to proceed.
Auto-clears when a successful completion (or edit) follows the
blocked event.
"""
"""A worker's kanban_complete named created_cards that don't exist / weren't
its own; the completion was blocked. Clears on a later completion/edit."""
hits = _active_hallucination_events(events, "completion_blocked_hallucination")
if not hits:
return []
@@ -343,13 +297,11 @@ def _rule_hallucinated_cards(task, events, runs, now, cfg) -> list[Diagnostic]:
if pid not in phantom_ids:
phantom_ids.append(pid)
running = _task_field(task, "status") == "running"
actions: list[DiagnosticAction] = []
actions.append(DiagnosticAction(
actions = [DiagnosticAction(
kind="comment",
label="Add a comment explaining what to do",
suggested=False,
))
actions.extend(_generic_recovery_actions(task, running=running))
)] + _generic_recovery_actions(task, running=running)
return [Diagnostic(
kind="hallucinated_cards",
severity="error",
@@ -370,22 +322,12 @@ def _rule_hallucinated_cards(task, events, runs, now, cfg) -> list[Diagnostic]:
def _rule_triage_aux_unavailable(task, events, runs, now, cfg) -> list[Diagnostic]:
"""A triage task cannot leave triage without an auxiliary helper.
With the auto-decompose dispatcher (kanban.auto_decompose, default True),
triage tasks fan out via ``auxiliary.kanban_decomposer`` and fall back to
``auxiliary.triage_specifier`` when the decomposer returns ``fanout=false``.
With auto-decompose off, the user must run ``hermes kanban specify``,
which only needs ``auxiliary.triage_specifier``.
The default slot is ``provider: auto`` → auto-falls back to the main model,
so this rule only fires when:
- the relevant slot is explicitly set to something broken, OR
- the auto fallback has no main model to fall back to.
Config context is required; pass {} from tests to keep the rule silent.
"""
"""A triage task can't leave triage without a usable aux model. With
auto-decompose on the primary slot is ``auxiliary.kanban_decomposer``
(specifier as fallback); off, it is ``auxiliary.triage_specifier``. The
default ``provider: auto`` falls back to the main model, so this fires only
when the slot isn't explicit AND no main model is visible. Requires config
context ({} keeps it silent)."""
if _task_field(task, "status") != "triage":
return []
@@ -421,9 +363,6 @@ def _rule_triage_aux_unavailable(task, events, runs, now, cfg) -> list[Diagnosti
"`hermes kanban specify`, which uses auxiliary.triage_specifier."
)
# The primary slot is usable when either: it was explicitly configured by
# the user, OR the default `provider: auto` can fall back to the main
# model. If both fail, we have a real configuration gap.
if primary_explicit or main_visible:
return []
@@ -482,12 +421,8 @@ def _rule_triage_aux_unavailable(task, events, runs, now, cfg) -> list[Diagnosti
def _rule_prose_phantom_refs(task, events, runs, now, cfg) -> list[Diagnostic]:
"""Advisory prose-scan: the completion summary mentions ``t_<hex>``
ids that don't resolve. Non-blocking; surfaced as a warning only.
Auto-clears when a fresh clean completion arrives AFTER the
suspected event.
"""
"""Advisory: the completion summary mentions ``t_<hex>`` ids that don't
resolve. Warning only; clears on a later clean completion."""
hits = _active_hallucination_events(events, "suspected_hallucinated_references")
if not hits:
return []
@@ -516,32 +451,14 @@ def _rule_prose_phantom_refs(task, events, runs, now, cfg) -> list[Diagnostic]:
def _rule_repeated_failures(task, events, runs, now, cfg) -> list[Diagnostic]:
"""Task's unified ``consecutive_failures`` counter is climbing —
something about this task+profile combo is broken and each retry
fails the same way. Triggers regardless of the specific failure
mode (spawn error, timeout, crash) because operationally they
all look the same: the kernel keeps retrying and the operator
needs to intervene.
"""``consecutive_failures`` >= cfg["failure_threshold"] (legacy key
``spawn_failure_threshold``), regardless of failure mode — the kernel keeps
retrying and the operator must intervene. Runtime callers derive the
threshold from ``kanban.failure_limit`` so it doesn't lag the breaker.
Threshold: cfg["failure_threshold"]. Runtime callers should derive
this from ``kanban.failure_limit`` unless the user explicitly set a
diagnostics threshold, so the signal does not lag behind the
dispatcher's circuit breaker.
Accepts the legacy ``spawn_failure_threshold`` config key for
back-compat.
Terminal statuses are exempt: a done/archived card has nothing left
to retry, so a lingering failure streak is history, not a signal.
(``complete_task`` resets the counter, but a manual done — e.g. a
dashboard drag — ends no run and used to leave the flag stuck.)
A fresh attempt in flight (``running``) is also exempt: retrying a
task should clear the stale failure banner until this attempt also
resolves. Otherwise a card that's actively trying again still shows
"failed Nx", which reads as a current failure. It re-fires if the new
run fails too (status leaves ``running`` with a recorded outcome).
"""
Exempt: done/archived (a manual done ends no run, so the streak is history)
and running (a retry in flight must not read as a current failure; re-fires
if it fails too)."""
if _task_field(task, "status") in ("done", "archived", "running"):
return []
threshold = _positive_int(cfg.get(
@@ -549,26 +466,13 @@ def _rule_repeated_failures(task, events, runs, now, cfg) -> list[Diagnostic]:
cfg.get("spawn_failure_threshold", 3),
), 3)
failure_limit = _positive_int(cfg.get("failure_limit"), threshold)
# Read the new unified counter name, with a fallback to the legacy
# column name so this rule keeps working against old DB rows the
# caller somehow materialised without running the migration.
failures = (
_task_field(task, "consecutive_failures", None)
if _task_field(task, "consecutive_failures", None) is not None
else _task_field(task, "spawn_failures", 0)
)
failures = _first_field(task, "consecutive_failures", "spawn_failures", 0)
if failures is None or failures < threshold:
return []
last_err = (
_task_field(task, "last_failure_error", None)
if _task_field(task, "last_failure_error", None) is not None
else _task_field(task, "last_spawn_error", None)
)
last_err = _first_field(task, "last_failure_error", "last_spawn_error")
assignee = _task_field(task, "assignee")
# Classify the most recent failure by peeking at run outcomes so
# the title + suggested action can be specific without a separate
# per-outcome rule.
# Most recent failure outcome makes the title/action specific.
ordered_runs = sorted(runs, key=lambda r: _task_field(r, "id", 0))
most_recent_outcome = None
for r in reversed(ordered_runs):
@@ -596,19 +500,13 @@ def _rule_repeated_failures(task, events, runs, now, cfg) -> list[Diagnostic]:
# to diagnose; reclaim/reassign are the recovery levers.
task_id = _task_field(task, "id")
if task_id:
actions.append(DiagnosticAction(
kind="cli_hint",
label=f"Check logs: hermes kanban log {task_id}",
payload={"command": f"hermes kanban log {task_id}"},
suggested=True,
))
actions.append(_log_hint_action(task_id))
actions.extend(_generic_recovery_actions(
task, running=_task_field(task, "status") == "running",
))
severity = "critical" if failures >= threshold * 2 else "error"
err_text = (last_err or "").strip() if last_err else ""
err_snippet = err_text[:500] + ("…" if len(err_text) > 500 else "") if err_text else ""
err_snippet = _error_snippet(last_err)
outcome_label = {
"spawn_failed": "spawn",
"timed_out": "timeout",
@@ -651,29 +549,14 @@ def _rule_repeated_failures(task, events, runs, now, cfg) -> list[Diagnostic]:
def _rule_repeated_crashes(task, events, runs, now, cfg) -> list[Diagnostic]:
"""The worker spawns fine but keeps crashing mid-run. Check the last
N runs' outcomes; N consecutive ``crashed`` without a successful
``completed`` means something about the task + profile combo is
broken (OOM, missing dependency, tool it needs is down).
"""Trailing run outcomes show >= cfg["crash_threshold"] (default 2)
consecutive ``crashed`` with no ``completed``/``reclaimed`` between. Fires
earlier than ``repeated_failures`` for a crash-specific heads-up and
suppresses itself when the unified rule is about to fire.
Threshold: cfg["crash_threshold"] (default 2).
Narrower than ``repeated_failures`` — fires earlier (2 crashes vs 3
total failures) so the operator gets a crash-specific heads-up
before the unified rule kicks in. Suppresses itself when the
unified rule is also about to fire, to avoid double-flagging.
Terminal statuses are exempt for the same reason as
``repeated_failures`` — with one extra wrinkle: this rule reads run
history, and a manual done (dashboard drag) appends no ``completed``
run to break the crash streak, so the flag was permanent (#kanban
desktop dogfood). Done means done.
``running`` is exempt too: a fresh attempt is in flight, and its
in-flight run (no outcome yet) doesn't break the trailing crash scan,
so a retried card kept showing "crashed Nx" over an active run. The
banner re-fires if the new attempt also crashes.
"""
Exempt: done/archived (a manual done appends no completed run, so the
streak would be permanent) and running (an in-flight run has no outcome
and wouldn't break the scan)."""
if _task_field(task, "status") in ("done", "archived", "running"):
return []
failure_threshold = int(cfg.get(
@@ -702,29 +585,19 @@ def _rule_repeated_crashes(task, events, runs, now, cfg) -> list[Diagnostic]:
# A success (or manual reclaim) breaks the streak.
break
else:
# Other outcomes (timed_out, blocked, spawn_failed, gave_up)
# aren't crash signals — don't count them, but they also
# don't break the crash streak.
# Other outcomes neither count as crashes nor break the streak.
continue
if consecutive < threshold:
return []
task_id = _task_field(task, "id")
actions: list[DiagnosticAction] = []
if task_id:
actions.append(DiagnosticAction(
kind="cli_hint",
label=f"Check logs: hermes kanban log {task_id}",
payload={"command": f"hermes kanban log {task_id}"},
suggested=True,
))
actions.append(_log_hint_action(task_id))
running = _task_field(task, "status") == "running"
actions.extend(_generic_recovery_actions(task, running=running))
severity = "critical" if consecutive >= threshold * 2 else "error"
# Put the actual error up-front so operators see WHAT broke without
# having to open the logs. Truncate defensively — these can be huge
# (full tracebacks).
err_text = (last_err or "").strip() if last_err else ""
err_snippet = err_text[:500] + ("…" if len(err_text) > 500 else "") if err_text else ""
# Error up-front so operators see WHAT broke without opening the logs.
err_snippet = _error_snippet(last_err)
if err_snippet:
title = f"Agent crashed {consecutive}x: {err_snippet.splitlines()[0][:160]}"
detail = (
@@ -751,14 +624,9 @@ def _rule_repeated_crashes(task, events, runs, now, cfg) -> list[Diagnostic]:
def _rule_review_dependency_deadlock(task, events, runs, now, cfg) -> list[Diagnostic]:
"""Detect a legacy review handoff that starves downstream children.
Older workers were instructed to sticky-block an implementation with a
``review-required:`` reason. A separately modelled reviewer child cannot
promote until that parent is terminal, so the lane has no autonomous next
step. This compatibility diagnostic is graph-aware but deliberately leaves
both the dependency graph and the user's sticky block unchanged.
"""
"""Legacy review handoff starving children: the implementation is
sticky-blocked with a ``review-required:`` reason while todo children wait
for it to be terminal. Graph-aware; deliberately mutates nothing."""
if _task_field(task, "status") != "blocked":
return []
@@ -829,21 +697,13 @@ def _rule_review_dependency_deadlock(task, events, runs, now, cfg) -> list[Diagn
def _rule_stuck_in_blocked(task, events, runs, now, cfg) -> list[Diagnostic]:
"""Task has been in ``blocked`` status for too long without a comment.
Threshold: cfg["blocked_stale_hours"] (default 24).
Surfaced as a warning so humans know there's a pending unblock.
"""
"""Blocked for >= cfg["blocked_stale_hours"] (default 24) with no comment
or unblock since the last ``blocked`` event."""
hours = float(cfg.get("blocked_stale_hours", 24))
status = _task_field(task, "status")
if status != "blocked":
return []
# Find the most recent ``blocked`` event.
last_blocked_ts = 0
for ev in events:
if _event_kind(ev) == "blocked":
t = _event_ts(ev)
last_blocked_ts = max(last_blocked_ts, t)
last_blocked_ts = _latest_event_ts(events, {"blocked"})
if last_blocked_ts == 0:
return []
age_hours = (now - last_blocked_ts) / 3600.0
@@ -879,29 +739,17 @@ def _rule_stuck_in_blocked(task, events, runs, now, cfg) -> list[Diagnostic]:
def _rule_block_unblock_cycling(task, events, runs, now, cfg) -> list[Diagnostic]:
"""Task has cycled through blocked → unblocked many times — the
``unblock`` is not fixing the underlying problem and the worker
keeps re-blocking for substantially the same reason.
``_rule_stuck_in_blocked`` resets its timer on any ``commented`` /
``unblocked`` event, so a task that cycles every few minutes is
invisible to it regardless of how many times it cycles (#29747
gap 1). This rule complements that one by counting block→unblock
cycles in a sliding window.
Threshold: cfg["block_cycle_threshold"] (default 3) cycles within
cfg["block_cycle_window_seconds"] (default 24h).
"""
""">= cfg["block_cycle_threshold"] (default 3) blocked-after-unblocked
cycles within cfg["block_cycle_window_seconds"] (default 24h). Complements
``_rule_stuck_in_blocked``, whose timer any unblock resets, so fast cyclers
are invisible to it."""
threshold = _positive_int(cfg.get("block_cycle_threshold"), 3)
window_seconds = float(cfg.get("block_cycle_window_seconds", 24 * 3600))
cycle_cutoff = now - window_seconds
# Walk events chronologically (arrival order — callers pre-sort by
# id, which is the canonical chronological order; ``created_at``
# alone is insufficient because multiple events can share the same
# second). Count "blocked after unblocked" transitions: every time
# a blocked event follows at least one unblocked event since the
# last cycle was counted, that's a new cycle.
# Walk in id (arrival) order — created_at alone can't order events that
# share a second. A blocked event after >= 1 unblocked since the last
# counted cycle is a new cycle.
cycles = 0
seen_unblock_since_last_cycle = False
initial_blocked_ts = 0
@@ -956,66 +804,31 @@ def _rule_block_unblock_cycling(task, events, runs, now, cfg) -> list[Diagnostic
def _rule_stranded_in_ready(task, events, runs, now, cfg) -> list[Diagnostic]:
"""Task has been in ``ready`` status for too long without any worker
claiming it.
Threshold: cfg["stranded_threshold_seconds"] (default 1800 = 30 min).
Catches every "task waiting for a worker that never comes" case
without caring WHY:
* Operator typo'd the assignee — no profile or external worker matches.
* Profile was deleted, leaving its tasks stranded.
* External worker pool (Codex CLI, Claude Code lane, custom daemon)
is down, hung, or wasn't started.
* Dispatcher is misconfigured (wrong board, wrong HERMES_HOME).
Pre-rule, all of these silently rotted in ``skipped_nonspawnable`` —
the dispatcher correctly skipped them (good — no respawn loop) but
nobody surfaced the fact that operator-actionable work was
accumulating. The rule fires when a ready task's promoted-to-ready
timestamp is older than the threshold AND the assignee is non-empty
(truly unassigned tasks have their own ``skipped_unassigned`` signal
on the dispatcher and a different operator response).
The signal is age-based on purpose: it's identity-agnostic, so it
works for Hermes profiles, registered lanes, external workers, and
typos uniformly. No registry to curate, no per-board allowlist.
"""
"""Assigned, unclaimed, ``ready`` for >= cfg["stranded_threshold_seconds"]
(default 30 min). Deliberately age-based and identity-agnostic so it
catches typo'd assignees, deleted profiles, and down external worker
pools alike without a registry to curate. Unassigned tasks are excluded —
the dispatcher's ``skipped_unassigned`` already covers them."""
threshold_seconds = float(
cfg.get("stranded_threshold_seconds", 30 * 60)
)
status = _task_field(task, "status")
if status != "ready":
return []
# Skip tasks with a live claim — they're being worked on, even if
# the worker hasn't reported progress yet (run-level liveness
# extends the claim TTL; we don't want to second-guess that here).
# A live claim means it's being worked on even without progress yet.
if _task_field(task, "claim_lock"):
return []
assignee = _task_field(task, "assignee") or ""
if not assignee.strip():
# Unassigned tasks: the dispatcher's ``skipped_unassigned`` is
# already the right signal. A separate diagnostic here would
# double-flag the same condition.
return []
# Find the most recent event that put this task into ready.
# ``created`` covers tasks born ready; ``promoted`` covers parent-
# done auto-promotion; ``reclaimed`` covers TTL/crash recovery;
# ``unblocked`` covers human-driven resumes.
READY_TRANSITION_KINDS = {
"created", "promoted", "reclaimed", "unblocked",
}
last_ready_ts = 0
for ev in events:
if _event_kind(ev) in READY_TRANSITION_KINDS:
t = _event_ts(ev)
last_ready_ts = max(last_ready_ts, t)
# Most recent event that put the task into ready.
last_ready_ts = _latest_event_ts(
events, {"created", "promoted", "reclaimed", "unblocked"},
)
# Fallback: if no qualifying event exists (very old task or events
# truncated), fall back to ``created_at`` on the task row. Better
# to occasionally over-flag an ancient task than miss a stranded one.
# No qualifying event (old task / truncated events): fall back to
# created_at — over-flagging an ancient task beats missing a stranded one.
if last_ready_ts == 0:
last_ready_ts = int(_task_field(task, "created_at", default=0) or 0)
if last_ready_ts == 0:
@@ -1031,9 +844,7 @@ def _rule_stranded_in_ready(task, events, runs, now, cfg) -> list[Diagnostic]:
else:
age_str = f"{int(age_seconds / 60)}m"
# Severity escalates with age. Below 2x threshold = warning;
# 2x – 6x = error; beyond 6x = critical (something is clearly
# broken, not just slow).
# Escalate with age: <2x threshold warning, 2x-6x error, >6x critical.
if age_seconds >= threshold_seconds * 6:
severity = "critical"
elif age_seconds >= threshold_seconds * 2:
@@ -1078,8 +889,7 @@ def _rule_stranded_in_ready(task, events, runs, now, cfg) -> list[Diagnostic]:
)]
# Registry — order matters: rules higher on the list render first when
# severity ties. Add new rules here.
# Order matters: earlier rules render first on severity ties.
_RULES: list[RuleFn] = [
_rule_hallucinated_cards,
_rule_triage_aux_unavailable,
@@ -1093,21 +903,6 @@ _RULES: list[RuleFn] = [
]
# Known kinds (for the UI's filter / legend / i18n keys). Update when
# rules are added.
DIAGNOSTIC_KINDS = (
"hallucinated_cards",
"triage_aux_unavailable",
"prose_phantom_refs",
"repeated_failures",
"repeated_crashes",
"review_dependency_deadlock",
"stuck_in_blocked",
"block_unblock_cycling",
"stranded_in_ready",
)
DEFAULT_CONFIG = {
# Match the dispatcher default (kanban.failure_limit) so repeated-failure
# diagnostics do not lag behind the default auto-block threshold.
@@ -1116,21 +911,16 @@ DEFAULT_CONFIG = {
"spawn_failure_threshold": 2,
"crash_threshold": 2,
"blocked_stale_hours": 24,
# Stranded-task threshold. 30 min by default — below that, the
# signal is dominated by tasks that are about to be claimed on the
# next dispatcher tick (default 60s) and would just be noise.
# Below 30 min the signal is dominated by tasks about to be claimed on
# the next dispatcher tick.
"stranded_threshold_seconds": 30 * 60,
}
def config_from_kanban_config(kanban_cfg: Optional[dict]) -> dict:
"""Build diagnostics config from the runtime ``kanban`` config section.
``kanban.diagnostics.failure_threshold`` remains an explicit override.
Otherwise, derive the repeated-failure threshold from
``kanban.failure_limit`` so CLI/dashboard diagnostics match the
dispatcher's actual circuit-breaker threshold.
"""
"""Diagnostics config from the ``kanban`` section. ``kanban.diagnostics.
failure_threshold`` is an explicit override; otherwise the threshold is
``kanban.failure_limit`` so diagnostics match the dispatcher's breaker."""
kanban_cfg = kanban_cfg or {}
diag_cfg = dict(kanban_cfg.get("diagnostics") or {})
diag_cfg.setdefault(
@@ -1146,13 +936,9 @@ def config_from_kanban_config(kanban_cfg: Optional[dict]) -> dict:
def config_from_runtime_config(raw_config: Optional[dict]) -> dict:
"""Build diagnostics config from the full Hermes runtime config.
Carries through ``kanban``, ``auxiliary``, and ``model`` keys so triage-
aware rules can inspect the active aux-helper and main-model state.
Folds the ``kanban`` block through ``config_from_kanban_config`` so the
repeated-failure threshold derivation still applies.
"""
"""Diagnostics config from the full runtime config: folds ``kanban`` through
``config_from_kanban_config`` and carries ``kanban``/``auxiliary``/``model``
through for the triage-aware rules."""
raw_config = raw_config or {}
if not isinstance(raw_config, dict):
return {}
@@ -1177,12 +963,8 @@ def compute_task_diagnostics(
config: Optional[dict] = None,
graph: Optional[dict] = None,
) -> list[Diagnostic]:
"""Run every rule against a single task's state and return a
severity-sorted list of active diagnostics.
Sorting: critical first, then error, then warning; ties broken by
most-recent ``last_seen_at``.
"""
"""Run every rule for one task; critical first, then error, warning; ties
broken by most-recent ``last_seen_at``."""
now_ts = int(now if now is not None else time.time())
config = config or {}
cfg = {**DEFAULT_CONFIG, **config}
@@ -1202,9 +984,7 @@ def compute_task_diagnostics(
try:
out.extend(rule(task, events, runs, now_ts, cfg))
except Exception:
# A broken rule must never crash the dashboard. Rule bugs
# get caught in tests; in production we'd rather drop the
# diagnostic than 500 a whole /board request.
# A broken rule must never 500 a whole /board request.
continue
severity_idx = {s: i for i, s in enumerate(SEVERITY_ORDER)}
out.sort(
+34 -69
View File
@@ -1,32 +1,13 @@
"""Kanban triage specifier — flesh out a one-liner into a real spec.
Used by ``hermes kanban specify [task_id | --all]``. Takes a task that
lives in the Triage column (a rough idea, typically only a title), calls
the auxiliary LLM to produce:
``hermes kanban specify [task_id | --all]`` asks the auxiliary LLM for a
tightened title + concrete body for a Triage task, then flips it
``triage -> todo`` via ``kanban_db.specify_triage_task``.
* A tightened title (optional — only replaces if the model proposes a
materially different one)
* A concrete body: goal, proposed approach, acceptance criteria
and then flips the task ``triage -> todo`` via
``kanban_db.specify_triage_task``. The dispatcher promotes it to
``ready`` on its next tick (or immediately if there are no open parents).
Design notes
------------
* This module intentionally mirrors ``hermes_cli/goals.py`` — same aux
client pattern, same "empty config => skip, don't crash" tolerance.
Keeps the surface area tiny and the failure modes predictable.
* The prompt is a short system + user pair. We ask for JSON with
``{title, body}``; if parsing fails, we fall back to treating the
whole response as the body and leave the title untouched. No
retry loop — one shot, keep cost bounded.
* Structured output / JSON mode is not requested explicitly so the
specifier works on providers that don't implement it. The parse
is lenient (tolerates markdown code fences around the JSON).
Mirrors ``hermes_cli/goals.py``: same aux-client pattern, same "empty config
=> skip, don't crash" tolerance. One shot, no retry loop. JSON mode is not
requested (works on providers without it); the parse is lenient and falls
back to "whole reply is the body" so a malformed reply never strands a task.
"""
from __future__ import annotations
@@ -108,13 +89,12 @@ def _truncate(text: str, limit: int) -> str:
_FENCE_RE = re.compile(r"^\s*```(?:json)?\s*|\s*```\s*$", re.IGNORECASE)
def _extract_json_blob(raw: str) -> Optional[dict]:
"""Lenient JSON extraction — tolerates fenced code blocks and
leading/trailing whitespace. Returns None if nothing parses."""
def _extract_json_blob(raw: str, fence_re: re.Pattern = _FENCE_RE) -> Optional[dict]:
"""Lenient JSON object extraction: strip code fences, take the first ``{``
to the last ``}``. None if nothing parses to a dict."""
if not raw:
return None
stripped = _FENCE_RE.sub("", raw.strip())
# Greedy: find the first `{` and last `}` and try that slice.
stripped = fence_re.sub("", raw.strip())
first = stripped.find("{")
last = stripped.rfind("}")
if first == -1 or last == -1 or last <= first:
@@ -124,19 +104,24 @@ def _extract_json_blob(raw: str) -> Optional[dict]:
val = json.loads(candidate)
except (ValueError, json.JSONDecodeError):
return None
if not isinstance(val, dict):
return None
return val
return val if isinstance(val, dict) else None
def _profile_author() -> str:
def _nonblank(v) -> Optional[str]:
return v if isinstance(v, str) and v.strip() else None
def _title_body(parsed: dict) -> tuple[Optional[str], Optional[str]]:
"""``(title, body)`` from an LLM reply: title stripped, body verbatim,
either None when missing/blank."""
title = _nonblank(parsed.get("title"))
return (title.strip() if title else None), _nonblank(parsed.get("body"))
def _profile_author(default: str = "specifier") -> str:
"""Mirror of ``hermes_cli.kanban._profile_author``. Kept local to
avoid a circular import when kanban.py imports this module."""
return (
os.environ.get("HERMES_PROFILE")
or os.environ.get("USER")
or "specifier"
)
return os.environ.get("HERMES_PROFILE") or os.environ.get("USER") or default
def specify_task(
@@ -145,13 +130,9 @@ def specify_task(
author: Optional[str] = None,
timeout: Optional[int] = None,
) -> SpecifyOutcome:
"""Specify a single triage task and promote it to ``todo``.
Returns an outcome describing what happened. Never raises for expected
failure modes (task not in triage, no aux client configured, API
error, malformed response) — those surface via ``ok=False`` so the
``--all`` sweep can continue past individual failures.
"""
"""Specify one triage task and promote it to ``todo``. Expected failures
(not in triage, no aux client, API error, malformed reply) surface as
``ok=False`` so an ``--all`` sweep continues."""
with kb.connect_closing() as conn:
task = kb.get_task(conn, task_id)
if task is None:
@@ -174,9 +155,8 @@ def specify_task(
)
try:
# Route through call_llm so auxiliary.triage_specifier.* config
# (provider/model/base_url, extra_body, reasoning_effort, retries)
# all apply — the direct-create path dropped extra_body (#35566).
# call_llm applies all auxiliary.triage_specifier.* config
# (provider/model/base_url, extra_body, reasoning_effort, retries).
resp = call_llm(
task="triage_specifier",
messages=[
@@ -206,9 +186,7 @@ def specify_task(
new_title: Optional[str]
new_body: Optional[str]
if parsed is None:
# Fall back: treat the whole reply as the body, leave title as-is.
# Worst case the user edits afterward — still better than stranding
# the task in triage on a malformed LLM reply.
# Whole reply becomes the body; the user can edit afterward.
stripped_raw = raw.strip()
if not stripped_raw:
return SpecifyOutcome(
@@ -217,16 +195,7 @@ def specify_task(
new_title = None
new_body = stripped_raw
else:
title_val = parsed.get("title")
body_val = parsed.get("body")
new_title = (
title_val.strip()
if isinstance(title_val, str) and title_val.strip()
else None
)
new_body = (
body_val if isinstance(body_val, str) and body_val.strip() else None
)
new_title, new_body = _title_body(parsed)
if new_body is None and new_title is None:
return SpecifyOutcome(
task_id, False, "LLM response missing title and body"
@@ -241,8 +210,7 @@ def specify_task(
author=author or _profile_author(),
)
if not ok:
# Race: someone else promoted / archived the task between our
# read above and the write. Report, don't crash.
# Race: promoted/archived between our read and the write.
return SpecifyOutcome(
task_id, False, "task moved out of triage before promotion"
)
@@ -250,10 +218,7 @@ def specify_task(
def list_triage_ids(*, tenant: Optional[str] = None) -> list[str]:
"""Return task ids currently in the triage column.
``tenant`` narrows the sweep; ``None`` returns every triage task.
"""
"""Task ids in the triage column; ``tenant`` narrows the sweep."""
with kb.connect_closing() as conn:
tasks = kb.list_tasks(
conn,
+32 -44
View File
@@ -19,6 +19,7 @@ from __future__ import annotations
from dataclasses import dataclass, field
import json
import sqlite3
import time
from typing import Any, Iterable, Optional
from hermes_cli import kanban_db as kb
@@ -83,16 +84,12 @@ def _activate_root_inline(
) -> bool:
"""Inline blocked→done CAS flip + event insert for the swarm root.
Runs INSIDE create_swarm's outer write_txn, so it must not call
``kb.complete_task`` — that helper opens its own transaction and fires
post-commit side effects (workspace cleanup, failure-counter clear,
``recompute_ready``) that would execute while the outer transaction can
still roll back. Instead we do the minimal durable writes here and let
the caller run ``recompute_ready`` after the outer commit.
Runs INSIDE create_swarm's write_txn, so it must not call
``kb.complete_task`` (own transaction + post-commit side effects that
would run while the outer txn can still roll back). The caller runs
``recompute_ready`` after the outer commit.
"""
import time as _time
now = int(_time.time())
now = int(time.time())
cur = conn.execute(
"""
UPDATE tasks
@@ -179,9 +176,8 @@ def create_swarm(
raise RuntimeError("could not activate the completed swarm topology")
activated = True
if activated:
# Outside the outer transaction: promote the root's children now
# that its 'done' flip is durable (recompute_ready opens its own
# txn and must never run under an open write_txn).
# After commit: recompute_ready opens its own txn and must never run
# under an open write_txn.
kb.recompute_ready(conn)
root = kb.get_task(conn, created.root_id)
run = kb.latest_run(conn, created.root_id)
@@ -249,9 +245,8 @@ def _create_swarm_uncommitted(
workspace_path=workspace_path,
)
# If idempotency returned an existing non-archived root, do not duplicate the
# swarm graph. Recover the topology from the root's latest blackboard, if it
# was created by this helper previously.
# Idempotency may return an existing root: recover its topology from the
# blackboard instead of duplicating the graph.
existing = latest_blackboard(conn, root).get("topology")
if isinstance(existing, dict):
worker_ids = [str(x) for x in existing.get("worker_ids", []) if x]
@@ -266,61 +261,54 @@ def _create_swarm_uncommitted(
)
context_suffix = _swarm_context(root, goal)
worker_ids: list[str] = []
for spec in worker_specs:
worker_id = kb.create_task(
common = dict(
created_by=created_by, tenant=tenant,
workspace_kind=workspace_kind, workspace_path=workspace_path,
)
worker_ids = [
kb.create_task(
conn,
title=spec.title,
body=(spec.body or "") + context_suffix,
assignee=spec.profile,
created_by=created_by,
parents=[root],
tenant=tenant,
priority=spec.priority or priority,
workspace_kind=workspace_kind,
workspace_path=workspace_path,
skills=spec.skills or None,
max_runtime_seconds=spec.max_runtime_seconds,
**common,
)
worker_ids.append(worker_id)
for spec in worker_specs
]
verifier_body = (
"Review every worker handoff and blackboard update. Gate the swarm: "
"complete only with metadata {\"gate\": \"pass\"} when evidence is "
"sufficient; otherwise block with exact missing work."
+ context_suffix
)
verifier = kb.create_task(
conn,
title=verifier_title,
body=verifier_body,
body=(
"Review every worker handoff and blackboard update. Gate the swarm: "
"complete only with metadata {\"gate\": \"pass\"} when evidence is "
"sufficient; otherwise block with exact missing work."
+ context_suffix
),
assignee=verifier_assignee,
created_by=created_by,
parents=worker_ids,
tenant=tenant,
priority=priority,
workspace_kind=workspace_kind,
workspace_path=workspace_path,
skills=["requesting-code-review"],
**common,
)
synthesizer_body = (
"Synthesize the verified worker outputs into the final deliverable. "
"Do not start until the verifier has passed the gate."
+ context_suffix
)
synthesizer = kb.create_task(
conn,
title=synthesizer_title,
body=synthesizer_body,
body=(
"Synthesize the verified worker outputs into the final deliverable. "
"Do not start until the verifier has passed the gate."
+ context_suffix
),
assignee=synthesizer_assignee,
created_by=created_by,
parents=[verifier],
tenant=tenant,
priority=priority,
workspace_kind=workspace_kind,
workspace_path=workspace_path,
skills=["humanizer"],
**common,
)
created = SwarmCreated(root, worker_ids, verifier, synthesizer)
+35 -70
View File
@@ -14,30 +14,21 @@ source board's slug)::
attachments/<task>/… attachment blobs (unless --no-attachments)
logs/<task>.log worker logs (only with --include-logs)
Two things make this more than a ``tar czf`` of the board directory.
Two things make this more than ``tar czf`` of the board directory:
**The database is live.** Kanban runs in WAL mode and a dispatcher may be
mid-write, so copying ``kanban.db`` off the filesystem yields a torn
snapshot that is missing whatever still sits in the ``-wal`` file. Export
goes through SQLite's online-backup API instead, which produces a
consistent single-file image of a database that is being written to.
* **The database is live** (WAL mode, dispatcher may be mid-write), so the
export uses SQLite's online-backup API for a consistent image instead of a
file copy that would miss the ``-wal`` sidecar.
* **Rows carry machine-local state** — claims, PIDs, heartbeats, absolute
paths, gateway chat subscriptions, session ids. Shipping them verbatim
would import claims owned by a stranger's process or push events into a
stranger's Telegram thread. Scrubbed on export and re-scrubbed on import
(an archive is untrusted); see :func:`_scrub_local_state` and
:func:`_relocate_imported_rows`.
**Rows carry machine-local state.** Claims, PIDs, heartbeats, absolute
workspace and attachment paths, gateway chat subscriptions, and session
ids are all meaningful only on the machine that wrote them. Shipping them
verbatim is how an imported board arrives holding claims owned by a
process on somebody else's laptop, or starts pushing task events into a
stranger's Telegram thread. Everything machine-local is scrubbed on the
export side (so the archive itself never carries it) and defensively
re-scrubbed on import; see :func:`_scrub_local_state` and
:func:`_relocate_imported_rows`.
Imports always land as a **new** board — the slug auto-suffixes on
collision — so an import can never mutate a board that is already there.
That also means an imported board is never ``default``, which is what
lets the import side ignore the default board's split on-disk layout
(``<root>/kanban.db`` beside ``<root>/kanban/attachments/``) and put
everything inside one ``boards/<slug>/`` directory.
Imports always land as a **new** board (slug auto-suffixes on collision), so
an import never mutates an existing board and is never ``default`` — which
lets the importer ignore the default board's split on-disk layout.
"""
from __future__ import annotations
@@ -74,13 +65,8 @@ _DISPATCHABLE_STATUSES = ("ready", "running", "todo", "scheduled")
# ---------------------------------------------------------------------------
def _snapshot_db(source: Path, target: Path) -> None:
"""Write a consistent copy of ``source`` to ``target``.
Uses SQLite's online-backup API rather than a file copy: in WAL mode
a just-committed page can still live in the ``-wal`` sidecar, so
copying only ``kanban.db`` loses recent writes and can produce a
torn image if the dispatcher commits mid-copy.
"""
"""Consistent copy of ``source`` via the online-backup API (a file copy
would miss pages still in the ``-wal`` sidecar and could tear)."""
src = sqlite3.connect(str(source))
try:
dst = sqlite3.connect(str(target))
@@ -93,14 +79,9 @@ def _snapshot_db(source: Path, target: Path) -> None:
def _scrub_local_state(conn: sqlite3.Connection) -> None:
"""Strip machine-local runtime state. Caller owns the transaction.
Runs on the export side so the archive itself never carries another
machine's claims, PIDs, or — the one that actually matters for a
board shared with someone else — the gateway chat ids subscribed to
its task events. Repeated on import because an archive is untrusted
input.
"""
"""Strip machine-local runtime state (claims, PIDs, and above all the
gateway chat ids subscribed to task events). Caller owns the transaction.
Run on export and again on import (an archive is untrusted input)."""
conn.execute("DELETE FROM kanban_notify_subs")
conn.execute(
"""
@@ -157,12 +138,9 @@ def export_board(
include_attachments: bool = True,
include_logs: bool = False,
) -> dict[str, Any]:
"""Export ``board`` to a ``tar.gz`` archive. Returns a summary dict.
``output_path`` may be given with or without the ``.tar.gz`` suffix.
Workspaces are never included: they are git worktrees and scratch
trees that are large, machine-local, and rebuilt on demand.
"""
"""Export ``board`` to a ``tar.gz`` (suffix optional on ``output_path``);
returns a summary dict. Workspaces are never included — large,
machine-local, rebuilt on demand."""
slug = kb._normalize_board_slug(board) or kb.get_current_board()
if not kb.board_exists(slug):
raise ValueError(f"board {slug!r} does not exist")
@@ -241,12 +219,8 @@ def export_board(
# ---------------------------------------------------------------------------
def _available_slug(preferred: str) -> str:
"""Return ``preferred``, or the first free ``<preferred>-N`` variant.
``default`` always reports as existing, so an archive exported from a
default board naturally lands as ``default-2`` instead of colliding
with the importer's own default board.
"""
"""``preferred`` or the first free ``<preferred>-N``. ``default`` always
exists, so a default-board export lands as ``default-2``."""
if not kb.board_exists(preferred):
return preferred
# Leave headroom for the suffix inside the 64-char slug limit.
@@ -295,22 +269,17 @@ def _read_board_metadata(path: Path) -> dict[str, Any]:
def _relocate_imported_rows(
conn: sqlite3.Connection, slug: str
) -> tuple[dict[str, int], list[str]]:
"""Re-anchor an imported board's rows to this machine.
"""Re-anchor an imported board's rows to this machine; returns
``(stats, warnings)``.
Returns ``(stats, warnings)``. Three things move:
* Attachment rows are repointed at this board's attachments tree.
Rows whose blob did not travel (an export made with
``--no-attachments``) are dropped, because a row pointing at a file
that does not exist breaks download in every UI that lists it.
* Workspace paths are cleared. ``scratch`` tasks regenerate one under
this board on the next claim, so they are simply reset. ``dir`` and
``worktree`` tasks cannot be resolved without a path that means
something here, so any that are still dispatchable are parked in
``triage`` — otherwise the dispatcher claims them, fails to build a
workspace, and burns them straight into the failure breaker.
* Runtime state is scrubbed again. Export already did this, but an
archive is an untrusted input and the cost is one UPDATE.
* Attachment rows are repointed at this board's tree; rows whose blob
did not travel (``--no-attachments``) are dropped, since a dangling row
breaks download in every UI.
* Workspace paths are cleared. ``scratch`` regenerates on next claim;
dispatchable ``dir``/``worktree`` tasks are parked in ``triage``,
otherwise the dispatcher claims them, fails to build a workspace, and
burns them into the failure breaker.
* Runtime state is scrubbed again (untrusted input, one UPDATE).
"""
warnings: list[str] = []
now = int(time.time())
@@ -389,12 +358,8 @@ def import_board(
*,
activate: bool = False,
) -> dict[str, Any]:
"""Import a board archive as a new board. Returns a summary dict.
``slug`` overrides the name from the archive. Either way the final
slug auto-suffixes if it is taken, so an import never merges into or
overwrites an existing board.
"""
"""Import an archive as a NEW board (``slug`` overrides the archive's;
either way it auto-suffixes if taken). Returns a summary dict."""
archive = Path(archive_path).expanduser()
if not archive.exists():
raise FileNotFoundError(f"archive not found: {archive}")
+367 -11202
View File
File diff suppressed because it is too large Load Diff
+179 -282
View File
@@ -1,9 +1,7 @@
"""Shared module-level constants for the SessionDB family of modules.
"""Shared constants and helpers for the SessionDB family of modules.
Extracted verbatim from hermes_state.py so the SessionDB mixin modules
(hermes_state_search / hermes_state_schema / hermes_state_portability) can
reference them without importing hermes_state (which would be a cycle).
hermes_state re-imports every name here for backward compatibility.
Lives outside hermes_state so the mixin modules can import it without a
cycle; hermes_state re-exports every name for backward compatibility.
"""
import contextlib
@@ -29,16 +27,11 @@ from agent.context_compressor import (
)
# Session preview = the head of the first user message, shown wherever a
# session has no title (sidebar rows, pickers, exports, the desktop's
# `sessionTitle` fallback).
#
# A /skill invocation expands into a message that embeds the whole skill body,
# so the plain head of it previews the SKILL's opening prose as if the user had
# written it. Scaffolded rows therefore carry a wider excerpt so
# ``_shape_preview`` can hand it to ``describe_skill_invocation`` and recover
# ``/work — fix the title leak``: the whole message while it stays under the
# budget, and head + tail (where the typed instruction lands) once it doesn't.
# Session preview = head of the first user message, shown wherever a session
# has no title. A /skill invocation embeds the whole skill body, so its plain
# head would preview the SKILL's prose; scaffolded rows carry a wider excerpt
# (whole message under budget, else head + tail where the typed instruction
# lands) so ``_shape_preview`` can recover ``/work — fix the title leak``.
_PREVIEW_HEAD_CHARS = 63
@@ -49,13 +42,9 @@ _PREVIEW_MAX_CHARS = 60
def escape_like(text: str) -> str:
"""Escape SQL LIKE wildcards so operator/session-derived text matches
literally. Pair with ``ESCAPE '\\'`` in the clause.
``%`` and ``_`` are wildcards to LIKE, and ``_`` in particular is common
in the values these patterns run against (branch names, session titles,
filesystem paths). A match documented as substring/prefix must not
silently widen.
"""Escape LIKE wildcards (``%``, ``_``) so derived text matches literally;
pair with ``ESCAPE '\\'``. ``_`` is common in branch names, titles and
paths, and a documented substring/prefix match must not silently widen.
"""
return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
@@ -90,10 +79,9 @@ def _sql_starts_with(expression: str, prefixes: tuple[str, ...]) -> str:
return "(" + " OR ".join(checks) + ")"
# Current and historical long-form prefixes share this complete introduction;
# their stale-item guidance diverges only after it. Matching the whole intro
# avoids treating an ordinary user message that merely starts with the short
# bracketed label as a compaction carrier.
# Current and legacy long-form prefixes share this whole introduction; matching
# all of it keeps an ordinary message that merely starts with the bracketed
# label from counting as a compaction carrier.
_PREVIEW_LONG_FORM_PREFIX = SUMMARY_PREFIX.split("Do NOT answer", 1)[0]
_PREVIEW_SUMMARY_PREFIXES = (
_PREVIEW_LONG_FORM_PREFIX,
@@ -127,9 +115,8 @@ _PREVIEW_FORCE_USER_REMAINDER_SQL = (
f" + {len(_SUMMARY_END_MARKER)})"
)
# Session preview subqueries select their first eligible user-authored content.
# Pure compaction rows are ineligible; force-user-leading and merged carriers
# remain eligible only when authentic content survives the wire boundary.
# Pure compaction rows are ineligible for previews; force-user-leading and
# merged carriers are eligible only when authentic content survives.
_PREVIEW_ELIGIBLE_SQL = (
f"((NOT {_PREVIEW_STANDALONE_SUMMARY_SQL} AND NOT {_PREVIEW_MERGED_SUMMARY_SQL})"
f" OR ({_PREVIEW_STANDALONE_SUMMARY_SQL}"
@@ -140,10 +127,8 @@ _PREVIEW_ELIGIBLE_SQL = (
)
# The shared ``_preview_raw`` SELECT expression, interpolated by every listing
# query. A scaffolded row gets a wider excerpt: the whole message while it fits
# the budget, else head + tail (where the typed instruction lands) spliced
# around SKILL_EXCERPT_JOINT.
# Shared ``_preview_raw`` SELECT expression for every listing query (scaffolded
# rows: head + tail spliced around SKILL_EXCERPT_JOINT when over budget).
_PREVIEW_RAW_SELECT = (
f"CASE WHEN {_PREVIEW_STANDALONE_SUMMARY_SQL}"
f" THEN {_PREVIEW_FORCE_USER_REMAINDER_SQL}"
@@ -173,8 +158,17 @@ def _shape_preview(raw: Any) -> str:
return text
# A child session counts as a /branch (kept visible, never cascade-deleted) if
# it carries the stable marker OR the legacy end_reason heuristic holds.
# Correlated ``_preview_raw`` column for a ``sessions s`` row.
_PREVIEW_RAW_SUBQUERY_SQL = (
f"COALESCE((SELECT {_PREVIEW_RAW_SELECT} FROM messages m"
f" WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL"
f" AND {_PREVIEW_ELIGIBLE_SQL}"
f" ORDER BY m.timestamp, m.id LIMIT 1), '') AS _preview_raw"
)
# A /branch child (kept visible, never cascade-deleted): stable marker OR the
# legacy end_reason heuristic.
_BRANCH_CHILD_SQL = (
"json_extract(COALESCE({a}.model_config, '{{}}'), '$._branched_from') IS NOT NULL"
" OR EXISTS (SELECT 1 FROM sessions p"
@@ -193,12 +187,10 @@ _COMPRESSION_CHILD_SQL = (
_RESET_END_REASONS = (
"session_reset",
# switch_session() never creates a child row, but pre-marker DBs can hold
# legacy reset children whose parent later ended with 'session_switch'
# (resumed then switched away before reopen-time stamping existed). Also
# keeps this set identical to the recovery fence in
# find_latest_gateway_session_for_peer, which interpolates
# _RESET_END_REASONS_SQL so the two cannot drift.
# switch_session() creates no child row, but pre-marker DBs hold legacy
# reset children whose parent later ended 'session_switch'. Also keeps
# this set identical to the recovery fence in
# find_latest_gateway_session_for_peer (which interpolates the SQL form).
"session_switch",
"idle",
"daily",
@@ -207,34 +199,25 @@ _RESET_END_REASONS = (
)
_RESET_END_REASONS_SQL = ", ".join(f"'{reason}'" for reason in _RESET_END_REASONS)
# Accidental end reasons that recovery treats as resumable (see
# docs/session-lifecycle.md "recoverable accidental reasons"). Interpolated
# into the recovery SQL below AND exposed as SessionDB.RECOVERABLE_END_REASONS
# so the tuple is the single source of truth — literals cannot drift.
# Accidental end reasons recovery treats as resumable (docs/session-lifecycle.md).
# Single source of truth: interpolated into recovery SQL AND exposed as
# SessionDB.RECOVERABLE_END_REASONS.
_RECOVERABLE_END_REASONS = (
"agent_close",
"ws_orphan_reap",
# A stale sentinel-parked runtime quietly superseded by a fresh
# session.resume of the same stored session (no reclaimed broadcast);
# the stored session stays resumable like any accidental end.
# Stale sentinel-parked runtime superseded by a fresh session.resume.
"superseded_by_resume",
# Startup sweep of rows orphaned by a dead gateway process (#65194):
# the in-process ws-orphan grace timer died with the process, so the
# row was closed at the next boot instead. Same accident class as
# ws_orphan_reap — kept distinct for forensics — and equally resumable.
# Startup sweep of rows orphaned by a dead gateway process: same accident
# class as ws_orphan_reap, kept distinct for forensics.
"startup_orphan_reap",
)
_RECOVERABLE_END_REASONS_SQL = ", ".join(f"'{reason}'" for reason in _RECOVERABLE_END_REASONS)
# End reasons written by AUTOMATIC infrastructure cleanup (server shutdown,
# orphan reapers, idle/LRU eviction) rather than by a deliberate conversation
# boundary (compression, session_reset, session_switch, explicit user close).
# An automatic stamp records "some runtime went away", NOT "this conversation
# ended" — so a writer that can prove the conversation is still live (e.g. an
# active compression rotation holding the lease, #88197) may treat the stamp
# as stale and clear it. Superset of the recoverable set: those are already
# resumable accidents; the extra TUI reasons are the same accident class but
# were historically only known to tui_gateway's _AUTOMATIC_SESSION_END_REASONS.
# End reasons written by AUTOMATIC cleanup (shutdown, orphan reapers, idle/LRU
# eviction), not by a deliberate conversation boundary. Such a stamp means
# "some runtime went away", NOT "this conversation ended", so a writer that can
# prove liveness (e.g. a compression rotation holding the lease) may clear it.
# Superset of the recoverable set plus the TUI gateway's automatic reasons.
_AUTOMATIC_END_REASONS = frozenset(_RECOVERABLE_END_REASONS) | {
"tui_shutdown",
"ws_disconnect",
@@ -244,23 +227,17 @@ _AUTOMATIC_END_REASONS = frozenset(_RECOVERABLE_END_REASONS) | {
def is_automatic_end_reason(reason) -> bool:
"""True when *reason* is an automatic-cleanup end stamp (see above).
Single owner of the "accidental vs deliberate end" predicate — every
compression-liveness site must call this instead of re-implementing the
reason taxonomy (#88197, never-patch-predicates).
"""
"""True when *reason* is an automatic-cleanup end stamp (see above). Single
owner of the accidental-vs-deliberate predicate; compression-liveness
sites must call this rather than re-implement the taxonomy."""
return isinstance(reason, str) and reason in _AUTOMATIC_END_REASONS
def _legacy_reset_child_sql(alias: str, reasons_sql: str) -> str:
"""Pre-marker reset-continuation heuristic.
A child is a legacy reset continuation when it rides its parent's exact
non-empty routing key and the parent ended at a reset boundary. Shared by
the listing predicate (``_RESET_CHILD_SQL``) and ``reopen_session()``'s
marker-stamping UPDATE so the two sites cannot drift; ``reasons_sql`` is
either the literal ``_RESET_END_REASONS_SQL`` or a bound-placeholder list.
"""Pre-marker reset-continuation heuristic: child rides its parent's exact
non-empty routing key and the parent ended at a reset boundary. Shared by
``_RESET_CHILD_SQL`` and ``reopen_session()``'s marker-stamping UPDATE so
the two cannot drift; ``reasons_sql`` is a literal or placeholder list.
"""
return (
f"EXISTS (SELECT 1 FROM sessions p"
@@ -272,19 +249,17 @@ def _legacy_reset_child_sql(alias: str, reasons_sql: str) -> str:
)
# A reset starts a separate user-visible conversation even though gateway rows
# retain parent_session_id for durable lineage. New rows carry the stable
# marker; the same-key fallback recovers rows written before the marker existed.
# Requiring the exact non-empty routing key keeps ordinary child/subagent rows
# out even when their parent is later reset.
# A reset starts a separate user-visible conversation though rows keep
# parent_session_id for lineage. Stable marker, or the same-key fallback for
# pre-marker rows (the exact-key requirement keeps subagent children out).
_RESET_CHILD_SQL = (
"json_extract(COALESCE({a}.model_config, '{{}}'), '$._reset_from') IS NOT NULL"
" OR " + _legacy_reset_child_sql("{a}", _RESET_END_REASONS_SQL)
)
# Rows that surface in pickers: roots + branch/reset children. Subagent runs
# and compression continuations stay hidden.
# Picker-visible rows: roots + branch/reset children (not subagent runs or
# compression continuations).
_LISTABLE_CHILD_SQL = (
f"(s.parent_session_id IS NULL OR {_BRANCH_CHILD_SQL.format(a='s')}"
f" OR {_RESET_CHILD_SQL.format(a='s')})"
@@ -305,14 +280,9 @@ def _ephemeral_child_sql(alias: str = "s") -> str:
def _sql_session_last_active(alias: str = "s") -> str:
"""SQL expression for session recency used by list/status surfaces.
Freshest of ``last_activity_at`` (mid-turn agent activity heartbeat) and
the latest message timestamp, then fall back to ``started_at``.
Must not prefer a stale heartbeat over a newer message: durable
heartbeats are rate-limited (~60s), so after a turn writes messages
``last_activity_at`` can lag ``MAX(messages.timestamp)``.
"""Session recency: freshest of ``last_activity_at`` and the latest message
timestamp, else ``started_at``. Heartbeats are rate-limited (~60s) so
``last_activity_at`` can lag a newer message; never prefer it alone.
"""
msg_max = (
f"(SELECT MAX(_act_m.timestamp) FROM messages _act_m "
@@ -356,24 +326,38 @@ def _sql_session_last_active_by_id(session_id_expr: str) -> str:
SCHEMA_VERSION = 28
# Auto-maintenance VACUUMs only when at least this fraction of pages is on the
# freelist; below it a full rewrite costs more I/O than it returns.
AUTO_VACUUM_MIN_FREELIST_RATIO = 0.25
# FTS storage-layout version, tracked INDEPENDENTLY of SCHEMA_VERSION in the
# state_meta key ``fts_storage_version``. The main schema version advances
# freely on open (so future migrations always land); the FTS *layout* only
# reaches the current version when a DB is either born fresh or explicitly
# optimized via ``hermes sessions optimize-storage``. A legacy DB sits at
# layout 0 (marker absent) with a working inline index until the user opts in.
# 1 = v23 external-content layout (content/tool_name/tool_calls,
# tool-row-excluded trigram)
# FTS storage layout, tracked INDEPENDENTLY of SCHEMA_VERSION (state_meta
# ``fts_storage_version``): the schema version advances freely on open, but
# the FTS layout only changes when a DB is born fresh or explicitly optimized
# via ``hermes sessions optimize-storage``. Legacy DBs sit at 0 (marker
# absent) with a working inline index. 1 = v23 external-content layout.
FTS_STORAGE_VERSION = 1
# Cap on user-controlled FTS5 query input before regex/sanitizer processing.
# Search queries do not need to be arbitrarily large, and bounding them keeps
# sanitizer/runtime behavior predictable under adversarial input.
# Cap on user-controlled FTS5 query input before sanitizer processing.
MAX_FTS5_QUERY_CHARS = 2_048
# ── Helpers shared by SessionDB, its mixins and the registry ──────────────
def stat_db_file_identity(path) -> "tuple[int, int] | None":
"""``(st_dev, st_ino)`` for *path*, or None. st_ino=0 (Windows, some
network FS) would false-positive every replaced-file check, so it counts
as unknown."""
try:
st = os.stat(path)
except OSError:
return None
if not st.st_dev or not st.st_ino:
return None
return (st.st_dev, st.st_ino)
_FTS_TRIGGERS = (
"messages_fts_insert",
"messages_fts_delete",
@@ -628,10 +612,8 @@ CREATE INDEX IF NOT EXISTS idx_async_delegations_delivery
"""
# Indexes that reference columns added in later schema versions must be
# created AFTER _reconcile_columns() has had a chance to ADD them on
# existing databases. SCHEMA_SQL above is run by sqlite executescript
# which would otherwise fail on legacy DBs ("no such column: active").
# Indexes on columns added in later schema versions must run AFTER
# _reconcile_columns() adds them, or executescript fails on legacy DBs.
DEFERRED_INDEX_SQL = """
CREATE INDEX IF NOT EXISTS idx_messages_session_active
ON messages(session_id, active, timestamp);
@@ -648,25 +630,17 @@ CREATE INDEX IF NOT EXISTS idx_sessions_system_prompt_hash
"""
# ── Deferred FTS rebuild bookkeeping (schema v23) ──
# While a background index rebuild is pending, two state_meta keys define
# which message rows are currently IN the FTS indexes:
# ── Deferred FTS rebuild bookkeeping ──
# While a background rebuild is pending, two state_meta keys define which rows
# are IN the FTS indexes: H = fts_rebuild_high_water (MAX(messages.id) when the
# old indexes were dropped), P = fts_rebuild_progress (highest backfilled id).
# A row is indexed iff id <= P OR id > H (AUTOINCREMENT ids, so post-drop rows
# are indexed live by the insert triggers); rows in (P, H] are not.
#
# fts_rebuild_high_water H — MAX(messages.id) at the moment the old
# indexes were dropped
# fts_rebuild_progress P — highest id the chunked backfill has indexed
#
# A row is indexed iff id <= P (backfilled) OR id > H (inserted after
# the drop; ids are AUTOINCREMENT so new rows are always > H and the insert
# triggers index them live). Rows in (P, H] are not yet indexed.
#
# Every trigger below gates on that same predicate: firing an FTS5
# external-content 'delete' for a row that is NOT in the index corrupts the
# index, and skipping it for a row that IS indexed leaves a stale entry.
# When no rebuild is pending both keys are absent and COALESCE turns the
# predicate into a tautology (id > -1 OR id <= -1), i.e. normal operation.
# The two state_meta PK probes per write are negligible next to the FTS
# insert itself.
# Every trigger gates on that predicate: an FTS5 external-content 'delete' for
# a row NOT in the index corrupts it, and skipping one for an indexed row
# leaves a stale entry. With no rebuild pending both keys are absent and
# COALESCE makes the predicate a tautology.
FTS_SQL = """
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
content,
@@ -717,20 +691,12 @@ END;
"""
# Trigram FTS5 table for CJK substring search. The default unicode61
# tokenizer splits CJK characters into individual tokens, breaking phrase
# matching. The trigram tokenizer creates overlapping 3-byte sequences so
# substring queries work natively for any script (CJK, Thai, etc.).
#
# The trigram index is the most expensive index in state.db (~2.6x the size
# of the text it covers), and ``role='tool'`` rows are ~90% of message bytes
# while being almost entirely machine noise (base64 payloads, file dumps,
# delegation transcripts). The index therefore reads through
# ``messages_fts_trigram_src``, a view that excludes tool rows — they stay
# fully stored in ``messages`` and fully searchable via the standard
# ``messages_fts`` index; they just don't get trigram (CJK substring)
# treatment. ``search_messages`` routes CJK queries that filter on
# ``role='tool'`` to the LIKE fallback for the same reason.
# Trigram FTS5 table for CJK substring search (unicode61 splits CJK into single
# tokens, breaking phrase matching). The trigram index is ~2.6x the text it
# covers and ``role='tool'`` rows are ~90% of message bytes of machine noise,
# so it reads through the ``messages_fts_trigram_src`` view, which excludes
# tool rows; those remain searchable via ``messages_fts``, and
# ``search_messages`` routes CJK queries filtered on role='tool' to LIKE.
FTS_TRIGRAM_SQL = """
CREATE VIEW IF NOT EXISTS messages_fts_trigram_src AS
SELECT id, role, content, tool_name, tool_calls
@@ -796,18 +762,15 @@ _FTS_CJK_TRIGGERS = (
)
# state_meta breadcrumb set when a tokenizer-less process had to drop the
# cjk triggers to keep message writes alive: rows written from that moment
# on are missing from the cjk index, so it must not serve reads until
# Set when a tokenizer-less process dropped the cjk triggers to keep writes
# alive: the cjk index is missing rows and must not serve reads until
# `hermes sessions optimize-storage` rebuilds it on a capable host.
FTS_CJK_STALE_KEY = "fts_cjk_stale"
# Durable breadcrumb for a base/trigram FTS index that was detached from the
# canonical messages table after runtime corruption. While present, startup
# must rebuild the complete index before reinstalling sync triggers: rows may
# have been written while those triggers were absent, so merely recreating
# them would preserve an unknown index gap.
# Set when a base/trigram FTS index was detached after runtime corruption.
# While present, startup must rebuild the complete index before reinstalling
# sync triggers: rows written while they were absent leave an unknown gap.
FTS_STALE_KEY = "fts_stale"
# Durable diagnostic for stale FTS recovery blocked across process restarts.
@@ -815,16 +778,11 @@ FTS_REBUILD_DEFERRAL_KEY = "fts_rebuild_deferral"
# ── Legacy (v22 / inline-content) FTS DDL ──────────────────────────────
# Used ONLY to keep an existing pre-v23 install's search working and its
# triggers repairable UNTIL the user opts into `hermes db optimize`. This is
# the exact inline shape v11..v22 shipped: each virtual table stores its own
# copy of ``content || tool_name || tool_calls`` and the trigram table indexes
# every row (including role='tool'). We never CREATE these on a fresh install —
# fresh installs are born on the v23 external-content schema above. These
# constants exist so a legacy DB is never accidentally handed the v23 DDL
# (which would create the external-content trigram source VIEW and leave the
# DB in a mixed, broken state). `optimize_fts_storage()` is what migrates a
# legacy DB to the v23 shape.
# Used ONLY to keep a pre-v23 install's search working and its triggers
# repairable until `optimize_fts_storage()` migrates it: inline copies of
# content || tool_name || tool_calls, trigram over every row. Never created
# on a fresh install. Handing a legacy DB the v23 DDL would create the
# external-content trigram VIEW and leave it in a mixed, broken state.
LEGACY_FTS_SQL = """
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
content
@@ -882,39 +840,25 @@ END;
# ── Cross-process full-FTS-rebuild admission (single authority) ──────────────
#
# Several independent Hermes processes routinely share one state.db (gateway
# service, the Desktop app's `hermes serve` backend, interactive CLI sessions,
# the TUI slash worker). A full structural FTS rebuild — the FTS5 'rebuild'
# command or the drop/recreate script in `_recover_stale_fts` — must only ever
# run in ONE of them at a time: two concurrent rebuilds collide on write and
# have structurally corrupted state.db in production (PR #93200; the
# 2026-08-15 / 2026-08-23 incidents and issues #89293 / #90950).
# Several Hermes processes share one state.db. A full structural FTS rebuild
# (FTS5 'rebuild' or the drop/recreate in `_recover_stale_fts`) must run in
# ONE of them at a time: concurrent rebuilds have structurally corrupted
# state.db in production. This is the single admission authority for
# `rebuild_fts()`, `_rebuild_fts_indexes()` and `_recover_stale_fts()`. The
# chunked backfill (`fts_rebuild_step`) is deliberately NOT routed through it:
# it claims progress under SQLite transaction authority and is multi-process.
#
# This is the single admission authority for every full structural rebuild
# entry point: `SessionSearchMixin.rebuild_fts()`,
# `SessionSchemaMixin._rebuild_fts_indexes()` (via `_init_schema`), and
# `SessionSchemaMixin._recover_stale_fts()`. The chunked deferred backfill
# (`fts_rebuild_step`) is deliberately NOT routed through it — it claims
# progress under `_execute_write`'s SQLite transaction authority and is
# intentionally multi-process.
# Semantics mirror `hermes_state._cross_process_repair_lock`: portable (msvcrt
# on Windows, flock elsewhere), bounded wait, FAIL CLOSED. The kernel drops
# the lock when the holder dies UNLESS a forked child inherited the fd (flock
# rides the open file description), which holds it forever; so the holder's
# pid + start time are recorded under the lock and a provably-dead holder's
# lock is broken by unlinking and retaking on a fresh inode. Indeterminate
# liveness still defers. Lives here because the mixins cannot import
# hermes_state (cycle).
#
# Semantics mirror `hermes_state._cross_process_repair_lock` (the schema-
# surgery authority): portable (msvcrt on Windows, flock elsewhere), bounded
# wait, and FAIL CLOSED — a caller that cannot acquire the lock must NOT
# rebuild. The kernel drops both lock types when the holder dies — UNLESS a
# forked child inherited the lock fd (flock rides the open file description,
# which fork() duplicates), in which case the orphaned descriptor holds the
# lock forever (issue #100108). `_acquire_db_flock` therefore records the
# holder's pid + start time under the lock and, when the recorded holder is
# provably dead, breaks the orphaned lock by unlinking and retaking it on a
# fresh inode; indeterminate liveness still defers. It lives here (not
# hermes_state) because the search/schema mixins cannot import hermes_state
# (cycle).
#
# The lock file is `<db>.fts_rebuild.lock`, distinct from `<db>.repair.lock`:
# schema surgery runs on an EXCLUSIVE offline connection and can legitimately
# take minutes in VACUUM, while runtime rebuilds run on live connections. The
# timeout is sized for a full 'rebuild' of both indexes on a large DB.
# `<db>.fts_rebuild.lock` is distinct from `<db>.repair.lock`: schema surgery
# runs on an EXCLUSIVE offline connection and may take minutes in VACUUM.
logger = logging.getLogger("hermes_state")
@@ -922,31 +866,23 @@ _FTS_REBUILD_LOCK_TIMEOUT_SECONDS = 120.0
_FTS_REBUILD_LOCK_POLL_SECONDS = 0.1
_IS_WINDOWS = sys.platform == "win32"
# Post-break re-acquire budget: once a provably-orphaned lock has been broken
# the fresh inode is uncontended (or contended only by live processes), so a
# short bounded wait suffices — never re-enter the full timeout.
# Post-break re-acquire budget: the fresh inode is contended only by live
# processes, so a short wait suffices — never re-enter the full timeout.
_LOCK_BREAK_REACQUIRE_SECONDS = 5.0
# errno set for "another process holds this advisory lock". flock() reports
# contention as EWOULDBLOCK/EAGAIN; msvcrt.locking() as EACCES (and EDEADLK
# when its internal retry gives up). Anything else — ESTALE on a dropped NFS
# handle, ENOTSUP/ENOLCK on a filesystem without advisory locks, EIO — is a
# persistent environment failure that no amount of polling turns into an
# acquire. Treating every OSError as contention made such a failure look
# like a live holder and burned the full 120s admission timeout on every
# attempt (#100108, PR #100130).
# "Another process holds the lock": flock → EWOULDBLOCK/EAGAIN, msvcrt.locking
# → EACCES (EDEADLK when its retry gives up). Anything else (ESTALE, ENOTSUP,
# ENOLCK, EIO) is a persistent environment failure that polling cannot fix;
# treating it as contention burned the full timeout on every attempt.
_LOCK_CONTENTION_ERRNOS = {errno.EAGAIN, errno.EACCES, errno.EWOULDBLOCK}
if hasattr(errno, "EDEADLK"):
_LOCK_CONTENTION_ERRNOS.add(errno.EDEADLK)
def is_advisory_lock_contention(exc: BaseException) -> bool:
"""True when *exc* means another process holds the advisory lock.
False for every other ``OSError`` (ESTALE, ENOTSUP, ENOLCK, EIO, ...):
callers must fail closed IMMEDIATELY rather than poll to the deadline,
because retrying cannot succeed and the wait only stalls the caller.
"""
"""True when *exc* means another process holds the advisory lock. For
any other ``OSError`` callers must fail closed IMMEDIATELY: retrying
cannot succeed and polling only stalls the caller."""
if isinstance(exc, BlockingIOError):
return True
if not isinstance(exc, OSError):
@@ -955,13 +891,9 @@ def is_advisory_lock_contention(exc: BaseException) -> bool:
def _proc_start_ticks(pid: int):
"""Kernel start time of *pid* in clock ticks, or None when unknowable.
Field 22 of ``/proc/<pid>/stat`` (``starttime``) uniquely identifies a
process together with its PID: a recycled PID gets a different start
time. Returns None off Linux or on any read/parse failure — callers must
treat None as "unknowable" and FAIL CLOSED.
"""
"""Kernel start time of *pid* (field 22 of ``/proc/<pid>/stat``), which
with the PID uniquely identifies a process; None off Linux or on any
failure — callers must treat None as unknowable and FAIL CLOSED."""
try:
with open(f"/proc/{pid}/stat", "rb") as fh:
stat = fh.read()
@@ -988,12 +920,8 @@ def _read_lock_holder_record(handle):
def _write_lock_holder_record(handle) -> None:
"""Record this process as the lock holder (advisory, best effort).
Written under the flock so contenders that time out can tell an
orphaned-fd holder (recorded process dead, flock inherited by a forked
child — issue #100108) from a live wedged holder.
"""
"""Record this process as holder (best effort), written under the flock so
timed-out contenders can tell an orphaned-fd holder from a live wedged one."""
try:
record = {
"pid": os.getpid(),
@@ -1009,12 +937,8 @@ def _write_lock_holder_record(handle) -> None:
def _clear_lock_holder_record(handle) -> None:
"""Erase holder metadata before a normal release.
Guarantees that a surviving record always describes an ABNORMAL exit
(holder died without releasing), which is the only condition under which
a contender may break the lock.
"""
"""Erase holder metadata before a normal release, so a surviving record
always means an ABNORMAL exit — the only condition allowing a break."""
try:
handle.seek(0)
handle.truncate()
@@ -1025,12 +949,8 @@ def _clear_lock_holder_record(handle) -> None:
def _lock_holder_provably_dead(record) -> bool:
"""True ONLY when the recorded holder is provably dead or PID-recycled.
Any indeterminate state (no record, malformed record, PID owned by
another user, /proc unavailable, start-time unknowable) returns False —
the caller must FAIL CLOSED and defer, never break a possibly-live
holder's lock.
"""
Anything indeterminate (no/malformed record, PID owned by another user,
/proc unavailable) is False — the caller must FAIL CLOSED and defer."""
if not isinstance(record, dict):
return False
try:
@@ -1052,34 +972,27 @@ def _lock_holder_provably_dead(record) -> bool:
current_ticks = _proc_start_ticks(pid)
if current_ticks is None:
return False
# Same PID, different kernel start time: the recorded holder is dead and
# its PID was recycled by an unrelated process.
# Same PID, different start time: recycled by an unrelated process.
return current_ticks != recorded_ticks
def _acquire_db_flock(lock_path, handle, timeout_seconds, poll_seconds, description):
"""Bounded POSIX flock acquire with orphaned-holder staleness break.
Returns ``(acquired, handle)``; *handle* may have been re-opened (the
caller owns closing whichever handle comes back). *acquired* is True on
success, False when a holder kept the lock past the deadline, and None
when a non-contention ``OSError`` (ESTALE/ENOTSUP/EIO) made acquisition
impossible — already logged here; callers treat None as "not acquired"
without emitting the held-by-another-process warning.
Returns ``(acquired, handle)``; *handle* may have been re-opened and the
caller closes whichever comes back. *acquired* is True, False (a holder
kept the lock past the deadline), or None (non-contention ``OSError``,
already logged; callers treat it as not acquired without the
held-by-another-process warning).
Why breaking exists at all (issue #100108): ``flock`` belongs to the open
file DESCRIPTION, which ``fork()`` duplicates into every child. A holder
that forks (multiprocessing worker, daemonized helper) and then dies
leaves the flock held by a child that will never release it — the
kernel's holder-death release never triggers, and every contender defers
forever. The recorded-holder liveness check distinguishes exactly that
case: the process that ACQUIRED is provably dead (so its critical section
died with it), yet the flock is still held. Only then is the lock file
unlinked and retaken on a fresh inode; the orphan's flock stays on the
old unlinked inode where it blocks nobody. Every successful acquire
verifies its inode still names *lock_path*, so a racer that locked a dead
inode retries instead of running concurrently with the breaker.
Indeterminate liveness always defers (fail closed).
``flock`` belongs to the open file DESCRIPTION, which ``fork()``
duplicates: a holder that forks then dies leaves the lock held forever by
a child that never releases. When the process that ACQUIRED is provably
dead (its critical section died with it) yet the flock is held, the file
is unlinked and retaken on a fresh inode; the orphan's flock stays on the
old inode blocking nobody. Every successful acquire verifies its inode
still names *lock_path*, so a racer that locked a dead inode retries
instead of running alongside the breaker. Indeterminate liveness defers.
"""
import fcntl
@@ -1090,9 +1003,7 @@ def _acquire_db_flock(lock_path, handle, timeout_seconds, poll_seconds, descript
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except (BlockingIOError, OSError) as exc:
if not is_advisory_lock_contention(exc):
# ESTALE / ENOTSUP / EIO: not a holder, and polling cannot
# fix it. Defer NOW instead of pretending a live process
# held the lock for the whole timeout (#100108).
# Not a holder and polling cannot fix it: defer NOW.
logger.warning(
"Could not acquire %s %s (%s) — deferring rather than "
"waiting out the %.0fs holder timeout on a "
@@ -1135,9 +1046,8 @@ def _acquire_db_flock(lock_path, handle, timeout_seconds, poll_seconds, descript
broke_lock = True
deadline = time.monotonic() + _LOCK_BREAK_REACQUIRE_SECONDS
continue
# flock acquired — verify the path still names our inode: a breaker
# may have unlinked/replaced the file while we waited, and a lock on
# a dead inode excludes nobody.
# Verify the path still names our inode: a breaker may have replaced
# the file while we waited, and a lock on a dead inode excludes nobody.
try:
fd_stat = os.fstat(handle.fileno())
path_stat = os.stat(lock_path)
@@ -1178,21 +1088,13 @@ def _describe_lock_holder(record) -> str:
def fts_rebuild_admission(db_path, *, timeout_seconds=None):
"""Serialize full structural FTS rebuilds on *db_path* across processes.
Yields True when this process holds the rebuild authority, False when the
bounded acquire timed out or the lock file could not be opened at all. A
caller that gets False must NOT perform a full rebuild — proceeding is
exactly the concurrent-rebuild interleaving this lock exists to prevent
(fail closed). The deferred/stale breadcrumb machinery already guarantees
a skipped rebuild is retried later.
Yields True when this process holds the authority, False when the bounded
acquire timed out or the lock file could not be opened. On False the
caller must NOT rebuild (fail closed); the stale breadcrumb guarantees a
retry. ``db_path`` None (in-memory DB) yields True.
``db_path`` may be a str or Path; None (in-memory DB / tests without a
file path) yields True — a private in-memory DB has no cross-process
surface.
*timeout_seconds* defaults to ``_FTS_REBUILD_LOCK_TIMEOUT_SECONDS``.
Opportunistic in-process retries (``retry_deferred_fts_recovery``) pass
``0`` so a live holder never stalls a long-lived writer for two minutes;
the orphaned-holder break still applies on the single attempt.
Opportunistic in-process retries pass ``timeout_seconds=0`` so a live
holder never stalls a long-lived writer; the orphan break still applies.
"""
if db_path is None:
yield True
@@ -1206,16 +1108,12 @@ def fts_rebuild_admission(db_path, *, timeout_seconds=None):
try:
handle = open(lock_path, "a+b")
except OSError as exc:
# Fail closed, exactly as a timed-out acquire does. A lock file we
# cannot even open means the filesystem is out of space, inodes or
# descriptors — and a sibling process that opened ITS handle before
# the disk filled is still holding the authority and rebuilding.
# Yielding True here handed every process on a full disk a concurrent
# structural rebuild of the same live state.db with no cross-process
# authority at all: the disk-full trigger and the re-corruption on
# every multi-writer boot in #100368. Deferring costs nothing that
# was reachable anyway — the breadcrumb retries, and on a read-only
# directory the rebuild's own writes could not have committed either.
# Fail closed like a timed-out acquire. An unopenable lock file means
# the FS is out of space/inodes/descriptors, and a sibling that opened
# its handle earlier may still be rebuilding; yielding True here gave
# every process on a full disk a concurrent rebuild of the same DB.
# Deferring costs nothing: the breadcrumb retries, and the rebuild's
# own writes could not have committed either.
logger.warning(
"Could not open FTS rebuild lock %s (%s) — deferring this rebuild "
"rather than running it without cross-process authority.",
@@ -1257,14 +1155,13 @@ def fts_rebuild_admission(db_path, *, timeout_seconds=None):
"FTS rebuild lock",
)
if acquired is None:
# Non-contention failure: already logged with the real errno;
# a "held by another process" line here would be a lie.
# Already logged with the real errno; "held by another process"
# would be a lie.
acquired = False
elif not acquired:
record = None if _IS_WINDOWS else _read_lock_holder_record(handle)
if timeout <= 0:
# Non-blocking probe from an in-process retry: a busy lock
# is expected and will be tried again, so keep it quiet.
# Non-blocking probe from an in-process retry: keep it quiet.
logger.info(
"FTS rebuild lock %s is busy — deferring this retry "
"(the stale-FTS breadcrumb keeps it retryable). "
File diff suppressed because it is too large Load Diff
+674
View File
@@ -0,0 +1,674 @@
"""state.db file-level health helpers.
Split out of ``hermes_state.py``: header probes (application_id / zeroed-file
detection), deleted-WAL-sidecar holder scans, quarantine of zeroed or
lock-poisoned databases, ``collect_state_db_stats`` and holder-process
classification. Every name is re-imported into ``hermes_state`` so
``hermes_state.<name>`` keeps resolving — and tests that monkeypatch it keep
intercepting, because intra-module calls to patched helpers go through a
lazy ``from hermes_state import ...`` at call time.
"""
from __future__ import annotations
import contextlib
import json
import logging
import os
import sqlite3
import struct
import sys
import threading
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Set, Tuple
from hermes_state_common import (
FTS_REBUILD_DEFERRAL_KEY,
stat_db_file_identity as _stat_db_file_identity,
)
# Log-record parity with the origin module (caplog tests pin "hermes_state").
logger = logging.getLogger("hermes_state")
# _read_sqlite_application_id runs on EVERY write via _raise_if_db_replaced,
# against the LIVE state.db. A bare open()/read()/close() there is the
# howtocorrupt §2.2 bug: close() cancels every POSIX advisory lock this
# process holds on the file — measured on Linux/SQLite 3.53.1, one probe call
# drops the WAL-mode DMS shared lock the writer connection holds on state.db
# (see hermes_cli/sqlite_safe_read.py for the module built around this rule).
# With the DMS lock gone, a fresh opener in another process can treat this
# writer as dead and rerun WAL-index recovery underneath it.
#
# The probe therefore reads through a per-path fd cached for the life of the
# process: opening an fd never cancels locks (only close() does), and
# os.pread takes no shared file position. When the path is re-pointed at a
# new inode (the very replacement this probe exists to detect), the stale fd
# is RETIRED, never closed — closing it would cancel the live connection's
# locks on the old file, the exact bug being avoided. Replacement events are
# rare and halt writes anyway, so the leak is bounded.
_HEADER_PROBE_LOCK = threading.Lock()
_HEADER_PROBE_FDS: "dict[str, tuple[int, int, int]]" = {} # key -> (fd, dev, ino)
_RETIRED_HEADER_PROBE_FDS: "list[int]" = [] # intentionally never closed
def _pread_db_header(db_path: Path, length: int) -> "Optional[bytes]":
"""Lock-safe raw header read of a possibly-live SQLite database.
POSIX: pread from a cached, never-closed fd (rebound when the path names
a new inode). Windows: plain read — advisory-lock cancellation is a
POSIX-only hazard and msvcrt locks do not share the failure mode.
"""
from hermes_state import _IS_WINDOWS
if _IS_WINDOWS:
try:
with db_path.open("rb") as handle:
return handle.read(length)
except OSError:
return None
key = str(db_path)
try:
st = os.stat(db_path)
except OSError:
return None
with _HEADER_PROBE_LOCK:
cached = _HEADER_PROBE_FDS.get(key)
if cached is not None and (cached[1], cached[2]) != (st.st_dev, st.st_ino):
# Path re-pointed at a new file. Retire (never close) the old fd.
_RETIRED_HEADER_PROBE_FDS.append(cached[0])
cached = None
del _HEADER_PROBE_FDS[key]
if cached is None:
try:
fd = os.open(db_path, os.O_RDONLY)
except OSError:
return None
try:
fst = os.fstat(fd)
except OSError:
_RETIRED_HEADER_PROBE_FDS.append(fd)
return None
cached = (fd, fst.st_dev, fst.st_ino)
_HEADER_PROBE_FDS[key] = cached
try:
return os.pread(cached[0], length, 0)
except OSError:
return None
def _read_sqlite_application_id(db_path: Path) -> "Optional[int]":
"""Read application_id from the SQLite header without opening a connection.
Safe against live databases: routed through :func:`_pread_db_header`,
which never issues a ``close()`` that would cancel this process's POSIX
locks on the file (howtocorrupt §2.2).
"""
from hermes_state import _STATE_DB_APPLICATION_ID_OFFSET
header = _pread_db_header(db_path, _STATE_DB_APPLICATION_ID_OFFSET + 4)
if header is None:
return None
if len(header) < _STATE_DB_APPLICATION_ID_OFFSET + 4:
return None
if header[:16] != b"SQLite format 3\x00":
return None
return int(
struct.unpack(
">I",
header[_STATE_DB_APPLICATION_ID_OFFSET:_STATE_DB_APPLICATION_ID_OFFSET + 4],
)[0]
)
def _stat_sqlite_sidecar_identity(db_path: Path) -> Dict[str, tuple]:
"""Snapshot ``(st_dev, st_ino)`` for existing WAL/SHM sidecars."""
identities: Dict[str, tuple] = {}
base = os.fspath(db_path)
for suffix in ("-wal", "-shm"):
ident = _stat_db_file_identity(Path(base + suffix))
if ident is not None:
identities[suffix] = ident
return identities
def _canonical_sqlite_path(path: str) -> str:
"""Normalize a /proc fd target, stripping the Linux `` (deleted)`` suffix."""
return os.path.normcase(os.path.abspath(path.removesuffix(" (deleted)")))
def _watched_sqlite_sidecar_paths(db_path) -> Set[str]:
base = os.path.abspath(os.fspath(db_path))
return {
_canonical_sqlite_path(base + "-wal"),
_canonical_sqlite_path(base + "-shm"),
}
def iter_deleted_sqlite_sidecar_holders(db_path) -> List[Tuple[int, str]]:
"""Return processes holding an unlinked ``state.db-wal`` / ``-shm``.
Linux-only (``/proc/<pid>/fd`` readlink). Windows and other hosts
return ``[]`` — Windows cannot unlink a sidecar another process still
holds, and macOS does not use the `` (deleted)`` suffix.
The scan includes this process: on the SessionDB open/write refuse
path, the in-process writer that still holds the orphan inode is the
one that must not mint a replacement WAL (and must stop committing).
``_foreign_state_db_holders`` keeps skipping this PID for FTS
maintenance so a process does not block its own optional repair.
"""
if not sys.platform.startswith("linux"):
return []
holders: List[Tuple[int, str]] = []
watched = _watched_sqlite_sidecar_paths(db_path)
try:
for pid_str in os.listdir("/proc"):
if not pid_str.isdigit():
continue
pid = int(pid_str)
fd_dir = f"/proc/{pid}/fd"
try:
fds = os.listdir(fd_dir)
except OSError:
continue
for fd in fds:
try:
target = os.readlink(f"{fd_dir}/{fd}")
except OSError:
continue
if " (deleted)" not in target:
continue
if _canonical_sqlite_path(target) in watched:
holders.append((pid, target))
except Exception as exc:
logger.debug("deleted-WAL holder scan failed for %s: %s", db_path, exc)
return holders
return holders
def refuse_deleted_wal_generation(db_path) -> None:
"""Raise if any process holds a deleted WAL/SHM generation for *db_path*.
Called *before* ``sqlite3.connect`` so a second opener cannot mint a
replacement WAL inode while a live writer still holds the orphan.
"""
from hermes_state import DeletedWalGenerationError, _DELETED_WAL_GENERATION_MSG
holders = iter_deleted_sqlite_sidecar_holders(db_path)
if not holders:
return
logger.error(_DELETED_WAL_GENERATION_MSG)
raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG)
def _connect_tracked_db(path, tracking_path=None, **kwargs):
"""``sqlite3.connect`` that registers the open fd for lock-safety.
While a connection is live, byte-level probes of the same file are
refused: an ``open()``/``close()`` cancels every POSIX advisory lock this
process holds on it -- including a running VACUUM's EXCLUSIVE lock.
Released automatically on ``close()``.
The ONLY tolerated fallback is the helper being absent entirely
(scaffold/embed installs that ship hermes_state without hermes_cli). A
real connection failure must propagate: silently retrying an *untracked*
connect would disable the guard for the lifetime of that connection,
which is precisely the failure mode this module exists to prevent.
"""
try:
from hermes_cli.sqlite_safe_read import connect_tracked
except ImportError:
logger.debug(
"hermes_cli.sqlite_safe_read unavailable; opening %s untracked "
"(byte-probe guard inactive in this install)",
path,
)
return sqlite3.connect(str(path), **kwargs)
# Open through THIS module's sqlite3.connect so callers (and tests) that
# patch hermes_state.sqlite3.connect keep control of connection creation;
# the helper still owns tracking.
return connect_tracked(
path,
tracking_path=tracking_path,
connect_fn=sqlite3.connect,
**kwargs,
)
def is_zeroed_state_db(
path: Path, *, probe_bytes: int = 100, force: bool = False
) -> bool:
"""Detect the #68474/#97568 zeroed state.db signature (0-byte or NUL header).
Byte-level probe, so it is only safe BEFORE any connection to *path*
exists in this process: ``close()`` cancels every POSIX advisory lock the
process holds on the file, which can pull the EXCLUSIVE lock out from
under a running VACUUM and corrupt the database. The read is routed
through ``read_header_bytes_preopen``, which refuses (returning False
here) once a connection is live. Pass ``force=True`` only for offline
files -- quarantined copies, snapshots, archives.
Prefer ``hermes_cli.backup.is_zeroed_sqlite_file`` when available; this
local copy keeps SessionDB openable without importing the CLI package
in constrained embed paths.
"""
try:
from hermes_cli.backup import is_zeroed_sqlite_file
return is_zeroed_sqlite_file(path, probe_bytes=probe_bytes, force=force)
except Exception:
pass
try:
if not path.is_file():
# Special files (FIFO, device, socket) are never "zeroed", and
# probing a FIFO would block until a writer appears.
return False
size = path.stat().st_size
except OSError:
return False
if size < 0:
return False
from hermes_cli.sqlite_safe_read import has_live_connection, read_header_bytes_preopen
if not force and has_live_connection(path):
return False
head = read_header_bytes_preopen(
path, length=max(16, probe_bytes), force=force
)
if head is None:
return False
if len(head) == 0:
return True
if head.startswith(b"SQLite format 3"):
return False
return all(byte == 0 for byte in head)
@contextlib.contextmanager
def quarantine_cross_process_lock(path: Path, timeout: float = 5.0):
"""Acquire the cross-process lock for path.quarantine.lock."""
import platform
lock_path = path.with_name(path.name + ".quarantine.lock")
lock_path.parent.mkdir(parents=True, exist_ok=True)
handle = lock_path.open("a+b")
acquired = False
try:
deadline = time.monotonic() + timeout
if platform.system() == "Windows":
import msvcrt
while True:
try:
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
acquired = True
break
except OSError:
if time.monotonic() >= deadline:
break
time.sleep(0.020)
else:
import fcntl
while True:
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
acquired = True
break
except (BlockingIOError, OSError):
if time.monotonic() >= deadline:
break
time.sleep(0.020)
yield acquired
finally:
try:
if acquired:
if platform.system() == "Windows":
import msvcrt
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
except (OSError, AttributeError):
pass
finally:
handle.close()
def quarantine_zeroed_state_db(
path: Path, *, already_locked: bool = False
) -> Optional[Path]:
"""Move a zeroed state.db aside (preserve bytes) and return quarantine path.
Uses a cross-process lock (``#68805``) so two concurrent startups cannot
race: the first process moves the zeroed file and the second re-checks
under the lock, finding the file already gone (or a fresh DB in its place)
instead of clobbering the quarantine.
"""
def _do_quarantine():
if not path.exists():
logger.info(
"quarantine_zeroed_state_db: %s already moved by another process",
path,
)
return None
if not is_zeroed_state_db(path):
logger.info(
"quarantine_zeroed_state_db: %s is no longer zeroed (another "
"process quarantined it and a fresh DB was created)",
path,
)
return None
try:
ts = time.strftime("%Y%m%d-%H%M%S")
except Exception:
ts = "unknown"
dest = path.with_name(
f"{path.name}.zeroed-{ts}-{os.getpid()}.bak"
)
n = 0
while dest.exists():
n += 1
dest = path.with_name(
f"{path.name}.zeroed-{ts}-{os.getpid()}-{n}.bak"
)
try:
path.rename(dest)
except OSError as exc:
logger.error("Failed to quarantine zeroed %s: %s", path, exc)
return None
for suffix in ("-wal", "-shm"):
side = Path(str(path) + suffix)
if side.exists():
try:
side.rename(Path(str(dest) + suffix))
except OSError:
pass
return dest
if already_locked:
return _do_quarantine()
with quarantine_cross_process_lock(path) as acquired:
if not acquired:
logger.error(
"quarantine lock for %s not acquired within 5s — refusing to "
"quarantine without the cross-process lock. The zeroed file "
"is left in place. If sessions fail to load, restore from "
"state-snapshots via `hermes snapshot list` / "
"`hermes snapshot restore <id>`.",
path,
)
return None
return _do_quarantine()
def collect_state_db_stats(db_path: Path) -> Dict[str, Any]:
"""Best-effort, strictly read-only stats snapshot of a state.db file.
Opens the database with ``mode=ro`` (URI) and a short timeout so it can
run against a *live* database held by a gateway without ever taking a
write lock or mutating the file. Every field is collected independently:
a failed pragma/SELECT yields ``None`` for that field, and the helper
itself never raises.
Deliberately does NOT instantiate :class:`SessionDB` — its constructor
runs schema DDL (migrations, FTS table creation), which is exactly the
kind of write a diagnostics probe must never perform.
Returned keys (all present, any may be None on failure):
- ``page_count``, ``page_size``, ``freelist_count`` — PRAGMA values
- ``logical_size_bytes`` — page_count * page_size (post-checkpoint size)
- ``wal_size_bytes`` — stat() of ``<db>-wal`` (0 when absent)
- ``journal_mode`` — PRAGMA journal_mode string
- ``messages`` / ``sessions`` — row counts
- ``fts_tables`` — dict of {table_name: bool} presence for
messages_fts / messages_fts_trigram / messages_fts_cjk
- ``fts_storage_version`` — int from state_meta, None when the marker is
absent (legacy pre-v23 inline layout)
- ``fts_rebuild_pending`` — True when the deferred v23 backfill has not
finished (high_water present and progress < high_water)
- ``fts_rebuild_high_water`` / ``fts_rebuild_progress`` — raw ints
- ``fts_rebuild_deferral`` — durable blocked-repair diagnostic, when present
"""
from hermes_state import _connect_tracked_db
stats: Dict[str, Any] = {
"page_count": None,
"page_size": None,
"freelist_count": None,
"logical_size_bytes": None,
"wal_size_bytes": None,
"journal_mode": None,
"messages": None,
"sessions": None,
"fts_tables": None,
"fts_storage_version": None,
"fts_rebuild_pending": None,
"fts_rebuild_high_water": None,
"fts_rebuild_progress": None,
"fts_rebuild_deferral": None,
}
# WAL sidecar size needs no connection at all.
try:
wal_path = Path(str(db_path) + "-wal")
stats["wal_size_bytes"] = wal_path.stat().st_size if wal_path.exists() else 0
except OSError:
pass
conn = None
try:
# mode=ro refuses to create the file and refuses every write; a
# short timeout keeps doctor snappy when a writer holds the lock.
# Route through the tracked connect so byte-probe helpers
# (read_header_bytes_preopen) see this connection and refuse raw
# opens that could cancel our POSIX locks mid-read.
conn = _connect_tracked_db(
f"file:{Path(db_path)}?mode=ro",
tracking_path=Path(db_path),
uri=True,
timeout=2.0,
)
except Exception as exc:
logger.debug("collect_state_db_stats: cannot open %s read-only: %s",
db_path, exc)
return stats
def _scalar(sql: str) -> Any:
try:
row = conn.execute(sql).fetchone()
return row[0] if row else None
except Exception:
return None
try:
pc = _scalar("PRAGMA page_count")
ps = _scalar("PRAGMA page_size")
stats["page_count"] = int(pc) if pc is not None else None
stats["page_size"] = int(ps) if ps is not None else None
if stats["page_count"] is not None and stats["page_size"] is not None:
stats["logical_size_bytes"] = stats["page_count"] * stats["page_size"]
fl = _scalar("PRAGMA freelist_count")
stats["freelist_count"] = int(fl) if fl is not None else None
jm = _scalar("PRAGMA journal_mode")
stats["journal_mode"] = str(jm) if jm is not None else None
msgs = _scalar("SELECT COUNT(*) FROM messages")
stats["messages"] = int(msgs) if msgs is not None else None
sess = _scalar("SELECT COUNT(*) FROM sessions")
stats["sessions"] = int(sess) if sess is not None else None
# FTS table presence via sqlite_master (never SELECTs from the
# virtual tables themselves — a corrupt index must not fail stats).
try:
names = {
row[0]
for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type = 'table' "
"AND name IN (?, ?, ?)",
("messages_fts", "messages_fts_trigram", "messages_fts_cjk"),
).fetchall()
}
stats["fts_tables"] = {
t: (t in names)
for t in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk")
}
except Exception:
pass
# Raw state_meta reads — cheap, and independent of SessionDB.
def _meta_int(key: str) -> Optional[int]:
try:
row = conn.execute(
"SELECT value FROM state_meta WHERE key = ?", (key,)
).fetchone()
return int(row[0]) if row and row[0] is not None else None
except Exception:
return None
stats["fts_storage_version"] = _meta_int("fts_storage_version")
high_water = _meta_int("fts_rebuild_high_water")
progress = _meta_int("fts_rebuild_progress")
stats["fts_rebuild_high_water"] = high_water
stats["fts_rebuild_progress"] = progress
if high_water is None:
stats["fts_rebuild_pending"] = False
else:
stats["fts_rebuild_pending"] = (progress or 0) < high_water
try:
row = conn.execute(
"SELECT value FROM state_meta WHERE key = ? LIMIT 1",
(FTS_REBUILD_DEFERRAL_KEY,),
).fetchone()
if row:
parsed = json.loads(row[0])
if isinstance(parsed, dict):
stats["fts_rebuild_deferral"] = parsed
except Exception:
pass
finally:
try:
conn.close()
except Exception:
pass
return stats
def count_db_holders(db_path: Path) -> Optional[int]:
"""Best-effort count of processes holding ``db_path`` open (Linux only).
Scans ``/proc/*/fd`` symlinks for the resolved database path. Returns
the number of distinct PIDs with the file open, or ``None`` on any
error or on non-Linux platforms. Never raises; no lsof dependency.
Unreadable per-process fd dirs (other users' processes without root)
are silently skipped, so the count is a lower bound.
"""
try:
if not sys.platform.startswith("linux"):
return None
target = os.path.realpath(str(db_path))
holders = 0
for pid in os.listdir("/proc"):
if not pid.isdigit():
continue
fd_dir = f"/proc/{pid}/fd"
try:
fds = os.listdir(fd_dir)
except OSError:
continue # process gone or not ours
for fd in fds:
try:
if os.readlink(f"{fd_dir}/{fd}") == target:
holders += 1
break # one hit per PID
except OSError:
continue
return holders
except Exception:
return None
def _is_inactive_orphan_desktop_holder(
*,
ppid: int,
age_seconds: float,
min_age_seconds: float,
ephemeral_backend: bool,
connection_statuses: List[str],
) -> bool:
"""Pure safety predicate for the narrow Desktop holder reap."""
return (
ppid in (0, 1)
and age_seconds >= min_age_seconds
and ephemeral_backend
and "ESTABLISHED" not in connection_statuses
)
def _concrete_state_db_holder_pids(
db_path: Path, holders: List[Tuple[int, str]]
) -> List[int]:
"""Return unique PIDs proven to hold this DB or one of its sidecars."""
canonical_db = os.path.normcase(os.path.abspath(os.fspath(db_path)))
watched = {
canonical_db,
canonical_db + "-wal",
canonical_db + "-shm",
}
pids: List[int] = []
seen = set()
for pid, path in holders:
canonical_path = os.path.normcase(
os.path.abspath(path.removesuffix(" (deleted)"))
)
if pid <= 0 or pid in seen or canonical_path not in watched:
continue
seen.add(pid)
pids.append(pid)
return pids
def _read_proc_cmdline(pid: int) -> Optional[str]:
"""Read /proc/<pid>/cmdline, world-readable even when fd table is not.
Returns the cmdline as a space-joined string, or None when unreadable
(process exited, or hidepid mount).
"""
try:
with open(f"/proc/{pid}/cmdline", "rb") as f:
raw = f.read()
if not raw:
return None
return raw.replace(b"\x00", b" ").decode("utf-8", "replace").strip()
except OSError:
return None
_HERMES_CMDLINE_MARKERS = ("hermes_cli.main", "hermes_cli/main", "hermes serve",
"hermes-agent", "hermes gateway", "hermes chat")
def _looks_like_hermes(cmdline: str) -> bool:
"""Heuristic: does this cmdline look like a Hermes process?
Used to decide whether an uninspectable process (fd table unreadable
due to different user) should be treated as a potential state.db holder.
We only flag processes that look like Hermes, not every system daemon.
"""
lower = cmdline.lower()
return any(marker in lower for marker in _HERMES_CMDLINE_MARKERS)
File diff suppressed because it is too large Load Diff
+699
View File
@@ -0,0 +1,699 @@
"""Retention pruning, stale-session archiving and VACUUM policy mixin for
SessionDB."""
from __future__ import annotations
import logging
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from hermes_state_common import (
AUTO_VACUUM_MIN_FREELIST_RATIO,
_sql_session_last_active,
escape_like as _escape_like,
)
# caplog tests pin the "hermes_state" logger name.
logger = logging.getLogger("hermes_state")
class SessionMaintenanceMixin:
"""Retention pruning, stale-session archiving and VACUUM policy for SessionDB."""
def prune_empty_ghost_sessions(self, sessions_dir: "Optional[Path]" = None) -> int:
"""Remove empty TUI ghost sessions (no messages, no title, >24hr old)."""
cutoff = time.time() - 86400
def _do(conn):
rows = conn.execute("""
SELECT id FROM sessions
WHERE source = 'tui'
AND title IS NULL
AND ended_at IS NOT NULL
AND started_at < ?
AND NOT EXISTS (
SELECT 1 FROM messages WHERE messages.session_id = sessions.id
)
""", (cutoff,)).fetchall()
ids = [r[0] for r in rows]
if ids:
placeholders = ",".join("?" * len(ids))
conn.execute(
f"DELETE FROM sessions WHERE id IN ({placeholders})", ids
)
self._delete_unreferenced_system_prompts(conn)
return ids
removed_ids = self._execute_write(_do) or []
if sessions_dir and removed_ids:
for sid in removed_ids:
self._remove_session_files(sessions_dir, sid)
return len(removed_ids)
def sweep_orphaned_sessions(
self,
*,
max_idle_seconds: float,
sources: Tuple[str, ...] = ("tui", "desktop", "subagent"),
exclude_ids: Tuple[str, ...] = (),
exclude_pinned: bool = False,
heartbeat_staleness_seconds: Optional[float] = None,
heartbeat_ownership_grace_seconds: Optional[float] = None,
respect_gateway_heartbeats: bool = True,
) -> List[str]:
"""Close session rows orphaned by a dead gateway process.
The TUI/desktop gateway reaps disconnected sessions with an in-process
grace timer; a restart destroys the timer and leaves ``ended_at IS
NULL`` forever. This closes rows for ``sources`` whose ``started_at``
AND canonical last activity (newest of ``last_activity_at`` and the
newest message, else ``started_at``) are both older than
``max_idle_seconds``, with ``end_reason='startup_orphan_reap'``. The
separate ``started_at`` predicate protects fresh compression/branch
children whose copied activity is old.
Only pass sources whose lifecycle the caller owns — never messaging
platforms like ``telegram`` (ending those triggers a routing loop).
``exclude_ids`` spares rows this process still holds in memory.
Non-destructive: messages are kept and the row stays resumable;
first-reason-wins via ``ended_at IS NULL``.
Cross-backend liveness: with ``respect_gateway_heartbeats``, a row is
reaped only when stale AND no live backend (heartbeat within
``heartbeat_staleness_seconds``, default ``2 * max_idle_seconds``) could
own it, where backend B owns session S if ``B.started_at <= S.started_at
+ heartbeat_ownership_grace_seconds`` (default = staleness). The grace
covers a migrating backend whose sessions predate its first heartbeat
but is bounded so a PID-reuse respawn cannot protect rows forever.
Disable the gate only for sources owned by state.db itself.
SELECT, live-lease validation and UPDATE run in one ``BEGIN IMMEDIATE``
transaction. Active turn leases / compression locks spare the row;
expired guards are removed so their former owner is fenced.
"""
from hermes_state import SessionCompressionInProgressError, SessionTurnLeaseLostError
srcs = tuple(s for s in sources if s)
if max_idle_seconds <= 0 or not srcs:
return []
hb_staleness = (
heartbeat_staleness_seconds
if heartbeat_staleness_seconds and heartbeat_staleness_seconds > 0
else max_idle_seconds * 2
)
hb_grace = (
heartbeat_ownership_grace_seconds
if heartbeat_ownership_grace_seconds is not None
and heartbeat_ownership_grace_seconds >= 0
else hb_staleness
)
now = time.time()
cutoff = now - max_idle_seconds
hb_cutoff = now - hb_staleness
placeholders = ",".join("?" for _ in srcs)
staleness = (
f"started_at < ? AND {_sql_session_last_active('sessions')} < ?"
)
pin_scope = " AND COALESCE(pinned, 0) = 0" if exclude_pinned else ""
heartbeat_params: Tuple[float, ...] = ()
orphan_predicate = staleness
if respect_gateway_heartbeats:
orphan_predicate += (
" AND NOT EXISTS ("
"SELECT 1 FROM gateway_heartbeats h"
" WHERE h.last_heartbeat >= ?"
" AND h.started_at <= sessions.started_at + ?"
")"
)
heartbeat_params = (hb_cutoff, hb_grace)
def _do(conn):
rows = conn.execute(
f"SELECT id FROM sessions WHERE ended_at IS NULL"
f" AND source IN ({placeholders}){pin_scope}"
f" AND {orphan_predicate}",
(*srcs, cutoff, cutoff, *heartbeat_params),
).fetchall()
excluded = {str(x) for x in exclude_ids if x}
victims = []
for row in rows:
sid = str(row["id"])
if sid in excluded:
continue
try:
self._check_transcript_write_guards(
conn,
sid,
compression_lock_holder=None,
turn_lease_holder=None,
reject_active_turn_lease=True,
reject_active_compression_lock=True,
)
except (
SessionCompressionInProgressError,
SessionTurnLeaseLostError,
):
continue
victims.append(sid)
if not victims:
return []
closed_at = time.time()
marks = ",".join("?" for _ in victims)
# Re-apply every predicate under the write lock.
conn.execute(
f"UPDATE sessions SET ended_at = ?, end_reason = 'startup_orphan_reap'"
f" WHERE id IN ({marks}) AND ended_at IS NULL"
f" AND source IN ({placeholders}){pin_scope}"
f" AND {orphan_predicate}",
(
closed_at,
*victims,
*srcs,
cutoff,
cutoff,
*heartbeat_params,
),
)
return victims
return self._execute_write(_do) or []
@staticmethod
def _prune_filter_where(
*,
last_active_before: Optional[float] = None,
last_active_after: Optional[float] = None,
started_before: Optional[float] = None,
started_after: Optional[float] = None,
source: Optional[str] = None,
title_like: Optional[str] = None,
end_reason: Optional[str] = None,
cwd_prefix: Optional[str] = None,
min_messages: Optional[int] = None,
max_messages: Optional[int] = None,
archived: Optional[bool] = None,
model_like: Optional[str] = None,
provider: Optional[str] = None,
user_id: Optional[str] = None,
chat_id: Optional[str] = None,
chat_type: Optional[str] = None,
branch_like: Optional[str] = None,
min_tokens: Optional[int] = None,
max_tokens: Optional[int] = None,
min_cost: Optional[float] = None,
max_cost: Optional[float] = None,
min_tool_calls: Optional[int] = None,
max_tool_calls: Optional[int] = None,
include_pinned: bool = False,
) -> Tuple[str, list]:
"""Shared WHERE clause for bulk prune/archive selection (alias ``s``).
Filters AND together; only ended sessions are ever candidates.
``archived`` is tri-state (None = both). ``*_like`` filters are
case-insensitive substrings; the rest are exact (provider
case-insensitive). Token bounds use input+output; cost bounds use
``COALESCE(actual_cost_usd, estimated_cost_usd)``.
"""
from hermes_state import _cwd_prefix_clause
clauses = ["s.ended_at IS NOT NULL"]
params: list = []
if last_active_before is not None:
clauses.append(
"""COALESCE(
(SELECT MAX(m.timestamp) FROM messages m
WHERE m.session_id = s.id),
s.started_at
) < ?"""
)
params.append(last_active_before)
# Orphan-swept rows age from the sweep, not their old activity, or
# the next prune pass deletes them before the user can recover.
clauses.append(
"(COALESCE(s.end_reason, '') != 'startup_orphan_reap' "
"OR s.ended_at < ?)"
)
params.append(last_active_before)
if last_active_after is not None:
clauses.append(
"""COALESCE(
(SELECT MAX(m.timestamp) FROM messages m
WHERE m.session_id = s.id),
s.started_at
) >= ?"""
)
params.append(last_active_after)
if started_before is not None:
clauses.append("s.started_at < ?")
params.append(started_before)
if started_after is not None:
clauses.append("s.started_at >= ?")
params.append(started_after)
if source:
clauses.append("s.source = ?")
params.append(source)
if title_like:
clauses.append("LOWER(COALESCE(s.title, '')) LIKE ? ESCAPE '\\'")
params.append(f"%{_escape_like(title_like.lower())}%")
if end_reason:
clauses.append("s.end_reason = ?")
params.append(end_reason)
if cwd_prefix:
clause, clause_params = _cwd_prefix_clause(cwd_prefix)
clauses.append(clause)
params.extend(clause_params)
if min_messages is not None:
clauses.append("s.message_count >= ?")
params.append(min_messages)
if max_messages is not None:
clauses.append("s.message_count <= ?")
params.append(max_messages)
if model_like:
clauses.append("LOWER(COALESCE(s.model, '')) LIKE ? ESCAPE '\\'")
params.append(f"%{_escape_like(model_like.lower())}%")
if provider:
clauses.append("LOWER(COALESCE(s.billing_provider, '')) = ?")
params.append(provider.lower())
if user_id:
clauses.append("s.user_id = ?")
params.append(user_id)
if chat_id:
clauses.append("s.chat_id = ?")
params.append(chat_id)
if chat_type:
clauses.append("s.chat_type = ?")
params.append(chat_type)
if branch_like:
clauses.append("LOWER(COALESCE(s.git_branch, '')) LIKE ? ESCAPE '\\'")
params.append(f"%{_escape_like(branch_like.lower())}%")
if min_tokens is not None:
clauses.append(
"(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) >= ?"
)
params.append(min_tokens)
if max_tokens is not None:
clauses.append(
"(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) <= ?"
)
params.append(max_tokens)
if min_cost is not None:
clauses.append(
"COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) >= ?"
)
params.append(min_cost)
if max_cost is not None:
clauses.append(
"COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) <= ?"
)
params.append(max_cost)
if min_tool_calls is not None:
clauses.append("COALESCE(s.tool_call_count, 0) >= ?")
params.append(min_tool_calls)
if max_tool_calls is not None:
clauses.append("COALESCE(s.tool_call_count, 0) <= ?")
params.append(max_tool_calls)
if archived is True:
clauses.append("s.archived = 1")
elif archived is False:
clauses.append("s.archived = 0")
# Pinned is a durable "keep" flag: bulk prune/delete/archive exclude
# pinned rows unless the caller explicitly opts in.
if not include_pinned:
clauses.append("COALESCE(s.pinned, 0) = 0")
return " AND ".join(clauses), params
@staticmethod
def _apply_prune_age_filter(
older_than_days: Optional[float], filters: Dict[str, Any]
) -> None:
"""Translate the legacy age window into the shared activity filter."""
if (
filters.get("last_active_before") is None
and filters.get("started_before") is None
and older_than_days is not None
):
filters["last_active_before"] = time.time() - (
older_than_days * 86400
)
def list_prune_candidates(
self,
older_than_days: Optional[float] = None,
source: str = None,
**filters,
) -> List[Dict[str, Any]]:
"""Sessions a matching prune/archive would touch (dry-run), oldest
first. Same filters as :meth:`_prune_filter_where`; ``older_than_days``
is an inactivity threshold (latest message, else ``started_at``)."""
self._apply_prune_age_filter(older_than_days, filters)
where, params = self._prune_filter_where(source=source, **filters)
with self._read_ctx() as conn:
cursor = conn.execute(
f"""SELECT s.id, s.source, s.title, s.model, s.started_at,
COALESCE(
(SELECT MAX(m.timestamp) FROM messages m
WHERE m.session_id = s.id),
s.started_at
) AS last_active,
s.ended_at, s.message_count, s.archived
FROM sessions s WHERE {where}
ORDER BY last_active ASC, s.started_at ASC""",
params,
)
return [dict(row) for row in cursor.fetchall()]
def count_prune_matches(
self,
older_than_days: Optional[float] = None,
source: str = None,
**filters,
) -> int:
"""Count-only variant of :meth:`list_prune_candidates` (the CLI uses it
to report how many pinned sessions are spared)."""
self._apply_prune_age_filter(older_than_days, filters)
where, params = self._prune_filter_where(source=source, **filters)
with self._read_ctx() as conn:
cursor = conn.execute(
f"SELECT COUNT(*) FROM sessions s WHERE {where}", params
)
return int(cursor.fetchone()[0])
def count_open_prune_matches(
self,
older_than_days: Optional[float] = None,
source: str = None,
**filters,
) -> int:
"""Count open sessions a matching prune skips: every normal filter with
only the ``ended_at`` guard inverted. Visibility-only; live sessions
never become prune-eligible."""
self._apply_prune_age_filter(older_than_days, filters)
where, params = self._prune_filter_where(source=source, **filters)
ended_guard = "s.ended_at IS NOT NULL"
if not where.startswith(ended_guard):
raise RuntimeError("prune filter lost its ended-session safety guard")
open_where = f"s.ended_at IS NULL{where[len(ended_guard):]}"
with self._read_ctx() as conn:
cursor = conn.execute(
f"SELECT COUNT(*) FROM sessions s WHERE {open_where}", params
)
return int(cursor.fetchone()[0])
def archive_stale_sessions(
self, idle_days: float, *, exclude_pinned: bool = True
) -> int:
"""Archive every session untouched for ``idle_days`` (real recency:
freshest of ``last_activity_at`` / latest message / ``started_at``).
Unlike :meth:`archive_sessions`, this can archive unended sessions.
Guards: ``pinned = 0`` when ``exclude_pinned``; ``archived = 0`` so
repeats are no-ops; only lineage tips (``end_reason <> 'compression'``)
are candidates — a stale tip archives its chain via
:meth:`set_session_archived`, so an old compressed-away root with a
recent continuation is never matched. Returns the count archived.
"""
if idle_days is None or idle_days < 0:
return 0
cutoff = time.time() - float(idle_days) * 86400.0
pin_clause = "AND s.pinned = 0" if exclude_pinned else ""
rows = self._read_all(
f"""
SELECT s.id FROM sessions s
WHERE s.archived = 0
AND COALESCE(s.end_reason, '') <> 'compression'
{pin_clause}
AND {_sql_session_last_active("s")} < ?
ORDER BY s.started_at ASC
""",
(cutoff,),
)
ids = [r[0] for r in rows]
for sid in ids:
self.set_session_archived(sid, True)
return len(ids)
def prune_sessions(
self,
older_than_days: Optional[float] = 90,
source: str = None,
sessions_dir: Optional[Path] = None,
exclude_active_write_guards: bool = False,
**filters,
) -> int:
"""Delete ended sessions matching the filters; returns the count.
Default: inactive for ``older_than_days`` (latest message, else
``started_at``), optionally by ``source``. Extra keyword filters are
those of :meth:`_prune_filter_where`; an explicit ``started_before`` /
``last_active_before`` overrides the ``older_than_days`` cutoff
(pass ``older_than_days=None`` for no implicit age bound).
Children outside the window are orphaned (parent NULLed), not cascade-
deleted. With *sessions_dir*, on-disk transcript files are removed
outside the DB transaction. ``exclude_active_write_guards`` (automatic
maintenance) skips rows under a live turn lease or compression lock,
while expired/dead holders are reclaimed and fenced in the same write.
"""
from hermes_state import SessionCompressionInProgressError, SessionTurnLeaseLostError
self._apply_prune_age_filter(older_than_days, filters)
where, where_params = self._prune_filter_where(source=source, **filters)
removed_ids: list[str] = []
def _do(conn):
cursor = conn.execute(
f"SELECT s.id FROM sessions s WHERE {where}", where_params
)
session_ids = {row["id"] for row in cursor.fetchall()}
if exclude_active_write_guards:
protected = set()
for sid in session_ids:
try:
self._check_transcript_write_guards(
conn,
sid,
compression_lock_holder=None,
turn_lease_holder=None,
reject_active_turn_lease=True,
reject_active_compression_lock=True,
allow_closed_compression_parent=True,
)
except (
SessionCompressionInProgressError,
SessionTurnLeaseLostError,
):
protected.add(sid)
session_ids.difference_update(protected)
if not session_ids:
return 0
placeholders = ",".join("?" * len(session_ids))
conn.execute(
f"UPDATE sessions SET parent_session_id = NULL "
f"WHERE parent_session_id IN ({placeholders})",
list(session_ids),
)
for sid in session_ids:
conn.execute("DELETE FROM messages WHERE session_id = ?", (sid,))
conn.execute("DELETE FROM sessions WHERE id = ?", (sid,))
removed_ids.append(sid)
self._delete_unreferenced_system_prompts(conn)
return len(session_ids)
count = self._execute_write(_do)
for sid in removed_ids:
self._remove_session_files(sessions_dir, sid)
return count
def logical_size_bytes(self) -> Optional[int]:
"""``page_count * page_size``: the main-file size once the WAL is
checkpointed back in. Prefer over ``os.path.getsize`` when reporting a
VACUUM: in WAL mode the rewrite lands in ``-wal`` and the checkpoint is
refused while another connection holds a read-mark, so a stat() delta
understates the win and can go negative. None if pragmas fail.
"""
try:
with self._read_ctx() as conn:
if self._conn is None:
return None
page_count = conn.execute("PRAGMA page_count").fetchone()[0]
page_size = conn.execute("PRAGMA page_size").fetchone()[0]
return int(page_count) * int(page_size)
except Exception as exc:
logger.debug("Could not read logical DB size: %s", exc)
return None
def _freelist_ratio(self) -> Optional[float]:
"""Reclaimable fraction (``freelist_count / page_count``) over the
existing connection — never a byte-level probe of the live file. Gates
VACUUM in :meth:`maybe_auto_prune_and_vacuum`. None if pragmas fail
(callers then fall back to the time throttle alone).
"""
try:
with self._read_ctx() as conn:
if self._conn is None:
return None
page_count = int(conn.execute("PRAGMA page_count").fetchone()[0])
freelist = int(conn.execute("PRAGMA freelist_count").fetchone()[0])
if page_count <= 0:
return 0.0
return freelist / page_count
except Exception as exc:
logger.debug("Could not read freelist ratio: %s", exc)
return None
def vacuum(self) -> int:
"""VACUUM to reclaim space after large deletes (SQLite never shrinks
the file on its own).
Rewrites the whole DB, cannot run inside a transaction, and takes an
exclusive lock — callers must ensure no other writers are active (safe
at startup before serving traffic). FTS5 segments are merged first via
:meth:`optimize_fts` so the VACUUM reclaims those pages too. Returns
the number of FTS indexes optimized (0 on merge failure / no FTS).
"""
# optimize_fts() manages its own lock.
optimized = 0
try:
optimized = self.optimize_fts()
except Exception as exc:
logger.warning("FTS optimize before VACUUM failed: %s", exc)
# VACUUM cannot be executed inside a transaction.
with self._lock:
# PASSIVE, not TRUNCATE: a manual `hermes sessions vacuum` runs in
# a transient CLI process, and a TRUNCATE reset here would race a
# live gateway writer and tear B-tree pages.
try:
self._conn.execute("PRAGMA wal_checkpoint(PASSIVE)")
except Exception as exc:
logger.debug("WAL checkpoint (PASSIVE) before VACUUM failed: %s", exc)
self._conn.execute("VACUUM")
# VACUUM rewrites every page THROUGH the WAL; without this TRUNCATE
# a 3 GB database leaves a 3 GB -wal behind and the command is a
# net loss on disk.
try:
self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
except Exception as exc:
logger.debug("WAL checkpoint (TRUNCATE) after VACUUM failed: %s", exc)
# TRUNCATE may replace the WAL inode; adopt the new sidecars so the
# write-path generation guard does not halt this connection.
self._record_db_file_identity()
return optimized
def maybe_auto_prune_and_vacuum(
self,
retention_days: int = 90,
min_interval_hours: int = 24,
vacuum: bool = True,
sessions_dir: Optional[Path] = None,
min_vacuum_interval_days: int = 30,
min_vacuum_freelist_ratio: float = AUTO_VACUUM_MIN_FREELIST_RATIO,
) -> Dict[str, Any]:
"""Idempotent startup auto-maintenance: prune inactive sessions,
reap stale open state-owned rows, optional VACUUM. Never raises.
Runs at most once per ``min_interval_hours`` (state_meta). VACUUM has
its own ``min_vacuum_interval_days`` throttle and additionally requires
``freelist_count / page_count`` > ``min_vacuum_freelist_ratio`` so a
small prune on a dense multi-GB database never triggers a full rewrite.
With *sessions_dir*, pruned transcripts are removed from disk too.
Stale-open reconciliation: cron/kanban/subagent/one-shot CLI rows never
set ``ended_at`` when their process dies, and prune only deletes ended
rows. After pruning, open rows from :attr:`_AUTO_PRUNE_STALE_OPEN_SOURCES`
older than ``retention_days`` are closed (``startup_orphan_reap``); they
stay resumable and age from their close, so they get one more full
retention window. Messaging and UI sources are never touched.
Returns ``{"skipped", "pruned", "closed", "vacuumed"}`` plus
``"freelist_ratio"`` when a VACUUM was considered and ``"error"`` on
failure.
"""
from hermes_state import _release_auto_maintenance_lock, _try_acquire_auto_maintenance_lock
result: Dict[str, Any] = {
"skipped": False,
"pruned": 0,
"closed": 0,
"vacuumed": False,
}
maintenance_lock = _try_acquire_auto_maintenance_lock(self.db_path)
if maintenance_lock is None:
result["skipped"] = True
return result
try:
last_raw = self.get_meta("last_auto_prune")
now = time.time()
if last_raw:
try:
last_ts = float(last_raw)
if now - last_ts < min_interval_hours * 3600:
result["skipped"] = True
return result
except (TypeError, ValueError):
pass # corrupt meta; treat as no prior run
# Prune first: orphans closed below get a full retention window.
pruned = self.prune_sessions(
older_than_days=retention_days,
sessions_dir=sessions_dir,
exclude_active_write_guards=True,
)
result["pruned"] = pruned
closed = self.sweep_orphaned_sessions(
max_idle_seconds=float(retention_days) * 86400.0,
sources=self._AUTO_PRUNE_STALE_OPEN_SOURCES,
exclude_pinned=True,
# State-owned lifecycles, not gateway heartbeats.
respect_gateway_heartbeats=False,
)
result["closed"] = len(closed)
# VACUUM only if rows were freed, the time throttle passed ("not
# too often") AND the freelist ratio passed ("only when it pays
# off") — it holds an exclusive lock for a full rewrite.
last_vacuum_raw = self.get_meta("last_vacuum")
vacuum_due = True
if last_vacuum_raw:
try:
vacuum_due = (now - float(last_vacuum_raw)) >= min_vacuum_interval_days * 86400
except (TypeError, ValueError):
vacuum_due = True
if vacuum and pruned > 0 and vacuum_due:
ratio = self._freelist_ratio()
result["freelist_ratio"] = ratio
if ratio is None or ratio > min_vacuum_freelist_ratio:
try:
self.vacuum()
result["vacuumed"] = True
self.set_meta("last_vacuum", str(now))
except Exception as exc:
logger.warning("state.db VACUUM failed: %s", exc)
else:
logger.debug(
"state.db auto-maintenance: skipping VACUUM, only "
"%.1f%% of pages reclaimable (threshold %.0f%%)",
ratio * 100.0,
min_vacuum_freelist_ratio * 100.0,
)
# Record even when pruned == 0 so the throttle holds.
self.set_meta("last_auto_prune", str(now))
if closed or pruned > 0:
logger.info(
"state.db auto-maintenance: closed %d stale open session(s), "
"pruned %d session(s) inactive for %d days%s",
len(closed),
pruned,
retention_days,
" + VACUUM" if result["vacuumed"] else "",
)
except Exception as exc:
# Maintenance must never block startup.
logger.warning("state.db auto-maintenance failed: %s", exc)
result["error"] = str(exc)
finally:
_release_auto_maintenance_lock(maintenance_lock)
return result
File diff suppressed because it is too large Load Diff
+119 -300
View File
@@ -1,11 +1,9 @@
"""Session listing/rich rows, export, and import (portability) for SessionDB.
Mixin contract: this is a plain mixin class consumed by
``hermes_state.SessionDB``. It defines no ``__init__`` and no state of its
own; methods access the host's attributes (``self._conn``, ``self.db_path``,
``self._execute_write`` and other SessionDB methods) established by
``SessionDB.__init__``. It must never import hermes_state (cycle) — shared
module-level constants live in hermes_state_common.
Plain mixin consumed by ``hermes_state.SessionDB``: no ``__init__``, no state
of its own; methods use host attributes established by ``SessionDB.__init__``.
Must never import hermes_state (cycle) — shared constants live in
hermes_state_common.
"""
import logging
@@ -16,14 +14,12 @@ from typing import Any, Dict, List, Optional
from agent.skill_commands import SKILL_SCAFFOLD_SQL_LIKE
from hermes_state_common import (
SCHEMA_SQL,
_PREVIEW_ELIGIBLE_SQL,
_PREVIEW_RAW_SELECT,
_PREVIEW_RAW_SUBQUERY_SQL,
_shape_preview,
_sql_session_last_active,
)
# Moved methods logged under the "hermes_state" logger before the split;
# keep that logger identity so log filtering/capture behavior is unchanged.
# Keep the pre-split logger identity so log filtering/capture is unchanged.
logger = logging.getLogger("hermes_state")
@@ -32,9 +28,8 @@ class SessionPortabilityMixin:
@classmethod
def _compact_session_cols(cls) -> str:
"""SELECT list for compact_rows: every ``sessions`` column declared in
SCHEMA_SQL except prompt storage internals, aliased with the ``s``
prefix used by list_sessions_rich/_get_session_rich_row queries."""
"""``s.``-prefixed SELECT list of every SCHEMA_SQL ``sessions`` column
except prompt storage internals (the compact_rows projection)."""
if cls._session_compact_cols_sql is None:
declared = cls._parse_schema_columns(SCHEMA_SQL)["sessions"]
cls._session_compact_cols_sql = ", ".join(
@@ -43,13 +38,19 @@ class SessionPortabilityMixin:
)
return cls._session_compact_cols_sql
@classmethod
def _rich_row(cls, row) -> Dict[str, Any]:
"""Session row dict with ``_preview_raw`` shaped into ``preview``."""
s = cls._session_row_dict(row)
s["preview"] = _shape_preview(s.pop("_preview_raw", ""))
return s
def distinct_session_cwds(self, include_archived: bool = False) -> List[Dict[str, Any]]:
"""Distinct non-empty session cwds with usage stats, for repo discovery.
Aggregates across ALL session history (not a single page), so the desktop
can surface every git repo the user has worked in — not just the repos
that happen to be in the currently-loaded recents. Children/branches
count: a worktree session is still a real workspace signal.
Aggregates across ALL history (not one page) so every repo the user
worked in surfaces. Children/branches count: a worktree session is a
real workspace signal.
"""
where = "cwd IS NOT NULL AND TRIM(cwd) != ''"
if not include_archived:
@@ -77,41 +78,24 @@ class SessionPortabilityMixin:
) -> List[Dict[str, Any]]:
"""List the run sessions produced by a single cron job, newest first.
Cron runs are flat, independent sessions whose id is
``cron_{job_id}_{timestamp}`` (see ``cron/scheduler.run_job``). They are
never compression roots and never branch, so this deliberately skips the
``list_sessions_rich`` recursive compression-chain CTE / leading-wildcard
``id_query`` path — that path seeds from *every* ``source='cron'`` row in
the DB and only filters to one job's runs after the scan, so it scales
with the whole cron pile (a heavy history makes the desktop run-history
endpoint time out before it eventually populates).
Cron runs are flat sessions with id ``cron_{job_id}_{timestamp}``; they
never compress or branch, so this skips ``list_sessions_rich``'s
compression-chain CTE / leading-wildcard ``id_query`` path, which seeds
from EVERY ``source='cron'`` row and scales with the whole cron pile.
Instead: a ``[prefix, prefix_hi)`` index range scan on id, filtered to
``source='cron'``, so work scales with the requested window.
Instead this binds to one job with a ``[prefix, prefix_hi)`` range over
the id (an index range scan, not a ``%...%`` substring), filters
``source='cron'``, and orders by ``started_at DESC``. Work scales with
the requested window, not the total cron history.
Returns the same enriched row shape as ``list_sessions_rich`` (adds
``preview`` + ``last_active``) so callers can reuse it.
Returns the ``list_sessions_rich`` row shape (``preview`` + ``last_active``).
"""
prefix = f"cron_{job_id}_"
# Half-open upper bound for an index range scan: increment the final
# byte of the prefix so the range covers exactly the ids that start
# with ``prefix`` and nothing else. ``prefix`` always ends in '_', but
# compute it generically rather than hardcoding the successor char.
# Half-open upper bound: bump the final byte so the range covers exactly
# the ids starting with ``prefix``.
prefix_hi = prefix[:-1] + chr(ord(prefix[-1]) + 1)
query = f"""
SELECT s.*,
COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved,
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
AND {_PREVIEW_ELIGIBLE_SQL}
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
{_PREVIEW_RAW_SUBQUERY_SQL},
{_sql_session_last_active("s")} AS last_active
FROM sessions s
LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash
@@ -120,27 +104,12 @@ class SessionPortabilityMixin:
LIMIT ? OFFSET ?
"""
with self._lock:
cursor = self._conn.execute(query, (prefix, prefix_hi, limit, offset))
rows = cursor.fetchall()
runs: List[Dict[str, Any]] = []
for row in rows:
s = self._session_row_dict(row)
s["preview"] = _shape_preview(s.pop("_preview_raw", ""))
runs.append(s)
return runs
rows = self._conn.execute(query, (prefix, prefix_hi, limit, offset)).fetchall()
return [self._rich_row(row) for row in rows]
def _get_session_rich_row(self, session_id: str, compact_rows: bool = False) -> Optional[Dict[str, Any]]:
"""Fetch a single session with the same enriched columns as
``list_sessions_rich`` (preview + last_active). Returns None if the
session doesn't exist.
Pass ``compact_rows=True`` to omit the ``system_prompt`` blob (see
``list_sessions_rich`` for details).
Thin wrapper over ``_get_session_rich_rows_batch`` so the enriched
SELECT lives in exactly one place.
"""
"""One session with the ``list_sessions_rich`` enriched columns, or
None. ``compact_rows=True`` omits the ``system_prompt`` blob."""
return self._get_session_rich_rows_batch(
[session_id], compact_rows=compact_rows
).get(session_id)
@@ -148,23 +117,15 @@ class SessionPortabilityMixin:
def _get_session_rich_rows_batch(
self, session_ids, compact_rows: bool = False
) -> Dict[str, Dict[str, Any]]:
"""Fetch multiple sessions with the same enriched columns as
``_get_session_rich_row``, in a single query.
Used by ``list_sessions_rich``'s compression-tip projection to resolve
every tip row for a page in one round trip instead of one query per
compression-root row. Returns a dict keyed by session id; ids that
don't exist are simply absent from the result (same as
``_get_session_rich_row`` returning ``None`` for them).
"""Enriched rows for many sessions in one query, keyed by id; missing
ids are simply absent. Resolves a page of compression tips in one
round trip instead of one query per root row.
"""
ids = [sid for sid in session_ids if sid]
if not ids:
return {}
# Old SQLite builds cap bound variables at 999
# (SQLITE_MAX_VARIABLE_NUMBER); large pages (limit=10000 callers
# exist) could exceed it. Chunk the IN list so the helper is safe at
# any page size — this is the single choke point for the enriched
# multi-row fetch, so the bound lives here, not at call sites.
# Old SQLite caps bound variables at 999 (SQLITE_MAX_VARIABLE_NUMBER);
# limit=10000 callers exist. Chunk here — the single choke point.
_CHUNK = 900
if len(ids) > _CHUNK:
result: Dict[str, Dict[str, Any]] = {}
@@ -189,45 +150,26 @@ class SessionPortabilityMixin:
)
query = f"""
SELECT {_sel}{prompt_select},
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
AND {_PREVIEW_ELIGIBLE_SQL}
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
{_PREVIEW_RAW_SUBQUERY_SQL},
{_sql_session_last_active("s")} AS last_active
FROM sessions s
{prompt_join}
WHERE s.id IN ({placeholders})
"""
with self._lock:
cursor = self._conn.execute(query, ids)
rows = cursor.fetchall()
result: Dict[str, Dict[str, Any]] = {}
for row in rows:
s = self._session_row_dict(row)
s["preview"] = _shape_preview(s.pop("_preview_raw", ""))
result[s["id"]] = s
return result
rows = self._conn.execute(query, ids).fetchall()
return {s["id"]: s for s in map(self._rich_row, rows)}
def get_session_rich_row(self, session_id: str, compact_rows: bool = False) -> Optional[Dict[str, Any]]:
"""Public wrapper for :meth:`_get_session_rich_row`.
Exposes the single-session enriched row (same columns as
``list_sessions_rich``: preview + last_active) for callers outside
this module, e.g. the web server's session-search hydration.
"""
"""Public wrapper for :meth:`_get_session_rich_row` (web server hydration)."""
return self._get_session_rich_row(session_id, compact_rows=compact_rows)
def list_skill_scaffolded_sessions(self, limit: int = 200) -> List[Dict[str, Any]]:
"""Titled sessions whose first user turn was a ``/skill`` invocation.
Those titles were generated from the expanded message, which embeds the
whole skill body — so they describe the skill rather than the request.
Returns ``id``, ``title``, and the full first-turn ``content`` so a
caller can re-derive what the user typed. Newest first.
Their titles were generated from the expanded skill body, so they
describe the skill, not the request. Returns ``id``, ``title`` and the
first-turn ``content`` so callers can re-derive what was typed. Newest first.
"""
with self._lock:
rows = self._conn.execute(
@@ -248,63 +190,36 @@ class SessionPortabilityMixin:
).fetchall()
return [dict(row) for row in rows]
def get_first_assistant_text(self, session_id: str) -> str:
"""The session's first assistant reply as plain text ('' when none).
Pairs with :meth:`list_skill_scaffolded_sessions` so a re-title can feed
the titler the same (request, reply) shape the live path uses.
"""
with self._lock:
row = self._conn.execute(
"SELECT content FROM messages "
"WHERE session_id = ? AND role = 'assistant' AND content IS NOT NULL "
"ORDER BY timestamp, id LIMIT 1",
(session_id,),
).fetchone()
if not row:
return ""
decoded = self._decode_content(row["content"])
return decoded if isinstance(decoded, str) else ""
def export_session(self, session_id: str) -> Optional[Dict[str, Any]]:
"""Export a single session with all its messages as a dict."""
session = self.get_session(session_id)
if not session:
return None
messages = self.get_messages(session_id)
return {**session, "messages": messages}
return {**session, "messages": self.get_messages(session_id)}
def export_session_lineage(self, session_id: str) -> Optional[Dict[str, Any]]:
"""Export a compression lineage as one logical session dict."""
lineage_ids = self.get_compression_lineage(session_id)
if not lineage_ids:
return None
segments = []
for sid in lineage_ids:
segment = self.export_session(sid)
if segment:
segments.append(segment)
segments = [seg for seg in map(self.export_session, lineage_ids) if seg]
if not segments:
return None
base = dict(segments[-1])
total_messages = sum(len(seg.get("messages") or []) for seg in segments)
base["segments"] = segments
base["lineage_session_ids"] = [seg["id"] for seg in segments]
base["message_count"] = total_messages
base["messages"] = [msg for seg in segments for msg in (seg.get("messages") or [])]
return base
messages = [msg for seg in segments for msg in (seg.get("messages") or [])]
return {
**segments[-1],
"segments": segments,
"lineage_session_ids": [seg["id"] for seg in segments],
"message_count": len(messages),
"messages": messages,
}
def export_all(self, source: str = None) -> List[Dict[str, Any]]:
"""
Export all sessions (with messages) as a list of dicts.
Suitable for writing to a JSONL file for backup/analysis.
"""
sessions = self.search_sessions(source=source, limit=100000)
results = []
for session in sessions:
messages = self.get_messages(session["id"])
results.append({**session, "messages": messages})
return results
"""Export all sessions (with messages) as dicts, e.g. for JSONL backup."""
return [
{**session, "messages": self.get_messages(session["id"])}
for session in self.search_sessions(source=source, limit=100000)
]
def adopt_session_lineage_from(
self,
@@ -313,52 +228,36 @@ class SessionPortabilityMixin:
*,
retire_donor: bool = True,
) -> Dict[str, Any]:
"""Adopt *session_id*'s full compression lineage from *donor_db* into
this store.
"""Adopt *session_id*'s full compression lineage from *donor_db*.
The stranded-bot-session heal (#93091 follow-up to #93296): before the
desktop routed session RPCs by their target session, a profile bot's
turns executed on whichever backend held window focus — usually the
default one — so the bot's canonical session rows and messages
accumulated in the DEFAULT profile's state.db. Once routing was fixed,
the profile backend correctly received the RPCs but had no such
session, so the same chat 4001'd for the opposite reason. This method
moves the conversation to where routing now looks for it.
Stranded-bot-session heal: before the desktop routed session RPCs by
target session, a profile bot's rows accumulated in the DEFAULT
profile's state.db; this moves the conversation to where routing now
looks. Pure composition: ``donor_db.export_session_lineage()`` ->
``self.import_sessions()`` — routing/handoff/activity fields reset,
already-present ids skipped (idempotent re-adoption).
Composition of existing primitives (no new import/export machinery):
``donor_db.export_session_lineage()`` -> ``self.import_sessions()``.
Import semantics apply unchanged: gateway routing, handoff, and live
activity fields are reset; already-present ids are skipped
(idempotent re-adoption after a partial run).
With ``retire_donor`` and a complete adoption, donor rows are ARCHIVED
(never deleted) with ``end_reason='adopted_by_profile'``. That
end_reason is deliberately NOT in the recoverable set
(agent_close/ws_orphan_reap): resurrection must not undo an adoption.
When ``retire_donor`` is True and at least one segment was imported
(or every segment already exists here), the donor rows are ARCHIVED —
never deleted — with ``end_reason='adopted_by_profile'`` so the
default profile's list stops advertising a conversation that now
lives elsewhere, while the bytes stay recoverable. The archive is
deliberately NOT in the recoverable set (agent_close/ws_orphan_reap):
canonical-lookup resurrection must not undo an adoption.
Returns the ``import_sessions`` result dict, plus ``adopted`` (bool)
and ``donor_retired`` (bool — True only when EVERY segment's
retirement actually applied).
Returns the ``import_sessions`` dict plus ``adopted`` and
``donor_retired`` (True only when EVERY segment's retirement applied).
"""
payload = donor_db.export_session_lineage(session_id)
if not payload:
return {
"ok": False,
"adopted": False,
"donor_retired": False,
"ok": False, "adopted": False, "donor_retired": False,
"error": f"session {session_id!r} not found in donor store",
}
segments = payload.get("segments") or [payload]
# Divergence guard: a segment we are about to SKIP (already present
# here) may have kept accumulating messages in the donor store after
# a partial earlier adoption. Retiring it would strand those newer
# messages behind a non-recoverable archive. Compare counts up front
# and refuse to retire (still adopt/import) when the donor is ahead.
# Divergence guard: a segment we will SKIP (already here) may have kept
# growing in the donor after a partial adoption; retiring it would strand
# those messages behind a non-recoverable archive. Still import, but
# refuse to retire when the donor is ahead.
donor_ahead = False
for seg in segments:
seg_id = seg.get("id")
@@ -394,15 +293,11 @@ class SessionPortabilityMixin:
if not seg_id:
continue
try:
# TOCTOU close-out: the guard above compared EXPORT-TIME
# counts, but another backend can append donor messages
# between export and this loop. Re-read both stores right
# before stamping; a donor-ahead signal here skips the
# stamp so growth never lands behind a non-recoverable
# archive. (Count comparison cannot see equal-count
# CONTENT divergence — e.g. a donor rewind+rewrite; that
# residual case is accepted: bytes stay in the donor
# store either way, only reachability differs.)
# TOCTOU close-out: the guard above used EXPORT-TIME counts;
# re-read both stores right before stamping so donor growth
# never lands behind a non-recoverable archive. (Equal-count
# CONTENT divergence is accepted: bytes stay in the donor
# either way, only reachability differs.)
donor_now = len(donor_db.get_messages(seg_id))
local_now = len(self.get_messages(seg_id))
if donor_now > local_now:
@@ -414,17 +309,15 @@ class SessionPortabilityMixin:
seg_id, donor_now, local_now,
)
continue
# First end_reason wins in end_session(); reopen first so
# the adoption boundary is stamped even on ended segments
# (e.g. 'compression' parents).
# First end_reason wins in end_session(); reopen so the
# adoption boundary is stamped even on ended segments.
donor_db.reopen_session(seg_id)
donor_db.end_session(seg_id, "adopted_by_profile")
donor_db.set_session_archived(seg_id, True)
except Exception:
# Best-effort by design: a retirement failure must not
# fail the adoption (the profile copy is already whole;
# a later resume retries retirement idempotently). But
# never claim success we didn't have.
# Best-effort: a retirement failure must not fail the adoption
# (a later resume retries idempotently) — but never claim
# success we didn't have.
retire_ok = False
logger.warning(
"failed to retire donor segment %s after adoption",
@@ -507,21 +400,16 @@ class SessionPortabilityMixin:
def import_sessions(self, sessions: List[Dict[str, Any]]) -> Dict[str, Any]:
"""Import sessions exported by :meth:`export_session` or ``export_all``.
Existing session IDs are skipped. Imported child sessions keep their
parent only when that parent already exists or is included in the same
import payload; otherwise the child is detached so partial imports don't
fail foreign-key validation. Gateway routing, handoff, rewind, and other
live runtime state are intentionally reset: this restores conversation
history, not ownership of a live channel or process.
Existing ids are skipped. A child keeps its parent only when the parent
exists or is in the same payload; otherwise it is detached so partial
imports pass FK validation. Gateway routing, handoff, rewind and other
live runtime state are reset: this restores history, not ownership of
a live channel or process.
Activity contract (#76354 review S4): export INCLUDES the live
activity fields (``last_activity_at`` / ``last_activity_description``
/ ``last_activity_provenance``) because they are part of the durable
row, but import deliberately RESETS them to NULL. Resurrecting a
stale "working ..." label on a machine where no agent is running
would fabricate activity the watchdog and session listings act on.
This asymmetry is intentional and covered by regression
(tests/gateway/test_watchdog_review_76354.py::test_s4_export_includes_activity_import_resets_it).
Activity contract: export INCLUDES ``last_activity_*`` (durable row
fields) but import RESETS them to NULL — resurrecting a stale
"working ..." label would fabricate activity the watchdog and listings
act on. Intentional asymmetry, pinned by regression test.
"""
if not isinstance(sessions, list):
raise ValueError("sessions must be a list")
@@ -536,32 +424,14 @@ class SessionPortabilityMixin:
total_messages = 0
total_bytes = 0
session_text_fields = (
"source",
"user_id",
"model",
"system_prompt",
"end_reason",
"cwd",
"git_branch",
"git_repo_root",
"billing_provider",
"billing_base_url",
"billing_mode",
"cost_status",
"cost_source",
"pricing_version",
"title",
"source", "user_id", "model", "system_prompt", "end_reason", "cwd",
"git_branch", "git_repo_root", "billing_provider", "billing_base_url",
"billing_mode", "cost_status", "cost_source", "pricing_version", "title",
)
# ``role`` is validated separately below (non-empty string).
message_text_fields = (
"role",
"tool_call_id",
"tool_name",
"effect_disposition",
"finish_reason",
"reasoning",
"reasoning_content",
"platform_message_id",
"message_id",
"tool_call_id", "tool_name", "effect_disposition", "finish_reason",
"reasoning", "reasoning_content", "platform_message_id", "message_id",
)
for index, raw in enumerate(sessions):
@@ -580,43 +450,23 @@ class SessionPortabilityMixin:
errors.append(self._import_error(index, session_id, "messages must be a list"))
continue
if len(messages) > self._IMPORT_MAX_MESSAGES_PER_SESSION:
errors.append(
self._import_error(
index,
session_id,
"messages exceeds the per-session import limit",
)
)
errors.append(self._import_error(index, session_id, "messages exceeds the per-session import limit"))
continue
if any(not isinstance(msg, dict) for msg in messages):
errors.append(
self._import_error(
index,
session_id,
"messages must contain only objects",
)
)
errors.append(self._import_error(index, session_id, "messages must contain only objects"))
continue
try:
session_bytes = len(
json.dumps(raw, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
)
session_bytes = len(json.dumps(raw, ensure_ascii=False, separators=(",", ":")).encode("utf-8"))
except (TypeError, ValueError):
errors.append(
self._import_error(index, session_id, "session must be JSON serializable")
)
errors.append(self._import_error(index, session_id, "session must be JSON serializable"))
continue
if session_bytes > self._IMPORT_MAX_SESSION_BYTES:
errors.append(
self._import_error(index, session_id, "session exceeds the import size limit")
)
errors.append(self._import_error(index, session_id, "session exceeds the import size limit"))
continue
total_bytes += session_bytes
if total_bytes > self._IMPORT_MAX_TOTAL_BYTES:
errors.append(
self._import_error(index, session_id, "import exceeds the total size limit")
)
errors.append(self._import_error(index, session_id, "import exceeds the total size limit"))
continue
try:
@@ -640,8 +490,6 @@ class SessionPortabilityMixin:
if not isinstance(role, str) or not role:
raise ValueError(f"messages[{message_index}].role must be a non-empty string")
for field in message_text_fields:
if field == "role":
continue
clean_message[field] = self._import_text_or_none(
clean_message.get(field), field
)
@@ -655,13 +503,7 @@ class SessionPortabilityMixin:
total_messages += len(clean_messages)
if total_messages > self._IMPORT_MAX_TOTAL_MESSAGES:
errors.append(
self._import_error(
index,
session_id,
"messages exceeds the total import limit",
)
)
errors.append(self._import_error(index, session_id, "messages exceeds the total import limit"))
continue
seen_ids.add(session_id)
normalized.append(
@@ -669,13 +511,7 @@ class SessionPortabilityMixin:
)
if errors:
return {
"ok": False,
"imported": 0,
"skipped": 0,
"detached": 0,
"errors": errors,
}
return {"ok": False, "imported": 0, "skipped": 0, "detached": 0, "errors": errors}
def _do(conn):
imported_ids: List[str] = []
@@ -738,27 +574,17 @@ class SessionPortabilityMixin:
"end_reason": raw.get("end_reason"),
"input_tokens": self._int_or_default(raw.get("input_tokens")),
"output_tokens": self._int_or_default(raw.get("output_tokens")),
"cache_read_tokens": self._int_or_default(
raw.get("cache_read_tokens")
),
"cache_write_tokens": self._int_or_default(
raw.get("cache_write_tokens")
),
"reasoning_tokens": self._int_or_default(
raw.get("reasoning_tokens")
),
"cache_read_tokens": self._int_or_default(raw.get("cache_read_tokens")),
"cache_write_tokens": self._int_or_default(raw.get("cache_write_tokens")),
"reasoning_tokens": self._int_or_default(raw.get("reasoning_tokens")),
"cwd": raw.get("cwd"),
"git_branch": raw.get("git_branch"),
"git_repo_root": raw.get("git_repo_root"),
"billing_provider": raw.get("billing_provider"),
"billing_base_url": raw.get("billing_base_url"),
"billing_mode": raw.get("billing_mode"),
"estimated_cost_usd": self._float_or_none(
raw.get("estimated_cost_usd")
),
"actual_cost_usd": self._float_or_none(
raw.get("actual_cost_usd")
),
"estimated_cost_usd": self._float_or_none(raw.get("estimated_cost_usd")),
"actual_cost_usd": self._float_or_none(raw.get("actual_cost_usd")),
"cost_status": raw.get("cost_status"),
"cost_source": raw.get("cost_source"),
"pricing_version": raw.get("pricing_version"),
@@ -771,18 +597,12 @@ class SessionPortabilityMixin:
sanitized_messages: List[Dict[str, Any]] = []
for msg in messages:
clean = dict(msg)
for key in (
"reasoning_details",
"codex_reasoning_items",
"codex_message_items",
):
for key in ("reasoning_details", "codex_reasoning_items", "codex_message_items"):
clean[key] = self._reasoning_json_value(clean.get(key))
sanitized_messages.append(clean)
total_messages, total_tool_calls = self._insert_message_rows(
conn,
session_id,
sanitized_messages,
conn, session_id, sanitized_messages
)
conn.execute(
"UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?",
@@ -826,9 +646,8 @@ class SessionPortabilityMixin:
(parent_id, session_id),
)
else:
# Drop only the closing edge. Later entries can still attach
# to this now-root session, preserving the acyclic portion
# of a malformed imported lineage.
# Drop only the closing edge; later entries can still attach
# to this now-root session.
parent_by_child.pop(session_id, None)
detached += 1
+64 -122
View File
@@ -1,40 +1,31 @@
"""Process-wide shared SessionDB registry (#90837).
"""Process-wide shared SessionDB registry.
A gateway process opens state.db from many call sites — the runner's
``AsyncSessionDB``, the ``SessionStore`` per-path cache, per-agent lazy
recall (``run_agent._get_session_db_for_recall``), per-job cron opens,
and per-message opens in mirror / channel_directory / slash_commands /
shutdown_flush / session_search / react_to_message. Each bare
``SessionDB()`` mints its own writer connection, ``self._lock``,
close-time WAL checkpoint, and async token-writer thread. With N
independent writer connections on one WAL file, mutual exclusion relies
only on SQLite's WAL write lock plus each instance's busy_timeout retry
ladder — and one connection's close-time checkpoint can race another's
growth, producing the lost/reordered-page-write signature reported
across 11+ incidents (#90837).
This module owns that boundary: one shared ``SessionDB`` per resolved
path per process, refcounted, with generation-aware retirement when the
underlying file is replaced (snapshot restore, recovery swap).
A gateway process opens state.db from many call sites (runner, SessionStore,
per-agent recall, cron, per-message helpers). Each bare ``SessionDB()`` mints
its own writer connection, lock, close-time WAL checkpoint and token-writer
thread; N independent writers on one WAL file rely only on SQLite's write lock
plus busy_timeout, and one connection's close-time checkpoint can race another's
growth (lost/reordered-page-write corruption). This module owns that boundary:
one shared ``SessionDB`` per resolved path per process, refcounted, with
generation-aware retirement when the file is replaced (snapshot restore,
recovery swap).
Lifecycle rules:
- ``acquire(path)`` returns the current generation for *path*,
incrementing its refcount. Same path ⇒ same instance ⇒ same writer
connection.
- ``close()`` on a shared instance is a NO-OP. The registry — not any
individual caller — owns the connection lifecycle, so one caller's
``close()`` can never tear down a writer other callers still hold.
- ``release(db)`` decrements the generation *db was acquired from*
(object-keyed, not pathname-keyed, so an inode replacement cannot
strand a still-owned generation). The final release of a retired
generation tears it down.
- On inode change, the old generation is RETIRED — never lent again —
but stays alive until its existing holders release. If a replacement
open fails, the registry is left WITHOUT a path entry (never a closed
stale object), so the next acquire retries fresh.
- All teardown happens OUTSIDE the registry lock: a final release's
WAL checkpoint must never stall acquisition for every state.db.
- ``acquire(path)`` returns the current generation for *path* and bumps its
refcount. Same path ⇒ same instance ⇒ same writer connection.
- ``close()`` on a shared instance is a NO-OP: the registry, not any caller,
owns the connection lifecycle, so one caller can never tear down a writer
others still hold.
- ``release(db)`` decrements the generation *db was acquired from* (object-
keyed, not pathname-keyed, so an inode replacement cannot strand a
still-owned generation). The final release of a retired generation tears
it down.
- On inode change the old generation is RETIRED (never lent again) but stays
alive until its holders release. If the replacement open fails the registry
keeps NO path entry (never a closed stale object) so the next acquire retries.
- All teardown happens OUTSIDE the registry lock: a final release's WAL
checkpoint must never stall acquisition for every state.db.
"""
from __future__ import annotations
@@ -44,34 +35,14 @@ import threading
from pathlib import Path
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
from hermes_state_common import stat_db_file_identity as _stat_db_file_identity
if TYPE_CHECKING: # pragma: no cover - import cycle guard, typed only
from hermes_state import SessionDB
logger = logging.getLogger(__name__)
def _stat_db_file_identity(path: Path) -> Optional[Tuple[int, int]]:
"""Return ``(st_dev, st_ino)`` for *path*, or None when unavailable.
Mirrors the hermes_state helper of the same name; kept local so this
module has no import-time dependency on hermes_state (which imports
this module — the cycle is resolved by deferring SessionDB lookup
to call time).
"""
import os
try:
st = os.stat(path)
except OSError:
return None
# Windows volumes (and some network FS) report st_ino=0; a (0, 0)
# identity would false-positive every check. Skip the inode half of
# the guard there.
if not st.st_dev or not st.st_ino:
return None
return (st.st_dev, st.st_ino)
class _Generation:
"""One shared SessionDB generation: instance, refcount, file identity."""
@@ -85,16 +56,13 @@ class _Generation:
_lock = threading.Lock()
# path → live generation (never retired). A retired generation leaves
# this table immediately on retirement and lives on in _retired until
# its last holder releases.
# path → live generation. Retired generations move to _retired (keyed by
# id(db)) until their last holder releases.
_generations: Dict[Path, _Generation] = {}
# Object-keyed retired generations still draining holders.
_retired: Dict[int, _Generation] = {} # id(db) → generation
# Paths whose next generation is currently being constructed. Construction
# stays outside _lock because schema reconciliation can take seconds, but peers
# for the SAME file must wait: otherwise every cold caller opens a writable
# SQLite connection before the registry chooses one winner.
_retired: Dict[int, _Generation] = {}
# Paths whose next generation is being constructed. Construction runs outside
# _lock (schema reconciliation can take seconds), but peers for the SAME file
# must wait or every cold caller opens its own writer before a winner is chosen.
_opening: Dict[Path, threading.Event] = {}
@@ -120,20 +88,13 @@ def _teardown(db: "SessionDB") -> None:
def acquire(db_path: Optional[Path] = None) -> "SessionDB":
"""Return the shared SessionDB for *db_path*, incrementing its refcount.
The same resolved path always returns the same ``SessionDB`` instance
within one process, so all long-lived in-process callers share one
writer connection, one ``self._lock``, and one token-writer thread.
If the file was replaced (different inode) since the generation opened —
``hermes sessions recover``, snapshot restore — that generation is RETIRED
but stays alive for its holders, and a fresh one is opened in its place.
If the underlying file was replaced (different inode) since the
shared generation was opened — e.g. by ``hermes sessions recover`` or
a snapshot restore — the current generation is RETIRED (never lent
again) but stays alive for its existing holders, and a fresh
generation is opened in its place.
Raises whatever ``SessionDB.__init__`` raises (malformed, locked,
etc.). On a replacement-open failure the registry holds NO entry for
the path, so the next acquire retries fresh rather than handing out
a closed stale object.
Raises whatever ``SessionDB.__init__`` raises. On a replacement-open
failure the registry holds NO entry for the path, so the next acquire
retries fresh instead of receiving a closed stale object.
"""
from hermes_state import _default_db_path
@@ -153,9 +114,8 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB":
and generation.identity is not None
and current != generation.identity
):
# File replaced: retire the live generation (its
# holders keep it until they release) and elect one
# caller to construct the replacement below.
# File replaced: retire, then elect one caller to open
# the replacement below.
_retire_generation_locked(path, generation)
else:
generation.refcount += 1
@@ -167,13 +127,12 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB":
_opening[path] = opening
break
# Another caller is constructing this path. Do not hold the global
# registry lock while waiting: unrelated databases continue opening.
# A failed opener signals too, so one waiter can retry as the successor.
# Another caller is constructing this path; wait without holding the
# global lock. A failed opener signals too, so a waiter can retry.
opening.wait()
# Open a fresh generation OUTSIDE the lock. The per-path opening marker
# prevents redundant writer connections without serialising other files.
# Open OUTSIDE the lock; the per-path marker prevents redundant writers
# without serialising other files.
try:
db = _open_session_db(path)
db._shared_registry_owned = True
@@ -188,8 +147,7 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB":
with _lock:
existing = _generations.get(path)
if existing is not None:
# Defensive: a generation may have been installed by explicit
# registry manipulation while this open was in flight.
# Defensive: installed by explicit registry manipulation mid-open.
existing.refcount += 1
winner = existing.db
else:
@@ -206,9 +164,8 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB":
def _retire_generation_locked(path: Path, generation: _Generation) -> None:
"""Retire *generation* so it is never lent again (caller holds _lock).
The instance stays alive — its holders still own references — and is
tracked in ``_retired`` keyed by ``id(db)`` so their releases find
the right generation even after the path maps to a new one.
It stays alive for its holders, tracked in ``_retired`` by ``id(db)`` so
their releases find it even after the path maps to a new generation.
"""
generation.retired = True
if _generations.get(path) is generation:
@@ -219,15 +176,11 @@ def _retire_generation_locked(path: Path, generation: _Generation) -> None:
def release(db: "SessionDB") -> bool:
"""Decrement the refcount of a shared SessionDB.
Returns ``True`` if *db* was a shared instance and its refcount was
decremented; ``False`` if *db* is not registry-managed (caller owns
its own close()). The final release of a generation tears it down —
OUTSIDE the registry lock, so a close-time WAL checkpoint never
stalls acquisition for every state.db in the process.
Object-keyed lookup means an inode replacement cannot strand a
still-owned generation: holders of the old generation release into
the retired record, not into whatever the path currently names.
Returns ``True`` if *db* was shared; ``False`` if it is not registry-managed
(caller owns close()). The final release tears the generation down OUTSIDE
the registry lock so a close-time WAL checkpoint never stalls acquisition.
Lookup is object-keyed, so holders of an old generation release into its
retired record, not into whatever the path currently names.
"""
if db is None:
return False
@@ -244,8 +197,7 @@ def release(db: "SessionDB") -> bool:
return False
generation = _generations.get(path)
if generation is None or generation.db is not db:
# Not a shared instance (caller used SessionDB()
# directly) — nothing to do; the caller owns close().
# Not shared (bare SessionDB()); the caller owns close().
return False
generation.refcount -= 1
needs_teardown = generation.refcount <= 0
@@ -259,21 +211,17 @@ def release(db: "SessionDB") -> bool:
_generations.pop(Path(path), None)
except (TypeError, ValueError):
pass
# Teardown OUTSIDE the lock: it stops the token writer, checkpoints
# the WAL, and drains the read pool — none of which may hold up
# acquisition for every other state.db in the process.
# Teardown OUTSIDE the lock: stopping the token writer, WAL checkpoint and
# read-pool drain must not block acquisition for every other state.db.
if needs_teardown:
_teardown(db)
return True
def close_all() -> int:
"""Close every shared SessionDB in this process, regardless of refcount.
"""Close every shared SessionDB regardless of refcount; returns the count.
Called at gateway shutdown (after all agents and cron jobs have
finished) to release every WAL write lock and drain every
token-writer thread cleanly. Returns the number of instances
closed. Idempotent.
For gateway shutdown, after all agents and cron jobs finished. Idempotent.
"""
closed = 0
with _lock:
@@ -282,7 +230,6 @@ def close_all() -> int:
_retired.clear()
for generation in generations:
generation.retired = True
# Teardown outside the lock, one generation at a time.
for generation in generations:
_teardown(generation.db)
closed += 1
@@ -290,12 +237,11 @@ def close_all() -> int:
def live_shared_session_dbs() -> List["SessionDB"]:
"""Snapshot of every live (non-retired) shared SessionDB in this process.
"""Snapshot of every live (non-retired) shared SessionDB.
For periodic in-process maintenance (the gateway housekeeping tick's
deferred-FTS retry). Refcounts are NOT touched: the caller only invokes
a method on an instance that some holder already keeps alive; a
concurrent final release closes it and the callee sees ``_conn is None``.
For in-process maintenance (housekeeping deferred-FTS retry). Refcounts
are NOT touched: a concurrent final release may close an instance, in
which case the callee sees ``_conn is None``.
"""
with _lock:
return [g.db for g in _generations.values() if not g.retired]
@@ -314,9 +260,7 @@ def stats() -> Dict[str, int]:
}
# ── Backwards-compatible aliases (hermes_state re-exports) ──
# Kept so call sites and tests can import either from hermes_state
# (the historical path) or from this module directly.
# ── Backwards-compatible aliases (hermes_state re-exports them) ──
def get_shared_session_db(db_path: Optional[Path] = None) -> "SessionDB":
return acquire(db_path)
@@ -333,10 +277,8 @@ def close_shared_session_dbs() -> int:
def release_or_close(db: "SessionDB") -> None:
"""Release a shared instance, or close it when it is not registry-managed.
The one-line cleanup for call sites that previously did a plain
``db.close()``: shared instances return their refcount to the
registry (the registry owns the lifecycle), anything else — read-only
opens, CLI one-shots, test fakes — falls back to a direct close.
Drop-in for a plain ``db.close()``: read-only opens, CLI one-shots and
test fakes fall back to a direct close.
"""
if not release(db):
try:
File diff suppressed because it is too large Load Diff
+291 -595
View File
File diff suppressed because it is too large Load Diff
+637 -1285
View File
File diff suppressed because it is too large Load Diff
+529
View File
@@ -0,0 +1,529 @@
"""Telegram DM topic-mode mixin for :class:`hermes_state.SessionDB`."""
from __future__ import annotations
import logging
import sqlite3
import time
from typing import Any, Dict, List, Optional
from hermes_state_common import (
_PREVIEW_ELIGIBLE_SQL,
_PREVIEW_RAW_SELECT,
_shape_preview,
_sql_session_last_active,
)
# caplog tests pin the "hermes_state" logger name.
logger = logging.getLogger("hermes_state")
def _normalize_telegram_topic_profile_name(profile_name: Optional[str] = None) -> str:
"""Empty/missing → ``"default"`` (single namespace for non-multiplexed
gateways). Multiplexed callers must pass the *routed* profile
(``source.profile``), never the process-global active profile."""
name = str(profile_name or "").strip()
return name if name else "default"
class SessionTelegramTopicsMixin:
"""Telegram DM topic-mode tables, bindings and lookups."""
def apply_telegram_topic_migration(self) -> None:
"""Create Telegram DM topic-mode tables on explicit /topic opt-in.
Deliberately NOT part of startup reconciliation: operators can upgrade
and keep the old bot behavior until a user runs /topic.
Schema versions: v1 initial; v2 session_id FK ON DELETE CASCADE (pruning
clears bindings); v3 ``profile_name`` on both tables so multiplexed
gateways sharing one state.db isolate topic state per profile.
"""
# (table, column list, DDL body). profile_name leads the PK: a private
# chat_id is the user id, identical across bots sharing one state.db.
tables = (
(
"telegram_dm_topic_mode",
"profile_name, chat_id, user_id, enabled, activated_at, updated_at, "
"has_topics_enabled, allows_users_to_create_topics, "
"capability_checked_at, intro_message_id, pinned_message_id",
"""
profile_name TEXT NOT NULL DEFAULT 'default',
chat_id TEXT NOT NULL,
user_id TEXT NOT NULL,
enabled INTEGER NOT NULL DEFAULT 1,
activated_at REAL NOT NULL,
updated_at REAL NOT NULL,
has_topics_enabled INTEGER,
allows_users_to_create_topics INTEGER,
capability_checked_at REAL,
intro_message_id TEXT,
pinned_message_id TEXT,
PRIMARY KEY (profile_name, chat_id)
""",
),
(
"telegram_dm_topic_bindings",
"profile_name, chat_id, thread_id, user_id, session_key, "
"session_id, managed_mode, linked_at, updated_at",
"""
profile_name TEXT NOT NULL DEFAULT 'default',
chat_id TEXT NOT NULL,
thread_id TEXT NOT NULL,
user_id TEXT NOT NULL,
session_key TEXT NOT NULL,
session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
managed_mode TEXT NOT NULL DEFAULT 'auto',
linked_at REAL NOT NULL,
updated_at REAL NOT NULL,
PRIMARY KEY (profile_name, chat_id, thread_id)
""",
),
)
def _do(conn):
for table, columns, ddl in tables:
conn.execute(f"CREATE TABLE IF NOT EXISTS {table} ({ddl})")
have = {row[1] for row in conn.execute(f"PRAGMA table_info('{table}')")}
if "profile_name" in have:
continue
# v1/v2 → v3. SQLite can't ALTER a PK or FK, so rebuild (this
# also supplies v2's ON DELETE CASCADE). Legacy rows land in
# "default" only, never replicated across profiles.
legacy_columns = columns.replace("profile_name, ", "", 1)
conn.executescript(
f"""
CREATE TABLE {table}_new ({ddl});
INSERT INTO {table}_new ({columns})
SELECT 'default', {legacy_columns} FROM {table};
DROP TABLE {table};
ALTER TABLE {table}_new RENAME TO {table};
"""
)
# Indexes after any rebuild: the user index needs profile_name.
conn.executescript(
"""
CREATE UNIQUE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_session
ON telegram_dm_topic_bindings(session_id);
CREATE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_user
ON telegram_dm_topic_bindings(profile_name, user_id, chat_id);
"""
)
conn.execute(
"INSERT INTO state_meta (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
("telegram_dm_topic_schema_version", "3"),
)
self._execute_write(_do)
def enable_telegram_topic_mode(
self,
*,
chat_id: str,
user_id: str,
profile_name: str = "default",
has_topics_enabled: Optional[bool] = None,
allows_users_to_create_topics: Optional[bool] = None,
) -> None:
"""Enable Telegram DM topic mode for one private chat/user.
Owns the explicit topic migration; SessionDB startup must not create
these tables. Multiplexed callers pass the routed ``source.profile``,
not the process-global active profile.
"""
self.apply_telegram_topic_migration()
now = time.time()
profile_name = _normalize_telegram_topic_profile_name(profile_name)
def _to_int(value: Optional[bool]) -> Optional[int]:
if value is None:
return None
return 1 if value else 0
self._write_sql(
"""
INSERT INTO telegram_dm_topic_mode (
profile_name, chat_id, user_id, enabled, activated_at, updated_at,
has_topics_enabled, allows_users_to_create_topics,
capability_checked_at
) VALUES (?, ?, ?, 1, ?, ?, ?, ?, ?)
ON CONFLICT(profile_name, chat_id) DO UPDATE SET
user_id = excluded.user_id,
enabled = 1,
updated_at = excluded.updated_at,
has_topics_enabled = excluded.has_topics_enabled,
allows_users_to_create_topics = excluded.allows_users_to_create_topics,
capability_checked_at = excluded.capability_checked_at
""",
(
profile_name,
str(chat_id),
str(user_id),
now,
now,
_to_int(has_topics_enabled),
_to_int(allows_users_to_create_topics),
now,
),
)
def disable_telegram_topic_mode(
self,
*,
chat_id: str,
profile_name: str = "default",
clear_bindings: bool = True,
) -> None:
"""Disable Telegram DM topic mode for one private chat.
``clear_bindings`` also drops the chat's bindings so a later re-enable
starts clean. Never creates the tables; absent tables are a no-op.
"""
profile_name = _normalize_telegram_topic_profile_name(profile_name)
def _do(conn):
try:
conn.execute(
"UPDATE telegram_dm_topic_mode SET enabled = 0, updated_at = ? "
"WHERE profile_name = ? AND chat_id = ?",
(time.time(), profile_name, str(chat_id)),
)
if clear_bindings:
conn.execute(
"DELETE FROM telegram_dm_topic_bindings "
"WHERE profile_name = ? AND chat_id = ?",
(profile_name, str(chat_id)),
)
except sqlite3.OperationalError:
return
self._execute_write(_do)
def is_telegram_topic_mode_enabled(
self,
*,
chat_id: str,
user_id: str,
profile_name: str = "default",
) -> bool:
"""Return whether Telegram DM topic mode is enabled for this chat/user."""
profile_name = _normalize_telegram_topic_profile_name(profile_name)
with self._read_ctx() as conn:
try:
row = conn.execute(
"""
SELECT enabled FROM telegram_dm_topic_mode
WHERE profile_name = ? AND chat_id = ? AND user_id = ?
""",
(profile_name, str(chat_id), str(user_id)),
).fetchone()
except sqlite3.OperationalError:
return False
if row is None:
return False
enabled = row[0]
return bool(enabled)
def get_telegram_topic_binding(
self,
*,
chat_id: str,
thread_id: str,
profile_name: str = "default",
) -> Optional[Dict[str, Any]]:
"""Return the session binding for a Telegram DM topic, if present."""
profile_name = _normalize_telegram_topic_profile_name(profile_name)
with self._read_ctx() as conn:
try:
row = conn.execute(
"""
SELECT * FROM telegram_dm_topic_bindings
WHERE profile_name = ? AND chat_id = ? AND thread_id = ?
""",
(profile_name, str(chat_id), str(thread_id)),
).fetchone()
except sqlite3.OperationalError:
return None
return dict(row) if row else None
def list_telegram_topic_bindings_for_chat(
self,
*,
chat_id: str,
profile_name: str = "default",
) -> List[Dict[str, Any]]:
"""All bindings for one chat, newest first. Read-only: [] when the
table is absent (never triggers the migration)."""
profile_name = _normalize_telegram_topic_profile_name(profile_name)
with self._read_ctx() as conn:
try:
rows = conn.execute(
"SELECT * FROM telegram_dm_topic_bindings "
"WHERE profile_name = ? AND chat_id = ? "
"ORDER BY updated_at DESC",
(profile_name, str(chat_id)),
).fetchall()
except sqlite3.OperationalError:
return []
return [dict(row) for row in rows]
def get_telegram_topic_binding_by_session(
self,
*,
session_id: str,
) -> Optional[Dict[str, Any]]:
"""Reverse lookup via the UNIQUE INDEX on session_id; None when
unbound or the table is absent."""
with self._read_ctx() as conn:
try:
row = conn.execute(
"""
SELECT * FROM telegram_dm_topic_bindings
WHERE session_id = ?
""",
(str(session_id),),
).fetchone()
except sqlite3.OperationalError:
return None
return dict(row) if row else None
def delete_telegram_topic_binding(
self,
*,
chat_id: str,
thread_id: str,
profile_name: str = "default",
) -> int:
"""Remove the binding row for one (chat, thread) pair.
Called when the Bot API confirms a topic was deleted externally
(``Thread not found`` after the same-thread retry failed); otherwise
``gateway.run._recover_telegram_topic_thread_id`` keeps redirecting
inbound messages to the dead topic.
If this removes the chat's *last* binding, ``telegram_dm_topic_mode``
is flipped to ``enabled = 0`` in the same transaction; otherwise the
chat stays in topic mode with zero lanes and a user who disabled topics
in the Telegram client (not via ``/topic off``) stays stuck.
Returns the number of rows deleted; absent binding or unmigrated tables
are silent no-ops (never raise from a cleanup hot path).
"""
chat_id = str(chat_id)
thread_id = str(thread_id)
profile_name = _normalize_telegram_topic_profile_name(profile_name)
deleted = {"count": 0}
def _do(conn):
try:
cursor = conn.execute(
"""
DELETE FROM telegram_dm_topic_bindings
WHERE profile_name = ? AND chat_id = ? AND thread_id = ?
""",
(profile_name, chat_id, thread_id),
)
deleted["count"] = cursor.rowcount or 0
except sqlite3.OperationalError:
deleted["count"] = 0
return
if not deleted["count"]:
return
# Last binding gone → disable topic mode. Same transaction, so no
# read-after-prune race.
try:
remaining = conn.execute(
"""
SELECT 1 FROM telegram_dm_topic_bindings
WHERE profile_name = ? AND chat_id = ? LIMIT 1
""",
(profile_name, chat_id),
).fetchone()
if remaining is None:
conn.execute(
"UPDATE telegram_dm_topic_mode "
"SET enabled = 0, updated_at = ? "
"WHERE profile_name = ? AND chat_id = ?",
(time.time(), profile_name, chat_id),
)
except sqlite3.OperationalError:
# telegram_dm_topic_mode absent — binding prune still stands.
pass
self._execute_write(_do)
return deleted["count"]
def bind_telegram_topic(
self,
*,
chat_id: str,
thread_id: str,
user_id: str,
session_key: str,
session_id: str,
managed_mode: str = "auto",
profile_name: str = "default",
) -> None:
"""Bind one Telegram DM topic thread to one Hermes session.
A session may be linked to only one topic: rebinding the same pair is
idempotent; linking the session to a different topic raises ValueError.
"""
self.apply_telegram_topic_migration()
now = time.time()
chat_id = str(chat_id)
thread_id = str(thread_id)
user_id = str(user_id)
session_key = str(session_key)
session_id = str(session_id)
profile_name = _normalize_telegram_topic_profile_name(profile_name)
def _do(conn):
existing_session = conn.execute(
"""
SELECT profile_name, chat_id, thread_id
FROM telegram_dm_topic_bindings
WHERE session_id = ?
""",
(session_id,),
).fetchone()
if existing_session is not None:
if isinstance(existing_session, sqlite3.Row):
linked_profile = existing_session["profile_name"]
linked_chat = existing_session["chat_id"]
linked_thread = existing_session["thread_id"]
else:
linked_profile, linked_chat, linked_thread = existing_session
if (
str(linked_profile) != profile_name
or str(linked_chat) != chat_id
or str(linked_thread) != thread_id
):
raise ValueError("session is already linked to another Telegram topic")
conn.execute(
"""
INSERT INTO telegram_dm_topic_bindings (
profile_name, chat_id, thread_id, user_id, session_key, session_id,
managed_mode, linked_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(profile_name, chat_id, thread_id) DO UPDATE SET
user_id = excluded.user_id,
session_key = excluded.session_key,
session_id = excluded.session_id,
managed_mode = excluded.managed_mode,
updated_at = excluded.updated_at
""",
(
profile_name,
chat_id,
thread_id,
user_id,
session_key,
session_id,
managed_mode,
now,
now,
),
)
self._execute_write(_do)
def is_telegram_session_linked_to_topic(self, *, session_id: str) -> bool:
"""True if the session is bound to any Telegram DM topic. Read-only:
absent tables (nobody ran ``/topic``) mean unbound → False."""
with self._read_ctx() as conn:
try:
row = conn.execute(
"""
SELECT 1 FROM telegram_dm_topic_bindings
WHERE session_id = ?
LIMIT 1
""",
(str(session_id),),
).fetchone()
except sqlite3.OperationalError:
return False
return row is not None
def list_unlinked_telegram_sessions_for_user(
self,
*,
chat_id: str,
user_id: str,
profile_name: str = "default",
limit: int = 10,
) -> List[Dict[str, Any]]:
"""This user's Telegram sessions not bound to a topic.
Read-only: if the bindings table is absent, every session is unlinked
and a simpler query is used. Scoped by ``profile_name`` so multiplexed
profiles do not surface each other's sessions.
"""
profile_name = _normalize_telegram_topic_profile_name(profile_name)
# sessions.profile_name is NULL/empty for legacy rows → treat as default.
profile_clause = "AND COALESCE(NULLIF(TRIM(s.profile_name), ''), 'default') = ?"
with self._read_ctx() as conn:
try:
rows = conn.execute(
f"""
SELECT s.*,
COALESCE(sp.prompt, s.system_prompt)
AS _system_prompt_resolved,
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
AND {_PREVIEW_ELIGIBLE_SQL}
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
{_sql_session_last_active("s")} AS last_active
FROM sessions s
LEFT JOIN system_prompts sp
ON sp.hash = s.system_prompt_hash
WHERE s.source = 'telegram'
AND s.user_id = ?
{profile_clause}
AND NOT EXISTS (
SELECT 1 FROM telegram_dm_topic_bindings b
WHERE b.session_id = s.id
)
ORDER BY last_active DESC, s.started_at DESC
LIMIT ?
""",
(str(user_id), profile_name, int(limit)),
).fetchall()
except sqlite3.OperationalError:
rows = conn.execute(
f"""
SELECT s.*,
COALESCE(sp.prompt, s.system_prompt)
AS _system_prompt_resolved,
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
AND {_PREVIEW_ELIGIBLE_SQL}
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
{_sql_session_last_active("s")} AS last_active
FROM sessions s
LEFT JOIN system_prompts sp
ON sp.hash = s.system_prompt_hash
WHERE s.source = 'telegram'
AND s.user_id = ?
ORDER BY last_active DESC, s.started_at DESC
LIMIT ?
""",
(str(user_id), int(limit)),
).fetchall()
sessions: List[Dict[str, Any]] = []
for row in rows:
session = self._session_row_dict(row)
session["preview"] = _shape_preview(session.pop("_preview_raw", ""))
sessions.append(session)
return sessions
+305
View File
@@ -0,0 +1,305 @@
"""Session title mixin for SessionDB: sanitizing, auto/user provenance
ranking, and lineage-aware lookups."""
from __future__ import annotations
import logging
import re
from typing import Any, Dict, Optional
from agent.message_sanitization import _sanitize_surrogates
from hermes_state_common import _COMPRESSION_CHILD_SQL, escape_like as _escape_like
# caplog tests pin the "hermes_state" logger name.
logger = logging.getLogger("hermes_state")
class SessionTitlesMixin:
"""Sanitizing, ranking auto/user titles, lineage-aware lookups."""
@classmethod
def _title_rank(cls, source: Optional[str]) -> int:
"""Rank a stored title_source.
NULL (pre-provenance rows) is indistinguishable from a manual ``/title``
of that era, so it ranks as ``user``: auto-titling only ever fills
genuinely empty legacy titles.
"""
if source is None:
return cls._TITLE_SOURCE_RANK[cls.TITLE_SOURCE_USER]
return cls._TITLE_SOURCE_RANK.get(str(source), 0)
@staticmethod
def sanitize_title(title: Optional[str]) -> Optional[str]:
"""Strip control/zero-width/bidi chars, collapse whitespace, normalize
empty to None. Raises ValueError if longer than MAX_TITLE_LENGTH
after cleaning."""
from hermes_state import SessionDB
if not title:
return None
# Lone surrogates cannot be bound by sqlite3 (UnicodeEncodeError).
title = _sanitize_surrogates(title)
# ASCII controls, keeping \t \n \r so the whitespace collapse below
# turns them into spaces.
cleaned = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', title)
# Zero-width, bidi override, object-replacement, interlinear annotation.
cleaned = re.sub(
r'[\u200b-\u200f\u2028-\u202e\u2060-\u2069\ufeff\ufffc\ufff9-\ufffb]',
'', cleaned,
)
cleaned = re.sub(r'\s+', ' ', cleaned).strip()
if not cleaned:
return None
if len(cleaned) > SessionDB.MAX_TITLE_LENGTH:
raise ValueError(
f"Title too long ({len(cleaned)} chars, max {SessionDB.MAX_TITLE_LENGTH})"
)
return cleaned
def _is_compression_ancestor(
self, conn, *, ancestor_id: str, descendant_id: str
) -> bool:
"""True if *ancestor_id* is a compression predecessor of *descendant_id*.
Uses the canonical continuation edge ``_COMPRESSION_CHILD_SQL`` (parent
ended with ``end_reason = 'compression'`` and child started at/after its
``ended_at``), which excludes delegate/branch children that also carry
``parent_session_id``. One recursive CTE so the edge is defined once.
"""
if not ancestor_id or not descendant_id or ancestor_id == descendant_id:
return False
edge = _COMPRESSION_CHILD_SQL.format(a="child")
row = conn.execute(
f"""
WITH RECURSIVE ancestors(id) AS (
SELECT ?
UNION
SELECT parent.id
FROM ancestors a
JOIN sessions child ON child.id = a.id
JOIN sessions parent ON parent.id = child.parent_session_id
WHERE {edge}
)
SELECT 1 FROM ancestors WHERE id = ? AND id != ? LIMIT 1
""",
(descendant_id, ancestor_id, descendant_id),
).fetchone()
return row is not None
def _set_session_title(
self,
session_id: str,
title: str,
*,
source: str,
) -> bool:
"""Write a title, enforcing provenance precedence.
A ``user`` write always lands. ``derived``/``llm`` land only when the
row is untitled or holds strictly lower authority, so derived upgrades
to llm exactly once, nothing overwrites a user name, and re-running the
titler on an llm row is a no-op (stops sessions renaming themselves).
No writer may move a hidden canonical Bot Chat off its title.
Read and write are one compare-and-swap in a single transaction, so a
manual ``/title`` racing an in-flight generation is not clobbered.
"""
title = self.sanitize_title(title)
is_user = source == self.TITLE_SOURCE_USER
new_rank = self._title_rank(source) if not is_user else None
def _do(conn):
current = conn.execute(
"SELECT title, title_source, hidden FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if current is None:
return 0
# The canonical Bot Chat's NAME is its identity: Bot Mode resolves it
# by exact-title lookup on every open, so a rename orphans the whole
# conversation (next open mints an empty replacement and UNIQUE(title)
# blocks renaming back). Refuse here, the single write path every
# surface funnels through. Hidden is the discriminator: canonical
# chats are born hidden; a visible session merely named "Bot Chat"
# stays renameable. Provenance-blind so the auto-titler no-ops too.
if (
(current["title"] or "") == self.CANONICAL_BOT_CHAT_TITLE
and bool(current["hidden"])
and title != self.CANONICAL_BOT_CHAT_TITLE
):
if is_user:
raise ValueError(
"This is the bot's canonical Bot Chat — its name is its "
"identity, and renaming it would orphan the conversation. "
"To start fresh, create a new bot instead."
)
return 0
if not is_user and current["title"] is not None:
if self._title_rank(current["title_source"]) >= new_rank:
return 0
if title:
cursor = conn.execute(
"SELECT id FROM sessions WHERE title = ? AND id != ?",
(title, session_id),
)
conflict = cursor.fetchone()
if conflict:
conflict_id = conflict["id"]
# If the conflicting holder is a hidden compressed ancestor
# of this continuation, the user cannot free the title, so
# transfer it onto the tip. Uniqueness and lineage are kept.
if self._is_compression_ancestor(
conn, ancestor_id=conflict_id, descendant_id=session_id
):
conn.execute(
"UPDATE sessions SET title = NULL WHERE id = ?",
(conflict_id,),
)
else:
raise ValueError(
f"Title '{title}' is already in use by session {conflict_id}"
)
# CAS on the values just read (``IS`` is NULL-safe): a concurrent
# write between the SELECT and here loses instead of being overwritten.
cursor = conn.execute(
"UPDATE sessions SET title = ?, title_source = ? "
"WHERE id = ? AND title IS ? AND title_source IS ?",
(
title,
source if title else None,
session_id,
current["title"],
current["title_source"],
),
)
return cursor.rowcount
rowcount = self._execute_write(_do)
return rowcount > 0
def set_session_title(self, session_id: str, title: str) -> bool:
"""Set a title on the user's behalf (``user`` provenance; auto-titling
never replaces it). Empty clears the title. Raises ValueError on a
title conflict or validation failure. Automatic callers must use
:meth:`set_auto_title`."""
return self._set_session_title(
session_id, title, source=self.TITLE_SOURCE_USER
)
def set_auto_title(self, session_id: str, title: str, *, source: str) -> bool:
"""Set an automatic title; False (untouched) when a higher-authority
title already holds the row."""
if source not in (self.TITLE_SOURCE_DERIVED, self.TITLE_SOURCE_LLM):
raise ValueError(f"invalid automatic title source: {source!r}")
return self._set_session_title(session_id, title, source=source)
def set_auto_title_if_empty(self, session_id: str, title: str) -> bool:
"""Back-compat shim (third-party plugins reference it by name); new
code calls :meth:`set_auto_title` with an explicit source."""
return self.set_auto_title(
session_id, title, source=self.TITLE_SOURCE_LLM
)
def get_session_title(self, session_id: str) -> Optional[str]:
"""Get the title for a session, or None."""
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT title FROM sessions WHERE id = ?", (session_id,)
)
row = cursor.fetchone()
return row["title"] if row else None
def get_session_title_source(self, session_id: str) -> Optional[str]:
"""Get the provenance of a session's title, or None when untitled."""
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT title, title_source FROM sessions WHERE id = ?",
(session_id,),
)
row = cursor.fetchone()
if not row or row["title"] is None:
return None
return row["title_source"]
def set_session_title_source(self, session_id: str, source: str) -> bool:
"""Overwrite a title's provenance without touching the text: a title
copied across a compression rotation keeps the original's authority."""
if source not in self._TITLE_SOURCE_RANK:
raise ValueError(f"invalid title source: {source!r}")
return self._write_rowcount(
"UPDATE sessions SET title_source = ? "
"WHERE id = ? AND title IS NOT NULL",
(source, session_id),
) > 0
def get_session_by_title(self, title: str) -> Optional[Dict[str, Any]]:
"""Look up a session by exact title. Returns session dict or None."""
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT s.*, "
"COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved "
"FROM sessions s "
"LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash "
"WHERE s.title = ?",
(title,),
)
row = cursor.fetchone()
return self._session_row_dict(row) if row else None
def resolve_session_by_title(self, title: str) -> Optional[str]:
"""Resolve a title to a session ID, preferring the latest "title #N"
continuation over the exact match."""
exact = self.get_session_by_title(title)
# Escape LIKE wildcards so "%"/"_" in titles cannot false-match.
escaped = _escape_like(title)
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT id, title, started_at FROM sessions "
"WHERE title LIKE ? ESCAPE '\\' ORDER BY started_at DESC",
(f"{escaped} #%",),
)
numbered = cursor.fetchall()
if numbered:
return numbered[0]["id"]
elif exact:
return exact["id"]
return None
def get_next_title_in_lineage(self, base_title: str) -> str:
"""Next title in a lineage ("my session" → "my session #2"): strip any
" #N" suffix, then increment the highest existing number."""
match = re.match(r'^(.*?) #(\d+)$', base_title)
if match:
base = match.group(1)
else:
base = base_title
escaped = _escape_like(base)
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT title FROM sessions WHERE title = ? OR title LIKE ? ESCAPE '\\'",
(base, f"{escaped} #%"),
)
existing = [row["title"] for row in cursor.fetchall()]
if not existing:
return base
max_num = 1 # The unnumbered original counts as #1
for t in existing:
m = re.match(r'^.* #(\d+)$', t)
if m:
max_num = max(max_num, int(m.group(1)))
return f"{base} #{max_num + 1}"
+597
View File
@@ -0,0 +1,597 @@
"""Token/usage accounting mixin for SessionDB: the coalescing background
token writer, per-model usage rows, and billing-route columns. Writer thread
state lives on the SessionDB instance."""
from __future__ import annotations
import atexit
import logging
import threading
import time
import weakref
from typing import Any, Dict, List, Optional, Tuple
# caplog tests pin the "hermes_state" logger name.
logger = logging.getLogger("hermes_state")
class SessionUsageMixin:
"""Coalesced token writer, per-model usage rows, billing route."""
def update_session_billing_route(
self,
session_id: str,
*,
provider: str,
base_url: str,
billing_mode: Optional[str] = None,
) -> None:
"""Unconditionally set the billing route (``update_token_counts`` only
COALESCE-fills NULLs) so the dashboard reflects the latest /model switch.
Also nulls ``system_prompt`` so the cached snapshot (stale ``Model:`` /
``Provider:`` header) is rebuilt, like ``update_session_model``.
"""
# Barrier against queued token deltas — see update_session_model.
self.flush_token_counts()
def _do(conn):
conn.execute(
"""UPDATE sessions SET
billing_provider = ?,
billing_base_url = ?,
billing_mode = COALESCE(?, billing_mode),
system_prompt = NULL,
system_prompt_hash = NULL
WHERE id = ?""",
(provider, base_url, billing_mode, session_id),
)
self._delete_unreferenced_system_prompts(conn)
self._execute_write(_do)
def queue_token_counts(self, session_id: str, **kwargs) -> None:
"""Enqueue a token/cost delta for the background writer.
Same kwargs and semantics as :meth:`update_token_counts`, applied
asynchronously; cheap enough for the turn thread. After close() has
stopped the writer, falls back to the synchronous path and may raise.
"""
with self._token_queue_cond:
thread = self._token_writer_thread
writer_stopped = self._token_writer_stop and (
thread is None or not thread.is_alive()
)
if not writer_stopped:
self._token_queue.append((session_id, kwargs))
if thread is None or not thread.is_alive():
# Daemon so exit never hangs on accounting; the atexit hook
# (registered once per instance) drains leftovers. Checking
# ``not is_alive()`` rather than ``is None`` respawns a writer
# that died from an unexpected escape, otherwise deltas
# would pile up until a reader's flush drained them.
thread = threading.Thread(
target=self._token_writer_loop,
name="session-db-token-writer",
daemon=True,
)
self._token_writer_thread = thread
thread.start()
if self._token_atexit_hook is None:
self_ref = weakref.ref(self)
def _drain_at_exit() -> None:
db = self_ref()
if db is not None:
db._drain_token_queue_at_exit()
self._token_atexit_hook = _drain_at_exit
atexit.register(_drain_at_exit)
self._token_queue_cond.notify_all()
if writer_stopped:
# close() ran (a stop-flagged but live writer still accepts; its
# loop drains before exiting). Enqueueing now would drop the delta
# silently — no writer, atexit hook gone — so apply inline and let a
# closed-connection failure raise at the call site.
self.update_token_counts(session_id, **kwargs)
def flush_token_counts(self, timeout: float = 5.0) -> bool:
"""Block until every queued token delta has been applied.
False on timeout (callers then read totals stale by the queued deltas).
Never raises: apply failures are logged by the writer.
"""
# Lock-free fast path: reads queue-then-busy (see ordering notes below).
if not self._token_queue and not self._token_writer_busy:
return True
batch = None
with self._token_queue_cond:
deadline = time.monotonic() + timeout
while self._token_queue or self._token_writer_busy:
# A live writer is authoritative even when stop-flagged: draining
# here would race its in-flight batch, and newer deltas committing
# before older ones breaks last-non-None-wins / first-accounted-
# route / COALESCE-backfill fields. Only a dead writer lets the
# caller take leftovers; re-checked each wakeup because the writer
# can exit mid-wait with deltas enqueued after its final check.
# busy is claimed while draining so a concurrent flush cannot
# report drained or pop a newer delta while this batch is
# unapplied: a claimed busy means "wait", never "drain alongside".
thread = self._token_writer_thread
if (
(thread is None or not thread.is_alive())
and not self._token_writer_busy
):
self._token_writer_busy = True
batch = list(self._token_queue)
self._token_queue.clear()
break
remaining = deadline - time.monotonic()
if remaining <= 0:
return False
self._token_queue_cond.wait(remaining)
if batch:
try:
self._apply_token_batch(batch)
finally:
with self._token_queue_cond:
self._token_writer_busy = False
self._token_queue_cond.notify_all()
return True
def _token_writer_loop(self) -> None:
while True:
with self._token_queue_cond:
idle_deadline = time.monotonic() + self._TOKEN_WRITER_IDLE_SECONDS
while not self._token_queue and not self._token_writer_stop:
remaining = idle_deadline - time.monotonic()
if remaining <= 0:
# Retire under the same lock queue_token_counts() uses to
# decide to spawn, so no delta strands behind an exiting worker.
self._token_writer_thread = None
return
self._token_queue_cond.wait(remaining)
if not self._token_queue:
self._token_writer_thread = None
return # stop requested and fully drained
# busy BEFORE clearing the queue: flush's lock-free fast path
# reads queue-then-busy and must never see "empty and idle"
# while a popped batch is unapplied.
self._token_writer_busy = True
batch = list(self._token_queue)
self._token_queue.clear()
try:
self._apply_token_batch(batch)
finally:
with self._token_queue_cond:
self._token_writer_busy = False
self._token_queue_cond.notify_all()
def _apply_token_batch(self, batch: List[Tuple[str, Dict[str, Any]]]) -> None:
"""Apply queued deltas in order, coalescing where safe. Never raises."""
try:
coalesced = self._coalesce_token_deltas(batch)
except Exception as exc:
# Coalescing must never kill the writer (callers cannot observe a
# dead one); the merge is only an optimization.
logger.warning(
"async token accounting: coalesce failed, applying raw "
"batch: %s", exc,
)
coalesced = batch
for session_id, kwargs in coalesced:
try:
self.update_token_counts(session_id, **kwargs)
except Exception as exc:
# Accounting loss is logged, never raised into a turn.
logger.warning(
"async token accounting: apply failed (session=%s): %s",
session_id, exc,
)
def _coalesce_token_deltas(
self, batch: List[Tuple[str, Dict[str, Any]]]
) -> List[Tuple[str, Dict[str, Any]]]:
"""Merge adjacent incremental deltas with an identical route, so
ordering across sessions and /model switches is preserved exactly.
absolute=True deltas never merge."""
groups: List[Tuple[Optional[tuple], str, Dict[str, Any]]] = []
for session_id, kwargs in batch:
key = None
if not kwargs.get("absolute"):
key = (session_id,) + tuple(
kwargs.get(f) for f in self._TOKEN_DELTA_ROUTE_FIELDS
)
if groups and key is not None and groups[-1][0] == key:
merged = groups[-1][2]
for f in self._TOKEN_DELTA_SUM_FIELDS:
merged[f] = merged.get(f, 0) + kwargs.get(f, 0)
for f in self._TOKEN_DELTA_COST_FIELDS:
value = kwargs.get(f)
if value is not None:
# All-None runs stay None so COALESCE keeps the stored value.
merged[f] = (merged.get(f) or 0.0) + value
else:
groups.append((key, session_id, dict(kwargs)))
return [(sid, kw) for _, sid, kw in groups]
def _stop_token_writer(self, join_timeout: float = 10.0) -> None:
"""Stop the writer thread and drain remaining deltas. Never raises."""
with self._token_queue_cond:
self._token_writer_stop = True
self._token_queue_cond.notify_all()
thread = self._token_writer_thread
if thread is not None and thread.is_alive():
thread.join(timeout=join_timeout)
if thread.is_alive():
# Writer stuck mid-apply: leave deltas unapplied rather than
# race it and misorder/double-count.
logger.warning(
"async token accounting: writer did not stop within %.0fs; "
"%d queued delta(s) not persisted",
join_timeout, len(self._token_queue),
)
return
# Writer gone: apply leftovers synchronously under the same busy
# protocol. Wait out a flush caller-drain that already claimed busy —
# close() nulls the connection right after this returns and must not
# yank it mid-batch.
with self._token_queue_cond:
deadline = time.monotonic() + join_timeout
while self._token_writer_busy:
remaining = deadline - time.monotonic()
if remaining <= 0:
logger.warning(
"async token accounting: concurrent drain did not "
"finish within %.0fs; %d queued delta(s) not persisted",
join_timeout, len(self._token_queue),
)
return
self._token_queue_cond.wait(remaining)
# busy BEFORE clearing the queue (same ordering as the writer loop),
# or flush's lock-free fast path could see "empty and idle".
batch = list(self._token_queue)
if batch:
self._token_writer_busy = True
self._token_queue.clear()
if batch:
try:
self._apply_token_batch(batch)
finally:
with self._token_queue_cond:
self._token_writer_busy = False
self._token_queue_cond.notify_all()
def _drain_token_queue_at_exit(self) -> None:
try:
self._stop_token_writer()
except Exception:
pass # never fatal at interpreter shutdown
def update_token_counts(
self,
session_id: str,
input_tokens: int = 0,
output_tokens: int = 0,
model: str = None,
cache_read_tokens: int = 0,
cache_write_tokens: int = 0,
reasoning_tokens: int = 0,
estimated_cost_usd: Optional[float] = None,
actual_cost_usd: Optional[float] = None,
cost_status: Optional[str] = None,
cost_source: Optional[str] = None,
pricing_version: Optional[str] = None,
billing_provider: Optional[str] = None,
billing_base_url: Optional[str] = None,
billing_mode: Optional[str] = None,
api_call_count: int = 0,
absolute: bool = False,
) -> None:
"""Update token counters and backfill model if unset.
*absolute*=False increments (per-API-call deltas, CLI path);
*absolute*=True sets directly (gateway path, where the cached agent
holds cumulative totals).
"""
# Ensure the row exists: under concurrent load the initial
# create_session() may have failed on SQLite locking, and the UPDATE
# would silently affect 0 rows.
self._insert_session_row(session_id, "unknown", model=model)
if absolute:
sql = """UPDATE sessions SET
input_tokens = ?,
output_tokens = ?,
cache_read_tokens = ?,
cache_write_tokens = ?,
reasoning_tokens = ?,
estimated_cost_usd = COALESCE(?, 0),
actual_cost_usd = CASE
WHEN ? IS NULL THEN actual_cost_usd
ELSE ?
END,
cost_status = COALESCE(?, cost_status),
cost_source = COALESCE(?, cost_source),
pricing_version = COALESCE(?, pricing_version),
billing_provider = COALESCE(billing_provider, ?),
billing_base_url = COALESCE(billing_base_url, ?),
billing_mode = COALESCE(billing_mode, ?),
model = COALESCE(model, ?),
api_call_count = ?
WHERE id = ?"""
else:
sql = """UPDATE sessions SET
input_tokens = input_tokens + ?,
output_tokens = output_tokens + ?,
cache_read_tokens = cache_read_tokens + ?,
cache_write_tokens = cache_write_tokens + ?,
reasoning_tokens = reasoning_tokens + ?,
estimated_cost_usd = COALESCE(estimated_cost_usd, 0) + COALESCE(?, 0),
actual_cost_usd = CASE
WHEN ? IS NULL THEN actual_cost_usd
ELSE COALESCE(actual_cost_usd, 0) + ?
END,
cost_status = COALESCE(?, cost_status),
cost_source = COALESCE(?, cost_source),
pricing_version = COALESCE(?, pricing_version),
billing_provider = COALESCE(billing_provider, ?),
billing_base_url = COALESCE(billing_base_url, ?),
billing_mode = COALESCE(billing_mode, ?),
model = COALESCE(model, ?),
api_call_count = COALESCE(api_call_count, 0) + ?
WHERE id = ?"""
has_accounted_usage = bool(
input_tokens or output_tokens or cache_read_tokens
or cache_write_tokens or reasoning_tokens or api_call_count
or estimated_cost_usd or actual_cost_usd
)
params = (
input_tokens,
output_tokens,
cache_read_tokens,
cache_write_tokens,
reasoning_tokens,
estimated_cost_usd,
actual_cost_usd,
actual_cost_usd,
cost_status,
cost_source,
pricing_version,
billing_provider if has_accounted_usage else None,
billing_base_url if has_accounted_usage else None,
billing_mode if has_accounted_usage else None,
model if has_accounted_usage else None,
api_call_count,
session_id,
)
# Per-model attribution: the sessions row keeps one (model, provider)
# pair, so a mid-session /model switch would attribute every token to
# the initial model. Each delta carries the route active at call time
# and is recorded into session_model_usage keyed by it. Only the
# incremental path records here: absolute cumulative updates cannot be
# split back into routes; Insights reconciles the residual instead.
record_model_usage = (not absolute) and (
input_tokens or output_tokens or cache_read_tokens
or cache_write_tokens or reasoning_tokens or api_call_count
or estimated_cost_usd
)
def _do(conn):
row = conn.execute(
"SELECT model, billing_provider, api_call_count FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
existing_model = row["model"] if row is not None else None
existing_provider = row["billing_provider"] if row is not None else None
existing_api_calls = int((row["api_call_count"] if row is not None else 0) or 0)
# create_session records the requested route before any API call.
# If that fails and fallback succeeds, the first accounted usage is
# the authoritative route; after that keep the row as is (one row
# cannot represent mixed-provider usage).
first_accounted_route = (
existing_api_calls == 0
and has_accounted_usage
and bool(model)
and bool(billing_provider)
and (existing_model != model or existing_provider != billing_provider)
)
if first_accounted_route:
conn.execute(
"""UPDATE sessions
SET model = ?, billing_provider = ?,
billing_base_url = ?, billing_mode = ?
WHERE id = ?""",
(model, billing_provider, billing_base_url, billing_mode, session_id),
)
conn.execute(sql, params)
if record_model_usage:
self._record_model_usage(
conn,
session_id,
model=model,
billing_provider=billing_provider,
billing_base_url=billing_base_url,
billing_mode=billing_mode,
input_tokens=input_tokens,
output_tokens=output_tokens,
cache_read_tokens=cache_read_tokens,
cache_write_tokens=cache_write_tokens,
reasoning_tokens=reasoning_tokens,
estimated_cost_usd=estimated_cost_usd,
actual_cost_usd=actual_cost_usd,
cost_status=cost_status,
cost_source=cost_source,
api_call_count=api_call_count,
)
self._execute_write(_do)
def _record_model_usage(
self,
conn,
session_id: str,
*,
model: Optional[str],
billing_provider: Optional[str],
billing_base_url: Optional[str],
billing_mode: Optional[str],
input_tokens: int,
output_tokens: int,
cache_read_tokens: int,
cache_write_tokens: int,
reasoning_tokens: int,
estimated_cost_usd: Optional[float],
actual_cost_usd: Optional[float],
cost_status: Optional[str],
cost_source: Optional[str],
api_call_count: int,
task: str = "",
) -> None:
"""Accumulate a per-API-call usage delta into session_model_usage.
Runs inside the caller's write transaction, after the ``sessions``
UPDATE, so per-model rows stay consistent with the summary row. A
missing model/provider falls back to the session row (same COALESCE
behaviour as the summary update). ``task`` is ``''`` for the main loop;
auxiliary calls record their task name via :meth:`record_auxiliary_usage`.
"""
row = conn.execute(
"SELECT model, billing_provider, billing_base_url, billing_mode "
"FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
sess_model = row["model"] if row is not None else None
sess_provider = row["billing_provider"] if row is not None else None
sess_base_url = row["billing_base_url"] if row is not None else None
sess_billing_mode = row["billing_mode"] if row is not None else None
# Aux rows must NOT inherit the main-loop route (vision on gemini while
# the main loop runs anthropic); missing info stays 'unknown'/empty.
if task:
eff_model = model or "unknown"
eff_provider = billing_provider or ""
eff_base_url = billing_base_url or ""
eff_billing_mode = billing_mode or ""
else:
eff_model = model or sess_model or "unknown"
eff_provider = billing_provider or sess_provider or ""
eff_base_url = billing_base_url or sess_base_url or ""
eff_billing_mode = billing_mode or sess_billing_mode or ""
now = time.time()
conn.execute(
"""INSERT INTO session_model_usage (
session_id, model, billing_provider, billing_base_url, billing_mode,
task, api_call_count, input_tokens, output_tokens,
cache_read_tokens, cache_write_tokens, reasoning_tokens,
estimated_cost_usd, actual_cost_usd, cost_status, cost_source,
first_seen, last_seen
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(session_id, model, billing_provider, billing_base_url, billing_mode, task)
DO UPDATE SET
api_call_count = api_call_count + excluded.api_call_count,
input_tokens = input_tokens + excluded.input_tokens,
output_tokens = output_tokens + excluded.output_tokens,
cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens,
cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
reasoning_tokens = reasoning_tokens + excluded.reasoning_tokens,
estimated_cost_usd = estimated_cost_usd + excluded.estimated_cost_usd,
actual_cost_usd = actual_cost_usd + excluded.actual_cost_usd,
cost_status = COALESCE(excluded.cost_status, cost_status),
cost_source = COALESCE(excluded.cost_source, cost_source),
last_seen = excluded.last_seen""",
(
session_id,
eff_model,
eff_provider,
eff_base_url,
eff_billing_mode,
task or "",
api_call_count or 0,
input_tokens or 0,
output_tokens or 0,
cache_read_tokens or 0,
cache_write_tokens or 0,
reasoning_tokens or 0,
float(estimated_cost_usd or 0.0),
float(actual_cost_usd or 0.0),
cost_status,
cost_source,
now,
now,
),
)
def record_auxiliary_usage(
self,
session_id: str,
task: str,
*,
model: Optional[str] = None,
billing_provider: Optional[str] = None,
billing_base_url: Optional[str] = None,
input_tokens: int = 0,
output_tokens: int = 0,
cache_read_tokens: int = 0,
cache_write_tokens: int = 0,
reasoning_tokens: int = 0,
estimated_cost_usd: Optional[float] = None,
api_call_count: int = 1,
) -> None:
"""Record an auxiliary LLM call's usage (vision, compression, title
generation, ...) against *session_id*.
Writes a per-(model, provider, task) delta into ``session_model_usage``
WITHOUT touching the ``sessions`` summary row: the gateway overwrites
session counters with absolute main-loop totals, so aux tokens there
would be clobbered or double-counted. Insights read the union.
``api_call_count`` may aggregate N calls (background-review forks).
Best-effort: callers must never fail an aux call over accounting.
"""
if not session_id or not task:
return
# FK to sessions.id: same INSERT OR IGNORE guard as update_token_counts.
self._insert_session_row(session_id, "unknown")
def _do(conn):
self._record_model_usage(
conn,
session_id,
model=model,
billing_provider=billing_provider,
billing_base_url=billing_base_url,
billing_mode=None,
input_tokens=input_tokens or 0,
output_tokens=output_tokens or 0,
cache_read_tokens=cache_read_tokens or 0,
cache_write_tokens=cache_write_tokens or 0,
reasoning_tokens=reasoning_tokens or 0,
estimated_cost_usd=estimated_cost_usd,
actual_cost_usd=None,
cost_status=None,
cost_source=None,
api_call_count=(
1 if api_call_count is None else int(api_call_count)
),
task=task,
)
self._execute_write(_do)
def usage_totals(self, *, min_message_count: int = 1, include_archived: bool = False) -> Dict[str, float]:
"""Tokens and spend across the whole store (one scan), so the sidebar
total does not shrink with paging. Spend prefers the billed figure over
the estimate, the same precedence a single row renders."""
where = ["parent_session_id IS NULL", "message_count >= ?"]
params: List[Any] = [min_message_count]
if not include_archived:
where.append("COALESCE(archived, 0) = 0")
row = self._read_one(
f"""
SELECT COALESCE(SUM(COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0)), 0),
COALESCE(SUM(COALESCE(actual_cost_usd, estimated_cost_usd, 0)), 0)
FROM sessions
WHERE {' AND '.join(where)}
""",
params,
)
return {"tokens": int(row[0] or 0), "cost_usd": float(row[1] or 0.0)}
+821
View File
@@ -0,0 +1,821 @@
"""SQLite journal-mode and PRAGMA policy for state.db.
Split out of ``hermes_state.py``. Every name is re-imported there so
``hermes_state.<name>`` keeps resolving, and tests that monkeypatch it keep
intercepting because intra-module calls to patched helpers go through a lazy
``from hermes_state import ...`` at call time.
"""
from __future__ import annotations
import logging
import sqlite3
import sys
import threading
import time
from typing import Any, Dict, Optional
from hermes_cli.sqlite_runtime import (
is_sqlite_wal_reset_vulnerable as _is_sqlite_wal_reset_vulnerable,
)
# Log-record parity with the origin module (caplog tests pin "hermes_state").
logger = logging.getLogger("hermes_state")
# ---------------------------------------------------------------------------
# WAL-compatibility fallback
# ---------------------------------------------------------------------------
# WAL needs mmap shared memory and fcntl byte-range locks, which network
# filesystems (NFS, SMB/CIFS, some FUSE, WSL1) don't provide reliably — there
# ``PRAGMA journal_mode=WAL`` raises ``locking protocol`` (SQLITE_PROTOCOL).
# ZFS instead corrupts the -shm file under concurrent connection bursts (COW +
# mmap), presenting as ``disk I/O error``. Propagating either would silently
# break everything backed by state.db/kanban.db, so we fall back to
# ``journal_mode=DELETE`` (pre-WAL default, works on NFS/ZFS): readers block
# during a write, but it works. The WAL-reset-bug gate and the
# never-live-downgrade invariant are documented on apply_wal_with_fallback.
_WAL_INCOMPAT_MARKERS = (
"locking protocol", # SQLITE_PROTOCOL on NFS/SMB
"not authorized", # Some FUSE mounts block WAL pragma outright
"disk i/o error", # ZFS SHM corruption under concurrent connections
)
# SQLite's default is -1 (unlimited), so state.db-wal would keep the high-water
# mark of the largest-ever transaction forever. See _apply_wal_size_limit().
_WAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024 # 64 MiB
# Dedup sets: kanban_db.connect() runs on every kanban operation, so an
# undeduped fallback log line would repeat per connection and fill errors.log.
_wal_fallback_warned_paths: set[str] = set()
_wal_fallback_warned_lock = threading.Lock()
_wal_reset_bug_warned_paths: set[str] = set()
_wal_reset_bug_warned_lock = threading.Lock()
# "configured delete overridden by on-disk WAL" ERROR.
_delete_overridden_warned_paths: set[str] = set()
_delete_overridden_warned_lock = threading.Lock()
def _on_disk_journal_mode(conn: sqlite3.Connection) -> Optional[str]:
"""Read the journal mode from the DB header; ``None`` if undeterminable.
``None`` (new DB, or PRAGMA failed) sends callers down their fail-closed
"unknown → refuse to downgrade" branch. ``disk i/o error`` can be transient
on virtualized block devices (XFS on cloud hosts), so it is retried a few
times first: transient EIO clears, deterministic filesystem errors do not.
"""
last_exc: Optional[Exception] = None
for _ in range(4):
try:
row = conn.execute("PRAGMA journal_mode").fetchone()
except sqlite3.OperationalError as exc:
last_exc = exc
if "disk i/o error" not in str(exc).lower():
return None
time.sleep(0.05)
continue
if row is None:
return None
mode = row[0]
if isinstance(mode, bytes): # defensive: sqlite3 occasionally returns bytes
try:
mode = mode.decode("ascii")
except UnicodeDecodeError:
return None
return str(mode).strip().lower() if mode is not None else None
if last_exc is not None:
logger.debug(
"_on_disk_journal_mode: retries exhausted on disk read (%s)", last_exc
)
return None
def _apply_wal_size_limit(conn: sqlite3.Connection) -> None:
"""Bound the WAL so it returns space to the OS after big transactions.
SQLite's default ``journal_size_limit`` is -1: a checkpointed WAL is reused
in place, never truncated, so ``state.db-wal`` keeps the high-water mark
of the largest transaction ever run. One bulk op strands gigabytes —
``hermes sessions optimize`` on a 3 GB state.db left a 3 GB WAL and filled
the disk, so the maintenance command was self-defeating on the largest
DBs. With a limit, each checkpoint truncates the WAL back to it; 64 MiB is
above normal transaction sizes (steady-state commits never pay a truncate)
while capping slack predictably. kanban_db uses ``wal_autocheckpoint=100``.
Best-effort: never raises — failure only costs disk slack and must not
prevent the database from opening.
"""
try:
conn.execute(f"PRAGMA journal_size_limit={_WAL_SIZE_LIMIT_BYTES}")
except sqlite3.OperationalError as exc: # pragma: no cover - defensive
logger.debug("journal_size_limit not applied: %s", exc)
def _apply_macos_checkpoint_barrier(conn: sqlite3.Connection) -> None:
"""Enable ``PRAGMA checkpoint_fullfsync`` on macOS (no-op elsewhere).
Apple's ``fsync(2)`` guarantees neither data-on-platter nor write ordering,
so WAL's corruption-safety assumption fails on Darwin without ``F_FULLFSYNC``.
A launchd shutdown drops the page cache (power-loss for in-flight pages), so
a checkpoint that "reported" durable can leave a malformed ``state.db``;
a plain in-session kill survives via the page cache. The barrier applies
only at checkpoint boundaries (~+0.1 ms/commit vs ~+4 ms for
``fullfsync=1``). Best-effort: never raises.
"""
if sys.platform != "darwin":
return
try:
conn.execute("PRAGMA checkpoint_fullfsync=1")
except sqlite3.OperationalError:
pass
def _enforce_macos_synchronous_full(conn: sqlite3.Connection) -> None:
"""Enforce ``PRAGMA synchronous=FULL`` on macOS to prevent btree corruption.
With NORMAL, a WAL checkpoint racing process termination (launchd shutdown)
can leave half-written btree pages (``btreeInitPage error 11``) because
Darwin's ``fsync()`` guarantees neither ordering nor durability. Called
after every successful WAL activation so a prior connection's NORMAL never
sticks. Best-effort: never raises.
"""
if sys.platform != "darwin":
return
try:
conn.execute("PRAGMA synchronous=FULL")
except sqlite3.OperationalError:
pass
def is_sqlite_wal_reset_vulnerable(
version_info: Optional[tuple] = None,
) -> bool:
"""True when the linked SQLite has the WAL-reset bug (3.7.0–3.51.2;
fixed 3.51.3+, backports 3.50.7 / 3.44.6). Pre-WAL libraries are safe.
https://sqlite.org/wal.html#walresetbug
"""
info = version_info if version_info is not None else sqlite3.sqlite_version_info
return _is_sqlite_wal_reset_vulnerable(info)
def sqlite_source_id() -> str:
"""Return ``sqlite_source_id()``, or an empty string when unavailable."""
try:
conn = sqlite3.connect(":memory:")
try:
row = conn.execute("SELECT sqlite_source_id()").fetchone()
finally:
conn.close()
except sqlite3.Error:
return ""
if not row or row[0] is None:
return ""
return str(row[0])
def _database_has_content(conn: sqlite3.Connection) -> bool:
"""Whether the file already holds pages (existing vs brand-new DB).
``PRAGMA page_count`` is a lock-free header read. Fail-quiet: any error
answers False, because the only caller gates a warning on this and an
unknown-answer warning would fire on every fresh database — exactly where
there is provably no operator choice being overwritten.
"""
try:
row = conn.execute("PRAGMA page_count").fetchone()
except sqlite3.Error:
return False
if not row or row[0] is None:
return False
try:
return int(row[0]) > 0
except (TypeError, ValueError):
return False
def resolve_journal_mode() -> str:
"""Return the configured journal mode (``wal`` or ``delete``).
``database.journal_mode`` in config.yaml is the canonical operator setting;
``wal`` is the default, ``delete`` is for filesystems without WAL-safe
durability (macOS virtiofs, NFS, SMB). Invalid values fail safe to ``wal``.
"""
try:
from hermes_cli.config import load_config_readonly
config = load_config_readonly() or {}
database = config.get("database", {})
if not isinstance(database, dict):
return "wal"
raw = database.get("journal_mode", "wal")
except Exception:
return "wal"
if not isinstance(raw, str):
return "wal"
mode = raw.strip().lower()
return mode if mode in ("wal", "delete") else "wal"
class WalUnsupportedError(sqlite3.OperationalError):
"""Raised by :func:`apply_wal_with_fallback` when ``require_wal=True`` and
the filesystem cannot provide WAL — whether SQLite *raised*
``SQLITE_PROTOCOL`` or (macOS NFS) silently returned the still-effective
mode. Subclasses ``OperationalError`` so existing DB-init handlers still
catch it while WAL-mandating callers can catch the narrower type.
"""
def apply_wal_with_fallback(
conn: sqlite3.Connection,
*,
db_label: str = "state.db",
require_wal: bool = False,
) -> str:
"""Set ``journal_mode=WAL`` on ``conn``, falling back to DELETE on failure.
Returns the mode actually set (``"wal"`` or ``"delete"``). Shared by
:class:`SessionDB` and ``hermes_cli.kanban_db.connect`` for identical
fallback behavior.
On WAL-incompatible filesystems (NFS, SMB, some FUSE, ZFS) SQLite either
raises ``OperationalError`` ("locking protocol" / "disk I/O error") or —
macOS NFS / SMB / AgentFS NFS overlay — silently refuses and leaves the DB
in DELETE. Either way we log at ERROR (a write now blocks readers — a real
concurrency loss) and fall back to DELETE so the feature keeps working.
``require_wal=True`` raises :class:`WalUnsupportedError` instead; all
current callers keep the default so NFS-homed installs work.
On SQLite builds with the WAL-reset bug (https://sqlite.org/wal.html#walresetbug,
fixed 3.51.3+, backports 3.50.7 / 3.44.6), refuse to enable WAL on
fresh / non-WAL databases; an already-WAL DB keeps WAL with a warning.
This gate is deliberately RETAINED: an attempt to revert it (theory: DELETE
was "the mode that corrupts") was confounded — its clean WAL result came
from SQLite 3.53.1, which also carries 3.51.0's close()-broken-POSIX-lock
defenses. Re-measured on the bundled 3.50.4 with the lock fix, WAL and
DELETE are both clean, so there is no evidence WAL is safer; keep new
databases out of WAL until a fixed runtime ships.
Invariant on every path (NFS and WAL-reset alike): never downgrade to
DELETE if the on-disk header reports WAL or the mode cannot be read (see
_on_disk_journal_mode). Other gateway/cron/worker connections may hold the
DB open, and a live downgrade destroys their committed-but-uncheckpointed
transactions.
The ERROR is deduplicated per ``db_label``: once per process per DB, so
state.db and kanban.db on one NFS mount each log once.
"""
from hermes_state import is_sqlite_wal_reset_vulnerable, resolve_journal_mode
configured = resolve_journal_mode()
# Vulnerable SQLite: never enable WAL on new/non-WAL files. Resolve the
# operator setting first so an explicit DELETE request still verifies SQLite
# accepted DELETE rather than silently returning MEMORY or another mode.
if is_sqlite_wal_reset_vulnerable():
return _apply_delete_for_wal_reset_bug(
conn,
db_label=db_label,
require_delete=configured == "delete",
)
# Read-only probe — no flock, no checkpoint, no WAL/SHM unlink — so
# WAL-init cannot unlink files other connections hold open.
current_mode = _on_disk_journal_mode(conn)
if current_mode == "wal":
if configured == "delete":
# Never-live-downgrade keeps WAL; tell the operator their delete did not apply.
_log_configured_delete_overridden_once(db_label)
_apply_wal_size_limit(conn)
_apply_macos_checkpoint_barrier(conn)
_enforce_macos_synchronous_full(conn)
return "wal"
# Honor the canonical database.journal_mode setting (on-disk WAL DBs were
# returned above and are never live-downgraded).
if configured == "delete":
if current_mode is None:
# Probe failed (locked/busy): another process may hold this DB open
# in WAL, so ownership is not provably exclusive and flipping modes
# could destroy a concurrent writer's committed-but-uncheckpointed
# transactions. Fail loudly — the operator asked for DELETE and we
# cannot verify it.
raise sqlite3.OperationalError(
"could not verify journal mode before applying configured "
"journal_mode=delete (database is locked — possible "
"concurrent openers); refusing to downgrade a database "
"this process does not exclusively own"
)
actual = _set_journal_mode_no_wait(conn, "DELETE")
if actual != "delete":
raise sqlite3.OperationalError(
f"could not set configured journal_mode=delete (got {actual or 'no result'})"
)
return actual
# Decide BEFORE the flip whether it would overwrite a mode somebody chose:
# the probe and page_count are only readable while the file is untouched.
# A 0-page DB has no prior choice, and every caller reaches this before
# creating schema, so brand-new databases stay quiet.
_upgrading_existing_db = (
current_mode is not None
and current_mode != "wal"
and _database_has_content(conn)
)
try:
# ``PRAGMA journal_mode=WAL`` RETURNS the resulting mode. Filesystems
# that refuse by *raising* SQLITE_PROTOCOL hit the except branch, but
# macOS NFS, SMB/CIFS and the AgentFS NFS overlay refuse WITHOUT raising
# and just return the still-effective mode. Trust the row, not the
# absence of an exception, or we report a false "wal", skip the
# fallback ERROR, and leave the DB silently in DELETE.
row = conn.execute("PRAGMA journal_mode=WAL").fetchone()
mode = str(row[0]).strip().lower() if row and row[0] is not None else ""
if mode == "wal":
if _upgrading_existing_db:
_log_journal_mode_upgrade_once(db_label, current_mode)
_apply_wal_size_limit(conn)
_apply_macos_checkpoint_barrier(conn)
_enforce_macos_synchronous_full(conn)
return "wal"
# Silent refusal: WAL was not honored, but nothing raised.
silent_exc = WalUnsupportedError(
f"journal_mode=WAL refused without raising (still {mode!r})"
)
if require_wal:
raise silent_exc
_log_wal_fallback_once(db_label, silent_exc)
return mode or "delete"
except sqlite3.OperationalError as exc:
# The require_wal silent-refusal raise above lands here (subclass of
# OperationalError) — propagate unchanged, skip the marker logic.
if isinstance(exc, WalUnsupportedError):
raise
msg = str(exc).lower()
if not any(marker in msg for marker in _WAL_INCOMPAT_MARKERS):
# Unrelated OperationalError — don't silently swallow.
raise
# ``disk i/o error`` is ambiguous: deterministic WAL-incompatibility on
# ZFS / APFS-CoW (SHM corruption under connection bursts), or a one-shot
# transient EIO (page-cache pressure, brief lock contention). Treating
# a transient EIO as a permanent downgrade signal produced mixed-mode
# corruption (process A downgrades to DELETE while siblings set WAL),
# so retry the pragma: transient EIO clears and we return "wal";
# deterministic cases keep failing into the guarded DELETE fallback.
if "disk i/o error" in msg:
for _ in range(2):
time.sleep(0.05)
try:
row = conn.execute("PRAGMA journal_mode=WAL").fetchone()
except sqlite3.OperationalError as retry_exc:
if "disk i/o error" not in str(retry_exc).lower():
raise
exc = retry_exc
continue
mode = (
str(row[0]).strip().lower()
if row and row[0] is not None
else ""
)
if mode == "wal":
# Transient EIO cleared and the switch went through; same
# header rewrite, so same upgrade signal.
if _upgrading_existing_db:
_log_journal_mode_upgrade_once(db_label, current_mode)
_apply_wal_size_limit(conn)
_apply_macos_checkpoint_barrier(conn)
_enforce_macos_synchronous_full(conn)
return "wal"
break
# Don't downgrade if another process already set WAL on disk, or if the
# mode cannot be read (probe blocked by a concurrent opener's locks) —
# ownership is not provably exclusive either way.
existing = _on_disk_journal_mode(conn)
if existing == "wal" or existing is None:
raise
if require_wal:
raise WalUnsupportedError(str(exc)) from exc
_log_wal_fallback_once(db_label, exc)
_set_journal_mode_no_wait(conn, "DELETE")
return "delete"
def _set_journal_mode_no_wait(conn: sqlite3.Connection, mode: str) -> str:
"""Execute ``PRAGMA journal_mode=<mode>`` without waiting on other openers.
The ONLY place a journal-mode switch may be issued for a non-WAL target.
Forces ``busy_timeout=0`` so SQLite's exclusivity requirement becomes a
concurrent-opener detector: leaving WAL needs exclusive access, so if ANY
other connection (this process or another) holds the DB the pragma fails
immediately with ``database is locked`` instead of waiting out a busy
timeout and sneaking the flip between a concurrent writer's transactions —
exactly how committed-but-uncheckpointed WAL transactions get destroyed.
Callers must treat a raised ``OperationalError`` as "not exclusively
owned: leave the journal mode alone", never as retryable. Returns SQLite's
reported mode (lowercase), or ``""`` if no row.
"""
previous_timeout = 0
try:
row = conn.execute("PRAGMA busy_timeout").fetchone()
if row and row[0] is not None:
previous_timeout = int(row[0])
except (sqlite3.OperationalError, TypeError, ValueError):
previous_timeout = 0
conn.execute("PRAGMA busy_timeout=0")
try:
row = conn.execute(f"PRAGMA journal_mode={mode}").fetchone()
return str(row[0]).strip().lower() if row and row[0] is not None else ""
finally:
try:
conn.execute(f"PRAGMA busy_timeout={previous_timeout}")
except sqlite3.OperationalError:
pass
def _apply_delete_for_wal_reset_bug(
conn: sqlite3.Connection,
*,
db_label: str,
require_delete: bool = False,
) -> str:
"""Avoid enabling WAL when the linked SQLite has the WAL-reset bug.
- Already-WAL on disk: leave WAL alone (no live downgrade) and warn.
- Mode unreadable (probe blocked by a concurrent opener's locks): not
provably exclusive — leave the mode alone and warn. Never treat "could
not read the mode" as "not WAL": that confusion once flipped a live WAL
state.db to DELETE under a concurrent writer, destroying its
committed-but-uncheckpointed transactions.
- Otherwise: set DELETE (refusing to wait out concurrent openers) and warn.
- For an explicit operator request, verify SQLite accepted DELETE.
"""
current = _on_disk_journal_mode(conn)
if current == "wal":
_log_wal_reset_bug_once(db_label, kept_wal=True)
if require_delete:
# Upgrading SQLite (the warning above) doesn't help on a
# WAL-incompatible filesystem; emit the actionable message last.
_log_configured_delete_overridden_once(db_label)
# No TRUNCATE / journal_mode=DELETE while other processes may still
# hold this WAL DB open; same safety rule as the NFS path.
_apply_wal_size_limit(conn)
_apply_macos_checkpoint_barrier(conn)
_enforce_macos_synchronous_full(conn)
return "wal"
if current is None:
# Probe failed — likely another opener's locks, and the DB may be in
# WAL under a live writer. Never flip a mode we cannot even read.
if require_delete:
raise sqlite3.OperationalError(
"could not verify journal mode before applying configured "
"journal_mode=delete (database is locked — possible "
"concurrent openers); refusing to downgrade a database "
"this process does not exclusively own"
)
_log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True)
return "wal"
actual = ""
try:
actual = _set_journal_mode_no_wait(conn, "DELETE")
except sqlite3.OperationalError as exc:
if require_delete:
raise
lowered = str(exc).lower()
if "locked" in lowered or "busy" in lowered:
# A concurrent opener appeared between probe and flip (or already
# held the DB): SQLite refused the exclusive lock. Leave the mode as is.
_log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True)
return current or "delete"
# Best-effort for the automatic fallback: DELETE is normally already
# the default for new file-backed databases.
if require_delete and actual != "delete":
raise sqlite3.OperationalError(
"could not set configured journal_mode=delete "
f"(got {actual or 'no result'})"
)
_log_wal_reset_bug_once(db_label, kept_wal=False)
return "delete"
def _wal_reset_repair_hint() -> str:
"""Repair hint matching what ``hermes update`` can actually do for this
install type (uv-managed venv vs git/pip/docker/nix)."""
try:
from hermes_cli.config import (
detect_install_method,
recommended_update_command_for_method,
get_project_root,
)
method = detect_install_method(get_project_root())
cmd = recommended_update_command_for_method(method)
if method in {"git", "unknown"}:
return f"Hermes-managed installs can repair the embedded runtime with `{cmd}`"
if method == "docker":
return f"update the container image with `{cmd}`"
# nix/nixos
return cmd
except Exception:
pass
return (
"install a Python build bundled with SQLite 3.51.3+ "
"(or backports 3.50.7 / 3.44.6) and restart Hermes"
)
# Dedup state for _log_journal_mode_upgrade_once.
_journal_upgrade_warned_paths: set = set()
_journal_upgrade_warned_lock = threading.Lock()
def _log_wal_reset_bug_once(
db_label: str,
*,
kept_wal: bool,
indeterminate: bool = False,
) -> None:
"""Log once per (process, db_label) about the WAL-reset vulnerability path."""
from hermes_state import _wal_reset_bug_warned_paths
with _wal_reset_bug_warned_lock:
if db_label in _wal_reset_bug_warned_paths:
return
_wal_reset_bug_warned_paths.add(db_label)
if indeterminate:
action = (
"journal mode could not be verified or exclusively switched "
"(database is locked — possible concurrent openers); leaving the "
"journal mode untouched (no live downgrade under concurrent "
"openers)"
)
elif kept_wal:
action = (
"is already in WAL mode — leaving WAL in place (no live "
"downgrade under concurrent openers)"
)
else:
action = "using journal_mode=DELETE instead of enabling WAL"
# Install-type-aware so the warning never promises a repair path that
# doesn't exist for git/pip/system Python installs.
repair_hint = _wal_reset_repair_hint()
logger.warning(
"%s: linked SQLite %s (interpreter %s) is vulnerable to the WAL-reset "
"corruption bug (https://sqlite.org/wal.html#walresetbug) — %s. "
"Upgrade to SQLite 3.51.3+ (or backports 3.50.7 / 3.44.6); "
"%s. See `hermes doctor`. This warning fires once per "
"process per database.",
db_label,
sqlite3.sqlite_version,
sys.executable,
action,
repair_hint,
)
def _log_journal_mode_upgrade_once(db_label: str, previous_mode: str) -> None:
"""Log a single WARNING per (process, db_label) about a non-WAL -> WAL flip.
``PRAGMA journal_mode`` is a property of the FILE: switching an existing DB
to WAL rewrites its header and outlives the process. Operators do set
DELETE on the file directly (the documented WAL-reset-bug mitigation), and
nothing told them the next open would silently put WAL back.
WARNING, not ERROR: the reverse move is ERROR in ``_log_wal_fallback_once``
because dropping to DELETE loses concurrency, whereas this direction is
normally desirable (managed_uv repairs DELETE-stuck DBs on update). The
only problem was invisibility, so this names the durable setting without
claiming a degradation. Deduped per process per ``db_label`` because
kanban opens a fresh connection per operation.
"""
from hermes_state import _journal_upgrade_warned_paths
with _journal_upgrade_warned_lock:
if db_label in _journal_upgrade_warned_paths:
return
_journal_upgrade_warned_paths.add(db_label)
logger.warning(
"%s: on-disk journal_mode was %s and has been switched to WAL. This "
"rewrites the database header and persists after this process exits. "
"If %s was a deliberate choice (for example the mitigation for the "
"SQLite WAL-reset bug, or a WAL-unsafe filesystem), setting it with "
"PRAGMA on the file will not survive -- every open re-applies the "
"configured mode. Set `database.journal_mode: delete` in config.yaml "
"to make it stick. This message fires once per process per database.",
db_label,
previous_mode,
previous_mode,
)
def _log_wal_fallback_once(db_label: str, exc: Exception) -> None:
"""Log a single ERROR per (process, db_label) about WAL fallback.
ERROR, not WARNING: silently dropping to DELETE is a real concurrency loss
(under kanban dispatcher + workers a write blocks readers as SQLITE_BUSY).
Deduped because kanban opens a fresh connection per operation.
"""
from hermes_state import _wal_fallback_warned_paths
with _wal_fallback_warned_lock:
if db_label in _wal_fallback_warned_paths:
return
_wal_fallback_warned_paths.add(db_label)
logger.error(
"%s: WAL journal_mode unsupported on this filesystem (%s) — "
"falling back to journal_mode=DELETE (slower rollback-journal "
"mode; reduces concurrency but works on NFS/SMB/FUSE/ZFS). See "
"https://www.sqlite.org/wal.html for details. This message "
"fires once per process per database.",
db_label,
exc,
)
def _log_configured_delete_overridden_once(db_label: str) -> None:
"""Log a single ERROR per (process, db_label) when the operator configured
``journal_mode=delete`` but the on-disk DB is already WAL.
Never-live-downgrade keeps WAL (a live downgrade causes mixed-mode
corruption); without this the operator would never learn that
``database.journal_mode: delete`` had no effect and that a one-time
offline ``PRAGMA journal_mode=DELETE`` (no open connections) is required.
"""
from hermes_state import _delete_overridden_warned_paths
with _delete_overridden_warned_lock:
if db_label in _delete_overridden_warned_paths:
return
_delete_overridden_warned_paths.add(db_label)
logger.error(
"%s: database.journal_mode=delete is configured but the on-disk "
"database is already WAL; keeping WAL (a live downgrade under open "
"connections can corrupt the DB). To apply journal_mode=DELETE, stop "
"all connections to this DB and run a one-time offline "
"'PRAGMA journal_mode=DELETE' on the file. This message fires once "
"per process per database.",
db_label,
)
# ---------------------------------------------------------------------------
# Config-driven database pragmas
# ---------------------------------------------------------------------------
# Operators write synchronous as a name; mapped here rather than passed through
# so a typo becomes a warning instead of a silently different durability level.
_SYNCHRONOUS_LEVELS: Dict[str, int] = {
"OFF": 0,
"NORMAL": 1,
"FULL": 2,
"EXTRA": 3,
}
_SYNCHRONOUS_NAMES: Dict[int, str] = {v: k for k, v in _SYNCHRONOUS_LEVELS.items()}
_SYNCHRONOUS_FULL = 2
def resolve_synchronous_level(raw_value: Any) -> Optional[int]:
"""Map a configured ``database.synchronous`` value to its PRAGMA integer.
Accepts SQLite's names (``OFF``/``NORMAL``/``FULL``/``EXTRA``, any case) or
``0``-``3``. Anything else returns None so the caller warns and leaves the
level untouched — guessing at a malformed durability setting is worse.
"""
if isinstance(raw_value, bool):
# bool is an int subclass and YAML turns bare `on`/`off` into one.
# "off" is a real durability choice; True is meaningless.
return 0 if raw_value is False else None
if isinstance(raw_value, int):
return raw_value if raw_value in _SYNCHRONOUS_NAMES else None
text = str(raw_value).strip()
if not text:
return None
upper = text.upper()
if upper in _SYNCHRONOUS_LEVELS:
return _SYNCHRONOUS_LEVELS[upper]
try:
value = int(text)
except (TypeError, ValueError):
return None
return value if value in _SYNCHRONOUS_NAMES else None
def _apply_synchronous_pragma(
conn: sqlite3.Connection,
raw_value: Any,
*,
db_label: str,
) -> None:
"""Set ``PRAGMA synchronous`` from config, never below FULL on macOS.
Kept out of the integer loop in :func:`apply_database_pragmas`: this PRAGMA
decides whether a commit is on the platter, so an unrecognised value must
not fall through to "SQLite default" the way a bad ``cache_size`` can.
Darwin floor: :func:`_enforce_macos_synchronous_full` runs during
``apply_wal_with_fallback()`` and this runs after it, so a configured
``NORMAL`` would otherwise silently undo the macOS btree protection.
Raising the level on macOS is allowed; lowering it is refused out loud.
"""
level = resolve_synchronous_level(raw_value)
if level is None:
logger.warning(
"%s: ignoring unrecognized database.synchronous=%r "
"(expected OFF, NORMAL, FULL, EXTRA, or 0-3)",
db_label,
raw_value,
)
return
if sys.platform == "darwin" and level < _SYNCHRONOUS_FULL:
logger.warning(
"%s: refusing database.synchronous=%s on macOS; keeping FULL. "
"Darwin's fsync() does not guarantee write ordering, so a lower "
"level readmits the half-written btree pages FULL exists to "
"prevent.",
db_label,
_SYNCHRONOUS_NAMES[level],
)
return
try:
conn.execute(f"PRAGMA synchronous={level}")
except sqlite3.OperationalError:
pass
def apply_database_pragmas(
conn: sqlite3.Connection,
*,
db_label: str = "state.db",
) -> None:
"""Apply optional performance and WAL-sizing PRAGMAs from ``config.yaml``.
Journal mode is NOT handled here — ``database.journal_mode`` is owned by
:func:`resolve_journal_mode` inside :func:`apply_wal_with_fallback`, under
all the safety guards.
Keys under ``database:``: ``cache_size`` (negative = KiB, positive =
pages), ``mmap_size`` (bytes, 0 = disabled), ``temp_store`` (0-3),
``wal_autocheckpoint`` (pages), ``journal_size_limit`` (bytes), and
``synchronous`` (``OFF``/``NORMAL``/``FULL``/``EXTRA`` or ``0``-``3``).
Unset ``synchronous`` leaves SQLite's default, a *compile-time* constant
(``SQLITE_DEFAULT_WAL_SYNCHRONOUS``) that differs between bundled, distro
and Homebrew builds; setting it explicitly is the only way to know.
Best-effort: config load or pragma failures are ignored so DB init never
breaks on a malformed ``database:`` section.
"""
try:
# Local import avoids a circular import with hermes_cli.config.
from hermes_cli.config import cfg_get, load_config_readonly
cfg = load_config_readonly()
except Exception:
return
# Applied to ALL connection types: writer, read_only, WAL per-thread readers.
for pragma_name in (
"cache_size",
"mmap_size",
"temp_store",
"wal_autocheckpoint",
"journal_size_limit",
):
raw_value = cfg_get(cfg, "database", pragma_name, default=None)
if raw_value is None:
continue
try:
value = int(str(raw_value).strip())
except (TypeError, ValueError):
logger.warning(
"%s: ignoring non-integer database.%s=%r",
db_label,
pragma_name,
raw_value,
)
continue
try:
conn.execute(f"PRAGMA {pragma_name}={value}")
except sqlite3.OperationalError:
pass
# Last: the sizing pragmas above cannot change durability, and the macOS
# enforcement ran earlier during WAL activation (see _apply_synchronous_pragma
# for why that ordering needs an explicit floor rather than an override).
raw_synchronous = cfg_get(cfg, "database", "synchronous", default=None)
if raw_synchronous is not None:
_apply_synchronous_pragma(conn, raw_synchronous, db_label=db_label)
+6 -2
View File
@@ -102,8 +102,12 @@ def test_repair_path_has_no_bare_connects() -> None:
Source-level guard: the bare form is exactly what regressed, and a unit
test on the helper alone would not notice a sixth site being added.
"""
source = Path(hermes_state.__file__).read_text(encoding="utf-8")
tree = ast.parse(source, filename=str(hermes_state.__file__))
# The repair/probe helpers live in hermes_state_repair; hermes_state only
# re-imports them.
import hermes_state_repair
source = Path(hermes_state_repair.__file__).read_text(encoding="utf-8")
tree = ast.parse(source, filename=str(hermes_state_repair.__file__))
def is_db_path_connect(node: ast.AST) -> bool:
if not isinstance(node, ast.Call):