diff --git a/hermes_cli/active_sessions.py b/hermes_cli/active_sessions.py index 4f21e6cd9a..015b8a636f 100644 --- a/hermes_cli/active_sessions.py +++ b/hermes_cli/active_sessions.py @@ -107,11 +107,8 @@ def active_session_limit_message( max_sessions: int, entries: Optional[list[dict[str, Any]]] = None, ) -> str: - # Name the holders: the slots are shared across CLI, desktop/TUI and the - # messaging gateway, so the surface that gets rejected is usually NOT the - # one squatting on them (idle desktop chats starving a Discord bot, say). - # Without this the message is unactionable and the only way to find out is - # reading runtime/active_sessions.json by hand. + # Name the holders: slots are shared across CLI, desktop/TUI and gateway, + # so the rejected surface is usually NOT the one squatting on them. held = summarize_holders(entries or []) detail = f" Held by: {held}." if held else "" return ( @@ -124,42 +121,25 @@ def _registry_home(registry_home: str | Path | None = None) -> Path: return Path(registry_home) if registry_home is not None else Path(get_hermes_home()) -# WHY A REFUSAL IS REFUSED, in a form a caller can branch on. -# -# The two refusals mean opposite things to an automated client. Capacity is -# "the machine is busy, come back later". Ownership is "this specific session -# has a live owner, and writing to it would interleave with theirs". -# -# Callers used to have only the human-readable message, so anything that needed -# to DECIDE had to match prose -- which silently changes meaning whenever the -# wording is improved. The reason is the contract; the message is for people. +# Machine-readable refusal reasons: the reason is the contract, the message is +# for people. Capacity = "busy, come back later"; ownership = "this session +# has a live owner and writing would interleave with theirs". SESSION_NOT_OWNED = "SESSION_NOT_OWNED" MAX_CONCURRENT_SESSIONS = "MAX_CONCURRENT_SESSIONS" -# Ownership could not be PROVEN either way: the registry was unreadable or -# corrupt. Distinct from SESSION_NOT_OWNED on purpose -- "someone else owns -# this" and "I cannot tell who owns this" call for different operator action, -# and collapsing the second into a silent go-ahead is exactly the fail-open -# hole that let two writers share one session (#94595 review, blocker 2). +# Ownership could not be PROVEN (registry unreadable/corrupt). Deliberately +# distinct from SESSION_NOT_OWNED: collapsing "can't tell" into a silent +# go-ahead is the fail-open hole that let two writers share one session. SESSION_COORDINATION_UNAVAILABLE = "SESSION_COORDINATION_UNAVAILABLE" -# Advertised through the gateway so a client can tell a build that enforces -# per-session exclusivity from one that does not. -# -# A module constant rather than a config flag, deliberately: it is true because -# try_acquire_active_session below performs the check atomically, so it cannot -# be turned on by an operator who has not got the enforcement, and cannot drift -# out of step with it without this file changing. +# Advertised through the gateway. A module constant, not a config flag: it is +# true because try_acquire_active_session checks atomically, so it cannot drift +# out of step with the enforcement without this file changing. PER_SESSION_EXCLUSIVE_SUBMIT = True class ActiveSessionRefusal(str): - """A refusal message that also carries a machine-readable ``reason``. - - A ``str`` subclass so every existing caller keeps working untouched -- they - format it, hand it back as a JSON-RPC message, or just test it for None -- - while a caller that must act on WHICH refusal happened reads ``.reason`` - instead of matching the wording. - """ + """A refusal message (``str`` subclass, so existing callers are untouched) + that also carries a machine-readable ``reason``.""" reason: str @@ -170,13 +150,10 @@ class ActiveSessionRefusal(str): def _is_same_writer(entry: dict[str, Any], metadata: Optional[dict[str, Any]]) -> bool: - """True when an existing lease belongs to the very caller now re-acquiring it. - - Both halves are required. A pid alone would let two live sessions in one - process steal each other's lease -- which is a real hazard, since each holds - its own snapshot of the transcript. A live session id alone would let another - process with a coincidentally equal id do the same. - """ + """True when an existing lease belongs to the very caller re-acquiring it. + Identity is (pid, live_session_id): pid alone would let two live sessions in + one process steal each other's lease; the live id alone would let another + process with an equal id do the same.""" try: if int(entry.get("pid") or -1) != os.getpid(): return False @@ -223,6 +200,19 @@ def _lease_paths( return _state_path(home), _lock_path(home) +def _flock(fh, *, lock: bool) -> None: + """Exclusive whole-file lock/unlock on ``fh`` (fcntl on POSIX, msvcrt on Windows).""" + if os.name == "nt": + import msvcrt + + fh.seek(0) + msvcrt.locking(fh.fileno(), msvcrt.LK_LOCK if lock else msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(fh.fileno(), fcntl.LOCK_EX if lock else fcntl.LOCK_UN) + + class _FileLock: def __init__(self, path: Path): self.path = path @@ -231,45 +221,21 @@ class _FileLock: def __enter__(self): self.path.parent.mkdir(parents=True, exist_ok=True) self._fh = open(self.path, "a+b") - if os.name == "nt": - try: - import msvcrt - - self._fh.seek(0) - msvcrt.locking(self._fh.fileno(), msvcrt.LK_LOCK, 1) - except Exception as exc: - self._fh.close() - self._fh = None - raise RuntimeError("active session file lock unavailable") from exc - else: - try: - import fcntl - - fcntl.flock(self._fh.fileno(), fcntl.LOCK_EX) - except Exception as exc: - self._fh.close() - self._fh = None - raise RuntimeError("active session file lock unavailable") from exc + try: + _flock(self._fh, lock=True) + except Exception as exc: + self._fh.close() + self._fh = None + raise RuntimeError("active session file lock unavailable") from exc return self def __exit__(self, exc_type, exc, tb): if self._fh is None: return - if os.name == "nt": - try: - import msvcrt - - self._fh.seek(0) - msvcrt.locking(self._fh.fileno(), msvcrt.LK_UNLCK, 1) - except Exception: - pass - else: - try: - import fcntl - - fcntl.flock(self._fh.fileno(), fcntl.LOCK_UN) - except Exception: - pass + try: + _flock(self._fh, lock=False) + except Exception: + pass try: self._fh.close() finally: @@ -306,58 +272,51 @@ def _read_entries(path: Path, *, strict: bool = False) -> list[dict[str, Any]]: seen_leases: set[str] = set() for entry in valid: lease_id = entry.get("lease_id") - session_id = entry.get("session_id") - pid = entry.get("pid") - if not isinstance(lease_id, str) or not lease_id.strip(): - raise ActiveSessionRegistryError( - f"active session registry contains an invalid lease id: {path}" - ) - if lease_id in seen_leases: - raise ActiveSessionRegistryError( - f"active session registry contains a duplicate lease id: {path}" - ) - seen_leases.add(lease_id) - if not isinstance(session_id, str) or not session_id.strip(): - raise ActiveSessionRegistryError( - f"active session registry contains an invalid session id: {path}" - ) - if isinstance(pid, bool) or not isinstance(pid, (int, str)): - pid_int = 0 - else: - try: - pid_int = int(pid) - except (TypeError, ValueError): - pid_int = 0 - if pid_int <= 0: - raise ActiveSessionRegistryError( - f"active session registry contains an invalid pid: {path}" - ) - surface = entry.get("surface") - if surface is not None and not isinstance(surface, str): - raise ActiveSessionRegistryError( - f"active session registry contains an invalid surface: {path}" - ) - tracked = entry.get("track_liveness") - if tracked is not None and not isinstance(tracked, bool): - raise ActiveSessionRegistryError( - f"active session registry contains an invalid liveness marker: {path}" - ) - metadata = entry.get("metadata") - if metadata is not None and not isinstance(metadata, dict): - raise ActiveSessionRegistryError( - f"active session registry contains invalid metadata: {path}" - ) - process_start = entry.get("process_start_time") - parsed_process_start = _optional_float(process_start) - if process_start not in (None, "") and ( - parsed_process_start is None or not math.isfinite(parsed_process_start) + # (problem-if-True predicate, message fragment) — checked lazily, in + # this order, so an unhashable lease id is reported before the dup check. + for bad, what in ( + (lambda: not _nonblank_str(lease_id), "an invalid lease id"), + (lambda: lease_id in seen_leases, "a duplicate lease id"), + (lambda: not _nonblank_str(entry.get("session_id")), "an invalid session id"), + (lambda: _registry_pid(entry.get("pid")) <= 0, "an invalid pid"), + (lambda: not _optional_isinstance(entry.get("surface"), str), "an invalid surface"), + (lambda: not _optional_isinstance(entry.get("track_liveness"), bool), "an invalid liveness marker"), + (lambda: not _optional_isinstance(entry.get("metadata"), dict), "invalid metadata"), + (lambda: not _valid_process_start(entry.get("process_start_time")), "an invalid process start time"), ): - raise ActiveSessionRegistryError( - f"active session registry contains an invalid process start time: {path}" - ) + if bad(): + raise ActiveSessionRegistryError( + f"active session registry contains {what}: {path}" + ) + seen_leases.add(lease_id) return valid +def _nonblank_str(v: Any) -> bool: + return isinstance(v, str) and bool(v.strip()) + + +def _optional_isinstance(v: Any, typ) -> bool: + return v is None or isinstance(v, typ) + + +def _registry_pid(pid: Any) -> int: + """Registry pid as int; 0 for bools, non-int/str, or unparseable values.""" + if isinstance(pid, bool) or not isinstance(pid, (int, str)): + return 0 + try: + return int(pid) + except (TypeError, ValueError): + return 0 + + +def _valid_process_start(v: Any) -> bool: + if v in (None, ""): + return True + parsed = _optional_float(v) + return parsed is not None and math.isfinite(parsed) + + def _write_entries(path: Path, entries: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(f"{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") @@ -392,20 +351,24 @@ def _optional_float(value: Any) -> Optional[float]: return None -def _pid_liveness(pid: Any, process_start_time: Any = None) -> Optional[bool]: - """Return True/False for live/dead, or None when liveness is unknowable.""" +def _pid_liveness(pid: Any, process_start_time: Any = None, *, lenient: bool = False) -> Optional[bool]: + """Return True/False for live/dead, or None when liveness is unknowable. + + ``lenient`` never returns None: an unparseable pid or a failed existence + probe counts as dead, an unreadable current start time as alive. + """ try: pid_int = int(pid) except (TypeError, ValueError): - return None + return False if lenient else None if pid_int <= 0: - return None + return False if lenient else None try: from gateway.status import _pid_exists exists = bool(_pid_exists(pid_int)) except Exception: - return None + return False if lenient else None if not exists: return False expected_start = _optional_float(process_start_time) @@ -413,32 +376,12 @@ def _pid_liveness(pid: Any, process_start_time: Any = None) -> Optional[bool]: return True current_start = _process_start_time(pid_int) if current_start is None: - return None + return True if lenient else None return abs(current_start - expected_start) < 0.001 def _pid_alive(pid: Any, process_start_time: Any = None) -> bool: - try: - pid_int = int(pid) - except (TypeError, ValueError): - return False - if pid_int <= 0: - return False - try: - from gateway.status import _pid_exists - - exists = bool(_pid_exists(pid_int)) - except Exception: - return False - if not exists: - return False - expected_start = _optional_float(process_start_time) - if expected_start is None: - return True - current_start = _process_start_time(pid_int) - if current_start is None: - return True - return abs(current_start - expected_start) < 0.001 + return bool(_pid_liveness(pid, process_start_time, lenient=True)) def _prune_dead( @@ -468,12 +411,9 @@ class ActiveSessionLease: surface: str enabled: bool = True released: bool = False - # Registry paths pinned at acquisition time. A lease acquired under the - # root ``HERMES_HOME`` must release against the same registry even when - # ``release()`` runs inside a profile home override (native multiplex - # routes turns under ``_profile_runtime_scope``), otherwise the root - # entry survives until process exit and the session cap fills with - # phantom leases (#85431). + # Pinned at acquisition: a lease acquired under the root HERMES_HOME must + # release against the same registry even when release() runs inside a + # profile-home override, or phantom leases fill the session cap. state_path: Optional[Path] = None lock_path: Optional[Path] = None track_liveness: bool = False @@ -484,6 +424,33 @@ class ActiveSessionLease: release_active_session(self) +def _clean_metadata(metadata: dict[str, Any]) -> dict[str, Any]: + return {str(k): v for k, v in metadata.items() if isinstance(k, str)} + + +def _without_lease(entries: list[dict[str, Any]], lease_id: str) -> list[dict[str, Any]]: + return [e for e in entries if str(e.get("lease_id") or "") != lease_id] + + +def _read_live_entries( + state_path: Path, *, track_liveness: bool, warn: str, +) -> Optional[tuple[list[dict[str, Any]], list[dict[str, Any]]]]: + """``(raw, pruned)`` from the registry, or None when it is unreadable. + + Liveness-tracked callers re-raise instead: they must not proceed on an + unprovable registry. Untracked callers get ``warn`` logged and decide + how to degrade themselves. + """ + try: + raw_entries = _read_entries(state_path, strict=True) + return raw_entries, _prune_dead(raw_entries, strict=track_liveness) + except ActiveSessionRegistryError: + if track_liveness: + raise + logger.warning(warn) + return None + + def _lease_entry( *, lease_id: str, @@ -505,9 +472,7 @@ def _lease_entry( if track_liveness: entry["track_liveness"] = True if metadata: - entry["metadata"] = { - str(k): v for k, v in metadata.items() if isinstance(k, str) - } + entry["metadata"] = _clean_metadata(metadata) return entry @@ -522,27 +487,20 @@ def try_acquire_active_session( ) -> tuple[Optional[ActiveSessionLease], Optional[str]]: """Acquire an active-session slot. - Per-session exclusivity is CORRECTNESS and is enforced unconditionally: - at most one live owner may run a given stored session, whether or not an - operator configured ``max_concurrent_sessions`` (#94595). The concurrency - cap remains a resource POLICY and applies only when configured. Liveness - tracking keeps richer desktop lifecycle semantics; ``registry_home`` lets - profile-scoped backends share the owning profile's registry even when - launched from another home. + Per-session exclusivity is CORRECTNESS, enforced unconditionally: at most + one live owner per stored session. ``max_concurrent_sessions`` is resource + POLICY and applies only when configured. ``registry_home`` lets + profile-scoped backends share the owning profile's registry. - Returns ``(lease, None)`` on success and ``(None, refusal)`` otherwise, - where ``refusal`` is an :class:`ActiveSessionRefusal` carrying a - machine-readable ``reason``. Ownership uncertainty fails CLOSED: when the - registry cannot be read, the caller gets ``SESSION_COORDINATION_UNAVAILABLE`` - rather than a silent go-ahead that could reopen the double-writer state. + Returns ``(lease, None)`` or ``(None, ActiveSessionRefusal)``. Ownership + uncertainty fails CLOSED with ``SESSION_COORDINATION_UNAVAILABLE``. """ max_sessions = resolve_max_concurrent_sessions(config) lease_id = uuid.uuid4().hex key = str(session_id or "") - # A session with no stored id yet cannot collide with another writer, and - # the strict registry schema (rightly) refuses entries with empty session - # ids. Nothing to fence, nothing to record: hand back a no-op lease. + # No stored id yet => nothing to fence or record (and the strict schema + # refuses empty session ids): hand back a no-op lease. if not key and not track_liveness: return ActiveSessionLease( lease_id=lease_id, @@ -560,22 +518,23 @@ def try_acquire_active_session( ) state_path, lock_path = _lease_paths(registry_home=registry_home) + lease = ActiveSessionLease( + lease_id=lease_id, + session_id=key, + surface=str(surface), + state_path=state_path, + lock_path=lock_path, + track_liveness=track_liveness, + ) with _FileLock(lock_path): - try: - raw_entries = _read_entries(state_path, strict=True) - entries = _prune_dead(raw_entries, strict=track_liveness) - except ActiveSessionRegistryError: - if track_liveness: - raise - # A capacity cap could afford to degrade open -- worst case, more - # sessions than the operator wanted. Exclusivity cannot: "could not - # prove ownership" must never be collapsed into "no owner exists", - # because that silently reopens the exact double-writer state this - # fence guarantees against. Refuse, and say which file to fix. - logger.warning( - "Active-session registry is unavailable; refusing the session " - "rather than risking a concurrent writer" - ) + # A capacity cap could degrade open; exclusivity cannot: "could not + # prove ownership" must never become "no owner exists". + loaded = _read_live_entries( + state_path, track_liveness=track_liveness, + warn="Active-session registry is unavailable; refusing the session " + "rather than risking a concurrent writer", + ) + if loaded is None: return None, ActiveSessionRefusal( ( "Hermes could not read the active-session registry at " @@ -584,47 +543,27 @@ def try_acquire_active_session( ), SESSION_COORDINATION_UNAVAILABLE, ) + raw_entries, entries = loaded pruned = len(raw_entries) - len(entries) if pruned: logger.info("Pruned %d stale active session lease(s)", pruned) - # Correctness first, and under the same lock that just pruned the dead - # owners -- so an owner that died is never mistaken for one that is - # running, and a live one is never overlooked. - # - # An empty key is exempt: a session with no stored id yet cannot collide - # with another, and treating "" as an identity would make every unsaved - # draft exclude every other one. + # Correctness first, under the same lock that just pruned dead owners. + # An empty key is exempt: treating "" as an identity would make every + # unsaved draft exclude every other one. if key: for index, existing in enumerate(entries): if str(existing.get("session_id") or "") != key: continue - # THE SAME WRITER IS NOT A SECOND WRITER. - # - # A live session that lost its lease reference -- its record was - # rebuilt in place, so the object holding the lease is unreachable - # while the session itself is still the one being driven -- would - # otherwise be fenced out of its own session by its own leak, and - # permanently: pruning only removes entries whose PROCESS is dead, - # and this process is very much alive. - # - # Identity here is (pid, live session id). Two processes never - # match, because their pids differ. Two live sessions inside one - # process never match, because their live ids differ. Only the - # exact same writer re-acquiring its own session matches, and that - # is re-entrancy rather than a concurrent writer. + # The same writer is not a second writer: a live session that + # leaked its lease reference would otherwise be fenced out of + # its own session permanently (pruning only removes entries + # whose PROCESS is dead). Re-entrancy, not concurrency. if _is_same_writer(existing, metadata): entries[index] = entry _write_entries(state_path, entries) - return ActiveSessionLease( - lease_id=lease_id, - session_id=key, - surface=str(surface), - state_path=state_path, - lock_path=lock_path, - track_liveness=track_liveness, - ), None + return lease, None _write_entries(state_path, entries) logger.info( @@ -656,14 +595,7 @@ def try_acquire_active_session( entries.append(entry) _write_entries(state_path, entries) - return ActiveSessionLease( - lease_id=lease_id, - session_id=key, - surface=str(surface), - state_path=state_path, - lock_path=lock_path, - track_liveness=track_liveness, - ), None + return lease, None def release_active_session(lease: ActiveSessionLease) -> None: @@ -673,25 +605,16 @@ def release_active_session(lease: ActiveSessionLease) -> None: with _FileLock(lock_path): if lease.released: return - try: - raw_entries = _read_entries(state_path, strict=True) - entries = _prune_dead(raw_entries, strict=lease.track_liveness) - except ActiveSessionRegistryError: - if lease.track_liveness: - raise - logger.warning( - "Active-session registry is unavailable; preserving it while " - "releasing an untracked lease" - ) - lease.released = True - return - kept = [ - entry - for entry in entries - if str(entry.get("lease_id") or "") != lease.lease_id - ] - if len(kept) != len(entries): - _write_entries(state_path, kept) + loaded = _read_live_entries( + state_path, track_liveness=lease.track_liveness, + warn="Active-session registry is unavailable; preserving it while " + "releasing an untracked lease", + ) + if loaded is not None: + entries = loaded[1] + kept = _without_lease(entries, lease.lease_id) + if len(kept) != len(entries): + _write_entries(state_path, kept) lease.released = True @@ -717,17 +640,14 @@ def transfer_active_session( # thread acquired the file lock. Never resurrect a durably removed lease. if lease.released: return False - try: - raw_entries = _read_entries(state_path, strict=True) - entries = _prune_dead(raw_entries, strict=lease.track_liveness) - except ActiveSessionRegistryError: - if lease.track_liveness: - raise - logger.warning( - "Active-session registry is unavailable; refusing to overwrite " - "it during lease transfer" - ) + loaded = _read_live_entries( + state_path, track_liveness=lease.track_liveness, + warn="Active-session registry is unavailable; refusing to overwrite " + "it during lease transfer", + ) + if loaded is None: return False + entries = loaded[1] updated = False for entry in entries: if str(entry.get("lease_id") or "") != lease.lease_id: @@ -735,9 +655,7 @@ def transfer_active_session( entry["session_id"] = new_session_id entry["updated_at"] = time.time() if metadata: - entry["metadata"] = { - str(k): v for k, v in metadata.items() if isinstance(k, str) - } + entry["metadata"] = _clean_metadata(metadata) updated = True break if not updated and lease.track_liveness: @@ -760,12 +678,10 @@ def transfer_active_session( def release_orphaned_leases(live_lease_ids: set[str]) -> int: """Drop this process's registry entries that no live session owns. - ``_prune_dead`` only reclaims leases whose owning process died. A server - that runs for days (``hermes dashboard`` / ``serve``) never trips that - check, so a lease whose session skipped teardown is held until restart. - The owning process is the only authority on which of its own leases are - real, so it drops the rest itself — exact, with no heartbeat write on the - turn path and no staleness threshold to tune. + ``_prune_dead`` only reclaims leases of dead processes; a days-long server + never trips it, so a lease whose session skipped teardown is held until + restart. The owning process is the only authority on its own leases — + exact, no heartbeat on the turn path, no staleness threshold. """ pid = os.getpid() state_path = _state_path() @@ -774,14 +690,13 @@ def release_orphaned_leases(live_lease_ids: set[str]) -> int: if not state_path.exists(): return 0 with _FileLock(_lock_path()): - try: - raw_entries = _read_entries(state_path, strict=True) - entries = _prune_dead(raw_entries) - except ActiveSessionRegistryError: - logger.warning( - "Active-session registry is unavailable; skipping orphaned-lease sweep" - ) + loaded = _read_live_entries( + state_path, track_liveness=False, + warn="Active-session registry is unavailable; skipping orphaned-lease sweep", + ) + if loaded is None: return 0 + entries = loaded[1] kept = [ entry for entry in entries @@ -813,12 +728,9 @@ def active_session_liveness_guard( *, registry_home: str | Path | None = None, ) -> Iterator[bool]: - """Hold the registry lock while reporting whether ``session_id`` is leased. - - Keeping the lock across the caller's lifecycle mutation prevents a new - backend from acquiring a lease and reopening the row between the liveness - check and the corresponding ``end_session`` write. - """ + """Hold the registry lock while reporting whether ``session_id`` is leased, + so no new backend can acquire a lease between the check and the caller's + ``end_session`` write.""" target = str(session_id or "") state_path, lock_path = _lease_paths(registry_home=registry_home) with _FileLock(lock_path): @@ -834,12 +746,9 @@ def release_active_session_liveness_guard( lease: ActiveSessionLease, session_id: str, ) -> Iterator[bool]: - """Remove ``lease`` and hold its registry lock through a lifecycle write. - - This makes automatic cleanup one atomic ownership decision: the local - runtime disappears, sibling liveness is checked, and the caller may end the - durable row before any new backend can acquire/reopen it. - """ + """Remove ``lease`` and hold its registry lock through a lifecycle write, + making cleanup one atomic ownership decision (release, check siblings, + end the durable row) before any new backend can reopen it.""" if not lease.enabled or lease.released: with active_session_liveness_guard( session_id, registry_home=_registry_home_for_lease(lease) @@ -850,13 +759,8 @@ def release_active_session_liveness_guard( target = str(session_id or "") state_path, lock_path = _lease_paths(lease) with _FileLock(lock_path): - raw_entries = _read_entries(state_path, strict=True) - entries = _prune_dead(raw_entries, strict=True) - kept = [ - entry - for entry in entries - if str(entry.get("lease_id") or "") != lease.lease_id - ] + entries = _prune_dead(_read_entries(state_path, strict=True), strict=True) + kept = _without_lease(entries, lease.lease_id) if len(kept) != len(entries): _write_entries(state_path, kept) lease.released = True diff --git a/hermes_cli/kanban.py b/hermes_cli/kanban.py index 7037941a46..806403224e 100644 --- a/hermes_cli/kanban.py +++ b/hermes_cli/kanban.py @@ -1,15 +1,8 @@ """CLI for the Hermes Kanban board — ``hermes kanban …`` subcommand. -Exposes the full Kanban command surface documented in the design spec -(``docs/hermes-kanban-v1-spec.pdf``). All DB work is delegated to -``kanban_db``. This module adds: - - * Argparse subcommand construction (``build_parser``). - * Argument dispatch (``kanban_command``). - * Output formatting (plain text + ``--json``). - * A short shared helper that parses a single slash-style string - (used by ``/kanban …`` in CLI and gateway) and forwards it to the - argparse surface. +All DB work is delegated to ``kanban_db``; this module adds argparse +construction (``build_parser``), dispatch (``kanban_command``), text/``--json`` +output, and ``run_slash`` for ``/kanban …`` from the CLI and gateway. """ from __future__ import annotations @@ -49,6 +42,57 @@ def _fmt_ts(ts: Optional[int]) -> str: return time.strftime("%Y-%m-%d %H:%M", time.localtime(ts)) +def _print_json(obj: Any, *, ascii: bool = False) -> None: + print(json.dumps(obj, indent=2, ensure_ascii=ascii)) + + +def _json_out(args: argparse.Namespace, obj: Any, *, ascii: bool = False) -> bool: + """Print ``obj`` as JSON and return True when ``--json`` was passed.""" + if not getattr(args, "json", False): + return False + _print_json(obj, ascii=ascii) + return True + + +def _fmt_counts(counts: dict, empty: str = "") -> str: + return ", ".join(f"{k}={v}" for k, v in sorted(counts.items())) or empty + + +def _err(msg: str, rc: int = 1) -> int: + print(msg, file=sys.stderr) + return rc + + +def _none_profile(value: str) -> Optional[str]: + """``none`` / ``-`` / ``null`` mean "unassign".""" + return None if value.lower() in {"none", "-", "null"} else value + + +def _parse_metadata_flag(raw: Optional[str]) -> tuple[Optional[dict], int]: + """Parse ``--metadata`` JSON; returns ``(dict|None, rc)`` with rc=2 on error.""" + if not raw: + return None, 0 + try: + metadata = json.loads(raw) + if not isinstance(metadata, dict): + raise ValueError("must be a JSON object") + except (ValueError, json.JSONDecodeError) as exc: + return None, _err(f"kanban: --metadata: {exc}", 2) + return metadata, 0 + + +def _bulk_apply(ids, op, ok_msg, fail_msg) -> int: + """Run ``op(tid) -> bool`` per id, print ok/fail lines, exit 1 if any failed.""" + failed = False + for tid in ids: + if not op(tid): + failed = True + print(fail_msg(tid), file=sys.stderr) + else: + print(ok_msg(tid)) + return 1 if failed else 0 + + def _fmt_task_line(t: kb.Task) -> str: icon = _STATUS_ICONS.get(t.status, "?") assignee = t.assignee or "(unassigned)" @@ -56,32 +100,34 @@ def _fmt_task_line(t: kb.Task) -> str: return f"{icon} {t.id} {t.status:8s} {assignee:20s}{tenant} {t.title}" +_TASK_DICT_FIELDS = ( + "id", "title", "body", "assignee", "status", "priority", "tenant", + "workspace_kind", "workspace_path", "branch_name", "project_id", + "created_by", "created_at", "started_at", "completed_at", "result", + "skills", "max_retries", "model_override", "provider_override", + "session_id", "workflow_template_id", "current_step_key", +) +_SHOW_RUN_FIELDS = ( + "id", "profile", "step_key", "status", "outcome", "summary", "error", + "metadata", "worker_pid", "started_at", "ended_at", +) +_RUNS_RUN_FIELDS = ( + "id", "profile", "status", "outcome", "started_at", "ended_at", + "summary", "error", "metadata", "worker_pid", "step_key", +) +_ATTACHMENT_FIELDS = ( + "id", "filename", "content_type", "size", "uploaded_by", "stored_path", "created_at", +) + + +def _obj_dict(obj: Any, fields: tuple[str, ...]) -> dict[str, Any]: + return {k: getattr(obj, k) for k in fields} + + def _task_to_dict(t: kb.Task) -> dict[str, Any]: - return { - "id": t.id, - "title": t.title, - "body": t.body, - "assignee": t.assignee, - "status": t.status, - "priority": t.priority, - "tenant": t.tenant, - "workspace_kind": t.workspace_kind, - "workspace_path": t.workspace_path, - "branch_name": t.branch_name, - "project_id": t.project_id, - "created_by": t.created_by, - "created_at": t.created_at, - "started_at": t.started_at, - "completed_at": t.completed_at, - "result": t.result, - "skills": list(t.skills) if t.skills else [], - "max_retries": t.max_retries, - "model_override": t.model_override, - "provider_override": t.provider_override, - "session_id": t.session_id, - "workflow_template_id": t.workflow_template_id, - "current_step_key": t.current_step_key, - } + d = _obj_dict(t, _TASK_DICT_FIELDS) + d["skills"] = list(t.skills) if t.skills else [] + return d def _run_state_kwargs(args: argparse.Namespace) -> Optional[dict[str, str]]: @@ -136,47 +182,33 @@ def _parse_branch_flag(value: Optional[str]) -> Optional[str]: def _check_dispatcher_presence( hermes_home: Optional[Path] = None, ) -> tuple[bool, str]: - """Return ``(running, message)``. + """Return ``(running, message)`` for the "will anything dispatch this?" warning. - - ``running=True``: a gateway is alive for this HERMES_HOME and its - config has ``kanban.dispatch_in_gateway`` on (default). Message - is a short status line. - - ``running=False``: either no gateway is running, or the gateway - is running but the config flag is off. Message is human guidance - explaining the next step. + ``running=True`` when a gateway is alive for this HERMES_HOME with + ``kanban.dispatch_in_gateway`` on (message is a status line); otherwise + ``False`` with human guidance. Fails OPEN — import/probe/config errors + return ``(True, "")`` — since a missed warning beats crying wolf. - Used by ``hermes kanban create`` (and callers) to warn when a task - will sit in ``ready`` because nothing is there to pick it up. - Defensive against import failures and config-read errors — if the - probe itself errors, we return ``(True, "")`` so we don't spam - false warnings (better to miss a warning than to cry wolf). - - ``hermes_home`` scopes the probe to a named profile's directory. The - dashboard plugin API passes it because the dashboard backend process can - be running under a different HERMES_HOME than the profile the request - targets, which otherwise produced a "no gateway is running" warning - against a perfectly healthy profile gateway (#71211). CLI callers leave - it ``None`` and keep the existing process-level behavior. + ``hermes_home`` scopes the probe to a profile's directory: the dashboard + backend may run under a different HERMES_HOME than the profile it serves, + which otherwise misreports a healthy gateway as absent. CLI callers pass + ``None``. """ try: from gateway.status import resolve_gateway_liveness # type: ignore except Exception: return (True, "") # can't probe — silent try: - # Same shared ladder the dashboard status endpoints use, so a - # PID-file-less (launch-service-managed) or cross-container gateway - # is not misreported as absent. use_cache=False: this is a one-shot - # CLI/create-time probe, not a polling loop, and it must observe the - # gateway's state right now rather than a cached snapshot. + # Same ladder the dashboard status endpoints use, so PID-file-less or + # cross-container gateways aren't misreported. use_cache=False: this + # one-shot probe must see the gateway's state right now. liveness = resolve_gateway_liveness( profile_dir=hermes_home, use_cache=False ) except Exception: return (True, "") # probe errored — silent if liveness.probe_error: - # The resolver swallows per-rung failures so status endpoints never - # 500. This caller must still fail OPEN: an unreadable probe means - # "can't tell", not "no gateway", and warning on it cries wolf. + # The resolver swallows per-rung failures; "can't tell" != "no gateway". return (True, "") pid = liveness.pid @@ -213,11 +245,38 @@ def _check_dispatcher_presence( # Argparse builder # --------------------------------------------------------------------------- -def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.ArgumentParser: - """Attach the ``kanban`` subcommand tree under an existing subparsers. +def _add_run_state_filters(p: argparse.ArgumentParser, type_help: str) -> None: + p.add_argument( + "--state-type", + choices=("status", "outcome"), + default=None, + help=f"With --state-name: {type_help}", + ) + p.add_argument( + "--state-name", + default=None, + metavar="VALUE", + help="With --state-type: keep runs whose column equals this value", + ) - Returns the top-level ``kanban`` parser so caller can ``set_defaults``. - """ + +def _add_triage_sweep_args(p: argparse.ArgumentParser, verb: str, Verb: str, noun: str) -> None: + """Shared ``specify`` / ``decompose`` arguments.""" + p.add_argument("task_id", nargs="?", default=None, + help=f"Task id to {verb} (required unless --all is given)") + p.add_argument("--all", dest="all_triage", action="store_true", + help=f"{Verb} every task currently in the triage column") + p.add_argument("--tenant", default=None, + help="When used with --all, restrict the sweep to this tenant") + p.add_argument("--author", default=None, + help="Author name recorded on the audit comment " + f"(default: $HERMES_PROFILE or '{noun}')") + p.add_argument("--json", action="store_true", + help="Emit one JSON object per task on stdout") + + +def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.ArgumentParser: + """Attach the ``kanban`` subcommand tree; returns the ``kanban`` parser.""" kanban_parser = parent_subparsers.add_parser( "kanban", help="Multi-profile collaboration board (tasks, links, comments)", @@ -229,28 +288,20 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu "or docs/hermes-kanban-v1-spec.pdf for the full design." ), ) - # --- global --board flag --- - # Applies to every subcommand below. When set, scopes all reads and - # writes to that board's DB. When omitted, resolves via the - # HERMES_KANBAN_BOARD env var, then the persisted current-board - # file, then "default". See kanban_db.get_current_board(). - kanban_parser.add_argument( - "--board", - default=None, - metavar="", - help=( - "Board slug to operate on. Defaults to the current board " - "(set via `hermes kanban boards switch ` or the " - "HERMES_KANBAN_BOARD env var). Use `hermes kanban boards list` " - "to see all boards." - ), - ) + # --board scopes every subcommand to one board's DB; when omitted the + # resolution is HERMES_KANBAN_BOARD, then the persisted current-board + # file, then "default" (kanban_db.get_current_board()). + kanban_parser.add_argument("--board", default=None, metavar="", + help="Board slug to operate on. Defaults to the current board (set " + "via `hermes kanban boards switch ` or the " + "HERMES_KANBAN_BOARD env var). Use `hermes kanban boards " + "list` to see all boards.") sub = kanban_parser.add_subparsers(dest="kanban_action") # --- init --- sub.add_parser("init", help="Create kanban.db if missing (idempotent)") - # --- boards (new in v2: multi-project support) --- + # --- boards --- p_boards = sub.add_parser( "boards", help="Manage kanban boards (one board per project / workstream)", @@ -264,24 +315,15 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu ) boards_sub = p_boards.add_subparsers(dest="boards_action") - b_list = boards_sub.add_parser( - "list", aliases=["ls"], - help="List all boards with task counts", - ) + b_list = boards_sub.add_parser("list", aliases=["ls"], help="List all boards with task counts") b_list.add_argument("--json", action="store_true") - b_list.add_argument("--all", action="store_true", - help="Include archived boards too") + b_list.add_argument("--all", action="store_true", help="Include archived boards too") - b_create = boards_sub.add_parser( - "create", aliases=["new"], - help="Create a new board", - ) - b_create.add_argument("slug", - help="Board slug (kebab-case, e.g. atm10-server)") + b_create = boards_sub.add_parser("create", aliases=["new"], help="Create a new board") + b_create.add_argument("slug", help="Board slug (kebab-case, e.g. atm10-server)") b_create.add_argument("--name", default=None, help="Human-readable display name (defaults to Title Case of slug)") - b_create.add_argument("--description", default=None, - help="Optional description") + b_create.add_argument("--description", default=None, help="Optional description") b_create.add_argument("--icon", default=None, help="Optional emoji or single-character icon for the dashboard") b_create.add_argument("--color", default=None, @@ -291,37 +333,27 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu b_create.add_argument("--default-workdir", default=None, help="Default workspace path for tasks created on this board") - b_rm = boards_sub.add_parser( - "rm", aliases=["remove", "delete"], - help="Archive (default) or delete a board", - ) + b_rm = boards_sub.add_parser("rm", aliases=["remove", "delete"], + help="Archive (default) or delete a board") b_rm.add_argument("slug") b_rm.add_argument("--delete", action="store_true", help="Hard-delete the board directory instead of archiving it. " "Default is to move it to boards/_archived/ so it's recoverable.") - b_switch = boards_sub.add_parser( - "switch", aliases=["use"], - help="Set the active board for subsequent CLI calls", - ) + b_switch = boards_sub.add_parser("switch", aliases=["use"], + help="Set the active board for subsequent CLI calls") b_switch.add_argument("slug") - boards_sub.add_parser( - "show", aliases=["current"], - help="Print the currently-active board slug", - ) + boards_sub.add_parser("show", aliases=["current"], help="Print the currently-active board slug") - b_rename = boards_sub.add_parser( - "rename", - help="Change a board's human-readable display name (slug is immutable)", - ) + b_rename = boards_sub.add_parser("rename", + help="Change a board's human-readable display name (slug is " + "immutable)") b_rename.add_argument("slug") b_rename.add_argument("name", help="New display name") - b_set_wd = boards_sub.add_parser( - "set-default-workdir", - help="Set the default workspace path for tasks on a board", - ) + b_set_wd = boards_sub.add_parser("set-default-workdir", + help="Set the default workspace path for tasks on a board") b_set_wd.add_argument("slug") b_set_wd.add_argument("path", nargs="?", default=None, help="Absolute path to use as default workdir. Omit to clear.") @@ -388,17 +420,15 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu help="Dedup key. If a non-archived task with this key exists, " "its id is returned instead of creating a duplicate.") p_create.add_argument("--max-runtime", default=None, - help="Per-task runtime cap. Accepts seconds (300) or " - "durations (90s, 30m, 2h, 1d). When exceeded, " - "the dispatcher SIGTERMs (then SIGKILLs) the worker " - "and re-queues the task.") + help="Per-task runtime cap. Accepts seconds (300) or durations (90s, " + "30m, 2h, 1d). When exceeded, the dispatcher SIGTERMs (then " + "SIGKILLs) the worker and re-queues the task.") p_create.add_argument("--created-by", default="user", help="Author name recorded on the task (default: user)") p_create.add_argument("--skill", action="append", default=[], dest="skills", - help="Skill to force-load into the worker " - "(repeatable). The kanban lifecycle is already " - "injected automatically. Example: " - "--skill translation --skill github-code-review") + help="Skill to force-load into the worker (repeatable). The kanban " + "lifecycle is already injected automatically. Example: --skill " + "translation --skill github-code-review") p_create.add_argument("--max-retries", type=int, default=None, metavar="N", help="Per-task override for the consecutive-failure " @@ -409,24 +439,19 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu "kanban.failure_limit config " f"(default {kb.DEFAULT_FAILURE_LIMIT}).") p_create.add_argument("--model", default=None, dest="model_override", - help="Pin the worker to this model (passed as " - "-m ) without changing the profile's " - "configured model. Combine with --provider " - "when the model belongs to a different " - "backend than the profile's default.") + help="Pin the worker to this model (passed as -m ) without " + "changing the profile's configured model. Combine with --provider " + "when the model belongs to a different backend than the profile's " + "default.") p_create.add_argument("--provider", default=None, dest="provider_override", - help="Provider the --model belongs to (passed as " - "--provider to the worker). Requires " - "--model.") + help="Provider the --model belongs to (passed as --provider to " + "the worker). Requires --model.") p_create.add_argument("--goal", action="store_true", dest="goal_mode", - help="Run the worker in a goal loop: after each " - "turn a judge checks the response against the " - "card title/body and, if not done, the worker " - "keeps going in the same session until the " - "judge agrees it's complete (or the turn " - "budget runs out, which blocks the card for " - "review). Best for open-ended cards one shot " - "rarely finishes.") + help="Run the worker in a goal loop: after each turn a judge checks the " + "response against the card title/body and, if not done, the worker " + "keeps going in the same session until the judge agrees it's " + "complete (or the turn budget runs out, which blocks the card for " + "review). Best for open-ended cards one shot rarely finishes.") p_create.add_argument("--goal-max-turns", type=int, default=None, metavar="N", dest="goal_max_turns", help="Turn budget for --goal workers (default 20). " @@ -440,10 +465,9 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu p_create.add_argument("--json", action="store_true", help="Emit JSON output") # --- swarm --- - p_swarm = sub.add_parser( - "swarm", - help="Create a Kanban Swarm v1 graph (parallel workers → verifier → synthesizer)", - ) + p_swarm = sub.add_parser("swarm", + help="Create a Kanban Swarm v1 graph (parallel workers → verifier → " + "synthesizer)") p_swarm.add_argument("goal", help="Swarm goal / final outcome") p_swarm.add_argument( "--worker", @@ -462,54 +486,27 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu # --- list --- p_list = sub.add_parser("list", aliases=["ls"], help="List tasks") - p_list.add_argument("--mine", action="store_true", - help="Filter by $HERMES_PROFILE as assignee") + p_list.add_argument("--mine", action="store_true", help="Filter by $HERMES_PROFILE as assignee") p_list.add_argument("--assignee", default=None) - p_list.add_argument("--status", default=None, - choices=sorted(kb.VALID_STATUSES)) + p_list.add_argument("--status", default=None, choices=sorted(kb.VALID_STATUSES)) p_list.add_argument("--tenant", default=None) p_list.add_argument("--session", default=None, help="Filter by originating chat/agent session id " "(set on tasks created from inside an ACP loop)") - p_list.add_argument("--archived", action="store_true", - help="Include archived tasks") + p_list.add_argument("--archived", action="store_true", help="Include archived tasks") p_list.add_argument("--json", action="store_true") - p_list.add_argument( - "--sort", - default=None, - choices=sorted(kb.VALID_SORT_ORDERS.keys()), - help="Sort order for listed tasks (default: priority)", - ) - p_list.add_argument( - "--workflow-template-id", - default=None, - metavar="ID", - help="Restrict to tasks with this workflow_template_id", - ) - p_list.add_argument( - "--step-key", - default=None, - dest="current_step_key", - metavar="KEY", - help="Restrict to tasks with this current_step_key", - ) + p_list.add_argument("--sort", default=None, choices=sorted(kb.VALID_SORT_ORDERS.keys()), + help="Sort order for listed tasks (default: priority)") + p_list.add_argument("--workflow-template-id", default=None, metavar="ID", + help="Restrict to tasks with this workflow_template_id") + p_list.add_argument("--step-key", default=None, dest="current_step_key", metavar="KEY", + help="Restrict to tasks with this current_step_key") # --- show --- p_show = sub.add_parser("show", help="Show a task with comments + events") p_show.add_argument("task_id") p_show.add_argument("--json", action="store_true") - p_show.add_argument( - "--state-type", - choices=("status", "outcome"), - default=None, - help="With --state-name: filter listed runs by task_runs column", - ) - p_show.add_argument( - "--state-name", - default=None, - metavar="VALUE", - help="With --state-type: keep runs whose column equals this value", - ) + _add_run_state_filters(p_show, "filter listed runs by task_runs column") # --- assign --- p_assign = sub.add_parser("assign", help="Assign or reassign a task") @@ -517,72 +514,41 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu p_assign.add_argument("profile", help="Profile name (or 'none' to unassign)") # --- set-model (per-task model/provider override) --- - p_set_model = sub.add_parser( - "set-model", - help="Set or clear a task's model/provider override " - "(takes effect on the next dispatch)", - ) + p_set_model = sub.add_parser("set-model", + help="Set or clear a task's model/provider override (takes " + "effect on the next dispatch)") p_set_model.add_argument("task_id") - p_set_model.add_argument( - "model", nargs="?", default=None, - help="Model to pin the worker to (or 'none' to clear the override)", - ) - p_set_model.add_argument( - "--provider", default=None, - help="Provider the model belongs to (worker is spawned with " - "--provider ). Cleared together with the model.", - ) + p_set_model.add_argument("model", nargs="?", default=None, + help="Model to pin the worker to (or 'none' to clear the override)") + p_set_model.add_argument("--provider", default=None, + help="Provider the model belongs to (worker is spawned with " + "--provider ). Cleared together with the model.") # --- reclaim / reassign (recovery) --- - p_reclaim = sub.add_parser( - "reclaim", - help="Release an active worker claim on a running task", - ) + p_reclaim = sub.add_parser("reclaim", help="Release an active worker claim on a running task") p_reclaim.add_argument("task_id") - p_reclaim.add_argument( - "--reason", default=None, - help="Human-readable reason (recorded on the reclaimed event)", - ) + p_reclaim.add_argument("--reason", default=None, + help="Human-readable reason (recorded on the reclaimed event)") - p_reassign = sub.add_parser( - "reassign", - help="Reassign a task to a different profile, optionally reclaiming first", - ) + p_reassign = sub.add_parser("reassign", + help="Reassign a task to a different profile, optionally " + "reclaiming first") p_reassign.add_argument("task_id") - p_reassign.add_argument( - "profile", - help="New profile name (or 'none' to unassign)", - ) - p_reassign.add_argument( - "--reclaim", action="store_true", - help="Release any active claim before reassigning (required if task is running)", - ) - p_reassign.add_argument( - "--reason", default=None, - help="Human-readable reason (recorded on the reclaimed event)", - ) + p_reassign.add_argument("profile", help="New profile name (or 'none' to unassign)") + p_reassign.add_argument("--reclaim", action="store_true", + help="Release any active claim before reassigning (required if task " + "is running)") + p_reassign.add_argument("--reason", default=None, + help="Human-readable reason (recorded on the reclaimed event)") # --- diagnostics (board-wide health) --- - p_diag = sub.add_parser( - "diagnostics", - aliases=["diag"], - help="List active diagnostics on the current board", - ) - p_diag.add_argument( - "--severity", - choices=["warning", "error", "critical"], - default=None, - help="Only show diagnostics at or above this severity", - ) - p_diag.add_argument( - "--task", - default=None, - help="Only show diagnostics for one task id", - ) - p_diag.add_argument( - "--json", action="store_true", - help="Emit JSON (structured) instead of the default human table", - ) + p_diag = sub.add_parser("diagnostics", aliases=["diag"], + help="List active diagnostics on the current board") + p_diag.add_argument("--severity", choices=["warning", "error", "critical"], default=None, + help="Only show diagnostics at or above this severity") + p_diag.add_argument("--task", default=None, help="Only show diagnostics for one task id") + p_diag.add_argument("--json", action="store_true", + help="Emit JSON (structured) instead of the default human table") # --- link / unlink --- p_link = sub.add_parser("link", help="Add a parent->child dependency") @@ -593,10 +559,8 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu p_unlink.add_argument("child_id") # --- claim --- - p_claim = sub.add_parser( - "claim", - help="Atomically claim a ready task (prints resolved workspace path)", - ) + p_claim = sub.add_parser("claim", + help="Atomically claim a ready task (prints resolved workspace path)") p_claim.add_argument("task_id") p_claim.add_argument("--ttl", type=int, default=kb.DEFAULT_CLAIM_TTL_SECONDS, help="Claim TTL in seconds (default: 900)") @@ -639,42 +603,26 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu help='JSON dict of structured facts (e.g. \'{"changed_files": [...], ' '"tests_run": 12}\'). Stored on the closing run.') - p_edit = sub.add_parser( - "edit", - help="Edit recovery fields on an already-completed task", - ) + p_edit = sub.add_parser("edit", help="Edit recovery fields on an already-completed task") p_edit.add_argument("task_id") - p_edit.add_argument( - "--result", - required=True, - help="Backfilled task result text for a done task", - ) - p_edit.add_argument( - "--summary", - default=None, - help="Structured handoff summary. Falls back to --result if omitted.", - ) - p_edit.add_argument( - "--metadata", - default=None, - help="JSON dict of structured facts to store on the latest completed run.", - ) + p_edit.add_argument("--result", required=True, + help="Backfilled task result text for a done task") + p_edit.add_argument("--summary", default=None, + help="Structured handoff summary. Falls back to --result if omitted.") + p_edit.add_argument("--metadata", default=None, + help="JSON dict of structured facts to store on the latest completed run.") p_block = sub.add_parser("block", help="Mark one or more tasks blocked") p_block.add_argument("task_id") p_block.add_argument("reason", nargs="*", help="Reason (also appended as a comment)") p_block.add_argument("--ids", nargs="+", default=None, help="Additional task ids to block with the same reason (bulk mode)") - p_block.add_argument( - "--kind", default=None, choices=sorted(kb.VALID_BLOCK_KINDS), - help=( - "Typed block reason. 'dependency' waits in todo (auto-promoted " - "when parents finish, no human); 'needs_input'/'capability' go to " - "blocked for a human; 'transient' marks a maybe-flaky failure. " - "Repeated same-kind re-blocks after unblock route the task to " - "triage to break unblock loops. Omit for a generic block." - ), - ) + p_block.add_argument("--kind", default=None, choices=sorted(kb.VALID_BLOCK_KINDS), + help="Typed block reason. 'dependency' waits in todo (auto-promoted when " + "parents finish, no human); 'needs_input'/'capability' go to " + "blocked for a human; 'transient' marks a maybe-flaky failure. " + "Repeated same-kind re-blocks after unblock route the task to " + "triage to break unblock loops. Omit for a generic block.") p_schedule = sub.add_parser("schedule", help="Park one or more tasks in Scheduled (waiting on time, not human input)") p_schedule.add_argument("task_id") @@ -682,104 +630,65 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu p_schedule.add_argument("--ids", nargs="+", default=None, help="Additional task ids to schedule with the same reason (bulk mode)") - p_unblock = sub.add_parser( - "unblock", - help="Return blocked/scheduled tasks to ready, or todo while parents remain open", - ) - p_unblock.add_argument( - "--reason", - default=None, - help="Optional reason/note — recorded as a comment before unblocking. Quote multi-word reasons.", - ) + p_unblock = sub.add_parser("unblock", + help="Return blocked/scheduled tasks to ready, or todo while " + "parents remain open") + p_unblock.add_argument("--reason", default=None, + help="Optional reason/note — recorded as a comment before unblocking. " + "Quote multi-word reasons.") p_unblock.add_argument("task_ids", nargs="+") - p_request_review = sub.add_parser( - "request-review", - help="Move a task to 'review' (implementation done, awaiting review) — NOT a block", - ) + p_request_review = sub.add_parser("request-review", + help="Move a task to 'review' (implementation done, " + "awaiting review) — NOT a block") p_request_review.add_argument("task_id") - p_request_review.add_argument( - "--summary", default=None, - help="What was implemented and how it was verified — shown to the reviewer.", - ) - p_request_review.add_argument( - "--reviewer", default=None, - help="Optional reviewer profile; reassigns the task before review dispatch.", - ) - p_request_review.add_argument( - "--metadata", default=None, - help="JSON object with structured reviewer handoff facts.", - ) - p_request_review.add_argument( - "--force", action="store_true", - help=( - "Override the live-claim guard: move a running, claimed task to " - "review even without owning its run (clears the worker's claim)." - ), - ) + p_request_review.add_argument("--summary", default=None, + help="What was implemented and how it was verified — shown to " + "the reviewer.") + p_request_review.add_argument("--reviewer", default=None, + help="Optional reviewer profile; reassigns the task before " + "review dispatch.") + p_request_review.add_argument("--metadata", default=None, + help="JSON object with structured reviewer handoff facts.") + p_request_review.add_argument("--force", action="store_true", + help="Override the live-claim guard: move a running, claimed " + "task to review even without owning its run (clears the " + "worker's claim).") - p_request_changes = sub.add_parser( - "request-changes", - help="Reviewer verdict: return the active review run to its implementer", - ) + p_request_changes = sub.add_parser("request-changes", + help="Reviewer verdict: return the active review run to " + "its implementer") p_request_changes.add_argument("task_id") - p_request_changes.add_argument( - "reason", nargs="+", help="Concrete changes required before re-review", - ) + p_request_changes.add_argument("reason", nargs="+", + help="Concrete changes required before re-review") - p_reopen_review = sub.add_parser( - "reopen-review", - help="Send one or more review tasks back for changes (review -> ready/todo)", - ) + p_reopen_review = sub.add_parser("reopen-review", + help="Send one or more review tasks back for changes (review " + "-> ready/todo)") p_reopen_review.add_argument("task_ids", nargs="+") - p_reopen_review.add_argument( - "--reason", default=None, - help="Optional reason/note — recorded as a comment before reopening. Quote multi-word reasons.", - ) + p_reopen_review.add_argument("--reason", default=None, + help="Optional reason/note — recorded as a comment before " + "reopening. Quote multi-word reasons.") - p_promote = sub.add_parser( - "promote", - help="Manually move one or more todo/blocked tasks to ready (recovery path)", - ) + p_promote = sub.add_parser("promote", + help="Manually move one or more todo/blocked tasks to ready " + "(recovery path)") p_promote.add_argument("task_id") - p_promote.add_argument( - "reason", - nargs="*", - help="Audit-trail reason (recorded on the task_events row)", - ) - p_promote.add_argument( - "--ids", - nargs="+", - default=None, - help="Additional task ids to promote with the same reason (bulk mode)", - ) - p_promote.add_argument( - "--force", - action="store_true", - help="Promote even if parent dependencies are not yet done/archived", - ) - p_promote.add_argument( - "--dry-run", - action="store_true", - help="Validate the promotion without mutating state", - ) - p_promote.add_argument( - "--json", - dest="json", - action="store_true", - help="Emit machine-readable JSON result", - ) + p_promote.add_argument("reason", nargs="*", + help="Audit-trail reason (recorded on the task_events row)") + p_promote.add_argument("--ids", nargs="+", default=None, + help="Additional task ids to promote with the same reason (bulk mode)") + p_promote.add_argument("--force", action="store_true", + help="Promote even if parent dependencies are not yet done/archived") + p_promote.add_argument("--dry-run", action="store_true", + help="Validate the promotion without mutating state") + p_promote.add_argument("--json", dest="json", action="store_true", + help="Emit machine-readable JSON result") p_archive = sub.add_parser("archive", help="Archive one or more tasks") - p_archive.add_argument("task_ids", nargs="*", - help="Task ids to archive (default mode)") - p_archive.add_argument( - "--rm", - dest="purge_ids", - nargs="+", - default=None, - help="Permanently delete already-archived task ids from the board", - ) + p_archive.add_argument("task_ids", nargs="*", help="Task ids to archive (default mode)") + p_archive.add_argument("--rm", dest="purge_ids", nargs="+", default=None, + help="Permanently delete already-archived task ids from the board") # --- tail --- p_tail = sub.add_parser("tail", help="Follow a task's event stream") @@ -787,14 +696,11 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu p_tail.add_argument("--interval", type=float, default=1.0) # --- dispatch --- - p_disp = sub.add_parser( - "dispatch", - help="One dispatcher pass: reclaim stale, promote ready, spawn workers", - ) + p_disp = sub.add_parser("dispatch", + help="One dispatcher pass: reclaim stale, promote ready, spawn workers") p_disp.add_argument("--dry-run", action="store_true", help="Don't actually spawn processes; just print what would happen") - p_disp.add_argument("--max", type=int, default=None, - help="Cap number of spawns this pass") + p_disp.add_argument("--max", type=int, default=None, help="Cap number of spawns this pass") p_disp.add_argument("--failure-limit", type=int, default=kb.DEFAULT_SPAWN_FAILURE_LIMIT, help=f"Auto-block a task after this many consecutive non-success attempts " @@ -802,31 +708,24 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu p_disp.add_argument("--json", action="store_true") # --- daemon (deprecated) --- - p_daemon = sub.add_parser( - "daemon", - help="DEPRECATED — dispatcher now runs in the gateway. Use `hermes gateway start`.", - ) + p_daemon = sub.add_parser("daemon", + help="DEPRECATED — dispatcher now runs in the gateway. Use `hermes " + "gateway start`.") p_daemon.add_argument("--interval", type=float, default=60.0, help="Seconds between dispatch ticks (default: 60)") - p_daemon.add_argument("--max", type=int, default=None, - help="Cap number of spawns per tick") - p_daemon.add_argument("--failure-limit", type=int, - default=kb.DEFAULT_SPAWN_FAILURE_LIMIT) + p_daemon.add_argument("--max", type=int, default=None, help="Cap number of spawns per tick") + p_daemon.add_argument("--failure-limit", type=int, default=kb.DEFAULT_SPAWN_FAILURE_LIMIT) p_daemon.add_argument("--pidfile", default=None, help="Write the daemon's PID to this file on start") p_daemon.add_argument("--verbose", "-v", action="store_true", help="Log each tick's outcome to stdout") - # Undocumented escape hatch for users who truly cannot run the gateway. - # Intentionally excluded from --help so nobody discovers it casually and - # keeps the old double-dispatcher pattern alive. - p_daemon.add_argument("--force", action="store_true", - help=argparse.SUPPRESS) + # Escape hatch for hosts that truly cannot run the gateway; hidden from + # --help so nobody casually keeps the double-dispatcher pattern alive. + p_daemon.add_argument("--force", action="store_true", help=argparse.SUPPRESS) # --- watch --- - p_watch = sub.add_parser( - "watch", - help="Live-stream task_events to the terminal (Ctrl+C to exit)", - ) + p_watch = sub.add_parser("watch", + help="Live-stream task_events to the terminal (Ctrl+C to exit)") p_watch.add_argument("--assignee", default=None, help="Only show events for tasks assigned to this profile") p_watch.add_argument("--tenant", default=None, @@ -838,35 +737,26 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu help="Poll interval in seconds (default: 0.5)") # --- stats --- - p_stats = sub.add_parser( - "stats", help="Per-status + per-assignee counts + oldest-ready age", - ) + p_stats = sub.add_parser("stats", help="Per-status + per-assignee counts + oldest-ready age") p_stats.add_argument("--json", action="store_true") # --- notify subscribe / list / remove --- - p_nsub = sub.add_parser( - "notify-subscribe", - help="Subscribe a gateway source to a task's terminal events " - "(used by /kanban subscribe in the gateway adapter)", - ) + p_nsub = sub.add_parser("notify-subscribe", + help="Subscribe a gateway source to a task's terminal events (used by " + "/kanban subscribe in the gateway adapter)") p_nsub.add_argument("task_id") p_nsub.add_argument("--platform", required=True) p_nsub.add_argument("--chat-id", required=True) p_nsub.add_argument("--thread-id", default=None) p_nsub.add_argument("--user-id", default=None) p_nsub.add_argument("--user-id-alt", default=None) - p_nsub.add_argument( - "--chat-type", - choices=("dm", "group", "channel", "thread"), - default=None, - help="Originating source chat_type, recorded so the active-wake " - "delivery modes resolve the operator's real session. Omit to " - "leave an existing sub unchanged (new subs default to 'dm').", - ) - p_nsub.add_argument( - "--notifier-profile", default=None, - help="Profile gateway that owns/delivers this subscription (default: active profile)", - ) + p_nsub.add_argument("--chat-type", choices=("dm", "group", "channel", "thread"), default=None, + help="Originating source chat_type, recorded so the active-wake delivery " + "modes resolve the operator's real session. Omit to leave an " + "existing sub unchanged (new subs default to 'dm').") + p_nsub.add_argument("--notifier-profile", default=None, + help="Profile gateway that owns/delivers this subscription (default: " + "active profile)") p_nsub.add_argument( "--delivery-mode", # Single source of truth shared with the DB/watcher enum. @@ -880,154 +770,69 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu "existing subscription's mode unchanged (new subs default to 'notify').", ) - p_nlist = sub.add_parser( - "notify-list", - help="List notification subscriptions (optionally for a single task)", - ) + p_nlist = sub.add_parser("notify-list", + help="List notification subscriptions (optionally for a single task)") p_nlist.add_argument("task_id", nargs="?", default=None) p_nlist.add_argument("--json", action="store_true") - p_nrm = sub.add_parser( - "notify-unsubscribe", - help="Remove a gateway subscription from a task", - ) + p_nrm = sub.add_parser("notify-unsubscribe", help="Remove a gateway subscription from a task") p_nrm.add_argument("task_id") p_nrm.add_argument("--platform", required=True) p_nrm.add_argument("--chat-id", required=True) p_nrm.add_argument("--thread-id", default=None) # --- log --- - p_log = sub.add_parser( - "log", - help="Print the worker log for a task (from /kanban/logs/)", - ) + p_log = sub.add_parser("log", + help="Print the worker log for a task (from /kanban/logs/)") p_log.add_argument("task_id") - p_log.add_argument("--tail", type=int, default=None, - help="Only print the last N bytes") + p_log.add_argument("--tail", type=int, default=None, help="Only print the last N bytes") # --- runs (per-attempt history for a task) --- - p_runs = sub.add_parser( - "runs", - help="Show attempt history for a task (one row per run: profile, " - "outcome, elapsed, summary)", - ) + p_runs = sub.add_parser("runs", + help="Show attempt history for a task (one row per run: profile, " + "outcome, elapsed, summary)") p_runs.add_argument("task_id") p_runs.add_argument("--json", action="store_true") - p_runs.add_argument( - "--state-type", - choices=("status", "outcome"), - default=None, - help="With --state-name: filter runs by task_runs column", - ) - p_runs.add_argument( - "--state-name", - default=None, - metavar="VALUE", - help="With --state-type: keep runs whose column equals this value", - ) + _add_run_state_filters(p_runs, "filter runs by task_runs column") # --- heartbeat (worker liveness signal) --- - p_hb = sub.add_parser( - "heartbeat", - help="Emit a heartbeat event for a running task (worker liveness signal)", - ) + p_hb = sub.add_parser("heartbeat", + help="Emit a heartbeat event for a running task (worker liveness signal)") p_hb.add_argument("task_id") p_hb.add_argument("--note", default=None, help="Optional short note attached to the heartbeat event") # --- assignees --- - p_asg = sub.add_parser( - "assignees", - help="List known profiles + per-profile task counts " - "(union of ~/.hermes/profiles/ and current assignees on the board)", - ) + p_asg = sub.add_parser("assignees", + help="List known profiles + per-profile task counts (union of " + "~/.hermes/profiles/ and current assignees on the board)") p_asg.add_argument("--json", action="store_true") # --- context --- (for spawned workers) - p_ctx = sub.add_parser( - "context", - help="Print the full context a worker sees for a task " - "(title + body + parent results + comments).", - ) + p_ctx = sub.add_parser("context", + help="Print the full context a worker sees for a task (title + body + " + "parent results + comments).") p_ctx.add_argument("task_id") # --- specify --- (triage → todo via auxiliary LLM) - p_specify = sub.add_parser( - "specify", - help="Flesh out a triage-column task into a concrete spec " - "(title + body) and promote it to todo. Uses the auxiliary " - "LLM configured under auxiliary.triage_specifier.", - ) - p_specify.add_argument( - "task_id", - nargs="?", - default=None, - help="Task id to specify (required unless --all is given)", - ) - p_specify.add_argument( - "--all", - dest="all_triage", - action="store_true", - help="Specify every task currently in the triage column", - ) - p_specify.add_argument( - "--tenant", - default=None, - help="When used with --all, restrict the sweep to this tenant", - ) - p_specify.add_argument( - "--author", - default=None, - help="Author name recorded on the audit comment " - "(default: $HERMES_PROFILE or 'specifier')", - ) - p_specify.add_argument( - "--json", - action="store_true", - help="Emit one JSON object per task on stdout", - ) + p_specify = sub.add_parser("specify", + help="Flesh out a triage-column task into a concrete spec (title + " + "body) and promote it to todo. Uses the auxiliary LLM " + "configured under auxiliary.triage_specifier.") + _add_triage_sweep_args(p_specify, "specify", "Specify", "specifier") # --- decompose --- (triage → fan-out via auxiliary LLM + orchestrator) - p_decompose = sub.add_parser( - "decompose", - help="Decompose a triage-column task into a graph of child tasks " - "routed to specialist profiles by description. Falls back to " - "specify-style single-task promotion when the task doesn't " - "benefit from fan-out. Uses auxiliary.kanban_decomposer.", - ) - p_decompose.add_argument( - "task_id", - nargs="?", - default=None, - help="Task id to decompose (required unless --all is given)", - ) - p_decompose.add_argument( - "--all", - dest="all_triage", - action="store_true", - help="Decompose every task currently in the triage column", - ) - p_decompose.add_argument( - "--tenant", - default=None, - help="When used with --all, restrict the sweep to this tenant", - ) - p_decompose.add_argument( - "--author", - default=None, - help="Author name recorded on the audit comment " - "(default: $HERMES_PROFILE or 'decomposer')", - ) - p_decompose.add_argument( - "--json", - action="store_true", - help="Emit one JSON object per task on stdout", - ) + p_decompose = sub.add_parser("decompose", + help="Decompose a triage-column task into a graph of child tasks " + "routed to specialist profiles by description. Falls back " + "to specify-style single-task promotion when the task " + "doesn't benefit from fan-out. Uses " + "auxiliary.kanban_decomposer.") + _add_triage_sweep_args(p_decompose, "decompose", "Decompose", "decomposer") # --- gc --- - p_gc = sub.add_parser( - "gc", help="Garbage-collect archived-task workspaces, old events, and old logs", - ) + p_gc = sub.add_parser("gc", + help="Garbage-collect archived-task workspaces, old events, and old logs") p_gc.add_argument("--event-retention-days", type=int, default=30, help="Delete task_events older than N days for terminal tasks (default: 30)") p_gc.add_argument("--log-retention-days", type=int, default=30, @@ -1049,8 +854,7 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu "is healthy or was repaired, non-zero when it is still corrupt." ), ) - p_repair.add_argument("--json", action="store_true", - help="Emit the repair report as JSON") + p_repair.add_argument("--json", action="store_true", help="Emit the repair report as JSON") kanban_parser.set_defaults(_kanban_parser=kanban_parser) return kanban_parser @@ -1061,10 +865,7 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu # --------------------------------------------------------------------------- def kanban_command(args: argparse.Namespace) -> int: - """Entry point from ``hermes kanban …`` argparse dispatch. - - Returns a shell-style exit code (0 on success, non-zero on error). - """ + """Entry point from ``hermes kanban …``; returns a shell-style exit code.""" action = getattr(args, "kanban_action", None) if not action: # No subaction given: print help via the stored parser reference. @@ -1082,123 +883,56 @@ def kanban_command(args: argparse.Namespace) -> int: # Fast-fail for clearer CLI UX only. The durable trust boundary is lower in # hermes_cli.kanban_db, because children can import DB mutators directly. if _is_delegated_child_cli_mutation(args): - print( - "kanban: delegate_task child contexts cannot mutate Kanban tasks via the CLI", - file=sys.stderr, - ) - return 1 + return _err("kanban: delegate_task child contexts cannot mutate Kanban tasks via the CLI") # Board-management commands operate on board metadata and the persisted - # current-board pointer itself. They must ignore the shared `--board` - # task-routing override; otherwise `/kanban --board beta boards show` - # reports beta as the current board even when the on-disk pointer is - # alpha. + # current-board pointer itself, so they must ignore the shared `--board` + # task-routing override (else `--board beta boards show` reports beta). if action == "boards": return _dispatch_boards(args) - # `--board ` applies to every subcommand below by way of an - # env-var pin for the duration of this call. Using HERMES_KANBAN_BOARD - # (rather than threading `board=` through 50+ kb.connect() sites) - # keeps the patch small and inherits the exact same resolution the - # dispatcher uses for workers — consistency is a feature here. + # `--board ` applies to every subcommand below via an env-var pin + # (HERMES_KANBAN_BOARD) for the duration of this call, so it inherits the + # exact resolution the dispatcher uses for workers. board_override = getattr(args, "board", None) board_scope = contextlib.nullcontext() if board_override: try: normed = kb._normalize_board_slug(board_override) except ValueError as exc: - print(f"kanban: {exc}", file=sys.stderr) - return 2 + return _err(f"kanban: {exc}", 2) if not normed: - print("kanban: --board requires a slug", file=sys.stderr) - return 2 + return _err("kanban: --board requires a slug", 2) # Boards other than 'default' must already exist — typoed slugs # would otherwise silently create an empty board. if normed != kb.DEFAULT_BOARD and not kb.board_exists(normed): - print( + return _err( f"kanban: board {normed!r} does not exist. " - f"Create it with `hermes kanban boards create {normed}`.", - file=sys.stderr, + f"Create it with `hermes kanban boards create {normed}`." ) - return 1 board_scope = kb.scoped_current_board(normed) - # Auto-initialize the DB before dispatching any subcommand. init_db - # is idempotent, so running it every invocation is cheap (one - # SELECT against sqlite_master when tables already exist) and - # prevents "no such table: tasks" on first use from a fresh - # HERMES_HOME. Previously only `init` and `daemon` triggered - # schema creation; `create` / `list` / every other command would - # error out on a fresh install. with board_scope: - # `repair` must dispatch BEFORE the auto-init below: on a corrupt DB - # init_db() itself raises KanbanDbCorruptError, which would turn - # every `hermes kanban repair` into "could not initialize database" - # without ever reaching the repair path. + # `repair` must dispatch BEFORE the auto-init: on a corrupt DB init_db() + # itself raises KanbanDbCorruptError, which would turn every + # `hermes kanban repair` into "could not initialize database". if action == "repair": return _cmd_repair(args) + # Auto-initialize the DB before any subcommand. init_db is idempotent + # (one SELECT against sqlite_master when tables exist) and prevents + # "no such table: tasks" on first use from a fresh HERMES_HOME. try: kb.init_db() except Exception as exc: - print(f"kanban: could not initialize database: {exc}", file=sys.stderr) - return 1 + return _err(f"kanban: could not initialize database: {exc}") - handlers = { - "init": _cmd_init, - "create": _cmd_create, - "swarm": _cmd_swarm, - "list": _cmd_list, - "ls": _cmd_list, - "show": _cmd_show, - "assign": _cmd_assign, - "set-model": _cmd_set_model, - "reclaim": _cmd_reclaim, - "reassign": _cmd_reassign, - "diagnostics": _cmd_diagnostics, - "diag": _cmd_diagnostics, - "link": _cmd_link, - "unlink": _cmd_unlink, - "claim": _cmd_claim, - "comment": _cmd_comment, - "attach": _cmd_attach, - "attachments": _cmd_attachments, - "attach-rm": _cmd_attach_rm, - "complete": _cmd_complete, - "edit": _cmd_edit, - "block": _cmd_block, - "schedule": _cmd_schedule, - "unblock": _cmd_unblock, - "request-review": _cmd_request_review, - "request-changes": _cmd_request_changes, - "reopen-review": _cmd_reopen_review, - "promote": _cmd_promote, - "archive": _cmd_archive, - "tail": _cmd_tail, - "dispatch": _cmd_dispatch, - "daemon": _cmd_daemon, - "watch": _cmd_watch, - "stats": _cmd_stats, - "log": _cmd_log, - "runs": _cmd_runs, - "heartbeat": _cmd_heartbeat, - "assignees": _cmd_assignees, - "notify-subscribe": _cmd_notify_subscribe, - "notify-list": _cmd_notify_list, - "notify-unsubscribe": _cmd_notify_unsubscribe, - "context": _cmd_context, - "specify": _cmd_specify, - "decompose": _cmd_decompose, - "gc": _cmd_gc, - } - handler = handlers.get(action) + handler = _HANDLERS.get(action) if not handler: - print(f"kanban: unknown action {action!r}", file=sys.stderr) - return 2 + return _err(f"kanban: unknown action {action!r}", 2) try: return int(handler(args) or 0) except (ValueError, RuntimeError) as exc: - print(f"kanban: {exc}", file=sys.stderr) - return 1 + return _err(f"kanban: {exc}") # --------------------------------------------------------------------------- @@ -1219,45 +953,15 @@ def _profile_author() -> str: _DELEGATED_CHILD_DENIED_ACTIONS: frozenset[str] = frozenset({ - "init", - "create", - "swarm", - "assign", - "reclaim", - "reassign", - "link", - "unlink", - "claim", - "comment", - "attach", - "attach-rm", - "complete", - "edit", - "block", - "schedule", - "unblock", - "promote", - "archive", - "dispatch", - "daemon", - "repair", - "heartbeat", - "notify-subscribe", - "notify-unsubscribe", - "specify", - "decompose", + "init", "create", "swarm", "assign", "reclaim", "reassign", "link", "unlink", + "claim", "comment", "attach", "attach-rm", "complete", "edit", "block", + "schedule", "unblock", "promote", "archive", "dispatch", "daemon", "repair", + "heartbeat", "notify-subscribe", "notify-unsubscribe", "specify", "decompose", "gc", }) _DELEGATED_CHILD_DENIED_BOARD_ACTIONS: frozenset[str] = frozenset({ - "create", - "new", - "rm", - "remove", - "delete", - "switch", - "use", - "rename", + "create", "new", "rm", "remove", "delete", "switch", "use", "rename", "set-default-workdir", }) @@ -1283,35 +987,13 @@ def _is_delegated_child_cli_mutation(args: argparse.Namespace) -> bool: # --------------------------------------------------------------------------- def _dispatch_boards(args: argparse.Namespace) -> int: - """Handle ``hermes kanban boards ``. - - Boards management is deliberately separate from the task-level - commands: it operates on the filesystem (board directories, - ``current`` pointer, ``board.json``), not on the per-board SQLite - DB, so a fresh HERMES_HOME that has never called ``kanban init`` - can still run ``boards create`` / ``boards list``. - """ + """``hermes kanban boards `` — filesystem-only (board dirs, the + ``current`` pointer, ``board.json``), so it works before ``kanban init``.""" sub = getattr(args, "boards_action", None) or "list" - if sub in {"list", "ls"}: - return _cmd_boards_list(args) - if sub in {"create", "new"}: - return _cmd_boards_create(args) - if sub in {"rm", "remove", "delete"}: - return _cmd_boards_rm(args) - if sub in {"switch", "use"}: - return _cmd_boards_switch(args) - if sub in {"show", "current"}: - return _cmd_boards_show(args) - if sub == "rename": - return _cmd_boards_rename(args) - if sub == "set-default-workdir": - return _cmd_boards_set_default_workdir(args) - if sub == "export": - return _cmd_boards_export(args) - if sub == "import": - return _cmd_boards_import(args) - print(f"kanban boards: unknown action {sub!r}", file=sys.stderr) - return 2 + handler = _BOARD_HANDLERS.get(sub) + if handler is None: + return _err(f"kanban boards: unknown action {sub!r}", 2) + return handler(args) def _board_task_counts(slug: str) -> dict[str, int]: @@ -1329,34 +1011,39 @@ def _board_task_counts(slug: str) -> dict[str, int]: return {} +def _board_slug_arg(args: argparse.Namespace, cmd: str, *, must_exist: bool) -> tuple[Optional[str], int]: + """Normalize ``args.slug`` for a ``boards`` subcommand; ``(slug, 0)`` or ``(None, rc)``.""" + try: + normed = kb._normalize_board_slug(args.slug) + except ValueError as exc: + return None, _err(f"kanban boards {cmd}: {exc}", 2) + if must_exist: + if not normed or not kb.board_exists(normed): + return None, _err(f"kanban boards {cmd}: board {args.slug!r} does not exist") + elif not normed: + return None, _err(f"kanban boards {cmd}: slug is required", 2) + return normed, 0 + + def _cmd_boards_list(args: argparse.Namespace) -> int: - include_archived = bool(getattr(args, "all", False)) - boards = kb.list_boards(include_archived=include_archived) - # Enrich each entry with task counts + whether it's the current board. + boards = kb.list_boards(include_archived=bool(getattr(args, "all", False))) current = kb.get_current_board() for b in boards: b["is_current"] = (b["slug"] == current) b["counts"] = _board_task_counts(b["slug"]) b["total"] = sum(b["counts"].values()) - if getattr(args, "json", False): - print(json.dumps(boards, indent=2, ensure_ascii=False)) + if _json_out(args, boards): return 0 - # Human table: marker (•) for current, slug, display name, counts. if not boards: print("(no boards — create one with `hermes kanban boards create `)") return 0 print(f"{'':2s} {'SLUG':24s} {'NAME':28s} COUNTS") for b in boards: marker = "●" if b["is_current"] else " " - counts = b["counts"] or {} - counts_str = ( - ", ".join(f"{k}={v}" for k, v in sorted(counts.items())) - or "(empty)" - ) name = b.get("name") or "" if b.get("archived"): name += " [archived]" - print(f"{marker:2s} {b['slug']:24s} {name:28s} {counts_str}") + print(f"{marker:2s} {b['slug']:24s} {name:28s} {_fmt_counts(b['counts'] or {}, '(empty)')}") print() print(f"Current board: {current}") if len(boards) > 1: @@ -1365,14 +1052,9 @@ def _cmd_boards_list(args: argparse.Namespace) -> int: def _cmd_boards_create(args: argparse.Namespace) -> int: - try: - normed = kb._normalize_board_slug(args.slug) - except ValueError as exc: - print(f"kanban boards create: {exc}", file=sys.stderr) - return 2 - if not normed: - print("kanban boards create: slug is required", file=sys.stderr) - return 2 + normed, rc = _board_slug_arg(args, "create", must_exist=False) + if rc: + return rc already = kb.board_exists(normed) and normed != kb.DEFAULT_BOARD meta = kb.create_board( normed, @@ -1395,16 +1077,13 @@ def _cmd_boards_create(args: argparse.Namespace) -> int: def _cmd_boards_rm(args: argparse.Namespace) -> int: - # When the user runs `hermes kanban boards delete ` (alias), the - # boards_action is 'delete' but args.delete is never set to True because - # the --delete flag belongs to the 'rm' subparser only. Detect the alias - # and treat it identically to `boards rm --delete` (fixes #23139). + # `boards delete ` (alias) never sets args.delete because --delete + # belongs to the 'rm' subparser only; treat the alias as `rm --delete`. force_delete = getattr(args, "delete", False) or getattr(args, "boards_action", "") == "delete" try: res = kb.remove_board(args.slug, archive=not force_delete) except ValueError as exc: - print(f"kanban boards rm: {exc}", file=sys.stderr) - return 1 + return _err(f"kanban boards rm: {exc}") if res["action"] == "archived": print(f"Board {res['slug']!r} archived → {res['new_path']}") print("Recover by moving the directory back to " @@ -1415,21 +1094,14 @@ def _cmd_boards_rm(args: argparse.Namespace) -> int: def _cmd_boards_switch(args: argparse.Namespace) -> int: - try: - normed = kb._normalize_board_slug(args.slug) - except ValueError as exc: - print(f"kanban boards switch: {exc}", file=sys.stderr) - return 2 - if not normed: - print("kanban boards switch: slug is required", file=sys.stderr) - return 2 + normed, rc = _board_slug_arg(args, "switch", must_exist=False) + if rc: + return rc if not kb.board_exists(normed): - print( + return _err( f"kanban boards switch: board {normed!r} does not exist. " - f"Create it with `hermes kanban boards create {normed}`.", - file=sys.stderr, + f"Create it with `hermes kanban boards create {normed}`." ) - return 1 kb.set_current_board(normed) print(f"Active board is now {normed!r}.") return 0 @@ -1439,43 +1111,29 @@ def _cmd_boards_show(args: argparse.Namespace) -> int: current = kb.get_current_board() meta = kb.read_board_metadata(current) counts = _board_task_counts(current) - total = sum(counts.values()) print(f"Current board: {current}") print(f" Display name: {meta.get('name', '')}") if meta.get("description"): print(f" Description: {meta['description']}") print(f" DB path: {meta['db_path']}") - print(f" Tasks: {total} total" - + (f" ({', '.join(f'{k}={v}' for k, v in sorted(counts.items()))})" - if counts else "")) + print(f" Tasks: {sum(counts.values())} total" + + (f" ({_fmt_counts(counts)})" if counts else "")) return 0 def _cmd_boards_rename(args: argparse.Namespace) -> int: - try: - normed = kb._normalize_board_slug(args.slug) - except ValueError as exc: - print(f"kanban boards rename: {exc}", file=sys.stderr) - return 2 - if not normed or not kb.board_exists(normed): - print(f"kanban boards rename: board {args.slug!r} does not exist", - file=sys.stderr) - return 1 + normed, rc = _board_slug_arg(args, "rename", must_exist=True) + if rc: + return rc meta = kb.write_board_metadata(normed, name=args.name) print(f"Board {normed!r} renamed to {meta['name']!r}.") return 0 def _cmd_boards_set_default_workdir(args: argparse.Namespace) -> int: - try: - normed = kb._normalize_board_slug(args.slug) - except ValueError as exc: - print(f"kanban boards set-default-workdir: {exc}", file=sys.stderr) - return 2 - if not normed or not kb.board_exists(normed): - print(f"kanban boards set-default-workdir: board {args.slug!r} does not exist", - file=sys.stderr) - return 1 + normed, rc = _board_slug_arg(args, "set-default-workdir", must_exist=True) + if rc: + return rc meta = kb.write_board_metadata(normed, default_workdir=args.path) new_val = meta.get("default_workdir") if new_val: @@ -1499,11 +1157,9 @@ def _cmd_boards_export(args: argparse.Namespace) -> int: include_logs=args.include_logs, ) except (OSError, ValueError) as exc: - print(f"kanban boards export: {exc}", file=sys.stderr) - return 1 + return _err(f"kanban boards export: {exc}") - if getattr(args, "json", False): - print(json.dumps(res, indent=2, ensure_ascii=False)) + if _json_out(args, res): return 0 counts = res["counts"] print(f"Exported board {res['board']!r} → {res['archive']}") @@ -1523,11 +1179,9 @@ def _cmd_boards_import(args: argparse.Namespace) -> int: args.archive, args.as_slug, activate=args.switch ) except (OSError, ValueError) as exc: - print(f"kanban boards import: {exc}", file=sys.stderr) - return 1 + return _err(f"kanban boards import: {exc}") - if getattr(args, "json", False): - print(json.dumps(res, indent=2, ensure_ascii=False)) + if _json_out(args, res): return 0 print(f"Imported board {res['board']!r} ({res['name']}).") if res["renamed"]: @@ -1543,15 +1197,25 @@ def _cmd_boards_import(args: argparse.Namespace) -> int: return 0 +_BOARD_HANDLERS = { + "list": _cmd_boards_list, "ls": _cmd_boards_list, + "create": _cmd_boards_create, "new": _cmd_boards_create, + "rm": _cmd_boards_rm, "remove": _cmd_boards_rm, "delete": _cmd_boards_rm, + "switch": _cmd_boards_switch, "use": _cmd_boards_switch, + "show": _cmd_boards_show, "current": _cmd_boards_show, + "rename": _cmd_boards_rename, + "set-default-workdir": _cmd_boards_set_default_workdir, + "export": _cmd_boards_export, + "import": _cmd_boards_import, +} + + # --------------------------------------------------------------------------- def _parse_duration(val) -> Optional[int]: - """Parse ``30s`` / ``5m`` / ``2h`` / ``1d`` or a raw integer → seconds. - - Returns None for empty input. Raises ValueError on malformed input so - the CLI can surface a usage error cleanly. - """ + """``30s`` / ``5m`` / ``2h`` / ``1d`` or a raw integer → seconds; None for + empty input; ValueError on malformed input.""" if val is None or val == "": return None s = str(val).strip().lower() @@ -1577,10 +1241,7 @@ def _cmd_init(args: argparse.Namespace) -> int: print() # Enumerate profiles on disk so the user knows what assignees are - # already addressable. Multica does this auto-detection on its - # daemon start; we do it here at init time instead because our - # dispatcher doesn't need to enumerate — we just pass the name - # through to `hermes -p `. + # already addressable. try: profiles = kb.list_profiles_on_disk() except Exception: @@ -1614,8 +1275,7 @@ def _cmd_heartbeat(args: argparse.Namespace) -> int: expected_run_id=_worker_run_id_for(args.task_id), ) if not ok: - print(f"cannot heartbeat {args.task_id} (not running?)", file=sys.stderr) - return 1 + return _err(f"cannot heartbeat {args.task_id} (not running?)") print(f"Heartbeat recorded for {args.task_id}") return 0 @@ -1623,19 +1283,15 @@ def _cmd_heartbeat(args: argparse.Namespace) -> int: def _cmd_assignees(args: argparse.Namespace) -> int: with kb.connect_closing() as conn: data = kb.known_assignees(conn) - if getattr(args, "json", False): - print(json.dumps(data, indent=2, ensure_ascii=False)) + if _json_out(args, data): return 0 if not data: print("(no assignees — create a profile with `hermes -p setup`)") return 0 - # Header print(f"{'NAME':20s} {'ON DISK':8s} COUNTS") for entry in data: on_disk = "yes" if entry["on_disk"] else "no" - counts = entry["counts"] or {} - count_str = ", ".join(f"{k}={v}" for k, v in sorted(counts.items())) or "(idle)" - print(f"{entry['name']:20s} {on_disk:8s} {count_str}") + print(f"{entry['name']:20s} {on_disk:8s} {_fmt_counts(entry['counts'] or {}, '(idle)')}") return 0 @@ -1644,24 +1300,20 @@ def _cmd_create(args: argparse.Namespace) -> int: ws_kind, ws_path = _parse_workspace_flag(args.workspace) branch_name = _parse_branch_flag(getattr(args, "branch", None)) except argparse.ArgumentTypeError as exc: - print(f"kanban: {exc}", file=sys.stderr) - return 2 + return _err(f"kanban: {exc}", 2) if branch_name and ws_kind != "worktree": - print("kanban: --branch is only valid with --workspace worktree", file=sys.stderr) - return 2 + return _err("kanban: --branch is only valid with --workspace worktree", 2) try: max_runtime = _parse_duration(getattr(args, "max_runtime", None)) except ValueError as exc: - print(f"kanban: --max-runtime: {exc}", file=sys.stderr) - return 2 + return _err(f"kanban: --max-runtime: {exc}", 2) max_retries = getattr(args, "max_retries", None) if max_retries is not None and max_retries < 1: - print( + return _err( f"kanban: --max-retries must be >= 1 (got {max_retries}); " "use 1 to trip on the first failure.", - file=sys.stderr, + 2, ) - return 2 with kb.connect_closing() as conn: task_id = kb.create_task( conn, @@ -1689,17 +1341,14 @@ def _cmd_create(args: argparse.Namespace) -> int: ) task = kb.get_task(conn, task_id) if getattr(args, "json", False): - print(json.dumps(_task_to_dict(task), indent=2, ensure_ascii=False)) + _print_json(_task_to_dict(task)) else: print(f"Created {task_id} ({task.status}, assignee={task.assignee or '-'})") # Warn when the task would sit in `ready` because no dispatcher is - # present. Only warn on ready+assigned tasks — triage/todo are - # expected to sit idle until promoted, and unassigned tasks - # can't be dispatched. Skipped in --json mode so the stdout - # stream stays strictly machine-parseable for callers (the JSON - # response itself carries enough info for them to decide if - # they want to check dispatcher presence separately). + # present. Only ready+assigned tasks — triage/todo idle by design, + # unassigned can't dispatch. Skipped in --json so stdout stays + # machine-parseable. if task.status == "ready" and task.assignee: running, message = _check_dispatcher_presence() if not running and message: @@ -1711,11 +1360,9 @@ def _cmd_swarm(args: argparse.Namespace) -> int: try: workers = [ks.parse_worker_arg(raw) for raw in (args.worker or [])] except ValueError as exc: - print(f"kanban swarm: {exc}", file=sys.stderr) - return 2 + return _err(f"kanban swarm: {exc}", 2) if not workers: - print("kanban swarm: at least one --worker is required", file=sys.stderr) - return 2 + return _err("kanban swarm: at least one --worker is required", 2) with kb.connect_closing() as conn: created = ks.create_swarm( conn, @@ -1729,7 +1376,7 @@ def _cmd_swarm(args: argparse.Namespace) -> int: idempotency_key=getattr(args, "idempotency_key", None), ) if getattr(args, "json", False): - print(json.dumps(created.as_dict(), indent=2, ensure_ascii=False)) + _print_json(created.as_dict()) else: print(f"Swarm root: {created.root_id}") print("Workers: " + ", ".join(created.worker_ids)) @@ -1757,12 +1404,9 @@ def _cmd_list(args: argparse.Namespace) -> int: workflow_template_id=args.workflow_template_id, current_step_key=args.current_step_key, ) - if getattr(args, "json", False): - print(json.dumps([_task_to_dict(t) for t in tasks], indent=2, ensure_ascii=False)) + if _json_out(args, [_task_to_dict(t) for t in tasks]): return 0 - # Passive discoverability: when the user has multiple boards, surface - # which one they're looking at in the list header. Single-board users - # never see this — the feature stays invisible until you opt in. + # Passive discoverability: only multi-board users see which board this is. try: all_boards = kb.list_boards(include_archived=False) except Exception: @@ -1783,69 +1427,54 @@ def _cmd_list(args: argparse.Namespace) -> int: return 0 +def _print_diagnostics(diags, indent: str, *, with_kind: bool) -> None: + """Shared human rendering for ``show`` and ``diagnostics`` (suggested actions only).""" + sev_marker = {"warning": "⚠", "error": "!!", "critical": "!!!"} + for d in diags: + head = f"{d.kind}: {d.title}" if with_kind else d.title + print(f"{indent}{sev_marker.get(d.severity, '?')} [{d.severity}] {head}") + if d.data: + bits = [ + f"{k}={','.join(str(x) for x in v)}" if isinstance(v, list) else f"{k}={v}" + for k, v in d.data.items() + ] + if bits: + print(f"{indent} data: {' | '.join(bits)}") + for a in d.actions: + if a.suggested: + print(f"{indent} → {a.label}") + + def _cmd_show(args: argparse.Namespace) -> int: rsk = _run_state_kwargs(args) if rsk is None: - print( - "kanban show: pass both --state-type and --state-name, or omit both", - file=sys.stderr, - ) - return 2 + return _err("kanban show: pass both --state-type and --state-name, or omit both", 2) graph = None with kb.connect_closing() as conn: task = kb.get_task(conn, args.task_id) if not task: - print(f"no such task: {args.task_id}", file=sys.stderr) - return 1 + return _err(f"no such task: {args.task_id}") comments = kb.list_comments(conn, args.task_id) events = kb.list_events(conn, args.task_id) parents = kb.parent_ids(conn, args.task_id) children = kb.child_ids(conn, args.task_id) runs = kb.list_runs(conn, args.task_id, **rsk) - # Workers hand off via ``task_runs.summary``; ``tasks.result`` is left NULL unless the caller explicitly passed - # ``result=``. Surfacing the latest summary here keeps ``show`` from - # looking like a no-op when the worker actually did real work. + # Workers hand off via task_runs.summary; tasks.result stays NULL unless + # explicitly set, so surface the latest summary here. latest_summary = kb.latest_summary(conn, args.task_id) if not getattr(args, "json", False): graph = kb.task_graph_context(conn, task.id) if getattr(args, "json", False): - payload = { + _print_json({ "task": _task_to_dict(task), "latest_summary": latest_summary, "parents": parents, "children": children, - "comments": [ - {"author": c.author, "body": c.body, "created_at": c.created_at} - for c in comments - ], - "events": [ - { - "kind": e.kind, - "payload": e.payload, - "created_at": e.created_at, - "run_id": e.run_id, - } - for e in events - ], - "runs": [ - { - "id": r.id, - "profile": r.profile, - "step_key": r.step_key, - "status": r.status, - "outcome": r.outcome, - "summary": r.summary, - "error": r.error, - "metadata": r.metadata, - "worker_pid": r.worker_pid, - "started_at": r.started_at, - "ended_at": r.ended_at, - } - for r in runs - ], - } - print(json.dumps(payload, indent=2, ensure_ascii=False)) + "comments": [_obj_dict(c, ("author", "body", "created_at")) for c in comments], + "events": [_obj_dict(e, ("kind", "payload", "created_at", "run_id")) for e in events], + "runs": [_obj_dict(r, _SHOW_RUN_FIELDS) for r in runs], + }) return 0 print(f"Task {task.id}: {task.title}") @@ -1862,10 +1491,8 @@ def _cmd_show(args: argparse.Namespace) -> int: if task.model_override: _prov = f" (provider: {task.provider_override})" if task.provider_override else "" print(f" model: {task.model_override}{_prov}") - # Effective retry threshold. Show the per-task override if set, - # otherwise the dispatcher's resolved value from config (or the - # default if config doesn't set it either). Helps operators see - # why a task auto-blocked earlier/later than they expected. + # Effective retry threshold: per-task override, else config, else default — + # so operators can see why a task auto-blocked when it did. if task.max_retries is not None: print(f" max-retries: {task.max_retries} (task)") else: @@ -1881,30 +1508,12 @@ def _cmd_show(args: argparse.Namespace) -> int: print(f" max-retries: {kb.DEFAULT_FAILURE_LIMIT} (default)") print(f" created: {_fmt_ts(task.created_at)} by {task.created_by or '-'}") - # Diagnostics section — surface active distress signals at the top - # of show output so CLI users see them before scrolling through - # comments / runs. + # Diagnostics up top so CLI users see distress signals before scrolling. from hermes_cli import kanban_diagnostics as kd diags = kd.compute_task_diagnostics(task, events, runs, graph=graph) if diags: - sev_marker = {"warning": "⚠", "error": "!!", "critical": "!!!"} print(f"\n Diagnostics ({len(diags)}):") - for d in diags: - print(f" {sev_marker.get(d.severity, '?')} [{d.severity}] {d.title}") - if d.data: - bits = [] - for k, v in d.data.items(): - if isinstance(v, list): - bits.append(f"{k}={','.join(str(x) for x in v)}") - else: - bits.append(f"{k}={v}") - if bits: - print(f" data: {' | '.join(bits)}") - # Only show suggested actions in show output to keep it tight; - # full list is available via `kanban diagnostics --task `. - for a in d.actions: - if a.suggested: - print(f" → {a.label}") + _print_diagnostics(diags, " ", with_kind=False) if task.started_at: print(f" started: {_fmt_ts(task.started_at)}") if task.completed_at: @@ -1922,9 +1531,6 @@ def _cmd_show(args: argparse.Namespace) -> int: print("Result:") print(task.result) elif latest_summary: - # Worker handoff lives on the latest run, not on tasks.result. - # Surface it at top-level so a glance at ``hermes kanban show `` - # tells you what the worker did even if tasks.result is empty. print() print("Latest summary:") print(latest_summary) @@ -1959,12 +1565,11 @@ def _cmd_show(args: argparse.Namespace) -> int: def _cmd_assign(args: argparse.Namespace) -> int: - profile = None if args.profile.lower() in {"none", "-", "null"} else args.profile + profile = _none_profile(args.profile) with kb.connect_closing() as conn: ok = kb.assign_task(conn, args.task_id, profile) if not ok: - print(f"no such task: {args.task_id}", file=sys.stderr) - return 1 + return _err(f"no such task: {args.task_id}") print(f"Assigned {args.task_id} to {profile or '(unassigned)'}") return 0 @@ -1978,11 +1583,9 @@ def _cmd_set_model(args: argparse.Namespace) -> int: with kb.connect_closing() as conn: ok = kb.set_model_override(conn, args.task_id, model, provider=provider) except (ValueError, RuntimeError) as exc: - print(f"kanban: {exc}", file=sys.stderr) - return 2 + return _err(f"kanban: {exc}", 2) if not ok: - print(f"no such task: {args.task_id}", file=sys.stderr) - return 1 + return _err(f"no such task: {args.task_id}") if model: label = f"{provider}:{model}" if provider else model print(f"Set model override on {args.task_id}: {label} " @@ -2000,17 +1603,13 @@ def _cmd_reclaim(args: argparse.Namespace) -> int: reason=getattr(args, "reason", None), ) if not ok: - print( - f"cannot reclaim {args.task_id} (not running or unknown id)", - file=sys.stderr, - ) - return 1 + return _err(f"cannot reclaim {args.task_id} (not running or unknown id)") print(f"Reclaimed {args.task_id}") return 0 def _cmd_reassign(args: argparse.Namespace) -> int: - profile = None if args.profile.lower() in {"none", "-", "null"} else args.profile + profile = _none_profile(args.profile) with kb.connect_closing() as conn: ok = kb.reassign_task( conn, args.task_id, profile, @@ -2018,12 +1617,10 @@ def _cmd_reassign(args: argparse.Namespace) -> int: reason=getattr(args, "reason", None), ) if not ok: - print( + return _err( f"cannot reassign {args.task_id} " - f"(unknown id, or still running — pass --reclaim to release first)", - file=sys.stderr, + f"(unknown id, or still running — pass --reclaim to release first)" ) - return 1 print( f"Reassigned {args.task_id} to " f"{profile or '(unassigned)'}" @@ -2032,10 +1629,19 @@ def _cmd_reassign(args: argparse.Namespace) -> int: return 0 +def _rows_by_task(conn, table: str, ids: list[str]) -> dict[str, list]: + """``{task_id: [rows ordered by id]}`` for every id (empty list when none).""" + by = {i: [] for i in ids} + placeholders = ",".join(["?"] * len(ids)) + for row in conn.execute( + f"SELECT * FROM {table} WHERE task_id IN ({placeholders}) ORDER BY id", tuple(ids), + ): + by.setdefault(row["task_id"], []).append(row) + return by + + def _cmd_diagnostics(args: argparse.Namespace) -> int: - """List active diagnostics on the board. Wraps the same rule engine - the dashboard uses, so CLI output matches what the UI shows. - """ + """List active diagnostics on the board via the same rule engine the dashboard uses.""" from hermes_cli import kanban_diagnostics as kd from hermes_cli.config import load_config @@ -2046,8 +1652,7 @@ def _cmd_diagnostics(args: argparse.Namespace) -> int: if getattr(args, "task", None): task = kb.get_task(conn, args.task) if task is None: - print(f"no such task: {args.task}", file=sys.stderr) - return 1 + return _err(f"no such task: {args.task}") diags_by_task = { args.task: kd.compute_task_diagnostics( task, @@ -2063,24 +1668,11 @@ def _cmd_diagnostics(args: argparse.Namespace) -> int: "SELECT * FROM tasks WHERE status != 'archived'" ).fetchall()) ids = [r["id"] for r in rows] - if not ids: - diags_by_task = {} - else: - placeholders = ",".join(["?"] * len(ids)) - ev_by = {i: [] for i in ids} - for row in conn.execute( - f"SELECT * FROM task_events WHERE task_id IN ({placeholders}) ORDER BY id", - tuple(ids), - ): - ev_by.setdefault(row["task_id"], []).append(row) - run_by = {i: [] for i in ids} - for row in conn.execute( - f"SELECT * FROM task_runs WHERE task_id IN ({placeholders}) ORDER BY id", - tuple(ids), - ): - run_by.setdefault(row["task_id"], []).append(row) + diags_by_task = {} + if ids: + ev_by = _rows_by_task(conn, "task_events", ids) + run_by = _rows_by_task(conn, "task_runs", ids) graph_by = kb.task_graph_contexts(conn, ids) - diags_by_task = {} for r in rows: tid = r["id"] dl = kd.compute_task_diagnostics( @@ -2096,12 +1688,12 @@ def _cmd_diagnostics(args: argparse.Namespace) -> int: # Severity filter. sev = getattr(args, "severity", None) if sev: - for tid in list(diags_by_task.keys()): - kept = [d for d in diags_by_task[tid] if kd.SEVERITY_ORDER.index(d.severity) >= kd.SEVERITY_ORDER.index(sev)] - if kept: - diags_by_task[tid] = kept - else: - del diags_by_task[tid] + floor = kd.SEVERITY_ORDER.index(sev) + diags_by_task = { + tid: kept + for tid, dl in diags_by_task.items() + if (kept := [d for d in dl if kd.SEVERITY_ORDER.index(d.severity) >= floor]) + } # Map task_id → title/status/assignee for the table output. meta: dict[str, dict] = {} @@ -2111,30 +1703,23 @@ def _cmd_diagnostics(args: argparse.Namespace) -> int: f"SELECT id, title, status, assignee FROM tasks WHERE id IN ({placeholders})", tuple(diags_by_task.keys()), ): - meta[r["id"]] = { - "title": r["title"], "status": r["status"], - "assignee": r["assignee"], - } + meta[r["id"]] = {k: r[k] for k in ("title", "status", "assignee")} if getattr(args, "json", False): - out_json = [ + _print_json([ { "task_id": tid, **meta.get(tid, {}), "diagnostics": [d.to_dict() for d in dl], } for tid, dl in diags_by_task.items() - ] - print(json.dumps(out_json, indent=2, ensure_ascii=False)) + ]) return 0 if not diags_by_task: print("No active diagnostics on this board.") return 0 - # Human-readable summary: grouped by task, severity-marked, with - # suggested actions inline. - sev_marker = {"warning": "⚠", "error": "!!", "critical": "!!!"} total = sum(len(dl) for dl in diags_by_task.values()) print( f"{total} active diagnostic(s) across " @@ -2146,22 +1731,7 @@ def _cmd_diagnostics(args: argparse.Namespace) -> int: status = m.get("status") or "?" assignee = m.get("assignee") or "(unassigned)" print(f" {tid} {status:8s} @{assignee:18s} {title}") - for d in dl: - print(f" {sev_marker.get(d.severity, '?')} [{d.severity}] {d.kind}: {d.title}") - if d.data: - # Compact key:value pairs on one line. - bits = [] - for k, v in d.data.items(): - if isinstance(v, list): - bits.append(f"{k}={','.join(str(x) for x in v)}") - else: - bits.append(f"{k}={v}") - if bits: - print(f" data: {' | '.join(bits)}") - # Suggested actions first. - for a in d.actions: - if a.suggested: - print(f" → {a.label}") + _print_diagnostics(dl, " ", with_kind=True) print() return 0 @@ -2177,8 +1747,7 @@ def _cmd_unlink(args: argparse.Namespace) -> int: with kb.connect_closing() as conn: ok = kb.unlink_tasks(conn, args.parent_id, args.child_id) if not ok: - print(f"No such link: {args.parent_id} -> {args.child_id}", file=sys.stderr) - return 1 + return _err(f"No such link: {args.parent_id} -> {args.child_id}") print(f"Unlinked {args.parent_id} -> {args.child_id}") return 0 @@ -2187,17 +1756,13 @@ def _cmd_claim(args: argparse.Namespace) -> int: with kb.connect_closing() as conn: task = kb.claim_task(conn, args.task_id, ttl_seconds=args.ttl) if task is None: - # Report why existing = kb.get_task(conn, args.task_id) if existing is None: - print(f"no such task: {args.task_id}", file=sys.stderr) - return 1 - print( + return _err(f"no such task: {args.task_id}") + return _err( f"cannot claim {args.task_id}: status={existing.status} " - f"lock={existing.claim_lock or '(none)'}", - file=sys.stderr, + f"lock={existing.claim_lock or '(none)'}" ) - return 1 workspace = kb.resolve_workspace(task) kb.set_workspace_path(conn, task.id, str(workspace)) print(f"Claimed {task.id}") @@ -2209,8 +1774,7 @@ def _cmd_comment(args: argparse.Namespace) -> int: body = " ".join(args.text).strip() if args.max_len is not None: if args.max_len < 1: - print("kanban: --max-len must be positive", file=sys.stderr) - return 2 + return _err("kanban: --max-len must be positive", 2) if len(body) > args.max_len: suffix = f"\n\n[trimmed to {args.max_len} chars by --max-len]" body = body[: max(0, args.max_len - len(suffix))].rstrip() + suffix @@ -2222,19 +1786,13 @@ def _cmd_comment(args: argparse.Namespace) -> int: def _cmd_attach(args: argparse.Namespace) -> int: - """Attach a local file to a task. - - Reads the file off disk, writes it under the task's attachments dir, - and records the metadata row via the shared ``store_attachment_bytes`` - path (same code the dashboard upload and the agent tool use), so the - 25 MB cap and name-sanitisation behave identically everywhere. - """ + """Attach a local file via the shared ``store_attachment_bytes`` path (same + 25 MB cap and name sanitisation as the dashboard upload and agent tool).""" import mimetypes src = Path(args.path).expanduser() if not src.is_file(): - print(f"kanban: no such file: {src}", file=sys.stderr) - return 1 + return _err(f"kanban: no such file: {src}") data = src.read_bytes() name = args.name or src.name content_type = args.content_type or mimetypes.guess_type(name)[0] @@ -2250,32 +1808,17 @@ def _cmd_attach(args: argparse.Namespace) -> int: uploaded_by=uploaded_by, ) except kb.AttachmentTooLarge as exc: - print(f"kanban: {exc}", file=sys.stderr) - return 1 + return _err(f"kanban: {exc}") print(f"Attached {name} to {args.task_id} (attachment {att_id}, {len(data)} bytes)") return 0 def _cmd_attachments(args: argparse.Namespace) -> int: - """List a task's attachments.""" with kb.connect_closing() as conn: if kb.get_task(conn, args.task_id) is None: - print(f"no such task: {args.task_id}", file=sys.stderr) - return 1 + return _err(f"no such task: {args.task_id}") atts = kb.list_attachments(conn, args.task_id) - if getattr(args, "json", False): - print(json.dumps([ - { - "id": a.id, - "filename": a.filename, - "content_type": a.content_type, - "size": a.size, - "uploaded_by": a.uploaded_by, - "stored_path": a.stored_path, - "created_at": a.created_at, - } - for a in atts - ], indent=2)) + if _json_out(args, [_obj_dict(a, _ATTACHMENT_FIELDS) for a in atts], ascii=True): return 0 if not atts: print(f"No attachments on {args.task_id}") @@ -2289,12 +1832,10 @@ def _cmd_attachments(args: argparse.Namespace) -> int: def _cmd_attach_rm(args: argparse.Namespace) -> int: - """Delete an attachment by id (removes the row and the on-disk blob).""" with kb.connect_closing() as conn: removed = kb.delete_attachment(conn, args.attachment_id) if removed is None: - print(f"no such attachment: {args.attachment_id}", file=sys.stderr) - return 1 + return _err(f"no such attachment: {args.attachment_id}") print(f"Deleted attachment {args.attachment_id} ({removed.filename}) from {removed.task_id}") return 0 @@ -2353,85 +1894,60 @@ def _cmd_complete(args: argparse.Namespace) -> int: """Mark one or more tasks done. Supports a single id or a list.""" ids = list(args.task_ids or []) if not ids: - print("at least one task_id is required", file=sys.stderr) - return 1 + return _err("at least one task_id is required") summary = getattr(args, "summary", None) raw_meta = getattr(args, "metadata", None) - # Guard: structured handoff fields are per-run, so they'd be - # copy-pasted identically across N runs — almost always a footgun. - # Refuse instead of silently doing the wrong thing. + # Structured handoff fields are per-run; copying them across N runs is + # almost always a footgun, so refuse rather than silently do it. if len(ids) > 1 and (summary or raw_meta): - print( + return _err( "kanban: --summary / --metadata are per-task and can't be used " "with multiple ids (would apply the same handoff to every task). " "Complete tasks one at a time, or drop the flags for the bulk close.", - file=sys.stderr, + 2, ) - return 2 - metadata = None - if raw_meta: - try: - metadata = json.loads(raw_meta) - if not isinstance(metadata, dict): - raise ValueError("must be a JSON object") - except (ValueError, json.JSONDecodeError) as exc: - print(f"kanban: --metadata: {exc}", file=sys.stderr) - return 2 - failed: list[str] = [] + metadata, rc = _parse_metadata_flag(raw_meta) + if rc: + return rc + fail_msg: dict[str, str] = {} with kb.connect_closing() as conn: - for tid in ids: - # Goal-mode judge gate (mirrors tools/kanban_tools.py). Apply it - # to every terminal handoff so request-review cannot bypass the - # acceptance contract that protects complete. - task = kb.get_task(conn, tid) + def op(tid): + # Goal-mode judge gate (mirrors tools/kanban_tools.py); applied to + # every terminal handoff so request-review can't bypass it. gate_verdict, rejection = _goal_mode_handoff_rejection( - task, + kb.get_task(conn, tid), (summary or args.result or "").strip(), ) if gate_verdict == "blocked": - print( + fail_msg[tid] = ( f"kanban: goal completion of {tid} rejected: judge ruled " f"the goal unachievable — {rejection}. Re-scope with " f"kanban edit, or record the block with kanban block " - f"instead of completing.", - file=sys.stderr, + f"instead of completing." ) - failed.append(tid) - continue + return False if rejection is not None: - print( + fail_msg[tid] = ( f"kanban: goal completion of {tid} rejected by judge: {rejection}. " - f"Provide evidence matching the task's acceptance criteria.", - file=sys.stderr, + f"Provide evidence matching the task's acceptance criteria." ) - failed.append(tid) - continue - - if not kb.complete_task( + return False + fail_msg[tid] = f"cannot complete {tid} (unknown id or terminal state)" + return kb.complete_task( conn, tid, result=args.result, summary=summary, metadata=metadata, expected_run_id=_worker_run_id_for(tid), - ): - failed.append(tid) - print(f"cannot complete {tid} (unknown id or terminal state)", file=sys.stderr) - else: - print(f"Completed {tid}") - return 0 if not failed else 1 + ) + + return _bulk_apply(ids, op, lambda tid: f"Completed {tid}", fail_msg.__getitem__) def _cmd_edit(args: argparse.Namespace) -> int: - raw_meta = getattr(args, "metadata", None) - metadata = None - if raw_meta: - try: - metadata = json.loads(raw_meta) - if not isinstance(metadata, dict): - raise ValueError("must be a JSON object") - except (ValueError, json.JSONDecodeError) as exc: - print(f"kanban: --metadata: {exc}", file=sys.stderr) - return 2 + metadata, rc = _parse_metadata_flag(getattr(args, "metadata", None)) + if rc: + return rc with kb.connect_closing() as conn: if not kb.edit_completed_task_result( conn, @@ -2440,11 +1956,7 @@ def _cmd_edit(args: argparse.Namespace) -> int: summary=getattr(args, "summary", None), metadata=metadata, ): - print( - f"cannot edit {args.task_id} (unknown id or task is not done)", - file=sys.stderr, - ) - return 1 + return _err(f"cannot edit {args.task_id} (unknown id or task is not done)") print(f"Edited {args.task_id}") return 0 @@ -2454,80 +1966,68 @@ def _cmd_block(args: argparse.Namespace) -> int: kind = getattr(args, "kind", None) author = _profile_author() ids = [args.task_id] + list(getattr(args, "ids", None) or []) - failed: list[str] = [] + suffix = f": {reason}" if reason else "" with kb.connect_closing() as conn: - for tid in ids: + def op(tid): if reason: kb.add_comment(conn, tid, author, f"BLOCKED: {reason}") - if not kb.block_task( - conn, - tid, - reason=reason, - kind=kind, + return kb.block_task( + conn, tid, reason=reason, kind=kind, expected_run_id=_worker_run_id_for(tid), - ): - failed.append(tid) - print(f"cannot block {tid}", file=sys.stderr) - else: - # Report where the task actually landed — dependency blocks go - # to todo, and a tripped unblock-loop breaker routes to triage. - landed = kb.get_task(conn, tid) - where = landed.status if landed else "blocked" - suffix = f": {reason}" if reason else "" - if where == "todo": - print(f"{tid} → todo (dependency wait){suffix}") - elif where == "triage": - print( - f"{tid} → triage (unblock loop detected — needs a " - f"human decision){suffix}" - ) - else: - print(f"Blocked {tid}{suffix}") - return 0 if not failed else 1 + ) + + def ok_msg(tid): + # Report where the task actually landed — dependency blocks go + # to todo, and a tripped unblock-loop breaker routes to triage. + landed = kb.get_task(conn, tid) + where = landed.status if landed else "blocked" + if where == "todo": + return f"{tid} → todo (dependency wait){suffix}" + if where == "triage": + return (f"{tid} → triage (unblock loop detected — needs a " + f"human decision){suffix}") + return f"Blocked {tid}{suffix}" + + return _bulk_apply(ids, op, ok_msg, lambda tid: f"cannot block {tid}") def _cmd_schedule(args: argparse.Namespace) -> int: reason = " ".join(args.reason).strip() if args.reason else None author = _profile_author() ids = [args.task_id] + list(getattr(args, "ids", None) or []) - failed: list[str] = [] + suffix = f": {reason}" if reason else "" with kb.connect_closing() as conn: - for tid in ids: + def op(tid): if reason: kb.add_comment(conn, tid, author, f"SCHEDULED: {reason}") - if not kb.schedule_task( - conn, - tid, - reason=reason, - expected_run_id=_worker_run_id_for(tid), - ): - failed.append(tid) - print(f"cannot schedule {tid}", file=sys.stderr) - else: - print(f"Scheduled {tid}" + (f": {reason}" if reason else "")) - return 0 if not failed else 1 + return kb.schedule_task( + conn, tid, reason=reason, expected_run_id=_worker_run_id_for(tid), + ) + + return _bulk_apply( + ids, op, lambda tid: f"Scheduled {tid}{suffix}", lambda tid: f"cannot schedule {tid}", + ) def _cmd_unblock(args: argparse.Namespace) -> int: ids = list(args.task_ids or []) if not ids: - print("at least one task_id is required", file=sys.stderr) - return 1 + return _err("at least one task_id is required") reason = getattr(args, "reason", None) if reason is not None: reason = reason.strip() or None author = _profile_author() if reason else None - failed: list[str] = [] + suffix = f": {reason}" if reason else "" with kb.connect_closing() as conn: - for tid in ids: + def op(tid): if reason: kb.add_comment(conn, tid, author, f"UNBLOCK: {reason}") - if not kb.unblock_task(conn, tid): - failed.append(tid) - print(f"cannot unblock {tid} (not blocked/scheduled?)", file=sys.stderr) - else: - print(f"Unblocked {tid}" + (f": {reason}" if reason else "")) - return 0 if not failed else 1 + return kb.unblock_task(conn, tid) + + return _bulk_apply( + ids, op, lambda tid: f"Unblocked {tid}{suffix}", + lambda tid: f"cannot unblock {tid} (not blocked/scheduled?)", + ) def _cmd_request_review(args: argparse.Namespace) -> int: @@ -2535,16 +2035,9 @@ def _cmd_request_review(args: argparse.Namespace) -> int: summary = getattr(args, "summary", None) if summary is not None: summary = summary.strip() or None - raw_metadata = getattr(args, "metadata", None) - metadata = None - if raw_metadata: - try: - metadata = json.loads(raw_metadata) - if not isinstance(metadata, dict): - raise ValueError("must be a JSON object") - except (ValueError, json.JSONDecodeError) as exc: - print(f"kanban: --metadata: {exc}", file=sys.stderr) - return 2 + metadata, rc = _parse_metadata_flag(getattr(args, "metadata", None)) + if rc: + return rc reviewer = getattr(args, "reviewer", None) with kb.connect_closing() as conn: gate_verdict, rejection = _goal_mode_handoff_rejection( @@ -2552,20 +2045,16 @@ def _cmd_request_review(args: argparse.Namespace) -> int: summary or "", ) if gate_verdict == "blocked": - print( + return _err( f"kanban: goal review handoff of {tid} rejected: judge ruled " f"the goal unachievable — {rejection}. Record the block with " - f"kanban block instead of requesting review.", - file=sys.stderr, + f"kanban block instead of requesting review." ) - return 1 if rejection is not None: - print( + return _err( f"kanban: goal review handoff of {tid} rejected by judge: " - f"{rejection}. Provide acceptance evidence matching the task.", - file=sys.stderr, + f"{rejection}. Provide acceptance evidence matching the task." ) - return 1 ok, reason = kb.request_review( conn, tid, @@ -2577,12 +2066,7 @@ def _cmd_request_review(args: argparse.Namespace) -> int: with_reason=True, ) if not ok: - detail = reason or "not running/ready?" - print( - f"cannot request review for {tid}: {detail}", - file=sys.stderr, - ) - return 1 + return _err(f"cannot request review for {tid}: {reason or 'not running/ready?'}") persisted_run = kb.latest_run(conn, tid) display_summary = persisted_run.summary if persisted_run else None print( @@ -2603,11 +2087,7 @@ def _cmd_request_changes(args: argparse.Namespace) -> int: expected_run_id=_worker_run_id_for(tid), ) if not ok: - print( - f"cannot request changes for {tid}: {detail or 'invalid review state'}", - file=sys.stderr, - ) - return 1 + return _err(f"cannot request changes for {tid}: {detail or 'invalid review state'}") print( f"Requested changes for {tid}" + (f"; routed to {detail}" if detail else "") @@ -2618,42 +2098,31 @@ def _cmd_request_changes(args: argparse.Namespace) -> int: def _cmd_reopen_review(args: argparse.Namespace) -> int: ids = list(args.task_ids or []) if not ids: - print("at least one task_id is required", file=sys.stderr) - return 1 + return _err("at least one task_id is required") reason = getattr(args, "reason", None) if reason is not None: reason = str(kb.redact_review_value(reason.strip())).strip() or None author = _profile_author() if reason else None - failed: list[str] = [] + suffix = f": {reason}" if reason else "" with kb.connect_closing() as conn: - for tid in ids: + def op(tid): if not kb.reopen_review_task(conn, tid): - failed.append(tid) - print(f"cannot reopen {tid} (not in review?)", file=sys.stderr) - else: - if reason: - kb.add_comment( - conn, - tid, - author or "operator", - f"CHANGES REQUESTED: {reason}", - ) - print(f"Reopened {tid}" + (f": {reason}" if reason else "")) - return 0 if not failed else 1 + return False + if reason: + kb.add_comment(conn, tid, author or "operator", f"CHANGES REQUESTED: {reason}") + return True + + return _bulk_apply( + ids, op, lambda tid: f"Reopened {tid}{suffix}", + lambda tid: f"cannot reopen {tid} (not in review?)", + ) def _cmd_promote(args: argparse.Namespace) -> int: reason = " ".join(args.reason).strip() if args.reason else None author = _profile_author() - as_json = getattr(args, "json", False) - extra_ids = list(getattr(args, "ids", None) or []) # Dedupe while preserving order; positional task_id always first. - ids: list[str] = [] - seen: set[str] = set() - for tid in [args.task_id, *extra_ids]: - if tid not in seen: - ids.append(tid) - seen.add(tid) + ids = list(dict.fromkeys([args.task_id, *(getattr(args, "ids", None) or [])])) results: list[dict[str, object]] = [] with kb.connect_closing() as conn: @@ -2676,10 +2145,9 @@ def _cmd_promote(args: argparse.Namespace) -> int: }) failed = [r for r in results if not r["promoted"]] - if as_json: + if getattr(args, "json", False): # Single-id stays a flat object for back-compat; bulk emits a list. - payload: object = results[0] if len(results) == 1 else results - print(json.dumps(payload, indent=2, ensure_ascii=False)) + _print_json(results[0] if len(results) == 1 else results) return 0 if not failed else 1 tag = " (dry)" if args.dry_run else "" @@ -2697,28 +2165,20 @@ def _cmd_archive(args: argparse.Namespace) -> int: ids = list(args.task_ids or []) purge_ids = list(getattr(args, "purge_ids", None) or []) if ids and purge_ids: - print("choose either task_ids to archive or --rm archived task_ids", file=sys.stderr) - return 1 + return _err("choose either task_ids to archive or --rm archived task_ids") if not ids and not purge_ids: - print("at least one task_id is required", file=sys.stderr) - return 1 - failed: list[str] = [] + return _err("at least one task_id is required") with kb.connect_closing() as conn: if purge_ids: - for tid in purge_ids: - if not kb.delete_archived_task(conn, tid): - failed.append(tid) - print(f"cannot delete {tid} (must already be archived)", file=sys.stderr) - else: - print(f"Deleted {tid}") - return 0 if not failed else 1 - for tid in ids: - if not kb.archive_task(conn, tid): - failed.append(tid) - print(f"cannot archive {tid}", file=sys.stderr) - else: - print(f"Archived {tid}") - return 0 if not failed else 1 + return _bulk_apply( + purge_ids, lambda tid: kb.delete_archived_task(conn, tid), + lambda tid: f"Deleted {tid}", + lambda tid: f"cannot delete {tid} (must already be archived)", + ) + return _bulk_apply( + ids, lambda tid: kb.archive_task(conn, tid), + lambda tid: f"Archived {tid}", lambda tid: f"cannot archive {tid}", + ) def _cmd_tail(args: argparse.Namespace) -> int: @@ -2739,39 +2199,33 @@ def _cmd_tail(args: argparse.Namespace) -> int: return 0 +def _coerce_positive_int(value): + if value is None: + return None + try: + ival = int(value) + except (TypeError, ValueError): + return None + return ival if ival >= 1 else None + + def _cmd_dispatch(args: argparse.Namespace) -> int: - # Honour kanban.default_assignee as the fallback for unassigned ready - # tasks (#27145), kanban.max_in_progress as the global concurrency cap - # (#33488), kanban.max_in_progress_per_profile as the per-profile - # cap (#21582), and kanban.max_spawn as the per-tick spawn limit - # (#28805). Same semantics as the gateway dispatch path so behavior - # matches whether the user runs the CLI directly or relies on the - # gateway-embedded dispatcher. + # Honour kanban.default_assignee, kanban.max_in_progress, + # kanban.max_in_progress_per_profile and kanban.max_spawn with the same + # semantics as the gateway dispatch path. try: from hermes_cli.config import load_config _cfg = load_config() _kanban_cfg = _cfg.get("kanban", {}) if isinstance(_cfg, dict) else {} default_assignee = (_kanban_cfg.get("default_assignee") or "").strip() or None - - def _coerce_positive_int(value): - if value is None: - return None - try: - ival = int(value) - except (TypeError, ValueError): - return None - return ival if ival >= 1 else None - max_in_progress_per_profile = _coerce_positive_int( _kanban_cfg.get("max_in_progress_per_profile") ) - max_in_progress = _coerce_positive_int(_kanban_cfg.get("max_in_progress")) - # Memory-derived default when unset (OOF-30/OOF-77) — same - # fallback the gateway-embedded dispatcher applies, so behaviour - # matches regardless of which path runs the tick. - max_in_progress = kb.resolve_max_in_progress(max_in_progress) - # CLI --max overrides config kanban.max_spawn when both are present; - # CLI is the more explicit signal so it wins. + # Memory-derived default when unset — same fallback the gateway applies. + max_in_progress = kb.resolve_max_in_progress( + _coerce_positive_int(_kanban_cfg.get("max_in_progress")) + ) + # CLI --max is the more explicit signal, so it wins over kanban.max_spawn. cli_max = getattr(args, "max", None) max_spawn = cli_max if cli_max is not None else _coerce_positive_int( _kanban_cfg.get("max_spawn") @@ -2792,7 +2246,7 @@ def _cmd_dispatch(args: argparse.Namespace) -> int: max_in_progress_per_profile=max_in_progress_per_profile, ) if getattr(args, "json", False): - print(json.dumps({ + _print_json({ "reclaimed": res.reclaimed, "crashed": res.crashed, "timed_out": res.timed_out, @@ -2810,25 +2264,22 @@ def _cmd_dispatch(args: argparse.Namespace) -> int: for (tid, who, current) in res.skipped_per_profile_capped ], "auto_assigned_default": res.auto_assigned_default, - }, indent=2)) + }, ascii=True) return 0 print(f"Reclaimed: {res.reclaimed}") - print(f"Crashed: {len(res.crashed)}") - if res.crashed: - print(f" {', '.join(res.crashed)}") - print(f"Timed out: {len(res.timed_out)}") - if res.timed_out: - print(f" {', '.join(res.timed_out)}") - print(f"Stale: {len(res.stale)}") - if res.stale: - print(f" {', '.join(res.stale)}") - print(f"Auto-blocked: {len(res.auto_blocked)}") - if res.auto_blocked: - print(f" {', '.join(res.auto_blocked)}") + for label, items in ( + ("Crashed: ", res.crashed), + ("Timed out: ", res.timed_out), + ("Stale: ", res.stale), + ("Auto-blocked:", res.auto_blocked), + ): + print(f"{label} {len(items)}") + if items: + print(f" {', '.join(items)}") print(f"Promoted: {res.promoted}") print(f"Spawned: {len(res.spawned)}") + tag = " (dry)" if args.dry_run else "" for tid, who, ws in res.spawned: - tag = " (dry)" if args.dry_run else "" print(f" - {tid} -> {who} @ {ws or '-'}{tag}") if res.auto_assigned_default: print( @@ -2837,11 +2288,8 @@ def _cmd_dispatch(args: argparse.Namespace) -> int: ) if res.skipped_unassigned: print(f"Skipped (unassigned): {', '.join(res.skipped_unassigned)}") - if res.skipped_per_profile_capped: - for tid, who, current in res.skipped_per_profile_capped: - print( - f"Deferred ({who} at per-profile cap, {current} running): {tid}" - ) + for tid, who, current in res.skipped_per_profile_capped: + print(f"Deferred ({who} at per-profile cap, {current} running): {tid}") if res.skipped_nonspawnable: print( f"Skipped (non-spawnable assignee — terminal lane, OK): " @@ -2853,20 +2301,13 @@ def _cmd_dispatch(args: argparse.Namespace) -> int: def _cmd_daemon(args: argparse.Namespace) -> int: """Deprecated — the dispatcher now runs inside the gateway. - Left in as a stub so users with the old command in scripts/systemd - units get a clear migration message instead of a cryptic - "no such command" error. A ``--force`` escape hatch keeps the old - standalone daemon alive for the rare edge case where someone truly - cannot run the gateway (e.g. running on a host that forbids - long-lived background services), but the default path exits 2 - with guidance so nobody accidentally keeps running two dispatchers - against the same kanban.db. + Kept as a stub so old scripts/systemd units get a clear migration message. + ``--force`` (hidden from --help) keeps the standalone loop for hosts that + truly cannot run the gateway; the default path exits 2 so nobody + accidentally runs two dispatchers against the same kanban.db. """ - # --force lets power users keep the standalone loop for one more - # release cycle. Undocumented in `--help` so nobody discovers it - # casually — intentional. if not getattr(args, "force", False): - print( + return _err( "hermes kanban daemon: DEPRECATED — the dispatcher now runs\n" "inside the gateway. To use kanban:\n" "\n" @@ -2883,13 +2324,11 @@ def _cmd_daemon(args: argparse.Namespace) -> int: "Running both the gateway AND this standalone daemon will\n" "race for claims. If you truly need the old standalone\n" "daemon (no gateway available), rerun with --force.", - file=sys.stderr, + 2, ) - return 2 - # Legacy path — same logic as before, kept behind --force. - # Make sure the DB exists before printing "started" so the user sees the - # correct DB path and any init error surfaces immediately. + # Init before printing "started" so the DB path is right and init errors + # surface immediately. kb.init_db() pidfile = getattr(args, "pidfile", None) @@ -2910,11 +2349,9 @@ def _cmd_daemon(args: argparse.Namespace) -> int: file=sys.stderr, ) - # Health telemetry: warn when every tick finds ready work but fails to - # spawn any worker. Catches broken profiles, PATH drift, missing venv, - # credential loss — cases where the per-task circuit breaker auto-blocks - # each task quietly but the operator has no signal that the dispatcher - # itself is dysfunctional. + # Health telemetry: warn when every tick finds ready work but spawns + # nothing (broken profile, PATH drift, missing venv, credential loss) — + # the per-task breaker auto-blocks quietly, so the operator needs a signal. HEALTH_WINDOW = 6 # ticks (default 30s at interval=5) health_state = {"bad_ticks": 0, "last_warn_at": 0} @@ -2925,11 +2362,9 @@ def _cmd_daemon(args: argparse.Namespace) -> int: health_state["bad_ticks"] += 1 else: health_state["bad_ticks"] = 0 - # Emit a warning once per HEALTH_WINDOW bad ticks (not every tick) - # so log volume stays bounded while the problem persists. + # Warn once per HEALTH_WINDOW bad ticks, at most every 5 minutes. if health_state["bad_ticks"] >= HEALTH_WINDOW: now = int(time.time()) - # Rate-limit repeats: at most one warning per 5 minutes. if now - health_state["last_warn_at"] >= 300: print( f"[{_fmt_ts(now)}] WARN dispatcher stuck: " @@ -2959,15 +2394,8 @@ def _cmd_daemon(args: argparse.Namespace) -> int: ) def _ready_queue_nonempty() -> bool: - """Cheap probe — is there at least one ready+assigned+unclaimed - task whose assignee maps to a real Hermes profile (i.e. one the - dispatcher would actually try to spawn for)? - - Filters out tasks assigned to control-plane lanes - (e.g. ``orion-cc``, ``orion-research``) that are pulled by - terminals via ``claim_task`` directly — those are correctly idle - from the dispatcher's perspective, not stuck. - """ + """Is there a ready+assigned+unclaimed task the dispatcher would spawn for? + Control-plane lanes pulled via ``claim_task`` are correctly idle, not stuck.""" try: with kb.connect_closing() as conn: return kb.has_spawnable_ready(conn) @@ -2997,7 +2425,6 @@ def _cmd_watch(args: argparse.Namespace) -> int: {k.strip() for k in args.kinds.split(",") if k.strip()} if args.kinds else None ) - cursor = 0 print("Watching kanban events. Ctrl-C to stop.", flush=True) # Seed cursor at the latest id so we don't replay history. with kb.connect_closing() as conn: @@ -3043,8 +2470,7 @@ def _cmd_watch(args: argparse.Namespace) -> int: def _cmd_stats(args: argparse.Namespace) -> int: with kb.connect_closing() as conn: stats = kb.board_stats(conn) - if getattr(args, "json", False): - print(json.dumps(stats, indent=2, ensure_ascii=False)) + if _json_out(args, stats): return 0 print("By status:") for k in ("triage", "todo", "scheduled", "ready", "running", "blocked", "done"): @@ -3052,8 +2478,7 @@ def _cmd_stats(args: argparse.Namespace) -> int: if stats["by_assignee"]: print("\nBy assignee:") for who, counts in sorted(stats["by_assignee"].items()): - parts = ", ".join(f"{k}={v}" for k, v in sorted(counts.items())) - print(f" {who:20s} {parts}") + print(f" {who:20s} {_fmt_counts(counts)}") age = stats["oldest_ready_age_seconds"] if age is not None: print(f"\nOldest ready task age: {int(age)}s") @@ -3063,8 +2488,7 @@ def _cmd_stats(args: argparse.Namespace) -> int: def _cmd_notify_subscribe(args: argparse.Namespace) -> int: with kb.connect_closing() as conn: if kb.get_task(conn, args.task_id) is None: - print(f"no such task: {args.task_id}", file=sys.stderr) - return 1 + return _err(f"no such task: {args.task_id}") kb.add_notify_sub( conn, task_id=args.task_id, platform=args.platform, chat_id=args.chat_id, @@ -3083,8 +2507,7 @@ def _cmd_notify_subscribe(args: argparse.Namespace) -> int: def _cmd_notify_list(args: argparse.Namespace) -> int: with kb.connect_closing() as conn: subs = kb.list_notify_subs(conn, args.task_id) - if getattr(args, "json", False): - print(json.dumps(subs, indent=2, ensure_ascii=False)) + if _json_out(args, subs): return 0 if not subs: print("(no subscriptions)") @@ -3110,8 +2533,7 @@ def _cmd_notify_unsubscribe(args: argparse.Namespace) -> int: thread_id=args.thread_id, ) if not ok: - print("(no such subscription)", file=sys.stderr) - return 1 + return _err("(no such subscription)") print(f"Unsubscribed from {args.task_id}") return 0 @@ -3119,9 +2541,7 @@ def _cmd_notify_unsubscribe(args: argparse.Namespace) -> int: def _cmd_log(args: argparse.Namespace) -> int: content = kb.read_worker_log(args.task_id, tail_bytes=args.tail) if content is None: - print(f"(no log for {args.task_id} — task may not have spawned yet)", - file=sys.stderr) - return 1 + return _err(f"(no log for {args.task_id} — task may not have spawned yet)") sys.stdout.write(content) if not content.endswith("\n"): sys.stdout.write("\n") @@ -3132,23 +2552,10 @@ def _cmd_runs(args: argparse.Namespace) -> int: """Show attempt history for a task.""" rsk = _run_state_kwargs(args) if rsk is None: - print( - "kanban runs: pass both --state-type and --state-name, or omit both", - file=sys.stderr, - ) - return 2 + return _err("kanban runs: pass both --state-type and --state-name, or omit both", 2) with kb.connect_closing() as conn: runs = kb.list_runs(conn, args.task_id, **rsk) - if getattr(args, "json", False): - print(json.dumps([ - { - "id": r.id, "profile": r.profile, "status": r.status, - "outcome": r.outcome, "started_at": r.started_at, - "ended_at": r.ended_at, "summary": r.summary, - "error": r.error, "metadata": r.metadata, - "worker_pid": r.worker_pid, "step_key": r.step_key, - } for r in runs - ], indent=2, ensure_ascii=False)) + if _json_out(args, [_obj_dict(r, _RUNS_RUN_FIELDS) for r in runs]): return 0 if not runs: print(f"(no runs yet for {args.task_id})") @@ -3167,9 +2574,7 @@ def _cmd_runs(args: argparse.Namespace) -> int: outcome = r.outcome or ("(running)" if not r.ended_at else r.status) print(f"{i:3d} {outcome:12s} {(r.profile or '-'):16s} {el:>8s} {_fmt_ts(r.started_at)}") if r.summary: - # Indent and truncate long summaries to keep the table readable. - summary = r.summary.splitlines()[0][:100] - print(f" → {summary}") + print(f" → {r.summary.splitlines()[0][:100]}") if r.error: print(f" ✖ {r.error[:100]}") return 0 @@ -3182,72 +2587,51 @@ def _cmd_context(args: argparse.Namespace) -> int: return 0 -def _cmd_specify(args: argparse.Namespace) -> int: - """Flesh out a triage task (or all of them) via auxiliary LLM, - then promote to todo. Thin wrapper over ``kanban_specify``.""" - from hermes_cli import kanban_specify as spec +def _triage_sweep_ids(args: argparse.Namespace, verb: str, list_triage_ids, json_key: str): + """Shared arg validation for ``specify`` / ``decompose``: ``(ids|None, rc)``. + ``ids is None`` with ``rc == 0`` means "nothing to do, already reported". + """ all_flag = bool(getattr(args, "all_triage", False)) tenant = getattr(args, "tenant", None) + if args.task_id and all_flag: + return None, _err("kanban: pass either a task id OR --all, not both", 2) + if all_flag: + ids = list_triage_ids(tenant=tenant) + if not ids: + if getattr(args, "json", False): + print(json.dumps({json_key: 0, "total": 0})) + else: + print("No triage tasks" + (f" for tenant {tenant!r}" if tenant else "") + ".") + return None, 0 + return ids, 0 + if args.task_id: + return [args.task_id], 0 + return None, _err(f"kanban: {verb} requires a task id or --all", 2) + + +def _run_triage_sweep(args: argparse.Namespace, verb: str, mod, run_one, json_key: str, + json_fields: tuple[str, ...], human_ok) -> int: + """Shared driver for ``specify`` / ``decompose``: validate ids, run + ``run_one(tid, author=...)`` per id, print JSON or human lines, exit code.""" + all_flag = bool(getattr(args, "all_triage", False)) author = getattr(args, "author", None) or _profile_author() want_json = bool(getattr(args, "json", False)) - - if args.task_id and all_flag: - print( - "kanban: pass either a task id OR --all, not both", - file=sys.stderr, - ) - return 2 - - if all_flag: - ids = spec.list_triage_ids(tenant=tenant) - if not ids: - msg = ( - "No triage tasks" - + (f" for tenant {tenant!r}" if tenant else "") - + "." - ) - if want_json: - print(json.dumps({"specified": 0, "total": 0})) - else: - print(msg) - return 0 - elif args.task_id: - ids = [args.task_id] - else: - print( - "kanban: specify requires a task id or --all", - file=sys.stderr, - ) - return 2 + ids, rc = _triage_sweep_ids(args, verb, mod.list_triage_ids, json_key) + if ids is None: + return rc ok_count = 0 - fail_count = 0 for tid in ids: - outcome = spec.specify_task(tid, author=author) + outcome = run_one(tid, author=author) if outcome.ok: ok_count += 1 - else: - fail_count += 1 if want_json: - print(json.dumps({ - "task_id": outcome.task_id, - "ok": outcome.ok, - "reason": outcome.reason, - "new_title": outcome.new_title, - })) + print(json.dumps(_obj_dict(outcome, json_fields))) elif outcome.ok: - title_suffix = ( - f" — retitled: {outcome.new_title!r}" - if outcome.new_title - else "" - ) - print(f"Specified {outcome.task_id} → todo{title_suffix}") + print(human_ok(outcome)) else: - print( - f"kanban: specify {outcome.task_id}: {outcome.reason}", - file=sys.stderr, - ) + print(f"kanban: {verb} {outcome.task_id}: {outcome.reason}", file=sys.stderr) if not all_flag: return 0 if ok_count == 1 else 1 # --all: succeed if at least one promotion landed; exit 1 only when @@ -3255,90 +2639,43 @@ def _cmd_specify(args: argparse.Namespace) -> int: return 0 if (ok_count > 0 or not ids) else 1 +def _retitled_suffix(outcome) -> str: + return f" — retitled: {outcome.new_title!r}" if outcome.new_title else "" + + +def _cmd_specify(args: argparse.Namespace) -> int: + """Flesh out a triage task (or all of them) via auxiliary LLM, then + promote to todo. Thin wrapper over ``kanban_specify``.""" + from hermes_cli import kanban_specify as spec + + return _run_triage_sweep( + args, "specify", spec, spec.specify_task, "specified", + ("task_id", "ok", "reason", "new_title"), + lambda o: f"Specified {o.task_id} → todo{_retitled_suffix(o)}", + ) + + +def _decompose_ok_line(o) -> str: + if o.fanout and o.child_ids: + return (f"Decomposed {o.task_id} → {len(o.child_ids)} " + f"children ({', '.join(o.child_ids)}); root promoted to todo") + return f"Specified {o.task_id} → todo (no fanout){_retitled_suffix(o)}" + + def _cmd_decompose(args: argparse.Namespace) -> int: - """Fan a triage task (or all of them) out into a graph of child - tasks via the auxiliary LLM, routed to specialist profiles by - description. Thin wrapper over ``kanban_decompose``.""" + """Fan a triage task (or all of them) out into a graph of child tasks via + the auxiliary LLM. Thin wrapper over ``kanban_decompose``.""" from hermes_cli import kanban_decompose as decomp - all_flag = bool(getattr(args, "all_triage", False)) - tenant = getattr(args, "tenant", None) - author = getattr(args, "author", None) or _profile_author() - want_json = bool(getattr(args, "json", False)) - - if args.task_id and all_flag: - print( - "kanban: pass either a task id OR --all, not both", - file=sys.stderr, - ) - return 2 - - if all_flag: - ids = decomp.list_triage_ids(tenant=tenant) - if not ids: - msg = ( - "No triage tasks" - + (f" for tenant {tenant!r}" if tenant else "") - + "." - ) - if want_json: - print(json.dumps({"decomposed": 0, "total": 0})) - else: - print(msg) - return 0 - elif args.task_id: - ids = [args.task_id] - else: - print( - "kanban: decompose requires a task id or --all", - file=sys.stderr, - ) - return 2 - - ok_count = 0 - for tid in ids: - outcome = decomp.decompose_task(tid, author=author) - if outcome.ok: - ok_count += 1 - if want_json: - print(json.dumps({ - "task_id": outcome.task_id, - "ok": outcome.ok, - "reason": outcome.reason, - "fanout": outcome.fanout, - "child_ids": outcome.child_ids, - "new_title": outcome.new_title, - })) - elif outcome.ok: - if outcome.fanout and outcome.child_ids: - child_summary = ", ".join(outcome.child_ids) - print( - f"Decomposed {outcome.task_id} → {len(outcome.child_ids)} " - f"children ({child_summary}); root promoted to todo" - ) - else: - title_suffix = ( - f" — retitled: {outcome.new_title!r}" - if outcome.new_title - else "" - ) - print( - f"Specified {outcome.task_id} → todo " - f"(no fanout){title_suffix}" - ) - else: - print( - f"kanban: decompose {outcome.task_id}: {outcome.reason}", - file=sys.stderr, - ) - if not all_flag: - return 0 if ok_count == 1 else 1 - return 0 if (ok_count > 0 or not ids) else 1 + return _run_triage_sweep( + args, "decompose", decomp, decomp.decompose_task, "decomposed", + ("task_id", "ok", "reason", "fanout", "child_ids", "new_title"), + _decompose_ok_line, + ) def _cmd_gc(args: argparse.Namespace) -> int: - """Remove scratch workspaces of archived tasks, prune old events, and - delete old worker logs.""" + """Remove archived tasks' scratch workspaces, old events, and old worker logs.""" import shutil scratch_root = kb.workspaces_root() removed_ws = 0 @@ -3349,9 +2686,8 @@ def _cmd_gc(args: argparse.Namespace) -> int: ).fetchall() for row in rows: if row["workspace_kind"] == "worktree": - # Backstop for worktrees that escaped the completion/archive hook - # (e.g. tasks archived before that hook existed). Same safety - # predicate: only clean, fully-pushed worktrees are removed. + # Backstop for worktrees that escaped the completion/archive hook. + # Same safety predicate: only clean, fully-pushed worktrees go. wt_path = row["workspace_path"] if wt_path and Path(wt_path).is_dir(): kb._cleanup_worktree_workspace(row["id"], wt_path, row["branch_name"]) @@ -3389,22 +2725,16 @@ def _cmd_gc(args: argparse.Namespace) -> int: def _cmd_repair(args: argparse.Namespace) -> int: - """Check DB integrity and apply the narrow index-REINDEX auto-repair. - - Dispatched BEFORE the auto ``kb.init_db()`` in :func:`kanban_command` - (init itself refuses corrupt DBs), so this is reachable on exactly the - boards that need it. Exit codes: 0 = healthy / repaired / no DB file, - 1 = still corrupt (non-index corruption, or REINDEX did not produce a - clean re-check). - """ + """Integrity check + narrow index-REINDEX auto-repair. Dispatched BEFORE + the auto ``kb.init_db()`` (init refuses corrupt DBs). Exit 0 = healthy / + repaired / no DB file, 1 = still corrupt.""" try: report = kb.repair_db() except Exception as exc: # locked/busy probe, unexpected I/O - print(f"kanban repair: {exc}", file=sys.stderr) - return 1 + return _err(f"kanban repair: {exc}") if getattr(args, "json", False): - print(json.dumps({ + _print_json({ "status": report.status, "db_path": str(report.db_path), "messages": report.messages, @@ -3413,7 +2743,7 @@ def _cmd_repair(args: argparse.Namespace) -> int: str(report.backup_path) if report.backup_path else None ), "reindexed": report.reindexed, - }, indent=2)) + }, ascii=True) return 0 if report.status in {"ok", "repaired", "missing"} else 1 if report.status == "missing": @@ -3458,6 +2788,29 @@ def _cmd_repair(args: argparse.Namespace) -> int: return 1 +_HANDLERS = { + "init": _cmd_init, "create": _cmd_create, "swarm": _cmd_swarm, + "list": _cmd_list, "ls": _cmd_list, "show": _cmd_show, + "assign": _cmd_assign, "set-model": _cmd_set_model, + "reclaim": _cmd_reclaim, "reassign": _cmd_reassign, + "diagnostics": _cmd_diagnostics, "diag": _cmd_diagnostics, + "link": _cmd_link, "unlink": _cmd_unlink, "claim": _cmd_claim, + "comment": _cmd_comment, "attach": _cmd_attach, + "attachments": _cmd_attachments, "attach-rm": _cmd_attach_rm, + "complete": _cmd_complete, "edit": _cmd_edit, "block": _cmd_block, + "schedule": _cmd_schedule, "unblock": _cmd_unblock, + "request-review": _cmd_request_review, "request-changes": _cmd_request_changes, + "reopen-review": _cmd_reopen_review, "promote": _cmd_promote, + "archive": _cmd_archive, "tail": _cmd_tail, "dispatch": _cmd_dispatch, + "daemon": _cmd_daemon, "watch": _cmd_watch, "stats": _cmd_stats, + "log": _cmd_log, "runs": _cmd_runs, "heartbeat": _cmd_heartbeat, + "assignees": _cmd_assignees, "notify-subscribe": _cmd_notify_subscribe, + "notify-list": _cmd_notify_list, "notify-unsubscribe": _cmd_notify_unsubscribe, + "context": _cmd_context, "specify": _cmd_specify, "decompose": _cmd_decompose, + "gc": _cmd_gc, +} + + # --------------------------------------------------------------------------- # Slash-command entry point (used by /kanban from CLI and gateway) # --------------------------------------------------------------------------- @@ -3490,26 +2843,22 @@ Read-only commands are safe while an agent is running.\ def run_slash(rest: str) -> str: """Execute a ``/kanban …`` string and return captured stdout/stderr. - ``rest`` is everything after ``/kanban`` (may be empty). Used from - both the interactive CLI (``self._handle_kanban_command``) and the - gateway (``_handle_kanban_command``) so formatting is identical. + ``rest`` is everything after ``/kanban``. Shared by the interactive CLI + and the gateway so formatting is identical. """ import io - import contextlib tokens = shlex.split(rest) if rest and rest.strip() else [] - # Bare ``/kanban`` or ``/kanban help`` / ``--help`` / ``-h`` / ``?``: - # show the curated short-help block instead of dumping argparse's full - # usage tree (which is enormous and reads as garbage in a chat - # bubble). Per-subcommand help still works via ``/kanban foo -h``. + # Bare ``/kanban`` / ``help`` / ``-h``: the curated short block, not + # argparse's full usage tree (garbage in a chat bubble). Per-subcommand + # help still works via ``/kanban foo -h``. if not tokens or tokens[0] in {"help", "--help", "-h", "?"}: return _SLASH_KANBAN_HELP - # Single argparse tree rooted at "/kanban". build_parser() expects a - # subparsers action to attach to, so build a throwaway one and pull - # the kanban_parser back out — then drive it directly so usage/error - # text reads as ``/kanban`` (not ``/kanban-wrap kanban``). + # build_parser() needs a subparsers action to attach to, so build a + # throwaway one and pull kanban_parser back out; drive it directly so + # usage/error text reads as ``/kanban`` (not ``/kanban-wrap kanban``). _wrap = argparse.ArgumentParser(prog="/kanban-wrap", add_help=False) _wrap.exit_on_error = False # type: ignore[attr-defined] _top_sub = _wrap.add_subparsers(dest="_top") @@ -3533,9 +2882,7 @@ def run_slash(rest: str) -> str: buf_out = io.StringIO() buf_err = io.StringIO() - # ``-h`` / ``--help`` makes argparse print to stdout and SystemExit(0). - # Capture both streams so neither the help text nor the error text - # bypasses our buffer. + # ``-h`` prints to stdout and SystemExit(0); capture both streams. try: with contextlib.redirect_stdout(buf_out), contextlib.redirect_stderr(buf_err): args = kanban_parser.parse_args(tokens) diff --git a/hermes_cli/kanban_db.py b/hermes_cli/kanban_db.py index 198669792e..88c5884979 100644 --- a/hermes_cli/kanban_db.py +++ b/hermes_cli/kanban_db.py @@ -1,71 +1,42 @@ """SQLite-backed Kanban board for multi-profile, multi-project collaboration. -In a fresh install the board lives at ``/kanban.db`` where -```` is the **shared Hermes root** (the parent of any active -profile). Profiles intentionally collapse onto a shared board: it IS -the cross-profile coordination primitive. A worker spawned with -``hermes -p `` joins the same board as the dispatcher that -claimed the task. The same applies to ``/kanban/workspaces/`` and -``/kanban/logs/``. +The board lives under the **shared Hermes root** ```` (the parent of any +active profile; ``HERMES_HOME`` itself in Docker / custom deployments). +Profiles intentionally collapse onto a shared board — it IS the cross-profile +coordination primitive: a worker spawned with ``hermes -p `` joins the +same board as the dispatcher that claimed its task. -**Multiple boards (projects):** users can create additional boards to -separate unrelated streams of work (e.g. one per project / repo / domain). -Each board is a directory under ``/kanban/boards//`` with -its own ``kanban.db``, ``workspaces/``, and ``logs/``. All boards share -the profile's Hermes home but are otherwise isolated: a worker spawned -for a task on board ``atm10-server`` sees only that board's tasks, -cannot enumerate other boards, and its dispatcher ticks don't touch -other boards' DBs. - -The first (and for single-project users, only) board is ``default``. -For back-compat its on-disk DB is ``/kanban.db`` (not -``boards/default/kanban.db``), so installs that predate the boards -feature keep working with zero migration. See :func:`kanban_db_path`. +**Boards:** each extra board is ``/kanban/boards//`` with its own +``kanban.db``, ``workspaces/`` and ``logs/``; a worker on one board cannot see +or enumerate others and its dispatcher ticks never touch their DBs. The first +board is ``default`` and, for back-compat, its DB stays at ``/kanban.db`` +so pre-boards installs need zero migration (see :func:`kanban_db_path`). Board resolution order (highest precedence first, all optional): -* ``board=`` argument passed directly to :func:`connect` / :func:`init_db` - (explicit — used by the CLI ``--board`` flag and the dashboard - ``?board=...`` query param). -* ``HERMES_KANBAN_BOARD`` env var (used by the dispatcher to pin workers - to the board their task lives on — workers cannot see other boards). -* ``HERMES_KANBAN_DB`` env var (pins the DB file path directly — legacy - override still honoured; highest precedence when the file path itself - is what the caller wants to force). -* ``/kanban/current`` — a one-line text file holding the slug of - the "currently selected" board. Written by ``hermes kanban boards - switch ``. When absent, the active board is ``default``. +* ``board=`` argument to :func:`connect` / :func:`init_db` (CLI ``--board``, + dashboard ``?board=``). +* ``HERMES_KANBAN_BOARD`` env var (dispatcher pins workers to their board). +* ``HERMES_KANBAN_DB`` env var (pins the DB file path directly; legacy + override, wins when the file path itself is what the caller forces). +* ``/kanban/current`` — one-line slug file written by + ``hermes kanban boards switch``; absent → ``default``. -In standard installs ```` is ``~/.hermes``. In Docker / custom -deployments where ``HERMES_HOME`` points outside ``~/.hermes`` (e.g. -``/opt/hermes``), ```` is ``HERMES_HOME``. Legacy env-var -overrides still work: +Legacy overrides ``HERMES_KANBAN_DB`` / ``HERMES_KANBAN_WORKSPACES_ROOT`` / +``HERMES_KANBAN_HOME`` (umbrella root; tests and unusual deployments) still +work. The dispatcher injects the DB, workspaces-root and board env vars into +worker subprocesses so they converge on the exact DB it claimed from, even +under unusual symlink or Docker layouts. -* ``HERMES_KANBAN_DB`` — pin the database file path directly. -* ``HERMES_KANBAN_WORKSPACES_ROOT`` — pin the workspaces root directly. -* ``HERMES_KANBAN_HOME`` — pin the umbrella root that anchors kanban - paths. Useful for tests and unusual deployments. +Schema: tasks, task_links, task_comments, task_events (+ runs, attachments, +notify subs). ``workspace_kind`` decouples coordination from git worktrees so +research / ops workloads run alongside coding. See +``docs/hermes-kanban-v1-spec.pdf``. -The dispatcher injects ``HERMES_KANBAN_DB``, -``HERMES_KANBAN_WORKSPACES_ROOT``, and ``HERMES_KANBAN_BOARD`` into -worker subprocess env so workers converge on the exact DB the -dispatcher used to claim their task — even under unusual symlink or -Docker layouts. - -Schema is intentionally small: tasks, task_links, task_comments, -task_events. The ``workspace_kind`` field decouples coordination from git -worktrees so that research / ops / digital-twin workloads work alongside -coding workloads. See ``docs/hermes-kanban-v1-spec.pdf`` for the full -design specification. - -Concurrency strategy: WAL mode + ``BEGIN IMMEDIATE`` for write -transactions + compare-and-swap (CAS) updates on ``tasks.status`` and -``tasks.claim_lock``. SQLite serializes writers via its WAL lock, so at -most one claimer can win any given task. Losers observe zero affected -rows and move on -- no retry loops, no distributed-lock machinery. -The CAS coordination is **per-board** — each board is a separate DB, -so multi-board installs get the same atomicity guarantees without any -new locking. +Concurrency: WAL + ``BEGIN IMMEDIATE`` write transactions + compare-and-swap +updates on ``tasks.status`` / ``tasks.claim_lock``. SQLite serializes writers, +so at most one claimer wins a task; losers see zero affected rows and move on +— no retry loops, no distributed locks. CAS is per-board (one DB per board). """ from __future__ import annotations @@ -95,6 +66,61 @@ from toolsets import get_toolset_names _log = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Shared micro-helpers (row access, JSON, env, git) +# --------------------------------------------------------------------------- + +def _row_get(row: Any, col: str, default: Any = None) -> Any: + """``row[col]`` tolerant of the column being absent from the SELECT / schema.""" + if row is None or col not in row.keys(): + return default + return row[col] + + +def _json_or(value: Any, default: Any = None) -> Any: + """Decode a JSON text column; any decode failure or empty value yields ``default``.""" + if not value: + return default + try: + return json.loads(value) + except Exception: + return default + + +def _json_dict(value: Any) -> dict: + """Decode a JSON text column that must be an object; anything else yields ``{}``.""" + parsed = _json_or(value, {}) + return parsed if isinstance(parsed, dict) else {} + + +def _env_int(name: str, default: int, *, minimum: int = 0) -> int: + """Integer env override: absent/empty/non-integer/below ``minimum`` falls back to ``default``.""" + raw = os.environ.get(name, "").strip() + if raw: + try: + parsed = int(raw) + except ValueError: + return default + if parsed >= minimum: + return parsed + return default + + +def _git_out(cwd: Path, *args: str, timeout: int = 30) -> Optional[str]: + """Run ``git -C cwd args`` and return stripped stdout, or ``None`` on any failure / empty output.""" + try: + result = subprocess.run( + ["git", "-C", str(cwd), *args], + capture_output=True, text=True, encoding="utf-8", errors="replace", + timeout=timeout, check=False, + ) + except Exception: + return None + if result.returncode != 0: + return None + return (result.stdout or "").strip() or None + + # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- @@ -200,16 +226,22 @@ def _fire_kanban_lifecycle_hook(event: str, task_id: str, **fields: Any) -> None """ try: from hermes_cli.lifecycle import invoke_hook - from hermes_cli.profiles import get_active_profile_name - try: - profile_name = get_active_profile_name() - except Exception: - profile_name = "default" - invoke_hook(event, task_id=task_id, profile_name=profile_name, **fields) + + invoke_hook(event, task_id=task_id, profile_name=_hook_profile_name(), **fields) except Exception as exc: # pragma: no cover - defensive _log.debug("kanban lifecycle hook %s failed: %s", event, exc) +def _hook_profile_name() -> str: + """Active profile for hook payloads; ``"default"`` when it cannot be resolved.""" + from hermes_cli.profiles import get_active_profile_name + + try: + return get_active_profile_name() + except Exception: + return "default" + + def _kanban_observer_consumed(event: str) -> bool: """Return whether any first-party observer or plugin consumes *event*. @@ -315,12 +347,8 @@ def _fire_dispatch_tick_hook( return try: from hermes_cli.lifecycle import invoke_hook - from hermes_cli.profiles import get_active_profile_name - try: - profile_name = get_active_profile_name() - except Exception: - profile_name = "default" + profile_name = _hook_profile_name() if board is None: try: board = get_current_board() @@ -398,16 +426,7 @@ def _resolve_claim_ttl_seconds(ttl_seconds: Optional[int] = None) -> int: if ttl_seconds is not None: return max(1, int(ttl_seconds)) - raw = os.environ.get("HERMES_KANBAN_CLAIM_TTL_SECONDS", "").strip() - if raw: - try: - parsed = int(raw) - except ValueError: - parsed = 0 - if parsed > 0: - return parsed - - return DEFAULT_CLAIM_TTL_SECONDS + return _env_int("HERMES_KANBAN_CLAIM_TTL_SECONDS", DEFAULT_CLAIM_TTL_SECONDS, minimum=1) # Grace period after a task transitions to ``running`` during which @@ -438,15 +457,7 @@ def _resolve_crash_grace_seconds() -> int: non-integer, or negative. A value of 0 restores immediate-reclaim behaviour (useful for tests). """ - raw = os.environ.get("HERMES_KANBAN_CRASH_GRACE_SECONDS", "").strip() - if raw: - try: - parsed = int(raw) - except ValueError: - parsed = -1 - if parsed >= 0: - return parsed - return DEFAULT_CRASH_GRACE_SECONDS + return _env_int("HERMES_KANBAN_CRASH_GRACE_SECONDS", DEFAULT_CRASH_GRACE_SECONDS) def _resolve_rate_limit_cooldown_seconds() -> int: @@ -458,17 +469,7 @@ def _resolve_rate_limit_cooldown_seconds() -> int: the next tick) — useful for tests that want to assert the task becomes spawnable again immediately. """ - raw = os.environ.get( - "HERMES_KANBAN_RATE_LIMIT_COOLDOWN_SECONDS", "" - ).strip() - if raw: - try: - parsed = int(raw) - except ValueError: - parsed = -1 - if parsed >= 0: - return parsed - return DEFAULT_RATE_LIMIT_COOLDOWN_SECONDS + return _env_int("HERMES_KANBAN_RATE_LIMIT_COOLDOWN_SECONDS", DEFAULT_RATE_LIMIT_COOLDOWN_SECONDS) # Worker-context caps so build_worker_context() stays bounded on @@ -622,34 +623,28 @@ def get_current_board() -> str: with a best-effort warning — the dispatcher must never crash because a user hand-edited a file or removed a board directory. """ - scoped = (_CURRENT_BOARD_OVERRIDE.get() or "").strip() - if scoped: + def _existing(candidate: str) -> Optional[str]: + if not candidate: + return None try: - normed = _normalize_board_slug(scoped) - if normed and board_exists(normed): - return normed + normed = _normalize_board_slug(candidate) except ValueError: - pass + return None + return normed if normed and board_exists(normed) else None - env = os.environ.get("HERMES_KANBAN_BOARD", "").strip() - if env: - try: - normed = _normalize_board_slug(env) - if normed and board_exists(normed): - return normed - except ValueError: - pass + for candidate in ( + (_CURRENT_BOARD_OVERRIDE.get() or "").strip(), + os.environ.get("HERMES_KANBAN_BOARD", "").strip(), + ): + found = _existing(candidate) + if found: + return found try: f = current_board_path() if f.exists(): - val = f.read_text(encoding="utf-8").strip() - if val: - try: - normed = _normalize_board_slug(val) - if normed and board_exists(normed): - return normed - except ValueError: - pass + found = _existing(f.read_text(encoding="utf-8").strip()) + if found: + return found except OSError: pass return DEFAULT_BOARD @@ -710,6 +705,23 @@ def board_exists(board: Optional[str] = None) -> bool: return (d / "board.json").exists() or (d / "kanban.db").exists() +def _board_path( + env_var: Optional[str], board: Optional[str], default_parts: tuple[str, ...], leaf: str, +) -> Path: + """Shared resolver: ``env_var`` override, else legacy ``/`` + for the ``default`` board, else ``board_dir(slug)/leaf``.""" + if env_var: + override = os.environ.get(env_var, "").strip() + if override: + return Path(override).expanduser() + slug = _normalize_board_slug(board) + if slug is None: + slug = get_current_board() + if slug == DEFAULT_BOARD: + return kanban_home().joinpath(*default_parts) + return board_dir(slug) / leaf + + def kanban_db_path(board: Optional[str] = None) -> Path: """Return the path to the ``kanban.db`` for ``board``. @@ -724,15 +736,7 @@ def kanban_db_path(board: Optional[str] = None) -> Path: 3. Board ``default`` → ``/kanban.db`` (back-compat path). Other boards → ``/kanban/boards//kanban.db``. """ - override = os.environ.get("HERMES_KANBAN_DB", "").strip() - if override: - return Path(override).expanduser() - slug = _normalize_board_slug(board) - if slug is None: - slug = get_current_board() - if slug == DEFAULT_BOARD: - return kanban_home() / "kanban.db" - return board_dir(slug) / "kanban.db" + return _board_path("HERMES_KANBAN_DB", board, ("kanban.db",), "kanban.db") def workspaces_root(board: Optional[str] = None) -> Path: @@ -746,15 +750,7 @@ def workspaces_root(board: Optional[str] = None) -> Path: that existing scratch workspaces from before the boards feature are preserved. Other boards use ``/kanban/boards//workspaces/``. """ - override = os.environ.get("HERMES_KANBAN_WORKSPACES_ROOT", "").strip() - if override: - return Path(override).expanduser() - slug = _normalize_board_slug(board) - if slug is None: - slug = get_current_board() - if slug == DEFAULT_BOARD: - return kanban_home() / "kanban" / "workspaces" - return board_dir(slug) / "workspaces" + return _board_path("HERMES_KANBAN_WORKSPACES_ROOT", board, ("kanban", "workspaces"), "workspaces") def attachments_root(board: Optional[str] = None) -> Path: @@ -776,15 +772,7 @@ def attachments_root(board: Optional[str] = None) -> Path: directly. Remote backends (Docker/Modal) need this directory mounted; see the kanban docs. """ - override = os.environ.get("HERMES_KANBAN_ATTACHMENTS_ROOT", "").strip() - if override: - return Path(override).expanduser() - slug = _normalize_board_slug(board) - if slug is None: - slug = get_current_board() - if slug == DEFAULT_BOARD: - return kanban_home() / "kanban" / "attachments" - return board_dir(slug) / "attachments" + return _board_path("HERMES_KANBAN_ATTACHMENTS_ROOT", board, ("kanban", "attachments"), "attachments") def task_attachments_dir(task_id: str, board: Optional[str] = None) -> Path: @@ -800,12 +788,7 @@ def worker_logs_dir(board: Optional[str] = None) -> Path: board — makes ``hermes kanban log`` unambiguous even when multiple boards have tasks with the same id. """ - slug = _normalize_board_slug(board) - if slug is None: - slug = get_current_board() - if slug == DEFAULT_BOARD: - return kanban_home() / "kanban" / "logs" - return board_dir(slug) / "logs" + return _board_path(None, board, ("kanban", "logs"), "logs") def board_metadata_path(board: Optional[str] = None) -> Path: @@ -1144,16 +1127,9 @@ class Task: @classmethod def from_row(cls, row: sqlite3.Row) -> "Task": - keys = set(row.keys()) - # Parse skills JSON blob if present - skills_value: Optional[list] = None - if "skills" in keys and row["skills"]: - try: - parsed = json.loads(row["skills"]) - if isinstance(parsed, list): - skills_value = [str(s) for s in parsed if s] - except Exception: - skills_value = None + g = lambda col, default=None: _row_get(row, col, default) # noqa: E731 + parsed = _json_or(g("skills")) + skills_value = [str(s) for s in parsed if s] if isinstance(parsed, list) else None return cls( id=row["id"], title=row["title"], @@ -1167,73 +1143,34 @@ class Task: completed_at=row["completed_at"], workspace_kind=row["workspace_kind"], workspace_path=row["workspace_path"], - branch_name=row["branch_name"] if "branch_name" in keys else None, - project_id=row["project_id"] if "project_id" in keys else None, + branch_name=g("branch_name"), + project_id=g("project_id"), claim_lock=row["claim_lock"], claim_expires=row["claim_expires"], - tenant=row["tenant"] if "tenant" in keys else None, - result=row["result"] if "result" in keys else None, - idempotency_key=row["idempotency_key"] if "idempotency_key" in keys else None, - consecutive_failures=( - row["consecutive_failures"] if "consecutive_failures" in keys - # Pre-migration fallback: ``_migrate_add_optional_columns`` always - # adds ``consecutive_failures`` now, so this branch is only reachable - # on a DB that was never opened since pre-#20410 code ran. Keep for - # belt-and-suspenders safety; in practice it is dead code post-migration. - else (row["spawn_failures"] if "spawn_failures" in keys else 0) - ), - worker_pid=row["worker_pid"] if "worker_pid" in keys else None, - last_failure_error=( - row["last_failure_error"] if "last_failure_error" in keys - # Same belt-and-suspenders fallback as consecutive_failures above. - else (row["last_spawn_error"] if "last_spawn_error" in keys else None) - ), - max_runtime_seconds=( - row["max_runtime_seconds"] if "max_runtime_seconds" in keys else None - ), - last_heartbeat_at=( - row["last_heartbeat_at"] if "last_heartbeat_at" in keys else None - ), - current_run_id=( - row["current_run_id"] if "current_run_id" in keys else None - ), - workflow_template_id=( - row["workflow_template_id"] if "workflow_template_id" in keys else None - ), - current_step_key=( - row["current_step_key"] if "current_step_key" in keys else None - ), + tenant=g("tenant"), + result=g("result"), + idempotency_key=g("idempotency_key"), + # Pre-migration fallbacks (spawn_failures / last_spawn_error) are only + # reachable on a DB never opened since the rename migration landed. + consecutive_failures=g("consecutive_failures", g("spawn_failures", 0)), + worker_pid=g("worker_pid"), + last_failure_error=g("last_failure_error", g("last_spawn_error")), + max_runtime_seconds=g("max_runtime_seconds"), + last_heartbeat_at=g("last_heartbeat_at"), + current_run_id=g("current_run_id"), + workflow_template_id=g("workflow_template_id"), + current_step_key=g("current_step_key"), skills=skills_value, - model_override=row["model_override"] if "model_override" in keys and row["model_override"] else None, - provider_override=( - row["provider_override"] - if "provider_override" in keys and row["provider_override"] - else None - ), - reasoning_effort=( - row["reasoning_effort"] - if "reasoning_effort" in keys and row["reasoning_effort"] - else None - ), - max_retries=( - row["max_retries"] if "max_retries" in keys else None - ), - goal_mode=( - bool(row["goal_mode"]) if "goal_mode" in keys and row["goal_mode"] else False - ), - goal_max_turns=( - row["goal_max_turns"] if "goal_max_turns" in keys and row["goal_max_turns"] else None - ), - session_id=( - row["session_id"] if "session_id" in keys else None - ), - block_kind=( - row["block_kind"] if "block_kind" in keys and row["block_kind"] else None - ), + model_override=g("model_override") or None, + provider_override=g("provider_override") or None, + reasoning_effort=g("reasoning_effort") or None, + max_retries=g("max_retries"), + goal_mode=bool(g("goal_mode")), + goal_max_turns=g("goal_max_turns") or None, + session_id=g("session_id"), + block_kind=g("block_kind") or None, block_recurrences=( - int(row["block_recurrences"]) - if "block_recurrences" in keys and row["block_recurrences"] is not None - else 0 + int(g("block_recurrences")) if g("block_recurrences") is not None else 0 ), ) @@ -1268,10 +1205,6 @@ class Run: @classmethod def from_row(cls, row: sqlite3.Row) -> "Run": - try: - meta = json.loads(row["metadata"]) if row["metadata"] else None - except Exception: - meta = None return cls( id=int(row["id"]), task_id=row["task_id"], @@ -1284,10 +1217,10 @@ class Run: max_runtime_seconds=row["max_runtime_seconds"], last_heartbeat_at=row["last_heartbeat_at"], started_at=int(row["started_at"]), - ended_at=(int(row["ended_at"]) if row["ended_at"] is not None else None), + ended_at=_opt_int(row["ended_at"]), outcome=row["outcome"], summary=row["summary"], - metadata=meta, + metadata=_json_or(row["metadata"]), error=row["error"], ) @@ -1300,6 +1233,13 @@ class Comment: body: str created_at: int + @classmethod + def from_row(cls, r: sqlite3.Row) -> "Comment": + return cls( + id=r["id"], task_id=r["task_id"], author=r["author"], + body=r["body"], created_at=r["created_at"], + ) + @dataclass class Attachment: @@ -1314,6 +1254,15 @@ class Attachment: uploaded_by: Optional[str] created_at: int + @classmethod + def from_row(cls, r: sqlite3.Row) -> "Attachment": + return cls( + id=r["id"], task_id=r["task_id"], filename=r["filename"], + stored_path=r["stored_path"], content_type=r["content_type"], + size=r["size"] or 0, uploaded_by=r["uploaded_by"], + created_at=r["created_at"], + ) + @dataclass class Event: @@ -1324,6 +1273,15 @@ class Event: created_at: int run_id: Optional[int] = None + @classmethod + def from_row(cls, row: sqlite3.Row) -> "Event": + run_id = _row_get(row, "run_id") + return cls( + id=row["id"], task_id=row["task_id"], kind=row["kind"], + payload=_json_or(row["payload"]), created_at=row["created_at"], + run_id=_opt_int(run_id), + ) + # --------------------------------------------------------------------------- # Schema @@ -1560,15 +1518,7 @@ def _resolve_busy_timeout_ms() -> int: expected. A long busy timeout lets SQLite serialize writers via WAL rather than surfacing transient ``database is locked`` failures during bursts. """ - raw = os.environ.get("HERMES_KANBAN_BUSY_TIMEOUT_MS", "").strip() - if raw: - try: - parsed = int(raw) - except ValueError: - parsed = 0 - if parsed > 0: - return parsed - return DEFAULT_BUSY_TIMEOUT_MS + return _env_int("HERMES_KANBAN_BUSY_TIMEOUT_MS", DEFAULT_BUSY_TIMEOUT_MS, minimum=1) def _sqlite_connect(path: Path) -> sqlite3.Connection: @@ -1607,6 +1557,39 @@ def _sqlite_connect(path: Path) -> sqlite3.Connection: return conn +def _try_lock_nb(handle) -> bool: + """One non-blocking exclusive lock attempt on ``handle``; False when held elsewhere. + + Windows uses a 1-byte ``msvcrt.locking`` range at offset 0, POSIX ``flock``. + """ + if _IS_WINDOWS: + import msvcrt + + handle.seek(0) + getattr(msvcrt, "locking")(handle.fileno(), getattr(msvcrt, "LK_NBLCK"), 1) + else: + import fcntl + + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return False + return True + + +def _unlock(handle) -> None: + """Release a lock taken by :func:`_try_lock_nb` (same byte range / flock).""" + if _IS_WINDOWS: + import msvcrt + + handle.seek(0) + getattr(msvcrt, "locking")(handle.fileno(), getattr(msvcrt, "LK_UNLCK"), 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + @contextlib.contextmanager def _cross_process_init_lock(path: Path): """Serialize first-connect WAL/schema/integrity setup across processes. @@ -1618,18 +1601,13 @@ def _cross_process_init_lock(path: Path): additive migrations single-file/single-writer across the whole host while leaving normal post-init DB usage concurrent under SQLite WAL. - The acquire is **bounded** (issue #36644): the original bare blocking - ``flock(LOCK_EX)`` had no timeout, so a single process stalled inside the - critical section (or a stale lock held by a wedged worker) blocked every - other ``connect()`` — including the long-lived gateway dispatcher's - next-tick connect — forever, with no traceback and no recovery short of a - restart. We now retry a non-blocking acquire up to a deadline; on timeout - we log a WARNING and proceed WITHOUT the cross-process lock. That is safe: - the in-process ``_INIT_LOCK`` still serializes same-process threads, and - the init work itself is idempotent (``CREATE TABLE IF NOT EXISTS`` + - additive migrations), so the worst case of two processes racing first-init - is redundant work, not corruption. A bounded "proceed anyway" beats an - unbounded hang that silently stops the board. + The acquire is **bounded**: a blocking ``flock`` let one stalled holder (or + a stale lock from a wedged worker) hang every other ``connect()`` — the + gateway dispatcher's next tick included — with no traceback. We retry a + non-blocking acquire until a deadline, then WARN and proceed WITHOUT the + lock. Safe because ``_INIT_LOCK`` still serializes same-process threads + and init is idempotent (``CREATE TABLE IF NOT EXISTS`` + additive + migrations): two racing first-inits mean redundant work, not corruption. """ path.parent.mkdir(parents=True, exist_ok=True) lock_path = path.with_name(path.name + ".init.lock") @@ -1637,33 +1615,14 @@ def _cross_process_init_lock(path: Path): acquired = False try: deadline = time.monotonic() + _INIT_LOCK_TIMEOUT_SECONDS - if _IS_WINDOWS: - import msvcrt - - locking = getattr(msvcrt, "locking") - nb_lock = getattr(msvcrt, "LK_NBLCK") - while True: - try: - handle.seek(0) - locking(handle.fileno(), nb_lock, 1) - acquired = True - break - except OSError: - if time.monotonic() >= deadline: - break - time.sleep(_INIT_LOCK_POLL_SECONDS) - else: - import fcntl - - while True: - try: - fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - acquired = True - break - except (BlockingIOError, OSError): - if time.monotonic() >= deadline: - break - time.sleep(_INIT_LOCK_POLL_SECONDS) + while True: + try: + acquired = _try_lock_nb(handle) + except OSError: + acquired = False + if acquired or time.monotonic() >= deadline: + break + time.sleep(_INIT_LOCK_POLL_SECONDS) if not acquired: _log.warning( "kanban init lock for %s not acquired within %.0fs — proceeding " @@ -1676,17 +1635,7 @@ def _cross_process_init_lock(path: Path): finally: try: if acquired: - if _IS_WINDOWS: - import msvcrt - - handle.seek(0) - locking = getattr(msvcrt, "locking") - unlock_mode = getattr(msvcrt, "LK_UNLCK") - locking(handle.fileno(), unlock_mode, 1) - else: - import fcntl - - fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + _unlock(handle) finally: handle.close() @@ -1699,22 +1648,17 @@ def _dispatch_tick_lock(db_path: Path): may proceed with the tick, or ``False`` when another process already holds it (the caller should skip the tick this round). - Motivation (issue #35240): a ``hermes gateway run --replace`` / - ``gateway restart`` invoked from a shell on a systemd/launchd host can - leave an orphan gateway whose dispatcher escapes the service cgroup, - survives ``systemctl restart``, and becomes a *second* long-lived - writer on the same ``kanban.db``. Two dispatchers that each believe - they own the file both pass SQLite ``busy_timeout`` and then race on - WAL frames — the documented root cause of multi-writer corruption. - The startup guard (``_guard_supervised_gateway_conflict``) blocks the - common way an orphan is born, but this lock is the defense-in-depth - that prevents two dispatchers from ever writing concurrently - *regardless of how the second one got there*. + An orphan gateway (``gateway run --replace`` / restart on a systemd or + launchd host escaping the service cgroup) becomes a second long-lived + writer on the same ``kanban.db``; two dispatchers both pass SQLite + ``busy_timeout`` and race on WAL frames — the root cause of multi-writer + corruption. ``_guard_supervised_gateway_conflict`` blocks the common way + an orphan is born; this lock is the defense-in-depth regardless of how + the second dispatcher got there. - The lock is **non-blocking** on purpose: the gateway's async watcher - must never stall on a held lock. A losing dispatcher simply skips its - tick (the winner is making progress on the same board), and tries - again next interval. + **Non-blocking** on purpose: the gateway's async watcher must never stall + on a held lock. A loser skips its tick (the winner is making progress on + the same board) and tries again next interval. Board-scoped: the lock file is a ``.dispatch.lock`` sibling of the board's ``kanban.db``, so unrelated boards tick independently. On @@ -1728,26 +1672,10 @@ def _dispatch_tick_lock(db_path: Path): try: lock_path.parent.mkdir(parents=True, exist_ok=True) handle = lock_path.open("a+b") - if _IS_WINDOWS: - try: - import msvcrt - - handle.seek(0) - locking = getattr(msvcrt, "locking") - # LK_NBLCK = non-blocking exclusive byte-range lock. - nb_lock = getattr(msvcrt, "LK_NBLCK") - locking(handle.fileno(), nb_lock, 1) - acquired = True - except (OSError, AttributeError): - acquired = False - else: - try: - import fcntl - - fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - acquired = True - except (BlockingIOError, OSError): - acquired = False + try: + acquired = _try_lock_nb(handle) + except (OSError, AttributeError): + acquired = False except OSError: # Could not even open the lock file (permissions, read-only FS). # Degrade to a no-op so a probe failure never blocks dispatch. @@ -1759,17 +1687,7 @@ def _dispatch_tick_lock(db_path: Path): if handle is not None: try: if acquired: - if _IS_WINDOWS: - import msvcrt - - handle.seek(0) - locking = getattr(msvcrt, "locking") - unlock_mode = getattr(msvcrt, "LK_UNLCK") - locking(handle.fileno(), unlock_mode, 1) - else: - import fcntl - - fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + _unlock(handle) except (OSError, AttributeError): pass finally: @@ -2052,6 +1970,20 @@ def _run_integrity_check(conn: sqlite3.Connection) -> list[str]: return [str(row[0]) for row in rows if row is not None and row[0] is not None] +def _probe_integrity(path: Path) -> list[str]: + """Open ``path`` read/write and return its ``integrity_check`` messages. + + Opening read/write lets SQLite recover or checkpoint a healthy WAL / + hot-journal DB before we judge it. ``sqlite3.OperationalError`` (locked, + busy, transient IO) propagates raw — it is not corruption. + """ + probe = _sqlite_connect(path) + try: + return _run_integrity_check(probe) + finally: + probe.close() + + def _repairable_index_names(messages: list[str]) -> Optional[list[str]]: """Return the distinct index names iff EVERY message is index-repairable. @@ -2165,11 +2097,7 @@ def _guard_existing_db_is_healthy(path: Path) -> None: reason: Optional[str] = None messages: list[str] = [] try: - probe = _sqlite_connect(resolved) - try: - messages = _run_integrity_check(probe) - finally: - probe.close() + messages = _probe_integrity(resolved) if not _integrity_messages_ok(messages): reason = ( f"integrity_check returned " @@ -2253,10 +2181,7 @@ def repair_db( raw, exactly like the guard: a locked healthy DB is not corruption and must not be quarantined. """ - if db_path is not None: - path = db_path - else: - path = kanban_db_path(board=board) + path = db_path if db_path is not None else kanban_db_path(board=board) try: resolved = path.resolve() except OSError: @@ -2270,11 +2195,7 @@ def repair_db( with _cross_process_init_lock(resolved): messages: list[str] = [] try: - probe = _sqlite_connect(resolved) - try: - messages = _run_integrity_check(probe) - finally: - probe.close() + messages = _probe_integrity(resolved) except sqlite3.OperationalError: # Locked/busy — not corruption; let the caller report it raw. raise @@ -2335,6 +2256,45 @@ def _schema_is_present(conn: sqlite3.Connection) -> bool: return row is not None +def _open_configured(path: Path, under_lock) -> tuple[sqlite3.Connection, Any]: + """Open ``path`` with the kanban PRAGMA set, then run ``under_lock(conn)``. + + WAL activation and ``under_lock`` share the process-local ``_INIT_LOCK`` + critical section: WAL setup can take an exclusive lock while SQLite + creates sidecar files for a fresh DB, and concurrent gateway startup + threads must not race before ``_INITIALIZED_PATHS`` is populated. The + connection is closed if anything raises. + """ + conn = _sqlite_connect(path) + try: + conn.row_factory = sqlite3.Row + with _INIT_LOCK: + # WAL doesn't work on network filesystems (NFS/SMB/FUSE); the shared + # helper falls back to DELETE with one ERROR log (see + # hermes_state._WAL_INCOMPAT_MARKERS). + from hermes_state import apply_wal_with_fallback + apply_wal_with_fallback(conn, db_label=f"kanban.db ({path.name})") + # FULL (was NORMAL): fsync before each checkpoint to narrow the + # crash window that can leave a b-tree page header torn. + conn.execute("PRAGMA synchronous=FULL") + conn.execute("PRAGMA wal_autocheckpoint=100") + # Bound the WAL file: the periodic explicit checkpoint is PASSIVE + # (never truncates), so SQLite trims the -wal file to this limit on + # the writer's natural post-checkpoint reset. 8 MiB is generous. + conn.execute("PRAGMA journal_size_limit=8388608") + conn.execute("PRAGMA foreign_keys=ON") + # Zero freed pages so a later torn write cannot expose stale cell + # content; persisted in the DB header for new DBs. + conn.execute("PRAGMA secure_delete=ON") + # Surface corrupt cells as read errors instead of silent wrong data. + conn.execute("PRAGMA cell_size_check=ON") + out = under_lock(conn) + except Exception: + conn.close() + raise + return conn, out + + def connect( db_path: Optional[Path] = None, *, @@ -2358,10 +2318,7 @@ def connect( ``HERMES_KANBAN_DB`` env → ``HERMES_KANBAN_BOARD`` env → ``/kanban/current`` → ``default``. """ - if db_path is not None: - path = db_path - else: - path = kanban_db_path(board=board) + path = db_path if db_path is not None else kanban_db_path(board=board) path.parent.mkdir(parents=True, exist_ok=True) # Fast path: once THIS process has initialized this path, the expensive @@ -2376,26 +2333,7 @@ def connect( # connection with WAL/pragmas under the cheap in-process _INIT_LOCK. resolved = str(path.resolve()) if resolved in _INITIALIZED_PATHS: - conn = _sqlite_connect(path) - try: - conn.row_factory = sqlite3.Row - with _INIT_LOCK: - from hermes_state import apply_wal_with_fallback - apply_wal_with_fallback(conn, db_label=f"kanban.db ({path.name})") - conn.execute("PRAGMA synchronous=FULL") - conn.execute("PRAGMA wal_autocheckpoint=100") - # Bound the WAL file size now that the periodic explicit - # checkpoint is PASSIVE (never truncates): on the writer's - # natural post-checkpoint reset SQLite trims the -wal file - # to this limit. 8 MiB is generous for a kanban board. - conn.execute("PRAGMA journal_size_limit=8388608") - conn.execute("PRAGMA foreign_keys=ON") - conn.execute("PRAGMA secure_delete=ON") - conn.execute("PRAGMA cell_size_check=ON") - schema_present = _schema_is_present(conn) - except Exception: - conn.close() - raise + conn, schema_present = _open_configured(path, _schema_is_present) if schema_present: return conn # The cache says "initialized", the file says otherwise: it was deleted @@ -2429,48 +2367,18 @@ def connect( # via _INITIALIZED_PATHS so it only runs once per process per path. _guard_existing_db_is_healthy(path) resolved = str(path.resolve()) - conn = _sqlite_connect(path) - try: - conn.row_factory = sqlite3.Row - with _INIT_LOCK: - # WAL activation can take an exclusive lock while SQLite creates the - # sidecar files for a fresh database. Keep it in the same process-local - # critical section as schema initialization so concurrent gateway - # startup threads do not race before _INITIALIZED_PATHS is populated. - # WAL doesn't work on network filesystems (NFS/SMB/FUSE). Shared helper - # falls back to DELETE with one ERROR log so kanban stays usable there. - # See hermes_state._WAL_INCOMPAT_MARKERS for detection logic. - from hermes_state import apply_wal_with_fallback - apply_wal_with_fallback(conn, db_label=f"kanban.db ({path.name})") - # FULL (was NORMAL): fsync before each checkpoint to narrow the - # crash window that can leave a b-tree page header torn. - conn.execute("PRAGMA synchronous=FULL") - conn.execute("PRAGMA wal_autocheckpoint=100") - # Bound the WAL file size now that the periodic explicit - # checkpoint is PASSIVE (never truncates): on the writer's - # natural post-checkpoint reset SQLite trims the -wal file - # to this limit. 8 MiB is generous for a kanban board. - conn.execute("PRAGMA journal_size_limit=8388608") - conn.execute("PRAGMA foreign_keys=ON") - # Zero freed pages so a later torn write cannot expose stale - # cell content; persisted in the DB header for new DBs. - conn.execute("PRAGMA secure_delete=ON") - # Surface corrupt cells as read errors instead of silent - # wrong-data returns. - conn.execute("PRAGMA cell_size_check=ON") - needs_init = resolved not in _INITIALIZED_PATHS - if needs_init: - # Idempotent: runs CREATE TABLE IF NOT EXISTS + the additive - # migrations. Cached so subsequent connect() calls in the same - # process are cheap. The lock prevents same-process dispatcher - # threads from racing through the additive ALTER TABLE pass with - # stale PRAGMA snapshots during gateway startup. - conn.executescript(SCHEMA_SQL) - _migrate_add_optional_columns(conn) - _INITIALIZED_PATHS.add(resolved) - except Exception: - conn.close() - raise + + def _init_if_needed(conn: sqlite3.Connection) -> None: + # Idempotent: CREATE TABLE IF NOT EXISTS + the additive migrations, + # cached so later connect() calls in this process are cheap. Runs + # under _INIT_LOCK so same-process dispatcher threads can't race + # through the ALTER TABLE pass with stale PRAGMA snapshots. + if resolved not in _INITIALIZED_PATHS: + conn.executescript(SCHEMA_SQL) + _migrate_add_optional_columns(conn) + _INITIALIZED_PATHS.add(resolved) + + conn, _ = _open_configured(path, _init_if_needed) return conn @@ -2482,22 +2390,12 @@ def connect_closing( ): """Open a kanban DB connection and guarantee it is closed on exit. - Use this instead of ``with kb.connect() as conn:`` — sqlite3's - built-in connection context manager only commits/rollbacks the - transaction; it does NOT close the file descriptor. In long-lived - processes (gateway, dashboard) that route every kanban operation - through ``connect()`` (e.g. ``run_slash`` dispatching ``/kanban …`` - commands, ``decompose_task_endpoint`` calling - ``kanban_decompose.decompose_task``), the unclosed connections - accumulate as open FDs to ``kanban.db`` and ``kanban.db-wal``. After - enough operations the process hits the kernel FD limit and dies - with ``[Errno 24] Too many open files``. - - See #33159 for the production incident. - - The ``connect()`` function itself remains unchanged so callers that - intentionally manage the connection lifetime (tests, long-lived - callers) continue to work. + Use this instead of ``with kb.connect() as conn:`` — sqlite3's connection + context manager only commits/rolls back; it does NOT close the file + descriptor. Long-lived processes (gateway, dashboard) that route every + kanban operation through ``connect()`` otherwise leak FDs to ``kanban.db`` + / ``kanban.db-wal`` until ``[Errno 24] Too many open files`` kills them. + ``connect()`` itself is unchanged for callers that manage the lifetime. """ conn = connect(db_path=db_path, board=board) try: @@ -2524,10 +2422,7 @@ def init_db( external tools that upgrade an old DB file — can call this to force re-migration. """ - if db_path is not None: - path = db_path - else: - path = kanban_db_path(board=board) + path = db_path if db_path is not None else kanban_db_path(board=board) path.parent.mkdir(parents=True, exist_ok=True) resolved = str(path.resolve()) # Clear the cache entry so the underlying connect() re-runs the @@ -2539,156 +2434,91 @@ def init_db( return path +# Additive ``tasks`` columns, in the order legacy DBs receive them (order is +# the physical column order for ``SELECT *`` on migrated boards). +_EARLY_TASK_COLUMNS = ( + ("tenant", "tenant TEXT"), + ("result", "result TEXT"), + ("branch_name", "branch_name TEXT"), + ("project_id", "project_id TEXT"), + ("idempotency_key", "idempotency_key TEXT"), +) +# (new column, ddl, legacy source column, copy statement) — see the +# RENAME-avoidance note in ``_migrate_add_optional_columns``. +_RENAMED_TASK_COLUMNS = ( + ( + "consecutive_failures", "consecutive_failures INTEGER NOT NULL DEFAULT 0", + "spawn_failures", "UPDATE tasks SET consecutive_failures = COALESCE(spawn_failures, 0)", + ), + ("worker_pid", "worker_pid INTEGER", None, None), + ( + "last_failure_error", "last_failure_error TEXT", + "last_spawn_error", "UPDATE tasks SET last_failure_error = last_spawn_error", + ), +) +# NULL / 0 defaults on every column below reproduce the behaviour existing +# rows had before the column existed (worker-profile model/provider/reasoning, +# global failure limit, single-shot worker, untyped human block, ...). +_LATER_TASK_COLUMNS = ( + ("max_runtime_seconds", "max_runtime_seconds INTEGER"), + ("last_heartbeat_at", "last_heartbeat_at INTEGER"), + ("current_run_id", "current_run_id INTEGER"), + ("workflow_template_id", "workflow_template_id TEXT"), + ("current_step_key", "current_step_key TEXT"), + # JSON array of skill names the dispatcher force-loads via --skills. + ("skills", "skills TEXT"), + # Per-task override for the consecutive-failure circuit breaker; NULL = + # ``kanban.failure_limit`` config, then ``DEFAULT_FAILURE_LIMIT``. + ("max_retries", "max_retries INTEGER"), + ("model_override", "model_override TEXT"), + ("provider_override", "provider_override TEXT"), + ("reasoning_effort", "reasoning_effort TEXT"), + # Ralph-style goal loop toggle; 0 = classic single-shot worker. + ("goal_mode", "goal_mode INTEGER NOT NULL DEFAULT 0"), + ("goal_max_turns", "goal_max_turns INTEGER"), + # Originating agent/chat session id (``HERMES_SESSION_ID``, e.g. ACP). + ("session_id", "session_id TEXT"), + # Typed block reason (VALID_BLOCK_KINDS); NULL = generic human blocker. + ("block_kind", "block_kind TEXT"), + # Unblock-loop counter; existing rows start at 0. + ("block_recurrences", "block_recurrences INTEGER NOT NULL DEFAULT 0"), +) + + def _migrate_add_optional_columns(conn: sqlite3.Connection) -> None: """Add columns that were introduced after v1 release to legacy DBs. Called by ``init_db`` so opening an old DB is always safe. """ cols = {row["name"] for row in conn.execute("PRAGMA table_info(tasks)")} - if "tenant" not in cols: - _add_column_if_missing(conn, "tasks", "tenant", "tenant TEXT") - if "result" not in cols: - _add_column_if_missing(conn, "tasks", "result", "result TEXT") - if "branch_name" not in cols: - _add_column_if_missing(conn, "tasks", "branch_name", "branch_name TEXT") - if "project_id" not in cols: - _add_column_if_missing(conn, "tasks", "project_id", "project_id TEXT") - if "idempotency_key" not in cols: - _add_column_if_missing( - conn, "tasks", "idempotency_key", "idempotency_key TEXT" - ) + for name, ddl in _EARLY_TASK_COLUMNS: + if name not in cols: + _add_column_if_missing(conn, "tasks", name, ddl) # ``idx_tasks_idempotency`` is created unconditionally below alongside - # the other additive-column indexes — see the block after the - # legacy-column migration. Creating it here too would be redundant. + # the other additive-column indexes. - # Refresh after early additive migrations above. Some existing DBs were - # partially migrated in older releases and can already contain the later - # columns (for example ``consecutive_failures``) even when this function's - # initial snapshot did not. Re-snapshot here so the legacy-column migration - # below is truly idempotent and never re-adds columns that already exist. + # Re-snapshot: DBs partially migrated by older releases may already carry + # later columns (e.g. ``consecutive_failures``) even when the first + # snapshot did not, so the legacy-column migration stays idempotent. cols = {row["name"] for row in conn.execute("PRAGMA table_info(tasks)")} # Legacy column migration: ``spawn_failures`` → ``consecutive_failures`` - # and ``last_spawn_error`` → ``last_failure_error``. - # - # Avoid ``ALTER TABLE ... RENAME COLUMN`` for two reasons: - # 1. Primary: very old DBs may never have had ``spawn_failures`` at - # all, so RENAME raises OperationalError: no such column (the crash - # reported in issue #20842 after the #20410 update). - # 2. Secondary: SQLite reparses the whole schema on any RENAME, which - # fails if related objects (views, triggers) reference the old name. - # - # ADD-first-then-copy is tolerant of both shapes and preserves - # historical counter values when the legacy columns do exist. - if "consecutive_failures" not in cols: - added = _add_column_if_missing( - conn, - "tasks", - "consecutive_failures", - "consecutive_failures INTEGER NOT NULL DEFAULT 0", - ) - if added and "spawn_failures" in cols: - conn.execute( - "UPDATE tasks SET consecutive_failures = COALESCE(spawn_failures, 0)" - ) - if "worker_pid" not in cols: - _add_column_if_missing(conn, "tasks", "worker_pid", "worker_pid INTEGER") - if "last_failure_error" not in cols: - added = _add_column_if_missing( - conn, "tasks", "last_failure_error", "last_failure_error TEXT" - ) - if added and "last_spawn_error" in cols: - conn.execute( - "UPDATE tasks SET last_failure_error = last_spawn_error" - ) - if "max_runtime_seconds" not in cols: - _add_column_if_missing( - conn, "tasks", "max_runtime_seconds", "max_runtime_seconds INTEGER" - ) - if "last_heartbeat_at" not in cols: - _add_column_if_missing( - conn, "tasks", "last_heartbeat_at", "last_heartbeat_at INTEGER" - ) - if "current_run_id" not in cols: - _add_column_if_missing( - conn, "tasks", "current_run_id", "current_run_id INTEGER" - ) - if "workflow_template_id" not in cols: - _add_column_if_missing( - conn, "tasks", "workflow_template_id", "workflow_template_id TEXT" - ) - if "current_step_key" not in cols: - _add_column_if_missing( - conn, "tasks", "current_step_key", "current_step_key TEXT" - ) - if "skills" not in cols: - # JSON array of skill names the dispatcher force-loads into the - # worker via --skills. NULL is fine for existing rows. - _add_column_if_missing(conn, "tasks", "skills", "skills TEXT") - - if "max_retries" not in cols: - # Per-task override for the consecutive-failure circuit breaker. - # NULL = fall through to the dispatcher-level ``kanban.failure_limit`` - # config, then ``DEFAULT_FAILURE_LIMIT``. Existing rows get NULL, - # which is the correct default (they keep the global behaviour - # they were getting before the column existed). - _add_column_if_missing(conn, "tasks", "max_retries", "max_retries INTEGER") - - if "model_override" not in cols: - conn.execute("ALTER TABLE tasks ADD COLUMN model_override TEXT") - - if "provider_override" not in cols: - # Provider the model_override belongs to. NULL = worker profile's - # provider resolves the model (the behaviour existing rows had). - _add_column_if_missing( - conn, "tasks", "provider_override", "provider_override TEXT" - ) - - if "reasoning_effort" not in cols: - # Per-task thinking depth for the worker. NULL = the worker profile's - # own agent.reasoning_effort, which is what existing rows were getting. - _add_column_if_missing( - conn, "tasks", "reasoning_effort", "reasoning_effort TEXT" - ) - - if "goal_mode" not in cols: - # Ralph-style goal loop toggle for the dispatched worker. 0 (the - # default) = classic single-shot worker, preserving the behaviour - # existing rows had before the column existed. - _add_column_if_missing( - conn, "tasks", "goal_mode", "goal_mode INTEGER NOT NULL DEFAULT 0" - ) - - if "goal_max_turns" not in cols: - # Per-task goal-loop turn budget. NULL = goals-engine default. - _add_column_if_missing( - conn, "tasks", "goal_max_turns", "goal_max_turns INTEGER" - ) - - if "session_id" not in cols: - # Originating agent/chat session id, populated when the task is - # created from within an agent loop that propagated - # ``HERMES_SESSION_ID`` (e.g. ACP). NULL on legacy rows and on any - # creation path that doesn't set the env var (CLI, dashboard). - _add_column_if_missing( - conn, "tasks", "session_id", "session_id TEXT" - ) - - if "block_kind" not in cols: - # Typed block reason (VALID_BLOCK_KINDS) or NULL for legacy/un-typed - # blocks. Existing blocked rows get NULL, which is treated as a - # generic human blocker — same behaviour they had before the column. - _add_column_if_missing(conn, "tasks", "block_kind", "block_kind TEXT") - - if "block_recurrences" not in cols: - # Unblock-loop counter. Existing rows start at 0, so the loop breaker - # only begins counting from the first re-block after this migration. - _add_column_if_missing( - conn, - "tasks", - "block_recurrences", - "block_recurrences INTEGER NOT NULL DEFAULT 0", - ) + # and ``last_spawn_error`` → ``last_failure_error``. ADD-then-copy rather + # than ``RENAME COLUMN``: very old DBs may lack the legacy column entirely + # (RENAME raises "no such column"), and RENAME reparses the whole schema, + # failing if views/triggers reference the old name. Preserves historical + # counter values when the legacy columns do exist. + for name, ddl, legacy, copy_sql in _RENAMED_TASK_COLUMNS: + if name not in cols: + added = _add_column_if_missing(conn, "tasks", name, ddl) + if added and legacy is not None and legacy in cols: + conn.execute(copy_sql) + for name, ddl in _LATER_TASK_COLUMNS: + if name not in cols: + if name == "model_override": + conn.execute("ALTER TABLE tasks ADD COLUMN model_override TEXT") + else: + _add_column_if_missing(conn, "tasks", name, ddl) # Indexes over additive ``tasks`` columns must be created after the # columns exist. Keeping them in SCHEMA_SQL breaks legacy boards: SQLite @@ -2726,54 +2556,30 @@ def _migrate_add_optional_columns(conn: sqlite3.Connection) -> None: notify_cols = { row["name"] for row in conn.execute("PRAGMA table_info(kanban_notify_subs)") } - if "notifier_profile" not in notify_cols: - _add_column_if_missing( - conn, "kanban_notify_subs", "notifier_profile", "notifier_profile TEXT" - ) - if "delivery_mode" not in notify_cols: - _add_column_if_missing( - conn, - "kanban_notify_subs", - "delivery_mode", - "delivery_mode TEXT NOT NULL DEFAULT 'notify'", - ) - # Backfill: before this column existed, the notifier woke the - # originating session unconditionally whenever the task carried a - # session_id — every pre-existing gateway subscription had de - # facto active wake. Defaulting them to plain 'notify' would - # silently disable that behavior on upgrade. TUI/CLI rows keep - # 'notify' (matching _maybe_auto_subscribe, which only requests - # 'notify+wake' for gateway sessions). Runs ONLY on first-add of - # the column, so a user's later explicit downgrade is never - # overwritten by a re-migration. - conn.execute( - "UPDATE kanban_notify_subs SET delivery_mode = 'notify+wake' " - "WHERE platform != 'tui'" - ) - if "chat_type" not in notify_cols: - _add_column_if_missing( - conn, - "kanban_notify_subs", - "chat_type", - "chat_type TEXT", - ) - if "user_id_alt" not in notify_cols: - # Records the originating source's platform-specific stable alt ID - # (Signal UUID, Feishu union_id, ...) alongside ``user_id`` so an - # active-wake replay reconstructs the SAME ``build_session_key`` as - # the original event. ``build_session_key`` prefers ``user_id_alt`` - # over ``user_id`` when both are present (gateway/session.py); a - # wake that only replayed ``user_id`` would key to a different, - # context-less session whenever the two diverge. Legacy rows - # default to NULL, which is inert: ``user_id_alt or user_id`` falls - # back to the already-persisted ``user_id``. - _add_column_if_missing( - conn, "kanban_notify_subs", "user_id_alt", "user_id_alt TEXT" - ) - if "delivery_metadata" not in notify_cols: - _add_column_if_missing( - conn, "kanban_notify_subs", "delivery_metadata", "delivery_metadata TEXT" - ) + for name, ddl in ( + ("notifier_profile", "notifier_profile TEXT"), + ("delivery_mode", "delivery_mode TEXT NOT NULL DEFAULT 'notify'"), + ("chat_type", "chat_type TEXT"), + # Platform-specific stable alt ID (Signal UUID, Feishu union_id, ...) + # so an active-wake replay reconstructs the SAME ``build_session_key`` + # (which prefers ``user_id_alt`` over ``user_id``). NULL is inert. + ("user_id_alt", "user_id_alt TEXT"), + ("delivery_metadata", "delivery_metadata TEXT"), + ): + if name in notify_cols: + continue + _add_column_if_missing(conn, "kanban_notify_subs", name, ddl) + if name == "delivery_mode": + # Backfill ONLY on first-add: pre-column gateway subscriptions + # had de facto active wake (the notifier woke the originating + # session whenever the task carried a session_id); defaulting + # them to 'notify' would silently disable that on upgrade. + # TUI/CLI rows keep 'notify' (matches _maybe_auto_subscribe). + # A user's later explicit downgrade is never overwritten. + conn.execute( + "UPDATE kanban_notify_subs SET delivery_mode = 'notify+wake' " + "WHERE platform != 'tui'" + ) # One-shot backfill: any task that is 'running' before runs existed # had its claim_lock / claim_expires / worker_pid on the task row. @@ -2833,17 +2639,12 @@ def _migrate_add_optional_columns(conn: sqlite3.Connection) -> None: # "spawn_auto_blocked") still worked but were awkward on the wire; # rename them in-place so existing DBs migrate cleanly. Fires once # per DB because after the UPDATE no rows match the old kinds. - _EVENT_RENAMES = ( - # (old, new) - ("ready", "promoted"), - ("priority", "reprioritized"), + for old, new in ( + ("ready", "promoted"), + ("priority", "reprioritized"), ("spawn_auto_blocked", "gave_up"), - ) - for old, new in _EVENT_RENAMES: - conn.execute( - "UPDATE task_events SET kind = ? WHERE kind = ?", - (new, old), - ) + ): + conn.execute("UPDATE task_events SET kind = ? WHERE kind = ?", (new, old)) _rebuild_drifted_tables(conn) @@ -3153,6 +2954,11 @@ def _claimer_id() -> str: return f"{host}:{os.getpid()}" +def _host_prefix() -> str: + """``":"`` prefix shared by every claim lock issued from this host.""" + return f"{_claimer_id().split(':', 1)[0]}:" + + # --------------------------------------------------------------------------- # Task creation / mutation # --------------------------------------------------------------------------- @@ -3449,23 +3255,22 @@ def create_task( # compose create_task calls under one outer commit so the # dispatcher can never observe a partially constructed graph. with write_txn(conn, allow_nested=True): + # Parent ids are validated in every mode (even triage) so the + # eventual link rows don't dangle. + if parents: + missing = _find_missing_parents(conn, parents) + if missing: + raise ValueError(f"unknown parent task(s): {', '.join(missing)}") # Determine task status from parent status, unless the caller # parks it directly in blocked for human-ops review or in # triage for a specifier. if initial_status == "blocked": task_status = "blocked" - if parents: - missing = _find_missing_parents(conn, parents) - if missing: - raise ValueError(f"unknown parent task(s): {', '.join(missing)}") elif triage: task_status = "triage" else: task_status = "ready" if parents: - missing = _find_missing_parents(conn, parents) - if missing: - raise ValueError(f"unknown parent task(s): {', '.join(missing)}") # If any parent is not yet done, we're todo. rows = conn.execute( "SELECT status FROM tasks WHERE id IN " @@ -3474,12 +3279,6 @@ def create_task( ).fetchall() if any(r["status"] != "done" for r in rows): task_status = "todo" - # Even in triage mode we still need to validate parent ids - # so the eventual link rows don't dangle. - if triage and parents: - missing = _find_missing_parents(conn, parents) - if missing: - raise ValueError(f"unknown parent task(s): {', '.join(missing)}") # Project-linked worktree: a fresh worktree dir under the repo # plus a deterministic branch (project slug + task id). Together @@ -3526,22 +3325,19 @@ def create_task( project_id, tenant, idempotency_key, - int(max_runtime_seconds) if max_runtime_seconds is not None else None, + _opt_int(max_runtime_seconds), json.dumps(skills_list) if skills_list is not None else None, - int(max_retries) if max_retries is not None else None, + _opt_int(max_retries), model_override, provider_override, reasoning_effort, 1 if goal_mode else 0, - int(goal_max_turns) if goal_max_turns is not None else None, + _opt_int(goal_max_turns), session_id, ), ) for pid in parents: - conn.execute( - "INSERT OR IGNORE INTO task_links (parent_id, child_id) VALUES (?, ?)", - (pid, task_id), - ) + _link(conn, pid, task_id) # Notify-sub inheritance (ACK-edge: the originating channel # still hears about a child that BLOCKs, not just the final # fan-in) is handled by the single-owner helper below — @@ -3575,6 +3371,13 @@ def create_task( raise RuntimeError("unreachable") +def _link(conn: sqlite3.Connection, parent_id: str, child_id: str) -> None: + conn.execute( + "INSERT OR IGNORE INTO task_links (parent_id, child_id) VALUES (?, ?)", + (parent_id, child_id), + ) + + def _find_missing_parents(conn: sqlite3.Connection, parents: Iterable[str]) -> list[str]: parents = list(parents) if not parents: @@ -3773,12 +3576,10 @@ def set_model_override( if not model: provider = None with write_txn(conn): - row = conn.execute( - "SELECT status FROM tasks WHERE id = ?", (task_id,) - ).fetchone() - if not row: + status = _task_status(conn, task_id) + if status is None: return False - if row["status"] == "archived": + if status == "archived": raise RuntimeError(f"cannot set model override on archived task {task_id}") conn.execute( "UPDATE tasks SET model_override = ?, provider_override = ? WHERE id = ?", @@ -3812,12 +3613,10 @@ def set_reasoning_effort( """ effort = normalize_reasoning_effort(effort) with write_txn(conn): - row = conn.execute( - "SELECT status FROM tasks WHERE id = ?", (task_id,) - ).fetchone() - if not row: + status = _task_status(conn, task_id) + if status is None: return False - if row["status"] == "archived": + if status == "archived": raise RuntimeError( f"cannot set reasoning effort on archived task {task_id}" ) @@ -3848,15 +3647,9 @@ def link_tasks(conn: sqlite3.Connection, parent_id: str, child_id: str) -> None: raise ValueError( f"linking {parent_id} -> {child_id} would create a cycle" ) - conn.execute( - "INSERT OR IGNORE INTO task_links (parent_id, child_id) VALUES (?, ?)", - (parent_id, child_id), - ) + _link(conn, parent_id, child_id) # If child was ready but parent is not yet done, demote child to todo. - parent_status = conn.execute( - "SELECT status FROM tasks WHERE id = ?", (parent_id,) - ).fetchone()["status"] - if parent_status != "done": + if _task_status(conn, parent_id) != "done": conn.execute( "UPDATE tasks SET status = 'todo' WHERE id = ? AND status = 'ready'", (child_id,), @@ -3971,21 +3764,6 @@ def task_graph_context(conn: sqlite3.Connection, task_id: str) -> dict: return task_graph_contexts(conn, [task_id])[task_id] -def parent_results(conn: sqlite3.Connection, task_id: str) -> list[tuple[str, Optional[str]]]: - """Return ``(parent_id, result)`` for every done parent of ``task_id``.""" - rows = conn.execute( - """ - SELECT t.id AS id, t.result AS result - FROM tasks t - JOIN task_links l ON l.parent_id = t.id - WHERE l.child_id = ? AND t.status = 'done' - ORDER BY t.completed_at ASC - """, - (task_id,), - ).fetchall() - return [(r["id"], r["result"]) for r in rows] - - # --------------------------------------------------------------------------- # Comments & events # --------------------------------------------------------------------------- @@ -4019,16 +3797,7 @@ def list_comments(conn: sqlite3.Connection, task_id: str) -> list[Comment]: "SELECT * FROM task_comments WHERE task_id = ? ORDER BY created_at ASC", (task_id,), ).fetchall() - return [ - Comment( - id=r["id"], - task_id=r["task_id"], - author=r["author"], - body=r["body"], - created_at=r["created_at"], - ) - for r in rows - ] + return [Comment.from_row(r) for r in rows] def list_comments_after( @@ -4046,16 +3815,7 @@ def list_comments_after( "WHERE task_id = ? AND id > ? ORDER BY id ASC", (task_id, int(after_id)), ).fetchall() - return [ - Comment( - id=r["id"], - task_id=r["task_id"], - author=r["author"], - body=r["body"], - created_at=r["created_at"], - ) - for r in rows - ] + return [Comment.from_row(r) for r in rows] # --------------------------------------------------------------------------- @@ -4227,37 +3987,14 @@ def list_attachments(conn: sqlite3.Connection, task_id: str) -> list[Attachment] "SELECT * FROM task_attachments WHERE task_id = ? ORDER BY created_at ASC, id ASC", (task_id,), ).fetchall() - return [ - Attachment( - id=r["id"], - task_id=r["task_id"], - filename=r["filename"], - stored_path=r["stored_path"], - content_type=r["content_type"], - size=r["size"] or 0, - uploaded_by=r["uploaded_by"], - created_at=r["created_at"], - ) - for r in rows - ] + return [Attachment.from_row(r) for r in rows] def get_attachment(conn: sqlite3.Connection, attachment_id: int) -> Optional[Attachment]: r = conn.execute( "SELECT * FROM task_attachments WHERE id = ?", (attachment_id,) ).fetchone() - if r is None: - return None - return Attachment( - id=r["id"], - task_id=r["task_id"], - filename=r["filename"], - stored_path=r["stored_path"], - content_type=r["content_type"], - size=r["size"] or 0, - uploaded_by=r["uploaded_by"], - created_at=r["created_at"], - ) + return None if r is None else Attachment.from_row(r) def delete_attachment(conn: sqlite3.Connection, attachment_id: int) -> Optional[Attachment]: @@ -4289,23 +4026,22 @@ def list_events(conn: sqlite3.Connection, task_id: str) -> list[Event]: "SELECT * FROM task_events WHERE task_id = ? ORDER BY created_at ASC, id ASC", (task_id,), ).fetchall() - out = [] - for r in rows: - try: - payload = json.loads(r["payload"]) if r["payload"] else None - except Exception: - payload = None - out.append( - Event( - id=r["id"], - task_id=r["task_id"], - kind=r["kind"], - payload=payload, - created_at=r["created_at"], - run_id=(int(r["run_id"]) if "run_id" in r.keys() and r["run_id"] is not None else None), - ) - ) - return out + return [Event.from_row(r) for r in rows] + + +def _insert_comment( + conn: sqlite3.Connection, task_id: str, author: str, body: str, created_at: int, +) -> None: + """Raw ``task_comments`` INSERT for callers already inside a write txn. + + ``add_comment`` opens its own ``write_txn`` (raises on nesting) and emits + a ``commented`` event; transitions that record their own event use this. + """ + conn.execute( + "INSERT INTO task_comments (task_id, author, body, created_at) " + "VALUES (?, ?, ?, ?)", + (task_id, author, body, created_at), + ) def _append_event( @@ -4352,12 +4088,9 @@ def _end_run( task that was never claimed). """ now = int(time.time()) - row = conn.execute( - "SELECT current_run_id FROM tasks WHERE id = ?", (task_id,), - ).fetchone() - if not row or not row["current_run_id"]: + run_id = _current_run_id(conn, task_id) + if run_id is None: return None - run_id = int(row["current_run_id"]) conn.execute( """ UPDATE task_runs @@ -4389,6 +4122,17 @@ def _end_run( return run_id +def _opt_int(value: Any) -> Optional[int]: + """``int(value)`` or ``None`` when ``value`` is ``None`` (NULL column passthrough).""" + return int(value) if value is not None else None + + +def _task_status(conn: sqlite3.Connection, task_id: str) -> Optional[str]: + """Current ``tasks.status`` for ``task_id``, or ``None`` when no such row.""" + row = conn.execute("SELECT status FROM tasks WHERE id = ?", (task_id,)).fetchone() + return row["status"] if row else None + + def _current_run_id(conn: sqlite3.Connection, task_id: str) -> Optional[int]: row = conn.execute( "SELECT current_run_id FROM tasks WHERE id = ?", (task_id,), @@ -4453,32 +4197,17 @@ def _synthesize_ended_run( def _has_sticky_block(conn: sqlite3.Connection, task_id: str) -> bool: """Return True when ``task_id`` is sticky-blocked by an explicit - worker/operator ``kanban_block`` call (#28712). + worker/operator ``kanban_block`` call. - A ``blocked`` status can come from two very different sources: - - * **Worker- or operator-initiated** — a worker called - ``kanban_block(reason="review-required: ...")`` (or somebody ran - ``hermes kanban block ``). This is a deliberate handoff that - should stay blocked until an operator unblocks it. The block tool - emits a ``"blocked"`` event row in ``task_events``. - - * **Circuit-breaker** — ``_record_task_failure`` tripped after - repeated crashes / spawn failures / timeouts. This emits - ``"gave_up"``, *not* ``"blocked"``, and is meant to recover - automatically once the underlying conditions change (e.g. parents - finish, transient infra error clears). - - The cheapest signal that distinguishes the two is the most recent - ``"blocked"`` / ``"unblocked"`` event for the task. If the most - recent one is ``"blocked"`` (or there is a ``"blocked"`` event and - no ``"unblocked"`` event has fired since), the task is sticky and - ``recompute_ready`` must *not* auto-promote it. - - Returns ``False`` when there is no such event at all (e.g. the task - was set to ``status='blocked'`` by the circuit breaker or by direct - DB manipulation) — preserves the pre-#28712 auto-recover semantics - for that path. + A ``blocked`` status has two sources: a deliberate worker/operator + ``kanban_block`` handoff (emits a ``"blocked"`` event; must stay blocked + until an operator unblocks it) and a circuit-breaker trip in + ``_record_task_failure`` (emits ``"gave_up"``, NOT ``"blocked"``; meant to + recover once conditions change). The cheapest discriminator is the most + recent ``"blocked"``/``"unblocked"`` event: if it is ``"blocked"`` the task + is sticky and ``recompute_ready`` must not auto-promote it. No such event + at all (breaker trip, direct DB edit) returns ``False`` — the legacy + auto-recover path. """ row = conn.execute( "SELECT kind FROM task_events " @@ -4489,6 +4218,18 @@ def _has_sticky_block(conn: sqlite3.Connection, task_id: str) -> bool: return bool(row) and row["kind"] == "blocked" +def _latest_event( + conn: sqlite3.Connection, task_id: str, kind: str, run_id: Optional[int] = None, +) -> Optional[sqlite3.Row]: + """Newest ``task_events`` row of ``kind`` (optionally scoped to one run).""" + sql = "SELECT payload FROM task_events WHERE task_id = ? AND kind = ?" + params: tuple[Any, ...] = (task_id, kind) + if run_id is not None: + sql += " AND run_id = ?" + params = (*params, int(run_id)) + return conn.execute(sql + " ORDER BY id DESC LIMIT 1", params).fetchone() + + def _resume_status_from_events(conn: sqlite3.Connection, task_id: str) -> str: """Return the durable phase a blocked/dependency-wait task should resume. @@ -4506,12 +4247,7 @@ def _resume_status_from_events(conn: sqlite3.Connection, task_id: str) -> str: ") ORDER BY id DESC LIMIT 1", (task_id,), ).fetchone() - try: - payload = json.loads(row["payload"]) if row and row["payload"] else {} - except (json.JSONDecodeError, TypeError): - payload = {} - if not isinstance(payload, dict): - payload = {} + payload = _json_dict(_row_get(row, "payload")) for key in ("resume_status", "retry_status", "source_status"): if payload.get(key) == "review": return "review" @@ -4527,28 +4263,17 @@ def recompute_ready( MUST be called OUTSIDE any open write transaction (plain ``write_txn`` raises on nesting); call it after the enclosing txn commits. - ``blocked`` tasks are also considered for promotion (so a task - blocked purely by a parent dependency unblocks itself when the - parent completes), *except* in two cases: + ``blocked`` tasks are also considered (a task blocked purely by a parent + dependency unblocks itself when the parent completes), *except* when the + most recent block event was a worker-initiated ``kanban_block`` (stays + blocked until explicit ``kanban_unblock``) or ``consecutive_failures`` has + reached the effective limit (otherwise the counter would reset on every + recovery cycle and the breaker could never trip). - 1. The most recent block event was a worker-initiated - ``kanban_block`` — those stay blocked until an explicit - ``kanban_unblock`` (#28712). - - 2. The task's ``consecutive_failures`` has reached the effective - failure limit. This prevents infinite retry loops when a task - repeatedly exhausts its iteration budget: without this guard the - counter would reset on every recovery cycle and the circuit - breaker could never trip (#35072). - - The effective failure limit resolves in the same order as the - circuit breaker in ``_record_task_failure`` so the two never - disagree about when a task is permanently blocked: - - 1. per-task ``max_retries`` if set - 2. caller-supplied ``failure_limit`` (the dispatcher passes the - ``kanban.failure_limit`` config value through ``dispatch_once``) - 3. ``DEFAULT_FAILURE_LIMIT`` + The effective limit resolves in the same order as ``_record_task_failure`` + so the two never disagree: per-task ``max_retries``, then the caller's + ``failure_limit`` (``kanban.failure_limit`` via ``dispatch_once``), then + ``DEFAULT_FAILURE_LIMIT``. """ if failure_limit is None: failure_limit = DEFAULT_FAILURE_LIMIT @@ -4625,6 +4350,73 @@ def _parents_satisfied(conn: sqlite3.Connection, task_id: str) -> bool: ).fetchone() is None +def _claim_and_open_run( + conn: sqlite3.Connection, + task_id: str, + source_status: str, + lock: str, + expires: int, + now: int, + *, + event_extra: Optional[dict] = None, +) -> Optional[int]: + """CAS ``source_status -> running``, open a ``task_runs`` row and emit ``claimed``. + + Caller holds the write transaction. Returns the new run id, or ``None`` + when the CAS lost (task already claimed / not in ``source_status``). + """ + cur = conn.execute( + f""" + UPDATE tasks + SET status = 'running', + claim_lock = ?, + claim_expires = ?, + started_at = COALESCE(started_at, ?) + WHERE id = ? + AND status = '{source_status}' + AND claim_lock IS NULL + """, + (lock, expires, now, task_id), + ) + if cur.rowcount != 1: + return None + # Populate the run with the task's assignee / step / runtime cap. + trow = conn.execute( + "SELECT assignee, max_runtime_seconds, current_step_key " + "FROM tasks WHERE id = ?", + (task_id,), + ).fetchone() + run_cur = conn.execute( + """ + INSERT INTO task_runs ( + task_id, profile, step_key, status, + claim_lock, claim_expires, max_runtime_seconds, + started_at + ) VALUES (?, ?, ?, 'running', ?, ?, ?, ?) + """, + ( + task_id, + trow["assignee"] if trow else None, + trow["current_step_key"] if trow else None, + lock, + expires, + trow["max_runtime_seconds"] if trow else None, + now, + ), + ) + run_id = run_cur.lastrowid + conn.execute( + "UPDATE tasks SET current_run_id = ? WHERE id = ?", + (run_id, task_id), + ) + _append_event( + conn, task_id, "claimed", + {"lock": lock, "expires": expires, "run_id": run_id, **(event_extra or {})}, + run_id=run_id, + ) + return run_id + + def claim_task( conn: sqlite3.Connection, task_id: str, @@ -4649,13 +4441,7 @@ def claim_task( # 'todo' here — recompute_ready will re-promote when the parents # actually finish. See RCA at # kanban/boards/cookai/workspaces/t_a6acd07d/root-cause.md. - undone = conn.execute( - "SELECT 1 FROM task_links l " - "JOIN tasks p ON p.id = l.parent_id " - "WHERE l.child_id = ? AND p.status NOT IN ('done', 'archived') LIMIT 1", - (task_id,), - ).fetchone() - if undone: + if not _parents_satisfied(conn, task_id): conn.execute( "UPDATE tasks SET status = 'todo' " "WHERE id = ? AND status = 'ready'", @@ -4666,76 +4452,15 @@ def claim_task( {"reason": "parents_not_done"}, ) return None - # Defensive: if a prior run somehow leaked (invariant violation from - # an unknown code path), close it as 'reclaimed' so we don't strand - # it when the CAS resets the pointer below. No-op when the invariant - # holds (the common case). - stale = conn.execute( - "SELECT current_run_id FROM tasks WHERE id = ? AND status = 'ready'", - (task_id,), - ).fetchone() - if stale and stale["current_run_id"]: - conn.execute( - """ - UPDATE task_runs - SET status = 'reclaimed', outcome = 'reclaimed', - summary = COALESCE(summary, 'invariant recovery on re-claim'), - ended_at = ?, - claim_lock = NULL, claim_expires = NULL, worker_pid = NULL - WHERE id = ? AND ended_at IS NULL - """, - (now, int(stale["current_run_id"])), - ) - cur = conn.execute( - """ - UPDATE tasks - SET status = 'running', - claim_lock = ?, - claim_expires = ?, - started_at = COALESCE(started_at, ?) - WHERE id = ? - AND status = 'ready' - AND claim_lock IS NULL - """, - (lock, expires, now, task_id), + # Defensive: close a leaked prior run as 'reclaimed' so the CAS below + # doesn't strand it. No-op when the runs invariant holds. + _reclaim_dangling_run( + conn, task_id, statuses=("ready",), now=now, + note="invariant recovery on re-claim", ) - if cur.rowcount != 1: + run_id = _claim_and_open_run(conn, task_id, "ready", lock, expires, now) + if run_id is None: return None - # Look up the current task row so we can populate the run with - # its assignee / step / runtime cap. - trow = conn.execute( - "SELECT assignee, max_runtime_seconds, current_step_key " - "FROM tasks WHERE id = ?", - (task_id,), - ).fetchone() - run_cur = conn.execute( - """ - INSERT INTO task_runs ( - task_id, profile, step_key, status, - claim_lock, claim_expires, max_runtime_seconds, - started_at - ) VALUES (?, ?, ?, 'running', ?, ?, ?, ?) - """, - ( - task_id, - trow["assignee"] if trow else None, - trow["current_step_key"] if trow else None, - lock, - expires, - trow["max_runtime_seconds"] if trow else None, - now, - ), - ) - run_id = run_cur.lastrowid - conn.execute( - "UPDATE tasks SET current_run_id = ? WHERE id = ?", - (run_id, task_id), - ) - _append_event( - conn, task_id, "claimed", - {"lock": lock, "expires": expires, "run_id": run_id}, - run_id=run_id, - ) claimed = get_task(conn, task_id) _fire_kanban_lifecycle_hook( "kanban_task_claimed", @@ -4786,55 +4511,12 @@ def claim_review_task( }, ) return None - cur = conn.execute( - """ - UPDATE tasks - SET status = 'running', - claim_lock = ?, - claim_expires = ?, - started_at = COALESCE(started_at, ?) - WHERE id = ? - AND status = 'review' - AND claim_lock IS NULL - """, - (lock, expires, now, task_id), + run_id = _claim_and_open_run( + conn, task_id, "review", lock, expires, now, + event_extra={"source_status": "review"}, ) - if cur.rowcount != 1: + if run_id is None: return None - trow = conn.execute( - "SELECT assignee, max_runtime_seconds, current_step_key " - "FROM tasks WHERE id = ?", - (task_id,), - ).fetchone() - run_cur = conn.execute( - """ - INSERT INTO task_runs ( - task_id, profile, step_key, status, - claim_lock, claim_expires, max_runtime_seconds, - started_at - ) VALUES (?, ?, ?, 'running', ?, ?, ?, ?) - """, - ( - task_id, - trow["assignee"] if trow else None, - trow["current_step_key"] if trow else None, - lock, - expires, - trow["max_runtime_seconds"] if trow else None, - now, - ), - ) - run_id = run_cur.lastrowid - conn.execute( - "UPDATE tasks SET current_run_id = ? WHERE id = ?", - (run_id, task_id), - ) - _append_event( - conn, task_id, "claimed", - {"lock": lock, "expires": expires, "run_id": run_id, - "source_status": "review"}, - run_id=run_id, - ) return get_task(conn, task_id) @@ -4851,25 +4533,11 @@ def _retry_status_for_run( reviewer run into an implementation run. """ if run_id is None: - row = conn.execute( - "SELECT current_run_id FROM tasks WHERE id = ?", - (task_id,), - ).fetchone() - run_id = row["current_run_id"] if row else None + run_id = _current_run_id(conn, task_id) if run_id is None: return "ready" - event = conn.execute( - "SELECT payload FROM task_events " - "WHERE task_id = ? AND run_id = ? AND kind = 'claimed' " - "ORDER BY id DESC LIMIT 1", - (task_id, int(run_id)), - ).fetchone() - try: - payload = json.loads(event["payload"]) if event and event["payload"] else {} - except (json.JSONDecodeError, TypeError): - payload = {} - if not isinstance(payload, dict): - payload = {} + event = _latest_event(conn, task_id, "claimed", run_id) + payload = _json_dict(_row_get(event, "payload")) return "review" if payload.get("source_status") == "review" else "ready" @@ -4963,31 +4631,24 @@ def release_stale_claims( """Reset any ``running`` task whose claim has expired. A stale-by-TTL claim whose host-local worker PID is still alive is - *extended* (with a ``claim_extended`` event) instead of being - reclaimed. Reclaiming a live worker mid-flight produces the spawn- - then-immediately-reclaim loop seen on slow models that spend longer - than ``DEFAULT_CLAIM_TTL_SECONDS`` inside a single tool-free LLM - call (#23025): no tool calls means no ``kanban_heartbeat``, even - though the subprocess is healthy. + *extended* (``claim_extended`` event) instead of reclaimed: reclaiming a + live worker mid-flight causes a spawn-then-reclaim loop on slow models + that spend longer than ``DEFAULT_CLAIM_TTL_SECONDS`` inside one tool-free + LLM call (no tool calls means no ``kanban_heartbeat``). - Backstop (#29747 gap 3): if the worker's PID is still alive but its - ``last_heartbeat_at`` is stale by more than - ``DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS`` (1h), the worker has - been making no observable progress and we reclaim anyway — even if - ``_pid_alive`` is still true. This catches the wedged-in-a-logic-loop - case where the process is technically running but accomplishing - nothing. ``_touch_activity`` (run_agent.py) bridges chunk-level - liveness into ``last_heartbeat_at`` via #31752, so any genuinely - active worker keeps its heartbeat fresh as a side effect of normal - API traffic. ``enforce_max_runtime`` and ``detect_crashed_workers`` - remain the upper bounds for genuinely wedged or dead workers. + Backstop: a live PID whose ``last_heartbeat_at`` is older than + ``DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS`` is reclaimed anyway — the + wedged-in-a-logic-loop case. ``_touch_activity`` (run_agent.py) bridges + chunk-level liveness into ``last_heartbeat_at``, so any genuinely active + worker stays fresh via normal API traffic. ``enforce_max_runtime`` and + ``detect_crashed_workers`` remain the upper bounds for wedged/dead workers. Returns the number of stale claims actually reclaimed (live-pid extensions don't count). Safe to call often. """ now = int(time.time()) reclaimed = 0 - host_prefix = f"{_claimer_id().split(':', 1)[0]}:" + host_prefix = _host_prefix() stale = conn.execute( "SELECT id, claim_lock, worker_pid, claim_expires, last_heartbeat_at, " " assignee " @@ -5040,11 +4701,7 @@ def release_stale_claims( "claim_lock": row["claim_lock"], "claim_expires_was": int(row["claim_expires"]), "claim_expires_now": new_expires, - "last_heartbeat_at": ( - int(row["last_heartbeat_at"]) - if row["last_heartbeat_at"] is not None - else None - ), + "last_heartbeat_at": _opt_int(row["last_heartbeat_at"]), }, run_id=run_id, ) @@ -5080,15 +4737,9 @@ def release_stale_claims( ) payload = { "stale_lock": row["claim_lock"], - "worker_pid": ( - int(row["worker_pid"]) - if row["worker_pid"] is not None else None - ), + "worker_pid": _opt_int(row["worker_pid"]), "claim_expires": int(row["claim_expires"]), - "last_heartbeat_at": ( - int(row["last_heartbeat_at"]) - if row["last_heartbeat_at"] is not None else None - ), + "last_heartbeat_at": _opt_int(row["last_heartbeat_at"]), "now": now, "host_local": host_local, "heartbeat_stale": bool(heartbeat_stale), @@ -5112,10 +4763,7 @@ def release_stale_claims( board=get_current_board(), assignee=row["assignee"], run_id=run_id, - worker_pid=( - int(row["worker_pid"]) - if row["worker_pid"] is not None else None - ), + worker_pid=_opt_int(row["worker_pid"]), heartbeat_stale=bool(heartbeat_stale), retry_status=retry_status, ) @@ -5288,11 +4936,11 @@ def _verify_created_cards( phantom.append(cid) continue # Accept if any of the three trust conditions holds. - if completing_assignee and created_by == completing_assignee: - verified.append(cid) - elif created_by == completing_task_id: - verified.append(cid) - elif cid in linked_children: + if ( + (completing_assignee and created_by == completing_assignee) + or created_by == completing_task_id + or cid in linked_children + ): verified.append(cid) else: phantom.append(cid) @@ -5445,14 +5093,8 @@ def complete_task( # ``review`` or ``running``. if not _parents_satisfied(conn, task_id): return False - prior = conn.execute( - "SELECT status FROM tasks WHERE id = ?", - (task_id,), - ).fetchone() - prior_status = prior["status"] if prior else None - if expected_run_id is None: - cur = conn.execute( - """ + prior_status = _task_status(conn, task_id) + sql = """ UPDATE tasks SET status = 'done', result = ?, @@ -5464,27 +5106,12 @@ def complete_task( block_recurrences = 0 WHERE id = ? AND status IN ('running', 'ready', 'blocked', 'review') - """, - (result, now, task_id), - ) - else: - cur = conn.execute( """ - UPDATE tasks - SET status = 'done', - result = ?, - completed_at = ?, - claim_lock = NULL, - claim_expires= NULL, - worker_pid = NULL, - block_kind = NULL, - block_recurrences = 0 - WHERE id = ? - AND status IN ('running', 'ready', 'blocked', 'review') - AND current_run_id = ? - """, - (result, now, task_id, int(expected_run_id)), - ) + params: tuple = (result, now, task_id) + if expected_run_id is not None: + sql += " AND current_run_id = ?" + params = (*params, int(expected_run_id)) + cur = conn.execute(sql, params) if cur.rowcount != 1: return False if isinstance(metadata, dict): @@ -5622,13 +5249,9 @@ def _merge_completion_prose_artifacts( deliverable path in ``summary``/``result``; discover it while scratch still exists so cleanup cannot erase the file the user was promised. """ - row = conn.execute( - "SELECT workspace_kind, workspace_path FROM tasks WHERE id = ?", - (task_id,), - ).fetchone() - if not row or row["workspace_kind"] != "scratch" or not row["workspace_path"]: + workspace = _scratch_workspace(conn, task_id) + if workspace is None: return metadata - workspace = Path(row["workspace_path"]).expanduser() if not _is_managed_scratch_path(workspace): return metadata text = "\n".join(part for part in (summary, result) if part) @@ -5665,14 +5288,9 @@ def _persist_scratch_completion_artifacts( if not isinstance(raw_artifacts, (list, tuple)): return - row = conn.execute( - "SELECT workspace_kind, workspace_path FROM tasks WHERE id = ?", - (task_id,), - ).fetchone() - if not row or row["workspace_kind"] != "scratch" or not row["workspace_path"]: + workspace = _scratch_workspace(conn, task_id) + if workspace is None: return - - workspace = Path(row["workspace_path"]).expanduser() is_managed, board = _managed_scratch_path_info(workspace) if not is_managed: return @@ -5857,6 +5475,17 @@ def _managed_scratch_path_info(p: Path) -> tuple[bool, Optional[str]]: return False, None +def _scratch_workspace(conn: sqlite3.Connection, task_id: str) -> Optional[Path]: + """Expanded ``workspace_path`` when the task uses a scratch workspace, else ``None``.""" + row = conn.execute( + "SELECT workspace_kind, workspace_path FROM tasks WHERE id = ?", + (task_id,), + ).fetchone() + if not row or row["workspace_kind"] != "scratch" or not row["workspace_path"]: + return None + return Path(row["workspace_path"]).expanduser() + + def _is_managed_scratch_path(p: Path) -> bool: """Return True iff *p* is a strict descendant of a kanban-managed scratch root. @@ -6106,21 +5735,11 @@ def _cleanup_worker_tmux(conn: sqlite3.Connection, task_id: str) -> None: # First-use tip for scratch workspaces # --------------------------------------------------------------------------- # -# Scratch workspaces are intentionally ephemeral — ``_cleanup_workspace`` -# removes them as soon as ``complete_task`` runs. New users often don't -# realize that and lose worker output (community report, May 2026). The -# behavior is right; the lack of warning is the bug. -# -# On the FIRST scratch workspace materialization across the whole install -# we: -# 1. Log a warning line on the dispatcher logger. -# 2. Append a ``tip_scratch_workspace`` event on the task so it's visible -# via ``hermes kanban show `` and the dashboard. -# 3. Touch a sentinel file under ``kanban_home() / '.scratch_tip_shown'`` -# so we don't repeat the tip — once you know, you know. -# -# Scope is per-install, not per-board: a user creating a second board -# already learned the lesson on board #1. +# Scratch workspaces are intentionally ephemeral (``_cleanup_workspace`` removes +# them on ``complete_task``); new users lose worker output without warning. On +# the FIRST scratch materialization per install: log a warning, append a +# ``tip_scratch_workspace`` event on the task, and touch a sentinel under +# ``kanban_home()`` so the tip never repeats. Per-install, not per-board. _SCRATCH_TIP_SENTINEL_NAME = ".scratch_tip_shown" @@ -6200,10 +5819,7 @@ def edit_completed_task_result( """Backfill the user-visible result for an already completed task.""" handoff_summary = summary if summary is not None else result with write_txn(conn): - row = conn.execute( - "SELECT status FROM tasks WHERE id = ?", (task_id,), - ).fetchone() - if not row or row["status"] != "done": + if _task_status(conn, task_id) != "done": return False conn.execute( "UPDATE tasks SET result = ? WHERE id = ?", @@ -6293,7 +5909,6 @@ def block_task( raise ValueError( f"block kind must be one of {sorted(VALID_BLOCK_KINDS)} or None" ) - recurrences = 0 with write_txn(conn): cur_row = conn.execute( "SELECT status, block_kind, block_recurrences FROM tasks WHERE id = ?", @@ -6306,54 +5921,73 @@ def block_task( if cur_row["status"] == "running" else "ready" ) - prev_kind = cur_row["block_kind"] if "block_kind" in cur_row.keys() else None - prev_recurrences = ( - int(cur_row["block_recurrences"]) - if "block_recurrences" in cur_row.keys() - and cur_row["block_recurrences"] is not None - else 0 - ) + prev_kind = _row_get(cur_row, "block_kind") + prev_recurrences = _row_get(cur_row, "block_recurrences") + prev_recurrences = int(prev_recurrences) if prev_recurrences is not None else 0 # Dependency blocks never enter the human ``blocked`` bucket — they # wait in ``todo`` and let ``recompute_ready`` gate on parents. Routing # here (rather than ``blocked``) is what keeps a cron from ever seeing # a dependency-wait as something to "unblock". if kind == "dependency": - cur = conn.execute( - """ + new_status, event_kind = "todo", "dependency_wait" + set_sql, params = "block_kind = ?", (kind,) + payload = {"reason": reason, "kind": kind, "source_status": source_status} + else: + # Truly-blocked kinds. Increment the unblock-loop counter when this + # is a re-block for the SAME reason after a prior unblock: block_task + # only fires from running/ready (i.e. AFTER an unblock returned the + # task to the work pool), so a stored block_kind matching the + # incoming kind means blocked → unblocked → re-block, same cause. + # An un-typed (None) block compares as "same" to a prior un-typed one. + recurrences = prev_recurrences + 1 if prev_kind == kind else 1 + set_sql = "block_kind = ?,\n block_recurrences = ?" + params = (kind, recurrences) + payload = { + "reason": reason, + "kind": kind, + "recurrences": recurrences, + "source_status": source_status, + } + if recurrences >= BLOCK_RECURRENCE_LIMIT: + # Loop detected — route to triage for a human-in-the-loop + # decision instead of letting the unblocker spin this task. + new_status, event_kind = "triage", "block_loop_detected" + payload["limit"] = BLOCK_RECURRENCE_LIMIT + else: + new_status, event_kind = "blocked", "blocked" + sql = f""" UPDATE tasks - SET status = 'todo', + SET status = '{new_status}', claim_lock = NULL, claim_expires = NULL, worker_pid = NULL, - block_kind = ? + {set_sql} WHERE id = ? AND status IN ('running', 'ready') - """ + ("" if expected_run_id is None else " AND current_run_id = ?"), - (kind, task_id) if expected_run_id is None - else (kind, task_id, int(expected_run_id)), + """ + params = (*params, task_id) + if expected_run_id is not None: + sql += " AND current_run_id = ?" + params = (*params, int(expected_run_id)) + cur = conn.execute(sql, params) + if cur.rowcount != 1: + return False + run_id = _end_run( + conn, task_id, + outcome="blocked", status="blocked", + summary=reason, + ) + # Synthesize a run when blocking a never-claimed task so the reason + # is preserved in attempt history. + if run_id is None and reason: + run_id = _synthesize_ended_run( + conn, task_id, outcome="blocked", summary=reason, ) - if cur.rowcount != 1: - return False - run_id = _end_run( - conn, task_id, - outcome="blocked", status="blocked", - summary=reason, - ) - if run_id is None and reason: - run_id = _synthesize_ended_run( - conn, task_id, outcome="blocked", summary=reason, - ) - _append_event( - conn, task_id, "dependency_wait", - { - "reason": reason, - "kind": kind, - "source_status": source_status, - }, - run_id=run_id, - ) - _blocked_task = get_task(conn, task_id) + _append_event(conn, task_id, event_kind, payload, run_id=run_id) + _blocked_task = get_task(conn, task_id) + + def _fire_blocked_hook() -> None: _fire_kanban_lifecycle_hook( "kanban_task_blocked", task_id, @@ -6362,127 +5996,15 @@ def block_task( run_id=run_id, reason=reason, ) + + if kind == "dependency": + # Historical ordering: the dependency lane fires inside the txn. + _fire_blocked_hook() return True - - # Truly-blocked kinds. Increment the unblock-loop counter when this is a - # re-block for the SAME reason after a prior unblock. block_task only - # fires from running/ready (i.e. AFTER an unblock returned the task to - # the work pool), so a stored block_kind that matches the incoming kind - # means: blocked → unblocked → about-to-re-block for the same cause. - # An un-typed (None) block compares as "same" to a prior un-typed block. - same_cause = prev_kind == kind - recurrences = prev_recurrences + 1 if same_cause else 1 - - if recurrences >= BLOCK_RECURRENCE_LIMIT: - # Loop detected — stop letting the unblocker spin this task. Route - # to triage for a human-in-the-loop decision instead of blocked. - cur = conn.execute( - """ - UPDATE tasks - SET status = 'triage', - claim_lock = NULL, - claim_expires = NULL, - worker_pid = NULL, - block_kind = ?, - block_recurrences = ? - WHERE id = ? - AND status IN ('running', 'ready') - """ + ("" if expected_run_id is None else " AND current_run_id = ?"), - (kind, recurrences, task_id) if expected_run_id is None - else (kind, recurrences, task_id, int(expected_run_id)), - ) - if cur.rowcount != 1: - return False - run_id = _end_run( - conn, task_id, - outcome="blocked", status="blocked", - summary=reason, - ) - if run_id is None and reason: - run_id = _synthesize_ended_run( - conn, task_id, outcome="blocked", summary=reason, - ) - _append_event( - conn, task_id, "block_loop_detected", - { - "reason": reason, - "kind": kind, - "recurrences": recurrences, - "limit": BLOCK_RECURRENCE_LIMIT, - "source_status": source_status, - }, - run_id=run_id, - ) - else: - if expected_run_id is None: - cur = conn.execute( - """ - UPDATE tasks - SET status = 'blocked', - claim_lock = NULL, - claim_expires = NULL, - worker_pid = NULL, - block_kind = ?, - block_recurrences = ? - WHERE id = ? - AND status IN ('running', 'ready') - """, - (kind, recurrences, task_id), - ) - else: - cur = conn.execute( - """ - UPDATE tasks - SET status = 'blocked', - claim_lock = NULL, - claim_expires = NULL, - worker_pid = NULL, - block_kind = ?, - block_recurrences = ? - WHERE id = ? - AND status IN ('running', 'ready') - AND current_run_id = ? - """, - (kind, recurrences, task_id, int(expected_run_id)), - ) - if cur.rowcount != 1: - return False - run_id = _end_run( - conn, task_id, - outcome="blocked", status="blocked", - summary=reason, - ) - # Synthesize a run when blocking a never-claimed task so the - # reason is preserved in attempt history. - if run_id is None and reason: - run_id = _synthesize_ended_run( - conn, task_id, - outcome="blocked", - summary=reason, - ) - _append_event( - conn, task_id, "blocked", - { - "reason": reason, - "kind": kind, - "recurrences": recurrences, - "source_status": source_status, - }, - run_id=run_id, - ) - _blocked_task = get_task(conn, task_id) - _fire_kanban_lifecycle_hook( - "kanban_task_blocked", - task_id, - board=get_current_board(), - assignee=_blocked_task.assignee if _blocked_task else None, - run_id=run_id, - reason=reason, - ) + _fire_blocked_hook() return True - def redact_review_value(value: Any) -> Any: """Redact secrets at the domain boundary for durable review handoffs.""" if isinstance(value, str): @@ -6567,26 +6089,10 @@ def request_review( ).fetchone() changes_event = None if changes_run is not None: - changes_event = conn.execute( - "SELECT payload FROM task_events " - "WHERE task_id = ? AND run_id = ? " - "AND kind = 'changes_requested' " - "ORDER BY id DESC LIMIT 1", - (task_id, int(changes_run["id"])), - ).fetchone() - try: - changes_payload = ( - json.loads(changes_event["payload"]) - if changes_event and changes_event["payload"] - else {} + changes_event = _latest_event( + conn, task_id, "changes_requested", changes_run["id"], ) - except (json.JSONDecodeError, TypeError): - changes_payload = {} - prior_reviewer = ( - changes_payload.get("reviewer") - if isinstance(changes_payload, dict) - else None - ) + prior_reviewer = _json_dict(_row_get(changes_event, "payload")).get("reviewer") if changes_run is not None: if not isinstance(prior_reviewer, str) or not prior_reviewer.strip(): return _ret( @@ -6692,44 +6198,15 @@ def request_changes( if expected_run_id is not None and int(current_run_id) != int(expected_run_id): return False, "run_id mismatch" - claimed_event = conn.execute( - "SELECT payload FROM task_events " - "WHERE task_id = ? AND run_id = ? AND kind = 'claimed' " - "ORDER BY id DESC LIMIT 1", - (task_id, int(current_run_id)), - ).fetchone() - try: - claimed_payload = ( - json.loads(claimed_event["payload"]) - if claimed_event and claimed_event["payload"] - else {} - ) - except (json.JSONDecodeError, TypeError): - claimed_payload = {} - if not isinstance(claimed_payload, dict): - claimed_payload = {} + claimed_event = _latest_event(conn, task_id, "claimed", current_run_id) + claimed_payload = _json_dict(_row_get(claimed_event, "payload")) if claimed_payload.get("source_status") != "review": return False, "active run was not claimed from review" - requested_event = conn.execute( - "SELECT payload FROM task_events " - "WHERE task_id = ? AND kind = 'review_requested' " - "ORDER BY id DESC LIMIT 1", - (task_id,), - ).fetchone() + requested_event = _latest_event(conn, task_id, "review_requested") if requested_event is None: return False, "no prior review_requested event" - try: - requested_payload = ( - json.loads(requested_event["payload"]) - if requested_event["payload"] - else {} - ) - except (json.JSONDecodeError, TypeError): - requested_payload = {} - if not isinstance(requested_payload, dict): - requested_payload = {} - implementer = requested_payload.get("implementer") + implementer = _json_dict(requested_event["payload"]).get("implementer") if not isinstance(implementer, str) or not implementer.strip(): return False, "review handoff has no valid implementer provenance" reviewer = task_row["assignee"] @@ -6798,13 +6275,10 @@ def promote_task( ``(False, reason)`` if refused. ``dry_run=True`` validates the promotion would succeed without mutating state. """ - row = conn.execute( - "SELECT status FROM tasks WHERE id = ?", (task_id,) - ).fetchone() - if row is None: + cur_status = _task_status(conn, task_id) + if cur_status is None: return False, f"task {task_id} not found" - cur_status = row["status"] if cur_status not in ("todo", "blocked"): return False, ( f"task {task_id} is {cur_status!r}; promote only applies to " @@ -6888,14 +6362,7 @@ def _landing_status_after_parents(conn: sqlite3.Connection, task_id: str) -> str kanban/boards/cookai/workspaces/t_a6acd07d/root-cause.md. Kept in one place so the two transitions can't drift. """ - undone_parents = conn.execute( - "SELECT 1 FROM task_links l " - "JOIN tasks p ON p.id = l.parent_id " - "WHERE l.child_id = ? " - "AND p.status NOT IN ('done', 'archived') LIMIT 1", - (task_id,), - ).fetchone() - return "todo" if undone_parents else "ready" + return "ready" if _parents_satisfied(conn, task_id) else "todo" def unblock_task(conn: sqlite3.Connection, task_id: str) -> bool: @@ -6910,13 +6377,9 @@ def unblock_task(conn: sqlite3.Connection, task_id: str) -> bool: """ now = int(time.time()) with write_txn(conn): - current = conn.execute( - "SELECT status FROM tasks WHERE id = ?", - (task_id,), - ).fetchone() resume_status = ( _resume_status_from_events(conn, task_id) - if current and current["status"] == "blocked" + if _task_status(conn, task_id) == "blocked" else "ready" ) _reclaim_dangling_run( @@ -6980,20 +6443,8 @@ def reopen_review_task(conn: sqlite3.Connection, task_id: str) -> bool: note="invariant recovery on review reopen", ) new_status = _landing_status_after_parents(conn, task_id) - review_event = conn.execute( - "SELECT payload FROM task_events " - "WHERE task_id = ? AND kind = 'review_requested' " - "ORDER BY id DESC LIMIT 1", - (task_id,), - ).fetchone() - try: - handoff = ( - json.loads(review_event["payload"]) - if review_event and review_event["payload"] - else {} - ) - except (json.JSONDecodeError, TypeError): - handoff = {} + review_event = _latest_event(conn, task_id, "review_requested") + handoff = _json_dict(_row_get(review_event, "payload")) implementer = handoff.get("implementer") if not isinstance(implementer, str) or not implementer.strip(): implementer = None @@ -7035,54 +6486,35 @@ def invalidate_descendants_for_parent_reopen( ) -> dict[str, Any]: """Retract every dispatchable/completed descendant of a reopened ancestor. - THE single domain implementation of done-reopen descendant invalidation. - When a ``done`` (or ``archived``) ancestor is reopened, every descendant - whose state assumed the ancestor's result — ``ready``, ``review``, - ``running`` or ``done`` — is building on a retracted premise, so it is - demoted to ``todo`` and re-gated on the graph. The CLI deliberately has - NO done-reopen verb on this branch (``reopen-review`` only handles the - review-phase transition via :func:`reopen_review_task`), so every surface - that reopens a done task (dashboard drag-drop / PATCH — single and bulk — - via ``_set_status_direct``) must route through this function; keeping the - implementation here means a future CLI or tool reopen verb inherits - identical semantics for free. + THE single implementation of done-reopen descendant invalidation. When a + ``done``/``archived`` ancestor is reopened, every descendant whose state + assumed its result — ``ready``, ``review``, ``running`` or ``done`` — is + demoted to ``todo`` and re-gated on the graph. The CLI has no done-reopen + verb (``reopen-review`` is the review-phase transition), so every surface + that reopens a done task (dashboard drag-drop / PATCH via + ``_set_status_direct``) must route through here. - Transactionality: composes under the caller's already-open transaction - via ``write_txn(conn, allow_nested=True)`` — the dashboard's status - writer must commit the ancestor's status flip and the descendant - retractions atomically (a crash between the two would leave stale done - descendants claiming a premise that no longer holds). Called standalone - it opens its own transaction. All SQL is inline per this file's txn - conventions (no calls into other txn-opening helpers). + Transactionality: composes under the caller's open transaction via + ``write_txn(conn, allow_nested=True)`` so the ancestor's status flip and + the retractions commit atomically; standalone it opens its own. All SQL + is inline (no txn-opening helpers). - Non-silent contract: every invalidated descendant gets - * a ``descendant_invalidated`` event with ``{ancestor, prior_status, - new_status}`` (plus ``resume_status``) for board/notifier surfaces, - * the legacy ``status`` event (``reason=ancestor_reopened``) the live - feed already renders, and - * a ``task_comments`` row naming the reopened ancestor, so operators see - WHY a card moved instead of watching it silently teleport. + Non-silent contract — every invalidated descendant gets a + ``descendant_invalidated`` event (``ancestor, prior_status, new_status, + resume_status``), the legacy ``status`` event + (``reason=ancestor_reopened``) the live feed renders, and a comment + naming the ancestor so operators see WHY a card moved. - Live ``running`` descendants keep the termination behavior (a running - child building on a retracted premise is wasted spend): their run is - closed ``reclaimed`` and their worker is killed via - :func:`_terminate_reclaimed_worker` — the same helper the reclaim paths - use. Events/comments are written inside the transaction and the kill - happens strictly post-commit, so the audit trail exists BEFORE the - worker dies. When this function opened its own transaction it performs - the terminations itself after commit; when composing under a caller's - transaction the caller MUST drain the returned ``terminations`` list - with ``_terminate_reclaimed_worker`` after its own commit. + Live ``running`` descendants are wasted spend: their run is closed + ``reclaimed`` and the worker killed via :func:`_terminate_reclaimed_worker` + strictly post-commit, so the audit trail exists BEFORE the worker dies. + When composing under a caller's transaction the caller MUST drain the + returned ``terminations`` after its own commit. - ``consecutive_failures`` is reset to 0 on every invalidated descendant: - ancestor reopen is a deliberate operator action, so demoted work gets a - fresh start with the breaker (a previously auto-blocked-then-completed - descendant should not re-enter the queue one failure from the breaker). - This is deliberately the OPPOSITE of the review-transition rule - (:func:`reopen_review_task` / #35072 preserves the counter) because the - autonomous review loop must not be able to launder its own failure - streak, while an operator invalidating a subtree is an explicit reset - signal. + ``consecutive_failures`` resets to 0: ancestor reopen is a deliberate + operator action, so demoted work gets a fresh breaker budget. This is the + OPPOSITE of :func:`reopen_review_task` (preserves the counter) because the + autonomous review loop must not launder its own failure streak. Returns ``{"invalidated": [...], "terminations": [...]}`` where each invalidated entry is ``{id, prior_status, new_status, resume_status}`` @@ -7164,21 +6596,12 @@ def invalidate_descendants_for_parent_reopen( }, run_id=run_id, ) - # Inline comment insert (not add_comment: no txn-opening helper - # calls inside a txn per file convention). - conn.execute( - "INSERT INTO task_comments (task_id, author, body, created_at) " - "VALUES (?, ?, ?, ?)", - ( - row["id"], - author, - ( - f"Invalidated: ancestor {task_id} was reopened; " - f"retracted from '{previous_status}' to 'todo' " - f"(will resume via '{resume_status}')." - ), - now, - ), + _insert_comment( + conn, row["id"], author, + f"Invalidated: ancestor {task_id} was reopened; " + f"retracted from '{previous_status}' to 'todo' " + f"(will resume via '{resume_status}').", + now, ) invalidated.append( { @@ -7261,17 +6684,10 @@ def specify_triage_task( # IMMEDIATE would raise OperationalError. We also skip the # 'commented' event that ``add_comment`` emits, since the # 'specified' event below already records the change. - conn.execute( - "INSERT INTO task_comments (task_id, author, body, created_at) " - "VALUES (?, ?, ?, ?)", - ( - task_id, - author.strip(), - "Specified — updated " - + ", ".join(changed_fields) - + " and promoted to todo.", - int(time.time()), - ), + _insert_comment( + conn, task_id, author.strip(), + "Specified — updated " + ", ".join(changed_fields) + " and promoted to todo.", + int(time.time()), ) _append_event( conn, @@ -7456,11 +6872,7 @@ def decompose_triage_task( for p_idx in child.get("parents") or []: parent_id = child_ids[p_idx] child_id = child_ids[idx] - conn.execute( - "INSERT OR IGNORE INTO task_links (parent_id, child_id) " - "VALUES (?, ?)", - (parent_id, child_id), - ) + _link(conn, parent_id, child_id) _append_event( conn, child_id, "linked", {"parent": parent_id, "child": child_id}, @@ -7471,11 +6883,7 @@ def decompose_triage_task( # link root under every child. Cycle-free because the root is # only ever a child here, never a parent of children. for cid in child_ids: - conn.execute( - "INSERT OR IGNORE INTO task_links (parent_id, child_id) " - "VALUES (?, ?)", - (cid, task_id), - ) + _link(conn, cid, task_id) # Flip the root: triage -> todo, set assignee to the orchestrator. sets = ["status = 'todo'"] @@ -7491,17 +6899,11 @@ def decompose_triage_task( # Audit comment + event on the root so the timeline shows the fan-out. if author and author.strip(): - conn.execute( - "INSERT INTO task_comments (task_id, author, body, created_at) " - "VALUES (?, ?, ?, ?)", - ( - task_id, - author.strip(), - "Decomposed into " - + ", ".join(child_ids) - + ". Root will wake when all children complete.", - now, - ), + _insert_comment( + conn, task_id, author.strip(), + "Decomposed into " + ", ".join(child_ids) + + ". Root will wake when all children complete.", + now, ) _append_event( conn, task_id, "decomposed", @@ -7550,6 +6952,15 @@ def archive_task(conn: sqlite3.Connection, task_id: str) -> bool: return True +def _delete_task_relations(conn: sqlite3.Connection, task_id: str) -> None: + """Delete every row referencing ``task_id`` (schema has no ON DELETE CASCADE).""" + conn.execute( + "DELETE FROM task_links WHERE parent_id = ? OR child_id = ?", (task_id, task_id), + ) + for table in ("task_comments", "task_events", "task_runs", "kanban_notify_subs"): + conn.execute(f"DELETE FROM {table} WHERE task_id = ?", (task_id,)) + + def delete_archived_task(conn: sqlite3.Connection, task_id: str) -> bool: """Permanently remove an already-archived task and its related rows. @@ -7558,20 +6969,9 @@ def delete_archived_task(conn: sqlite3.Connection, task_id: str) -> bool: second deliberate action. """ with write_txn(conn): - row = conn.execute( - "SELECT status FROM tasks WHERE id = ?", - (task_id,), - ).fetchone() - if not row or row["status"] != "archived": + if _task_status(conn, task_id) != "archived": return False - conn.execute( - "DELETE FROM task_links WHERE parent_id = ? OR child_id = ?", - (task_id, task_id), - ) - conn.execute("DELETE FROM task_comments WHERE task_id = ?", (task_id,)) - conn.execute("DELETE FROM task_events WHERE task_id = ?", (task_id,)) - conn.execute("DELETE FROM task_runs WHERE task_id = ?", (task_id,)) - conn.execute("DELETE FROM kanban_notify_subs WHERE task_id = ?", (task_id,)) + _delete_task_relations(conn, task_id) cur = conn.execute("DELETE FROM tasks WHERE id = ?", (task_id,)) return cur.rowcount == 1 @@ -7590,11 +6990,7 @@ def delete_task(conn: sqlite3.Connection, task_id: str) -> bool: cur = conn.execute("DELETE FROM tasks WHERE id = ?", (task_id,)) if cur.rowcount != 1: return False - conn.execute("DELETE FROM task_links WHERE parent_id = ? OR child_id = ?", (task_id, task_id)) - conn.execute("DELETE FROM task_comments WHERE task_id = ?", (task_id,)) - conn.execute("DELETE FROM task_events WHERE task_id = ?", (task_id,)) - conn.execute("DELETE FROM task_runs WHERE task_id = ?", (task_id,)) - conn.execute("DELETE FROM kanban_notify_subs WHERE task_id = ?", (task_id,)) + _delete_task_relations(conn, task_id) recompute_ready(conn) return True @@ -7605,20 +7001,8 @@ def delete_task(conn: sqlite3.Connection, task_id: str) -> bool: def _git_toplevel(path: Path) -> Optional[Path]: """Return the git toplevel containing ``path``, or ``None`` if not in a repo.""" - try: - result = subprocess.run( - ["git", "-C", str(path), "rev-parse", "--show-toplevel"], - capture_output=True, - text=True, encoding='utf-8', errors='replace', - timeout=30, - check=False, - ) - except Exception: - return None - if result.returncode != 0: - return None - out = (result.stdout or "").strip() - if not out: + out = _git_out(path, "rev-parse", "--show-toplevel") + if out is None: return None try: return Path(out).expanduser().resolve() @@ -7641,58 +7025,17 @@ def _git_branch_exists(repo_root: Path, branch_name: str) -> bool: def _git_common_dir(path: Path) -> Optional[Path]: - try: - result = subprocess.run( - ["git", "-C", str(path), "rev-parse", "--path-format=absolute", "--git-common-dir"], - capture_output=True, - text=True, encoding='utf-8', errors='replace', - timeout=30, - check=False, - ) - except Exception: - return None - if result.returncode != 0: - return None - out = (result.stdout or "").strip() - if not out: - return None - return Path(out).expanduser().resolve(strict=False) + out = _git_out(path, "rev-parse", "--path-format=absolute", "--git-common-dir") + return Path(out).expanduser().resolve(strict=False) if out else None def _git_dir(path: Path) -> Optional[Path]: - try: - result = subprocess.run( - ["git", "-C", str(path), "rev-parse", "--path-format=absolute", "--git-dir"], - capture_output=True, - text=True, encoding='utf-8', errors='replace', - timeout=30, - check=False, - ) - except Exception: - return None - if result.returncode != 0: - return None - out = (result.stdout or "").strip() - if not out: - return None - return Path(out).expanduser().resolve(strict=False) + out = _git_out(path, "rev-parse", "--path-format=absolute", "--git-dir") + return Path(out).expanduser().resolve(strict=False) if out else None def _git_current_branch(path: Path) -> Optional[str]: - try: - result = subprocess.run( - ["git", "-C", str(path), "branch", "--show-current"], - capture_output=True, - text=True, encoding='utf-8', errors='replace', - timeout=30, - check=False, - ) - except Exception: - return None - if result.returncode != 0: - return None - branch = (result.stdout or "").strip() - return branch or None + return _git_out(path, "branch", "--show-current") def _is_linked_worktree_checkout(path: Path) -> bool: @@ -8280,7 +7623,7 @@ def _terminate_reclaimed_worker( if not pid or pid <= 0 or not claim_lock: return info - host_prefix = f"{_claimer_id().split(':', 1)[0]}:" + host_prefix = _host_prefix() if not str(claim_lock).startswith(host_prefix): return info info["host_local"] = True @@ -8451,7 +7794,7 @@ def enforce_max_runtime( import signal timed_out: list[str] = [] now = int(time.time()) - host_prefix = f"{_claimer_id().split(':', 1)[0]}:" + host_prefix = _host_prefix() rows = conn.execute( "SELECT t.id, t.worker_pid, " @@ -8648,12 +7991,8 @@ def detect_stale_running( payload = { "elapsed_seconds": int(elapsed), - "last_heartbeat_at": ( - int(last_hb) if last_hb is not None else None - ), - "heartbeat_age_seconds": ( - int(hb_age) if hb_age is not None else None - ), + "last_heartbeat_at": _opt_int(last_hb), + "heartbeat_age_seconds": _opt_int(hb_age), "timeout_seconds": stale_timeout_seconds, "pid": int(pid) if pid else None, "retry_status": retry_status, @@ -8710,7 +8049,6 @@ def reconcile_orphaned_running( Returns the list of reconciled task ids. Safe to call every tick. - Idea from openai/symphony's tracker reconciliation (Apache-2.0). """ now = int(time.time()) reconciled: list[str] = [] @@ -8744,10 +8082,7 @@ def reconcile_orphaned_running( payload = { "reason": "orphaned_running", "claim_lock": row["claim_lock"], - "claim_expires": ( - int(row["claim_expires"]) - if row["claim_expires"] is not None else None - ), + "claim_expires": _opt_int(row["claim_expires"]), "worker_pid": int(pid) if pid else None, "now": now, } @@ -8757,17 +8092,11 @@ def reconcile_orphaned_running( error="orphaned running card (broken claim bookkeeping)", metadata=payload, ) - # Inline comment INSERT — add_comment opens its own write_txn - # and would raise on nesting (see write_txn pitfalls). - conn.execute( - "INSERT INTO task_comments (task_id, author, body, created_at) " - "VALUES (?, ?, ?, ?)", - ( - tid, "dispatcher", - "reconciliation: card was 'running' with no valid claim " - "(dead/gone worker) — requeued to ready", - now, - ), + _insert_comment( + conn, tid, "dispatcher", + "reconciliation: card was 'running' with no valid claim " + "(dead/gone worker) — requeued to ready", + now, ) _append_event(conn, tid, "reconciled", payload, run_id=run_id) reconciled.append(tid) @@ -8789,24 +8118,17 @@ def _error_fingerprint(error_text: str) -> str: return fp.lower().strip() -# Empirically ~96% of "clean exit without a terminal tool call" tasks complete -# on a later run (a goal-mode finalize nudge, or the model simply emitting the -# tool call next time), so a protocol violation is NOT deterministic — give it a -# bounded retry before the breaker trips instead of blocking on the first hit. -# -# The budget is a violation-only STREAK, not a share of the unified -# ``consecutive_failures`` counter: it counts consecutive clean-exit protocol -# violations (derived from run history by ``_protocol_violation_streak``), so -# earlier timeouts / nonzero exits neither consume nor extend it, and a -# below-budget violation does not tick the unified counter either. A per-task -# ``max_retries`` overrides this bound — the same "task override wins" -# precedence ``_record_task_failure`` documents for every other failure kind. +# ~96% of "clean exit without a terminal tool call" tasks complete on a later +# run, so a protocol violation is NOT deterministic — bounded retry before the +# breaker trips. The budget is a violation-only STREAK +# (``_protocol_violation_streak``), independent of ``consecutive_failures``: +# other failure kinds neither consume nor extend it, and a below-budget +# violation doesn't tick the unified counter. Per-task ``max_retries`` +# overrides it, as for every other failure kind. _PROTOCOL_VIOLATION_FAILURE_LIMIT = 3 -# How far back to walk a task's closed runs when counting the violation -# streak. The streak trips at a handful of violations, so anything beyond a -# few dozen rows (violations interleaved with neutral rate-limited requeues) -# can only mean "way past the bound" anyway. +# Closed runs to walk when counting the streak; the streak trips at a handful, +# so anything past a few dozen rows means "way past the bound" anyway. _PROTOCOL_VIOLATION_SCAN_LIMIT = 50 @@ -8842,15 +8164,7 @@ def _protocol_violation_streak(conn: sqlite3.Connection, task_id: str) -> int: if outcome == "rate_limited": continue if outcome == "crashed": - is_violation = False - raw_meta = row["metadata"] - if raw_meta: - try: - is_violation = bool( - json.loads(raw_meta).get("protocol_violation") - ) - except (ValueError, TypeError): - is_violation = False + is_violation = bool(_json_dict(row["metadata"]).get("protocol_violation")) if not is_violation: is_violation = "protocol violation" in (row["error"] or "") if is_violation: @@ -8867,26 +8181,21 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: Different from ``release_stale_claims``: this checks liveness immediately rather than waiting for the claim TTL. - Only considers tasks claimed by *this host* — PIDs from other hosts - are meaningless here. The host-local check is enough because - ``_default_spawn`` always runs the worker on the same host as the - dispatcher (the whole design is single-host). + Only tasks claimed by *this host* are considered — PIDs from other hosts + are meaningless, and ``_default_spawn`` always runs workers on the + dispatcher's host. - When the reap registry shows the worker exited cleanly (rc=0) but - the task was still ``running`` in the DB, treat it as a protocol - violation (worker answered conversationally without calling - ``kanban_complete`` / ``kanban_block``) and trip the circuit breaker - on the first occurrence — retrying a worker whose CLI keeps - returning 0 without a terminal transition just loops forever. + A clean exit (rc=0) with the task still ``running`` is a protocol + violation (worker answered conversationally without ``kanban_complete`` / + ``kanban_block``); it gets a bounded violation-only retry budget before + the breaker trips (see ``_protocol_violation_streak``). - When the reap registry shows the worker exited with the rate-limit - sentinel (``KANBAN_RATE_LIMIT_EXIT_CODE``), the worker bailed on a - provider quota wall, NOT a task failure. Such tasks are released back - to its source phase WITHOUT counting a failure (so a long quota window can't - trip the breaker) and stamped with a quota-blocker error so - ``check_respawn_guard`` defers their respawn until the window clears. - The ids are returned via the ``_last_rate_limited`` function attribute - (the public return stays the crashed-only ``list[str]``). + An exit with ``KANBAN_RATE_LIMIT_EXIT_CODE`` is a provider quota wall, + NOT a task failure: released to its source phase WITHOUT counting a + failure (a long quota window must not trip the breaker) and stamped with + a quota-blocker error so ``check_respawn_guard`` defers the respawn. Those + ids surface via the ``_last_rate_limited`` function attribute; the public + return stays the crashed-only ``list[str]``. """ crashed: list[str] = [] rate_limited: list[str] = [] @@ -8907,7 +8216,7 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: "FROM tasks " "WHERE status = 'running' AND worker_pid IS NOT NULL" ).fetchall() - host_prefix = f"{_claimer_id().split(':', 1)[0]}:" + host_prefix = _host_prefix() for row in rows: # Only check liveness for claims owned by this host. lock = row["claim_lock"] or "" @@ -8916,7 +8225,7 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: # Skip liveness check inside the launch-window grace period # so a freshly-spawned worker isn't reclaimed before its PID # is visible on /proc. - started_at = row["started_at"] if "started_at" in row.keys() else None + started_at = _row_get(row, "started_at") if started_at is not None: grace = _resolve_crash_grace_seconds() if time.time() - started_at < grace: @@ -8928,13 +8237,10 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: kind, code = _classify_worker_exit(pid) rate_limited_exit = False if kind == "clean_exit": - # Worker subprocess returned 0 but its task is still - # ``running`` in the DB — it exited without calling - # ``kanban_complete`` / ``kanban_block``. Overwhelmingly the - # work itself succeeded and only the paperwork was skipped, so - # a retry usually completes; the corrective sentence below is - # surfaced to the retry worker via the prior-attempt error in - # ``build_worker_context`` (guidance approach from #61817). + # rc=0 with the task still ``running``: exited without + # ``kanban_complete`` / ``kanban_block``. Usually the work + # succeeded and only the paperwork was skipped; the corrective + # sentence reaches the retry worker via ``build_worker_context``. protocol_violation = True error_text = ( "worker exited cleanly (rc=0) without calling " @@ -8955,13 +8261,9 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: "protocol_violation": True, } elif kind == "rate_limited": - # Worker bailed because the provider rate-limited / exhausted - # quota (EX_TEMPFAIL sentinel). This is NOT a task failure — - # the task is fine, the account just hit a wall. Release it - # back to its source phase so the respawn guard defers it until the - # quota window clears, and crucially do NOT count a failure - # (skip ``_record_task_failure``) so a long quota window can't - # trip the circuit breaker and permanently block the card. + # EX_TEMPFAIL quota wall — NOT a task failure. Release to the + # source phase (respawn guard defers it) and do NOT count a + # failure so a long quota window can't trip the breaker. protocol_violation = False rate_limited_exit = True error_text = ( @@ -9024,11 +8326,8 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: "retry_status": retry_status, }) if rate_limited_exit: - # Stamp the failure-error column so ``check_respawn_guard`` - # recognizes this as a quota blocker and defers the - # respawn until the window clears — WITHOUT touching - # ``consecutive_failures`` (that's the whole point: no - # breaker trip on a throttle). + # Stamp last_failure_error so ``check_respawn_guard`` sees a + # quota blocker — WITHOUT touching ``consecutive_failures``. conn.execute( "UPDATE tasks SET last_failure_error = ? WHERE id = ?", (error_text[:500], row["id"]), @@ -9036,12 +8335,10 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: rate_limited.append(row["id"]) else: if protocol_violation: - # Stamp the failure error now: a below-budget - # violation never reaches ``_record_task_failure`` - # (which stamps this column for every other failure - # kind), yet the board UI and the retry worker's - # context still need the violation message + the - # corrective guidance it carries. + # A below-budget violation never reaches + # ``_record_task_failure`` (which stamps this column + # for every other kind), yet the board UI and the retry + # worker's context need the corrective message. conn.execute( "UPDATE tasks SET last_failure_error = ? " "WHERE id = ?", @@ -9052,22 +8349,13 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: (row["id"], pid, row["claim_lock"], protocol_violation, error_text) ) - # Outside the main txn: account each crashed task and maybe trip the - # breaker (the retried task transitions to blocked with a ``gave_up`` event - # on top of the event we already emitted). - # - # Protocol-violation crashes (clean exit, no terminal tool call) get a - # BOUNDED retry, not an immediate trip: empirically ~96% of these tasks - # complete on a later run (a goal-mode finalize nudge, or the model simply - # emitting kanban_complete/kanban_block next time), so blocking on the first - # occurrence just churned them through the respawn cycle. The retry budget - # is a violation-only streak (``_protocol_violation_streak``): earlier - # timeouts / nonzero exits neither consume nor extend it, and a - # below-budget violation does not tick the unified - # ``consecutive_failures`` counter, so the two budgets stay independent. - # A per-task ``max_retries`` overrides the violation bound with the same - # top precedence it has for every other failure kind. Systemic same-error - # crashes still trip immediately. + # Outside the main txn: account each crash and maybe trip the breaker + # (blocked + ``gave_up`` on top of the event already emitted). + # Protocol violations get a BOUNDED violation-only retry budget + # (``_protocol_violation_streak``; see ``_PROTOCOL_VIOLATION_FAILURE_LIMIT`` + # for why) — independent of ``consecutive_failures``, with per-task + # ``max_retries`` taking top precedence. Systemic same-error crashes still + # trip immediately. auto_blocked: list[str] = [] if crash_details: # Fingerprint errors to detect systemic failures. @@ -9083,27 +8371,20 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]: ).fetchone() if trow is None: continue # task deleted mid-loop - task_override = ( - trow["max_retries"] if "max_retries" in trow.keys() else None - ) + task_override = _row_get(trow, "max_retries") violation_limit = ( int(task_override) if task_override is not None else _PROTOCOL_VIOLATION_FAILURE_LIMIT ) if streak < violation_limit: - # Below budget: the task is already back at ``ready`` - # (respawn allowed) with ``last_failure_error`` stamped. - # Deliberately no ``_record_task_failure`` call — a - # below-budget violation must not consume the unified - # failure budget, just as other failure kinds don't - # consume this one. + # Below budget: already back at ``ready`` with + # ``last_failure_error`` stamped. Deliberately no + # ``_record_task_failure`` — must not consume the unified + # failure budget. continue - # Streak reached the bound: trip the breaker. ``force_trip`` - # skips the threshold resolution inside - # ``_record_task_failure`` because the decision — including - # the per-task ``max_retries`` override — was already made - # against the violation streak above. + # ``force_trip``: the decision (incl. per-task ``max_retries``) + # was already made against the violation streak above. tripped = _record_task_failure( conn, tid, error=error_text, @@ -9175,45 +8456,24 @@ def _record_task_failure( """Record a non-success outcome (spawn_failed / crashed / timed_out) and maybe trip the circuit breaker. - Unified replacement for the old spawn-only ``_record_spawn_failure``. - Every path that ends a task with a non-success outcome funnels - through here so the ``consecutive_failures`` counter and the - auto-block threshold stay consistent. + Every non-success path funnels through here so ``consecutive_failures`` + and the auto-block threshold stay consistent. Returns True when the task + was auto-blocked, False when just updated in place. - Returns True when the task was auto-blocked (counter reached - ``failure_limit``), False when it was just updated in place. + Modes: ``release_claim=True, end_run=True`` is the spawn-failure path + (task still running with an open run: restore its source phase — or + ``blocked`` on trip — release the claim, close the run with + ``outcome=``). ``release_claim=False, end_run=False`` is the + timeout/crash path (caller ALREADY restored the phase and closed the run; + only the counter moves, and a trip re-transitions to ``blocked`` with a + ``gave_up`` event). ``event_payload_extra`` merges into that payload. - Modes: - - * ``release_claim=True, end_run=True`` — spawn-failure path. - Caller has a running task with an open run; this transitions - it back to its source phase (or ``blocked`` when the breaker trips), - releases the claim, and closes the run with ``outcome=``. - - * ``release_claim=False, end_run=False`` — timeout/crash path. - Caller has ALREADY restored the task's source phase and closed the - run with the appropriate outcome. This just increments the - counter; if the breaker trips, the task is re-transitioned - into ``blocked`` and a ``gave_up`` event is emitted. - - ``event_payload_extra`` merges into the ``gave_up`` event payload - when the breaker trips, so callers can include outcome-specific - context (e.g. pid on crash, elapsed on timeout). - - Resolution order for the effective threshold: - 1. per-task ``max_retries`` if set (nothing else overrides) - 2. caller-supplied ``failure_limit`` (gateway passes the config - value from ``kanban.failure_limit``; tests pass fixed values) - 3. ``DEFAULT_FAILURE_LIMIT`` - - ``force_trip=True`` trips the breaker unconditionally, skipping the - counter-vs-threshold comparison (the resolution order above is then - only reported in the ``gave_up`` payload, not re-evaluated). Callers - use it when they have already applied their own bounded-retry policy - — e.g. the clean-exit protocol-violation streak in - ``detect_crashed_workers``, which resolves the per-task - ``max_retries`` override against the violation streak itself. The - failure is still counted into ``consecutive_failures``. + Effective threshold: per-task ``max_retries`` (nothing overrides it), then + the caller's ``failure_limit`` (``kanban.failure_limit``), then + ``DEFAULT_FAILURE_LIMIT``. ``force_trip=True`` trips unconditionally — + the caller already applied its own bounded-retry policy (the + protocol-violation streak in ``detect_crashed_workers``); the order above + is then only reported in the payload. The failure is still counted. """ if failure_limit is None: failure_limit = DEFAULT_FAILURE_LIMIT @@ -9234,9 +8494,7 @@ def _record_task_failure( # Per-task override wins over both caller-supplied and default # thresholds. None (the common case) falls through. - task_override = ( - row["max_retries"] if "max_retries" in row.keys() else None - ) + task_override = _row_get(row, "max_retries") if task_override is not None: effective_limit = int(task_override) limit_source = "task" @@ -9245,26 +8503,17 @@ def _record_task_failure( limit_source = "dispatcher" if force_trip or failures >= effective_limit: - # Trip the breaker. - if release_claim: - # Spawn path: still running, also clear claim state. - conn.execute( - "UPDATE tasks SET status = 'blocked', claim_lock = NULL, " - "claim_expires = NULL, worker_pid = NULL, " - "consecutive_failures = ?, last_failure_error = ? " - "WHERE id = ? AND status IN ('running', 'ready', 'review')", - (failures, error[:500], task_id), - ) - else: - # Timeout/crash path: source phase already restored with claim - # cleared; just flip to blocked + update - # counter fields. - conn.execute( - "UPDATE tasks SET status = 'blocked', " - "consecutive_failures = ?, last_failure_error = ? " - "WHERE id = ? AND status IN ('ready', 'review', 'running')", - (failures, error[:500], task_id), - ) + # Trip the breaker. Spawn path (release_claim) is still running + # and also clears claim state; the timeout/crash path already + # restored the source phase with the claim cleared. + conn.execute( + "UPDATE tasks SET status = 'blocked', " + + ("claim_lock = NULL, claim_expires = NULL, worker_pid = NULL, " + if release_claim else "") + + "consecutive_failures = ?, last_failure_error = ? " + "WHERE id = ? AND status IN ('running', 'ready', 'review')", + (failures, error[:500], task_id), + ) run_id = None if end_run: # Only the spawn path has an open run to close. @@ -9406,54 +8655,40 @@ def check_respawn_guard( Returning a reason defers the spawn this tick; the task stays in its source phase and gets another chance on the next dispatcher tick. - ``lane`` names the dispatch column the task is being spawned from - (``"ready"`` or ``"review"``). In the review lane the - ``recent_success`` and ``active_pr`` rules are skipped: a recent PR - URL comment (and often a recent completed run) is the *precondition* - of the canonical review handoff — a worker opened a PR and requested - review — not a duplicate-work signal. Rate-limit cooldown and the - auth-blocker check still apply in every lane. + ``lane`` is the dispatch column (``"ready"`` / ``"review"``). The review + lane skips ``recent_success`` and ``active_pr``: a recent PR comment (and + often a completed run) is the *precondition* of a review handoff, not a + duplicate-work signal. Cooldown and auth-blocker apply in every lane. Checks in priority order: ``"rate_limit_cooldown"`` - The task's most recent run ended with the ``rate_limited`` outcome - (a worker bailed on a provider quota wall via the EX_TEMPFAIL - sentinel) within ``_resolve_rate_limit_cooldown_seconds()``. The - quota almost certainly hasn't reset yet, so defer the respawn until - the cooldown elapses — then allow a cheap probe. This is checked - BEFORE ``blocker_auth`` because the rate-limit requeue stamps a - quota-flavored ``last_failure_error`` that would otherwise match the - auth-blocker regex and park the task forever (the rate-limit path - never increments ``consecutive_failures``, so the breaker can't free - it). Once the cooldown elapses the task falls through and respawns. + Latest run ended ``rate_limited`` (EX_TEMPFAIL quota wall) within + ``_resolve_rate_limit_cooldown_seconds()``; defer until it elapses, + then allow a cheap probe. Checked BEFORE ``blocker_auth`` because the + rate-limit requeue stamps a quota-flavored ``last_failure_error`` that + would otherwise match the auth regex and park the task forever (that + path never increments ``consecutive_failures``, so the breaker can't + free it). ``"blocker_auth"`` - The task's last failure error matches a quota / authentication - pattern. Retrying immediately is unlikely to help (rate limits - reset on a timer; auth needs human action), so we defer to the - next tick. The existing ``consecutive_failures`` counter still - trips the auto-block circuit breaker after ``failure_limit`` - consecutive failures, so a persistent auth error eventually - blocks via the normal path — but a transient 429 gets a few - ticks of recovery first. + ``last_failure_error`` matches a quota / auth pattern; retrying now + won't help. ``consecutive_failures`` still trips the breaker after + ``failure_limit``, so a persistent auth error eventually blocks while + a transient 429 gets a few ticks of recovery. ``"recent_success"`` - A completed run exists within ``_RESPAWN_GUARD_SUCCESS_WINDOW`` - seconds. Useful work already succeeded for this task; wait for an - explicit re-queue rather than immediately re-spawning. Bypassed when an - explicit re-queue event (status change, promote, unblock, reclaim) - arrives AFTER that completion — that's a deliberate re-run request. + A completed run within ``_RESPAWN_GUARD_SUCCESS_WINDOW``; wait for an + explicit re-queue. Bypassed when a re-queue event (status, promote, + unblock, reclaim) arrives AFTER that completion — a deliberate re-run. ``"active_pr"`` - A GitHub PR URL appears in a recent task comment (within - ``_RESPAWN_GUARD_PR_WINDOW`` seconds). A prior worker already - opened a PR; re-spawning risks a duplicate PR on the same task. + A GitHub PR URL in a comment within ``_RESPAWN_GUARD_PR_WINDOW``; + re-spawning risks a duplicate PR. - Stale / dead claim locks are NOT a guard reason — they are handled - by ``release_stale_claims`` and ``detect_crashed_workers`` which - reset the task to ``ready`` only after verifying the lock is - genuinely dead (no live PID on this host). + Stale / dead claim locks are NOT a guard reason — ``release_stale_claims`` + and ``detect_crashed_workers`` reset those only after verifying the lock + is genuinely dead (no live PID on this host). """ row = conn.execute( "SELECT last_failure_error FROM tasks WHERE id = ?", @@ -9548,6 +8783,22 @@ def check_respawn_guard( return None +def _has_spawnable(conn: sqlite3.Connection, status: str) -> bool: + rows = conn.execute( + "SELECT DISTINCT assignee FROM tasks " + "WHERE status = ? AND assignee IS NOT NULL AND claim_lock IS NULL", + (status,), + ).fetchall() + if not rows: + return False + try: + from hermes_cli.profiles import profile_exists # local import: avoids cycle + except Exception: + # Can't introspect — assume spawnable, preserve legacy behavior. + return True + return any(profile_exists(row["assignee"]) for row in rows) + + def has_spawnable_ready(conn: sqlite3.Connection) -> bool: """Return True iff there is at least one ready+assigned+unclaimed task whose assignee maps to a real Hermes profile. @@ -9562,22 +8813,7 @@ def has_spawnable_ready(conn: sqlite3.Connection) -> bool: importable (e.g. partial install) — preserves the old behavior so the warning still fires in degraded environments. """ - rows = conn.execute( - "SELECT DISTINCT assignee FROM tasks " - "WHERE status = 'ready' AND assignee IS NOT NULL " - " AND claim_lock IS NULL" - ).fetchall() - if not rows: - return False - try: - from hermes_cli.profiles import profile_exists # local import: avoids cycle - except Exception: - # Can't introspect — assume spawnable, preserve legacy behavior. - return True - for row in rows: - if profile_exists(row["assignee"]): - return True - return False + return _has_spawnable(conn, "ready") def has_spawnable_review(conn: sqlite3.Connection) -> bool: @@ -9588,21 +8824,7 @@ def has_spawnable_review(conn: sqlite3.Connection) -> bool: used by the health telemetry to decide whether the dispatcher should have spawned a review agent. """ - rows = conn.execute( - "SELECT DISTINCT assignee FROM tasks " - "WHERE status = 'review' AND assignee IS NOT NULL " - " AND claim_lock IS NULL" - ).fetchall() - if not rows: - return False - try: - from hermes_cli.profiles import profile_exists # local import: avoids cycle - except Exception: - return True - for row in rows: - if profile_exists(row["assignee"]): - return True - return False + return _has_spawnable(conn, "review") def review_dispatch_enabled() -> bool: @@ -9622,25 +8844,15 @@ def review_dispatch_enabled() -> bool: # --------------------------------------------------------------------------- -# Memory-aware dispatch guard (OOF-30 / OOF-77) +# Memory-aware dispatch guard # -# Two production incidents ("larrikin-lollies", "synclare-task-manager") -# followed the same shape: no ``kanban.max_in_progress`` configured, a busy -# board, and a 1 GiB VM — the dispatcher fanned out 26-31 concurrent workers, -# the host went into swap-thrash/OOM, and the dashboard (and everything else -# on the machine) became unreachable. Two complementary safeguards: -# -# 1. A memory-DERIVED default concurrency cap when the operator never set -# ``kanban.max_in_progress`` (``resolve_max_in_progress``) — sized from -# MemTotal so a 1 GiB VM defaults to 2 workers, not unlimited. -# 2. A live memory-PRESSURE guard inside the dispatch tick itself -# (``_memory_pressure_level``) — even a correctly-sized static cap can't -# see other tenants of the box, so under real observed pressure the -# dispatcher stops adding workers regardless of configured caps. -# -# Both fail open: on non-Linux hosts or any read error the sample is empty, -# the derived default is None (no cap — unchanged behaviour), and the -# pressure level is "unknown" (no spawn restriction). +# With no ``kanban.max_in_progress`` on a busy board the dispatcher fanned out +# 26-31 workers on a 1 GiB VM and OOM'd the host. Two safeguards: a +# memory-DERIVED default cap when the operator never set one +# (``resolve_max_in_progress``, sized from MemTotal), and a live +# memory-PRESSURE guard inside the tick (``_memory_pressure_level``) because +# a static cap can't see other tenants of the box. Both fail open: non-Linux +# or any read error → empty sample → no cap / "unknown" (no restriction). # --------------------------------------------------------------------------- # Assumed per-worker memory footprint for the derived default cap. Hermes @@ -9898,6 +9110,157 @@ def dispatch_once( return result +def _dispatch_lane_task( + conn: sqlite3.Connection, + row: sqlite3.Row, + assignee: str, + result: "DispatchResult", + *, + lane: str, + dry_run: bool, + ttl_seconds: Optional[int], + board: Optional[str], + failure_limit: int, + spawn_fn, + per_profile_cap: Optional[int], + per_profile_running: dict[str, int], +) -> bool: + """Guard, claim, resolve the workspace and spawn one ready/review row. + + Shared tail of both dispatch lanes. Returns True when a spawn slot was + consumed (real spawn, or a would-be spawn under ``dry_run``); every skip + is recorded on ``result``. + """ + task_id = row["id"] + # Skip tasks whose assignee is not a real Hermes profile: ``_default_spawn`` + # runs ``hermes -p ``, which fails on startup for control-plane + # lanes (interactive terminals that pull via ``claim_task``), and the task + # would loop ready→crash→ready forever. Bucketed apart from + # skipped_unassigned: the operator cannot fix it by assigning a profile, + # and health telemetry uses the distinction to suppress "stuck" warnings. + try: + from hermes_cli.profiles import profile_exists # local import: avoids cycle + except Exception: + profile_exists = None # type: ignore[assignment] + if profile_exists is not None and not profile_exists(assignee): + result.skipped_nonspawnable.append(task_id) + return False + # Per-profile concurrency cap: even with global headroom, refuse to spawn + # for an assignee already at its in-flight cap so one profile's local + # model / API quota / browser pool isn't overwhelmed by a fan-out. + if per_profile_cap is not None: + current = per_profile_running.get(assignee, 0) + if current >= per_profile_cap: + result.skipped_per_profile_capped.append((task_id, assignee, current)) + return False + # Respawn guard: defer this tick when useful work is already in-flight / + # recent or the last failure is a deterministic blocker (quota / auth). + # consecutive_failures still trips the auto-block breaker eventually. + guard_reason = check_respawn_guard(conn, task_id, lane=lane) + if guard_reason is not None: + result.respawn_guarded.append((task_id, guard_reason)) + # Event so ``hermes kanban tail`` shows why the task looks stuck. + if not dry_run: + with write_txn(conn): + _append_event(conn, task_id, "respawn_guarded", {"reason": guard_reason}) + return False + if dry_run: + result.spawned.append((task_id, assignee, "")) + # Count the would-be spawn so the cap check sees it on later rows; + # otherwise dry_run under-reports the capped subset. + if per_profile_cap is not None and assignee: + per_profile_running[assignee] = per_profile_running.get(assignee, 0) + 1 + return True + claim = claim_review_task if lane == "review" else claim_task + claimed = claim(conn, task_id, ttl_seconds=ttl_seconds) + if claimed is None: + return False + try: + resolved_branch_name = None + if claimed.workspace_kind == "worktree": + workspace, resolved_branch_name = _resolve_worktree_workspace(claimed, board=board) + else: + workspace = resolve_workspace(claimed, board=board) + except Exception as exc: + if _record_spawn_failure(conn, claimed.id, f"workspace: {exc}", failure_limit=failure_limit): + result.auto_blocked.append(claimed.id) + return False + # Persist the resolved workspace path so the worker can cd there. + set_workspace_path(conn, claimed.id, str(workspace)) + if claimed.workspace_kind == "worktree": + set_branch_name(conn, claimed.id, resolved_branch_name or (claimed.branch_name or "").strip() or f"wt/{claimed.id}") + _maybe_emit_scratch_tip(conn, claimed.id, claimed.workspace_kind) + if lane == "review": + # Force-load the sdlc-review skill (AC verification, merge, ...). The + # mandatory kanban lifecycle is already in every worker's system + # prompt via KANBAN_GUIDANCE, so this is the only extra skill needed. + claimed.skills = list(dict.fromkeys([*(claimed.skills or []), "sdlc-review"])) + _spawn = spawn_fn if spawn_fn is not None else _default_spawn + try: + # Back-compat: older spawn_fn signatures (and test stubs) accept only + # (task, workspace); pass ``board`` only when the callable supports it. + import inspect + try: + sig = inspect.signature(_spawn) + if "board" in sig.parameters: + pid = _spawn(claimed, str(workspace), board=board) + else: + pid = _spawn(claimed, str(workspace)) + except (TypeError, ValueError): + pid = _spawn(claimed, str(workspace)) + if pid: + _set_worker_pid(conn, claimed.id, int(pid)) + # Worker-lifecycle observer: fires AFTER spawn_fn returned and the PID + # (when reported) is durably persisted. Best-effort. + _fire_worker_spawned_hook(conn, claimed, str(workspace), pid, board=board) + # consecutive_failures is deliberately NOT reset here: a successful + # spawn doesn't prove the run will succeed, and resetting on spawn + # would let a task that keeps timing out loop forever. Cleared only + # on successful completion (complete_task). + result.spawned.append((claimed.id, claimed.assignee or "", str(workspace))) + # Track the new in-flight count so later rows in this tick respect + # the per-profile cap; subsequent ticks re-query from the DB. + if per_profile_cap is not None and claimed.assignee: + per_profile_running[claimed.assignee] = per_profile_running.get(claimed.assignee, 0) + 1 + return True + except Exception as exc: + if _record_spawn_failure(conn, claimed.id, str(exc), failure_limit=failure_limit): + result.auto_blocked.append(claimed.id) + return False + + +def _apply_default_assignee( + conn: sqlite3.Connection, task_id: str, assignee: str, *, dry_run: bool, +) -> bool: + """Persist ``kanban.default_assignee`` on an unassigned ready row. + + Mutating the row (not just the in-memory view) keeps diagnostics and board + state consistent: the task is now legitimately owned by the default, not + "unassigned but secretly routed". ``dry_run`` reports the would-be + assignment without writing. Returns False when the write failed. + """ + if dry_run: + return True + try: + with write_txn(conn): + conn.execute( + "UPDATE tasks SET assignee = ? WHERE id = ? " + "AND (assignee IS NULL OR assignee = '')", + (assignee, task_id), + ) + _append_event( + conn, task_id, "assigned", + {"assignee": assignee, "source": "kanban.default_assignee"}, + ) + except Exception: + _log.debug( + "kanban dispatch: failed to apply default_assignee=%r to task %s", + assignee, task_id, exc_info=True, + ) + return False + return True + + def _dispatch_once_locked( conn: sqlite3.Connection, *, @@ -9929,20 +9292,15 @@ def _dispatch_once_locked( failures the task is auto-blocked with the last error as its reason — prevents the dispatcher from thrashing forever on an unfixable task. - ``max_spawn`` is a **live concurrency cap**, not a per-tick spawn budget: - it counts tasks already in ``status='running'`` plus this tick's spawns - against the limit. So ``max_spawn=4`` means "at most 4 workers running - at any time across the whole board" — matching the gateway's stated - intent ("limit concurrent kanban tasks"). With a per-tick interpretation - a 60-second tick interval could grow concurrency by N every minute on a - busy board and accumulate without bound. + ``max_spawn`` is a **live concurrency cap** (running + this tick's spawns + across the board), not a per-tick budget — a per-tick reading would grow + concurrency by N every tick on a busy board without bound. - ``max_in_progress`` is a **host-level** concurrency cap (OOF-30): it - counts running tasks on every active board — not just this one — plus - this tick's spawns. Workers are OS processes sharing one machine's - memory, so a per-board interpretation would multiply the cap by the - number of active boards. ``max_spawn`` retains its historical per-board - semantics. + ``max_in_progress`` is a **host-level** cap: it counts running tasks on + every active board plus this tick's spawns, because workers are OS + processes sharing one machine's memory and a per-board reading would + multiply the cap by the number of boards. ``max_spawn`` keeps its + historical per-board semantics. ``spawn_fn`` defaults to ``_default_spawn``. Tests pass a stub. ``board`` pins workspace/log/db resolution for this tick to a specific @@ -9966,19 +9324,11 @@ def _dispatch_once_locked( # detect_crashed_workers stashes protocol-violation auto-blocks on # itself so the public list-return stays stable. Pull them into the # DispatchResult here so telemetry / tests see the trip. - _crash_auto_blocked = getattr( - detect_crashed_workers, "_last_auto_blocked", [] - ) - if _crash_auto_blocked: - result.auto_blocked.extend(_crash_auto_blocked) + result.auto_blocked.extend(getattr(detect_crashed_workers, "_last_auto_blocked", [])) # Rate-limited requeues (quota wall, no failure counted) — surface for # telemetry / tests. These tasks went back to ``ready`` and the respawn # guard will defer them until the quota window clears. - _crash_rate_limited = getattr( - detect_crashed_workers, "_last_rate_limited", [] - ) - if _crash_rate_limited: - result.rate_limited.extend(_crash_rate_limited) + result.rate_limited.extend(getattr(detect_crashed_workers, "_last_rate_limited", [])) result.timed_out = enforce_max_runtime(conn) result.promoted = recompute_ready(conn, failure_limit=failure_limit) @@ -10058,17 +9408,13 @@ def _dispatch_once_locked( "WHERE status = 'review' AND claim_lock IS NULL " "ORDER BY priority DESC, created_at ASC" ).fetchall() - # Review-lane reservation (OOF-30 review finding): the ready loop runs - # first and used to consume the ENTIRE shared budget, so a sustained - # ready backlog permanently starved autonomous reviews — completed work - # sat in 'review' forever while new work kept spawning. When spawnable - # review work exists and the tick has any budget, hold one slot back - # from the ready loop so the review lane always gets a spawn - # opportunity. The reservation is per-tick and self-releasing: with no - # spawnable review work (or no cap at all) the ready loop keeps the - # full budget. "Spawnable" mirrors the review loop's own gate - # (assigned + real profile) so a review column full of human-pulled - # control-plane lanes doesn't permanently tax ready throughput. + # Review-lane reservation: the ready loop runs first and used to consume + # the ENTIRE shared budget, starving autonomous reviews under a sustained + # ready backlog. When spawnable review work exists and there is any + # budget, hold one slot back from the ready loop. Per-tick and + # self-releasing: no spawnable review work (or no cap) → full budget. + # "Spawnable" mirrors the review loop's own gate (assigned + real + # profile) so human-pulled control-plane lanes don't tax ready throughput. def _any_spawnable_review() -> bool: if not review_rows: return False @@ -10086,14 +9432,11 @@ def _dispatch_once_locked( if spawn_budget is not None and spawn_budget > 0 and _any_spawnable_review(): ready_budget = max(spawn_budget - 1, 0) spawned = 0 - # Per-profile concurrency cap (#21582): when set, track how many - # workers each assignee already has in flight, and refuse to spawn - # when this would push that assignee past the cap. Prevents - # fan-out workloads from melting a single profile's local model / - # API quota / browser pool while leaving other profiles idle. - # Tasks blocked this way go to skipped_per_profile_capped (not - # skipped_unassigned — the operator-actionable signal is different: - # "this profile is busy, try again later" not "this needs routing"). + # Per-profile concurrency cap: track in-flight workers per assignee and + # refuse spawns past the cap so a fan-out can't melt one profile's local + # model / API quota / browser pool. Deferred tasks go to + # skipped_per_profile_capped, not skipped_unassigned — "busy, retry + # later" is a different operator signal from "needs routing". _per_profile_cap = max_in_progress_per_profile if ( isinstance(max_in_progress_per_profile, int) and max_in_progress_per_profile > 0 @@ -10106,326 +9449,63 @@ def _dispatch_once_locked( "GROUP BY assignee" ): _per_profile_running[prow["assignee"]] = int(prow["n"]) - # Normalize default_assignee once: empty/whitespace string → None so the - # rest of the loop can use ``if default_assignee:`` as a single check. - # We also resolve profile_exists once here for the same reason. + # Normalize default_assignee once (empty → None) and resolve + # profile_exists once. When the profiles module isn't importable (test + # stubs, exotic envs) trust the operator's config: the downstream + # profile_exists check on the assigned row still buckets a missing + # profile as nonspawnable with the existing diagnostic. _default_assignee = (default_assignee or "").strip() or None - _default_assignee_resolved = False if _default_assignee: try: from hermes_cli.profiles import profile_exists as _pe - _default_assignee_resolved = bool(_pe(_default_assignee)) + if not _pe(_default_assignee): + _default_assignee = None except Exception: - # Profiles module not importable (test stubs, exotic envs). - # Trust the operator's config and try the assignment; the - # downstream profile_exists check on the assigned row will - # bucket it as nonspawnable if the profile genuinely isn't - # there, with the existing diagnostic. - _default_assignee_resolved = True + pass for row in ready_rows: if ready_budget is not None and spawned >= ready_budget: break row_assignee = row["assignee"] if not row_assignee: - # Honour kanban.default_assignee: when the dispatcher hits an - # unassigned ready task and an operator-configured fallback - # exists, persist the assignment and proceed. This removes the - # dashboard footgun where a task created without an assignee - # parks in 'ready' forever even though the operator's intent - # ("default") was perfectly clear (#27145). Mutating the row - # (not just the in-memory view) keeps diagnostics and the - # board state consistent: the task is now legitimately owned - # by ``kanban.default_assignee``, not "unassigned but secretly - # routed". - if _default_assignee and _default_assignee_resolved: - # Dry-run: show what WOULD happen (auto-assign + spawn) without - # mutating the DB. Real run: mutate the row + emit the - # 'assigned' event so the board state matches what just happened. - if not dry_run: - try: - with write_txn(conn): - conn.execute( - "UPDATE tasks SET assignee = ? WHERE id = ? " - "AND (assignee IS NULL OR assignee = '')", - (_default_assignee, row["id"]), - ) - _append_event( - conn, row["id"], "assigned", - { - "assignee": _default_assignee, - "source": "kanban.default_assignee", - }, - ) - except Exception: - _log.debug( - "kanban dispatch: failed to apply default_assignee=%r " - "to task %s", - _default_assignee, row["id"], exc_info=True, - ) - result.skipped_unassigned.append(row["id"]) - continue - row_assignee = _default_assignee - result.auto_assigned_default.append(row["id"]) - else: + # Honour kanban.default_assignee so a task created without an + # assignee doesn't park in 'ready' forever when the operator's + # intent ("default") was clear. + if not _default_assignee or not _apply_default_assignee( + conn, row["id"], _default_assignee, dry_run=dry_run, + ): result.skipped_unassigned.append(row["id"]) continue - # Skip ready tasks whose assignee is not a real Hermes profile. - # `_default_spawn` invokes ``hermes -p `` which fails - # with "Profile 'X' does not exist" when the assignee names a - # control-plane lane (e.g. an interactive Claude Code terminal - # like ``orion-cc`` / ``orion-research``) rather than a Hermes - # profile. Those task lanes are pulled by terminals via - # ``claim_task`` directly and should NEVER auto-spawn — the - # subprocess would crash on startup, get reaped as a zombie, - # the task would loop back to ``ready`` on next tick, and we'd - # burn CPU forever (#kanban-dispatcher-crash-loop 2026-05-05). - try: - from hermes_cli.profiles import profile_exists # local import: avoids cycle - except Exception: - profile_exists = None # type: ignore[assignment] - if profile_exists is not None and not profile_exists(row_assignee): - # Bucket separately from skipped_unassigned: the operator - # cannot fix this by assigning a profile (the assignee IS the - # intended owner — a terminal lane). Health telemetry uses - # this distinction to suppress spurious "stuck" warnings on - # multi-lane setups where the ready queue is steadily full - # of human-pulled work. - result.skipped_nonspawnable.append(row["id"]) - continue - # Per-profile concurrency cap (#21582): even if there's global - # headroom, refuse to spawn for an assignee that's already at - # its in-flight cap. Prevents one profile's local model / API - # quota / browser pool from being overwhelmed by a fan-out - # while the global max_in_progress / max_spawn caps still allow - # work on OTHER profiles. - if _per_profile_cap is not None: - current = _per_profile_running.get(row_assignee, 0) - if current >= _per_profile_cap: - result.skipped_per_profile_capped.append( - (row["id"], row_assignee, current) - ) - continue - # Respawn guard: refuse to re-spawn when useful work is already - # in-flight/recent, or when the last failure is a deterministic - # blocker (quota / auth). The guard defers the spawn this tick so - # the task gets a chance to clear (rate limits often reset in - # seconds-to-minutes); the existing consecutive_failures counter - # still trips the auto-block circuit breaker after failure_limit - # consecutive failures, so a persistent auth error eventually - # blocks via the normal path rather than on first occurrence. - guard_reason = check_respawn_guard(conn, row["id"]) - if guard_reason is not None: - result.respawn_guarded.append((row["id"], guard_reason)) - # Emit an event so operators can see why the task was - # skipped when reading `hermes kanban tail` — without - # this the task appears stuck in ready with no diagnosis. - if not dry_run: - with write_txn(conn): - _append_event( - conn, row["id"], "respawn_guarded", - {"reason": guard_reason}, - ) - continue - if dry_run: - result.spawned.append((row["id"], row_assignee, "")) + row_assignee = _default_assignee + result.auto_assigned_default.append(row["id"]) + if _dispatch_lane_task( + conn, row, row_assignee, result, lane="ready", dry_run=dry_run, + ttl_seconds=ttl_seconds, board=board, failure_limit=failure_limit, + spawn_fn=spawn_fn, per_profile_cap=_per_profile_cap, + per_profile_running=_per_profile_running, + ): spawned += 1 - # Increment per-profile counter even in dry_run so the cap - # check sees the would-be spawn on subsequent iterations. - # Without this, dry_run reports every task as spawnable and - # under-reports the capped subset (#21582). - if _per_profile_cap is not None and row_assignee: - _per_profile_running[row_assignee] = ( - _per_profile_running.get(row_assignee, 0) + 1 - ) - continue - claimed = claim_task(conn, row["id"], ttl_seconds=ttl_seconds) - if claimed is None: - continue - try: - resolved_branch_name = None - if claimed.workspace_kind == "worktree": - workspace, resolved_branch_name = _resolve_worktree_workspace(claimed, board=board) - else: - workspace = resolve_workspace(claimed, board=board) - except Exception as exc: - auto = _record_spawn_failure( - conn, claimed.id, f"workspace: {exc}", - failure_limit=failure_limit, - ) - if auto: - result.auto_blocked.append(claimed.id) - continue - # Persist the resolved workspace path so the worker can cd there. - set_workspace_path(conn, claimed.id, str(workspace)) - if claimed.workspace_kind == "worktree": - set_branch_name(conn, claimed.id, resolved_branch_name or (claimed.branch_name or "").strip() or f"wt/{claimed.id}") - _maybe_emit_scratch_tip(conn, claimed.id, claimed.workspace_kind) - _spawn = spawn_fn if spawn_fn is not None else _default_spawn - try: - # Back-compat: older spawn_fn signatures accept only - # (task, workspace). Test stubs in the suite rely on that. - # Introspect the callable and pass `board` only when supported. - import inspect - try: - sig = inspect.signature(_spawn) - if "board" in sig.parameters: - pid = _spawn(claimed, str(workspace), board=board) - else: - pid = _spawn(claimed, str(workspace)) - except (TypeError, ValueError): - pid = _spawn(claimed, str(workspace)) - if pid: - _set_worker_pid(conn, claimed.id, int(pid)) - # Worker-lifecycle observer (RFC #58548): fires AFTER spawn_fn - # returned and the PID (when reported) is durably persisted, - # per the RFC timing contract. Best-effort — can never break - # the dispatch loop. - _fire_worker_spawned_hook( - conn, claimed, str(workspace), pid, board=board, - ) - # NOTE: we intentionally do NOT reset consecutive_failures - # here. A successful spawn proves the worker can start but - # doesn't prove the run will succeed. Under unified - # failure counting, resetting on spawn would let a task - # that keeps timing out after spawn loop forever. The - # counter is cleared only on successful completion (see - # complete_task). - result.spawned.append((claimed.id, claimed.assignee or "", str(workspace))) - spawned += 1 - # Track the new in-flight count for this profile so later - # iterations in this same tick respect the per-profile cap - # (#21582). Subsequent ticks re-query from the DB. - if _per_profile_cap is not None and claimed.assignee: - _per_profile_running[claimed.assignee] = ( - _per_profile_running.get(claimed.assignee, 0) + 1 - ) - except Exception as exc: - auto = _record_spawn_failure( - conn, claimed.id, str(exc), - failure_limit=failure_limit, - ) - if auto: - result.auto_blocked.append(claimed.id) # ---- review column dispatch ---- - # Review tasks are tasks that a worker moved to 'review' after - # creating a PR. The dispatcher spawns a review agent (loading - # sdlc-review skill) that verifies the candidate and either approves - # (→ done) or requests changes (→ ready/todo for the implementer). - # - # Same concurrency model as ready dispatch: review spawns count - # against max_spawn alongside ready tasks, so the total number of - # running workers stays bounded. - # Auto-dispatch is enabled by default because Hermes bundles the - # ``sdlc-review`` skill and reviewer workers can now approve, request - # changes without block-loop accounting, or escalate a genuine blocker. - # Human-only boards can disable it with ``kanban.review_dispatch``. - # - # ``review_rows`` was enumerated before the ready loop; when it is - # non-empty the ready loop ran against ``ready_budget`` (one slot held - # back) so this lane cannot be permanently starved by a sustained - # ready backlog. The review loop itself still checks the FULL shared - # ``spawn_budget`` — the reservation caps the ready lane, it does not - # grant the review lane extra capacity. + # A worker moved the task to 'review' after creating a PR; the dispatcher + # spawns a review agent (sdlc-review skill) that approves (→ done) or + # requests changes (→ ready/todo for the implementer). Review spawns share + # max_spawn with ready tasks so total running workers stay bounded; enabled + # by default, disable with ``kanban.review_dispatch`` for human-only boards. + # The review loop checks the FULL shared ``spawn_budget`` — the + # reservation above caps the ready lane, it grants no extra capacity here. for row in review_rows: if spawn_budget is not None and spawned >= spawn_budget: break if not row["assignee"]: result.skipped_unassigned.append(row["id"]) continue - try: - from hermes_cli.profiles import profile_exists - except Exception: - profile_exists = None # type: ignore[assignment] - if profile_exists is not None and not profile_exists(row["assignee"]): - result.skipped_nonspawnable.append(row["id"]) - continue - if _per_profile_cap is not None: - current = _per_profile_running.get(row["assignee"], 0) - if current >= _per_profile_cap: - result.skipped_per_profile_capped.append( - (row["id"], row["assignee"], current) - ) - continue - guard_reason = check_respawn_guard(conn, row["id"], lane="review") - if guard_reason is not None: - result.respawn_guarded.append((row["id"], guard_reason)) - if not dry_run: - with write_txn(conn): - _append_event( - conn, row["id"], "respawn_guarded", - {"reason": guard_reason}, - ) - continue - if dry_run: - result.spawned.append((row["id"], row["assignee"], "")) + if _dispatch_lane_task( + conn, row, row["assignee"], result, lane="review", dry_run=dry_run, + ttl_seconds=ttl_seconds, board=board, failure_limit=failure_limit, + spawn_fn=spawn_fn, per_profile_cap=_per_profile_cap, + per_profile_running=_per_profile_running, + ): spawned += 1 - if _per_profile_cap is not None: - _per_profile_running[row["assignee"]] = ( - _per_profile_running.get(row["assignee"], 0) + 1 - ) - continue - claimed = claim_review_task(conn, row["id"], ttl_seconds=ttl_seconds) - if claimed is None: - continue - try: - resolved_branch_name = None - if claimed.workspace_kind == "worktree": - workspace, resolved_branch_name = _resolve_worktree_workspace(claimed, board=board) - else: - workspace = resolve_workspace(claimed, board=board) - except Exception as exc: - auto = _record_spawn_failure( - conn, claimed.id, f"workspace: {exc}", - failure_limit=failure_limit, - ) - if auto: - result.auto_blocked.append(claimed.id) - continue - # Persist the resolved workspace path so the worker can cd there. - set_workspace_path(conn, claimed.id, str(workspace)) - if claimed.workspace_kind == "worktree": - set_branch_name(conn, claimed.id, resolved_branch_name or (claimed.branch_name or "").strip() or f"wt/{claimed.id}") - _maybe_emit_scratch_tip(conn, claimed.id, claimed.workspace_kind) - # Force-load the sdlc-review skill for review agents — it carries - # the review logic (AC verification, merge, etc.). The mandatory - # kanban lifecycle is already injected into every worker's system - # prompt via KANBAN_GUIDANCE, so this is the only extra skill the - # review agent needs. - claimed.skills = list( - dict.fromkeys([*(claimed.skills or []), "sdlc-review"]) - ) - _spawn = spawn_fn if spawn_fn is not None else _default_spawn - try: - import inspect - try: - sig = inspect.signature(_spawn) - if "board" in sig.parameters: - pid = _spawn(claimed, str(workspace), board=board) - else: - pid = _spawn(claimed, str(workspace)) - except (TypeError, ValueError): - pid = _spawn(claimed, str(workspace)) - if pid: - _set_worker_pid(conn, claimed.id, int(pid)) - # Worker-lifecycle observer (RFC #58548): same contract as the - # ready-lane fire above — after spawn + PID persistence. - _fire_worker_spawned_hook( - conn, claimed, str(workspace), pid, board=board, - ) - result.spawned.append((claimed.id, claimed.assignee or "", str(workspace))) - spawned += 1 - if _per_profile_cap is not None and claimed.assignee: - _per_profile_running[claimed.assignee] = ( - _per_profile_running.get(claimed.assignee, 0) + 1 - ) - except Exception as exc: - auto = _record_spawn_failure( - conn, claimed.id, str(exc), - failure_limit=failure_limit, - ) - if auto: - result.auto_blocked.append(claimed.id) return result @@ -10753,47 +9833,30 @@ def _default_spawn( env.pop(key, None) # Inject HERMES_HOME so the worker reads the profile-scoped config.yaml - # (fallback_providers, toolsets, agent settings, etc.) instead of the root - # config. Without this, `env = dict(os.environ)` copies only the parent's - # env, and when the child process starts `hermes -p ` the - # _apply_profile_override() runs *before* hermes_constants is imported. - # If HERMES_HOME is absent from the child's env, get_hermes_home() falls - # back to Path.home() / ".hermes" (the DEFAULT profile root), ignoring the - # profile-specific config entirely. Fixes profile-scoped fallback_providers - # being invisible to kanban workers. + # (fallback_providers, toolsets, ...). Without it the child's + # get_hermes_home() falls back to the DEFAULT profile root because + # `hermes -p` applies its override before hermes_constants is imported. from hermes_cli.profiles import resolve_profile_env try: env["HERMES_HOME"] = resolve_profile_env(profile_arg) except FileNotFoundError: - # Profile dir doesn't exist — defer resolution to the CLI's - # _apply_profile_override() via HERMES_PROFILE (set below). - # This only happens in test fixtures where the isolated - # HERMES_HOME never had profiles created. + # No profile dir (isolated test fixtures) — the CLI resolves it from + # HERMES_PROFILE (set below) instead. pass if task.tenant: env["HERMES_TENANT"] = task.tenant env["HERMES_KANBAN_TASK"] = task.id env["HERMES_KANBAN_WORKSPACE"] = workspace - # Tag the worker's session so it lands in state.db as `kanban`, not as an - # untitled `cli` row. A worker is a dispatcher-owned run whose transcript is - # read on the board and in `hermes kanban log` — it is not a conversation - # the user started, so every session-browsing surface (desktop sidebar, TUI - # resume picker, session_search) filters it out by source. Without this the - # sidebar renders one row per attempt, labeled with the worker's own prompt - # ("work kanban task t_…"). + # Tag the session as `kanban` (not an untitled `cli` row): a worker is a + # dispatcher-owned run read on the board / `hermes kanban log`, so every + # session-browsing surface filters it out by source instead of rendering + # one sidebar row per attempt. env["HERMES_SESSION_SOURCE"] = "kanban" - # Pin TERMINAL_CWD to the task's workspace so the worker's file tools and - # context-file loader anchor on the workspace, not whatever cwd the - # dispatching gateway happened to export. The worker subprocess is already - # launched with cwd=workspace, but TERMINAL_CWD takes precedence over the - # process cwd in both file_tools._resolve_base_dir (#41312 — relative - # write_file paths were landing in the gateway user's home) and - # build_context_files_prompt (#34619 — workers loaded the dispatching - # gateway's AGENTS.md instead of the task's). Setting it to the workspace - # fixes both: the workspace is where the task's work actually happens. - # Only pin a real, absolute directory — file_tools rejects relative / - # sentinel TERMINAL_CWD values, so a non-dir workspace must NOT be set - # here (leave the inherited value rather than write a meaningless one). + # Pin TERMINAL_CWD to the workspace: it takes precedence over the process + # cwd in file_tools._resolve_base_dir and build_context_files_prompt, so + # without it relative writes land in the gateway user's home and workers + # load the gateway's AGENTS.md. Only a real absolute directory — file_tools + # rejects relative / sentinel values, so leave the inherited value otherwise. if workspace and os.path.isabs(workspace) and os.path.isdir(workspace): env["TERMINAL_CWD"] = workspace if task.branch_name: @@ -10821,32 +9884,24 @@ def _default_spawn( ) if foreground_timeout is not None: env["TERMINAL_MAX_FOREGROUND_TIMEOUT"] = foreground_timeout - # Pin the shared board + workspaces root the dispatcher resolved, so - # that even when the worker activates a profile (`hermes -p ` - # rewrites HERMES_HOME), its kanban paths still match the - # dispatcher's. Belt-and-braces with the `get_default_hermes_root()` - # resolution in `kanban_home()` — symmetric resolution is the norm, - # but unusual symlink / Docker layouts are caught here too. + # Pin the board DB + workspaces root the dispatcher resolved so the + # worker's kanban paths still match after `hermes -p` rewrites + # HERMES_HOME (belt-and-braces for symlink / Docker layouts). env["HERMES_KANBAN_DB"] = str(kanban_db_path(board=board)) env["HERMES_KANBAN_WORKSPACES_ROOT"] = str(workspaces_root(board=board)) _retag_legacy_worker_sessions(env["HERMES_KANBAN_WORKSPACES_ROOT"]) - # Board slug — the final defense-in-depth pin. If the worker ever - # resolves kanban paths without the DB / workspaces env vars, the - # board slug still forces it to the right directory. + # Board slug — final defense-in-depth pin if a path is ever resolved + # without the DB / workspaces env vars. resolved_board = _normalize_board_slug(board) or get_current_board() env["HERMES_KANBAN_BOARD"] = resolved_board - # HERMES_PROFILE is the author the kanban_comment tool defaults to. - # `hermes -p ` activates the profile, but the env var is - # what the tool reads — set it explicitly here so comments are - # attributed correctly regardless of how the child loads config. + # kanban_comment reads HERMES_PROFILE for its default author; `-p` alone + # doesn't set the env var. env["HERMES_PROFILE"] = profile_arg - # A worker must NEVER boot the interactive TUI: an inherited HERMES_TUI=1 - # or a `display.interface: tui` in the profile's config would send the - # quiet chat run into the Ink TUI, whose no-TTY bail-out exits 0 without - # doing the task → "protocol violation" on every attempt. `--cli` is the - # highest-precedence interface override; dropping the env var covers - # older hermes builds on PATH that predate the flag's precedence. + # A worker must NEVER boot the interactive TUI: its no-TTY bail-out exits + # 0 without doing the task → "protocol violation" every attempt. `--cli` + # is the highest-precedence override; dropping HERMES_TUI covers older + # hermes builds on PATH that predate the flag's precedence. env.pop("HERMES_TUI", None) cmd = [ @@ -11053,6 +10108,30 @@ def build_worker_context(conn: sqlite3.Connection, task_id: str) -> str: return s return s[:limit] + f"… [truncated, {len(s) - limit} chars omitted]" + def _stamp(ts: int) -> str: + """``YYYY-MM-DD HH:MM`` plus a relative age when one is available.""" + disp = time.strftime("%Y-%m-%d %H:%M", time.localtime(ts)) + age = _relative_age(ts, _now) + return f"{disp}, {age}" if age else disp + + def _metadata_line(metadata: Any) -> Optional[str]: + if not metadata: + return None + try: + return f"_metadata_: `{_cap(json.dumps(metadata, ensure_ascii=False, sort_keys=True))}`" + except Exception: + return None + + def _tail(items: list, cap: int, noun: str) -> tuple[list, Optional[str]]: + """Keep the newest ``cap`` items; describe the omitted head, if any.""" + omitted = max(0, len(items) - cap) + if not omitted: + return items, None + return items[-cap:], ( + f"_({omitted} earlier {noun}{'s' if omitted != 1 else ''} " + f"omitted; showing most recent {cap})_" + ) + lines: list[str] = [] lines.append(f"# Kanban task {task.id}: {task.title}") lines.append("") @@ -11105,39 +10184,24 @@ def build_worker_context(conn: sqlite3.Connection, task_id: str) -> str: # more exist without bloating the prompt. all_prior = [r for r in list_runs(conn, task_id) if r.ended_at is not None] # list_runs returns ascending by started_at; "most recent" = last N - if len(all_prior) > _CTX_MAX_PRIOR_ATTEMPTS: - omitted = len(all_prior) - _CTX_MAX_PRIOR_ATTEMPTS - shown = all_prior[-_CTX_MAX_PRIOR_ATTEMPTS:] - first_shown_idx = omitted + 1 - else: - omitted = 0 - shown = all_prior - first_shown_idx = 1 + shown, omitted_note = _tail(all_prior, _CTX_MAX_PRIOR_ATTEMPTS, "attempt") + first_shown_idx = len(all_prior) - len(shown) + 1 if shown: lines.append("## Prior attempts on this task") - if omitted: - lines.append( - f"_({omitted} earlier attempt{'s' if omitted != 1 else ''} " - f"omitted; showing most recent {len(shown)})_" - ) + if omitted_note: + lines.append(omitted_note) for offset, run in enumerate(shown): idx = first_shown_idx + offset - ts = time.strftime("%Y-%m-%d %H:%M", time.localtime(run.started_at)) - age = _relative_age(run.started_at, _now) - ts_disp = f"{ts}, {age}" if age else ts profile = run.profile or "(unknown)" outcome = run.outcome or run.status - lines.append(f"### Attempt {idx} — {outcome} ({profile}, {ts_disp})") + lines.append(f"### Attempt {idx} — {outcome} ({profile}, {_stamp(run.started_at)})") if run.summary and run.summary.strip(): lines.append(_cap(run.summary)) if run.error and run.error.strip(): lines.append(f"_error_: {_cap(run.error)}") - if run.metadata: - try: - meta_str = json.dumps(run.metadata, ensure_ascii=False, sort_keys=True) - lines.append(f"_metadata_: `{_cap(meta_str)}`") - except Exception: - pass + meta_line = _metadata_line(run.metadata) + if meta_line: + lines.append(meta_line) lines.append("") # Parents: prefer the most-recent 'completed' run's summary + metadata, @@ -11188,12 +10252,9 @@ def build_worker_context(conn: sqlite3.Connection, task_id: str) -> str: else: body_lines.append("(no result recorded)") - if run is not None and run.metadata: - try: - meta_str = json.dumps(run.metadata, ensure_ascii=False, sort_keys=True) - body_lines.append(f"_metadata_: `{_cap(meta_str)}`") - except Exception: - pass + meta_line = _metadata_line(run.metadata) if run is not None else None + if meta_line: + body_lines.append(meta_line) lines.extend(body_lines) lines.append("") @@ -11215,45 +10276,29 @@ def build_worker_context(conn: sqlite3.Connection, task_id: str) -> str: if role_rows: lines.append(f"## Recent work by @{task.assignee}") for row in role_rows: - ts = time.strftime( - "%Y-%m-%d %H:%M", time.localtime(int(row["ended_at"])) - ) - age = _relative_age(row["ended_at"], _now) - ts_disp = f"{ts}, {age}" if age else ts s = (row["summary"] or "").strip().splitlines() first = s[0][:200] if s else "(no summary)" - lines.append(f"- {row['id']} — {row['title']} ({ts_disp}): {first}") + lines.append( + f"- {row['id']} — {row['title']} ({_stamp(int(row['ended_at']))}): {first}" + ) lines.append("") # Comments: cap at the most-recent _CTX_MAX_COMMENTS so # comment-storm tasks don't blow out the worker's prompt. Older # comments summarised in a one-line marker like prior attempts. - all_comments = list_comments(conn, task_id) - if len(all_comments) > _CTX_MAX_COMMENTS: - omitted_c = len(all_comments) - _CTX_MAX_COMMENTS - shown_c = all_comments[-_CTX_MAX_COMMENTS:] - else: - omitted_c = 0 - shown_c = all_comments + shown_c, omitted_note = _tail(list_comments(conn, task_id), _CTX_MAX_COMMENTS, "comment") if shown_c: lines.append("## Comment thread") - if omitted_c: - lines.append( - f"_({omitted_c} earlier comment{'s' if omitted_c != 1 else ''} " - f"omitted; showing most recent {len(shown_c)})_" - ) + if omitted_note: + lines.append(omitted_note) for c in shown_c: - ts = time.strftime("%Y-%m-%d %H:%M", time.localtime(c.created_at)) - age = _relative_age(c.created_at, _now) - ts_disp = f"{ts}, {age}" if age else ts - # Render author with explicit "comment from worker" framing so - # operator-controlled HERMES_PROFILE values like "hermes-system" - # or "operator" can't be misread by the next worker as a system - # directive above the (attacker-influenceable) comment body. - # Defense-in-depth — the LLM-controlled author-forgery surface - # was already closed in #22435. See #22452. + # Explicit "comment from worker" framing so operator-controlled + # HERMES_PROFILE values like "hermes-system" or "operator" can't be + # misread by the next worker as a system directive above the + # (attacker-influenceable) comment body. Defense-in-depth on top of + # the closed LLM-controlled author-forgery surface. safe_author = (c.author or "").replace("`", "") - lines.append(f"comment from worker `{safe_author}` at {ts_disp}:") + lines.append(f"comment from worker `{safe_author}` at {_stamp(c.created_at)}:") lines.append(_cap(c.body, _CTX_MAX_COMMENT_BYTES)) lines.append("") @@ -11427,14 +10472,12 @@ def add_notify_sub( default for a fresh row); an explicit value is last-write-wins, so an operator can intentionally re-subscribe to change the mode (e.g. ``notify`` -> ``wake``). An unknown value falls back to ``"notify"``. - New subscriptions start "caught up": ``last_event_id`` snaps to the - task's current ``MAX(task_events.id)`` at creation instead of the - schema default 0. A cursor of 0 on an already-active task made the - gateway notifier replay every historical terminal event on its next - tick — and with many stale subs, a single boot-time burst of 100+ - messages (issue #29905). Subscribers only want events that occur - AFTER they subscribe; the gateway/tool auto-subscribe paths run at - task creation, where the snapshot is 0 anyway. + New subscriptions start "caught up": ``last_event_id`` snaps to the task's + current ``MAX(task_events.id)`` instead of the schema default 0, which made + the notifier replay every historical terminal event on its next tick (a + boot-time burst of 100+ messages with many stale subs). Subscribers only + want events AFTER they subscribe; auto-subscribe at task creation snapshots + 0 anyway. """ insert_mode = delivery_mode if delivery_mode in _NOTIFY_DELIVERY_MODES else ( # api_server is stateless: the adapter has no send() — the wake @@ -11473,57 +10516,25 @@ def add_notify_sub( task_id, ), ) - if chat_type: - # Explicit chat_type is last-write-wins on re-subscribe. + # Re-subscribe semantics per column: chat_type / delivery_mode / + # delivery_metadata are last-write-wins; user_id_alt and + # notifier_profile only self-heal legacy rows that never had one. + key = (task_id, platform, chat_id, thread_id or "") + for column, value, fill_only in ( + ("chat_type", chat_type, False), + ("user_id_alt", user_id_alt, True), + ("notifier_profile", notifier_profile, True), + ("delivery_mode", delivery_mode if delivery_mode in _NOTIFY_DELIVERY_MODES else None, False), + ("delivery_metadata", metadata_json, False), + ): + if not value: + continue + guard = f" AND ({column} IS NULL OR {column} = '')" if fill_only else "" conn.execute( - """ - UPDATE kanban_notify_subs - SET chat_type = ? - WHERE task_id = ? AND platform = ? AND chat_id = ? AND thread_id = ? - """, - (chat_type, task_id, platform, chat_id, thread_id or ""), - ) - if user_id_alt: - # Self-heal legacy rows created before alternate IDs were tracked. - conn.execute( - """ - UPDATE kanban_notify_subs - SET user_id_alt = ? - WHERE task_id = ? AND platform = ? AND chat_id = ? AND thread_id = ? - AND (user_id_alt IS NULL OR user_id_alt = '') - """, - (user_id_alt, task_id, platform, chat_id, thread_id or ""), - ) - if notifier_profile: - # Self-heal legacy rows that predate notifier ownership. - conn.execute( - """ - UPDATE kanban_notify_subs - SET notifier_profile = ? - WHERE task_id = ? AND platform = ? AND chat_id = ? AND thread_id = ? - AND (notifier_profile IS NULL OR notifier_profile = '') - """, - (notifier_profile, task_id, platform, chat_id, thread_id or ""), - ) - if delivery_mode in _NOTIFY_DELIVERY_MODES: - # Explicit delivery_mode is last-write-wins on re-subscribe. - conn.execute( - """ - UPDATE kanban_notify_subs - SET delivery_mode = ? - WHERE task_id = ? AND platform = ? AND chat_id = ? AND thread_id = ? - """, - (delivery_mode, task_id, platform, chat_id, thread_id or ""), - ) - if metadata_json: - # Refresh the routing anchor for duplicate subscriptions. - conn.execute( - """ - UPDATE kanban_notify_subs - SET delivery_metadata = ? - WHERE task_id = ? AND platform = ? AND chat_id = ? AND thread_id = ? - """, - (metadata_json, task_id, platform, chat_id, thread_id or ""), + f"UPDATE kanban_notify_subs SET {column} = ? " + "WHERE task_id = ? AND platform = ? AND chat_id = ? AND thread_id = ?" + + guard, + (value, *key), ) @@ -11728,6 +10739,18 @@ def purge_stale_done_notify_subs( return int(cur.rowcount or 0) +def _notify_cursor( + conn: sqlite3.Connection, task_id: str, platform: str, chat_id: str, thread_id: Optional[str], +) -> Optional[int]: + """``last_event_id`` of one subscription row, or ``None`` when unsubscribed.""" + row = conn.execute( + "SELECT last_event_id FROM kanban_notify_subs " + "WHERE task_id = ? AND platform = ? AND chat_id = ? AND thread_id = ?", + (task_id, platform, chat_id, thread_id or ""), + ).fetchone() + return None if row is None else int(row["last_event_id"]) + + def unseen_events_for_sub( conn: sqlite3.Connection, *, @@ -11743,14 +10766,9 @@ def unseen_events_for_sub( cursor is NOT advanced here; call :func:`advance_notify_cursor` after the gateway has successfully delivered the notifications. """ - row = conn.execute( - "SELECT last_event_id FROM kanban_notify_subs " - "WHERE task_id = ? AND platform = ? AND chat_id = ? AND thread_id = ?", - (task_id, platform, chat_id, thread_id or ""), - ).fetchone() - if row is None: + cursor = _notify_cursor(conn, task_id, platform, chat_id, thread_id) + if cursor is None: return 0, [] - cursor = int(row["last_event_id"]) kind_list = list(kinds) if kinds else None q = ( "SELECT * FROM task_events WHERE task_id = ? AND id > ? " @@ -11764,15 +10782,7 @@ def unseen_events_for_sub( out: list[Event] = [] max_id = cursor for r in rows: - try: - payload = json.loads(r["payload"]) if r["payload"] else None - except Exception: - payload = None - out.append(Event( - id=r["id"], task_id=r["task_id"], kind=r["kind"], - payload=payload, created_at=r["created_at"], - run_id=(int(r["run_id"]) if "run_id" in r.keys() and r["run_id"] is not None else None), - )) + out.append(Event.from_row(r)) max_id = max(max_id, int(r["id"])) return max_id, out @@ -11801,14 +10811,9 @@ def claim_unseen_events_for_sub( failed before any terminal unsubscribe removed the row. """ with write_txn(conn): - row = conn.execute( - "SELECT last_event_id FROM kanban_notify_subs " - "WHERE task_id = ? AND platform = ? AND chat_id = ? AND thread_id = ?", - (task_id, platform, chat_id, thread_id or ""), - ).fetchone() - if row is None: + old_cursor = _notify_cursor(conn, task_id, platform, chat_id, thread_id) + if old_cursor is None: return 0, 0, [] - old_cursor = int(row["last_event_id"]) new_cursor, events = unseen_events_for_sub( conn, task_id=task_id, @@ -12063,9 +11068,8 @@ def list_runs( """ if (state_type is None) ^ (state_name is None): raise ValueError("state_type and state_name must both be set or both omitted") - if state_type is not None: - if state_type not in ("status", "outcome"): - raise ValueError("state_type must be 'status' or 'outcome'") + if state_type is not None and state_type not in ("status", "outcome"): + raise ValueError("state_type must be 'status' or 'outcome'") q = "SELECT * FROM task_runs WHERE task_id = ?" params: list[Any] = [task_id] if not include_active: diff --git a/hermes_cli/kanban_decompose.py b/hermes_cli/kanban_decompose.py index d6c248c658..fffe52850f 100644 --- a/hermes_cli/kanban_decompose.py +++ b/hermes_cli/kanban_decompose.py @@ -11,40 +11,26 @@ so when the whole graph completes the root wakes back up — its assignee (the orchestrator profile) gets a chance to judge completion and add more tasks if the work isn't done yet. -Design notes ------------- - -* Mirrors the shape of ``hermes_cli/kanban_specify.py``: lazy aux - client import inside the function, lenient response parse, never - raises on expected failure modes. - -* The system prompt sees the *configured* profile roster — names plus - descriptions plus the default fallback. Profiles without a - description are still listed (with a note) so the decomposer can - match on name as a fallback, but the user has an obvious incentive - to describe them. - -* ``fanout=false`` collapses to the same effect as ``kanban specify``: - we tighten the body and flip ``triage -> todo`` as a single task, - no children created. This makes ``decompose`` a strict superset of - ``specify`` from the user's perspective. - -* If the LLM picks an assignee that doesn't exist as a profile, we - rewrite it to the configured ``default_assignee`` (or the default - profile if unset). A child task NEVER ends up with ``assignee=None``. +Design notes: mirrors ``kanban_specify`` (lazy aux import, lenient parse, +never raises on expected failures). The prompt sees the configured profile +roster; undescribed profiles are listed with a note so name-matching still +works. ``fanout=false`` collapses to the ``specify`` behaviour (tighten + +promote, no children), making ``decompose`` a strict superset. Unknown +assignees are rewritten to ``default_assignee`` — a child NEVER ends up with +``assignee=None``. """ from __future__ import annotations -import json import logging -import os import re from dataclasses import dataclass from typing import Optional from hermes_cli import kanban_db as kb from hermes_cli import profiles as profiles_mod +from hermes_cli.kanban_specify import _extract_json_blob, _title_body, _truncate +from hermes_cli.kanban_specify import _profile_author as _specify_author logger = logging.getLogger(__name__) @@ -136,37 +122,9 @@ class DecomposeOutcome: new_title: Optional[str] = None -def _truncate(text: str, limit: int) -> str: - if len(text) <= limit: - return text - return text[: limit - 1] + "…" - - -def _extract_json_blob(raw: str) -> Optional[dict]: - if not raw: - return None - stripped = _FENCE_RE.sub("", raw.strip()) - first = stripped.find("{") - last = stripped.rfind("}") - if first == -1 or last == -1 or last <= first: - return None - candidate = stripped[first : last + 1] - try: - val = json.loads(candidate) - except (ValueError, json.JSONDecodeError): - return None - if not isinstance(val, dict): - return None - return val - - def _profile_author() -> str: """Mirror of ``hermes_cli.kanban._profile_author``.""" - return ( - os.environ.get("HERMES_PROFILE") - or os.environ.get("USER") - or "decomposer" - ) + return _specify_author("decomposer") def _load_config() -> dict: @@ -177,31 +135,15 @@ def _load_config() -> dict: return {} -def _resolve_orchestrator_profile(cfg: dict) -> str: - """Resolve which profile owns the root/orchestration task after fan-out. +def _resolve_profile_from_cfg(cfg: dict, key: str) -> str: + """``kanban.`` if it names an existing profile, else the active + default profile — so a task is never stranded for lack of an owner. - Falls back to the active default profile when ``kanban.orchestrator_profile`` - is unset, so a task is never stranded for lack of an orchestrator. + ``orchestrator_profile`` owns the root after fan-out; ``default_assignee`` + catches children the decomposer can't route. """ kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {} - explicit = (kanban_cfg.get("orchestrator_profile") or "").strip() - if explicit: - try: - if profiles_mod.profile_exists(explicit): - return explicit - except Exception: - pass - # Fall back to the active default profile. - try: - return profiles_mod.get_active_profile_name() or "default" - except Exception: - return "default" - - -def _resolve_default_assignee(cfg: dict) -> str: - """Resolve which profile catches child tasks the orchestrator can't route.""" - kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {} - explicit = (kanban_cfg.get("default_assignee") or "").strip() + explicit = (kanban_cfg.get(key) or "").strip() if explicit: try: if profiles_mod.profile_exists(explicit): @@ -215,12 +157,8 @@ def _resolve_default_assignee(cfg: dict) -> str: def _build_roster() -> tuple[list[dict], set[str]]: - """Return (roster_for_prompt, valid_assignee_names). - - Each roster entry is ``{name, description, has_description}``. The - valid-set is used after the LLM responds to rewrite invalid - assignees to the default fallback. - """ + """``(roster_for_prompt, valid_assignee_names)``; entries are + ``{name, description, has_description}``.""" roster: list[dict] = [] valid: set[str] = set() try: @@ -255,11 +193,8 @@ def _normalize_assignee_choice( default_assignee: str, valid_names: set[str], ) -> str: - """Return a valid assignee, falling back to ``default_assignee``. - - Fan-out children and the single-task fallback should share the same - routing guarantee: promoted work must not be left unassigned. - """ + """A valid assignee, else ``default_assignee`` — promoted work is never + left unassigned.""" if not isinstance(assignee, str) or not assignee.strip(): return default_assignee chosen = assignee.strip() @@ -274,13 +209,9 @@ def decompose_task( author: Optional[str] = None, timeout: Optional[int] = None, ) -> DecomposeOutcome: - """Decompose a triage task into a graph of child tasks. - - Returns an outcome describing what happened. Never raises for - expected failure modes (task not in triage, no aux client - configured, API error, malformed response, decomposer returned - fanout=true with empty task list) — those surface via ``ok=False``. - """ + """Decompose a triage task into a graph of child tasks. Expected failures + (not in triage, no aux client, API error, malformed/empty reply) surface + as ``ok=False``.""" with kb.connect_closing() as conn: task = kb.get_task(conn, task_id) if task is None: @@ -291,8 +222,8 @@ def decompose_task( ) cfg = _load_config() - orchestrator = _resolve_orchestrator_profile(cfg) - default_assignee = _resolve_default_assignee(cfg) + orchestrator = _resolve_profile_from_cfg(cfg, "orchestrator_profile") + default_assignee = _resolve_profile_from_cfg(cfg, "default_assignee") kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {} auto_promote = bool(kanban_cfg.get("auto_promote_children", True)) roster, valid_names = _build_roster() @@ -312,10 +243,8 @@ def decompose_task( ) try: - # Route through call_llm so auxiliary.kanban_decomposer.* config - # (provider/model/base_url, extra_body, reasoning_effort, retries) - # all apply — the previous direct client.chat.completions.create() - # path dropped auxiliary..extra_body entirely (#35566). + # call_llm applies all auxiliary.kanban_decomposer.* config + # (provider/model/base_url, extra_body, reasoning_effort, retries). resp = call_llm( task="kanban_decomposer", messages=[ @@ -337,7 +266,7 @@ def decompose_task( except Exception: raw = "" - parsed = _extract_json_blob(raw) + parsed = _extract_json_blob(raw, _FENCE_RE) if parsed is None: return DecomposeOutcome(task_id, False, "LLM returned malformed JSON") @@ -346,10 +275,7 @@ def decompose_task( if not fanout: # Fall back to single-task spec promotion (same effect as specify). - new_title = parsed.get("title") - new_body = parsed.get("body") - title_val = new_title.strip() if isinstance(new_title, str) and new_title.strip() else None - body_val = new_body if isinstance(new_body, str) and new_body.strip() else None + title_val, body_val = _title_body(parsed) assignee_val = None if not task.assignee: assignee_val = _normalize_assignee_choice( @@ -385,8 +311,7 @@ def decompose_task( task_id, False, "decomposer returned fanout=true with empty tasks list", ) - # Rewrite invalid assignees to the default fallback. Never leave a - # task with assignee=None — the user explicitly does not want that. + # Unknown assignees route to the default; never assignee=None. children: list[dict] = [] for idx, entry in enumerate(raw_tasks): if not isinstance(entry, dict): diff --git a/hermes_cli/kanban_diagnostics.py b/hermes_cli/kanban_diagnostics.py index 1a0309aa01..61d1e201ce 100644 --- a/hermes_cli/kanban_diagnostics.py +++ b/hermes_cli/kanban_diagnostics.py @@ -1,30 +1,13 @@ """Kanban diagnostics — structured, actionable distress signals for tasks. -A ``Diagnostic`` is a machine-readable description of something that's wrong -with a kanban task: a hallucinated card id, a spawn crash-loop, a task -stuck blocked for too long, etc. Each one carries: +A ``Diagnostic`` carries a **kind** (canonical code the UI/tests match on), a +**severity**, title/detail text, and **actions** the dashboard renders as +buttons and the CLI as hints. Rules are stateless and read-only over +(task, events, runs, optional graph); callers compute on demand. -* A **kind** (canonical code; UI/tests match on this). -* A **severity** (``warning`` / ``error`` / ``critical``). -* A **title** (one-line human description) and **detail** (longer text). -* A list of **suggested actions** — structured entries the dashboard - turns into buttons and the CLI turns into hints. - -Rules run over (task, recent events, recent runs, optional graph context) and -emit diagnostics. They are stateless and read-only — no DB writes. Callers compute -diagnostics on demand (on ``/board`` load, ``/tasks/:id`` fetch, or -``hermes kanban diagnostics``). - -Design goals: - -* Fixable-on-the-operator's-side signals only (missing config, phantom - ids, crash loop). Not "the provider returned 502 once" — that's a - transient runtime blip, not a diagnostic. -* Recoverable: every diagnostic comes with at least one suggested - recovery action the operator can actually take from the UI. -* Auto-clearing: when the underlying failure mode resolves (a clean - ``completed`` event arrives, a spawn succeeds, the task gets - unblocked), the diagnostic stops firing. The audit event trail stays. +Design goals: operator-fixable signals only (not a one-off provider 502); +every diagnostic has at least one recovery action; diagnostics auto-clear +when the failure mode resolves (the audit event trail stays). """ from __future__ import annotations @@ -35,9 +18,7 @@ import json import time -# Severity rungs, ordered least → most urgent. The UI colors them -# amber (warning), orange (error), red (critical). Sorted outputs put -# critical first so operators see the worst fires at the top. +# Least → most urgent; sorted outputs put critical first. SEVERITY_ORDER = ("warning", "error", "critical") @@ -52,24 +33,10 @@ def severity_at_or_above(severity: Optional[str], threshold: Optional[str]) -> b @dataclass class DiagnosticAction: - """A single recovery action attached to a diagnostic. - - The ``kind`` determines how both the UI and CLI render it: - - * ``reclaim`` / ``reassign`` — POST to the matching /tasks/:id/* - endpoint; dashboard wires into the existing recovery popover. - * ``unblock`` — PATCH status back to ``ready`` (for stuck-blocked - diagnostics). - * ``cli_hint`` — print/copy a shell command (e.g. - ``hermes -p auth``). No HTTP side effect. - * ``open_docs`` — deep-link to the docs URL named in ``payload.url``. - * ``comment`` — nudge the operator to add a comment (for - stuck-blocked tasks that need human input). - - ``suggested=True`` marks the action as the recommended first step; - the UI highlights it. Multiple actions can be suggested if they're - equally valid. - """ + """A recovery action. ``kind`` drives rendering: ``reclaim``/``reassign`` + POST to /tasks/:id/*; ``unblock`` PATCHes status to ready; ``cli_hint`` + shows ``payload.command``; ``open_docs`` links ``payload.url``; ``comment`` + nudges the operator. ``suggested=True`` = recommended first step.""" kind: str label: str @@ -122,20 +89,10 @@ class Diagnostic: # --------------------------------------------------------------------------- def _task_field(task, name, default=None): - """Read a field from a task regardless of representation. - - Callers pass sqlite3.Row (dict-like with [] but no attribute - access), kanban_db.Task dataclasses (attribute access), or plain - dicts (both). This normalises them so rule functions don't have - to branch on type each time. - """ + """Read a field from a sqlite3.Row, a kanban_db.Task dataclass, or a dict.""" if task is None: return default - # sqlite Row + plain dicts both support mapping access; Row also - # supports .keys(). try: - # Row raises IndexError if the key isn't a column in the query; - # dicts return default via .get. Handle both. if hasattr(task, "keys") and name in task.keys(): return task[name] except Exception: @@ -169,20 +126,42 @@ def _event_ts(ev) -> int: return int(t or 0) +def _first_field(task, primary: str, legacy: str, default=None): + """``task[primary]`` unless it is None, else ``task[legacy]`` (old DB rows).""" + v = _task_field(task, primary, None) + return v if v is not None else _task_field(task, legacy, default) + + +def _latest_event_ts(events: Iterable[Any], kinds: set[str]) -> int: + """Max ``created_at`` over events whose kind is in ``kinds`` (0 if none).""" + latest = 0 + for ev in events: + if _event_kind(ev) in kinds: + latest = max(latest, _event_ts(ev)) + return latest + + +def _log_hint_action(task_id: str) -> DiagnosticAction: + return DiagnosticAction( + kind="cli_hint", + label=f"Check logs: hermes kanban log {task_id}", + payload={"command": f"hermes kanban log {task_id}"}, + suggested=True, + ) + + +def _error_snippet(last_err) -> str: + """First 500 chars of the error (with ellipsis), or "" when absent.""" + err_text = (last_err or "").strip() if last_err else "" + return err_text[:500] + ("…" if len(err_text) > 500 else "") if err_text else "" + + def _active_hallucination_events( events: Iterable[Any], kind: str, ) -> list[Any]: - """Return events of ``kind`` that have no ``completed``/``edited`` - event *strictly after* them. Walks chronologically: each clean - event resets the accumulator; each matching event gets appended. - - Events must be sorted by id (i.e. arrival order); callers pass the - task's full event list which the DB already returns in that order. - """ - # Events arrive sorted by id asc (chronological). Walk once, track - # which hallucination events are still "active" (no clean event - # supersedes them). + """Events of ``kind`` with no ``completed``/``edited`` event strictly after + them. Requires id-sorted (arrival-order) input, which the DB provides.""" active: list[Any] = [] for ev in events: k = _event_kind(ev) @@ -191,9 +170,9 @@ def _active_hallucination_events( elif k == kind: active.append(ev) return active -# Standard always-available actions. Every diagnostic can offer these as -# fallbacks regardless of kind — they're the two baseline recovery -# primitives the kernel supports. + + +# Baseline recovery primitives every diagnostic can fall back on. def _generic_recovery_actions(task: Any, *, running: bool) -> list[DiagnosticAction]: out: list[DiagnosticAction] = [] if running: @@ -214,22 +193,16 @@ def _generic_recovery_actions(task: Any, *, running: bool) -> list[DiagnosticAct # Rule implementations # --------------------------------------------------------------------------- -# Each rule takes (task, events, runs, now_ts, config) and returns -# zero or more Diagnostic instances. ``events`` / ``runs`` are lists of -# kanban_db.Event / kanban_db.Run (or plain dicts matching the same -# shape — for test convenience). +# Each rule: (task, events, runs, now_ts, config) -> list[Diagnostic]. +# ``events``/``runs`` are kanban_db rows/dataclasses or same-shaped dicts. RuleFn = Callable[[Any, list[Any], list[Any], int, dict], list[Diagnostic]] def _aux_slot_explicit(slot: Any) -> bool: - """Return True if the auxiliary slot has user-supplied non-default fields. - - Defaults from ``DEFAULT_CONFIG`` use ``provider: "auto"`` with empty - model/base_url/api_key — that path falls through to the main model. An - "explicit" config is one where the user actively set a provider (not - "auto"), or supplied a model / base_url / api_key. - """ + """True if the aux slot was user-configured: provider other than "auto", + or any of model/base_url/api_key set (the default falls through to the + main model).""" if not isinstance(slot, dict): return False provider = str(slot.get("provider") or "").strip().lower() @@ -242,12 +215,9 @@ def _aux_slot_explicit(slot: Any) -> bool: def _main_model_visible(raw_config: Any) -> bool: - """Best-effort check that a main model is configured. - - Diagnostics runs in the dashboard process which may not share the CLI's - runtime state, so we read the raw config dict. If we cannot prove the - main model is set, we err on the side of NOT firing the diagnostic. - """ + """Best-effort "a main model is configured" from the raw config dict (the + dashboard process may not share CLI runtime state). Unprovable => False, + which errs toward NOT firing the diagnostic.""" if not isinstance(raw_config, dict): return False model_cfg = raw_config.get("model") @@ -264,17 +234,9 @@ def _main_model_visible(raw_config: Any) -> bool: def triage_aux_status(config: Optional[dict]) -> Optional[dict]: - """Inspect raw config and report whether triage paths look configured. - - Returns ``None`` when config context is unavailable (suppress diagnostic - to avoid noisy false positives in tests / low-level callers). Otherwise - returns a dict with: - - - ``auto_decompose``: bool — whether the dispatcher auto-runs decompose - - ``decomposer_explicit``: bool — user-supplied decomposer slot - - ``specifier_explicit``: bool — user-supplied specifier slot - - ``main_model_visible``: bool — main model can serve as auto fallback - """ + """Report whether the triage aux paths look configured: ``{auto_decompose, + decomposer_explicit, specifier_explicit, main_model_visible}``. ``None`` + when no config context is present (keeps low-level callers/tests silent).""" if not isinstance(config, dict): return None @@ -285,9 +247,7 @@ def triage_aux_status(config: Optional[dict]) -> Optional[dict]: aux = config.get("auxiliary") kanban_cfg = config.get("kanban") if isinstance(config.get("kanban"), dict) else {} - # Have we been handed any config context at all? When neither auxiliary - # nor kanban nor model keys are present, the caller is a low-level test - # passing {} — stay silent. + # No auxiliary/kanban/model keys at all => a low-level caller passing {}. if ( not isinstance(aux, dict) and not kanban_cfg @@ -323,14 +283,8 @@ def _positive_int(value: Any, default: int) -> int: def _rule_hallucinated_cards(task, events, runs, now, cfg) -> list[Diagnostic]: - """Blocked-hallucination gate fires: a worker called kanban_complete - with created_cards that didn't exist or weren't created by the - completing profile. Task stayed in its prior state; the operator - needs to decide how to proceed. - - Auto-clears when a successful completion (or edit) follows the - blocked event. - """ + """A worker's kanban_complete named created_cards that don't exist / weren't + its own; the completion was blocked. Clears on a later completion/edit.""" hits = _active_hallucination_events(events, "completion_blocked_hallucination") if not hits: return [] @@ -343,13 +297,11 @@ def _rule_hallucinated_cards(task, events, runs, now, cfg) -> list[Diagnostic]: if pid not in phantom_ids: phantom_ids.append(pid) running = _task_field(task, "status") == "running" - actions: list[DiagnosticAction] = [] - actions.append(DiagnosticAction( + actions = [DiagnosticAction( kind="comment", label="Add a comment explaining what to do", suggested=False, - )) - actions.extend(_generic_recovery_actions(task, running=running)) + )] + _generic_recovery_actions(task, running=running) return [Diagnostic( kind="hallucinated_cards", severity="error", @@ -370,22 +322,12 @@ def _rule_hallucinated_cards(task, events, runs, now, cfg) -> list[Diagnostic]: def _rule_triage_aux_unavailable(task, events, runs, now, cfg) -> list[Diagnostic]: - """A triage task cannot leave triage without an auxiliary helper. - - With the auto-decompose dispatcher (kanban.auto_decompose, default True), - triage tasks fan out via ``auxiliary.kanban_decomposer`` and fall back to - ``auxiliary.triage_specifier`` when the decomposer returns ``fanout=false``. - With auto-decompose off, the user must run ``hermes kanban specify``, - which only needs ``auxiliary.triage_specifier``. - - The default slot is ``provider: auto`` → auto-falls back to the main model, - so this rule only fires when: - - - the relevant slot is explicitly set to something broken, OR - - the auto fallback has no main model to fall back to. - - Config context is required; pass {} from tests to keep the rule silent. - """ + """A triage task can't leave triage without a usable aux model. With + auto-decompose on the primary slot is ``auxiliary.kanban_decomposer`` + (specifier as fallback); off, it is ``auxiliary.triage_specifier``. The + default ``provider: auto`` falls back to the main model, so this fires only + when the slot isn't explicit AND no main model is visible. Requires config + context ({} keeps it silent).""" if _task_field(task, "status") != "triage": return [] @@ -421,9 +363,6 @@ def _rule_triage_aux_unavailable(task, events, runs, now, cfg) -> list[Diagnosti "`hermes kanban specify`, which uses auxiliary.triage_specifier." ) - # The primary slot is usable when either: it was explicitly configured by - # the user, OR the default `provider: auto` can fall back to the main - # model. If both fail, we have a real configuration gap. if primary_explicit or main_visible: return [] @@ -482,12 +421,8 @@ def _rule_triage_aux_unavailable(task, events, runs, now, cfg) -> list[Diagnosti def _rule_prose_phantom_refs(task, events, runs, now, cfg) -> list[Diagnostic]: - """Advisory prose-scan: the completion summary mentions ``t_`` - ids that don't resolve. Non-blocking; surfaced as a warning only. - - Auto-clears when a fresh clean completion arrives AFTER the - suspected event. - """ + """Advisory: the completion summary mentions ``t_`` ids that don't + resolve. Warning only; clears on a later clean completion.""" hits = _active_hallucination_events(events, "suspected_hallucinated_references") if not hits: return [] @@ -516,32 +451,14 @@ def _rule_prose_phantom_refs(task, events, runs, now, cfg) -> list[Diagnostic]: def _rule_repeated_failures(task, events, runs, now, cfg) -> list[Diagnostic]: - """Task's unified ``consecutive_failures`` counter is climbing — - something about this task+profile combo is broken and each retry - fails the same way. Triggers regardless of the specific failure - mode (spawn error, timeout, crash) because operationally they - all look the same: the kernel keeps retrying and the operator - needs to intervene. + """``consecutive_failures`` >= cfg["failure_threshold"] (legacy key + ``spawn_failure_threshold``), regardless of failure mode — the kernel keeps + retrying and the operator must intervene. Runtime callers derive the + threshold from ``kanban.failure_limit`` so it doesn't lag the breaker. - Threshold: cfg["failure_threshold"]. Runtime callers should derive - this from ``kanban.failure_limit`` unless the user explicitly set a - diagnostics threshold, so the signal does not lag behind the - dispatcher's circuit breaker. - - Accepts the legacy ``spawn_failure_threshold`` config key for - back-compat. - - Terminal statuses are exempt: a done/archived card has nothing left - to retry, so a lingering failure streak is history, not a signal. - (``complete_task`` resets the counter, but a manual done — e.g. a - dashboard drag — ends no run and used to leave the flag stuck.) - - A fresh attempt in flight (``running``) is also exempt: retrying a - task should clear the stale failure banner until this attempt also - resolves. Otherwise a card that's actively trying again still shows - "failed Nx", which reads as a current failure. It re-fires if the new - run fails too (status leaves ``running`` with a recorded outcome). - """ + Exempt: done/archived (a manual done ends no run, so the streak is history) + and running (a retry in flight must not read as a current failure; re-fires + if it fails too).""" if _task_field(task, "status") in ("done", "archived", "running"): return [] threshold = _positive_int(cfg.get( @@ -549,26 +466,13 @@ def _rule_repeated_failures(task, events, runs, now, cfg) -> list[Diagnostic]: cfg.get("spawn_failure_threshold", 3), ), 3) failure_limit = _positive_int(cfg.get("failure_limit"), threshold) - # Read the new unified counter name, with a fallback to the legacy - # column name so this rule keeps working against old DB rows the - # caller somehow materialised without running the migration. - failures = ( - _task_field(task, "consecutive_failures", None) - if _task_field(task, "consecutive_failures", None) is not None - else _task_field(task, "spawn_failures", 0) - ) + failures = _first_field(task, "consecutive_failures", "spawn_failures", 0) if failures is None or failures < threshold: return [] - last_err = ( - _task_field(task, "last_failure_error", None) - if _task_field(task, "last_failure_error", None) is not None - else _task_field(task, "last_spawn_error", None) - ) + last_err = _first_field(task, "last_failure_error", "last_spawn_error") assignee = _task_field(task, "assignee") - # Classify the most recent failure by peeking at run outcomes so - # the title + suggested action can be specific without a separate - # per-outcome rule. + # Most recent failure outcome makes the title/action specific. ordered_runs = sorted(runs, key=lambda r: _task_field(r, "id", 0)) most_recent_outcome = None for r in reversed(ordered_runs): @@ -596,19 +500,13 @@ def _rule_repeated_failures(task, events, runs, now, cfg) -> list[Diagnostic]: # to diagnose; reclaim/reassign are the recovery levers. task_id = _task_field(task, "id") if task_id: - actions.append(DiagnosticAction( - kind="cli_hint", - label=f"Check logs: hermes kanban log {task_id}", - payload={"command": f"hermes kanban log {task_id}"}, - suggested=True, - )) + actions.append(_log_hint_action(task_id)) actions.extend(_generic_recovery_actions( task, running=_task_field(task, "status") == "running", )) severity = "critical" if failures >= threshold * 2 else "error" - err_text = (last_err or "").strip() if last_err else "" - err_snippet = err_text[:500] + ("…" if len(err_text) > 500 else "") if err_text else "" + err_snippet = _error_snippet(last_err) outcome_label = { "spawn_failed": "spawn", "timed_out": "timeout", @@ -651,29 +549,14 @@ def _rule_repeated_failures(task, events, runs, now, cfg) -> list[Diagnostic]: def _rule_repeated_crashes(task, events, runs, now, cfg) -> list[Diagnostic]: - """The worker spawns fine but keeps crashing mid-run. Check the last - N runs' outcomes; N consecutive ``crashed`` without a successful - ``completed`` means something about the task + profile combo is - broken (OOM, missing dependency, tool it needs is down). + """Trailing run outcomes show >= cfg["crash_threshold"] (default 2) + consecutive ``crashed`` with no ``completed``/``reclaimed`` between. Fires + earlier than ``repeated_failures`` for a crash-specific heads-up and + suppresses itself when the unified rule is about to fire. - Threshold: cfg["crash_threshold"] (default 2). - - Narrower than ``repeated_failures`` — fires earlier (2 crashes vs 3 - total failures) so the operator gets a crash-specific heads-up - before the unified rule kicks in. Suppresses itself when the - unified rule is also about to fire, to avoid double-flagging. - - Terminal statuses are exempt for the same reason as - ``repeated_failures`` — with one extra wrinkle: this rule reads run - history, and a manual done (dashboard drag) appends no ``completed`` - run to break the crash streak, so the flag was permanent (#kanban - desktop dogfood). Done means done. - - ``running`` is exempt too: a fresh attempt is in flight, and its - in-flight run (no outcome yet) doesn't break the trailing crash scan, - so a retried card kept showing "crashed Nx" over an active run. The - banner re-fires if the new attempt also crashes. - """ + Exempt: done/archived (a manual done appends no completed run, so the + streak would be permanent) and running (an in-flight run has no outcome + and wouldn't break the scan).""" if _task_field(task, "status") in ("done", "archived", "running"): return [] failure_threshold = int(cfg.get( @@ -702,29 +585,19 @@ def _rule_repeated_crashes(task, events, runs, now, cfg) -> list[Diagnostic]: # A success (or manual reclaim) breaks the streak. break else: - # Other outcomes (timed_out, blocked, spawn_failed, gave_up) - # aren't crash signals — don't count them, but they also - # don't break the crash streak. + # Other outcomes neither count as crashes nor break the streak. continue if consecutive < threshold: return [] task_id = _task_field(task, "id") actions: list[DiagnosticAction] = [] if task_id: - actions.append(DiagnosticAction( - kind="cli_hint", - label=f"Check logs: hermes kanban log {task_id}", - payload={"command": f"hermes kanban log {task_id}"}, - suggested=True, - )) + actions.append(_log_hint_action(task_id)) running = _task_field(task, "status") == "running" actions.extend(_generic_recovery_actions(task, running=running)) severity = "critical" if consecutive >= threshold * 2 else "error" - # Put the actual error up-front so operators see WHAT broke without - # having to open the logs. Truncate defensively — these can be huge - # (full tracebacks). - err_text = (last_err or "").strip() if last_err else "" - err_snippet = err_text[:500] + ("…" if len(err_text) > 500 else "") if err_text else "" + # Error up-front so operators see WHAT broke without opening the logs. + err_snippet = _error_snippet(last_err) if err_snippet: title = f"Agent crashed {consecutive}x: {err_snippet.splitlines()[0][:160]}" detail = ( @@ -751,14 +624,9 @@ def _rule_repeated_crashes(task, events, runs, now, cfg) -> list[Diagnostic]: def _rule_review_dependency_deadlock(task, events, runs, now, cfg) -> list[Diagnostic]: - """Detect a legacy review handoff that starves downstream children. - - Older workers were instructed to sticky-block an implementation with a - ``review-required:`` reason. A separately modelled reviewer child cannot - promote until that parent is terminal, so the lane has no autonomous next - step. This compatibility diagnostic is graph-aware but deliberately leaves - both the dependency graph and the user's sticky block unchanged. - """ + """Legacy review handoff starving children: the implementation is + sticky-blocked with a ``review-required:`` reason while todo children wait + for it to be terminal. Graph-aware; deliberately mutates nothing.""" if _task_field(task, "status") != "blocked": return [] @@ -829,21 +697,13 @@ def _rule_review_dependency_deadlock(task, events, runs, now, cfg) -> list[Diagn def _rule_stuck_in_blocked(task, events, runs, now, cfg) -> list[Diagnostic]: - """Task has been in ``blocked`` status for too long without a comment. - - Threshold: cfg["blocked_stale_hours"] (default 24). - Surfaced as a warning so humans know there's a pending unblock. - """ + """Blocked for >= cfg["blocked_stale_hours"] (default 24) with no comment + or unblock since the last ``blocked`` event.""" hours = float(cfg.get("blocked_stale_hours", 24)) status = _task_field(task, "status") if status != "blocked": return [] - # Find the most recent ``blocked`` event. - last_blocked_ts = 0 - for ev in events: - if _event_kind(ev) == "blocked": - t = _event_ts(ev) - last_blocked_ts = max(last_blocked_ts, t) + last_blocked_ts = _latest_event_ts(events, {"blocked"}) if last_blocked_ts == 0: return [] age_hours = (now - last_blocked_ts) / 3600.0 @@ -879,29 +739,17 @@ def _rule_stuck_in_blocked(task, events, runs, now, cfg) -> list[Diagnostic]: def _rule_block_unblock_cycling(task, events, runs, now, cfg) -> list[Diagnostic]: - """Task has cycled through blocked → unblocked many times — the - ``unblock`` is not fixing the underlying problem and the worker - keeps re-blocking for substantially the same reason. - - ``_rule_stuck_in_blocked`` resets its timer on any ``commented`` / - ``unblocked`` event, so a task that cycles every few minutes is - invisible to it regardless of how many times it cycles (#29747 - gap 1). This rule complements that one by counting block→unblock - cycles in a sliding window. - - Threshold: cfg["block_cycle_threshold"] (default 3) cycles within - cfg["block_cycle_window_seconds"] (default 24h). - """ + """>= cfg["block_cycle_threshold"] (default 3) blocked-after-unblocked + cycles within cfg["block_cycle_window_seconds"] (default 24h). Complements + ``_rule_stuck_in_blocked``, whose timer any unblock resets, so fast cyclers + are invisible to it.""" threshold = _positive_int(cfg.get("block_cycle_threshold"), 3) window_seconds = float(cfg.get("block_cycle_window_seconds", 24 * 3600)) cycle_cutoff = now - window_seconds - # Walk events chronologically (arrival order — callers pre-sort by - # id, which is the canonical chronological order; ``created_at`` - # alone is insufficient because multiple events can share the same - # second). Count "blocked after unblocked" transitions: every time - # a blocked event follows at least one unblocked event since the - # last cycle was counted, that's a new cycle. + # Walk in id (arrival) order — created_at alone can't order events that + # share a second. A blocked event after >= 1 unblocked since the last + # counted cycle is a new cycle. cycles = 0 seen_unblock_since_last_cycle = False initial_blocked_ts = 0 @@ -956,66 +804,31 @@ def _rule_block_unblock_cycling(task, events, runs, now, cfg) -> list[Diagnostic def _rule_stranded_in_ready(task, events, runs, now, cfg) -> list[Diagnostic]: - """Task has been in ``ready`` status for too long without any worker - claiming it. - - Threshold: cfg["stranded_threshold_seconds"] (default 1800 = 30 min). - - Catches every "task waiting for a worker that never comes" case - without caring WHY: - - * Operator typo'd the assignee — no profile or external worker matches. - * Profile was deleted, leaving its tasks stranded. - * External worker pool (Codex CLI, Claude Code lane, custom daemon) - is down, hung, or wasn't started. - * Dispatcher is misconfigured (wrong board, wrong HERMES_HOME). - - Pre-rule, all of these silently rotted in ``skipped_nonspawnable`` — - the dispatcher correctly skipped them (good — no respawn loop) but - nobody surfaced the fact that operator-actionable work was - accumulating. The rule fires when a ready task's promoted-to-ready - timestamp is older than the threshold AND the assignee is non-empty - (truly unassigned tasks have their own ``skipped_unassigned`` signal - on the dispatcher and a different operator response). - - The signal is age-based on purpose: it's identity-agnostic, so it - works for Hermes profiles, registered lanes, external workers, and - typos uniformly. No registry to curate, no per-board allowlist. - """ + """Assigned, unclaimed, ``ready`` for >= cfg["stranded_threshold_seconds"] + (default 30 min). Deliberately age-based and identity-agnostic so it + catches typo'd assignees, deleted profiles, and down external worker + pools alike without a registry to curate. Unassigned tasks are excluded — + the dispatcher's ``skipped_unassigned`` already covers them.""" threshold_seconds = float( cfg.get("stranded_threshold_seconds", 30 * 60) ) status = _task_field(task, "status") if status != "ready": return [] - # Skip tasks with a live claim — they're being worked on, even if - # the worker hasn't reported progress yet (run-level liveness - # extends the claim TTL; we don't want to second-guess that here). + # A live claim means it's being worked on even without progress yet. if _task_field(task, "claim_lock"): return [] assignee = _task_field(task, "assignee") or "" if not assignee.strip(): - # Unassigned tasks: the dispatcher's ``skipped_unassigned`` is - # already the right signal. A separate diagnostic here would - # double-flag the same condition. return [] - # Find the most recent event that put this task into ready. - # ``created`` covers tasks born ready; ``promoted`` covers parent- - # done auto-promotion; ``reclaimed`` covers TTL/crash recovery; - # ``unblocked`` covers human-driven resumes. - READY_TRANSITION_KINDS = { - "created", "promoted", "reclaimed", "unblocked", - } - last_ready_ts = 0 - for ev in events: - if _event_kind(ev) in READY_TRANSITION_KINDS: - t = _event_ts(ev) - last_ready_ts = max(last_ready_ts, t) + # Most recent event that put the task into ready. + last_ready_ts = _latest_event_ts( + events, {"created", "promoted", "reclaimed", "unblocked"}, + ) - # Fallback: if no qualifying event exists (very old task or events - # truncated), fall back to ``created_at`` on the task row. Better - # to occasionally over-flag an ancient task than miss a stranded one. + # No qualifying event (old task / truncated events): fall back to + # created_at — over-flagging an ancient task beats missing a stranded one. if last_ready_ts == 0: last_ready_ts = int(_task_field(task, "created_at", default=0) or 0) if last_ready_ts == 0: @@ -1031,9 +844,7 @@ def _rule_stranded_in_ready(task, events, runs, now, cfg) -> list[Diagnostic]: else: age_str = f"{int(age_seconds / 60)}m" - # Severity escalates with age. Below 2x threshold = warning; - # 2x – 6x = error; beyond 6x = critical (something is clearly - # broken, not just slow). + # Escalate with age: <2x threshold warning, 2x-6x error, >6x critical. if age_seconds >= threshold_seconds * 6: severity = "critical" elif age_seconds >= threshold_seconds * 2: @@ -1078,8 +889,7 @@ def _rule_stranded_in_ready(task, events, runs, now, cfg) -> list[Diagnostic]: )] -# Registry — order matters: rules higher on the list render first when -# severity ties. Add new rules here. +# Order matters: earlier rules render first on severity ties. _RULES: list[RuleFn] = [ _rule_hallucinated_cards, _rule_triage_aux_unavailable, @@ -1093,21 +903,6 @@ _RULES: list[RuleFn] = [ ] -# Known kinds (for the UI's filter / legend / i18n keys). Update when -# rules are added. -DIAGNOSTIC_KINDS = ( - "hallucinated_cards", - "triage_aux_unavailable", - "prose_phantom_refs", - "repeated_failures", - "repeated_crashes", - "review_dependency_deadlock", - "stuck_in_blocked", - "block_unblock_cycling", - "stranded_in_ready", -) - - DEFAULT_CONFIG = { # Match the dispatcher default (kanban.failure_limit) so repeated-failure # diagnostics do not lag behind the default auto-block threshold. @@ -1116,21 +911,16 @@ DEFAULT_CONFIG = { "spawn_failure_threshold": 2, "crash_threshold": 2, "blocked_stale_hours": 24, - # Stranded-task threshold. 30 min by default — below that, the - # signal is dominated by tasks that are about to be claimed on the - # next dispatcher tick (default 60s) and would just be noise. + # Below 30 min the signal is dominated by tasks about to be claimed on + # the next dispatcher tick. "stranded_threshold_seconds": 30 * 60, } def config_from_kanban_config(kanban_cfg: Optional[dict]) -> dict: - """Build diagnostics config from the runtime ``kanban`` config section. - - ``kanban.diagnostics.failure_threshold`` remains an explicit override. - Otherwise, derive the repeated-failure threshold from - ``kanban.failure_limit`` so CLI/dashboard diagnostics match the - dispatcher's actual circuit-breaker threshold. - """ + """Diagnostics config from the ``kanban`` section. ``kanban.diagnostics. + failure_threshold`` is an explicit override; otherwise the threshold is + ``kanban.failure_limit`` so diagnostics match the dispatcher's breaker.""" kanban_cfg = kanban_cfg or {} diag_cfg = dict(kanban_cfg.get("diagnostics") or {}) diag_cfg.setdefault( @@ -1146,13 +936,9 @@ def config_from_kanban_config(kanban_cfg: Optional[dict]) -> dict: def config_from_runtime_config(raw_config: Optional[dict]) -> dict: - """Build diagnostics config from the full Hermes runtime config. - - Carries through ``kanban``, ``auxiliary``, and ``model`` keys so triage- - aware rules can inspect the active aux-helper and main-model state. - Folds the ``kanban`` block through ``config_from_kanban_config`` so the - repeated-failure threshold derivation still applies. - """ + """Diagnostics config from the full runtime config: folds ``kanban`` through + ``config_from_kanban_config`` and carries ``kanban``/``auxiliary``/``model`` + through for the triage-aware rules.""" raw_config = raw_config or {} if not isinstance(raw_config, dict): return {} @@ -1177,12 +963,8 @@ def compute_task_diagnostics( config: Optional[dict] = None, graph: Optional[dict] = None, ) -> list[Diagnostic]: - """Run every rule against a single task's state and return a - severity-sorted list of active diagnostics. - - Sorting: critical first, then error, then warning; ties broken by - most-recent ``last_seen_at``. - """ + """Run every rule for one task; critical first, then error, warning; ties + broken by most-recent ``last_seen_at``.""" now_ts = int(now if now is not None else time.time()) config = config or {} cfg = {**DEFAULT_CONFIG, **config} @@ -1202,9 +984,7 @@ def compute_task_diagnostics( try: out.extend(rule(task, events, runs, now_ts, cfg)) except Exception: - # A broken rule must never crash the dashboard. Rule bugs - # get caught in tests; in production we'd rather drop the - # diagnostic than 500 a whole /board request. + # A broken rule must never 500 a whole /board request. continue severity_idx = {s: i for i, s in enumerate(SEVERITY_ORDER)} out.sort( diff --git a/hermes_cli/kanban_specify.py b/hermes_cli/kanban_specify.py index e7aba34a37..24605d3fb5 100644 --- a/hermes_cli/kanban_specify.py +++ b/hermes_cli/kanban_specify.py @@ -1,32 +1,13 @@ """Kanban triage specifier — flesh out a one-liner into a real spec. -Used by ``hermes kanban specify [task_id | --all]``. Takes a task that -lives in the Triage column (a rough idea, typically only a title), calls -the auxiliary LLM to produce: +``hermes kanban specify [task_id | --all]`` asks the auxiliary LLM for a +tightened title + concrete body for a Triage task, then flips it +``triage -> todo`` via ``kanban_db.specify_triage_task``. - * A tightened title (optional — only replaces if the model proposes a - materially different one) - * A concrete body: goal, proposed approach, acceptance criteria - -and then flips the task ``triage -> todo`` via -``kanban_db.specify_triage_task``. The dispatcher promotes it to -``ready`` on its next tick (or immediately if there are no open parents). - -Design notes ------------- - -* This module intentionally mirrors ``hermes_cli/goals.py`` — same aux - client pattern, same "empty config => skip, don't crash" tolerance. - Keeps the surface area tiny and the failure modes predictable. - -* The prompt is a short system + user pair. We ask for JSON with - ``{title, body}``; if parsing fails, we fall back to treating the - whole response as the body and leave the title untouched. No - retry loop — one shot, keep cost bounded. - -* Structured output / JSON mode is not requested explicitly so the - specifier works on providers that don't implement it. The parse - is lenient (tolerates markdown code fences around the JSON). +Mirrors ``hermes_cli/goals.py``: same aux-client pattern, same "empty config +=> skip, don't crash" tolerance. One shot, no retry loop. JSON mode is not +requested (works on providers without it); the parse is lenient and falls +back to "whole reply is the body" so a malformed reply never strands a task. """ from __future__ import annotations @@ -108,13 +89,12 @@ def _truncate(text: str, limit: int) -> str: _FENCE_RE = re.compile(r"^\s*```(?:json)?\s*|\s*```\s*$", re.IGNORECASE) -def _extract_json_blob(raw: str) -> Optional[dict]: - """Lenient JSON extraction — tolerates fenced code blocks and - leading/trailing whitespace. Returns None if nothing parses.""" +def _extract_json_blob(raw: str, fence_re: re.Pattern = _FENCE_RE) -> Optional[dict]: + """Lenient JSON object extraction: strip code fences, take the first ``{`` + to the last ``}``. None if nothing parses to a dict.""" if not raw: return None - stripped = _FENCE_RE.sub("", raw.strip()) - # Greedy: find the first `{` and last `}` and try that slice. + stripped = fence_re.sub("", raw.strip()) first = stripped.find("{") last = stripped.rfind("}") if first == -1 or last == -1 or last <= first: @@ -124,19 +104,24 @@ def _extract_json_blob(raw: str) -> Optional[dict]: val = json.loads(candidate) except (ValueError, json.JSONDecodeError): return None - if not isinstance(val, dict): - return None - return val + return val if isinstance(val, dict) else None -def _profile_author() -> str: +def _nonblank(v) -> Optional[str]: + return v if isinstance(v, str) and v.strip() else None + + +def _title_body(parsed: dict) -> tuple[Optional[str], Optional[str]]: + """``(title, body)`` from an LLM reply: title stripped, body verbatim, + either None when missing/blank.""" + title = _nonblank(parsed.get("title")) + return (title.strip() if title else None), _nonblank(parsed.get("body")) + + +def _profile_author(default: str = "specifier") -> str: """Mirror of ``hermes_cli.kanban._profile_author``. Kept local to avoid a circular import when kanban.py imports this module.""" - return ( - os.environ.get("HERMES_PROFILE") - or os.environ.get("USER") - or "specifier" - ) + return os.environ.get("HERMES_PROFILE") or os.environ.get("USER") or default def specify_task( @@ -145,13 +130,9 @@ def specify_task( author: Optional[str] = None, timeout: Optional[int] = None, ) -> SpecifyOutcome: - """Specify a single triage task and promote it to ``todo``. - - Returns an outcome describing what happened. Never raises for expected - failure modes (task not in triage, no aux client configured, API - error, malformed response) — those surface via ``ok=False`` so the - ``--all`` sweep can continue past individual failures. - """ + """Specify one triage task and promote it to ``todo``. Expected failures + (not in triage, no aux client, API error, malformed reply) surface as + ``ok=False`` so an ``--all`` sweep continues.""" with kb.connect_closing() as conn: task = kb.get_task(conn, task_id) if task is None: @@ -174,9 +155,8 @@ def specify_task( ) try: - # Route through call_llm so auxiliary.triage_specifier.* config - # (provider/model/base_url, extra_body, reasoning_effort, retries) - # all apply — the direct-create path dropped extra_body (#35566). + # call_llm applies all auxiliary.triage_specifier.* config + # (provider/model/base_url, extra_body, reasoning_effort, retries). resp = call_llm( task="triage_specifier", messages=[ @@ -206,9 +186,7 @@ def specify_task( new_title: Optional[str] new_body: Optional[str] if parsed is None: - # Fall back: treat the whole reply as the body, leave title as-is. - # Worst case the user edits afterward — still better than stranding - # the task in triage on a malformed LLM reply. + # Whole reply becomes the body; the user can edit afterward. stripped_raw = raw.strip() if not stripped_raw: return SpecifyOutcome( @@ -217,16 +195,7 @@ def specify_task( new_title = None new_body = stripped_raw else: - title_val = parsed.get("title") - body_val = parsed.get("body") - new_title = ( - title_val.strip() - if isinstance(title_val, str) and title_val.strip() - else None - ) - new_body = ( - body_val if isinstance(body_val, str) and body_val.strip() else None - ) + new_title, new_body = _title_body(parsed) if new_body is None and new_title is None: return SpecifyOutcome( task_id, False, "LLM response missing title and body" @@ -241,8 +210,7 @@ def specify_task( author=author or _profile_author(), ) if not ok: - # Race: someone else promoted / archived the task between our - # read above and the write. Report, don't crash. + # Race: promoted/archived between our read and the write. return SpecifyOutcome( task_id, False, "task moved out of triage before promotion" ) @@ -250,10 +218,7 @@ def specify_task( def list_triage_ids(*, tenant: Optional[str] = None) -> list[str]: - """Return task ids currently in the triage column. - - ``tenant`` narrows the sweep; ``None`` returns every triage task. - """ + """Task ids in the triage column; ``tenant`` narrows the sweep.""" with kb.connect_closing() as conn: tasks = kb.list_tasks( conn, diff --git a/hermes_cli/kanban_swarm.py b/hermes_cli/kanban_swarm.py index ef1477f675..5a047eb67d 100644 --- a/hermes_cli/kanban_swarm.py +++ b/hermes_cli/kanban_swarm.py @@ -19,6 +19,7 @@ from __future__ import annotations from dataclasses import dataclass, field import json import sqlite3 +import time from typing import Any, Iterable, Optional from hermes_cli import kanban_db as kb @@ -83,16 +84,12 @@ def _activate_root_inline( ) -> bool: """Inline blocked→done CAS flip + event insert for the swarm root. - Runs INSIDE create_swarm's outer write_txn, so it must not call - ``kb.complete_task`` — that helper opens its own transaction and fires - post-commit side effects (workspace cleanup, failure-counter clear, - ``recompute_ready``) that would execute while the outer transaction can - still roll back. Instead we do the minimal durable writes here and let - the caller run ``recompute_ready`` after the outer commit. + Runs INSIDE create_swarm's write_txn, so it must not call + ``kb.complete_task`` (own transaction + post-commit side effects that + would run while the outer txn can still roll back). The caller runs + ``recompute_ready`` after the outer commit. """ - import time as _time - - now = int(_time.time()) + now = int(time.time()) cur = conn.execute( """ UPDATE tasks @@ -179,9 +176,8 @@ def create_swarm( raise RuntimeError("could not activate the completed swarm topology") activated = True if activated: - # Outside the outer transaction: promote the root's children now - # that its 'done' flip is durable (recompute_ready opens its own - # txn and must never run under an open write_txn). + # After commit: recompute_ready opens its own txn and must never run + # under an open write_txn. kb.recompute_ready(conn) root = kb.get_task(conn, created.root_id) run = kb.latest_run(conn, created.root_id) @@ -249,9 +245,8 @@ def _create_swarm_uncommitted( workspace_path=workspace_path, ) - # If idempotency returned an existing non-archived root, do not duplicate the - # swarm graph. Recover the topology from the root's latest blackboard, if it - # was created by this helper previously. + # Idempotency may return an existing root: recover its topology from the + # blackboard instead of duplicating the graph. existing = latest_blackboard(conn, root).get("topology") if isinstance(existing, dict): worker_ids = [str(x) for x in existing.get("worker_ids", []) if x] @@ -266,61 +261,54 @@ def _create_swarm_uncommitted( ) context_suffix = _swarm_context(root, goal) - worker_ids: list[str] = [] - for spec in worker_specs: - worker_id = kb.create_task( + common = dict( + created_by=created_by, tenant=tenant, + workspace_kind=workspace_kind, workspace_path=workspace_path, + ) + worker_ids = [ + kb.create_task( conn, title=spec.title, body=(spec.body or "") + context_suffix, assignee=spec.profile, - created_by=created_by, parents=[root], - tenant=tenant, priority=spec.priority or priority, - workspace_kind=workspace_kind, - workspace_path=workspace_path, skills=spec.skills or None, max_runtime_seconds=spec.max_runtime_seconds, + **common, ) - worker_ids.append(worker_id) + for spec in worker_specs + ] - verifier_body = ( - "Review every worker handoff and blackboard update. Gate the swarm: " - "complete only with metadata {\"gate\": \"pass\"} when evidence is " - "sufficient; otherwise block with exact missing work." - + context_suffix - ) verifier = kb.create_task( conn, title=verifier_title, - body=verifier_body, + body=( + "Review every worker handoff and blackboard update. Gate the swarm: " + "complete only with metadata {\"gate\": \"pass\"} when evidence is " + "sufficient; otherwise block with exact missing work." + + context_suffix + ), assignee=verifier_assignee, - created_by=created_by, parents=worker_ids, - tenant=tenant, priority=priority, - workspace_kind=workspace_kind, - workspace_path=workspace_path, skills=["requesting-code-review"], + **common, ) - synthesizer_body = ( - "Synthesize the verified worker outputs into the final deliverable. " - "Do not start until the verifier has passed the gate." - + context_suffix - ) synthesizer = kb.create_task( conn, title=synthesizer_title, - body=synthesizer_body, + body=( + "Synthesize the verified worker outputs into the final deliverable. " + "Do not start until the verifier has passed the gate." + + context_suffix + ), assignee=synthesizer_assignee, - created_by=created_by, parents=[verifier], - tenant=tenant, priority=priority, - workspace_kind=workspace_kind, - workspace_path=workspace_path, skills=["humanizer"], + **common, ) created = SwarmCreated(root, worker_ids, verifier, synthesizer) diff --git a/hermes_cli/kanban_transfer.py b/hermes_cli/kanban_transfer.py index a5bf3aaab4..41b1a0daeb 100644 --- a/hermes_cli/kanban_transfer.py +++ b/hermes_cli/kanban_transfer.py @@ -14,30 +14,21 @@ source board's slug):: attachments//… attachment blobs (unless --no-attachments) logs/.log worker logs (only with --include-logs) -Two things make this more than a ``tar czf`` of the board directory. +Two things make this more than ``tar czf`` of the board directory: -**The database is live.** Kanban runs in WAL mode and a dispatcher may be -mid-write, so copying ``kanban.db`` off the filesystem yields a torn -snapshot that is missing whatever still sits in the ``-wal`` file. Export -goes through SQLite's online-backup API instead, which produces a -consistent single-file image of a database that is being written to. +* **The database is live** (WAL mode, dispatcher may be mid-write), so the + export uses SQLite's online-backup API for a consistent image instead of a + file copy that would miss the ``-wal`` sidecar. +* **Rows carry machine-local state** — claims, PIDs, heartbeats, absolute + paths, gateway chat subscriptions, session ids. Shipping them verbatim + would import claims owned by a stranger's process or push events into a + stranger's Telegram thread. Scrubbed on export and re-scrubbed on import + (an archive is untrusted); see :func:`_scrub_local_state` and + :func:`_relocate_imported_rows`. -**Rows carry machine-local state.** Claims, PIDs, heartbeats, absolute -workspace and attachment paths, gateway chat subscriptions, and session -ids are all meaningful only on the machine that wrote them. Shipping them -verbatim is how an imported board arrives holding claims owned by a -process on somebody else's laptop, or starts pushing task events into a -stranger's Telegram thread. Everything machine-local is scrubbed on the -export side (so the archive itself never carries it) and defensively -re-scrubbed on import; see :func:`_scrub_local_state` and -:func:`_relocate_imported_rows`. - -Imports always land as a **new** board — the slug auto-suffixes on -collision — so an import can never mutate a board that is already there. -That also means an imported board is never ``default``, which is what -lets the import side ignore the default board's split on-disk layout -(``/kanban.db`` beside ``/kanban/attachments/``) and put -everything inside one ``boards//`` directory. +Imports always land as a **new** board (slug auto-suffixes on collision), so +an import never mutates an existing board and is never ``default`` — which +lets the importer ignore the default board's split on-disk layout. """ from __future__ import annotations @@ -74,13 +65,8 @@ _DISPATCHABLE_STATUSES = ("ready", "running", "todo", "scheduled") # --------------------------------------------------------------------------- def _snapshot_db(source: Path, target: Path) -> None: - """Write a consistent copy of ``source`` to ``target``. - - Uses SQLite's online-backup API rather than a file copy: in WAL mode - a just-committed page can still live in the ``-wal`` sidecar, so - copying only ``kanban.db`` loses recent writes and can produce a - torn image if the dispatcher commits mid-copy. - """ + """Consistent copy of ``source`` via the online-backup API (a file copy + would miss pages still in the ``-wal`` sidecar and could tear).""" src = sqlite3.connect(str(source)) try: dst = sqlite3.connect(str(target)) @@ -93,14 +79,9 @@ def _snapshot_db(source: Path, target: Path) -> None: def _scrub_local_state(conn: sqlite3.Connection) -> None: - """Strip machine-local runtime state. Caller owns the transaction. - - Runs on the export side so the archive itself never carries another - machine's claims, PIDs, or — the one that actually matters for a - board shared with someone else — the gateway chat ids subscribed to - its task events. Repeated on import because an archive is untrusted - input. - """ + """Strip machine-local runtime state (claims, PIDs, and above all the + gateway chat ids subscribed to task events). Caller owns the transaction. + Run on export and again on import (an archive is untrusted input).""" conn.execute("DELETE FROM kanban_notify_subs") conn.execute( """ @@ -157,12 +138,9 @@ def export_board( include_attachments: bool = True, include_logs: bool = False, ) -> dict[str, Any]: - """Export ``board`` to a ``tar.gz`` archive. Returns a summary dict. - - ``output_path`` may be given with or without the ``.tar.gz`` suffix. - Workspaces are never included: they are git worktrees and scratch - trees that are large, machine-local, and rebuilt on demand. - """ + """Export ``board`` to a ``tar.gz`` (suffix optional on ``output_path``); + returns a summary dict. Workspaces are never included — large, + machine-local, rebuilt on demand.""" slug = kb._normalize_board_slug(board) or kb.get_current_board() if not kb.board_exists(slug): raise ValueError(f"board {slug!r} does not exist") @@ -241,12 +219,8 @@ def export_board( # --------------------------------------------------------------------------- def _available_slug(preferred: str) -> str: - """Return ``preferred``, or the first free ``-N`` variant. - - ``default`` always reports as existing, so an archive exported from a - default board naturally lands as ``default-2`` instead of colliding - with the importer's own default board. - """ + """``preferred`` or the first free ``-N``. ``default`` always + exists, so a default-board export lands as ``default-2``.""" if not kb.board_exists(preferred): return preferred # Leave headroom for the suffix inside the 64-char slug limit. @@ -295,22 +269,17 @@ def _read_board_metadata(path: Path) -> dict[str, Any]: def _relocate_imported_rows( conn: sqlite3.Connection, slug: str ) -> tuple[dict[str, int], list[str]]: - """Re-anchor an imported board's rows to this machine. + """Re-anchor an imported board's rows to this machine; returns + ``(stats, warnings)``. - Returns ``(stats, warnings)``. Three things move: - - * Attachment rows are repointed at this board's attachments tree. - Rows whose blob did not travel (an export made with - ``--no-attachments``) are dropped, because a row pointing at a file - that does not exist breaks download in every UI that lists it. - * Workspace paths are cleared. ``scratch`` tasks regenerate one under - this board on the next claim, so they are simply reset. ``dir`` and - ``worktree`` tasks cannot be resolved without a path that means - something here, so any that are still dispatchable are parked in - ``triage`` — otherwise the dispatcher claims them, fails to build a - workspace, and burns them straight into the failure breaker. - * Runtime state is scrubbed again. Export already did this, but an - archive is an untrusted input and the cost is one UPDATE. + * Attachment rows are repointed at this board's tree; rows whose blob + did not travel (``--no-attachments``) are dropped, since a dangling row + breaks download in every UI. + * Workspace paths are cleared. ``scratch`` regenerates on next claim; + dispatchable ``dir``/``worktree`` tasks are parked in ``triage``, + otherwise the dispatcher claims them, fails to build a workspace, and + burns them into the failure breaker. + * Runtime state is scrubbed again (untrusted input, one UPDATE). """ warnings: list[str] = [] now = int(time.time()) @@ -389,12 +358,8 @@ def import_board( *, activate: bool = False, ) -> dict[str, Any]: - """Import a board archive as a new board. Returns a summary dict. - - ``slug`` overrides the name from the archive. Either way the final - slug auto-suffixes if it is taken, so an import never merges into or - overwrites an existing board. - """ + """Import an archive as a NEW board (``slug`` overrides the archive's; + either way it auto-suffixes if taken). Returns a summary dict.""" archive = Path(archive_path).expanduser() if not archive.exists(): raise FileNotFoundError(f"archive not found: {archive}") diff --git a/hermes_state.py b/hermes_state.py index e374fea4e0..d43979fea2 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -16,7 +16,6 @@ Key design decisions: import asyncio import atexit -import contextlib import errno import hashlib import json @@ -26,7 +25,6 @@ import queue import random import re import sqlite3 -import struct import sys import threading import time @@ -36,7 +34,6 @@ from collections import deque from contextlib import contextmanager from pathlib import Path -from agent.memory_manager import sanitize_context from agent.session_activity import ActivityProvenance from agent.message_sanitization import _sanitize_surrogates # Intrinsic persistence marker stamped on message dicts that are known-durable @@ -45,22 +42,14 @@ from agent.message_sanitization import _sanitize_surrogates # transitive dependency via hermes_state_common). run_agent keeps its own # predating copy — hermes_state cannot import run_agent (circular) — guarded # by test_marker_constant_in_sync. -from agent.context_compressor import ( +from agent.context_compressor import ( # noqa: F401 (re-exported; tests import it from here) _DB_PERSISTED_MARKER as _DB_PERSISTED_MARKER_KEY, ) -from agent.skill_commands import ( - SKILL_EXCERPT_JOINT, - SKILL_SCAFFOLD_SQL_LIKE, - describe_skill_invocation, -) from hermes_constants import get_hermes_home -from hermes_startup_watchdog import report_startup_progress -from hermes_cli.sqlite_runtime import ( - is_sqlite_wal_reset_vulnerable as _is_sqlite_wal_reset_vulnerable, -) -from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple, TypeVar, cast +from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TypeVar, cast from hermes_state_common import ( # noqa: F401 (re-exported for back-compat) + AUTO_VACUUM_MIN_FREELIST_RATIO, _BRANCH_CHILD_SQL, _COMPRESSION_CHILD_SQL, _FTS_CJK_TRIGGERS, @@ -101,9 +90,120 @@ from hermes_state_common import ( # noqa: F401 (re-exported for back-compat) _describe_lock_holder, _read_lock_holder_record, is_advisory_lock_contention, + stat_db_file_identity as _stat_db_file_identity, ) from hermes_state_portability import SessionPortabilityMixin +from hermes_state_telegram import SessionTelegramTopicsMixin, _normalize_telegram_topic_profile_name # noqa: F401 (re-exported for back-compat) from hermes_state_schema import SessionSchemaMixin +from hermes_state_dbfile import ( # noqa: F401 (re-exported; tests patch hermes_state.) + _HEADER_PROBE_FDS, + _HEADER_PROBE_LOCK, + _HERMES_CMDLINE_MARKERS, + _RETIRED_HEADER_PROBE_FDS, + _canonical_sqlite_path, + _concrete_state_db_holder_pids, + _connect_tracked_db, + _is_inactive_orphan_desktop_holder, + _looks_like_hermes, + _pread_db_header, + _read_proc_cmdline, + _read_sqlite_application_id, + _stat_sqlite_sidecar_identity, + _watched_sqlite_sidecar_paths, + collect_state_db_stats, + count_db_holders, + is_zeroed_state_db, + iter_deleted_sqlite_sidecar_holders, + quarantine_cross_process_lock, + quarantine_zeroed_state_db, + refuse_deleted_wal_generation, +) +from hermes_state_messages import SessionMessagesMixin +from hermes_state_wal import ( # noqa: F401 (re-exported; tests patch hermes_state.) + WalUnsupportedError, + _SYNCHRONOUS_FULL, + _SYNCHRONOUS_LEVELS, + _SYNCHRONOUS_NAMES, + _WAL_INCOMPAT_MARKERS, + _WAL_SIZE_LIMIT_BYTES, + _apply_delete_for_wal_reset_bug, + _apply_macos_checkpoint_barrier, + _apply_synchronous_pragma, + _apply_wal_size_limit, + _database_has_content, + _delete_overridden_warned_lock, + _delete_overridden_warned_paths, + _enforce_macos_synchronous_full, + _journal_upgrade_warned_lock, + _journal_upgrade_warned_paths, + _log_configured_delete_overridden_once, + _log_journal_mode_upgrade_once, + _log_wal_fallback_once, + _log_wal_reset_bug_once, + _on_disk_journal_mode, + _set_journal_mode_no_wait, + _wal_fallback_warned_lock, + _wal_fallback_warned_paths, + _wal_reset_bug_warned_lock, + _wal_reset_bug_warned_paths, + _wal_reset_repair_hint, + apply_database_pragmas, + apply_wal_with_fallback, + is_sqlite_wal_reset_vulnerable, + resolve_journal_mode, + resolve_synchronous_level, + sqlite_source_id, +) +from hermes_state_repair import ( # noqa: F401 (re-exported; tests patch hermes_state.) + _DB_SIDECAR_SUFFIXES, + _FINGERPRINT_SAMPLE_BYTES, + _FINGERPRINT_VOLATILE_HEADER_RANGES, + _MAX_MALFORMED_BACKUPS, + _MAX_PERSISTENT_REPAIR_ATTEMPTS, + _REPAIR_BACKUP_FREE_FRACTION, + _REPAIR_BACKUP_MIN_FREE_BYTES, + _REPAIR_LOCK_POLL_SECONDS, + _REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND, + _backup_content_identity, + _backup_db_file, + _bump_schema_cookie, + _claim_repair_attempt, + _connect_repair_durable, + _copy_database_snapshot, + _cross_process_repair_lock, + _db_fingerprint, + _db_opens_cleanly, + _exclusive_repair_db_guard, + _existing_malformed_backups, + _live_writer_holds_db, + _mask_volatile_header, + _persistent_repair_attempts_exhausted, + _persistent_repair_exhausted_error, + _probe_journal_mode_for_repair, + _prune_malformed_backups, + _read_repair_ledger, + _reapply_durability_barriers, + _record_repair_outcome, + _release_auto_maintenance_lock, + _repair_backup_headroom_bytes, + _repair_failure_consumes_attempt, + _repair_ledger_path, + _repair_scratch_space_error, + _repair_snapshot_timeout_seconds, + _repair_state_db_schema_locked, + _restore_journal_mode_after_repair, + _run_repair_strategies, + _try_acquire_auto_maintenance_lock, + _unlink_db_triple, + apply_durability_barriers, + preflight_db_writability, + repair_state_db_schema, +) +from hermes_state_titles import SessionTitlesMixin +from hermes_state_usage import SessionUsageMixin +from hermes_state_maintenance import SessionMaintenanceMixin +from hermes_state_gateway import SessionGatewayMixin +from hermes_state_compression import SessionCompressionMixin from hermes_state_search import SessionSearchMixin try: # Hard dependency, but tolerate scaffold-phase imports before pip install. @@ -116,13 +216,6 @@ logger = logging.getLogger(__name__) MAX_SAFE_RESUME_MESSAGES = 20_000 MAX_SAFE_EXPORT_MESSAGES = 20_000 -# Auto-maintenance only VACUUMs when at least this fraction of the database -# file is reclaimable (``PRAGMA freelist_count / PRAGMA page_count``). Below -# it a full rewrite costs more I/O than it returns — pruning a handful of small -# sessions on a dense multi-GB state.db should never rewrite the whole file to -# reclaim a few MB (#54189). Composes with ``min_vacuum_interval_days``. -AUTO_VACUUM_MIN_FREELIST_RATIO = 0.25 - def _configured_transcript_limit(key: str, fallback: int) -> int: """Resolve a transcript safety limit from config at call time. @@ -943,45 +1036,6 @@ def _ensure_test_isolation(db_path: Path) -> None: "its environment." ) -# --------------------------------------------------------------------------- -# WAL-compatibility fallback -# --------------------------------------------------------------------------- -# SQLite's WAL mode requires shared-memory (mmap) coordination and fcntl -# byte-range locks that don't reliably work on network filesystems (NFS, -# SMB/CIFS, some FUSE mounts, WSL1). Upstream documents this explicitly: -# https://www.sqlite.org/wal.html#sometimes_queries_return_sqlite_busy_in_wal_mode -# -# On those filesystems ``PRAGMA journal_mode=WAL`` raises -# ``sqlite3.OperationalError: locking protocol`` (SQLITE_PROTOCOL). If we -# propagate that, every feature backed by state.db / kanban.db breaks -# silently — /resume, /title, /history, /branch, kanban dispatcher, etc. -# -# ZFS is a separate case: its COW + mmap semantics can corrupt the WAL -# shared-memory (-shm) file under concurrent connection bursts, presenting -# as ``disk I/O error`` rather than ``locking protocol``. -# -# Instead, fall back to ``journal_mode=DELETE`` (the pre-WAL default) which -# works on NFS and ZFS. Concurrency drops — concurrent readers are blocked -# during a write — but the feature works. -# -# Separately, SQLite's WAL-reset bug can corrupt multi-process WAL databases -# on unfixed library builds (issue #69784). See: -# https://sqlite.org/wal.html#walresetbug -# Fixed in 3.51.3+ with backports 3.50.7 and 3.44.6. On vulnerable builds we -# refuse to *enable* WAL for fresh / non-WAL databases (prefer DELETE). We do -# NOT live-downgrade an on-disk WAL database — other gateway/cron/worker -# connections may still hold it open, and flipping journal_mode under them is -# unsafe (same invariant as the NFS path below). -_WAL_INCOMPAT_MARKERS = ( - "locking protocol", # SQLITE_PROTOCOL on NFS/SMB - "not authorized", # Some FUSE mounts block WAL pragma outright - "disk i/o error", # ZFS SHM corruption under concurrent connections -) - -# Upper bound for the write-ahead log. SQLite defaults to -1 (unlimited), -# which lets state.db-wal keep the high-water mark of the largest-ever -# transaction forever. See _apply_wal_size_limit(). -_WAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024 # 64 MiB # Last SessionDB() init error, per-process. Surfaced in /resume and # related slash-command error strings so users know WHY the DB is @@ -992,20 +1046,6 @@ _WAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024 # 64 MiB _last_init_error: Optional[str] = None _last_init_error_lock = threading.Lock() -# Paths for which we've already logged a WAL-fallback WARNING. Without -# this, kanban_db.connect() (called on every kanban operation — see -# hermes_cli/kanban_db.py for ~30 call sites) would re-log the same -# filesystem-incompat warning on every connection, filling errors.log. -_wal_fallback_warned_paths: set[str] = set() -_wal_fallback_warned_lock = threading.Lock() - -# Dedup WARNING for the WAL-reset vulnerability fallback (issue #69784). -_wal_reset_bug_warned_paths: set[str] = set() -_wal_reset_bug_warned_lock = threading.Lock() - -# Dedup ERROR for the "configured delete overridden by on-disk WAL" warning. -_delete_overridden_warned_paths: set[str] = set() -_delete_overridden_warned_lock = threading.Lock() def _set_last_init_error(msg: Optional[str]) -> None: """Record (or clear) the most recent state.db init failure. @@ -1141,17 +1181,6 @@ def _strip_stale_tool_call_markers( return messages -def _normalize_telegram_topic_profile_name(profile_name: Optional[str] = None) -> str: - """Normalize profile namespace for Telegram topic-mode tables. - - Empty / missing values map to ``\"default\"`` so non-multiplexed gateways - keep a single namespace. Multiplexed callers must pass the *routed* - profile (``source.profile``), never the process-global active profile. - """ - name = str(profile_name or "").strip() - return name if name else "default" - - def format_session_db_unavailable(prefix: str = "Session database not available") -> str: """Format a user-facing 'session DB unavailable' message with cause. @@ -1175,900 +1204,6 @@ def format_session_db_unavailable(prefix: str = "Session database not available" return f"{prefix}: {cause}{hint}." -def _on_disk_journal_mode(conn: sqlite3.Connection) -> Optional[str]: - """Read the journal mode from the SQLite DB header on disk. - - Returns the mode string (e.g. ``"wal"``, ``"delete"``), or ``None`` - if the value cannot be determined (new DB, or PRAGMA read failed). - - A PRAGMA read can fail transiently with ``disk i/o error`` on - virtualized block devices (XFS on cloud hosts). Treating that as - "mode unknown" pushes callers onto their fail-closed unknown-mode - branch even though the on-disk mode is perfectly readable a few - milliseconds later. Retry the read a few times before giving up: - transient EIO clears, deterministic unsupported-filesystem errors do - not. ``None`` is still returned on final failure so the caller's - existing "unknown → refuse to downgrade" logic applies. - """ - last_exc: Optional[Exception] = None - for _ in range(4): - try: - row = conn.execute("PRAGMA journal_mode").fetchone() - except sqlite3.OperationalError as exc: - last_exc = exc - if "disk i/o error" not in str(exc).lower(): - return None - time.sleep(0.05) - continue - if row is None: - return None - mode = row[0] - if isinstance(mode, bytes): # defensive: sqlite3 occasionally returns bytes - try: - mode = mode.decode("ascii") - except UnicodeDecodeError: - return None - return str(mode).strip().lower() if mode is not None else None - if last_exc is not None: - logger.debug( - "_on_disk_journal_mode: retries exhausted on disk read (%s)", last_exc - ) - return None - - -def _apply_wal_size_limit(conn: sqlite3.Connection) -> None: - """Bound the WAL so it returns space to the OS after big transactions. - - SQLite's default ``journal_size_limit`` is -1 (unlimited): after a - checkpoint the WAL file is *reused in place* and never truncated, so - ``state.db-wal`` permanently retains the high-water mark of the largest - transaction ever run against it. - - A single bulk operation is enough to strand gigabytes. Observed on a - 3.0 GB ``state.db``: ``hermes sessions optimize`` (FTS merge + VACUUM) - rewrites every page through the WAL, leaving a **3.07 GB** - ``state.db-wal`` sitting next to the database indefinitely — the host - went from 6.9 GB free to 772 MB (100% full) and stayed there, because - nothing shrinks the WAL back down. An explicit - ``PRAGMA wal_checkpoint(TRUNCATE)`` reclaimed the full 3.07 GB, which - confirms the space was pure slack rather than live data. - - That also makes the maintenance command self-defeating on exactly the - databases that need it most: the larger the DB, the larger the WAL it - strands, so ``optimize`` can consume more disk than it frees. - - ``journal_size_limit`` makes SQLite truncate the WAL back to the limit - at each checkpoint. 64 MiB is comfortably above normal transaction - sizes (so steady-state commits never pay a truncate) while capping the - stranded slack at a bounded, predictable figure. - - ``hermes_cli/kanban_db.py`` already bounds its WAL growth with - ``wal_autocheckpoint=100``; the session store — by far the larger - database — had no equivalent. - - Best-effort: never raises. A failure here only costs disk slack, and - must not prevent the database from opening. - """ - try: - conn.execute(f"PRAGMA journal_size_limit={_WAL_SIZE_LIMIT_BYTES}") - except sqlite3.OperationalError as exc: # pragma: no cover - defensive - logger.debug("journal_size_limit not applied: %s", exc) - - -def _apply_macos_checkpoint_barrier(conn: sqlite3.Connection) -> None: - """Enable ``PRAGMA checkpoint_fullfsync`` on macOS (no-op elsewhere). - - On Darwin, ``synchronous=FULL`` (the WAL default) issues a plain - ``fsync()``, which Apple documents does *not* guarantee that data - has reached stable storage or that writes are not reordered — see - the ``fsync(2)`` man page. SQLite's WAL corruption-safety guarantee - assumes the OS honors the fsync write barrier; macOS does not unless - the app uses ``F_FULLFSYNC``. - - During a launchd *system* shutdown/reboot the OS page cache is - dropped (effectively a power-loss event for in-flight pages), so a - WAL checkpoint whose ``fsync()`` "reported" durable may never have - hit the platter — corrupting ``state.db`` with a malformed image. - This is the trigger in issue #30636 ("SIGTERM during launchd - shutdown under high load"), distinct from a plain in-session kill - (which the page cache survives and SQLite recovers from). - - ``checkpoint_fullfsync=1`` forces an ``F_FULLFSYNC`` barrier only at - checkpoint boundaries — where WAL frames land in the main DB — so the - cost amortizes to roughly +0.1 ms/commit (vs ~+4 ms for the broader - ``fullfsync=1`` that flushes on every commit's WAL sync). Guarded by - ``sys.platform == "darwin"`` because ``F_FULLFSYNC`` is macOS-only; - on other platforms the PRAGMA is a no-op, so we skip it entirely. - - Best-effort: never raises. - """ - if sys.platform != "darwin": - return - try: - conn.execute("PRAGMA checkpoint_fullfsync=1") - except sqlite3.OperationalError: - pass - - -def _enforce_macos_synchronous_full(conn: sqlite3.Connection) -> None: - """Enforce ``PRAGMA synchronous=FULL`` on macOS to prevent btree corruption. - - On Darwin, the default ``synchronous=NORMAL`` only calls ``fsync()``, - which Apple's fsync(2) man page explicitly states does *not* guarantee - data-on-platter or write-ordering. During a WAL checkpoint race with - process termination (e.g., launchd shutdown), this can leave the main - DB with half-written btree pages → ``btreeInitPage error 11``. - - WAL mode's durability guarantee assumes the OS honors fsync barriers; - macOS does not unless we explicitly set ``synchronous=FULL``, which issues - a real ``fsync()`` on every transaction commit. The ``F_FULLFSYNC`` - barrier at checkpoint boundaries is handled separately by - :func:`_apply_macos_checkpoint_barrier`. - - This function is called after any successful WAL activation (either - from ``apply_wal_with_fallback()`` setting a fresh WAL or when probing - an existing WAL mode). It ensures macOS connections always use FULL - synchronous mode, even if a prior connection set ``synchronous=NORMAL``. - - Best-effort: never raises. - """ - if sys.platform != "darwin": - return - try: - conn.execute("PRAGMA synchronous=FULL") - except sqlite3.OperationalError: - pass - - -def is_sqlite_wal_reset_vulnerable( - version_info: Optional[tuple] = None, -) -> bool: - """Return True when the linked SQLite library has the WAL-reset bug. - - Upstream documents the bug in versions 3.7.0 through 3.51.2, fixed in - 3.51.3+, with backports 3.50.7 and 3.44.6: - https://sqlite.org/wal.html#walresetbug - - Pre-WAL libraries (< 3.7.0) cannot hit the race and are treated as safe. - """ - info = version_info if version_info is not None else sqlite3.sqlite_version_info - return _is_sqlite_wal_reset_vulnerable(info) - - -def sqlite_source_id() -> str: - """Return ``sqlite_source_id()``, or an empty string when unavailable.""" - try: - conn = sqlite3.connect(":memory:") - try: - row = conn.execute("SELECT sqlite_source_id()").fetchone() - finally: - conn.close() - except sqlite3.Error: - return "" - if not row or row[0] is None: - return "" - return str(row[0]) - - -def _database_has_content(conn: sqlite3.Connection) -> bool: - """Return whether the database file already holds pages. - - Used to tell an EXISTING database apart from a brand-new one before - rewriting its journal mode. ``PRAGMA page_count`` is a header read, so - this costs nothing and takes no lock. - - Fail-quiet: any error, or a database we cannot measure, answers False. - The only caller uses this to decide whether to emit a warning, and a - warning that fires when the answer is unknown would fire on every fresh - database -- precisely the case where there is provably no operator choice - being overwritten. - """ - try: - row = conn.execute("PRAGMA page_count").fetchone() - except sqlite3.Error: - return False - if not row or row[0] is None: - return False - try: - return int(row[0]) > 0 - except (TypeError, ValueError): - return False - - -def resolve_journal_mode() -> str: - """Return the configured journal mode (``wal`` or ``delete``). - - ``database.journal_mode`` in config.yaml is the canonical operator - setting. ``wal`` remains the default; use ``delete`` when the backing - filesystem does not provide WAL-safe durability (for example macOS - virtiofs, NFS, or SMB). Invalid or malformed values fail safely to the - existing default. - """ - try: - from hermes_cli.config import load_config_readonly - - config = load_config_readonly() or {} - database = config.get("database", {}) - if not isinstance(database, dict): - return "wal" - raw = database.get("journal_mode", "wal") - except Exception: - return "wal" - - if not isinstance(raw, str): - return "wal" - mode = raw.strip().lower() - return mode if mode in ("wal", "delete") else "wal" - - -class WalUnsupportedError(sqlite3.OperationalError): - """Raised by :func:`apply_wal_with_fallback` when ``require_wal=True`` and - the filesystem cannot provide WAL journal mode. - - Covers both shapes of WAL refusal on network filesystems (NFS / SMB / FUSE - / the AgentFS NFS overlay): SQLite *raising* ``SQLITE_PROTOCOL`` ("locking - protocol"), and the quieter macOS-NFS case where ``PRAGMA journal_mode=WAL`` - silently returns the still-effective mode without raising. Subclasses - ``sqlite3.OperationalError`` so existing ``except sqlite3.OperationalError`` - DB-init handling still catches it, while callers that specifically mandate - WAL can catch this narrower type. - """ - - -def apply_wal_with_fallback( - conn: sqlite3.Connection, - *, - db_label: str = "state.db", - require_wal: bool = False, -) -> str: - """Set ``journal_mode=WAL`` on ``conn``, falling back to DELETE on failure. - - Returns the journal mode actually set (``"wal"`` or ``"delete"``). - - On WAL-incompatible filesystems (NFS, SMB, some FUSE, ZFS), SQLite either - raises ``OperationalError("locking protocol")`` / - ``OperationalError("disk I/O error")`` or — on macOS NFS / SMB / - the AgentFS NFS overlay — silently refuses the switch and leaves the DB in - DELETE. Either way the degradation is logged at ERROR level (it is a real - loss of concurrency — a write blocks concurrent readers — not a cosmetic - warning) and, by default, the function falls back to DELETE (the pre-WAL - default, which works on NFS and ZFS) so the feature keeps working. - - On SQLite builds that still contain the WAL-reset corruption bug - (issue #69784), refuse to enable WAL on fresh / non-WAL databases - (prefer DELETE). If the on-disk DB is already WAL, keep WAL and warn - — never live-downgrade under possible concurrent openers. - - This gate (#70055) is deliberately RETAINED. An earlier revision of the - lock-cancellation fix (#71724) reverted it on the theory that DELETE was - "the mode that corrupts", but that comparison was confounded: the clean - WAL result came from SQLite 3.53.1, which carries BOTH the WAL-reset fix - AND 3.51.0's defenses against close()-broken POSIX locks, so it says - nothing about 3.50.4. Re-measured on the actually-bundled 3.50.4 with - the lock fix in place, WAL and DELETE are both clean (0/3 each) — i.e. - there is no evidence that WAL is safer here, and upstream still documents - the WAL-reset bug as real through 3.51.2 with serious consequences. Until - a fixed runtime is delivered, keep new databases out of WAL. - - Callers that genuinely require WAL concurrency (and would rather fail loudly - than run silently degraded) pass ``require_wal=True``; the function then - raises :class:`WalUnsupportedError` instead of returning ``"delete"``. All - current callers deliberately keep the default ``require_wal=False`` so - NFS-homed installs keep working. - - The ERROR is deduplicated per ``db_label``: repeated connections to the - same underlying DB (e.g. kanban_db.connect() which is called on every - kanban operation) log once per process, not once per call. Different - db_labels log independently, so state.db and kanban.db each get one error - on the same NFS mount. - - Shared by :class:`SessionDB` and ``hermes_cli.kanban_db.connect`` so - both databases get identical fallback behavior. - - Never downgrades to DELETE if the on-disk DB header reports WAL — see - _on_disk_journal_mode. That holds for both the NFS path and the - WAL-reset vulnerability path. - """ - configured = resolve_journal_mode() - - # Vulnerable SQLite: do not enable WAL on new/non-WAL files. Resolve the - # operator setting first so an explicit DELETE request still verifies that - # SQLite actually accepted DELETE rather than silently returning MEMORY or - # another connection-specific mode. - if is_sqlite_wal_reset_vulnerable(): - return _apply_delete_for_wal_reset_bug( - conn, - db_label=db_label, - require_delete=configured == "delete", - ) - - # Read-only probe — no flock, no checkpoint, no WAL/SHM unlink. - # Skipping the set-pragma prevents WAL-init from unlinking files other connections hold open. - current_mode = _on_disk_journal_mode(conn) - if current_mode == "wal": - if configured == "delete": - # Never-live-downgrade keeps this WAL; tell the operator their - # configured delete did not apply (see _log_configured_delete_overridden_once). - _log_configured_delete_overridden_once(db_label) - _apply_wal_size_limit(conn) - _apply_macos_checkpoint_barrier(conn) - _enforce_macos_synchronous_full(conn) - return "wal" - - # #68545: honor the canonical database.journal_mode setting. Existing - # on-disk WAL databases were returned above and are never live-downgraded. - if configured == "delete": - if current_mode is None: - # The mode probe failed (database locked / busy): another - # process may hold this DB open in WAL. Ownership is not - # provably exclusive, so flipping journal modes here could - # destroy committed-but-uncheckpointed WAL transactions of a - # concurrent writer. Fail loudly instead of downgrading — the - # operator explicitly requested DELETE and we cannot verify it. - raise sqlite3.OperationalError( - "could not verify journal mode before applying configured " - "journal_mode=delete (database is locked — possible " - "concurrent openers); refusing to downgrade a database " - "this process does not exclusively own" - ) - actual = _set_journal_mode_no_wait(conn, "DELETE") - if actual != "delete": - raise sqlite3.OperationalError( - f"could not set configured journal_mode=delete (got {actual or 'no result'})" - ) - return actual - - # Decide BEFORE the flip whether it would silently overwrite a mode - # somebody chose. Both inputs are only readable while the file is still - # in its original state: `current_mode` is the probe above, and - # page_count distinguishes an existing database from a fresh one. - # - # A 0-page database has no prior choice to overwrite, and every caller - # reaches this before creating any schema (SessionDB._connect_and_init - # applies WAL, then _init_schema), so brand-new databases land here empty - # and stay quiet. - _upgrading_existing_db = ( - current_mode is not None - and current_mode != "wal" - and _database_has_content(conn) - ) - - try: - # ``PRAGMA journal_mode=WAL`` is a query-that-sets: it RETURNS the - # resulting journal mode. Network filesystems that refuse WAL by - # *raising* SQLITE_PROTOCOL ("locking protocol") are handled in the - # except branch below. But macOS NFS — and SMB/CIFS, and the AgentFS - # NFS overlay — refuse the switch WITHOUT raising: the pragma simply - # returns the still-effective mode (e.g. ``delete``). Trust the - # returned row, not the mere absence of an exception; otherwise we - # report a false ``"wal"`` AND skip the fallback WARNING, leaving the - # DB silently in DELETE (reader-blocks-writer) with no signal. - row = conn.execute("PRAGMA journal_mode=WAL").fetchone() - mode = str(row[0]).strip().lower() if row and row[0] is not None else "" - if mode == "wal": - if _upgrading_existing_db: - _log_journal_mode_upgrade_once(db_label, current_mode) - _apply_wal_size_limit(conn) - _apply_macos_checkpoint_barrier(conn) - _enforce_macos_synchronous_full(conn) - return "wal" - # Silent refusal (macOS NFS / SMB / AgentFS overlay): WAL was not - # honored, but nothing raised. - silent_exc = WalUnsupportedError( - f"journal_mode=WAL refused without raising (still {mode!r})" - ) - if require_wal: - raise silent_exc - _log_wal_fallback_once(db_label, silent_exc) - return mode or "delete" - except sqlite3.OperationalError as exc: - # The require_wal silent-refusal raise above is a WalUnsupportedError - # (an OperationalError subclass) and lands here — propagate it - # unchanged rather than re-running it through the marker logic. - if isinstance(exc, WalUnsupportedError): - raise - msg = str(exc).lower() - if not any(marker in msg for marker in _WAL_INCOMPAT_MARKERS): - # Unrelated OperationalError — don't silently swallow. - raise - # ``disk i/o error`` is ambiguous: on ZFS / APFS-CoW it is a - # deterministic WAL-incompatibility (SHM corruption under concurrent - # connection bursts — #55305, #71498), but it can also be a one-shot - # transient EIO (page-cache pressure, brief lock contention). - # Treating a transient EIO as a permanent downgrade signal produced - # the mixed-journal-mode corruption pattern fixed in 5c49cd0ed0 - # (process A downgrades to DELETE while sibling processes set WAL). - # Disambiguate by retrying the pragma a couple of times: transient - # EIO clears and we return "wal"; the deterministic filesystem cases - # keep failing and fall through to the guarded DELETE fallback. - if "disk i/o error" in msg: - for _ in range(2): - time.sleep(0.05) - try: - row = conn.execute("PRAGMA journal_mode=WAL").fetchone() - except sqlite3.OperationalError as retry_exc: - if "disk i/o error" not in str(retry_exc).lower(): - raise - exc = retry_exc - continue - mode = ( - str(row[0]).strip().lower() - if row and row[0] is not None - else "" - ) - if mode == "wal": - # Same flip, later door: a transient EIO cleared and the - # switch went through. The header rewrite is identical, so - # the signal must be too. - if _upgrading_existing_db: - _log_journal_mode_upgrade_once(db_label, current_mode) - _apply_wal_size_limit(conn) - _apply_macos_checkpoint_barrier(conn) - _enforce_macos_synchronous_full(conn) - return "wal" - break - # Don't downgrade if another process already set WAL on disk, or if - # the mode cannot be verified at all (probe blocked by a concurrent - # opener's locks) — ownership is not provably exclusive either way. - existing = _on_disk_journal_mode(conn) - if existing == "wal" or existing is None: - raise - if require_wal: - # Caller mandates WAL — fail loudly instead of degrading to DELETE. - raise WalUnsupportedError(str(exc)) from exc - _log_wal_fallback_once(db_label, exc) - _set_journal_mode_no_wait(conn, "DELETE") - return "delete" - - -def _set_journal_mode_no_wait(conn: sqlite3.Connection, mode: str) -> str: - """Execute ``PRAGMA journal_mode=`` without waiting on other openers. - - This is the ONLY place a journal-mode switch pragma may be issued for a - non-WAL target. It temporarily forces ``busy_timeout=0`` so SQLite's own - exclusivity requirement becomes a concurrent-opener detector: leaving WAL - mode requires exclusive access to the database, so if ANY other connection - (this process or another) holds the DB open, the pragma fails immediately - with ``database is locked`` instead of waiting out a busy timeout and - sneaking the flip in between a concurrent writer's transactions — which is - exactly how committed-but-uncheckpointed WAL transactions get destroyed. - - Callers must treat a raised ``OperationalError`` as "not exclusively - owned: leave the journal mode alone", never as a retryable condition. - - Returns the resulting journal mode as reported by SQLite (lowercase), or - ``""`` when SQLite returned no row. - """ - previous_timeout = 0 - try: - row = conn.execute("PRAGMA busy_timeout").fetchone() - if row and row[0] is not None: - previous_timeout = int(row[0]) - except (sqlite3.OperationalError, TypeError, ValueError): - previous_timeout = 0 - conn.execute("PRAGMA busy_timeout=0") - try: - row = conn.execute(f"PRAGMA journal_mode={mode}").fetchone() - return str(row[0]).strip().lower() if row and row[0] is not None else "" - finally: - try: - conn.execute(f"PRAGMA busy_timeout={previous_timeout}") - except sqlite3.OperationalError: - pass - - -def _apply_delete_for_wal_reset_bug( - conn: sqlite3.Connection, - *, - db_label: str, - require_delete: bool = False, -) -> str: - """Avoid enabling WAL when the linked SQLite has the WAL-reset bug. - - - Already-WAL on disk: leave WAL alone (no live downgrade) and warn. - - Mode unreadable (probe blocked by a concurrent opener's locks): - ownership is not provably exclusive — leave the journal mode alone - and warn. Never treat "could not read the mode" as "not WAL": that - exact confusion let a vulnerable-SQLite process flip a live WAL - state.db to DELETE under a concurrent WAL writer, destroying its - committed-but-uncheckpointed transactions. - - Otherwise: set DELETE (refusing to wait out concurrent openers) and - warn. - - For an explicit operator request, verify SQLite accepted DELETE. - """ - current = _on_disk_journal_mode(conn) - - if current == "wal": - _log_wal_reset_bug_once(db_label, kept_wal=True) - if require_delete: - # The vulnerability warning above suggests upgrading SQLite, which - # does not help on a WAL-incompatible filesystem; surface that the - # configured delete is not in effect (see _log_configured_delete_overridden_once). - # Emitted last so the actionable message is the final one in the log. - _log_configured_delete_overridden_once(db_label) - # Do not TRUNCATE / journal_mode=DELETE while other processes may - # still hold this WAL DB open; same safety rule as the NFS path. - _apply_wal_size_limit(conn) - _apply_macos_checkpoint_barrier(conn) - _enforce_macos_synchronous_full(conn) - return "wal" - - if current is None: - # The mode probe itself failed — another opener's locks are the - # most likely cause, and the DB may well be in WAL under a live - # writer. Never flip a journal mode we cannot even read. - if require_delete: - raise sqlite3.OperationalError( - "could not verify journal mode before applying configured " - "journal_mode=delete (database is locked — possible " - "concurrent openers); refusing to downgrade a database " - "this process does not exclusively own" - ) - _log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True) - return "wal" - - actual = "" - try: - actual = _set_journal_mode_no_wait(conn, "DELETE") - except sqlite3.OperationalError as exc: - if require_delete: - raise - lowered = str(exc).lower() - if "locked" in lowered or "busy" in lowered: - # A concurrent opener appeared between the probe and the flip - # (or already held the DB): SQLite refused the exclusive lock. - # Leave the journal mode exactly as it is. - _log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True) - return current or "delete" - # Best-effort for the automatic vulnerable-runtime fallback: DELETE is - # normally already the default for new file-backed databases. - if require_delete and actual != "delete": - raise sqlite3.OperationalError( - "could not set configured journal_mode=delete " - f"(got {actual or 'no result'})" - ) - _log_wal_reset_bug_once(db_label, kept_wal=False) - return "delete" - - -def _wal_reset_repair_hint() -> str: - """Return a context-appropriate hint for repairing the SQLite runtime. - - Uses the codebase's install-type detection so the hint matches what - ``hermes update`` can actually do for this install (#75153). - """ - try: - from hermes_cli.config import ( - detect_install_method, - recommended_update_command_for_method, - get_project_root, - ) - method = detect_install_method(get_project_root()) - cmd = recommended_update_command_for_method(method) - if method in {"git", "unknown"}: - return f"Hermes-managed installs can repair the embedded runtime with `{cmd}`" - if method == "docker": - return f"update the container image with `{cmd}`" - # nix/nixos - return cmd - except Exception: - pass - return ( - "install a Python build bundled with SQLite 3.51.3+ " - "(or backports 3.50.7 / 3.44.6) and restart Hermes" - ) - - -# Dedup state for _log_journal_mode_upgrade_once, mirroring the -# _wal_fallback_warned_* pair below it. -_journal_upgrade_warned_paths: set = set() -_journal_upgrade_warned_lock = threading.Lock() - - -def _log_wal_reset_bug_once( - db_label: str, - *, - kept_wal: bool, - indeterminate: bool = False, -) -> None: - """Log once per (process, db_label) about the WAL-reset vulnerability path.""" - with _wal_reset_bug_warned_lock: - if db_label in _wal_reset_bug_warned_paths: - return - _wal_reset_bug_warned_paths.add(db_label) - if indeterminate: - action = ( - "journal mode could not be verified or exclusively switched " - "(database is locked — possible concurrent openers); leaving the " - "journal mode untouched (no live downgrade under concurrent " - "openers)" - ) - elif kept_wal: - action = ( - "is already in WAL mode — leaving WAL in place (no live " - "downgrade under concurrent openers)" - ) - else: - action = "using journal_mode=DELETE instead of enabling WAL" - # Check whether this is a Hermes-managed install (uv-managed venv) - # so the warning doesn't promise a repair path that doesn't exist - # for git/pip/system Python installs (#75153). - repair_hint = _wal_reset_repair_hint() - logger.warning( - "%s: linked SQLite %s (interpreter %s) is vulnerable to the WAL-reset " - "corruption bug (https://sqlite.org/wal.html#walresetbug) — %s. " - "Upgrade to SQLite 3.51.3+ (or backports 3.50.7 / 3.44.6); " - "%s. See `hermes doctor`. This warning fires once per " - "process per database.", - db_label, - sqlite3.sqlite_version, - sys.executable, - action, - repair_hint, - ) - - -def _log_journal_mode_upgrade_once(db_label: str, previous_mode: str) -> None: - """Log a single WARNING per (process, db_label) about a non-WAL -> WAL flip. - - ``PRAGMA journal_mode`` is a property of the FILE, not of the connection: - switching an existing database to WAL rewrites its header and outlives the - process that did it. Operators do set it directly on the file -- that was - the documented mitigation for the SQLite 3.50.4 WAL-reset bug -- and - nothing here told them the next open would silently put it back. - - WARNING, not ERROR, and deliberately so. The reverse move is logged at - ERROR by ``_log_wal_fallback_once`` because dropping to DELETE is a real - loss of concurrency; this direction is normally the desirable one (see - ``hermes_cli/managed_uv._default_live_venv``, which treats a database - stuck on DELETE as a bug worth repairing on update). The problem is not - the change, it is that the change was invisible: an operator who chose - DELETE deliberately had no way to learn their choice had been overwritten, - or which lever makes it stick. So this says what happened and names the - durable setting, without claiming a degradation that is not there. - - Deduped per process per ``db_label`` like its siblings: kanban opens a - fresh connection per operation, so an undeduped line here would be a log - flood rather than a signal. - """ - with _journal_upgrade_warned_lock: - if db_label in _journal_upgrade_warned_paths: - return - _journal_upgrade_warned_paths.add(db_label) - logger.warning( - "%s: on-disk journal_mode was %s and has been switched to WAL. This " - "rewrites the database header and persists after this process exits. " - "If %s was a deliberate choice (for example the mitigation for the " - "SQLite WAL-reset bug, or a WAL-unsafe filesystem), setting it with " - "PRAGMA on the file will not survive -- every open re-applies the " - "configured mode. Set `database.journal_mode: delete` in config.yaml " - "to make it stick. This message fires once per process per database.", - db_label, - previous_mode, - previous_mode, - ) - - -def _log_wal_fallback_once(db_label: str, exc: Exception) -> None: - """Log a single ERROR per (process, db_label) about WAL fallback. - - ERROR (not WARNING): a DB silently dropped to DELETE means a real loss of - concurrency — under the kanban dispatcher + workers a write blocks readers, - surfacing as SQLITE_BUSY/lock contention — so it must be loud, not cosmetic. - - Without this dedup, NFS users running kanban (which opens a fresh - connection on every operation — see hermes_cli/kanban_db.py) would - fill errors.log with hundreds of identical errors per hour. - """ - with _wal_fallback_warned_lock: - if db_label in _wal_fallback_warned_paths: - return - _wal_fallback_warned_paths.add(db_label) - logger.error( - "%s: WAL journal_mode unsupported on this filesystem (%s) — " - "falling back to journal_mode=DELETE (slower rollback-journal " - "mode; reduces concurrency but works on NFS/SMB/FUSE/ZFS). See " - "https://www.sqlite.org/wal.html for details. This message " - "fires once per process per database.", - db_label, - exc, - ) - - -def _log_configured_delete_overridden_once(db_label: str) -> None: - """Log a single ERROR per (process, db_label) when the operator configured - ``journal_mode=delete`` but the on-disk DB is already WAL, so the configured - mode is not in effect. - - Counterpart to :func:`_log_wal_fallback_once` for the opposite direction: - there WAL was refused by the filesystem and we silently fell back to DELETE; - here the operator asked for DELETE but an inherited on-disk WAL header means - we keep WAL (the never-live-downgrade rule prevents a live downgrade, which - causes mixed-mode corruption). The signal matters because otherwise the - operator has no indication that ``database.journal_mode: delete`` had no - effect and the DB still requires a one-time offline ``PRAGMA - journal_mode=DELETE`` (with no open connections) to apply. - - Fires once per process per database. - """ - with _delete_overridden_warned_lock: - if db_label in _delete_overridden_warned_paths: - return - _delete_overridden_warned_paths.add(db_label) - logger.error( - "%s: database.journal_mode=delete is configured but the on-disk " - "database is already WAL; keeping WAL (a live downgrade under open " - "connections can corrupt the DB). To apply journal_mode=DELETE, stop " - "all connections to this DB and run a one-time offline " - "'PRAGMA journal_mode=DELETE' on the file. This message fires once " - "per process per database.", - db_label, - ) - - -# --------------------------------------------------------------------------- -# Config-driven database pragmas -# --------------------------------------------------------------------------- -# PRAGMA synchronous accepts either an integer or a symbolic name, and operators -# write the names. Mapped here rather than passed through so a typo becomes a -# warning instead of a silently different durability level. -_SYNCHRONOUS_LEVELS: Dict[str, int] = { - "OFF": 0, - "NORMAL": 1, - "FULL": 2, - "EXTRA": 3, -} -_SYNCHRONOUS_NAMES: Dict[int, str] = {v: k for k, v in _SYNCHRONOUS_LEVELS.items()} -_SYNCHRONOUS_FULL = 2 - - -def resolve_synchronous_level(raw_value: Any) -> Optional[int]: - """Map a configured ``database.synchronous`` value to its PRAGMA integer. - - Accepts the symbolic names SQLite documents (``OFF``/``NORMAL``/``FULL``/ - ``EXTRA``, any case) and the equivalent integers ``0``-``3``. Returns None - for anything else so the caller can warn and leave the level untouched — - guessing at a malformed durability setting is worse than ignoring it. - """ - if isinstance(raw_value, bool): - # bool is an int subclass, and YAML turns a bare `on`/`off` into one. - # "off" as a durability level is a real choice; True is meaningless. - return 0 if raw_value is False else None - if isinstance(raw_value, int): - return raw_value if raw_value in _SYNCHRONOUS_NAMES else None - text = str(raw_value).strip() - if not text: - return None - upper = text.upper() - if upper in _SYNCHRONOUS_LEVELS: - return _SYNCHRONOUS_LEVELS[upper] - try: - value = int(text) - except (TypeError, ValueError): - return None - return value if value in _SYNCHRONOUS_NAMES else None - - -def _apply_synchronous_pragma( - conn: sqlite3.Connection, - raw_value: Any, - *, - db_label: str, -) -> None: - """Set ``PRAGMA synchronous`` from config, never below FULL on macOS. - - Split out of the integer loop in :func:`apply_database_pragmas` because - this PRAGMA is not interchangeable with the sizing ones around it. Those - trade memory or disk for speed; this one decides whether a commit is on - the platter, so an unrecognised value must not fall through to "SQLite - default" the way a bad ``cache_size`` harmlessly can. - - The Darwin floor exists because :func:`_enforce_macos_synchronous_full` - runs during ``apply_wal_with_fallback()`` and this function runs after it, - so a configured ``NORMAL`` would otherwise silently undo the macOS btree - protection that fix put there deliberately. Raising the level on macOS is - allowed; lowering it is refused out loud. - """ - level = resolve_synchronous_level(raw_value) - if level is None: - logger.warning( - "%s: ignoring unrecognized database.synchronous=%r " - "(expected OFF, NORMAL, FULL, EXTRA, or 0-3)", - db_label, - raw_value, - ) - return - if sys.platform == "darwin" and level < _SYNCHRONOUS_FULL: - logger.warning( - "%s: refusing database.synchronous=%s on macOS; keeping FULL. " - "Darwin's fsync() does not guarantee write ordering, so a lower " - "level readmits the half-written btree pages FULL exists to " - "prevent.", - db_label, - _SYNCHRONOUS_NAMES[level], - ) - return - try: - conn.execute(f"PRAGMA synchronous={level}") - except sqlite3.OperationalError: - pass - - -def apply_database_pragmas( - conn: sqlite3.Connection, - *, - db_label: str = "state.db", -) -> None: - """Apply optional performance and WAL-sizing PRAGMAs from ``config.yaml``. - - Reads the ``database:`` section and applies configurable PRAGMAs when set - to integer values. The journal mode itself is NOT handled here — - ``database.journal_mode`` is owned by :func:`resolve_journal_mode` inside - :func:`apply_wal_with_fallback`, which layers the operator setting under - all the safety guards (never live-downgrading an on-disk WAL DB, - filesystem fallback, WAL-reset-bug gating). - - Supported keys under ``database:`` in config.yaml: - - * ``cache_size`` — negative value = KiB, positive = pages - (e.g. ``-262144`` = 256 MB page cache) - * ``mmap_size`` — max bytes for memory-mapped I/O (0 = disabled) - * ``temp_store`` — 0=DEFAULT(file), 1=FILE, 2=MEMORY, 3=ALWAYS - * ``wal_autocheckpoint`` — WAL auto-checkpoint threshold in pages - * ``journal_size_limit`` — max journal/WAL size in bytes - * ``synchronous`` — durability level: ``OFF``/``NORMAL``/``FULL``/``EXTRA`` - or ``0``-``3``. Unset leaves SQLite's own default, which is a - *compile-time* constant (``SQLITE_DEFAULT_WAL_SYNCHRONOUS``) and so - differs between the bundled, distro and Homebrew builds an operator - might be running. Setting it explicitly is the only way to know. - - Best-effort: config load or pragma failures are ignored so DB init - never breaks on a malformed ``database:`` section. - """ - try: - # Local import avoids a circular import with hermes_cli.config. - from hermes_cli.config import cfg_get, load_config_readonly - - cfg = load_config_readonly() - except Exception: - return - - # Performance PRAGMAs (applied to ALL connection types: writer, read_only, - # and WAL per-thread readers). - for pragma_name in ( - "cache_size", - "mmap_size", - "temp_store", - "wal_autocheckpoint", - "journal_size_limit", - ): - raw_value = cfg_get(cfg, "database", pragma_name, default=None) - if raw_value is None: - continue - try: - value = int(str(raw_value).strip()) - except (TypeError, ValueError): - logger.warning( - "%s: ignoring non-integer database.%s=%r", - db_label, - pragma_name, - raw_value, - ) - continue - try: - conn.execute(f"PRAGMA {pragma_name}={value}") - except sqlite3.OperationalError: - pass - - # Last, so it wins over nothing and loses to nothing: the sizing pragmas - # above cannot change durability, and the macOS enforcement ran earlier - # during WAL activation (see _apply_synchronous_pragma for why that - # ordering needs an explicit floor rather than an explicit override). - raw_synchronous = cfg_get(cfg, "database", "synchronous", default=None) - if raw_synchronous is not None: - _apply_synchronous_pragma(conn, raw_synchronous, db_label=db_label) - - # --------------------------------------------------------------------------- # Malformed-schema recovery # --------------------------------------------------------------------------- @@ -2318,21 +1453,6 @@ def classify_persistence_error(exc_or_str) -> str: return "unknown" -def _claim_repair_attempt(db_path: Path) -> bool: - """Claim the one-shot repair attempt for *db_path* in this process. - - Returns True for the first caller, False afterwards. Keeps a malformed - DB from triggering an unbounded repair/reopen loop and stops concurrent - callers from racing surgery on the same file. - """ - key = str(db_path) - with _repair_attempt_lock: - if key in _repair_attempted_paths: - return False - _repair_attempted_paths.add(key) - return True - - # Cross-process serialisation for the schema-surgery paths below. The # ``_repair_attempt_lock`` above is a ``threading.Lock`` — it only covers # threads inside ONE interpreter, yet a normal Hermes host runs several @@ -2347,208 +1467,9 @@ def _claim_repair_attempt(db_path: Path) -> bool: # lock the losing caller spent the same minutes running its own surgery, it # just did so on top of the winner's. _REPAIR_LOCK_TIMEOUT_SECONDS = 120.0 -_REPAIR_LOCK_POLL_SECONDS = 0.1 -# Copying a multi-GB database through SQLite's online-backup API is a data -# transfer, not an inter-process locking operation. Keep the repair-lock -# bound separate and give a conservative 10 MiB/s budget to each snapshot or -# promotion, with the historical two-minute floor for ordinary state.db files. -_REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND = 10 * 1024 * 1024 _IS_WINDOWS = sys.platform == "win32" -@contextlib.contextmanager -def _cross_process_repair_lock(db_path: Path): - """Serialize state.db schema surgery across processes. - - Yields True when this process holds the repair lock for *db_path*, False - when the bounded acquire timed out or the lock file could not be opened at - all. Unlike the kanban init lock — whose critical section is idempotent, - so proceeding without the lock is merely redundant work — proceeding here - would be exactly the unsafe interleaving we are trying to prevent, so a - caller that gets False must NOT do surgery. - - ``flock`` is the right primitive for this: the kernel drops the lock when - the holding process dies, so a crashed repairer cannot leave a stale lock - that wedges every future repair (a pidfile would). One exception exists - (issue #100108): a forked child that inherited the lock fd keeps the - flock alive after the acquirer dies, so the acquire path records the - holder's pid + start time and breaks the lock when that holder is - provably dead (see ``_acquire_db_flock``). The acquire is still - bounded because a *live* repairer can legitimately sit in ``VACUUM`` for - minutes on a large DB, and an unbounded wait would hang the caller's open - with no traceback (the failure shape of #36644). - """ - lock_path = db_path.with_name(db_path.name + ".repair.lock") - try: - lock_path.parent.mkdir(parents=True, exist_ok=True) - handle = lock_path.open("a+b") - except OSError as exc: - # Fail closed, exactly as a timed-out acquire does. A lock file we - # cannot even open means the filesystem is out of space, inodes or - # descriptors — and a sibling that opened ITS handle before the disk - # filled is still inside writable_schema surgery or VACUUM. Yielding - # True here let two processes run schema surgery on the same live - # state.db concurrently, which is itself the corruption source this - # lock exists to remove (#100368: the disk-full trigger, then a fresh - # corruption on every boot with other writers alive). Callers already - # handle False by re-probing and reporting, and on a read-only - # directory no repair strategy could have written anyway. - logger.warning( - "Could not open state.db repair lock %s (%s) — skipping schema " - "surgery rather than running it without cross-process authority.", - lock_path, exc, - ) - yield False - return - - acquired = False - try: - if _IS_WINDOWS: - deadline = time.monotonic() + _REPAIR_LOCK_TIMEOUT_SECONDS - while True: - try: - import msvcrt - - handle.seek(0) - msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) - acquired = True - break - except (BlockingIOError, OSError) as exc: - if not is_advisory_lock_contention(exc): - logger.warning( - "Could not acquire state.db repair lock %s (%s) — " - "skipping schema surgery on a non-contention error.", - lock_path, exc, - ) - acquired = None - break - if time.monotonic() >= deadline: - break - time.sleep(_REPAIR_LOCK_POLL_SECONDS) - else: - acquired, handle = _acquire_db_flock( - str(lock_path), - handle, - _REPAIR_LOCK_TIMEOUT_SECONDS, - _REPAIR_LOCK_POLL_SECONDS, - "state.db repair lock", - ) - if acquired is None: - # Non-contention failure already logged with its errno. - acquired = False - elif not acquired: - record = None if _IS_WINDOWS else _read_lock_holder_record(handle) - logger.warning( - "state.db repair lock %s held by another process for more " - "than %.0fs — skipping schema surgery in this process to " - "avoid racing the repairer. Recorded holder: %s.", - lock_path, _REPAIR_LOCK_TIMEOUT_SECONDS, - _describe_lock_holder(record), - ) - yield acquired - finally: - try: - if acquired: - if _IS_WINDOWS: - import msvcrt - - handle.seek(0) - msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) - else: - import fcntl - - _clear_lock_holder_record(handle) - fcntl.flock(handle.fileno(), fcntl.LOCK_UN) - except OSError: # pragma: no cover - best effort release - pass - finally: - handle.close() - - -def _try_acquire_auto_maintenance_lock(db_path: Path) -> Optional[Any]: - """Non-blocking cross-process lock for one auto-maintenance pass. - - The kernel releases this advisory lock if the holder exits, unlike a - durable pid/meta marker. A caller that cannot acquire it must skip the - pass: otherwise two startups can both pass the interval check and the - second can prune a row the first has only just closed recoverably. - """ - lock_path = db_path.with_name(db_path.name + ".auto-maintenance.lock") - try: - lock_path.parent.mkdir(parents=True, exist_ok=True) - handle = lock_path.open("a+b") - except OSError as exc: - logger.warning( - "Could not open state.db auto-maintenance lock %s (%s) — skipping " - "automatic maintenance.", - lock_path, - exc, - ) - return None - - try: - if _IS_WINDOWS: - import msvcrt - - handle.seek(0) - msvcrt.locking( # type: ignore[attr-defined] - handle.fileno(), msvcrt.LK_NBLCK, 1 # type: ignore[attr-defined] - ) - else: - import fcntl - - fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - except (BlockingIOError, OSError): - handle.close() - return None - return handle - - -def _release_auto_maintenance_lock(handle: Any) -> None: - """Release a handle returned by :func:`_try_acquire_auto_maintenance_lock`.""" - try: - if _IS_WINDOWS: - import msvcrt - - handle.seek(0) - msvcrt.locking( # type: ignore[attr-defined] - handle.fileno(), msvcrt.LK_UNLCK, 1 # type: ignore[attr-defined] - ) - else: - import fcntl - - fcntl.flock(handle.fileno(), fcntl.LOCK_UN) - except OSError: # pragma: no cover - best effort release - pass - finally: - handle.close() - - -def _bump_schema_cookie(conn: sqlite3.Connection) -> None: - """Increment the schema cookie after direct ``sqlite_master`` surgery. - - Ordinary DDL bumps this counter for free, and every other connection - compares it before running a prepared statement — that is how they learn - to discard a cached schema. Editing ``sqlite_master`` under - ``PRAGMA writable_schema=ON`` does NOT bump it, so live connections in - other processes keep compiling statements against the schema we just - deleted objects from — e.g. writing ``messages`` rows through triggers - into ``messages_fts*`` shadow tables that no longer exist. SQLite's - writable_schema documentation calls out incrementing ``schema_version`` - as the required companion to such an edit. - - Best-effort and never raises: a failed bump leaves exactly the - pre-existing behaviour, and the repair itself is still worth completing. - """ - try: - current = conn.execute("PRAGMA schema_version").fetchone()[0] - # Wraps within the 32-bit signed range SQLite stores this in; the - # comparison other connections make is equality, not ordering. - conn.execute(f"PRAGMA schema_version={(int(current) + 1) & 0x7FFFFFFF}") - except (sqlite3.DatabaseError, TypeError, IndexError) as exc: - logger.warning("Could not bump state.db schema cookie: %s", exc) - - # ── Repair-loop bounding + dead-backup hygiene (#86747) ───────────────────── # # ``_claim_repair_attempt`` above is an in-memory set: it bounds the loop @@ -2565,1629 +1486,6 @@ def _bump_schema_cookie(conn: sqlite3.Connection) -> None: # damaged file is never copied twice, and only the newest # ``_MAX_MALFORMED_BACKUPS`` forensic copies are kept. -_MAX_PERSISTENT_REPAIR_ATTEMPTS = 3 -_MAX_MALFORMED_BACKUPS = 3 - -# Sidecars copied alongside a damaged DB and pruned with it. ``-journal`` is -# included because rollback-journal (DELETE) mode — Hermes's fallback on -# NFS/SMB/FUSE/ZFS and on WAL-reset-vulnerable SQLite builds — leaves a hot -# journal on disk whenever a transaction was open, and that file is what -# interprets the damaged bytes. Omitting it from the forensic copy means the -# backup cannot be rolled back to a consistent state by hand. -_DB_SIDECAR_SUFFIXES = ("-wal", "-shm", "-journal") - -# Head/tail bytes sampled by ``_db_fingerprint``. Enough to change whenever -# the DB is genuinely repaired, truncated or restored (SQLite rewrites the -# header on any real recovery), while staying O(1) on a multi-GB file. -_FINGERPRINT_SAMPLE_BYTES = 65536 - -# Byte ranges inside SQLite's 100-byte database header that move on ordinary -# commits rather than on repair, and are therefore masked out of the content -# sample. In rollback-journal (DELETE) mode a commit writes the main file -# directly, bumping the file change counter (24-27) and version-valid-for -# (92-95); a malformed-SCHEMA DB still accepts those writes, so without the -# mask any live session write re-keys the ledger and the repair budget resets -# to 1 forever — the exact unbounded loop this ledger exists to stop. (WAL mode -# routes commits to the -wal sidecar, so the main file's header only moves on -# checkpoint; masking is harmless there and correct for both.) Everything that -# matters for repair identity — the page-1 sqlite_master b-tree — sits after -# byte 100 and stays in the sample. -_FINGERPRINT_VOLATILE_HEADER_RANGES = ((24, 28), (92, 96)) - - -def _mask_volatile_header(head: bytes) -> bytes: - """Zero the commit-counter fields so ordinary writes don't re-key the ledger.""" - if len(head) < 96: - return head - buf = bytearray(head) - for start, end in _FINGERPRINT_VOLATILE_HEADER_RANGES: - buf[start:end] = b"\x00" * (end - start) - return bytes(buf) - -# Free-space headroom for the pre-repair forensic backup. The backup is a -# full raw copy of the damaged DB (plus its -wal/-shm sidecars), so a repair -# loop on a large state.db is a disk amplifier: the reporting incident wrote -# ~98MB every ~10s until the volume was nearly full, which would have taken -# down every agent on the host. -# -# Proportional, not a flat floor: an absolute multi-GB reserve would refuse -# backups that fit comfortably on small container/VM volumes, and because a -# refused backup is a HARD STOP (#69603) that would silently convert "repair -# loops" into "repair never runs" for those deployments. Require the copy -# itself plus a small slice of the volume, clamped to a modest floor. -_REPAIR_BACKUP_MIN_FREE_BYTES = 256 * 1024 * 1024 # 256 MiB absolute floor -_REPAIR_BACKUP_FREE_FRACTION = 0.02 # plus 2% of the volume - - -def _repair_backup_headroom_bytes(total_bytes: int) -> int: - """Free space required *beyond* the copy itself, for a volume of *total_bytes*.""" - return max( - _REPAIR_BACKUP_MIN_FREE_BYTES, - int(total_bytes * _REPAIR_BACKUP_FREE_FRACTION), - ) - - -def _repair_scratch_space_error(db_path: Path) -> Optional[str]: - """Return an error unless snapshot, VACUUM and promotion can fit safely.""" - import shutil - - try: - main_bytes = db_path.stat().st_size - snapshot_bytes = main_bytes - for suffix in _DB_SIDECAR_SUFFIXES: - sidecar = db_path.with_name(db_path.name + suffix) - if sidecar.exists(): - snapshot_bytes += sidecar.stat().st_size - usage = shutil.disk_usage(db_path.parent) - headroom = _repair_backup_headroom_bytes(usage.total) - # Strategy 2 runs VACUUM on the staged database. SQLite documents that - # VACUUM may need up to twice the database size in additional free - # space while it builds the replacement and journals the overwrite. - # Reserve that beyond the snapshot itself; after VACUUM releases its - # temporary files, the same reserve also covers transactional - # promotion into the live database. - required = snapshot_bytes + (2 * snapshot_bytes) + headroom - if usage.free >= required: - return None - return ( - f"only {usage.free / 1e9:.2f}GB free on {db_path.parent}; the " - f"repair snapshot needs up to {snapshot_bytes / 1e9:.2f}GB, " - f"VACUUM may need another {(2 * snapshot_bytes) / 1e9:.2f}GB, and " - f"{headroom / 1e9:.2f}GB must remain as headroom. Free disk space, " - "then retry." - ) - except OSError as exc: - return ( - f"could not determine free space on {db_path.parent} ({exc}); " - "refusing the repair snapshot rather than risk filling the volume" - ) - - -def _repair_snapshot_timeout_seconds(source_path: Path) -> float: - """Bound one SQLite snapshot by source size, including live sidecars. - - A WAL can contain committed canonical rows which are not yet present in - the main database file. Count it (and the rollback journal where - present), both to describe the work honestly and to avoid applying the - repair-lock timeout to an otherwise healthy large-database copy. - """ - source_bytes = 0 - for suffix in ("", *_DB_SIDECAR_SUFFIXES): - candidate = ( - source_path - if not suffix - else source_path.with_name(source_path.name + suffix) - ) - try: - source_bytes += candidate.stat().st_size - except FileNotFoundError: - continue - return max( - _REPAIR_LOCK_TIMEOUT_SECONDS, - source_bytes / _REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND, - ) - - -def _repair_failure_consumes_attempt(exc: BaseException) -> bool: - """Whether a pre-strategy SQLite failure proves deterministic corruption. - - Lock contention, timeouts, disk-full, I/O and filesystem failures are - environmental aborts: retrying later may succeed and must not exhaust the - repair ledger. Only SQLite's corruption/image result codes prove the - deterministic damage the bounded ledger exists to stop from retrying - forever, even when SQLite cannot stage a snapshot far enough to run a - named strategy. - """ - if not isinstance(exc, sqlite3.DatabaseError): - return False - error_code = getattr(exc, "sqlite_errorcode", None) - if isinstance(error_code, int): - # Extended result codes retain the primary code in the low byte. - primary_code = error_code & 0xFF - return primary_code in (sqlite3.SQLITE_CORRUPT, sqlite3.SQLITE_NOTADB) - - # sqlite3 versions before exception result-code attributes need a narrow, - # conservative compatibility path. Do not turn generic DatabaseError - # messages such as "disk is full" or "readonly" into permanent failures. - message = str(exc).lower() - return ( - "file is not a database" in message - or "database disk image is malformed" in message - ) - - -def _repair_ledger_path(db_path: Path) -> Path: - return db_path.with_name(db_path.name + ".repair-attempts.json") - - -def _db_fingerprint(db_path: Path) -> "Optional[str]": - """Cheap identity for a damaged DB file: size + a bounded content sample. - - Deliberately EXCLUDES mtime. The original ledger keyed on - ``size:mtime_ns`` on the assumption that "nothing can successfully write - to a damaged file", but that does not hold for the malformed-schema - class: the DB still opens and accepts writes (only ``sqlite_master`` is - unreadable), so live writers, WAL checkpoints and the in-place repair - strategies themselves all move mtime between passes. Every pass then - looked like a NEW file — the attempt counter reset to 1 forever, never - reaching ``_MAX_PERSISTENT_REPAIR_ATTEMPTS``, and the ``_backup_db_file`` - dedupe (which compares mtime too) never matched, so each pass wrote - another full-size forensic copy. Observed: a repair every ~10s, a fresh - 98MB copy each time, 2.3GB in 20 minutes, disk heading to zero. - - Hashing a multi-GB corrupt file on every open is the repeated cost this - ledger exists to avoid, so sample instead of digesting the whole file: - size plus the head/tail slices that any real repair, truncation or - restore necessarily changes. Stable across passes that merely touch - mtime; still resets the attempt count after genuine recovery. - - The content read runs under ``offline_file_access`` because it takes a raw - descriptor, and ``close()`` on ANY descriptor cancels every POSIX advisory - lock this process holds on the file — including a peer connection's - RESERVED lock (see ``hermes_cli.sqlite_safe_read`` rule 1). This function - is reached from ``repair_state_db_schema``'s exhaustion probe BEFORE - ``_backup_db_file``'s ``has_live_connection`` guard, and the repair path is - entered by one SessionDB while the gateway holds others, so a live peer is - the expected case rather than a theoretical one. - - Returns ``None`` when a live connection makes the read unsafe. Callers MUST - NOT substitute a differently-shaped key (an earlier revision fell back to - ``size:mtime_ns``): the ledger compares keys for equality, so alternating - between a content key and an mtime key across passes never matches, the - counter resets to 1 every time and the unbounded repair loop this ledger - exists to stop comes straight back. ``None`` means "identity unavailable", - and the ledger helpers below keep using the key already on record. - """ - try: - st = db_path.stat() - try: - from hermes_cli.sqlite_safe_read import ( - LiveConnectionError, - offline_file_access, - ) - except ImportError: - # Scaffold/embed installs ship hermes_state without hermes_cli. No - # tracked connections exist there, so the raw read is safe. - @contextmanager - def offline_file_access(_path, **_kw): - yield - - class LiveConnectionError(Exception): - pass - - try: - with offline_file_access(db_path, what="fingerprint"): - with open(db_path, "rb") as fh: - head = fh.read(_FINGERPRINT_SAMPLE_BYTES) - if st.st_size > _FINGERPRINT_SAMPLE_BYTES: - fh.seek(max(0, st.st_size - _FINGERPRINT_SAMPLE_BYTES)) - tail = fh.read(_FINGERPRINT_SAMPLE_BYTES) - else: - tail = b"" - except LiveConnectionError: - return None - digest = hashlib.sha256(_mask_volatile_header(head) + tail).hexdigest()[:32] - return f"{st.st_size}:{digest}" - except OSError: - return None - - -def _backup_content_identity(db_path: Path) -> "Optional[str]": - """Recovery-image identity for forensic-backup dedupe: whole-file + sidecars. - - This is a DIFFERENT equivalence relation from :func:`_db_fingerprint`, and - the two MUST NOT be conflated. ``_db_fingerprint`` answers "same repair - epoch?" — it masks SQLite's commit counters and samples only the head/tail - so an ordinary write does not mint a fresh repair budget. That is exactly - the wrong predicate for "may I reuse an existing forensic copy?": a live - writer can commit new transcript/session rows into an *interior* page while - preserving file size and leaving the first/last 64 KiB untouched, so two - materially different recovery images share one ``_db_fingerprint``. Reusing - a backup on that basis hands the operator a snapshot that predates real - user data (and #87409 shows a failed in-place repair can still VACUUM - canonical tables away), so the forensic copy must claim byte identity, not - epoch identity. - - So this digests the ENTIRE main file plus every present sidecar - (``-wal``/``-shm``/``-journal``) — the WAL can hold committed frames not yet - checkpointed, so it is part of the recovery image. The cost is an O(n) read; - on a miss the caller is about to do an O(n) *write* (the full raw copy), so - the read is the cheaper half and never the dominant cost. Runs under - ``offline_file_access`` for the same POSIX-advisory-lock reason as - ``_db_fingerprint``; returns ``None`` when a live connection makes the read - unsafe (caller then declines to dedupe and takes a fresh backup — the safe - side, never a false reuse). - """ - try: - from hermes_cli.sqlite_safe_read import ( - LiveConnectionError, - offline_file_access, - ) - except ImportError: - @contextmanager - def offline_file_access(_path, **_kw): - yield - - class LiveConnectionError(Exception): - pass - - def _hash_whole(path: Path, hasher: "Any") -> None: - with open(path, "rb") as fh: - for chunk in iter(lambda: fh.read(1024 * 1024), b""): - hasher.update(chunk) - - try: - hasher = hashlib.sha256() - with offline_file_access(db_path, what="backup-identity"): - # Length-delimit every member (main file included) so the - # concatenation is prefix-free — otherwise a main-file tail could - # coincide with a main+sidecar split and dedupe two different - # recovery images together. - hasher.update(f"\0main:{db_path.stat().st_size}\0".encode()) - _hash_whole(db_path, hasher) - for suffix in _DB_SIDECAR_SUFFIXES: - sidecar = db_path.with_name(db_path.name + suffix) - if sidecar.exists(): - hasher.update(f"\0{suffix}:{sidecar.stat().st_size}\0".encode()) - _hash_whole(sidecar, hasher) - return hasher.hexdigest() - except LiveConnectionError: - return None - except OSError: - return None - - -def _read_repair_ledger(db_path: Path) -> "Dict[str, Any]": - try: - raw = json.loads(_repair_ledger_path(db_path).read_text(encoding="utf-8")) - if isinstance(raw, dict): - return raw - except (OSError, ValueError): - pass - return {} - - -def _persistent_repair_attempts_exhausted(db_path: Path) -> bool: - """Whether *db_path* has already burned its cross-restart repair budget. - - True only when the ledger records ``_MAX_PERSISTENT_REPAIR_ATTEMPTS`` - failed attempts against the CURRENT file fingerprint. Never raises; a - missing/corrupt ledger or unstatable DB reads as "not exhausted" (the - in-process claim and cross-process lock still bound a single run). - - When the fingerprint is unavailable because a live connection makes the - content read unsafe, fall back to the SIZE the ledger recorded rather than - reading as "not exhausted". Otherwise a peer connection is enough to hide - an exhausted budget on every pass, which is the unbounded loop again. - """ - ledger = _read_repair_ledger(db_path) - recorded = ledger.get("fingerprint") - fp = _db_fingerprint(db_path) - if fp is None: - # Size is the one component both key shapes share and that a raw read - # is not needed for; an unchanged size means the damaged file is very - # likely the same one the budget was burned on. - try: - size_prefix = f"{db_path.stat().st_size}:" - except OSError: - return False - if not isinstance(recorded, str) or not recorded.startswith(size_prefix): - return False - elif recorded != fp: - return False - return int(ledger.get("failed_attempts", 0)) >= _MAX_PERSISTENT_REPAIR_ATTEMPTS - - -def _persistent_repair_exhausted_error(db_path: Path) -> str: - """The stable operator-facing diagnostic for an exhausted repair budget.""" - return ( - f"automatic repair has already failed " - f"{_MAX_PERSISTENT_REPAIR_ATTEMPTS} times on this exact file — " - "the corruption is beyond the schema/FTS repair strategies " - "(likely b-tree page damage). Manual recovery required: restore " - f"a backup, or salvage with `sqlite3 {db_path} \".recover\"`. " - f"Delete {_repair_ledger_path(db_path).name} to force another " - "automatic attempt." - ) - - -def _record_repair_outcome( - db_path: Path, *, repaired: bool, fingerprint: "Optional[str]" = None -) -> None: - """Update the persistent attempt ledger after a repair pass. Never raises. - - Defaults to the post-attempt fingerprint — the file state the NEXT - attempt's exhaustion probe will observe. - - When the fingerprint is unavailable (a live connection makes the content - read unsafe), keep the key already on record and still increment: dropping - the pass would let a peer connection reset the budget every time, which is - the unbounded loop this ledger exists to stop. Never write a differently - shaped key — the probe compares for equality, so mixing key shapes across - passes never matches. - """ - ledger_path = _repair_ledger_path(db_path) - try: - if repaired: - ledger_path.unlink(missing_ok=True) - return - ledger = _read_repair_ledger(db_path) - recorded = ledger.get("fingerprint") - fp = fingerprint if fingerprint is not None else _db_fingerprint(db_path) - if fp is None: - if not isinstance(recorded, str): - # No prior key to extend and no way to mint one safely: the - # in-process claim and cross-process lock still bound this run. - return - fp = recorded - attempts = ( - int(ledger.get("failed_attempts", 0)) + 1 if recorded == fp else 1 - ) - import datetime - - ledger_path.write_text( - json.dumps( - { - "fingerprint": fp, - "failed_attempts": attempts, - "last_attempt": datetime.datetime.now().isoformat( - timespec="seconds" - ), - } - ), - encoding="utf-8", - ) - except Exception as exc: # pragma: no cover - best effort - logger.warning("Could not update state.db repair ledger: %s", exc) - - -def _existing_malformed_backups(db_path: Path) -> "List[Path]": - """Timestamped forensic backups of *db_path*, newest first.""" - prefix = f"{db_path.name}.malformed-backup-" - try: - found = [ - p - for p in db_path.parent.iterdir() - if p.name.startswith(prefix) - and not p.name.endswith(_DB_SIDECAR_SUFFIXES) - ] - except OSError: - return [] - return sorted(found, key=lambda p: p.name, reverse=True) - - -def _prune_malformed_backups(db_path: Path, keep: int = _MAX_MALFORMED_BACKUPS) -> None: - """Delete all but the *keep* newest forensic backups (and sidecars).""" - for stale in _existing_malformed_backups(db_path)[keep:]: - for victim in ( - stale, - *(stale.with_name(stale.name + suffix) for suffix in _DB_SIDECAR_SUFFIXES), - ): - try: - victim.unlink(missing_ok=True) - except OSError as exc: # pragma: no cover - best effort - logger.warning("Could not prune stale DB backup %s: %s", victim, exc) - - -def _backup_db_file(db_path: Path) -> "Tuple[Optional[Path], Optional[str]]": - """Copy a (possibly malformed) DB file to a timestamped backup beside it. - Raw file copy on purpose: the DB won't open cleanly, so we preserve the - bytes exactly for forensics / manual restore. WAL, SHM and rollback-journal - sidecars are copied too when present. Returns ``(backup_path, None)`` on success or - ``(None, reason)`` on failure — callers on the repair path treat a - refused backup as a HARD STOP (see #69603). Repair strategies run on a - scratch snapshot, but the forensic bundle remains the recovery path when - corruption defeats them. - - Refuses when a connection to this database is still live in the process: - reading the file would ``close()`` a descriptor for it and cancel that - connection's POSIX advisory locks (see ``hermes_cli.sqlite_safe_read``). - The repair path can be entered by one SessionDB while the gateway holds - others, so this is a real possibility rather than a theoretical one. - """ - import datetime - import shutil - - try: - from hermes_cli.sqlite_safe_read import has_live_connection - except ImportError: - has_live_connection = None # type: ignore[assignment] - - if has_live_connection is not None and has_live_connection(db_path): - reason = ( - f"a connection to {db_path} is still open in this process; " - "raw-copying it would cancel that connection's POSIX advisory " - "locks. Close all SessionDB handles first." - ) - logger.error("Refusing to raw-copy %s for backup: %s", db_path, reason) - return None, reason - - stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - backup_path = db_path.with_name(f"{db_path.name}.malformed-backup-{stamp}") - # Same-second collision (two distinct damaged states within one second) - # must not silently overwrite the earlier forensic copy. - seq = 1 - while backup_path.exists(): - backup_path = db_path.with_name( - f"{db_path.name}.malformed-backup-{stamp}_{seq}" - ) - seq += 1 - try: - # Sweep staging debris from an earlier interrupted pass (kill mid-copy) - # BEFORE the dedupe below. A leftover staging file is a byte-identical - # copy of the damaged DB, so its fingerprint MATCHES and the dedupe - # would otherwise hand it back as a legitimate forensic backup. - # Matches sidecar staging names (``.backup-staging--wal``) too. - # The second pattern is the pre-merge ``.incomplete`` spelling, swept so - # a host that ran that build does not keep prefix-matching debris that - # sorts NEWEST and survives prune forever. - for pattern in ( - f"{db_path.name}.backup-staging-*", - f"{db_path.name}.malformed-backup-*.incomplete*", - ): - for old in db_path.parent.glob(pattern): - try: - old.unlink(missing_ok=True) - except OSError: # pragma: no cover - best effort - pass - # Dedupe (#86747): a repair loop used to copy the SAME damaged bytes - # on every restart — ~900MB a pass, 89GB over 11 days in the - # reporting install. If the newest existing backup is byte-identical to - # the current recovery image, reuse it. - # - # Matching on mtime made this dedupe miss exactly when it mattered - # most: the malformed-SCHEMA class still accepts writes, so live - # writers and the in-place repair strategies move mtime between - # passes and every pass wrote another full-size copy (2.3GB in 20 - # minutes). - # - # Use ``_backup_content_identity`` (whole file + sidecars), NOT the - # repair-epoch ``_db_fingerprint``. They are different equivalence - # relations: the fingerprint masks commit counters and samples only - # head/tail so an ordinary interior-page write does not re-key the - # repair budget — but that same write DOES change the recovery image, - # and deduping on the fingerprint would hand back a stale backup that - # predates the write. A forensic copy must prove byte identity, so it - # pays the O(n) read (cheaper than the O(n) write it avoids on a hit). - try: - # Only hash the source when there is actually a candidate to dedupe - # against — on the common first-corruption pass there is no prior - # backup, and hashing the (possibly multi-GB) source then would be - # pure waste right before the copy reads it again anyway. - existing_backups = _existing_malformed_backups(db_path)[:1] - if existing_backups: - src_id = _backup_content_identity(db_path) - for existing in existing_backups: - if src_id is not None and _backup_content_identity(existing) == src_id: - logger.info( - "Reusing existing forensic backup %s (identical to the " - "damaged DB).", existing, - ) - return existing, None - except OSError: - pass - # Disk guard: this is a full raw copy of a possibly multi-GB DB plus - # its sidecars. On a host whose volume is already nearly full — which - # a preceding repair loop may itself have caused — taking it can - # finish off the disk and take down every process on the machine. - # Refuse while there is still room to refuse in. - try: - need = db_path.stat().st_size - for suffix in _DB_SIDECAR_SUFFIXES: - sidecar = db_path.with_name(db_path.name + suffix) - if sidecar.exists(): - need += sidecar.stat().st_size - usage = shutil.disk_usage(db_path.parent) - headroom = _repair_backup_headroom_bytes(usage.total) - if usage.free - need < headroom: - reason = ( - f"only {usage.free / 1e9:.2f}GB free on {db_path.parent}; " - f"copying the damaged DB needs {need / 1e9:.2f}GB and must " - f"leave {headroom / 1e9:.2f}GB headroom. Free disk space, " - f"then retry (or recover manually with `sqlite3 {db_path} " - '".recover"`).' - ) - logger.error("Refusing forensic backup of %s: %s", db_path, reason) - return None, reason - except OSError as exc: - # Fail CLOSED. This guard exists for the nearly-full volume, which - # is exactly where stat()/disk_usage() is most likely to fail — and - # proceeding would take the multi-GB copy that finishes off the - # disk. A refused backup is a HARD STOP (#69603), so repair simply - # does not run until a human frees space, which is the safe side. - reason = ( - f"could not determine free space on {db_path.parent} ({exc}); " - "refusing the forensic copy rather than risk filling the " - f"volume. Free disk space, then retry (or recover manually " - f'with `sqlite3 {db_path} ".recover"`).' - ) - logger.error("Refusing forensic backup of %s: %s", db_path, reason) - return None, reason - # Copy to a staging name OUTSIDE the ``.malformed-backup-`` prefix, then - # rename into place only once every copy has succeeded. The prefix - # matters: ``_existing_malformed_backups`` matches on - # ``startswith(f"{db}.malformed-backup-")`` and excludes only ``-wal``/ - # ``-shm`` suffixes, so a staging name derived from the backup name (e.g. - # ``…malformed-backup-.incomplete``) still counts as a backup — - # it sorts NEWEST (``.incomplete`` > the bare stamp), so prune's - # keep-3-newest slice retained partials and deleted intact copies, and - # the dedupe could hand a partial back as the official ``backup_path``, - # passing the #69603 hard-stop gate with no real forensic copy on disk. - staging = db_path.with_name(f"{db_path.name}.backup-staging-{stamp}") - # (staging_src, final_dst) pairs. ORDER MATTERS for publication: the - # main-DB backup name is the bundle's commit marker — - # ``_existing_malformed_backups`` matches ``{db}.malformed-backup-*`` - # and excludes only the ``-wal``/``-shm``/``-journal`` suffixes, so the - # main file appearing is what makes the bundle "count". Sidecars are - # therefore staged/published FIRST and the main DB LAST, so a failure - # partway through never leaves a countable main backup standing over a - # missing sidecar (an incomplete recovery image that would pass the - # #69603 hard stop and dedupe as legitimate on the next pass). - staged_sidecars: "List[Tuple[Path, Path, Path]]" = [] - for suffix in _DB_SIDECAR_SUFFIXES: - sidecar = db_path.with_name(db_path.name + suffix) - if sidecar.exists(): - side_staging = staging.with_name(staging.name + suffix) - side_dst = backup_path.with_name(backup_path.name + suffix) - staged_sidecars.append((sidecar, side_staging, side_dst)) - main_pair = (staging, backup_path) - published: "List[Path]" = [] - all_staging_srcs = [staging] + [s for _src, s, _d in staged_sidecars] - try: - shutil.copy2(db_path, staging) - for sidecar, side_staging, _side_dst in staged_sidecars: - shutil.copy2(sidecar, side_staging) - # Publish sidecars first, main DB LAST (the commit marker), so a - # mid-publish failure never leaves a countable-but-incomplete bundle. - publish_order = [ - (s, d) for _src, s, d in staged_sidecars - ] + [main_pair] - for src, dst in publish_order: - os.replace(src, dst) - published.append(dst) - except Exception: - # Roll back BOTH unpublished staging files AND anything already - # promoted — the old code unlinked only staging srcs, so a failure - # after the main os.replace left the official backup_path on disk. - for src in all_staging_srcs: - try: - src.unlink(missing_ok=True) - except OSError: - pass - for dst in published: - try: - dst.unlink(missing_ok=True) - except OSError: - pass - try: - staging.unlink(missing_ok=True) - except OSError: - pass - raise - # Retention cap (#86747): keep only the newest few forensic copies. - _prune_malformed_backups(db_path) - return backup_path, None - except Exception as exc: # pragma: no cover - best effort - logger.warning("Could not back up malformed DB %s: %s", db_path, exc) - return None, f"backup copy failed: {exc}" - - -def preflight_db_writability( - db_path: Path, - *, - db_label: str = "state.db", -) -> None: - """Refuse-or-repair read-only DB files BEFORE the first connection opens. - - Port of Kilo-Org/kilocode#12508's startup preflight. A stray read-only - ``state.db`` / ``-wal`` / ``-shm`` (sudo run, restored backup, copied - dotfiles) previously surfaced as an opaque - ``sqlite3.OperationalError: attempt to write a readonly database`` raised - from deep inside ``_init_schema`` — naming no file and no fix — and the - obvious wrong "fix" (deleting the ``-wal``) silently loses committed - transactions. This preflight: - - - **Repairs** permissions with ``chmod u+rw`` when the file lives inside - the Hermes home tree (``get_hermes_home()``) — the safe repair scope: - Hermes owns those files, and the OS makes ``chmod`` fail on files the - user doesn't own, which bounds the repair exactly. - - **Fails fast with an actionable error** naming the exact file and the - exact ``chmod`` command for anything else (root-owned files, read-only - mounts, custom paths outside the home tree). - - Never deletes or truncates a WAL sidecar — once writable, the normal - open path checkpoints its committed frames into the DB as intended. - - ``:memory:`` and ``file:`` URI paths are skipped (no plain on-disk files - to check). Shared by :class:`SessionDB` and ``hermes_cli.kanban_db``. - """ - raw = str(db_path) - if raw == ":memory:" or raw.startswith("file:"): - return - - try: - home: Optional[Path] = Path(get_hermes_home()).resolve() - except Exception: # pragma: no cover - defensive - home = None - - def _in_repair_scope(p: Path) -> bool: - if home is None: - return False - try: - return p.resolve().is_relative_to(home) - except (OSError, ValueError): - return False - - def _ensure_writable(p: Path, *, is_dir: bool = False) -> None: - import stat as _stat - - if os.access(p, os.R_OK | os.W_OK): - return - if _in_repair_scope(p): - try: - add = _stat.S_IRUSR | _stat.S_IWUSR | (_stat.S_IXUSR if is_dir else 0) - os.chmod(p, p.stat().st_mode | add) - except OSError: - pass - if os.access(p, os.R_OK | os.W_OK): - logger.info( - "%s preflight: repaired read-only %s (chmod u+rw%s)", - db_label, - p, - "x" if is_dir else "", - ) - return - kind = "directory" if is_dir else "file" - wal_note = ( - " Do NOT delete the -wal file — it contains committed data that " - "will be merged into the database once it is writable." - if p.name.endswith("-wal") - else "" - ) - raise sqlite3.OperationalError( - f"{db_label} is not writable: {kind} {p} is read-only for this " - f"user. Hermes needs read-write access to open the database. " - f"Fix with: chmod u+rw{'x' if is_dir else ''} '{p}'" - f" (files owned by another user may need sudo/chown).{wal_note}" - ) - - parent = db_path.parent - if parent.is_dir(): - # SQLite needs a writable directory in every journal mode (WAL and - # SHM sidecars in WAL mode; the rollback journal in DELETE mode). - _ensure_writable(parent, is_dir=True) - - for suffix in ("", "-wal", "-shm"): - p = db_path.with_name(db_path.name + suffix) if suffix else db_path - if p.is_file(): - _ensure_writable(p) - - -def _connect_repair_durable( - db_path: Path, *, timeout: float = 5.0 -) -> sqlite3.Connection: - """``sqlite3.connect`` for the repair/probe paths, with macOS write barriers. - - These paths open ``state.db`` directly rather than through ``SessionDB`` - (which routes via :func:`apply_wal_with_fallback`), so they inherited - SQLite's ``synchronous=NORMAL`` default and no ``checkpoint_fullfsync``. - On Darwin that is exactly the combination :func:`_enforce_macos_synchronous_full` - exists to prevent: ``fsync()`` there guarantees neither data-on-platter nor - write ordering, so a rewrite interrupted by process or OS termination can - leave half-written b-tree pages behind. - - That matters more here than anywhere else in the module, because what runs - through these connections is ``REINDEX``, ``VACUUM`` and ``writable_schema`` - surgery — the operations that rewrite nearly every page of the file. The - 2026-08-19 recurrence tore ``messages`` (root page 5) and - ``idx_messages_session``, reporting the unmistakable signature: repeated - "2nd reference to page", a rowid out of order, and long runs of leaked - "never used" pages. - - Autocommit (``isolation_level=None``) is preserved: callers run DDL and - ``VACUUM``, which are illegal inside an implicit transaction. - - Applying the barriers is best-effort *by necessity*: SQLite loads the - schema before it runs any statement, so on a malformed schema even - ``PRAGMA synchronous=FULL`` raises ``DatabaseError`` ("malformed database - schema (messages_fts) - table messages_fts already exists"). A malformed - database is precisely this helper's input, so raising there would leave - repair unable to open the file it exists to fix. Strategies that go on to - rewrite the whole file call :func:`_reapply_durability_barriers` once the - schema parses again, which is the point at which the pragmas can stick. - """ - conn = sqlite3.connect(str(db_path), timeout=timeout, isolation_level=None) - _reapply_durability_barriers(conn) - return conn - - -def _reapply_durability_barriers(conn: sqlite3.Connection) -> bool: - """Best-effort (re)application of the macOS write barriers. Never raises. - - Returns True when the pragmas were accepted. Callers about to rewrite the - file wholesale (``VACUUM``, ``REINDEX``) should call this after the schema - becomes parseable, because a connection opened against a malformed schema - could not take them at open time. - """ - try: - _apply_macos_checkpoint_barrier(conn) - _enforce_macos_synchronous_full(conn) - return True - except sqlite3.DatabaseError: - # Schema still unparseable — the pragmas cannot be set yet. - return False - except Exception: - return False - - -def apply_durability_barriers(conn: sqlite3.Connection) -> bool: - """Apply state-store durability barriers without changing journal mode. - - This is the public entry point for secondary users of ``state.db`` that - must inherit its owner's journal mode while retaining per-connection - durability settings. Also applies the configured ``database.synchronous`` - level (a per-connection pragma that would otherwise only ride on the - journal-mode setup path guest connections must not run). - """ - ok = _reapply_durability_barriers(conn) - try: - # Local import avoids a circular import with hermes_cli.config. - from hermes_cli.config import cfg_get, load_config_readonly - - cfg = load_config_readonly() - raw_synchronous = cfg_get(cfg, "database", "synchronous", default=None) - if raw_synchronous is not None: - _apply_synchronous_pragma( - conn, raw_synchronous, db_label="state.db (guest)" - ) - except Exception: - pass - return ok - - -@contextmanager -def _exclusive_repair_db_guard(db_path: Path): - """Yield one live connection that excludes writers for repair surgery. - - ``locking_mode=EXCLUSIVE`` retains SQLite's file-level exclusion after the - short ``BEGIN EXCLUSIVE`` transaction is rolled back. That rollback is - essential: ``Connection.backup`` may use the guarded connection as a - *source* while it is transaction-free, and it must be transaction-free - when it is later the promotion *destination*. The connection itself - remains open for the entire snapshot -> strategies -> promotion window, - so another writer cannot commit a change that promotion could overwrite. - - Existing WAL readers make exclusive acquisition fail rather than being - disturbed. In DELETE mode an existing reader similarly prevents - ``BEGIN EXCLUSIVE``; a future reader/writer waits behind the guard. A - repair therefore fails closed whenever this process cannot own that whole - window. - """ - guard: Optional[sqlite3.Connection] = None - try: - # The cross-process repair lock already serializes repairers. Do not - # wait behind an ordinary application connection: a partial repair is - # less safe than an explicit "stop the gateway and retry" result. - guard = _connect_repair_durable(db_path, timeout=0.0) - guard.execute("PRAGMA locking_mode=EXCLUSIVE") - guard.execute("BEGIN EXCLUSIVE") - guard.execute("ROLLBACK") - except (sqlite3.Error, OSError) as exc: - if guard is not None: - try: - guard.execute("PRAGMA locking_mode=NORMAL") - except Exception: - pass - guard.close() - yield None, exc - return - - try: - yield guard, None - finally: - try: - # Let SQLite release the exclusive locks before close; this also - # avoids a connection-close checkpoint being mistaken for a - # repair write in callers that immediately reopen state.db. - guard.execute("PRAGMA locking_mode=NORMAL") - except Exception: - pass - guard.close() - - -def _copy_database_snapshot( - source_path: Path, - destination_path: Path, - *, - source_connection: Optional[sqlite3.Connection] = None, - destination_connection: Optional[sqlite3.Connection] = None, -) -> None: - """Copy one complete SQLite snapshot without replacing either file inode. - - SQLite's online backup API incorporates committed WAL frames into the - source snapshot and writes the destination inside one transaction. This - avoids both the main-file-only staging gap and replacing ``state.db`` from - under handles that already refer to it. If backup is interrupted, SQLite - rolls the destination transaction back. - """ - # Work out the deadline before opening an owned source connection. A - # sidecar disappearing while we stat it is an ordinary staging failure, - # but it must not leak a just-opened SQLite descriptor. - deadline_seconds = _repair_snapshot_timeout_seconds(source_path) - deadline = time.monotonic() + deadline_seconds - source = source_connection or _connect_repair_durable(source_path) - destination = destination_connection - own_source = source_connection is None - own_destination = destination_connection is None - - def _check_deadline(_status: int, _remaining: int, _total: int) -> None: - if time.monotonic() >= deadline: - raise TimeoutError( - "timed out copying SQLite repair snapshot after " - f"{deadline_seconds:.0f}s" - ) - - try: - if destination is None: - destination = _connect_repair_durable(destination_path) - elif destination.in_transaction: - # sqlite3_backup requires a transaction-free destination. The - # exclusive repair guard deliberately retains file exclusion via - # locking_mode, not an active transaction, so it satisfies this. - raise sqlite3.ProgrammingError( - "SQLite repair backup destination has an active transaction" - ) - source.backup( - destination, - pages=256, - progress=_check_deadline, - sleep=_REPAIR_LOCK_POLL_SECONDS, - ) - finally: - if own_destination and destination is not None: - destination.close() - if own_source: - source.close() - - -def _db_opens_cleanly(db_path: Path) -> Optional[str]: - """Probe a DB on a fresh connection. Returns None if healthy, else a reason. - - Runs the same first-statement (``PRAGMA journal_mode``) that trips the - malformed-schema parse, then ``PRAGMA integrity_check`` and a canonical - ``sessions`` read, and finally a rolled-back ``messages`` write so that - FTS5 index corruption — which leaves base-table reads and - ``integrity_check`` passing while every ``INSERT INTO messages`` fails - through the FTS triggers — is reported as unhealthy rather than slipping - past as a false "ok" (#50502). - """ - conn = _connect_repair_durable(db_path) - try: - # Best-effort tokenizer load: a DB carrying the messages_fts_cjk - # index needs the cjk_unicode61 extension before any statement can - # touch that table — including the trigger-driven write probe below. - # Without it, this probe sees the DB exactly as a tokenizer-less - # SessionDB open would (which drops the cjk triggers to keep writes - # working), so tokenizer absence must never classify as corruption. - load_fts5_cjk_extension(conn) - conn.execute("PRAGMA journal_mode").fetchone() - rows = conn.execute("PRAGMA integrity_check").fetchall() - problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != "ok"] - if problems: - return "; ".join(problems[:3]) - conn.execute("SELECT COUNT(*) FROM sessions").fetchone() - - # FTS5 read probe: run a representative MATCH query against the - # messages_fts* virtual tables. The FTS *write* probe below catches - # the corruption class where base tables read fine but writes fail - # through the triggers (#50502). It does NOT catch partial FTS5 - # index corruption — bad shadow-table segments where reads still - # parse but MATCH / snippet / rank queries error out with - # "database disk image is malformed" (a `sqlite3.DatabaseError`, - # not `OperationalError`). session_search, /resume title resolution, - # and any feature relying on FTS5 discovery then break silently - # because the official repair tool's check-only path reports the - # DB as healthy. #66724. - # Catch the full sqlite3 exception hierarchy (not just - # OperationalError) so the malformed-shadow-table class is reported - # rather than letting it crash the caller. - for fts_table in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"): - try: - # No-op queries against the actual FTS5 APIs the search - # tools use. The trigram table is included because it backs - # the title-resolution path; either corruption mode would - # break session recall without this probe. MATCH '""' is - # the empty phrase-token probe — FTS5 rejects MATCH '' - # outright ("fts5: syntax error"), but a quoted empty - # phrase parses, scans zero rows, and exercises the same - # shadow-table read path the search tools use. - conn.execute( - f"SELECT 1 FROM {fts_table} WHERE {fts_table} MATCH '\"\"' LIMIT 1" - ).fetchone() - except sqlite3.OperationalError as exc: - # Use the canonical capability classifier instead of a - # hand-rolled substring check. On SQLite builds without the - # fts5 module, the legacy messages_fts table may exist on - # disk (from a prior build that had FTS5) and MATCH queries - # against it raise OperationalError("no such module: fts5"); - # the substring check below would misclassify that as - # corruption and send the DB into the repair path, whose - # final fallback deletes the messages_fts% schema - # (hermes_state.py:645-723). The supported degraded-runtime - # path (SessionDB._is_fts5_unavailable_error + the - # regression suite in tests/test_hermes_state.py:600-632) - # treats both "no such module: fts5" and - # "no such tokenizer: trigram" as the capability error. - if SessionDB._is_fts5_unavailable_error(exc): - # Degraded runtime — not the corruption class we probe. - continue - msg = str(exc).lower() - if "no such table" in msg or "no such column" in msg: - # FTS5 not built yet (brand new file mid-init) — not the - # corruption class we probe. - continue - return f"fts5 read probe failed on {fts_table}: {exc}" - except sqlite3.DatabaseError as exc: - # This is the corruption class #66724 actually wants caught: - # partial shadow-table damage where MATCH / snippet / rank - # queries raise DatabaseError("database disk image is malformed") - # while reads of the FTS5 table itself parse fine. - return f"fts5 read probe failed on {fts_table}: {exc}" - - # FTS write probe: drive a row through the messages_fts* triggers in a - # transaction that is always rolled back, so a corrupt FTS index that - # rejects writes is caught even though reads look healthy. The probe is - # best-effort — if the messages/sessions tables don't exist yet (brand - # new file mid-init) the OperationalError is treated as "not yet a - # populated DB", not corruption. - probe_session_id = f"_hermes_fts_health_probe_{time.time_ns()}" - try: - conn.execute("BEGIN IMMEDIATE") - conn.execute( - "INSERT INTO sessions (id, source, started_at) VALUES (?, ?, ?)", - (probe_session_id, "_health_probe", time.time()), - ) - conn.execute( - "INSERT INTO messages (session_id, role, content, timestamp) " - "VALUES (?, ?, ?, ?)", - (probe_session_id, "user", "_fts_health_probe", time.time()), - ) - conn.execute("ROLLBACK") - except sqlite3.OperationalError as exc: - # Missing tables / FTS disabled — not the corruption class we probe. - try: - conn.execute("ROLLBACK") - except sqlite3.Error: - pass - msg = str(exc).lower() - if "no such table" in msg or "no such column" in msg: - return None - if "no such tokenizer: cjk_unicode61" in msg: - # This probe process couldn't load the cjk extension while - # the DB carries the cjk index — capability gap, not - # corruption. A tokenizer-capable SessionDB serves it fine; - # a tokenizer-less one self-heals by dropping the triggers. - return None - return str(exc) - return None - except sqlite3.DatabaseError as exc: - return str(exc) - finally: - conn.close() - - -def _live_writer_holds_db(db_path: Path) -> bool: - """True when a connection outside this call still holds ``db_path`` open. - - Detection works by asking SQLite for the thing a repair actually needs and - a live writer cannot grant: ``PRAGMA locking_mode=EXCLUSIVE`` followed by - ``BEGIN IMMEDIATE``. In WAL mode, entering exclusive locking mode - requires exclusive locks on the WAL index, so any other open connection — - reader or writer — makes it fail with SQLITE_BUSY. Neither statement - parses the schema, so this works on the malformed databases repair exists - to handle. - - Fails **open** (returns False) on anything other than a positive - busy/locked signal: refusing to repair a database that nobody is actually - holding would strand the very self-heal path this guard protects. - - Scope: the WAL-index exclusive lock is what makes this detect a holder, so - the guard is effective in WAL mode. On SQLite builds carrying the WAL-reset - bug and on NFS/SMB, Hermes deliberately runs ``state.db`` in - ``journal_mode=DELETE`` (see :func:`apply_wal_with_fallback`); there a held - reader takes only a SHARED lock, ``BEGIN IMMEDIATE`` still acquires - RESERVED, and this probe returns False. In that mode repair is serialised - only by the cross-process repairer lock rather than by this holder probe. - The 2026-08 incident that motivated the guard was in WAL mode, which this - covers; broadening detection to DELETE mode is left to a follow-up. - """ - probe = None - try: - probe = _connect_repair_durable(db_path, timeout=0.0) - probe.execute("PRAGMA locking_mode=EXCLUSIVE") - probe.execute("BEGIN IMMEDIATE") - probe.execute("ROLLBACK") - return False - except sqlite3.OperationalError as exc: - lowered = str(exc).lower() - return "locked" in lowered or "busy" in lowered - except sqlite3.DatabaseError: - # Malformed/unreadable: no evidence of a live holder either way. - return False - except Exception: - return False - finally: - if probe is not None: - try: - # Drop exclusive locking mode before closing so the probe - # itself never leaves the file pinned. - probe.execute("PRAGMA locking_mode=NORMAL") - except Exception: - pass - try: - probe.close() - except Exception: - pass - - -def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, Any]: - """Repair a state.db whose ``sqlite_master`` schema is malformed or whose - FTS indexes reject writes. - - Handles two corruption classes: the "duplicate object definition" / - malformed-schema class where even ``PRAGMA`` statements fail, and the FTS - write-corruption class (#50502) where base tables read fine and - ``integrity_check`` passes but writes fail through the ``messages_fts*`` - triggers. Tries least-destructive recovery first and escalates: - - 1. **Rebuild FTS indexes in place** via the FTS5 ``'rebuild'`` command, - which rewrites the internal b-tree segments from the canonical - ``messages`` rows without dropping or recreating anything. Fixes the - FTS write-corruption class while preserving the schema intact. - 2. **De-duplicate** ``sqlite_master`` (keep the lowest rowid per - ``type``/``name``). Fixes the canonical "table X already exists" - case and PRESERVES the existing FTS index intact. - 3. **Drop the FTS schema** (every ``messages_fts*`` object) + ``VACUUM``. - The next ``SessionDB()`` open rebuilds the FTS indexes from the - canonical ``messages`` table. - - Canonical ``sessions`` / ``messages`` rows are never modified by a failed - attempt. Mutating strategies run against a complete SQLite snapshot and a - successful result is copied back transactionally. A timestamped raw backup - is taken first unless ``backup=False``. - - The surgery below is serialised across processes (see - :func:`_cross_process_repair_lock`): the gateway service, the Desktop - app's backend and interactive CLI sessions all open the same file, and - two of them running ``writable_schema`` surgery concurrently is itself a - corruption source. - - Returns a report dict: ``{repaired: bool, strategy: str|None, - backup_path: str|None, error: str|None}``. - """ - report: Dict[str, Any] = { - "repaired": False, - "strategy": None, - "backup_path": None, - "error": None, - } - - # Startup-watchdog progress lease: repair (raw backup copy + surgery + - # VACUUM) is I/O-bound — near-zero CPU on a multi-GB file — which the - # watchdog's CPU fallback would misread as a parked deadlock (OOF-298). - # Single lease is deliberate (clamped to _MAX_LEASE_S=900): honest worst - # case is up to the lease duration of zombie time on a wedged repair, - # accepted over per-chunk renewal complexity in the repair loop. - report_startup_progress(900.0, phase="state_db_repair") - - db_path = Path(db_path) - if not db_path.exists(): - report["error"] = f"{db_path} does not exist" - return report - - # Cross-restart attempt cap (#86747): the in-memory claim bounds one - # process, but a corruption class the strategies below cannot heal - # (b-tree page damage) previously re-ran the whole surgery — and took a - # fresh multi-hundred-MB forensic backup — on EVERY restart, forever. - # After _MAX_PERSISTENT_REPAIR_ATTEMPTS failures against the same - # damaged file, stop retrying and surface a terminal, actionable error. - if _persistent_repair_attempts_exhausted(db_path): - report["error"] = _persistent_repair_exhausted_error(db_path) - logger.error("state.db repair skipped: %s", report["error"]) - return report - - result = report - with _cross_process_repair_lock(db_path) as holding_lock: - if not holding_lock: - # Another process is still inside its critical section, or the - # lock file itself could not be opened (full disk / no fds). It - # may nonetheless have healed the file already (long VACUUM after - # a successful strategy), so re-probe before reporting failure. - if _db_opens_cleanly(db_path) is None: - report["repaired"] = True - report["strategy"] = "repaired_by_other_process" - else: - report["error"] = ( - "could not obtain the state.db repair lock (held by " - "another process, or the lock file was unopenable); " - "skipped schema surgery to avoid racing a concurrent " - "repairer" - ) - else: - # The fast check above avoids taking the lock for a known-exhausted - # image. Recheck after acquisition: a queued repairer can have - # recorded the final failure while this process waited, and this - # process must not start a fourth attempt. - if _persistent_repair_attempts_exhausted(db_path): - report["error"] = _persistent_repair_exhausted_error(db_path) - logger.error("state.db repair skipped: %s", report["error"]) - # Keep the existing WAL-holder preflight: it preserves the - # established fail-closed behaviour for active readers before we - # create a forensic backup. It is not the race defence; the - # retained exclusive guard inside the locked routine is what - # excludes writers continuously through promotion. DELETE-mode - # readers which this probe cannot see are still rejected by the - # later BEGIN EXCLUSIVE acquisition. - elif _live_writer_holds_db(db_path): - report["error"] = ( - "a live writer still holds state.db; skipped schema surgery " - "to avoid tearing b-tree pages under a concurrent writer. " - "Stop the gateway (hermes gateway stop) and retry." - ) - logger.error("state.db repair skipped: %s", report["error"]) - else: - # Probe the mode BEFORE surgery (#89674): every repair - # strategy rewrites the file, and a rebuilt SQLite file comes - # back in the default journal mode (delete) — silently moving - # a WAL store out of WAL with nothing in the logs recording - # the flip. The open-time WAL-reset gate never sees this flip - # because it happens inside the repair path (distinct from - # the open-time flip #89393 warns about). A probe of the - # damaged file may fail, in which case the canonical - # database.journal_mode setting is the restore target. - before_mode = _probe_journal_mode_for_repair(db_path) - result = _repair_state_db_schema_locked( - db_path, backup=backup, report=report - ) - if result.get("repaired"): - result["journal_mode_before"] = before_mode - _restore_journal_mode_after_repair(db_path, before_mode) - # Environmental aborts happen before a strategy gets to mutate the - # isolated snapshot. They are retriable operating conditions, not - # proof that the damaged database exhausted a repair strategy. - # Keep that private signal out of the public report while - # successful health checks still clear a stale persistent failure - # record. This ledger update stays under the same cross-process - # lock as surgery, so two repairers cannot lose each other's - # attempt updates. A queued loser must not record at all: its - # owner is responsible for its outcome. - attempted = bool(result.pop("_repair_attempted", False)) - if attempted or result.get("repaired"): - _record_repair_outcome( - db_path, repaired=bool(result.get("repaired")) - ) - return result - - -def _probe_journal_mode_for_repair(db_path: Path) -> Optional[str]: - """Best-effort journal-mode probe for a (possibly malformed) DB file. - - Returns the on-disk mode (``wal``/``delete``), or ``None`` when the file - cannot be opened or probed — a malformed header or a concurrent opener's - locks are both expected on the repair path. Callers fall back to the - configured ``database.journal_mode`` for ``None``. - """ - try: - conn = _connect_repair_durable(db_path) - try: - return _on_disk_journal_mode(conn) - finally: - conn.close() - except (sqlite3.Error, OSError): - return None - - -def _restore_journal_mode_after_repair(db_path: Path, before_mode: Optional[str]) -> None: - """Re-apply the journal mode after schema surgery (#89674). - - A repaired/rebuilt SQLite file comes back in the default journal mode - (delete). Without this restore, a corruption event deterministically - moves a WAL store out of WAL and nothing records the change — the - WAL-reset gate at open time never sees the flip because it happened - inside the repair path, not at open (the open-time flip #89393 warns - about is a different door). - - The restore runs through :func:`apply_wal_with_fallback` — the canonical - journal-mode path — rather than issuing a switch pragma directly, so it - inherits the vulnerable-SQLite WAL-reset gate (a rebuilt file IS a new - database: on a vulnerable runtime the gate deliberately keeps it in - DELETE, and "restore could not reach WAL" there is the expected outcome, - not a failure), the macOS-NFS silent-refusal handling, and the WAL - companions (size limit, checkpoint barrier, synchronous=FULL) that the - front door applies. ``before_mode`` is the pre-surgery probe (None when - the damaged file could not be probed) and is only used for the log - comparison — the restore target itself is whatever the canonical path - resolves from ``database.journal_mode``. - - Best-effort by design: the repair itself already succeeded, so failures - to re-apply are logged at WARNING, never raised. - """ - try: - conn = _connect_repair_durable(db_path) - try: - after = apply_wal_with_fallback(conn, db_label=db_path.name) - finally: - conn.close() - if before_mode and after != before_mode: - logger.warning( - "state.db repair changed journal_mode %r -> %r " - "(pre-surgery probe %r; restore resolved through " - "apply_wal_with_fallback per database.journal_mode and the " - "WAL-reset gate)", - before_mode, after, before_mode, - ) - except (sqlite3.Error, OSError) as exc: - logger.warning( - "state.db repair at %s: post-surgery journal-mode restore " - "failed (%s); verify with PRAGMA journal_mode on the next open", - db_path, exc, - ) - - -def _repair_state_db_schema_locked( - db_path: Path, *, backup: bool, report: Dict[str, Any] -) -> Dict[str, Any]: - """Repair strategies for :func:`repair_state_db_schema`. - - Caller must hold the cross-process repair lock for *db_path*. - - The strategies run on a SCRATCH COPY and the result is copied back through - SQLite's transactional backup API only once it is proven to open cleanly. - A repair that does not succeed therefore cannot modify or lose committed - canonical data. In WAL mode SQLite may checkpoint already-committed WAL - frames into the main file while the exclusive guard is released; that is - not a repair mutation and does not change the committed database image. - - They used to run in place, and Strategy 2 ends in ``VACUUM``. VACUUM does - not preserve what it cannot parse: it rebuilds the file from the schema - SQLite can still read, so when the damage IS in the schema b-tree — page - 1's child pointers resolving to data pages, which is exactly the - ``malformed database schema ()`` class this function exists to handle — - every table hanging off the unreadable part is silently dropped. The probe - afterwards then correctly reports the file is STILL malformed, so the - function returns ``repaired=False`` and advises a manual restore, having - already destroyed the thing it was asked to save. Destroying the data and - reporting the repair failed are not mutually exclusive outcomes, and - nothing here treated them as a contradiction. - - The pre-repair backup (#69603) does not close this: it is a forensic - artefact that nothing reads back, so recovery still depends on a human - noticing a ``.malformed-backup-*`` file and knowing what to do with it. - Not mutating the original in the first place is the property that holds - without a human in the loop. - """ - scratch = db_path.with_name(f"{db_path.name}.repair-scratch") - cleanup_error = _unlink_db_triple(scratch) - if cleanup_error is not None: - report["error"] = ( - "could not remove a stale repair snapshot before probing state.db: " - f"{cleanup_error}" - ) - logger.error("state.db repair aborted: %s", report["error"]) - return report - - # Re-probe under the lock: a process we queued behind may have just - # repaired the file, in which case redoing the surgery would undo its - # work on a now-healthy DB (the repair/re-corrupt cascade this lock - # exists to break). - if _db_opens_cleanly(db_path) is None: - report["repaired"] = True - report["strategy"] = "already_healthy" - return report - - if backup: - bpath, backup_error = _backup_db_file(db_path) - report["backup_path"] = str(bpath) if bpath else None - if bpath is None: - # HARD STOP (#69603). The forensic recovery image remains required - # when corruption defeats every strategy, even though the - # strategies themselves now run against an isolated snapshot. - report["error"] = ( - "pre-repair backup refused; aborting schema repair to avoid " - f"mutating the only copy of the damaged DB: {backup_error}" - ) - logger.error("state.db repair aborted: %s", report["error"]) - return report - - # The forensic copy intentionally happens before this guard: its raw-copy - # safety checks inspect real live holders and would be poisoned by our - # exclusive connection. Everything that can affect the repair image or - # live promotion happens only after writer exclusion is held. - with _exclusive_repair_db_guard(db_path) as (live_guard, guard_error): - if live_guard is None: - report["error"] = ( - "could not acquire exclusive state.db repair ownership; " - "skipped schema surgery to avoid overwriting a concurrent " - f"writer. Stop the gateway and retry: {guard_error}" - ) - if guard_error is not None and _repair_failure_consumes_attempt( - guard_error - ): - report["_repair_attempted"] = True - logger.error("state.db repair skipped: %s", report["error"]) - return report - - space_error = _repair_scratch_space_error(db_path) - if space_error is not None: - report["error"] = space_error - logger.error("state.db repair aborted: %s", report["error"]) - return report - - try: - # Reuse live_guard rather than opening a second source connection: - # the guard owns the exclusion, so a second connection could be - # blocked by our own EXCLUSIVE lock on some SQLite builds. - _copy_database_snapshot( - db_path, scratch, source_connection=live_guard - ) - except (OSError, sqlite3.Error, TimeoutError) as exc: - report["error"] = ( - f"could not stage a complete SQLite repair snapshot of {db_path}: {exc}" - ) - if _repair_failure_consumes_attempt(exc): - report["_repair_attempted"] = True - logger.error("state.db repair aborted: %s", report["error"]) - _unlink_db_triple(scratch) - return report - - try: - # This private marker is consumed by the outer wrapper. A strategy - # failure is a genuine repair outcome and consumes the persistent - # budget. A later promotion failure is classified separately: - # full disks, I/O, permission and lock failures are environmental - # aborts, not evidence that the strategy cannot repair this image. - report["_repair_attempted"] = True - _run_repair_strategies(scratch, report) - if report.get("repaired"): - try: - # Do not os.replace the live DB: Windows rejects - # replacement under open handles, while POSIX would leave - # those handles on the old inode. The same transaction- - # free guard that staged the live image receives the - # promotion, retaining writer exclusion throughout. - _copy_database_snapshot( - scratch, - db_path, - destination_connection=live_guard, - ) - except (OSError, sqlite3.Error, TimeoutError) as exc: - report["repaired"] = False - report["strategy"] = None - report["_repair_attempted"] = _repair_failure_consumes_attempt( - exc - ) - report["error"] = ( - "repaired snapshot could not be promoted transactionally: " - f"{exc}" - ) - logger.error("state.db repair promotion failed: %s", exc) - else: - logger.warning( - "state.db repaired via '%s' and promoted transactionally: %s", - report.get("strategy"), - db_path, - ) - if not report.get("repaired"): - # Logged HERE, not inside the strategies: they run against the - # scratch copy, and naming that throwaway path in the one - # message a human is meant to act on would send them to a file - # that no longer exists by the time they read it. - logger.error( - "state.db schema repair could not recover %s automatically " - "(no committed canonical data was modified or lost; backup: %s); " - "manual restore from backup may be required.", - db_path, - report["backup_path"], - ) - return report - finally: - # Never leave a half-repaired file beside the DB for a later probe - # — or a later human — to mistake for the real thing. - cleanup_error = _unlink_db_triple(scratch) - if cleanup_error is not None: - logger.warning( - "Could not remove state.db repair snapshot after repair: %s", - cleanup_error, - ) - - -def _unlink_db_triple(path: Path) -> Optional[str]: - """Remove *path* and every SQLite sidecar; return any cleanup failure.""" - failures: List[str] = [] - for suffix in ("", *_DB_SIDECAR_SUFFIXES): - victim = path if not suffix else path.with_name(path.name + suffix) - for attempt in range(10): - try: - victim.unlink() - break - except FileNotFoundError: - break - except PermissionError as exc: - # Windows may retain a just-closed SQLite handle for a few - # scheduler ticks. Bound the retry; a later backup open still - # fails safely if the handle truly remains live. - if _IS_WINDOWS and attempt < 9: - time.sleep(0.05) - continue - failures.append(f"{victim}: {exc}") - break - except OSError as exc: - failures.append(f"{victim}: {exc}") - break - return "; ".join(failures) or None - - -def _run_repair_strategies( - db_path: Path, report: Dict[str, Any] -) -> Dict[str, Any]: - """Escalating repair attempts, applied to *db_path* IN PLACE. - - Every strategy here mutates its argument — FTS rebuild, REINDEX, - ``writable_schema`` surgery, ``VACUUM``. It is therefore only ever called - by :func:`_repair_state_db_schema_locked` on a scratch copy that nothing - else holds open, never on the user's database. - """ - # ── Strategy 0: rebuild FTS indexes in place (FTS write-corruption) ── - # The FTS5 'rebuild' command rewrites the internal index from the canonical - # content table. This is the recommended, least-destructive recovery for a - # corrupt FTS index that rejects message writes while reads still succeed. - try: - conn = _connect_repair_durable(db_path) - try: - # The cjk index can only be rebuilt with its tokenizer loaded; - # best-effort (a tokenizer-less host skips it at the probe below). - load_fts5_cjk_extension(conn) - for table_name in ( - "messages_fts", "messages_fts_trigram", "messages_fts_cjk" - ): - try: - conn.execute( - f"INSERT INTO {table_name}({table_name}) VALUES('rebuild')" - ) - except sqlite3.OperationalError: - # Table absent (FTS disabled / trigram off / cjk not - # present or tokenizer unavailable) — skip it. - continue - finally: - conn.close() - if _db_opens_cleanly(db_path) is None: - report["repaired"] = True - report["strategy"] = "rebuild_fts" - logger.warning( - "state.db FTS indexes rebuilt in place (schema preserved): %s", - db_path, - ) - return report - except sqlite3.DatabaseError as exc: - logger.warning("state.db FTS in-place rebuild pass failed: %s", exc) - - # ── Strategy 0.5: rebuild stale B-tree indexes (#63386) ── - # PRAGMA integrity_check can report "wrong # of entries in index" when a - # B-tree index (e.g. idx_sessions_handoff_state) falls out of sync with its - # base table. REINDEX rewrites the index b-tree from the canonical table - # rows using the existing index definition, fixing the mismatch without - # touching data or FTS schema. - try: - conn = _connect_repair_durable(db_path) - try: - # REINDEX rewrites every index b-tree; take the barriers now that - # the schema parses, in case the open-time attempt was refused. - _reapply_durability_barriers(conn) - conn.execute("REINDEX") - conn.commit() - finally: - conn.close() - if _db_opens_cleanly(db_path) is None: - report["repaired"] = True - report["strategy"] = "reindex_btree" - logger.warning( - "state.db B-tree indexes rebuilt via REINDEX: %s", db_path - ) - return report - except sqlite3.DatabaseError as exc: - logger.warning("state.db REINDEX pass failed: %s", exc) - - # ── Strategy 1: de-duplicate sqlite_master (keeps FTS index) ── - try: - conn = _connect_repair_durable(db_path) - try: - conn.execute("PRAGMA writable_schema=ON") - dupes = conn.execute( - "SELECT type, name, COUNT(*) AS c, MIN(rowid) AS keep " - "FROM sqlite_master GROUP BY type, name HAVING c > 1" - ).fetchall() - for type_, name, _count, keep in dupes: - conn.execute( - "DELETE FROM sqlite_master " - "WHERE type IS ? AND name IS ? AND rowid <> ?", - (type_, name, keep), - ) - if dupes: - _bump_schema_cookie(conn) - conn.execute("PRAGMA writable_schema=OFF") - conn.commit() - finally: - conn.close() - if _db_opens_cleanly(db_path) is None: - report["repaired"] = True - report["strategy"] = "dedup_schema" - logger.warning( - "state.db schema repaired by de-duplicating sqlite_master " - "(FTS index preserved): %s", db_path - ) - return report - except sqlite3.DatabaseError as exc: - logger.warning("state.db dedup repair pass failed: %s", exc) - - # ── Strategy 2: drop all FTS schema, VACUUM, rebuild on next open ── - # - # The destructive one, and the reason this whole path now runs on a - # scratch copy. VACUUM rebuilds the file from the schema SQLite can still - # parse, so on a damaged schema b-tree it silently drops every table - # hanging off the unreadable part — and the probe below then correctly - # reports the result is still malformed. On a scratch copy that is merely - # a discarded attempt; on the live file it was data loss. - try: - conn = _connect_repair_durable(db_path) - try: - conn.execute("PRAGMA writable_schema=ON") - conn.execute("DELETE FROM sqlite_master WHERE name LIKE 'messages_fts%'") - _bump_schema_cookie(conn) - conn.execute("PRAGMA writable_schema=OFF") - conn.commit() - # The schema is repaired and parseable now, so the barriers can - # finally stick — and VACUUM, which rewrites the entire file, is - # the single most damaging operation to lose halfway. - _reapply_durability_barriers(conn) - conn.execute("VACUUM") - finally: - conn.close() - reason = _db_opens_cleanly(db_path) - if reason is None: - report["repaired"] = True - report["strategy"] = "drop_fts_rebuild" - logger.warning( - "state.db schema repaired by dropping FTS schema; indexes " - "will rebuild from messages on next open: %s", db_path - ) - return report - report["error"] = reason - except sqlite3.DatabaseError as exc: - report["error"] = str(exc) - - # The "could not recover" log lives in the caller: it must name the user's - # database, not the scratch copy these strategies were handed. - return report - # ── CJK-bigram FTS index (replaces the trigram index when available) ──── # @@ -4470,186 +1768,6 @@ def divert_session_transcript_jsonl(session_id: str, messages) -> "Optional[Path return path -# _read_sqlite_application_id runs on EVERY write via _raise_if_db_replaced, -# against the LIVE state.db. A bare open()/read()/close() there is the -# howtocorrupt §2.2 bug: close() cancels every POSIX advisory lock this -# process holds on the file — measured on Linux/SQLite 3.53.1, one probe call -# drops the WAL-mode DMS shared lock the writer connection holds on state.db -# (see hermes_cli/sqlite_safe_read.py for the module built around this rule). -# With the DMS lock gone, a fresh opener in another process can treat this -# writer as dead and rerun WAL-index recovery underneath it. -# -# The probe therefore reads through a per-path fd cached for the life of the -# process: opening an fd never cancels locks (only close() does), and -# os.pread takes no shared file position. When the path is re-pointed at a -# new inode (the very replacement this probe exists to detect), the stale fd -# is RETIRED, never closed — closing it would cancel the live connection's -# locks on the old file, the exact bug being avoided. Replacement events are -# rare and halt writes anyway, so the leak is bounded. -_HEADER_PROBE_LOCK = threading.Lock() -_HEADER_PROBE_FDS: "dict[str, tuple[int, int, int]]" = {} # key -> (fd, dev, ino) -_RETIRED_HEADER_PROBE_FDS: "list[int]" = [] # intentionally never closed - - -def _pread_db_header(db_path: Path, length: int) -> "Optional[bytes]": - """Lock-safe raw header read of a possibly-live SQLite database. - - POSIX: pread from a cached, never-closed fd (rebound when the path names - a new inode). Windows: plain read — advisory-lock cancellation is a - POSIX-only hazard and msvcrt locks do not share the failure mode. - """ - if _IS_WINDOWS: - try: - with db_path.open("rb") as handle: - return handle.read(length) - except OSError: - return None - key = str(db_path) - try: - st = os.stat(db_path) - except OSError: - return None - with _HEADER_PROBE_LOCK: - cached = _HEADER_PROBE_FDS.get(key) - if cached is not None and (cached[1], cached[2]) != (st.st_dev, st.st_ino): - # Path re-pointed at a new file. Retire (never close) the old fd. - _RETIRED_HEADER_PROBE_FDS.append(cached[0]) - cached = None - del _HEADER_PROBE_FDS[key] - if cached is None: - try: - fd = os.open(db_path, os.O_RDONLY) - except OSError: - return None - try: - fst = os.fstat(fd) - except OSError: - _RETIRED_HEADER_PROBE_FDS.append(fd) - return None - cached = (fd, fst.st_dev, fst.st_ino) - _HEADER_PROBE_FDS[key] = cached - try: - return os.pread(cached[0], length, 0) - except OSError: - return None - - -def _read_sqlite_application_id(db_path: Path) -> "Optional[int]": - """Read application_id from the SQLite header without opening a connection. - - Safe against live databases: routed through :func:`_pread_db_header`, - which never issues a ``close()`` that would cancel this process's POSIX - locks on the file (howtocorrupt §2.2). - """ - header = _pread_db_header(db_path, _STATE_DB_APPLICATION_ID_OFFSET + 4) - if header is None: - return None - if len(header) < _STATE_DB_APPLICATION_ID_OFFSET + 4: - return None - if header[:16] != b"SQLite format 3\x00": - return None - return int( - struct.unpack( - ">I", - header[_STATE_DB_APPLICATION_ID_OFFSET:_STATE_DB_APPLICATION_ID_OFFSET + 4], - )[0] - ) - - -def _stat_db_file_identity(path: Path) -> "Optional[tuple]": - """Return ``(st_dev, st_ino)`` for *path*, or None when identity is unavailable.""" - try: - st = os.stat(path) - except OSError: - return None - # Windows volumes (and some network FS) report st_ino=0; a (0, 0) - # identity would false-positive every check. Skip the inode half of - # the guard there; generation stamp still applies. - if not st.st_dev or not st.st_ino: - return None - return (st.st_dev, st.st_ino) - - -def _stat_sqlite_sidecar_identity(db_path: Path) -> Dict[str, tuple]: - """Snapshot ``(st_dev, st_ino)`` for existing WAL/SHM sidecars.""" - identities: Dict[str, tuple] = {} - base = os.fspath(db_path) - for suffix in ("-wal", "-shm"): - ident = _stat_db_file_identity(Path(base + suffix)) - if ident is not None: - identities[suffix] = ident - return identities - - -def _canonical_sqlite_path(path: str) -> str: - """Normalize a /proc fd target, stripping the Linux `` (deleted)`` suffix.""" - return os.path.normcase(os.path.abspath(path.removesuffix(" (deleted)"))) - - -def _watched_sqlite_sidecar_paths(db_path) -> Set[str]: - base = os.path.abspath(os.fspath(db_path)) - return { - _canonical_sqlite_path(base + "-wal"), - _canonical_sqlite_path(base + "-shm"), - } - - -def iter_deleted_sqlite_sidecar_holders(db_path) -> List[Tuple[int, str]]: - """Return processes holding an unlinked ``state.db-wal`` / ``-shm``. - - Linux-only (``/proc//fd`` readlink). Windows and other hosts - return ``[]`` — Windows cannot unlink a sidecar another process still - holds, and macOS does not use the `` (deleted)`` suffix. - - The scan includes this process: on the SessionDB open/write refuse - path, the in-process writer that still holds the orphan inode is the - one that must not mint a replacement WAL (and must stop committing). - ``_foreign_state_db_holders`` keeps skipping this PID for FTS - maintenance so a process does not block its own optional repair. - """ - if not sys.platform.startswith("linux"): - return [] - - holders: List[Tuple[int, str]] = [] - watched = _watched_sqlite_sidecar_paths(db_path) - try: - for pid_str in os.listdir("/proc"): - if not pid_str.isdigit(): - continue - pid = int(pid_str) - fd_dir = f"/proc/{pid}/fd" - try: - fds = os.listdir(fd_dir) - except OSError: - continue - for fd in fds: - try: - target = os.readlink(f"{fd_dir}/{fd}") - except OSError: - continue - if " (deleted)" not in target: - continue - if _canonical_sqlite_path(target) in watched: - holders.append((pid, target)) - except Exception as exc: - logger.debug("deleted-WAL holder scan failed for %s: %s", db_path, exc) - return holders - return holders - - -def refuse_deleted_wal_generation(db_path) -> None: - """Raise if any process holds a deleted WAL/SHM generation for *db_path*. - - Called *before* ``sqlite3.connect`` so a second opener cannot mint a - replacement WAL inode while a live writer still holds the orphan. - """ - holders = iter_deleted_sqlite_sidecar_holders(db_path) - if not holders: - return - logger.error(_DELETED_WAL_GENERATION_MSG) - raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG) - - # ── Process-wide shared SessionDB registry (#90837) ── # # The registry itself lives in hermes_state_registry.py — a bounded @@ -4673,478 +1791,9 @@ from hermes_state_registry import ( # noqa: F401 (re-export) ) - -def _connect_tracked_db(path, tracking_path=None, **kwargs): - """``sqlite3.connect`` that registers the open fd for lock-safety. - - While a connection is live, byte-level probes of the same file are - refused: an ``open()``/``close()`` cancels every POSIX advisory lock this - process holds on it -- including a running VACUUM's EXCLUSIVE lock. - Released automatically on ``close()``. - - The ONLY tolerated fallback is the helper being absent entirely - (scaffold/embed installs that ship hermes_state without hermes_cli). A - real connection failure must propagate: silently retrying an *untracked* - connect would disable the guard for the lifetime of that connection, - which is precisely the failure mode this module exists to prevent. - """ - try: - from hermes_cli.sqlite_safe_read import connect_tracked - except ImportError: - logger.debug( - "hermes_cli.sqlite_safe_read unavailable; opening %s untracked " - "(byte-probe guard inactive in this install)", - path, - ) - return sqlite3.connect(str(path), **kwargs) - - # Open through THIS module's sqlite3.connect so callers (and tests) that - # patch hermes_state.sqlite3.connect keep control of connection creation; - # the helper still owns tracking. - return connect_tracked( - path, - tracking_path=tracking_path, - connect_fn=sqlite3.connect, - **kwargs, - ) - - -def is_zeroed_state_db( - path: Path, *, probe_bytes: int = 100, force: bool = False -) -> bool: - """Detect the #68474/#97568 zeroed state.db signature (0-byte or NUL header). - - Byte-level probe, so it is only safe BEFORE any connection to *path* - exists in this process: ``close()`` cancels every POSIX advisory lock the - process holds on the file, which can pull the EXCLUSIVE lock out from - under a running VACUUM and corrupt the database. The read is routed - through ``read_header_bytes_preopen``, which refuses (returning False - here) once a connection is live. Pass ``force=True`` only for offline - files -- quarantined copies, snapshots, archives. - - Prefer ``hermes_cli.backup.is_zeroed_sqlite_file`` when available; this - local copy keeps SessionDB openable without importing the CLI package - in constrained embed paths. - """ - try: - from hermes_cli.backup import is_zeroed_sqlite_file - - return is_zeroed_sqlite_file(path, probe_bytes=probe_bytes, force=force) - except Exception: - pass - try: - if not path.is_file(): - # Special files (FIFO, device, socket) are never "zeroed", and - # probing a FIFO would block until a writer appears. - return False - size = path.stat().st_size - except OSError: - return False - if size < 0: - return False - from hermes_cli.sqlite_safe_read import has_live_connection, read_header_bytes_preopen - - if not force and has_live_connection(path): - return False - - head = read_header_bytes_preopen( - path, length=max(16, probe_bytes), force=force - ) - if head is None: - return False - if len(head) == 0: - return True - if head.startswith(b"SQLite format 3"): - return False - return all(byte == 0 for byte in head) - - -@contextlib.contextmanager -def quarantine_cross_process_lock(path: Path, timeout: float = 5.0): - """Acquire the cross-process lock for path.quarantine.lock.""" - import platform - - lock_path = path.with_name(path.name + ".quarantine.lock") - lock_path.parent.mkdir(parents=True, exist_ok=True) - handle = lock_path.open("a+b") - acquired = False - try: - deadline = time.monotonic() + timeout - if platform.system() == "Windows": - import msvcrt - - while True: - try: - handle.seek(0) - msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) - acquired = True - break - except OSError: - if time.monotonic() >= deadline: - break - time.sleep(0.020) - else: - import fcntl - - while True: - try: - fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - acquired = True - break - except (BlockingIOError, OSError): - if time.monotonic() >= deadline: - break - time.sleep(0.020) - yield acquired - finally: - try: - if acquired: - if platform.system() == "Windows": - import msvcrt - - handle.seek(0) - msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) - else: - import fcntl - - fcntl.flock(handle.fileno(), fcntl.LOCK_UN) - except (OSError, AttributeError): - pass - finally: - handle.close() - - -def quarantine_zeroed_state_db( - path: Path, *, already_locked: bool = False -) -> Optional[Path]: - """Move a zeroed state.db aside (preserve bytes) and return quarantine path. - - Uses a cross-process lock (``#68805``) so two concurrent startups cannot - race: the first process moves the zeroed file and the second re-checks - under the lock, finding the file already gone (or a fresh DB in its place) - instead of clobbering the quarantine. - """ - def _do_quarantine(): - if not path.exists(): - logger.info( - "quarantine_zeroed_state_db: %s already moved by another process", - path, - ) - return None - if not is_zeroed_state_db(path): - logger.info( - "quarantine_zeroed_state_db: %s is no longer zeroed (another " - "process quarantined it and a fresh DB was created)", - path, - ) - return None - - try: - ts = time.strftime("%Y%m%d-%H%M%S") - except Exception: - ts = "unknown" - dest = path.with_name( - f"{path.name}.zeroed-{ts}-{os.getpid()}.bak" - ) - n = 0 - while dest.exists(): - n += 1 - dest = path.with_name( - f"{path.name}.zeroed-{ts}-{os.getpid()}-{n}.bak" - ) - try: - path.rename(dest) - except OSError as exc: - logger.error("Failed to quarantine zeroed %s: %s", path, exc) - return None - for suffix in ("-wal", "-shm"): - side = Path(str(path) + suffix) - if side.exists(): - try: - side.rename(Path(str(dest) + suffix)) - except OSError: - pass - return dest - - if already_locked: - return _do_quarantine() - - with quarantine_cross_process_lock(path) as acquired: - if not acquired: - logger.error( - "quarantine lock for %s not acquired within 5s — refusing to " - "quarantine without the cross-process lock. The zeroed file " - "is left in place. If sessions fail to load, restore from " - "state-snapshots via `hermes snapshot list` / " - "`hermes snapshot restore `.", - path, - ) - return None - return _do_quarantine() - - # ── Read-only health/stats probes (hermes doctor, dashboards) ────────── -def collect_state_db_stats(db_path: Path) -> Dict[str, Any]: - """Best-effort, strictly read-only stats snapshot of a state.db file. - - Opens the database with ``mode=ro`` (URI) and a short timeout so it can - run against a *live* database held by a gateway without ever taking a - write lock or mutating the file. Every field is collected independently: - a failed pragma/SELECT yields ``None`` for that field, and the helper - itself never raises. - - Deliberately does NOT instantiate :class:`SessionDB` — its constructor - runs schema DDL (migrations, FTS table creation), which is exactly the - kind of write a diagnostics probe must never perform. - - Returned keys (all present, any may be None on failure): - - - ``page_count``, ``page_size``, ``freelist_count`` — PRAGMA values - - ``logical_size_bytes`` — page_count * page_size (post-checkpoint size) - - ``wal_size_bytes`` — stat() of ``-wal`` (0 when absent) - - ``journal_mode`` — PRAGMA journal_mode string - - ``messages`` / ``sessions`` — row counts - - ``fts_tables`` — dict of {table_name: bool} presence for - messages_fts / messages_fts_trigram / messages_fts_cjk - - ``fts_storage_version`` — int from state_meta, None when the marker is - absent (legacy pre-v23 inline layout) - - ``fts_rebuild_pending`` — True when the deferred v23 backfill has not - finished (high_water present and progress < high_water) - - ``fts_rebuild_high_water`` / ``fts_rebuild_progress`` — raw ints - - ``fts_rebuild_deferral`` — durable blocked-repair diagnostic, when present - """ - stats: Dict[str, Any] = { - "page_count": None, - "page_size": None, - "freelist_count": None, - "logical_size_bytes": None, - "wal_size_bytes": None, - "journal_mode": None, - "messages": None, - "sessions": None, - "fts_tables": None, - "fts_storage_version": None, - "fts_rebuild_pending": None, - "fts_rebuild_high_water": None, - "fts_rebuild_progress": None, - "fts_rebuild_deferral": None, - } - - # WAL sidecar size needs no connection at all. - try: - wal_path = Path(str(db_path) + "-wal") - stats["wal_size_bytes"] = wal_path.stat().st_size if wal_path.exists() else 0 - except OSError: - pass - - conn = None - try: - # mode=ro refuses to create the file and refuses every write; a - # short timeout keeps doctor snappy when a writer holds the lock. - # Route through the tracked connect so byte-probe helpers - # (read_header_bytes_preopen) see this connection and refuse raw - # opens that could cancel our POSIX locks mid-read. - conn = _connect_tracked_db( - f"file:{Path(db_path)}?mode=ro", - tracking_path=Path(db_path), - uri=True, - timeout=2.0, - ) - except Exception as exc: - logger.debug("collect_state_db_stats: cannot open %s read-only: %s", - db_path, exc) - return stats - - def _scalar(sql: str) -> Any: - try: - row = conn.execute(sql).fetchone() - return row[0] if row else None - except Exception: - return None - - try: - pc = _scalar("PRAGMA page_count") - ps = _scalar("PRAGMA page_size") - stats["page_count"] = int(pc) if pc is not None else None - stats["page_size"] = int(ps) if ps is not None else None - if stats["page_count"] is not None and stats["page_size"] is not None: - stats["logical_size_bytes"] = stats["page_count"] * stats["page_size"] - - fl = _scalar("PRAGMA freelist_count") - stats["freelist_count"] = int(fl) if fl is not None else None - - jm = _scalar("PRAGMA journal_mode") - stats["journal_mode"] = str(jm) if jm is not None else None - - msgs = _scalar("SELECT COUNT(*) FROM messages") - stats["messages"] = int(msgs) if msgs is not None else None - sess = _scalar("SELECT COUNT(*) FROM sessions") - stats["sessions"] = int(sess) if sess is not None else None - - # FTS table presence via sqlite_master (never SELECTs from the - # virtual tables themselves — a corrupt index must not fail stats). - try: - names = { - row[0] - for row in conn.execute( - "SELECT name FROM sqlite_master WHERE type = 'table' " - "AND name IN (?, ?, ?)", - ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"), - ).fetchall() - } - stats["fts_tables"] = { - t: (t in names) - for t in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk") - } - except Exception: - pass - - # Raw state_meta reads — cheap, and independent of SessionDB. - def _meta_int(key: str) -> Optional[int]: - try: - row = conn.execute( - "SELECT value FROM state_meta WHERE key = ?", (key,) - ).fetchone() - return int(row[0]) if row and row[0] is not None else None - except Exception: - return None - - stats["fts_storage_version"] = _meta_int("fts_storage_version") - high_water = _meta_int("fts_rebuild_high_water") - progress = _meta_int("fts_rebuild_progress") - stats["fts_rebuild_high_water"] = high_water - stats["fts_rebuild_progress"] = progress - if high_water is None: - stats["fts_rebuild_pending"] = False - else: - stats["fts_rebuild_pending"] = (progress or 0) < high_water - try: - row = conn.execute( - "SELECT value FROM state_meta WHERE key = ? LIMIT 1", - (FTS_REBUILD_DEFERRAL_KEY,), - ).fetchone() - if row: - parsed = json.loads(row[0]) - if isinstance(parsed, dict): - stats["fts_rebuild_deferral"] = parsed - except Exception: - pass - finally: - try: - conn.close() - except Exception: - pass - - return stats - - -def count_db_holders(db_path: Path) -> Optional[int]: - """Best-effort count of processes holding ``db_path`` open (Linux only). - - Scans ``/proc/*/fd`` symlinks for the resolved database path. Returns - the number of distinct PIDs with the file open, or ``None`` on any - error or on non-Linux platforms. Never raises; no lsof dependency. - Unreadable per-process fd dirs (other users' processes without root) - are silently skipped, so the count is a lower bound. - """ - try: - if not sys.platform.startswith("linux"): - return None - target = os.path.realpath(str(db_path)) - holders = 0 - for pid in os.listdir("/proc"): - if not pid.isdigit(): - continue - fd_dir = f"/proc/{pid}/fd" - try: - fds = os.listdir(fd_dir) - except OSError: - continue # process gone or not ours - for fd in fds: - try: - if os.readlink(f"{fd_dir}/{fd}") == target: - holders += 1 - break # one hit per PID - except OSError: - continue - return holders - except Exception: - return None - - -def _is_inactive_orphan_desktop_holder( - *, - ppid: int, - age_seconds: float, - min_age_seconds: float, - ephemeral_backend: bool, - connection_statuses: List[str], -) -> bool: - """Pure safety predicate for the narrow Desktop holder reap.""" - return ( - ppid in (0, 1) - and age_seconds >= min_age_seconds - and ephemeral_backend - and "ESTABLISHED" not in connection_statuses - ) - - -def _concrete_state_db_holder_pids( - db_path: Path, holders: List[Tuple[int, str]] -) -> List[int]: - """Return unique PIDs proven to hold this DB or one of its sidecars.""" - canonical_db = os.path.normcase(os.path.abspath(os.fspath(db_path))) - watched = { - canonical_db, - canonical_db + "-wal", - canonical_db + "-shm", - } - pids: List[int] = [] - seen = set() - for pid, path in holders: - canonical_path = os.path.normcase( - os.path.abspath(path.removesuffix(" (deleted)")) - ) - if pid <= 0 or pid in seen or canonical_path not in watched: - continue - seen.add(pid) - pids.append(pid) - return pids - - -def _read_proc_cmdline(pid: int) -> Optional[str]: - """Read /proc//cmdline, world-readable even when fd table is not. - - Returns the cmdline as a space-joined string, or None when unreadable - (process exited, or hidepid mount). - """ - try: - with open(f"/proc/{pid}/cmdline", "rb") as f: - raw = f.read() - if not raw: - return None - return raw.replace(b"\x00", b" ").decode("utf-8", "replace").strip() - except OSError: - return None - - -_HERMES_CMDLINE_MARKERS = ("hermes_cli.main", "hermes_cli/main", "hermes serve", - "hermes-agent", "hermes gateway", "hermes chat") - - -def _looks_like_hermes(cmdline: str) -> bool: - """Heuristic: does this cmdline look like a Hermes process? - - Used to decide whether an uninspectable process (fd table unreadable - due to different user) should be treated as a potential state.db holder. - We only flag processes that look like Hermes, not every system daemon. - """ - lower = cmdline.lower() - return any(marker in lower for marker in _HERMES_CMDLINE_MARKERS) - - # Lifecycle statuses surfaced by session pickers. Classification looks ONLY at # a session's final message row — role, whether it carries tool_calls, and its # finish_reason — so it stays O(1) per session (see @@ -5199,7 +1848,18 @@ _SAME_KEY_NAMESPACE_SQL = ( ) -class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin): +class SessionDB( + SessionSearchMixin, + SessionSchemaMixin, + SessionPortabilityMixin, + SessionTelegramTopicsMixin, + SessionCompressionMixin, + SessionGatewayMixin, + SessionMaintenanceMixin, + SessionUsageMixin, + SessionTitlesMixin, + SessionMessagesMixin, +): """ SQLite-backed session storage with FTS5 search. @@ -6077,7 +2737,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ).fetchone() if row is None: return False - sql = (row[0] if not isinstance(row, sqlite3.Row) else row["sql"]) or "" + sql = row[0] or "" # The v23 table declares tool_name/tool_calls columns. Their absence # means a legacy shape that doesn't index tool metadata → optimize. return "tool_name" not in sql @@ -6448,6 +3108,46 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) continue raise + def _write_sql( + self, + sql: str, + params: Any = (), + *, + many: bool = False, + patience_s: Optional[float] = None, + ) -> None: + """Run one INSERT/UPDATE/DELETE through ``_execute_write``.""" + def _do(conn): + (conn.executemany if many else conn.execute)(sql, params) + + self._execute_write(_do, patience_s=patience_s) + + def _write_rowcount( + self, sql: str, params: Any = (), *, patience_s: Optional[float] = None + ) -> int: + """Run one UPDATE/DELETE through ``_execute_write``; return rows changed. + + Falls back to ``SELECT changes()`` when the driver reports an unknown + rowcount (None / negative). + """ + def _do(conn): + rowcount = conn.execute(sql, params).rowcount + if rowcount is None or rowcount < 0: + rowcount = conn.execute("SELECT changes()").fetchone()[0] + return rowcount + + return self._execute_write(_do, patience_s=patience_s) + + def _read_one(self, sql: str, params: Any = ()) -> Optional[sqlite3.Row]: + """``fetchone()`` of one read-only statement via ``_read_ctx``.""" + with self._read_ctx() as conn: + return conn.execute(sql, params).fetchone() + + def _read_all(self, sql: str, params: Any = ()) -> List[sqlite3.Row]: + """``fetchall()`` of one read-only statement via ``_read_ctx``.""" + with self._read_ctx() as conn: + return conn.execute(sql, params).fetchall() + def _ensure_db_file_generation(self) -> None: """Mint a once-per-file generation stamp (state_meta + application_id). @@ -6498,8 +3198,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._db_file_application_id = disk_id elif self._conn is not None and not self._db_file_application_id: try: - with self._read_ctx() as conn: - pragma_row = conn.execute("PRAGMA application_id").fetchone() + pragma_row = self._read_one("PRAGMA application_id") if pragma_row and pragma_row[0]: self._db_file_application_id = int(pragma_row[0]) except sqlite3.Error: @@ -6819,66 +3518,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return holders or [(-1, f"open-file scan failed: {exc}")] return holders - def _reap_inactive_orphan_desktop_holders( - self, holders: List[Tuple[int, str]], *, min_age_seconds: float - ) -> List[int]: - """Terminate old PPID-1 Desktop ephemeral backends with no client. - - Inspection fails closed: anything whose parent, age, argv, or network - connections cannot be proved safe remains a repair-blocking holder. - """ - if not sys.platform.startswith("linux") or psutil is None: - return [] - try: - from hermes_cli.dashboard_procs import _is_ephemeral_port_zero_backend - except Exception: - return [] - - now = time.time() - candidates = [] - for pid in _concrete_state_db_holder_pids(self.db_path, holders): - try: - process = psutil.Process(pid) - statuses = [ - conn.status for conn in process.net_connections(kind="inet") - ] - if not _is_inactive_orphan_desktop_holder( - ppid=process.ppid(), - age_seconds=now - process.create_time(), - min_age_seconds=min_age_seconds, - ephemeral_backend=_is_ephemeral_port_zero_backend(process.cmdline()), - connection_statuses=statuses, - ): - continue - except Exception: - continue - candidates.append(process) - - signalled: List[int] = [] - for process in candidates: - try: - process.terminate() - signalled.append(process.pid) - except (psutil.Error, OSError): - continue - if not signalled: - return [] - - try: - _gone, alive = psutil.wait_procs(candidates, timeout=1.5) - except Exception: - alive = [] - for process in alive: - try: - process.kill() - except (psutil.Error, OSError): - continue - if alive: - try: - psutil.wait_procs(alive, timeout=1.5) - except Exception: - pass - return signalled def _enter_fts_fail_open(self, exc: sqlite3.DatabaseError) -> bool: """Detach corrupt FTS indexes so canonical writes can continue. @@ -7233,51 +3872,33 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ) -> None: """Insert a session row, enriching NULL metadata on conflict. - The gateway's ``get_or_create_session`` creates a bare row (source + - user_id) *before* the agent exists; the agent's later - ``create_session`` then carries the real ``model`` / ``model_config`` / - ``system_prompt``. A plain ``INSERT OR IGNORE`` silently dropped that - enrichment, leaving gateway sessions with NULL model/billing metadata. - The ``ON CONFLICT`` upsert backfills those fields via ``COALESCE`` — - only filling columns that are still NULL, never overwriting values an - earlier writer already set (so a later bare call with source="unknown" - can't clobber a real source/model). + The gateway creates a bare row (source + user_id) before the agent + exists and the agent's later ``create_session`` carries the real + model/model_config/system_prompt; ``INSERT OR IGNORE`` dropped that + enrichment, so the upsert ``COALESCE``-fills columns that are still + NULL and never overwrites what an earlier writer set (a later bare + source="unknown" call cannot clobber a real source/model). - ``chat_id``/``thread_id`` record the messaging origin (the chat/room and - thread the session was started in) so that gateway ``/resume`` can prove - a persisted, now-inactive row belongs to the caller's chat/thread before - switching to it (IDOR scoping — without them the ``sessions`` table has - no chat/thread to compare). + ``chat_id``/``thread_id`` record the messaging origin so gateway + ``/resume`` can prove an inactive row belongs to the caller's + chat/thread (IDOR scoping). - When ``parent_session_id`` is set (compression fork, delegate/subagent - spawn, branch continuation) and this row's own ``cwd``/``git_repo_root``/ - ``git_branch``/``profile_name`` are still NULL after the insert, they are - backfilled from the parent row. Callers of ``create_session`` for a child - session historically didn't propagate these fields themselves (e.g. the - compression-fork path), so a lineage could silently lose its working - directory and drop out of the project sidebar every time it forked - (#64709), or lose its owning profile and be aggregated as "default" every - time it rotated or branched (the cross-profile session-jump bug). This - only fills NULLs — an explicit value on the child is never overwritten. - For compression forks specifically - (parent ended with ``end_reason='compression'``), the gateway origin - columns (``user_id``/``session_key``/``chat_id``/``chat_type``/ - ``thread_id``/``display_name``/``origin_json``) are inherited too, so a - crash before the gateway re-records the peer can't strand the child - without a recoverable routing mapping (#59527). + With ``parent_session_id`` (compression fork, delegate spawn, branch), + NULL ``cwd``/``git_repo_root``/``git_branch``/``profile_name`` are + backfilled from the parent — child creators historically did not + propagate them, so lineages dropped out of the project sidebar or were + aggregated as "default" on every fork. NULL-fill only. Compression + forks (parent ``end_reason='compression'``) also inherit the gateway + origin columns (user_id/session_key/chat_id/chat_type/thread_id/ + display_name/origin_json) so a crash before the gateway re-records the + peer cannot strand the child without a routing mapping. - When the caller passes no ``profile_name`` at all, the row is stamped - with THIS store's own profile (:meth:`_own_profile_name`) instead of - NULL. Every ``state.db`` belongs to exactly one profile — the same - single-match contract :meth:`backfill_null_session_profiles` relies - on — so the stamp is derivation, not a guess. Rows minted NULL after - that one-shot #94724 backfill ran stayed NULL forever, and - profile-keyed consumers (desktop sidebar scope matching, - ``@session:/`` deep links, the fail-closed owner ladder) - treat NULL as unowned: the session vanishes from the sidebar even - though its transcript is intact (#99222). Stores outside the profile - tree (explicit ``db_path`` in tests, ad-hoc copies) derive nothing - and keep NULL — never guess. + With no ``profile_name`` the row is stamped with THIS store's own + profile (:meth:`_own_profile_name`): every state.db belongs to exactly + one profile (the contract :meth:`backfill_null_session_profiles` + relies on), and profile-keyed consumers treat NULL as unowned, hiding + the session from the sidebar. Stores outside the profile tree derive + nothing and keep NULL — never guess. """ if not (profile_name or "").strip(): profile_name = self._own_profile_name() @@ -7426,140 +4047,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._insert_session_row(session_id, source, **kwargs) return session_id - def record_gateway_session_peer( - self, - session_id: str, - *, - source: str, - user_id: str = None, - session_key: str = None, - chat_id: str = None, - chat_type: str = None, - thread_id: str = None, - display_name: str = None, - origin_json: str = None, - include_compression_ancestors: bool = False, - ) -> None: - """Persist the gateway routing peer for an existing session row. - - ``display_name`` / ``origin_json`` carry the gateway's presentation - and full origin metadata (#9006) so consumers (mcp_serve, mirror, - channel directory) can read routing data from state.db instead of - sessions.json. They are COALESCE'd only in the sense that ``None`` - leaves the existing value untouched. - - ``include_compression_ancestors`` keeps a logical compression lineage - on one routing peer when an explicit gateway resume moves its tip to a - different lane. Normal per-turn metadata refreshes update only the - supplied row. - - Self-healing (#82616): when the target row does not exist yet — the - gateway's ``create_session`` write failed and was deferred, or a - crash landed between routing publication and row creation — this - recorder INSERTs the row with the full identity instead of silently - no-opping. Every per-turn peer refresh is therefore a repair - opportunity: a gateway session row can no longer be first-created by - an identity-less lazy writer (``update_token_counts`` / - ``record_auxiliary_usage``) and stay unroutable forever. - """ - if not session_id or not session_key: - return - - def _do(conn): - lineage_cte = "" - target_clause = "WHERE id = ?" - query_params = [] - if include_compression_ancestors: - lineage_cte = """ - WITH RECURSIVE compression_lineage(id) AS ( - SELECT ? - UNION - SELECT parent.id - FROM compression_lineage lineage - JOIN sessions child ON child.id = lineage.id - JOIN sessions parent ON parent.id = child.parent_session_id - WHERE parent.end_reason = 'compression' - AND json_extract( - COALESCE(child.model_config, '{}'), - '$._branched_from' - ) IS NULL - AND json_extract( - COALESCE(child.model_config, '{}'), - '$._delegate_from' - ) IS NULL - AND COALESCE(child.source, '') != 'tool' - ) - """ - target_clause = "WHERE id IN (SELECT id FROM compression_lineage)" - query_params.append(session_id) - query_params.extend( - ( - session_key, - source, - user_id, - chat_id, - chat_type, - thread_id, - display_name, - origin_json, - ) - ) - if not include_compression_ancestors: - query_params.append(session_id) - conn.execute( - f"""{lineage_cte} - UPDATE sessions - SET session_key = ?, source = ?, user_id = ?, chat_id = ?, - chat_type = ?, thread_id = ?, - display_name = COALESCE(?, display_name), - origin_json = COALESCE(?, origin_json) - {target_clause}""", - query_params, - ) - # Self-heal (#82616): the UPDATE is a silent no-op when the row - # is missing (create_session failed earlier, or a crash landed - # between routing publication and row creation). Insert it with - # the full identity so the session is durably routable — never - # leave first-creation to an identity-less lazy writer. - if not include_compression_ancestors: - cur = conn.execute( - "SELECT 1 FROM sessions WHERE id = ? LIMIT 1", (session_id,) - ) - if cur.fetchone() is None: - conn.execute( - """INSERT INTO sessions ( - id, source, user_id, session_key, chat_id, - chat_type, thread_id, display_name, origin_json, - profile_name, started_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(id) DO UPDATE SET - session_key = COALESCE(sessions.session_key, excluded.session_key), - chat_id = COALESCE(sessions.chat_id, excluded.chat_id), - chat_type = COALESCE(sessions.chat_type, excluded.chat_type), - thread_id = COALESCE(sessions.thread_id, excluded.thread_id), - display_name = COALESCE(sessions.display_name, excluded.display_name), - origin_json = COALESCE(sessions.origin_json, excluded.origin_json)""", - ( - session_id, - source, - user_id, - session_key, - chat_id, - chat_type, - thread_id, - display_name, - origin_json, - # Same ownership stamp as _insert_session_row: a - # self-healed row is a first creation too, and an - # unowned (NULL) row vanishes from profile-keyed - # consumers (#99222). - self._own_profile_name(), - time.time(), - ), - ) - - self._execute_write(_do) def set_expiry_finalized(self, session_id: str, finalized: bool = True) -> None: """Mark a gateway session's expiry-finalization flag in state.db. @@ -7570,248 +4057,13 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) if not session_id: return - def _do(conn): - conn.execute( - "UPDATE sessions SET expiry_finalized = ? WHERE id = ?", - (1 if finalized else 0, session_id), - ) - - self._execute_write(_do) + self._write_sql( + "UPDATE sessions SET expiry_finalized = ? WHERE id = ?", + (1 if finalized else 0, session_id), + ) # ── Gateway routing index (replaces sessions.json, #9006 follow-up) ──── - def save_gateway_routing_entry( - self, session_key: str, entry_json: str, *, scope: str = "" - ) -> None: - """Upsert one gateway routing entry (session_key -> SessionEntry JSON). - - The gateway_routing table is the durable replacement for - sessions.json: one row per routing key, holding the full serialized - ``SessionEntry`` so the gateway can rehydrate exactly what it wrote. - - ``scope`` namespaces the index the way separate sessions.json files - did (one per sessions_dir) — callers pass their sessions_dir path so - two stores with different directories never share routing state. - """ - if not session_key or not entry_json: - return - - def _do(conn): - conn.execute( - """INSERT INTO gateway_routing (scope, session_key, entry_json, updated_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(scope, session_key) DO UPDATE SET - entry_json = excluded.entry_json, - updated_at = excluded.updated_at""", - (scope, session_key, entry_json, time.time()), - ) - - self._execute_write(_do) - - def replace_gateway_routing_entries( - self, entries: Dict[str, str], *, scope: str = "" - ) -> None: - """Atomically replace the routing index for *scope* with *entries*. - - Mirrors the sessions.json full-rewrite semantics: keys absent from - *entries* are removed (pruned/reset sessions disappear from the - index). Runs as a single write transaction. Other scopes are - untouched. - """ - now = time.time() - - def _do(conn): - conn.execute("DELETE FROM gateway_routing WHERE scope = ?", (scope,)) - if entries: - conn.executemany( - "INSERT INTO gateway_routing (scope, session_key, entry_json, updated_at) " - "VALUES (?, ?, ?, ?)", - [(scope, k, v, now) for k, v in entries.items() if k and v], - ) - - self._execute_write(_do) - - def load_gateway_routing_entries(self, *, scope: str = "") -> Dict[str, str]: - """Load routing entries for *scope* as {session_key: entry_json}.""" - with self._read_ctx() as conn: - rows = conn.execute( - "SELECT session_key, entry_json FROM gateway_routing WHERE scope = ?", - (scope,), - ).fetchall() - return {r["session_key"]: r["entry_json"] for r in rows} - - def delete_gateway_routing_entries( - self, session_keys: List[str], *, scope: str = "" - ) -> None: - """Remove routing entries for the given session keys in *scope*.""" - if not session_keys: - return - - def _do(conn): - conn.executemany( - "DELETE FROM gateway_routing WHERE scope = ? AND session_key = ?", - [(scope, k) for k in session_keys], - ) - - self._execute_write(_do) - - def list_never_active_keyed_sessions( - self, *, older_than_days: float - ) -> List[Dict[str, Any]]: - """Keyed gateway rows that were opened and then never used at all. - - Selects rows that are keyed (``session_key IS NOT NULL``), still open - (``ended_at IS NULL``) and carry no evidence of a single turn: no - messages, no tokens, no tool or API calls, no recorded activity, no - title. Such a row is indistinguishable from "never happened". - - That is exactly the shape of a leaked test fixture (#82770) — and - also of a chat that was routed but never answered. Both are safe to - drop: there is no transcript to lose, and the gateway mints a fresh - session on the next inbound message either way. - - ``bulk prune``/``archive`` cannot reach these rows: their shared - selector is pinned to ``ended_at IS NOT NULL`` so that a live session - is never picked, which permanently excludes every never-closed row. - Hence a separate, narrower selector rather than another filter flag. - - ``pinned`` and ``archived`` rows are excluded — both are explicit - user intent to keep the row around. - """ - cutoff = time.time() - (float(older_than_days) * 86400.0) - with self._read_ctx() as conn: - rows = conn.execute( - """ - SELECT s.id, s.session_key, s.source, s.chat_id, - s.chat_type, s.user_id, s.started_at - FROM sessions s - WHERE s.session_key IS NOT NULL - AND s.ended_at IS NULL - AND s.title IS NULL - AND s.last_activity_at IS NULL - AND COALESCE(s.message_count, 0) = 0 - AND COALESCE(s.tool_call_count, 0) = 0 - AND COALESCE(s.api_call_count, 0) = 0 - AND COALESCE(s.input_tokens, 0) = 0 - AND COALESCE(s.output_tokens, 0) = 0 - AND COALESCE(s.pinned, 0) = 0 - AND COALESCE(s.archived, 0) = 0 - AND s.started_at IS NOT NULL - AND s.started_at < ? - AND NOT EXISTS ( - SELECT 1 FROM messages m WHERE m.session_id = s.id - ) - ORDER BY s.started_at - """, - (cutoff,), - ).fetchall() - return [dict(r) for r in rows] - - def _delete_routing_entries_for_sessions(self, session_ids: Set[str]) -> int: - """Drop ``gateway_routing`` rows pointing at any of *session_ids*. - - Routing entries are keyed by ``(scope, session_key)`` and record their - target session inside ``entry_json``, so there is no way to reach them - by session id in SQL — the match is done in Python over all scopes. - """ - if not session_ids: - return 0 - with self._read_ctx() as conn: - rows = conn.execute( - "SELECT scope, session_key, entry_json FROM gateway_routing" - ).fetchall() - doomed: List[Tuple[str, str]] = [] - for row in rows: - try: - entry = json.loads(row["entry_json"] or "{}") - except Exception: - continue - if isinstance(entry, dict) and entry.get("session_id") in session_ids: - doomed.append((row["scope"], row["session_key"])) - if not doomed: - return 0 - - def _do(conn): - conn.executemany( - "DELETE FROM gateway_routing WHERE scope = ? AND session_key = ?", - doomed, - ) - - self._execute_write(_do) - return len(doomed) - - def prune_never_active_keyed_sessions( - self, - *, - older_than_days: float, - sessions_dir: Optional[Path] = None, - ) -> Tuple[int, int]: - """Delete never-active keyed rows and the routing entries naming them. - - Returns ``(sessions_deleted, routing_entries_deleted)``. - - The routing entries go first: a stale entry that outlived its target - would leave the gateway resuming a session id that no longer exists. - Deleting the pair is what leaving them both would have amounted to - anyway — the target had no transcript to resume. - - Deletion goes through :meth:`delete_session` rather than a bulk - ``DELETE`` so the delegate cascade, FTS bookkeeping and on-disk - transcript cleanup stay owned by one implementation. - """ - candidates = self.list_never_active_keyed_sessions( - older_than_days=older_than_days - ) - if not candidates: - return (0, 0) - ids = {str(row["id"]) for row in candidates} - routing_deleted = self._delete_routing_entries_for_sessions(ids) - deleted = 0 - for session_id in ids: - if self.delete_session(session_id, sessions_dir=sessions_dir): - deleted += 1 - return (deleted, routing_deleted) - - def list_gateway_sessions( - self, - *, - platform: Optional[str] = None, - active_only: bool = True, - ) -> List[Dict[str, Any]]: - """List gateway sessions (rows with a session_key) from state.db. - - Returns the newest row per session_key — the same shape consumers got - from sessions.json: one live mapping per routing key. ``platform`` - filters on ``source``; ``active_only`` restricts to sessions that - have not ended. - """ - # Full rows carry token/cost totals (MCP listings, /status) — drain - # queued async accounting deltas so consumers see exact counters. - self.flush_token_counts() - query = f""" - SELECT sessions.*, - COALESCE(sp.prompt, sessions.system_prompt) - AS _system_prompt_resolved, - {_sql_session_last_active("sessions")} AS last_active - FROM sessions - LEFT JOIN system_prompts sp - ON sp.hash = sessions.system_prompt_hash - WHERE session_key IS NOT NULL - AND started_at = ( - SELECT MAX(s2.started_at) FROM sessions s2 - WHERE s2.session_key = sessions.session_key - ) - """ - params: list = [] - if platform: - query += " AND LOWER(source) = LOWER(?)" - params.append(platform) - if active_only: - query += " AND ended_at IS NULL" - query += " ORDER BY last_active DESC" - with self._read_ctx() as conn: - rows = conn.execute(query, params).fetchall() - return [self._session_row_dict(r) for r in rows] def find_session_by_origin( self, @@ -7863,132 +4115,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return None return str(rows[0]["id"]) - def find_latest_gateway_session_for_peer( - self, - *, - source: str, - user_id: Optional[str] = None, - session_key: Optional[str] = None, - chat_id: Optional[str] = None, - chat_type: Optional[str] = None, - thread_id: Optional[str] = None, - ) -> Optional[Dict[str, Any]]: - """Find the latest recoverable gateway session for a routing peer. - - ``sessions.json`` is the fast routing index, but it can be missing or - pruned after process-level restart bugs. New gateway sessions persist - the deterministic ``session_key`` on the durable session row so the - mapping can be rebuilt exactly. Rows ended only by older gateway - cleanup's ``agent_close`` bug or a mistaken TUI ``ws_orphan_reap`` - (dashboard viewer disconnect before #60609) are treated as recoverable; - explicit conversation boundaries such as /new, /resume switches, and - compression splits are not. - - Ordering and emptiness (#82616): candidates are ranked by actual - conversation recency (``last_activity_at``, falling back to - ``started_at``) — ``started_at`` alone resurrected days-old zombie - rows over the live conversation. Rows with messages are preferred, - but an empty keyed row is still returned rather than ``None``: - returning ``None`` mints a brand-new session id, which is a worse - outcome than resuming an empty-but-correctly-keyed row (and "empty" - may just mean the transcript lives under a compression child). - - Reset boundaries fence recovery (#68539): an intentional boundary - such as ``session_reset`` (or any explicit non-recoverable - end_reason) must block fallback to an *older* row for the same - peer. Without the fence, the has-messages ranking above could reach - behind a /new reset and silently restore the exact context the user - reset. Each candidate is therefore rejected when a boundary row for - the peer ended *after* the candidate's last activity — if the - conversation's most recent event is an intentional reset, recovery - returns nothing rather than reaching behind it. - """ - if not session_key: - return None - with self._read_ctx() as conn: - row = conn.execute( - f""" - SELECT s.*, - COALESCE(sp.prompt, s.system_prompt) - AS _system_prompt_resolved, - (COALESCE(s.message_count, 0) > 0 OR EXISTS ( - SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1 - )) AS _has_messages - FROM sessions s - LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash - WHERE s.session_key = ? - AND s.source = ? - AND (s.ended_at IS NULL OR s.end_reason IN ({_RECOVERABLE_END_REASONS_SQL})) - AND NOT EXISTS ( - SELECT 1 FROM sessions b - WHERE b.session_key = s.session_key - AND b.source = s.source - AND b.ended_at IS NOT NULL - AND b.end_reason IN ({_RESET_END_REASONS_SQL}) - AND b.ended_at - > COALESCE(s.last_activity_at, s.started_at) - ) - ORDER BY _has_messages DESC, - COALESCE(s.last_activity_at, s.started_at) DESC - LIMIT 1 - """, - (session_key, source), - ).fetchone() - if row is not None: - return self._session_row_dict(row) - - # Conservative fallback for rows created by current code but with a - # temporarily-missing exact key: still require the complete peer - # tuple so we never cross chats/threads/users. - if chat_id is None or chat_type is None: - return None - # Profile fence (#74285): a Telegram DM's peer tuple is identical - # for every bot (chat_id == user_id, no thread), so a sibling - # profile's row written into this store before the per-profile - # partition (legacy data) would otherwise be adopted here. Every - # profile-tree store has one owner; a row is ours when its - # profile_name is the owner or NULL (legacy rows this store - # minted). Stores outside the tree derive no owner and keep the - # historical unfenced behavior. - owner = self._own_profile_name() - row = conn.execute( - f""" - SELECT s.*, - COALESCE(sp.prompt, s.system_prompt) - AS _system_prompt_resolved, - (COALESCE(s.message_count, 0) > 0 OR EXISTS ( - SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1 - )) AS _has_messages - FROM sessions s - LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash - WHERE s.source = ? - AND COALESCE(s.user_id, '') = COALESCE(?, '') - AND COALESCE(s.chat_id, '') = COALESCE(?, '') - AND COALESCE(s.chat_type, '') = COALESCE(?, '') - AND COALESCE(s.thread_id, '') = COALESCE(?, '') - AND (? IS NULL OR COALESCE(s.profile_name, ?) = ?) - AND (s.ended_at IS NULL OR s.end_reason IN ({_RECOVERABLE_END_REASONS_SQL})) - AND (COALESCE(s.message_count, 0) > 0 OR EXISTS ( - SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1 - )) - AND NOT EXISTS ( - SELECT 1 FROM sessions b - WHERE b.source = s.source - AND COALESCE(b.user_id, '') = COALESCE(s.user_id, '') - AND COALESCE(b.chat_id, '') = COALESCE(s.chat_id, '') - AND COALESCE(b.chat_type, '') = COALESCE(s.chat_type, '') - AND COALESCE(b.thread_id, '') = COALESCE(s.thread_id, '') - AND b.ended_at IS NOT NULL - AND b.end_reason IN ({_RESET_END_REASONS_SQL}) - AND b.ended_at - > COALESCE(s.last_activity_at, s.started_at) - ) - ORDER BY COALESCE(s.last_activity_at, s.started_at) DESC - LIMIT 1 - """, - (source, user_id, chat_id, chat_type, thread_id, owner, owner, owner), - ).fetchone() - return self._session_row_dict(row) if row else None # ── Orphaned gateway-session repair (#82616) ────────────────────────── # A write-path failure (corrupt FTS, crash between routing publication @@ -8005,219 +4131,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # 15 minutes stays generous without spanning unrelated conversations. _ORPHAN_ADOPTION_MAX_GAP_S = 900.0 - def find_orphaned_gateway_sessions( - self, *, max_gap_s: Optional[float] = None - ) -> List[Dict[str, Any]]: - """Report message-bearing session rows that lost their routing identity. - - A row is a candidate orphan when it has messages but no - ``session_key``. It is only *adoptable* when exactly one keyed - predecessor can be named as the conversation it continues: - - * ``lineage`` — ``parent_session_id`` points at a keyed row of the - same source. That is a recorded fact, so no time window applies. - * ``contiguity`` — exactly one keyed row of the same source (and - compatible ``user_id``) fell quiet within *max_gap_s* of the - orphan's start, and is older than the orphan's own last activity. - - Anything ambiguous is reported with ``adoptable=False`` and a reason - rather than guessed at: mis-adopting would splice one person's - conversation into another person's chat. Branch/delegate/tool rows - are excluded outright — they are unkeyed by design, not by damage. - """ - gap = ( - self._ORPHAN_ADOPTION_MAX_GAP_S - if max_gap_s is None - else float(max_gap_s) - ) - orphan_active = _sql_session_last_active("o") - donor_active = _sql_session_last_active("d") - donor_columns = ( - "d.id, d.session_key, d.chat_id, d.chat_type, d.thread_id, " - "d.user_id, d.origin_json, d.display_name, d.end_reason" - ) - records: List[Dict[str, Any]] = [] - - with self._read_ctx() as conn: - orphans = conn.execute( - f""" - SELECT o.id, o.source, o.user_id, o.started_at, - o.parent_session_id, - {orphan_active} AS last_active, - (SELECT COUNT(*) FROM messages m - WHERE m.session_id = o.id) AS message_count - FROM sessions o - WHERE o.session_key IS NULL - AND EXISTS (SELECT 1 FROM messages m - WHERE m.session_id = o.id) - AND COALESCE(o.source, '') != 'tool' - AND json_extract(COALESCE(o.model_config, '{{}}'), - '$._branched_from') IS NULL - AND json_extract(COALESCE(o.model_config, '{{}}'), - '$._delegate_from') IS NULL - ORDER BY o.started_at ASC - """ - ).fetchall() - - for orphan in orphans: - donor = None - evidence = "" - reason = "" - - if orphan["parent_session_id"]: - evidence = "lineage" - donor = conn.execute( - f""" - SELECT {donor_columns} - FROM sessions d - WHERE d.id = ? - AND d.session_key IS NOT NULL - AND COALESCE(d.source, '') = COALESCE(?, '') - """, - (orphan["parent_session_id"], orphan["source"]), - ).fetchone() - if donor is None: - reason = ( - "parent session carries no gateway identity of " - "this source" - ) - else: - evidence = "contiguity" - candidates = conn.execute( - f""" - SELECT {donor_columns}, {donor_active} AS last_active - FROM sessions d - WHERE d.session_key IS NOT NULL - AND d.id != ? - AND COALESCE(d.source, '') = COALESCE(?, '') - AND (COALESCE(d.user_id, '') = '' - OR COALESCE(?, '') = '' - OR d.user_id = ?) - AND {donor_active} BETWEEN ? AND ? - AND {donor_active} < ? - ORDER BY last_active DESC - LIMIT 2 - """, - ( - orphan["id"], - orphan["source"], - orphan["user_id"], - orphan["user_id"], - (orphan["started_at"] or 0) - gap, - (orphan["started_at"] or 0) + gap, - orphan["last_active"], - ), - ).fetchall() - if not candidates: - reason = ( - f"no keyed predecessor fell quiet within {gap:.0f}s " - "of this session's start" - ) - elif len(candidates) > 1: - reason = ( - "ambiguous: more than one keyed predecessor " - "matches this window" - ) - else: - donor = candidates[0] - - records.append( - { - "orphan_id": orphan["id"], - "source": orphan["source"], - "message_count": orphan["message_count"], - "started_at": orphan["started_at"], - "last_active": orphan["last_active"], - "donor_id": donor["id"] if donor else None, - "session_key": donor["session_key"] if donor else None, - "evidence": evidence if donor else "", - "adoptable": donor is not None, - "reason": reason, - } - ) - - # Two unkeyed successors claiming the same predecessor means at most - # one of them continues that chat, and nothing here says which. - contested = { - r["donor_id"] - for r in records - if r["adoptable"] - and sum(1 for x in records if x["donor_id"] == r["donor_id"]) > 1 - } - for record in records: - if record["donor_id"] in contested: - record["adoptable"] = False - record["reason"] = ( - "ambiguous: more than one unkeyed session claims this " - "predecessor" - ) - return records - - def adopt_orphaned_gateway_session( - self, orphan_id: str, donor_id: str - ) -> bool: - """Stamp *orphan_id* with *donor_id*'s routing identity, retire *donor_id*. - - Re-verifies the pair inside the write transaction, so a concurrent - gateway that healed either row in the meantime turns this into a - no-op instead of a conflicting write. Existing non-NULL columns on - the orphan are preserved. Returns True when the adoption applied. - """ - if not orphan_id or not donor_id or orphan_id == donor_id: - return False - - def _do(conn): - donor = conn.execute( - "SELECT session_key, chat_id, chat_type, thread_id, user_id, " - "origin_json, display_name, source FROM sessions WHERE id = ?", - (donor_id,), - ).fetchone() - orphan = conn.execute( - "SELECT session_key, source FROM sessions WHERE id = ?", - (orphan_id,), - ).fetchone() - if donor is None or orphan is None: - return False - if not donor["session_key"] or orphan["session_key"]: - return False - if (donor["source"] or "") != (orphan["source"] or ""): - return False - - conn.execute( - """UPDATE sessions - SET session_key = ?, - chat_id = COALESCE(chat_id, ?), - chat_type = COALESCE(chat_type, ?), - thread_id = COALESCE(thread_id, ?), - user_id = COALESCE(user_id, ?), - origin_json = COALESCE(origin_json, ?), - display_name = COALESCE(display_name, ?), - parent_session_id = COALESCE(parent_session_id, ?) - WHERE id = ? AND session_key IS NULL""", - ( - donor["session_key"], - donor["chat_id"], - donor["chat_type"], - donor["thread_id"], - donor["user_id"], - donor["origin_json"], - donor["display_name"], - donor_id, - orphan_id, - ), - ) - # Retire the predecessor under a reason recovery does NOT treat - # as resumable — 'agent_close'/'ws_orphan_reap' would keep it in - # the running, and the newly keyed orphan could lose the chat - # again on the next restart. - conn.execute( - "UPDATE sessions SET ended_at = COALESCE(ended_at, ?), " - "end_reason = 'superseded_by_repair' WHERE id = ?", - (time.time(), donor_id), - ) - return True - - return self._execute_write(_do) # Children that carry a ``parent_session_id`` but are NOT compression # continuations: branches, delegate/subagent runs, and tool sessions. @@ -8237,371 +4150,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) " AND COALESCE({alias}source, '') != 'tool'\n" ) - def find_live_compression_child( - self, parent_session_id: str - ) -> Optional[Dict[str, Any]]: - """Return the unique live direct child of a compression-ended session. - - A stale agent may observe that another compression path already rotated - its parent. Recovery is safe only when the durable lineage identifies - exactly one live direct continuation. Multiple children are treated as - ambiguous and fail closed rather than guessing which transcript owns - subsequent messages. - """ - if not parent_session_id: - return None - with self._read_ctx() as conn: - parent = conn.execute( - "SELECT ended_at, end_reason FROM sessions WHERE id = ?", - (parent_session_id,), - ).fetchone() - if ( - parent is None - or parent["ended_at"] is None - or parent["end_reason"] != "compression" - ): - return None - rows = conn.execute( - """ - SELECT s.*, - COALESCE(sp.prompt, s.system_prompt) - AS _system_prompt_resolved - FROM sessions s - LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash - WHERE s.parent_session_id = ? - AND s.ended_at IS NULL - """ - + self._NON_CONTINUATION_CHILD_FILTER_SQL.format(alias="s.") - + """ - ORDER BY s.started_at ASC - LIMIT 2 - """, - (parent_session_id, parent_session_id, parent_session_id), - ).fetchall() - return self._session_row_dict(rows[0]) if len(rows) == 1 else None - - def reopen_orphaned_compression_session(self, session_id: str) -> bool: - """Reopen a compression parent only when no continuation was published. - - Compression publication is atomic in current builds, but older builds - could leave a closed parent behind after an interrupted handoff. This - recovery is deliberately conservative: an active compression lease or - any canonical child means the lineage is still owned by another path, - so the caller must fail closed instead of reopening the parent. - """ - if not session_id: - return False - - def _do(conn): - parent = conn.execute( - "SELECT ended_at, end_reason FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if ( - parent is None - or parent["ended_at"] is None - or parent["end_reason"] != "compression" - ): - return False - - # Treat any direct non-branch/non-delegate/non-tool child as a - # continuation, regardless of its current ended state. Reopening - # in that case could create a second live head for one lineage. - child = conn.execute( - """ - SELECT 1 - FROM sessions - WHERE parent_session_id = ? - """ - + self._NON_CONTINUATION_CHILD_FILTER_SQL.format(alias="") - + """ - LIMIT 1 - """, - (session_id, session_id, session_id), - ).fetchone() - if child is not None: - return False - - # refresh_compression_lock() deliberately lets an owner revive its - # own expired row. Reclaim that row inside this write transaction - # before reopening: refresh-first makes the lease active and aborts - # recovery; recovery-first deletes the holder identity so a later - # refresh cannot resurrect it. - now = time.time() - lock_row = conn.execute( - "SELECT holder, expires_at FROM compression_locks " - "WHERE session_id = ?", - (session_id,), - ).fetchone() - if lock_row is not None: - expires_at = lock_row["expires_at"] - if expires_at is None or float(expires_at) >= now: - return False - deleted = conn.execute( - "DELETE FROM compression_locks " - "WHERE session_id = ? AND holder = ? AND expires_at = ?", - (session_id, lock_row["holder"], expires_at), - ) - if deleted.rowcount != 1: - return False - - updated = conn.execute( - "UPDATE sessions SET ended_at = NULL, end_reason = NULL " - "WHERE id = ? AND ended_at IS NOT NULL " - "AND end_reason = 'compression'", - (session_id,), - ) - # rowcount==1 is guaranteed by the parent SELECT at the top of - # this same BEGIN IMMEDIATE transaction. If this is ever edited - # to return False past this point, note that the lease DELETE - # above will still COMMIT (_execute_write commits unless _do - # raises) — raise instead of returning False to roll back. - return updated.rowcount == 1 - - return bool(self._execute_write(_do)) - - def publish_compression_child( - self, - *, - parent_session_id: str, - child_session_id: str, - source: str, - messages: List[Dict[str, Any]], - model: str = None, - model_config: Dict[str, Any] = None, - system_prompt: str = None, - cwd: str = None, - profile_name: str = None, - compression_lock_holder: str = None, - require_compression_lease: bool = True, - require_lease_refresh: bool = False, - lease_ttl_seconds: float = 300.0, - watermark: Optional[int] = None, - watermark_ceiling: Optional[int] = None, - ) -> None: - """Atomically close a parent and publish its durable compression child. - - The parent closure, child row, and compacted handoff become visible in - one transaction. Readers can therefore observe either the live parent or - a complete child, never an ended parent with a missing/empty child. - - Concurrent-append safety (#75316): when *watermark* is provided (the - parent's :meth:`get_active_message_watermark` captured at compression - start), parent rows that arrived during the slow summary call - (``id > watermark``) are cloned into the child AFTER the handoff — - same pure-SQL column clone as :meth:`archive_and_compact`, with the - session id rewritten — so a mid-compression append survives rotation - instead of stranding in the closed parent. - - *watermark_ceiling* bounds the clone from above: the rotation path - flushes its OWN un-persisted input transcript to the parent right - before publishing (#47202), and those rows are already represented in - the compacted handoff — cloning them would duplicate the transcript. - The caller captures ``MAX(id)`` immediately BEFORE that flush; only - rows in ``(watermark, watermark_ceiling]`` are foreign concurrent - tail. ``None`` = unbounded (no internal flush happened). - - When *require_lease_refresh* is True and *compression_lock_holder* is - set, the lease is refreshed inside the same transaction before the - expiry check. This gives a refresher - that stopped due to transient DB failures one final chance to extend - the lease, preventing wasted compression work. The refresh uses the - same ``conn`` as the publication, so there is no TOCTOU window. - """ - def _do(conn): - if require_lease_refresh and compression_lock_holder: - conn.execute( - "UPDATE compression_locks SET expires_at = ? " - "WHERE session_id = ? AND holder = ?", - (time.time() + lease_ttl_seconds, parent_session_id, - compression_lock_holder), - ) - lock_row = conn.execute( - "SELECT holder, expires_at FROM compression_locks WHERE session_id = ?", - (parent_session_id,), - ).fetchone() - if require_compression_lease and ( - lock_row is None - or not compression_lock_holder - or lock_row["holder"] != compression_lock_holder - or float(lock_row["expires_at"]) <= time.time() - ): - raise CompressionSessionBusyError( - f"Compression lease lost before publication: {parent_session_id}" - ) - parent = conn.execute( - """SELECT ended_at, end_reason, cwd, git_branch, git_repo_root, - user_id, session_key, chat_id, chat_type, - thread_id, display_name, origin_json, profile_name - FROM sessions WHERE id = ?""", - (parent_session_id,), - ).fetchone() - if parent is None: - raise RuntimeError(f"Compression parent not found: {parent_session_id}") - if parent["ended_at"] is not None: - # A parent stamped ended by AUTOMATIC cleanup (tui_shutdown, - # ws_disconnect, orphan reap, idle/LRU evict) while a live - # agent is publishing its rotation is stale by construction — - # this writer holds the compression lease and is actively - # continuing the conversation the stamp claims is over. - # Left in place it wedges rotation forever: every attempt - # aborts here, nothing clears the stamp, and each attempt's - # pre-publish flush re-grows the parent until the provider - # rejects the request (#88197: 303 unique messages → 2,611 - # rows → HTTP 400). Clear it in this same transaction and - # proceed; the closure UPDATE below re-stamps the parent with - # its true boundary (end_reason='compression'). Deliberate - # boundaries (compression, session_reset, explicit close) - # still fail closed — those mean another path owns lineage. - if is_automatic_end_reason(parent["end_reason"]): - conn.execute( - "UPDATE sessions SET ended_at = NULL, end_reason = NULL " - "WHERE id = ?", - (parent_session_id,), - ) - else: - raise RuntimeError( - f"Compression parent already ended: {parent_session_id}" - ) - if not messages: - raise RuntimeError("Compression child handoff must not be empty") - system_prompt_hash = self._store_system_prompt(conn, system_prompt) - - conn.execute( - """INSERT INTO sessions ( - id, source, model, model_config, system_prompt, - system_prompt_hash, - parent_session_id, cwd, git_branch, git_repo_root, - profile_name, user_id, session_key, chat_id, chat_type, - thread_id, display_name, origin_json, started_at - ) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - child_session_id, - source, - model, - json.dumps(model_config) if model_config else None, - system_prompt_hash, - parent_session_id, - cwd or parent["cwd"], - parent["git_branch"], - parent["git_repo_root"], - # Same inheritance contract as _insert_session_row's - # compression-fork backfill (#59527 / cross-profile jump - # fix): the child stays on the parent's profile and keeps - # the gateway routing/origin columns so peer recovery - # still works after a crash at the boundary. When neither - # names an owner (legacy NULL parent), stamp this store's - # own profile so the rotated child doesn't extend the - # unowned lineage (#99222). - profile_name - or parent["profile_name"] - or self._own_profile_name(), - parent["user_id"], - parent["session_key"], - parent["chat_id"], - parent["chat_type"], - parent["thread_id"], - parent["display_name"], - parent["origin_json"], - time.time(), - ), - ) - total_messages, total_tool_calls = self._insert_message_rows( - conn, child_session_id, messages - ) - if watermark is not None: - # Clone the parent's concurrent tail (rows landed after the - # watermark, at or below the ceiling — see docstring) into the - # child, after the handoff. Column-exact except id/session_id; - # originals stay in the (closed) parent for lineage recovery. - _ceiling_clause = "" - _params: list = [parent_session_id, int(watermark)] - if watermark_ceiling is not None: - _ceiling_clause = " AND id <= ?" - _params.append(int(watermark_ceiling)) - tail_rows = conn.execute( - "SELECT id, tool_calls FROM messages " - "WHERE session_id = ? AND active = 1 AND id > ?" - f"{_ceiling_clause} ORDER BY id", - _params, - ).fetchall() - if tail_rows: - tail_ids = [int(r["id"]) for r in tail_rows] - placeholders = ",".join("?" for _ in tail_ids) - clone_cols = [ - c for c in self._message_column_names(conn) - if c not in ("id", "session_id", "active", "compacted") - ] - col_list = ", ".join(clone_cols) - conn.execute( - f"INSERT INTO messages ({col_list}, session_id, active, compacted) " - f"SELECT {col_list}, ?, 1, 0 FROM messages " - f"WHERE id IN ({placeholders}) ORDER BY id", - [child_session_id, *tail_ids], - ) - total_messages += len(tail_ids) - for r in tail_rows: - raw = r["tool_calls"] - if raw: - try: - parsed = json.loads(raw) if isinstance(raw, str) else raw - total_tool_calls += len(parsed) if isinstance(parsed, list) else 0 - except (TypeError, ValueError): - pass - conn.execute( - "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", - (total_messages, total_tool_calls, child_session_id), - ) - updated = conn.execute( - "UPDATE sessions SET ended_at = ?, end_reason = 'compression' " - "WHERE id = ? AND ended_at IS NULL", - (time.time(), parent_session_id), - ) - if updated.rowcount != 1: - raise RuntimeError( - f"Compression parent changed during publication: {parent_session_id}" - ) - - self._execute_write(_do) - - def _bump_conversation_generation(self, conn, session_id: str, end_reason: str) -> None: - """Advance this peer's conversation generation past a boundary. - - Called inside the transaction that writes the boundary, so the - generation and the ``end_reason`` that caused it commit together. - - Only ``_RESET_END_REASONS`` count: ``compression`` continues one - conversation, and an accidental close is not a replacement. Rows with - no ``session_key`` have no routing peer to advance. - - The counter deliberately does NOT read the session rows. An aggregate - over them (COUNT/MAX of boundaries) can return a pair it already - emitted once ``delete_session()`` or bulk pruning removes an ended row, - which would hand a new conversation a retired affinity identity. This - value only ever increments, so a generation is never reused for a peer - even if every row behind it is gone. - """ - if end_reason not in _RESET_END_REASONS: - return - row = conn.execute( - "SELECT source, session_key FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if row is None: - return - source = str(row["source"] or "").strip() - session_key = str(row["session_key"] or "").strip() - if not source or not session_key: - return - conn.execute( - """ - INSERT INTO conversation_generations (source, session_key, generation) - VALUES (?, ?, 1) - ON CONFLICT(source, session_key) DO UPDATE - SET generation = conversation_generations.generation + 1 - """, - (source, session_key), - ) def end_session(self, session_id: str, end_reason: str) -> None: """Mark a session as ended. @@ -8753,7 +4301,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) if current is None: return None - current_cwd = current["cwd"] if isinstance(current, sqlite3.Row) else current[0] + current_cwd = current[0] sets = [ "cwd = ?", "git_metadata_generation = COALESCE(git_metadata_generation, 0) + 1", @@ -8778,7 +4326,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ).fetchone() if row is None: return None - value = row["git_metadata_generation"] if isinstance(row, sqlite3.Row) else row[0] + value = row[0] return int(value) return self._execute_write(_do) @@ -8816,16 +4364,12 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) params.append(repo_root) params.extend((session_id, cwd, generation)) - def _do(conn): - cursor = conn.execute( - f"UPDATE sessions SET {', '.join(sets)} " - "WHERE id = ? AND cwd = ? " - "AND git_metadata_generation = ?", - params, - ) - return cursor.rowcount == 1 - - return bool(self._execute_write(_do)) + return self._write_rowcount( + f"UPDATE sessions SET {', '.join(sets)} " + "WHERE id = ? AND cwd = ? " + "AND git_metadata_generation = ?", + params, + ) == 1 def backfill_repo_roots(self, cwd_to_root: Dict[str, str]) -> None: """Persist resolved git repo roots for cwds that don't have one yet. @@ -8848,344 +4392,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._execute_write(_do) - def record_compression_failure_cooldown( - self, - session_id: str, - cooldown_until: float, - error: Optional[str] = None, - ) -> None: - """Persist the active compression-failure cooldown for a session.""" - if not session_id: - return - - def _do(conn): - # Merge-max with any longer live deadline so a later shorter - # write cannot reopen the thrash window (#96775). The error - # column always takes the latest diagnostic. - conn.execute( - "UPDATE sessions SET compression_failure_cooldown_until = CASE " - "WHEN compression_failure_cooldown_until IS NOT NULL " - " AND compression_failure_cooldown_until > ? " - "THEN compression_failure_cooldown_until ELSE ? END, " - "compression_failure_error = ? WHERE id = ?", - (cooldown_until, cooldown_until, error, session_id), - ) - - try: - self._execute_write(_do) - except sqlite3.Error as exc: - logger.warning( - "record_compression_failure_cooldown(%s) failed: %s", - session_id, exc, - ) - - def get_compression_failure_cooldown( - self, - session_id: str, - ) -> Optional[Dict[str, Any]]: - """Return the active compression-failure cooldown for ``session_id``.""" - if not session_id: - return None - now = time.time() - with self._read_ctx() as conn: - row = conn.execute( - "SELECT compression_failure_cooldown_until, compression_failure_error " - "FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if row is None: - return None - cooldown_until = ( - row["compression_failure_cooldown_until"] - if isinstance(row, sqlite3.Row) - else row[0] - ) - if cooldown_until is None: - return None - cooldown_until = float(cooldown_until) - if cooldown_until <= now: - return None - error = ( - row["compression_failure_error"] - if isinstance(row, sqlite3.Row) - else row[1] - ) - return { - "cooldown_until": cooldown_until, - "remaining_seconds": cooldown_until - now, - "error": error, - } - - def get_compression_failure_cooldown_row( - self, - session_id: str, - ) -> Dict[str, Any]: - """Return the exact stored cooldown columns without expiry filtering. - - Compression cancellation uses this under its session lease so rollback - can preserve an expired row, a partially-null row, or an absent session - exactly instead of converting those states through the active-cooldown - API. - """ - if not session_id: - return {"session_exists": False, "cooldown_until": None, "error": None} - with self._read_ctx() as conn: - row = conn.execute( - "SELECT compression_failure_cooldown_until, compression_failure_error " - "FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if row is None: - return {"session_exists": False, "cooldown_until": None, "error": None} - cooldown_until = ( - row["compression_failure_cooldown_until"] - if isinstance(row, sqlite3.Row) - else row[0] - ) - error = ( - row["compression_failure_error"] - if isinstance(row, sqlite3.Row) - else row[1] - ) - return { - "session_exists": True, - "cooldown_until": ( - float(cooldown_until) if cooldown_until is not None else None - ), - "error": error, - } - - def restore_compression_failure_cooldown_row( - self, - session_id: str, - snapshot: Dict[str, Any], - ) -> None: - """Restore and verify an exact cooldown-row snapshot. - - Unlike the ordinary record/clear helpers, this transactional rollback - API deliberately propagates write and verification failures. A caller - must not report cancellation as mutation-free when compensation failed. - """ - expected_exists = bool(snapshot.get("session_exists", False)) - if not expected_exists: - actual = self.get_compression_failure_cooldown_row(session_id) - if actual.get("session_exists", False): - raise RuntimeError( - "cannot restore absent compression cooldown row: session now exists" - ) - return - - deadline = snapshot.get("cooldown_until") - error = snapshot.get("error") - - def _do(conn): - cursor = conn.execute( - "UPDATE sessions SET compression_failure_cooldown_until = ?, " - "compression_failure_error = ? WHERE id = ?", - (deadline, error, session_id), - ) - if cursor.rowcount != 1: - raise RuntimeError( - f"compression cooldown rollback session missing: {session_id}" - ) - - self._execute_write(_do) - actual = self.get_compression_failure_cooldown_row(session_id) - expected = { - "session_exists": True, - "cooldown_until": float(deadline) if deadline is not None else None, - "error": error, - } - if actual != expected: - raise RuntimeError( - f"compression cooldown rollback verification failed: " - f"expected={expected!r}, actual={actual!r}" - ) - - def clear_compression_failure_cooldown(self, session_id: str) -> None: - """Clear any persisted compression-failure cooldown for a session.""" - if not session_id: - return - - def _do(conn): - conn.execute( - "UPDATE sessions SET compression_failure_cooldown_until = NULL, " - "compression_failure_error = NULL WHERE id = ?", - (session_id,), - ) - - try: - self._execute_write(_do) - except sqlite3.Error as exc: - logger.warning( - "clear_compression_failure_cooldown(%s) failed: %s", - session_id, exc, - ) - - def get_compression_fallback_streak(self, session_id: str) -> int: - """Return the persisted deterministic-fallback streak.""" - if not session_id: - return 0 - with self._read_ctx() as conn: - if conn is None: - return 0 - row = conn.execute( - "SELECT compression_fallback_streak FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if row is None: - return 0 - value = ( - row["compression_fallback_streak"] - if isinstance(row, sqlite3.Row) - else row[0] - ) - try: - return max(0, int(value or 0)) - except (TypeError, ValueError): - return 0 - - def set_compression_fallback_streak(self, session_id: str, streak: int) -> None: - """Persist the deterministic-fallback streak for one session.""" - if not session_id: - return - normalized = max(0, int(streak)) - - def _do(conn): - conn.execute( - "UPDATE sessions SET compression_fallback_streak = ? WHERE id = ?", - (normalized, session_id), - ) - - self._execute_write(_do) - - def increment_hygiene_failure_streak(self, session_key: str) -> int: - """Atomically increment the session-hygiene failure streak for one chat.""" - if not session_key: - return 1 - result = [] - - def _do(conn): - conn.execute( - """INSERT INTO gateway_hygiene_state (session_key, failure_streak) - VALUES (?, 1) - ON CONFLICT(session_key) DO UPDATE SET - failure_streak = gateway_hygiene_state.failure_streak + 1""", - (session_key,), - ) - row = conn.execute( - "SELECT failure_streak FROM gateway_hygiene_state WHERE session_key = ?", - (session_key,), - ).fetchone() - result.append(int(row[0])) - - self._execute_write(_do) - return result[0] - - def reset_hygiene_failure_streak(self, session_key: str) -> None: - """Clear the persisted session-hygiene failure streak for one chat.""" - if not session_key: - return - - def _do(conn): - conn.execute( - "DELETE FROM gateway_hygiene_state WHERE session_key = ?", - (session_key,), - ) - - self._execute_write(_do) - - def get_compression_ineffective_count(self, session_id: str) -> int: - """Return the persisted ineffective-compaction strike count. - - Mirrors ``get_compression_fallback_streak``: this is the durable half - of the anti-thrash guard (``_ineffective_compression_count`` on the - built-in compressor), persisted so that a fresh compressor bound to a - resumed session inherits an armed/tripped guard instead of starting - from zero across process restarts (#54923). - """ - if not session_id: - return 0 - with self._read_ctx() as conn: - if conn is None: - return 0 - row = conn.execute( - "SELECT compression_ineffective_count FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if row is None: - return 0 - value = ( - row["compression_ineffective_count"] - if isinstance(row, sqlite3.Row) - else row[0] - ) - try: - return max(0, int(value or 0)) - except (TypeError, ValueError): - return 0 - - def set_compression_ineffective_count(self, session_id: str, count: int) -> None: - """Persist the ineffective-compaction strike count for one session.""" - if not session_id: - return - normalized = max(0, int(count)) - - def _do(conn): - conn.execute( - "UPDATE sessions SET compression_ineffective_count = ? WHERE id = ?", - (normalized, session_id), - ) - - self._execute_write(_do) - - def get_compression_recovery_deadline(self, session_id: str) -> float: - """Return the persisted anti-thrash recovery deadline (wall-clock epoch). - - ``0.0`` means "not armed". The deadline is the durable half of the - #14694 recovery clock: the gateway rebuilds the compressor on every - turn / cache eviction, so a process-local deadline restarted the - wait on each rebuild and a tripped session never earned its probe - (#100185). - """ - if not session_id: - return 0.0 - with self._read_ctx() as conn: - if conn is None: - return 0.0 - row = conn.execute( - "SELECT compression_recovery_deadline FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if row is None: - return 0.0 - value = ( - row["compression_recovery_deadline"] - if isinstance(row, sqlite3.Row) - else row[0] - ) - try: - return max(0.0, float(value or 0.0)) - except (TypeError, ValueError): - return 0.0 - - def set_compression_recovery_deadline(self, session_id: str, deadline: float) -> None: - """Persist the anti-thrash recovery deadline; ``0`` / ``None`` disarms it.""" - if not session_id: - return - try: - normalized = max(0.0, float(deadline or 0.0)) - except (TypeError, ValueError): - normalized = 0.0 - stored = normalized if normalized > 0.0 else None - - def _do(conn): - conn.execute( - "UPDATE sessions SET compression_recovery_deadline = ? WHERE id = ?", - (stored, session_id), - ) - - self._execute_write(_do) # ────────────────────────────────────────────────────────────────────── # Compression locks @@ -9208,398 +4414,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # the compress() call plus the rotation. ``holder`` identifies the # current owner (pid:tid:nonce) for diagnostics; the lock is recovered # via ``expires_at`` if the holder process crashed without releasing. - def refresh_compression_lock( - self, - session_id: str, - holder: str, - ttl_seconds: float = 300.0, - ) -> bool: - """Extend the compression lock lease if ``holder`` still owns it. - Ownership is decided by the ``holder`` column alone, deliberately NOT - by ``expires_at``: a live owner whose refresher thread was starved - (GC pause, loaded CI runner, a slow write escaping ``_execute_write``'s - retry budget) past its own TTL must be able to revive its still-unclaimed - row on the next tick. Requiring ``expires_at >= now`` here made such a - stall permanent — every later refresh matched 0 rows, so the owner kept - compressing and rotating with no lease at all, which is exactly the - unprotected window a competing path can fork the session lineage in. - - This does not resurrect a lock somebody else already took: SQLite - serialises writes, so a reclaim (DELETE-expired + INSERT-or-IGNORE in - :meth:`try_acquire_compression_lock`) and this UPDATE never interleave. - Reclaim-first replaces ``holder``, so this UPDATE matches nothing and - returns False; refresh-first pushes ``expires_at`` into the future, so - the reclaimer's DELETE-expired matches nothing and its acquire fails. - """ - if not session_id or not holder: - return False - now = time.time() - expires_at = now + ttl_seconds - - def _do(conn): - cur = conn.execute( - "UPDATE compression_locks SET expires_at = ? " - "WHERE session_id = ? AND holder = ?", - (expires_at, session_id, holder), - ) - return cur.rowcount > 0 - - try: - return bool(self._execute_write(_do)) - except sqlite3.Error as exc: - logger.warning( - "refresh_compression_lock(%s) failed: %s", - session_id, exc, - ) - return False - - def try_acquire_compression_lock( - self, - session_id: str, - holder: str, - ttl_seconds: float = 300.0, - ) -> bool: - """Try to atomically acquire the compression lock for ``session_id``. - - Returns ``True`` on success (caller now owns the lock and must - release via :meth:`release_compression_lock`). Returns ``False`` - if another holder already owns a non-expired lock — the caller - MUST NOT proceed with compression in that case (its rotation would - race against the holder's, splitting the session lineage). - - Expired locks (``expires_at < now``) are reclaimed transparently. - Structured holders whose local ``pid=`` no longer exists are reclaimed - immediately, so a gateway killed during compression does not stall the - replacement process for the full lease TTL. - - Implementation: single-transaction DELETE-expired + INSERT-or-IGNORE, - followed by a SELECT to confirm we got the row. SQLite serialises - writes, so the whole sequence is atomic against other writers. - """ - if not session_id: - return False - now = time.time() - expires_at = now + ttl_seconds - - def _do(conn): - reclaimed_holder = None - row = conn.execute( - "SELECT holder, expires_at FROM compression_locks " - "WHERE session_id = ?", - (session_id,), - ).fetchone() - if row is not None: - current_holder = ( - row["holder"] if isinstance(row, sqlite3.Row) else row[0] - ) - current_expires_at = ( - row["expires_at"] if isinstance(row, sqlite3.Row) else row[1] - ) - if ( - current_expires_at < now - or _compression_lock_holder_process_is_dead(current_holder) - ): - conn.execute( - "DELETE FROM compression_locks " - "WHERE session_id = ? AND holder = ?", - (session_id, current_holder), - ) - reclaimed_holder = current_holder - # Then: try to insert. INSERT OR IGNORE returns no rowcount - # difference — verify ownership via SELECT. - conn.execute( - "INSERT OR IGNORE INTO compression_locks " - "(session_id, holder, acquired_at, expires_at) " - "VALUES (?, ?, ?, ?)", - (session_id, holder, now, expires_at), - ) - row = conn.execute( - "SELECT holder FROM compression_locks WHERE session_id = ?", - (session_id,), - ).fetchone() - acquired = row is not None and ( - row["holder"] if isinstance(row, sqlite3.Row) else row[0] - ) == holder - return acquired, reclaimed_holder - - try: - acquired, reclaimed_holder = self._execute_write(_do) - if reclaimed_holder: - logger.warning( - "Reclaimed stale compression lock for session=%s " - "(holder=%s)", - session_id, - reclaimed_holder, - ) - return bool(acquired) - except sqlite3.Error as exc: - logger.warning( - "try_acquire_compression_lock(%s) failed: %s", - session_id, exc, - ) - # Fail open: returning False makes the caller skip compression, - # which is the safe behaviour when the lock subsystem is broken. - return False - - def release_compression_lock(self, session_id: str, holder: str) -> None: - """Release the compression lock for ``session_id`` iff we own it. - - Idempotent: no-op when the lock has already expired and been - reclaimed by a different holder, or when no lock exists. The - ``holder`` check prevents a late-returning compressor from - clobbering a fresh lock held by someone else. - """ - if not session_id: - return - - def _do(conn): - conn.execute( - "DELETE FROM compression_locks " - "WHERE session_id = ? AND holder = ?", - (session_id, holder), - ) - - try: - self._execute_write(_do) - except sqlite3.Error as exc: - logger.warning( - "release_compression_lock(%s) failed: %s", - session_id, exc, - ) - - def _session_turn_lease_key_on_conn(self, conn, session_id: str) -> str: - """Walk compression parents on ``conn`` to the conversation lease key. - - Must run on the same connection as the lease INSERT/UPDATE/DELETE. - A prior ``get_session`` failure must not compute a child id that the - later write then persists: refresh would walk to the parent and - fail-close. Markers bind to ``parent_session_id`` (same contract as - ``_NON_CONTINUATION_CHILD_FILTER_SQL``). Lock errors propagate so - ``_execute_write`` / ``acquire_session_turn_lease`` can retry. - """ - if not session_id: - return session_id - - def _row(sid: str): - row = conn.execute( - "SELECT id, parent_session_id, source, model_config, end_reason " - "FROM sessions WHERE id = ?", - (sid,), - ).fetchone() - return dict(row) if row else None - - current = _row(session_id) - seen = {session_id} - while current: - parent_id = current.get("parent_session_id") - if ( - not parent_id - or parent_id in seen - or self._is_explicit_fork_child_row(current) - ): - break - parent = _row(parent_id) - if not parent or parent.get("end_reason") != "compression": - break - seen.add(parent_id) - current = parent - return str(current.get("id") or session_id) if current else session_id - - def _session_turn_lease_key(self, session_id: str) -> str: - """Return the stable serialization key for every compression segment. - - Acquire/refresh/release resolve this inside their write transaction. - This helper is for tests and diagnostics; it does not swallow lock - errors (a swallowed walk plus a later successful write was the - fail-open that replayed the post-rotation refresh miss). - """ - if not session_id: - return session_id - with self._read_ctx() as conn: - return self._session_turn_lease_key_on_conn(conn, session_id) - - def try_acquire_session_turn_lease( - self, - session_id: str, - holder: str, - *, - ttl_seconds: float = 300.0, - patience_s: Optional[float] = None, - ) -> bool: - """Atomically acquire the cross-process turn lease for a conversation. - - Compression rotates a session into child segments, so the durable key - is the lineage root rather than the current segment id. The walk and - INSERT share one write transaction. Expired leases and leases whose - structured local holder PID is known dead are reclaimed in that same - transaction. - """ - if not session_id or not holder: - return False - now = time.time() - expires_at = now + max(0.1, float(ttl_seconds)) - - def _do(conn): - conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) - row = conn.execute( - "SELECT holder, expires_at FROM session_turn_leases " - "WHERE conversation_id = ?", - (conversation_id,), - ).fetchone() - if row is not None: - current_holder = row["holder"] - if ( - float(row["expires_at"]) <= now - or _compression_lock_holder_process_is_dead(current_holder) - ): - conn.execute( - "DELETE FROM session_turn_leases " - "WHERE conversation_id = ? AND holder = ?", - (conversation_id, current_holder), - ) - conn.execute( - "INSERT OR IGNORE INTO session_turn_leases " - "(conversation_id, holder, acquired_at, expires_at) " - "VALUES (?, ?, ?, ?)", - (conversation_id, holder, now, expires_at), - ) - owner = conn.execute( - "SELECT holder FROM session_turn_leases WHERE conversation_id = ?", - (conversation_id,), - ).fetchone() - return owner is not None and owner["holder"] == holder - - return bool(self._execute_write(_do, patience_s=patience_s)) - - def acquire_session_turn_lease( - self, - session_id: str, - holder: str, - *, - ttl_seconds: float = 300.0, - wait_seconds: float = 1800.0, - poll_interval_seconds: float = 1.0, - on_wait=None, - wait_notice_interval_seconds: float = 15.0, - should_abort=None, - acquire_patience_s: float = 0.5, - ) -> bool: - """Wait for a cross-process turn lease without holding a SQLite lock. - - ``on_wait(elapsed_seconds)`` is best-effort: invoked when the first - attempt fails (elapsed ~0) and again about every - ``wait_notice_interval_seconds`` while still waiting, so UIs can show - that another process holds the conversation. - - When ``should_abort()`` returns True (for example the agent received - ``/stop`` while waiting), acquisition stops immediately and returns - False without consuming the full ``wait_seconds`` budget. - """ - deadline = time.monotonic() + max(0.0, float(wait_seconds)) - wait_started = None - last_notice_at = None - notice_every = max(0.0, float(wait_notice_interval_seconds)) - while True: - if should_abort is not None: - try: - if should_abort(): - return False - except Exception: - logger.debug( - "session turn lease should_abort callback failed", - exc_info=True, - ) - try: - if self.try_acquire_session_turn_lease( - session_id, - holder, - ttl_seconds=ttl_seconds, - patience_s=acquire_patience_s, - ): - return True - except sqlite3.Error as exc: - # Long holder transactions (compression publish, large - # flushes) can exhaust a single write-patience budget. - # Keep polling until wait_seconds or should_abort. - if classify_persistence_error(exc) != "locked": - raise - now = time.monotonic() - remaining = deadline - now - if remaining <= 0: - return False - if wait_started is None: - wait_started = now - if on_wait is not None and ( - last_notice_at is None - or notice_every == 0.0 - or (now - last_notice_at) >= notice_every - ): - try: - on_wait(max(0.0, now - wait_started)) - except Exception: - logger.debug( - "session turn lease on_wait callback failed", - exc_info=True, - ) - last_notice_at = now - time.sleep(min(max(0.01, float(poll_interval_seconds)), remaining)) - - def refresh_session_turn_lease( - self, - session_id: str, - holder: str, - *, - ttl_seconds: float = 300.0, - ) -> bool: - """Extend a turn lease only while ``holder`` still owns it.""" - if not session_id or not holder: - return False - expires_at = time.time() + max(0.1, float(ttl_seconds)) - - def _do(conn): - conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) - cursor = conn.execute( - "UPDATE session_turn_leases SET expires_at = ? " - "WHERE conversation_id = ? AND holder = ?", - (expires_at, conversation_id, holder), - ) - return cursor.rowcount > 0 - - return bool(self._execute_write(_do)) - - def release_session_turn_lease(self, session_id: str, holder: str) -> None: - """Release a turn lease iff ``holder`` still owns it; idempotent.""" - if not session_id or not holder: - return - - def _do(conn): - conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) - conn.execute( - "DELETE FROM session_turn_leases " - "WHERE conversation_id = ? AND holder = ?", - (conversation_id, holder), - ) - - self._execute_write(_do) - - def get_compression_lock_holder(self, session_id: str) -> Optional[str]: - """Return the current (non-expired) holder for ``session_id``, or None. - - Diagnostic helper — not used by the locking protocol itself. - """ - if not session_id: - return None - now = time.time() - with self._read_ctx() as conn: - row = conn.execute( - "SELECT holder FROM compression_locks " - "WHERE session_id = ? AND expires_at >= ?", - (session_id, now), - ).fetchone() - if row is None: - return None - return row["holder"] if isinstance(row, sqlite3.Row) else row[0] def touch_session_activity( self, @@ -9631,22 +4446,20 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) desc = bound_activity_description(description) prov = normalize_activity_provenance(provenance).value - def _do(conn): - conn.execute( - "UPDATE sessions SET " - "last_activity_at = ?, " - "last_activity_description = ?, " - "last_activity_provenance = ? " - "WHERE id = ? AND (last_activity_at IS NULL OR last_activity_at < ?)", - (when, desc, prov, session_id, when), - ) - # Observation-only write: never let it ride the full routine # write-patience budget (#76354 review S1). Under contention a # heartbeat that waits ~20s would delay the response-critical path # it is merely observing; give up after a sub-second budget instead # (the next due window retries naturally). - self._execute_write(_do, patience_s=self._ACTIVITY_WRITE_PATIENCE_S) + self._write_sql( + "UPDATE sessions SET " + "last_activity_at = ?, " + "last_activity_description = ?, " + "last_activity_provenance = ? " + "WHERE id = ? AND (last_activity_at IS NULL OR last_activity_at < ?)", + (when, desc, prov, session_id, when), + patience_s=self._ACTIVITY_WRITE_PATIENCE_S, + ) def clear_session_activity_labels(self, session_id: str) -> None: """Clear mid-turn activity labels after a turn ends. @@ -9670,32 +4483,29 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # No-op fast path: skip the transaction when there is nothing to # clear. Read-only, no write lock. try: - with self._read_ctx() as conn: - row = conn.execute( - "SELECT last_activity_description, last_activity_provenance " - "FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() + row = self._read_one( + "SELECT last_activity_description, last_activity_provenance " + "FROM sessions WHERE id = ?", + (session_id,), + ) except sqlite3.Error: row = None if row is not None: - desc = row[0] if not isinstance(row, sqlite3.Row) else row["last_activity_description"] - prov = row[1] if not isinstance(row, sqlite3.Row) else row["last_activity_provenance"] + desc = row[0] + prov = row[1] if not desc and ( not prov or prov == ActivityProvenance.UNKNOWN.value ): return - def _do(conn): - conn.execute( - "UPDATE sessions SET " - "last_activity_description = ?, " - "last_activity_provenance = ? " - "WHERE id = ?", - ("", ActivityProvenance.UNKNOWN.value, session_id), - ) - - self._execute_write(_do, patience_s=self._ACTIVITY_WRITE_PATIENCE_S) + self._write_sql( + "UPDATE sessions SET " + "last_activity_description = ?, " + "last_activity_provenance = ? " + "WHERE id = ?", + ("", ActivityProvenance.UNKNOWN.value, session_id), + patience_s=self._ACTIVITY_WRITE_PATIENCE_S, + ) def get_session_activity(self, session_id: str) -> Optional[Dict[str, Any]]: """Return the durable activity snapshot for *session_id*, or None.""" @@ -9727,12 +4537,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # Barrier against queued token deltas — see update_session_model. self.flush_token_counts() - def _do(conn): - conn.execute( - "UPDATE sessions SET model_config = ?, model = COALESCE(?, model) WHERE id = ?", - (model_config_json, model, session_id), - ) - self._execute_write(_do) + self._write_sql( + "UPDATE sessions SET model_config = ?, model = COALESCE(?, model) WHERE id = ?", + (model_config_json, model, session_id), + ) def update_system_prompt( self, session_id: str, system_prompt: Optional[str] @@ -9760,12 +4568,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) """ payload = json.dumps(list(tool_names)) if tool_names is not None else None - def _do(conn): - conn.execute( - "UPDATE sessions SET tool_names = ? WHERE id = ?", - (payload, session_id), - ) - self._execute_write(_do) + self._write_sql( + "UPDATE sessions SET tool_names = ? WHERE id = ?", + (payload, session_id), + ) def update_session_model( self, session_id: str, model: str, provider: Optional[str] = None @@ -9848,7 +4654,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) if on_missing == "raise": raise ValueError(f"Session not found: {session_id}") return _MODEL_CONFIG_ROW_MISSING - raw = row["model_config"] if isinstance(row, sqlite3.Row) else row[0] + raw = row[0] config: Dict[str, Any] = {} if isinstance(raw, str) and raw.strip(): try: @@ -9997,93 +4803,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return False return bool(raw.get("yolo_mode")) - @staticmethod - def session_gateway_runtime(session_meta: Optional[Dict[str, Any]]) -> Dict[str, Any]: - """Read the persisted runtime route off a session row dict. - - Accepts the dict returned by ``get_session`` (``model_config`` is a - JSON string) or an already-parsed dict. Prefers the nested - ``gateway_runtime`` key (written by the gateway's - ``_sync_session_model_from_agent`` and the CLI ``/model`` persist), - falling back to the top-level ``provider``/``base_url``/``api_mode`` - keys the TUI gateway's ``_runtime_model_config`` writes. As a last - resort, falls back to the ``billing_provider`` column (written on - every session's first accounted API call) so sessions that never ran - ``/model`` still restore the provider that actually served them. - Returns an empty dict on any parse failure — resume falls back to - ambient config resolution. - """ - raw = (session_meta or {}).get("model_config") - if isinstance(raw, str): - try: - raw = json.loads(raw) - except Exception: - raw = {} - if not isinstance(raw, dict): - raw = {} - runtime = raw.get("gateway_runtime") - if isinstance(runtime, dict) and runtime.get("provider"): - # Filter None values: the persist path writes or-None to trigger - # deletion in the top-level merge, but gateway_runtime is replaced - # as a whole dict (not deep-merged), so None values survive here. - return {k: v for k, v in runtime.items() if v is not None} - top_level = { - key: raw.get(key) - for key in ("provider", "base_url", "api_mode") - if raw.get(key) - } - if top_level: - return top_level - # Last resort: billing_provider column. Written via COALESCE on every - # session's first accounted API call — the only durable record for - # sessions that never ran /model. Mirrors the TUI gateway's - # _stored_session_runtime_overrides fallback. Bare billing buckets - # ("auto"/"custom") are not routable identities — filter them out so - # resume falls back to the ambient config default instead. - billing_provider = str( - (session_meta or {}).get("billing_provider") or "" - ).strip() - if ( - billing_provider - and billing_provider.lower() not in _BARE_BILLING_PROVIDERS - ): - return {"provider": billing_provider} - return {k: v for k, v in (runtime or {}).items() if v is not None} if isinstance(runtime, dict) else {} - - def update_session_billing_route( - self, - session_id: str, - *, - provider: str, - base_url: str, - billing_mode: Optional[str] = None, - ) -> None: - """Unconditionally update the billing provider/base_url for a session. - - Unlike ``update_token_counts`` which uses ``COALESCE(billing_provider, ?)`` - (only filling in NULL), this unconditionally sets the billing fields so - that the dashboard reflects the user's latest /model switch. - - Also nulls ``system_prompt`` so the cached snapshot (which embeds a - stale ``Model:`` / ``Provider:`` header) is rebuilt — matching the - behavior of ``update_session_model`` (see #48173, #48248). - """ - # Barrier against queued token deltas — see update_session_model. - self.flush_token_counts() - - def _do(conn): - conn.execute( - """UPDATE sessions SET - billing_provider = ?, - billing_base_url = ?, - billing_mode = COALESCE(?, billing_mode), - system_prompt = NULL, - system_prompt_hash = NULL - WHERE id = ?""", - (provider, base_url, billing_mode, session_id), - ) - self._delete_unreferenced_system_prompts(conn) - self._execute_write(_do) # ── Async token accounting ── # update_token_counts() runs a sessions UPDATE (plus a per-model usage @@ -10112,524 +4831,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) "billing_provider", "billing_base_url", "billing_mode", ) - def queue_token_counts(self, session_id: str, **kwargs) -> None: - """Enqueue a token/cost delta for the background writer. - - Accepts the same keyword arguments as :meth:`update_token_counts` - and applies them asynchronously with identical semantics. Cheap - (append + notify) — safe to call on the turn thread after every - API call. After close() has stopped the writer, falls back to the - synchronous path and may raise like :meth:`update_token_counts`. - """ - with self._token_queue_cond: - thread = self._token_writer_thread - writer_stopped = self._token_writer_stop and ( - thread is None or not thread.is_alive() - ) - if not writer_stopped: - self._token_queue.append((session_id, kwargs)) - if thread is None or not thread.is_alive(): - # Daemon so process exit never hangs on accounting; the - # atexit hook drains anything still queued at interpreter - # shutdown (registered once per instance, on first use). - # ``not is_alive()`` (rather than ``is None`` only) - # respawns the writer if it ever died from an unexpected - # escape — otherwise a dead thread object would block - # respawn forever and deltas would pile up on the deque - # until a reader's flush drained them synchronously. - thread = threading.Thread( - target=self._token_writer_loop, - name="session-db-token-writer", - daemon=True, - ) - self._token_writer_thread = thread - thread.start() - if self._token_atexit_hook is None: - self_ref = weakref.ref(self) - - def _drain_at_exit() -> None: - db = self_ref() - if db is not None: - db._drain_token_queue_at_exit() - - self._token_atexit_hook = _drain_at_exit - atexit.register(_drain_at_exit) - self._token_queue_cond.notify_all() - if writer_stopped: - # Writer permanently stopped (close() ran; a stop-flagged but - # still-live writer keeps accepting — its loop drains before - # exiting). Enqueueing now would drop the delta silently: no - # writer will run and close() already unregistered the atexit - # hook. Apply inline instead so a closed-connection failure - # raises at the call site, exactly like the old synchronous - # update_token_counts path these call sites still guard for. - self.update_token_counts(session_id, **kwargs) - - def flush_token_counts(self, timeout: float = 5.0) -> bool: - """Block until every queued token delta has been applied. - - Returns True when the queue is fully drained, False on timeout - (callers then read totals that are stale by the still-queued - deltas — no worse than reading before the flush existed). - Never raises: apply failures are logged by the writer. - """ - # Fast path — nothing queued, nothing in flight. - if not self._token_queue and not self._token_writer_busy: - return True - batch = None - with self._token_queue_cond: - deadline = time.monotonic() + timeout - while self._token_queue or self._token_writer_busy: - # A live writer is authoritative even when stop-flagged - # (close() in progress): its loop drains the queue before - # exiting, and draining here instead would race its - # in-flight batch — newer deltas committing before older - # ones breaks the last-non-None-wins / first-accounted- - # route / COALESCE-backfill fields. Only when the writer is - # dead (or never started for these deltas) does the caller - # take the leftovers. Re-checked each wakeup: the writer - # can exit mid-wait with deltas enqueued after its final - # empty-queue check. busy is claimed while draining (same - # protocol as the writer) so a concurrent flush cannot - # report drained — or pop a newer delta — while this batch - # is still unapplied; a claimed busy therefore also means - # "wait", never "drain alongside". - thread = self._token_writer_thread - if ( - (thread is None or not thread.is_alive()) - and not self._token_writer_busy - ): - self._token_writer_busy = True - batch = list(self._token_queue) - self._token_queue.clear() - break - remaining = deadline - time.monotonic() - if remaining <= 0: - return False - self._token_queue_cond.wait(remaining) - if batch: - try: - self._apply_token_batch(batch) - finally: - with self._token_queue_cond: - self._token_writer_busy = False - self._token_queue_cond.notify_all() - return True - - def _token_writer_loop(self) -> None: - while True: - with self._token_queue_cond: - idle_deadline = time.monotonic() + self._TOKEN_WRITER_IDLE_SECONDS - while not self._token_queue and not self._token_writer_stop: - remaining = idle_deadline - time.monotonic() - if remaining <= 0: - # Publish retirement under the same lock used by - # queue_token_counts() to decide whether to spawn. An - # enqueue cannot strand a delta behind an exiting worker. - self._token_writer_thread = None - return - self._token_queue_cond.wait(remaining) - if not self._token_queue: - self._token_writer_thread = None - return # stop requested and fully drained - # busy is set BEFORE the queue is cleared: the lock-free - # fast path in flush_token_counts() reads queue-then-busy, - # so this order guarantees it can never observe an empty - # queue while the popped batch is still unapplied. - self._token_writer_busy = True - batch = list(self._token_queue) - self._token_queue.clear() - try: - self._apply_token_batch(batch) - finally: - with self._token_queue_cond: - self._token_writer_busy = False - self._token_queue_cond.notify_all() - - def _apply_token_batch(self, batch: List[Tuple[str, Dict[str, Any]]]) -> None: - """Apply queued deltas in order, coalescing where safe. Never raises.""" - try: - coalesced = self._coalesce_token_deltas(batch) - except Exception as exc: - # Coalescing must never kill the writer thread (a dead writer - # can't be observed by callers). Fall back to applying the raw - # batch delta-by-delta — the merge is an optimization only. - logger.warning( - "async token accounting: coalesce failed, applying raw " - "batch: %s", exc, - ) - coalesced = batch - for session_id, kwargs in coalesced: - try: - self.update_token_counts(session_id, **kwargs) - except Exception as exc: - # Same contract as the old inline call sites: accounting - # loss is logged, never raised into a turn. - logger.warning( - "async token accounting: apply failed (session=%s): %s", - session_id, exc, - ) - - def _coalesce_token_deltas( - self, batch: List[Tuple[str, Dict[str, Any]]] - ) -> List[Tuple[str, Dict[str, Any]]]: - """Merge consecutive incremental deltas with an identical route. - - Only adjacent deltas merge, so ordering across sessions and across - a mid-session /model switch is preserved exactly. absolute=True - deltas (cumulative overwrites) never merge. - """ - groups: List[Tuple[Optional[tuple], str, Dict[str, Any]]] = [] - for session_id, kwargs in batch: - key = None - if not kwargs.get("absolute"): - key = (session_id,) + tuple( - kwargs.get(f) for f in self._TOKEN_DELTA_ROUTE_FIELDS - ) - if groups and key is not None and groups[-1][0] == key: - merged = groups[-1][2] - for f in self._TOKEN_DELTA_SUM_FIELDS: - merged[f] = merged.get(f, 0) + kwargs.get(f, 0) - for f in self._TOKEN_DELTA_COST_FIELDS: - value = kwargs.get(f) - if value is not None: - # None-preserving sum: an all-None run must stay - # None so COALESCE keeps the stored value untouched. - merged[f] = (merged.get(f) or 0.0) + value - else: - groups.append((key, session_id, dict(kwargs))) - return [(sid, kw) for _, sid, kw in groups] - - def _stop_token_writer(self, join_timeout: float = 10.0) -> None: - """Stop the writer thread and drain remaining deltas. Never raises.""" - with self._token_queue_cond: - self._token_writer_stop = True - self._token_queue_cond.notify_all() - thread = self._token_writer_thread - if thread is not None and thread.is_alive(): - thread.join(timeout=join_timeout) - if thread.is_alive(): - # Writer stuck mid-apply (pathological lock contention). - # Leave any queued deltas unapplied rather than racing the - # stuck apply and misordering/double-counting. - logger.warning( - "async token accounting: writer did not stop within %.0fs; " - "%d queued delta(s) not persisted", - join_timeout, len(self._token_queue), - ) - return - # Writer exited (or never started) — apply leftovers synchronously. - # Claim busy like the writer/flush drains do, so a concurrent - # flush_token_counts cannot fast-path True while this batch is - # still being applied; conversely, wait out a flush caller-drain - # that already claimed busy — close() nulls the connection right - # after this returns, and must not yank it mid-batch. - with self._token_queue_cond: - deadline = time.monotonic() + join_timeout - while self._token_writer_busy: - remaining = deadline - time.monotonic() - if remaining <= 0: - logger.warning( - "async token accounting: concurrent drain did not " - "finish within %.0fs; %d queued delta(s) not persisted", - join_timeout, len(self._token_queue), - ) - return - self._token_queue_cond.wait(remaining) - # busy is claimed BEFORE the queue is cleared — same ordering - # as the writer loop and the flush caller-drain. The lock-free - # fast path in flush_token_counts() reads queue-then-busy - # without the cond, so clearing first would let a concurrent - # flush observe "empty and idle" and return True while this - # popped batch is still unapplied. - batch = list(self._token_queue) - if batch: - self._token_writer_busy = True - self._token_queue.clear() - if batch: - try: - self._apply_token_batch(batch) - finally: - with self._token_queue_cond: - self._token_writer_busy = False - self._token_queue_cond.notify_all() - - def _drain_token_queue_at_exit(self) -> None: - try: - self._stop_token_writer() - except Exception: - pass # Best effort — never fatal at interpreter shutdown. - - def update_token_counts( - self, - session_id: str, - input_tokens: int = 0, - output_tokens: int = 0, - model: str = None, - cache_read_tokens: int = 0, - cache_write_tokens: int = 0, - reasoning_tokens: int = 0, - estimated_cost_usd: Optional[float] = None, - actual_cost_usd: Optional[float] = None, - cost_status: Optional[str] = None, - cost_source: Optional[str] = None, - pricing_version: Optional[str] = None, - billing_provider: Optional[str] = None, - billing_base_url: Optional[str] = None, - billing_mode: Optional[str] = None, - api_call_count: int = 0, - absolute: bool = False, - ) -> None: - """Update token counters and backfill model if not already set. - - When *absolute* is False (default), values are **incremented** — use - this for per-API-call deltas (CLI path). - - When *absolute* is True, values are **set directly** — use this when - the caller already holds cumulative totals (gateway path, where the - cached agent accumulates across messages). - """ - # Ensure the session row exists so the UPDATE doesn't silently affect - # 0 rows. Under concurrent load (cron + kanban + delegate_task) the - # initial create_session() may have failed due to SQLite locking. - # INSERT OR IGNORE is cheap and idempotent. - self._insert_session_row(session_id, "unknown", model=model) - if absolute: - sql = """UPDATE sessions SET - input_tokens = ?, - output_tokens = ?, - cache_read_tokens = ?, - cache_write_tokens = ?, - reasoning_tokens = ?, - estimated_cost_usd = COALESCE(?, 0), - actual_cost_usd = CASE - WHEN ? IS NULL THEN actual_cost_usd - ELSE ? - END, - cost_status = COALESCE(?, cost_status), - cost_source = COALESCE(?, cost_source), - pricing_version = COALESCE(?, pricing_version), - billing_provider = COALESCE(billing_provider, ?), - billing_base_url = COALESCE(billing_base_url, ?), - billing_mode = COALESCE(billing_mode, ?), - model = COALESCE(model, ?), - api_call_count = ? - WHERE id = ?""" - else: - sql = """UPDATE sessions SET - input_tokens = input_tokens + ?, - output_tokens = output_tokens + ?, - cache_read_tokens = cache_read_tokens + ?, - cache_write_tokens = cache_write_tokens + ?, - reasoning_tokens = reasoning_tokens + ?, - estimated_cost_usd = COALESCE(estimated_cost_usd, 0) + COALESCE(?, 0), - actual_cost_usd = CASE - WHEN ? IS NULL THEN actual_cost_usd - ELSE COALESCE(actual_cost_usd, 0) + ? - END, - cost_status = COALESCE(?, cost_status), - cost_source = COALESCE(?, cost_source), - pricing_version = COALESCE(?, pricing_version), - billing_provider = COALESCE(billing_provider, ?), - billing_base_url = COALESCE(billing_base_url, ?), - billing_mode = COALESCE(billing_mode, ?), - model = COALESCE(model, ?), - api_call_count = COALESCE(api_call_count, 0) + ? - WHERE id = ?""" - has_accounted_usage = bool( - input_tokens or output_tokens or cache_read_tokens - or cache_write_tokens or reasoning_tokens or api_call_count - or estimated_cost_usd or actual_cost_usd - ) - params = ( - input_tokens, - output_tokens, - cache_read_tokens, - cache_write_tokens, - reasoning_tokens, - estimated_cost_usd, - actual_cost_usd, - actual_cost_usd, - cost_status, - cost_source, - pricing_version, - billing_provider if has_accounted_usage else None, - billing_base_url if has_accounted_usage else None, - billing_mode if has_accounted_usage else None, - model if has_accounted_usage else None, - api_call_count, - session_id, - ) - # Per-model usage attribution. ``update_token_counts`` is the single - # chokepoint every per-API-call delta flows through (CLI, gateway, cron, - # delegated runs — see conversation_loop / codex_runtime), and each call - # carries the model/provider *active at the time of that call*. The - # ``sessions`` row only keeps one (model, billing_provider) pair, so a - # mid-session ``/model`` switch otherwise attributes every token to the - # initial model (issue #51607). Recording the per-call delta into - # session_model_usage keyed by the live model preserves an accurate - # per-model breakdown regardless of how many times the user switches. - # - # Only the incremental path records here. Absolute cumulative updates - # cannot be split back into routes; Insights reconciles any positive - # residual against the aggregate session row instead. - record_model_usage = (not absolute) and ( - input_tokens or output_tokens or cache_read_tokens - or cache_write_tokens or reasoning_tokens or api_call_count - or estimated_cost_usd - ) - - def _do(conn): - row = conn.execute( - "SELECT model, billing_provider, api_call_count FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - existing_model = row["model"] if row is not None else None - existing_provider = row["billing_provider"] if row is not None else None - existing_api_calls = int((row["api_call_count"] if row is not None else 0) or 0) - - # Session creation records the requested primary route before any API - # call. If it fails and fallback succeeds, the first accounted usage - # event is the first authoritative route. After that, preserve the - # legacy row: one row cannot represent mixed-provider usage. - first_accounted_route = ( - existing_api_calls == 0 - and has_accounted_usage - and bool(model) - and bool(billing_provider) - and (existing_model != model or existing_provider != billing_provider) - ) - if first_accounted_route: - conn.execute( - """UPDATE sessions - SET model = ?, billing_provider = ?, - billing_base_url = ?, billing_mode = ? - WHERE id = ?""", - (model, billing_provider, billing_base_url, billing_mode, session_id), - ) - conn.execute(sql, params) - if record_model_usage: - self._record_model_usage( - conn, - session_id, - model=model, - billing_provider=billing_provider, - billing_base_url=billing_base_url, - billing_mode=billing_mode, - input_tokens=input_tokens, - output_tokens=output_tokens, - cache_read_tokens=cache_read_tokens, - cache_write_tokens=cache_write_tokens, - reasoning_tokens=reasoning_tokens, - estimated_cost_usd=estimated_cost_usd, - actual_cost_usd=actual_cost_usd, - cost_status=cost_status, - cost_source=cost_source, - api_call_count=api_call_count, - ) - self._execute_write(_do) - - def _record_model_usage( - self, - conn, - session_id: str, - *, - model: Optional[str], - billing_provider: Optional[str], - billing_base_url: Optional[str], - billing_mode: Optional[str], - input_tokens: int, - output_tokens: int, - cache_read_tokens: int, - cache_write_tokens: int, - reasoning_tokens: int, - estimated_cost_usd: Optional[float], - actual_cost_usd: Optional[float], - cost_status: Optional[str], - cost_source: Optional[str], - api_call_count: int, - task: str = "", - ) -> None: - """Accumulate a per-API-call usage delta into session_model_usage. - - Runs inside the caller's write transaction (after the ``sessions`` - UPDATE) so the per-model rows stay consistent with the summary row. - When the caller omits the model/provider (some paths only pass token - deltas), fall back to the values already recorded on the session row — - the same COALESCE-from-session behaviour the summary update uses. - - ``task`` distinguishes what kind of work consumed the tokens: - ``''`` (empty) is the main agent loop; auxiliary calls record their - task name (``vision``, ``compression``, ``title_generation``, ...) - via :meth:`record_auxiliary_usage` (issue #23270). - """ - row = conn.execute( - "SELECT model, billing_provider, billing_base_url, billing_mode " - "FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - sess_model = row["model"] if row is not None else None - sess_provider = row["billing_provider"] if row is not None else None - sess_base_url = row["billing_base_url"] if row is not None else None - sess_billing_mode = row["billing_mode"] if row is not None else None - - # Aux-task rows (task != '') must NOT inherit the session's main-loop - # route: an aux call may use a completely different provider/model - # (vision on gemini while the main loop runs anthropic). Missing info - # stays 'unknown'/empty rather than borrowing a misleading route. - if task: - eff_model = model or "unknown" - eff_provider = billing_provider or "" - eff_base_url = billing_base_url or "" - eff_billing_mode = billing_mode or "" - else: - eff_model = model or sess_model or "unknown" - eff_provider = billing_provider or sess_provider or "" - eff_base_url = billing_base_url or sess_base_url or "" - eff_billing_mode = billing_mode or sess_billing_mode or "" - now = time.time() - conn.execute( - """INSERT INTO session_model_usage ( - session_id, model, billing_provider, billing_base_url, billing_mode, - task, api_call_count, input_tokens, output_tokens, - cache_read_tokens, cache_write_tokens, reasoning_tokens, - estimated_cost_usd, actual_cost_usd, cost_status, cost_source, - first_seen, last_seen - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(session_id, model, billing_provider, billing_base_url, billing_mode, task) - DO UPDATE SET - api_call_count = api_call_count + excluded.api_call_count, - input_tokens = input_tokens + excluded.input_tokens, - output_tokens = output_tokens + excluded.output_tokens, - cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens, - cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens, - reasoning_tokens = reasoning_tokens + excluded.reasoning_tokens, - estimated_cost_usd = estimated_cost_usd + excluded.estimated_cost_usd, - actual_cost_usd = actual_cost_usd + excluded.actual_cost_usd, - cost_status = COALESCE(excluded.cost_status, cost_status), - cost_source = COALESCE(excluded.cost_source, cost_source), - last_seen = excluded.last_seen""", - ( - session_id, - eff_model, - eff_provider, - eff_base_url, - eff_billing_mode, - task or "", - api_call_count or 0, - input_tokens or 0, - output_tokens or 0, - cache_read_tokens or 0, - cache_write_tokens or 0, - reasoning_tokens or 0, - float(estimated_cost_usd or 0.0), - float(actual_cost_usd or 0.0), - cost_status, - cost_source, - now, - now, - ), - ) def ensure_session( self, @@ -10642,281 +4843,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._insert_session_row(session_id, source, model=model, **kwargs) return session_id - def record_auxiliary_usage( - self, - session_id: str, - task: str, - *, - model: Optional[str] = None, - billing_provider: Optional[str] = None, - billing_base_url: Optional[str] = None, - input_tokens: int = 0, - output_tokens: int = 0, - cache_read_tokens: int = 0, - cache_write_tokens: int = 0, - reasoning_tokens: int = 0, - estimated_cost_usd: Optional[float] = None, - api_call_count: int = 1, - ) -> None: - """Record an auxiliary LLM call's usage against *session_id* (issue #23270). - - Auxiliary calls (vision, compression, title_generation, web_extract, - session_search, ...) historically discarded their usage, leaving the - dashboard's per-model analytics blind to aux model spend. This writes - a per-(model, provider, task) delta into ``session_model_usage`` — - the same table the main loop's ``update_token_counts`` feeds — WITHOUT - touching the ``sessions`` summary row. That separation is deliberate: - the gateway overwrites session counters with absolute main-loop totals, - so folding aux tokens into the summary row would either be clobbered - or double-counted. Insights/analytics read the union of both. - - ``api_call_count`` defaults to 1 (one aux LLM call). Background-review - forks record an aggregate of N fork API calls in one write with - ``task='background_review'`` (issue #87250). - - Best-effort by contract: callers must never fail an aux call because - accounting failed. - """ - if not session_id or not task: - return - # FK on session_model_usage.session_id → sessions.id: ensure the row - # exists (same INSERT OR IGNORE guard update_token_counts uses — the - # initial create_session() can fail under concurrent SQLite locking). - self._insert_session_row(session_id, "unknown") - - def _do(conn): - self._record_model_usage( - conn, - session_id, - model=model, - billing_provider=billing_provider, - billing_base_url=billing_base_url, - billing_mode=None, - input_tokens=input_tokens or 0, - output_tokens=output_tokens or 0, - cache_read_tokens=cache_read_tokens or 0, - cache_write_tokens=cache_write_tokens or 0, - reasoning_tokens=reasoning_tokens or 0, - estimated_cost_usd=estimated_cost_usd, - actual_cost_usd=None, - cost_status=None, - cost_source=None, - api_call_count=( - 1 if api_call_count is None else int(api_call_count) - ), - task=task, - ) - self._execute_write(_do) - - def prune_empty_ghost_sessions(self, sessions_dir: "Optional[Path]" = None) -> int: - """Remove empty TUI ghost sessions (no messages, no title, >24hr old).""" - cutoff = time.time() - 86400 # Only sessions older than 24 hours - - def _do(conn): - rows = conn.execute(""" - SELECT id FROM sessions - WHERE source = 'tui' - AND title IS NULL - AND ended_at IS NOT NULL - AND started_at < ? - AND NOT EXISTS ( - SELECT 1 FROM messages WHERE messages.session_id = sessions.id - ) - """, (cutoff,)).fetchall() - ids = [r[0] if isinstance(r, (tuple, list)) else r["id"] for r in rows] - if ids: - placeholders = ",".join("?" * len(ids)) - conn.execute( - f"DELETE FROM sessions WHERE id IN ({placeholders})", ids - ) - self._delete_unreferenced_system_prompts(conn) - return ids - - removed_ids = self._execute_write(_do) or [] - # Clean up any on-disk session files (belt-and-suspenders) - if sessions_dir and removed_ids: - for sid in removed_ids: - self._remove_session_files(sessions_dir, sid) - return len(removed_ids) - - def finalize_orphaned_compression_sessions(self) -> int: - """Mark orphaned compression continuation sessions as ended. - - Targets child sessions that were never finalized: parent is ended - with reason='compression', child has messages but no end_reason/ended_at - and api_call_count=0. Non-destructive: preserves all messages and sets - end_reason='orphaned_compression'. Fix for #20001. - """ - cutoff = time.time() - 604800 # 7 days - - def _do(conn): - now = time.time() - result = conn.execute( - """ - UPDATE sessions - SET ended_at = ?, - end_reason = 'orphaned_compression' - WHERE api_call_count = 0 - AND end_reason IS NULL - AND ended_at IS NULL - AND started_at < ? - AND parent_session_id IS NOT NULL - AND EXISTS ( - SELECT 1 FROM sessions p - WHERE p.id = sessions.parent_session_id - AND p.end_reason = 'compression' - AND p.ended_at IS NOT NULL - ) - AND EXISTS ( - SELECT 1 FROM messages m - WHERE m.session_id = sessions.id - ) - """, - (now, cutoff), - ) - return result.rowcount - - return self._execute_write(_do) or 0 - - def sweep_orphaned_sessions( - self, - *, - max_idle_seconds: float, - sources: Tuple[str, ...] = ("tui", "desktop", "subagent"), - exclude_ids: Tuple[str, ...] = (), - exclude_pinned: bool = False, - heartbeat_staleness_seconds: Optional[float] = None, - heartbeat_ownership_grace_seconds: Optional[float] = None, - respect_gateway_heartbeats: bool = True, - ) -> List[str]: - """Close session rows orphaned by a dead gateway process (#65194, #94895). - - The TUI/desktop gateway reaps disconnected websocket sessions with an - in-process ``threading.Timer`` grace timer; a gateway restart destroys - the timer and leaves the row ``ended_at IS NULL`` forever. This is the - startup-time complement: it closes rows for the given ``sources`` whose - ``started_at`` and canonical last-activity time are both older than - ``max_idle_seconds``, with a distinct - ``end_reason='startup_orphan_reap'`` for traceability. - - Canonical activity is the newest of ``last_activity_at`` (the in-turn - heartbeat) and the newest durable message timestamp, falling back to - ``started_at``. The separate ``started_at`` predicate protects freshly - created compression/branch children whose copied activity is old. - - Only pass sources whose lifecycle the caller owns (never messaging-gateway - platforms like ``telegram`` — ending those triggers the #60609 routing - loop). ``exclude_ids`` spares rows this process still holds in memory - (a ``session.resume`` that landed during the startup grace window). - ``exclude_pinned`` is intended for broad automatic sweeps; pinned rows - remain explicitly recoverable. Non-destructive: messages are preserved - and the row remains resumable. First-reason-wins is preserved via - ``ended_at IS NULL``. - - Cross-backend liveness (#94895): when one ``state.db`` is shared by N - serve / gateway processes, each backend refreshes a row in - ``gateway_heartbeats``. With ``respect_gateway_heartbeats`` enabled, a - row is only reaped when activity staleness holds AND no live backend - (heartbeat refreshed within ``heartbeat_staleness_seconds``, default - ``2 * max_idle_seconds``) could plausibly own it. Disable that gate only - for sources whose lifecycle is explicitly owned by state.db itself. - - Ownership inference: a live backend B ``owns`` a session S if - ``B.started_at <= S.started_at + heartbeat_ownership_grace_seconds`` - (default ``heartbeat_staleness_seconds``). The grace window covers a - migrating backend whose existing sessions predate its first heartbeat, - but is bounded so a fresh PID-reuse respawn cannot protect rows forever. - With no fresh heartbeat the predicate falls back to the legacy sweep. - - The SELECT, live-lease validation, and UPDATE run in one - ``BEGIN IMMEDIATE`` transaction. Active turn leases or compression - locks spare the row; expired/reclaimed guards are removed so their - former owner is fenced. Returns the swept session ids. - """ - srcs = tuple(s for s in sources if s) - if max_idle_seconds <= 0 or not srcs: - return [] - hb_staleness = ( - heartbeat_staleness_seconds - if heartbeat_staleness_seconds and heartbeat_staleness_seconds > 0 - else max_idle_seconds * 2 - ) - hb_grace = ( - heartbeat_ownership_grace_seconds - if heartbeat_ownership_grace_seconds is not None - and heartbeat_ownership_grace_seconds >= 0 - else hb_staleness - ) - now = time.time() - cutoff = now - max_idle_seconds - hb_cutoff = now - hb_staleness - placeholders = ",".join("?" for _ in srcs) - staleness = ( - f"started_at < ? AND {_sql_session_last_active('sessions')} < ?" - ) - pin_scope = " AND COALESCE(pinned, 0) = 0" if exclude_pinned else "" - heartbeat_params: Tuple[float, ...] = () - orphan_predicate = staleness - if respect_gateway_heartbeats: - orphan_predicate += ( - " AND NOT EXISTS (" - "SELECT 1 FROM gateway_heartbeats h" - " WHERE h.last_heartbeat >= ?" - " AND h.started_at <= sessions.started_at + ?" - ")" - ) - heartbeat_params = (hb_cutoff, hb_grace) - - def _do(conn): - rows = conn.execute( - f"SELECT id FROM sessions WHERE ended_at IS NULL" - f" AND source IN ({placeholders}){pin_scope}" - f" AND {orphan_predicate}", - (*srcs, cutoff, cutoff, *heartbeat_params), - ).fetchall() - excluded = {str(x) for x in exclude_ids if x} - victims = [] - for row in rows: - sid = str(row["id"]) - if sid in excluded: - continue - try: - self._check_transcript_write_guards( - conn, - sid, - compression_lock_holder=None, - turn_lease_holder=None, - reject_active_turn_lease=True, - reject_active_compression_lock=True, - ) - except ( - SessionCompressionInProgressError, - SessionTurnLeaseLostError, - ): - continue - victims.append(sid) - if not victims: - return [] - closed_at = time.time() - marks = ",".join("?" for _ in victims) - # Re-apply every scope/liveness predicate under the write lock. - conn.execute( - f"UPDATE sessions SET ended_at = ?, end_reason = 'startup_orphan_reap'" - f" WHERE id IN ({marks}) AND ended_at IS NULL" - f" AND source IN ({placeholders}){pin_scope}" - f" AND {orphan_predicate}", - ( - closed_at, - *victims, - *srcs, - cutoff, - cutoff, - *heartbeat_params, - ), - ) - return victims - - return self._execute_write(_do) or [] # ── Cross-backend heartbeat API (#94895) ─────────────────────────── # Each serve / tui_gateway process registers a heartbeat row at startup @@ -10926,98 +4852,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # their own row on graceful shutdown; a row that survives a crash is # reclaimed by the staleness sweep once ``last_heartbeat`` ages out. - def register_backend_heartbeat( - self, - *, - backend_id: str, - pid: int, - started_at: float, - last_heartbeat: Optional[float] = None, - profile: str = "", - host: str = "", - ) -> None: - """Upsert this backend's liveness row (#94895). - - ``backend_id`` MUST be stable for the lifetime of the process - (e.g. ``f"{profile}@{host}:{pid}"``) so a respawn cannot accidentally - inherit the dead predecessor's heartbeat and protect stale rows. - ``started_at`` records when THIS process started (not the wall clock - at first refresh) so a long-lived backend whose previous run died - cannot be confused with a freshly-spawned sibling. - """ - if not backend_id: - return - ts = time.time() if last_heartbeat is None else float(last_heartbeat) - def _do(conn): - conn.execute( - "INSERT INTO gateway_heartbeats" - " (backend_id, pid, started_at, last_heartbeat, profile, host)" - " VALUES (?, ?, ?, ?, ?, ?)" - " ON CONFLICT(backend_id) DO UPDATE SET" - " pid = excluded.pid," - " started_at = excluded.started_at," - " last_heartbeat = excluded.last_heartbeat," - " profile = excluded.profile," - " host = excluded.host", - (str(backend_id), int(pid), float(started_at), ts, - str(profile), str(host)), - ) - self._execute_write(_do) - - def clear_backend_heartbeat(self, backend_id: str) -> bool: - """Remove this backend's heartbeat row (#94895). - - Called from ``atexit`` so a graceful shutdown doesn't leave a stale - row behind. A crashed backend's row is reclaimed later by - ``prune_stale_heartbeats``. Returns True if a row was removed. - """ - if not backend_id: - return False - def _do(conn): - cur = conn.execute( - "DELETE FROM gateway_heartbeats WHERE backend_id = ?", - (str(backend_id),), - ) - return cur.rowcount > 0 - return bool(self._execute_write(_do)) - - def prune_stale_heartbeats(self, *, max_age_seconds: float) -> List[str]: - """Drop heartbeat rows whose ``last_heartbeat`` is older than the - staleness window. Returns the removed backend ids. Safe to call - from any process; only stale rows are touched. - """ - if max_age_seconds <= 0: - return [] - cutoff = time.time() - max_age_seconds - def _do(conn): - cur = conn.execute( - "DELETE FROM gateway_heartbeats WHERE last_heartbeat < ?" - " RETURNING backend_id", - (cutoff,), - ) - return [str(r[0]) for r in cur.fetchall()] - return list(self._execute_write(_do) or []) - - def list_backend_heartbeats(self) -> List[Dict[str, Any]]: - """Snapshot of every registered backend's heartbeat (for diagnostics - and tests). The fields mirror ``gateway_heartbeats`` exactly. - """ - with self._read_ctx() as conn: - rows = conn.execute( - "SELECT backend_id, pid, started_at, last_heartbeat," - " profile, host FROM gateway_heartbeats" - " ORDER BY last_heartbeat DESC" - ).fetchall() - out: List[Dict[str, Any]] = [] - for r in rows: - if isinstance(r, sqlite3.Row): - out.append({k: r[k] for k in r.keys()}) - else: - out.append({ - "backend_id": r[0], "pid": r[1], "started_at": r[2], - "last_heartbeat": r[3], "profile": r[4], "host": r[5], - }) - return out def get_session(self, session_id: str) -> Optional[Dict[str, Any]]: """Get a session by ID.""" @@ -11025,16 +4859,14 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # row through here; drain queued token deltas so they see exact # totals. No-op attribute check when nothing is queued. self.flush_token_counts() - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT s.*, " - "COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved " - "FROM sessions s " - "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash " - "WHERE s.id = ?", - (session_id,), - ) - row = cursor.fetchone() + row = self._read_one( + "SELECT s.*, " + "COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved " + "FROM sessions s " + "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash " + "WHERE s.id = ?", + (session_id,), + ) return self._session_row_dict(row) if row else None def get_dominant_session_model_route( @@ -11048,22 +4880,21 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) its dominant main-loop route when one is available. """ self.flush_token_counts() - with self._read_ctx() as conn: - row = conn.execute( - """SELECT model, billing_provider, billing_base_url, billing_mode, - api_call_count - FROM session_model_usage - WHERE session_id = ? - AND task = '' - AND model <> 'unknown' - AND billing_provider <> '' - ORDER BY api_call_count DESC, - (input_tokens + output_tokens + cache_read_tokens + - cache_write_tokens + reasoning_tokens) DESC, - last_seen DESC - LIMIT 1""", - (session_id,), - ).fetchone() + row = self._read_one( + """SELECT model, billing_provider, billing_base_url, billing_mode, + api_call_count + FROM session_model_usage + WHERE session_id = ? + AND task = '' + AND model <> 'unknown' + AND billing_provider <> '' + ORDER BY api_call_count DESC, + (input_tokens + output_tokens + cache_read_tokens + + cache_write_tokens + reasoning_tokens) DESC, + last_seen DESC + LIMIT 1""", + (session_id,), + ) return dict(row) if row else None def resolve_session_id(self, session_id_or_prefix: str) -> Optional[str]: @@ -11078,12 +4909,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return exact["id"] escaped = _escape_like(session_id_or_prefix) - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT id FROM sessions WHERE id LIKE ? ESCAPE '\\' ORDER BY started_at DESC LIMIT 2", - (f"{escaped}%",), - ) - matches = [row["id"] for row in cursor.fetchall()] + matches = [row["id"] for row in self._read_all( + "SELECT id FROM sessions WHERE id LIKE ? ESCAPE '\\' ORDER BY started_at DESC LIMIT 2", + (f"{escaped}%",), + )] if len(matches) == 1: return matches[0] return None @@ -11111,291 +4940,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # row holding it (#92473). CANONICAL_BOT_CHAT_TITLE = "Bot Chat" - @classmethod - def _title_rank(cls, source: Optional[str]) -> int: - """Rank a stored title_source. NULL means a pre-provenance row. - - Rows written before this column existed carry NULL. They were almost - always set by the old auto-titler, but a manual ``/title`` from that - era is indistinguishable — so treat NULL as ``user`` and refuse to - overwrite it. Auto-titling only ever fills genuinely empty titles on - legacy rows, which is the conservative direction. - """ - if source is None: - return cls._TITLE_SOURCE_RANK[cls.TITLE_SOURCE_USER] - return cls._TITLE_SOURCE_RANK.get(str(source), 0) - - @staticmethod - def sanitize_title(title: Optional[str]) -> Optional[str]: - """Validate and sanitize a session title. - - - Strips leading/trailing whitespace - - Removes ASCII control characters (0x00-0x1F, 0x7F) and problematic - Unicode control chars (zero-width, RTL/LTR overrides, etc.) - - Collapses internal whitespace runs to single spaces - - Normalizes empty/whitespace-only strings to None - - Enforces MAX_TITLE_LENGTH - - Returns the cleaned title string or None. - Raises ValueError if the title exceeds MAX_TITLE_LENGTH after cleaning. - """ - if not title: - return None - - # Lone surrogates cannot be bound by sqlite3 (UnicodeEncodeError at - # UTF-8 encode time) — scrub them like every other write path here. - title = _sanitize_surrogates(title) - - # Remove ASCII control characters (0x00-0x1F, 0x7F) but keep - # whitespace chars (\t=0x09, \n=0x0A, \r=0x0D) so they can be - # normalized to spaces by the whitespace collapsing step below - cleaned = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', title) - - # Remove problematic Unicode control characters: - # - Zero-width chars (U+200B-U+200F, U+FEFF) - # - Directional overrides (U+202A-U+202E, U+2066-U+2069) - # - Object replacement (U+FFFC), interlinear annotation (U+FFF9-U+FFFB) - cleaned = re.sub( - r'[\u200b-\u200f\u2028-\u202e\u2060-\u2069\ufeff\ufffc\ufff9-\ufffb]', - '', cleaned, - ) - - # Collapse internal whitespace runs and strip - cleaned = re.sub(r'\s+', ' ', cleaned).strip() - - if not cleaned: - return None - - if len(cleaned) > SessionDB.MAX_TITLE_LENGTH: - raise ValueError( - f"Title too long ({len(cleaned)} chars, max {SessionDB.MAX_TITLE_LENGTH})" - ) - - return cleaned - - def _is_compression_ancestor( - self, conn, *, ancestor_id: str, descendant_id: str - ) -> bool: - """Return True if *ancestor_id* is a compression predecessor of - *descendant_id* (walking parent links up the continuation chain). - - The continuation edge is the canonical one shared with - :func:`_ephemeral_child_sql` / :meth:`set_session_archived` - (``_COMPRESSION_CHILD_SQL``): a parent → child edge counts only when the - parent ended with ``end_reason = 'compression'`` and the child started - at or after the parent's ``ended_at``, which distinguishes continuations - from delegate subagents / branch children that also carry a - ``parent_session_id``. Expressed as a single recursive CTE rather than a - per-hop Python walk so the edge definition lives in exactly one place. - """ - if not ancestor_id or not descendant_id or ancestor_id == descendant_id: - return False - # Walk parent links up from the descendant, following only compression - # continuation edges, and check whether ancestor_id is reached. - edge = _COMPRESSION_CHILD_SQL.format(a="child") - row = conn.execute( - f""" - WITH RECURSIVE ancestors(id) AS ( - SELECT ? - UNION - SELECT parent.id - FROM ancestors a - JOIN sessions child ON child.id = a.id - JOIN sessions parent ON parent.id = child.parent_session_id - WHERE {edge} - ) - SELECT 1 FROM ancestors WHERE id = ? AND id != ? LIMIT 1 - """, - (descendant_id, ancestor_id, descendant_id), - ).fetchone() - return row is not None - - def _set_session_title( - self, - session_id: str, - title: str, - *, - source: str, - ) -> bool: - """Write a title, enforcing provenance precedence. - - ``source`` is one of ``TITLE_SOURCE_{DERIVED,LLM,USER}``. A ``user`` - write always lands — an explicit rename is authoritative. An automatic - write (``derived``/``llm``) lands only when the row is untitled or the - stored title has strictly lower authority, so the instant ``derived`` - title upgrades to ``llm`` exactly once and neither can ever overwrite a - name the user typed. Re-running the titler on an already-``llm`` row is - a no-op, which is what stops a session renaming itself. The one thing - no writer may do is move a hidden canonical Bot Chat off its title. - - The read and the write are one compare-and-swap inside a single - transaction, so a manual ``/title`` racing an in-flight generation - cannot be clobbered by the late arrival. - """ - title = self.sanitize_title(title) - is_user = source == self.TITLE_SOURCE_USER - new_rank = self._title_rank(source) if not is_user else None - - def _do(conn): - current = conn.execute( - "SELECT title, title_source, hidden FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if current is None: - return 0 - # The canonical Bot Chat's NAME is its identity: Bot Mode resolves - # the forever-chat by exact-title lookup on every open, so renaming - # the row orphans the entire conversation — the next click mints an - # empty replacement and UNIQUE(title) then blocks ever renaming - # back (#92473). Refuse the rename at the single write path every - # surface funnels through (gateway session.title, /title, CLI - # rename, REST). Hidden is the discriminator: canonical chats are - # born hidden; an ordinary visible session a user happens to call - # "Bot Chat" stays freely renameable. Provenance-blind: an - # automatic llm write outranks a derived title, so the auto-titler - # would otherwise rename the row too (#99517) — it no-ops instead. - if ( - (current["title"] or "") == self.CANONICAL_BOT_CHAT_TITLE - and bool(current["hidden"]) - and title != self.CANONICAL_BOT_CHAT_TITLE - ): - if is_user: - raise ValueError( - "This is the bot's canonical Bot Chat — its name is its " - "identity, and renaming it would orphan the conversation. " - "To start fresh, create a new bot instead." - ) - return 0 - if not is_user and current["title"] is not None: - if self._title_rank(current["title_source"]) >= new_rank: - return 0 - - if title: - # Check uniqueness (allow the same session to keep its own title) - cursor = conn.execute( - "SELECT id FROM sessions WHERE title = ? AND id != ?", - (title, session_id), - ) - conflict = cursor.fetchone() - if conflict: - conflict_id = conflict["id"] - # A compression continuation is the live, projected-forward - # head of its conversation; its compressed predecessors are - # ended and hidden from the session list (list_sessions_rich - # projects roots → tip). When the title that "conflicts" is - # held by such a hidden ancestor, the user has no way to free - # it — renaming the visible tip back to the base name would - # dead-end with "already in use by ". - # Treat this as a transfer: move the title off the ancestor - # onto the continuation. Uniqueness is preserved (still only - # one session carries the exact title) and the parent-link - # lineage is untouched. - if self._is_compression_ancestor( - conn, ancestor_id=conflict_id, descendant_id=session_id - ): - conn.execute( - "UPDATE sessions SET title = NULL WHERE id = ?", - (conflict_id,), - ) - else: - raise ValueError( - f"Title '{title}' is already in use by session {conflict_id}" - ) - # Compare-and-swap on the exact values we just read (``IS`` is - # NULL-safe in SQLite), so a concurrent write between the SELECT - # and here loses instead of being silently overwritten. - cursor = conn.execute( - "UPDATE sessions SET title = ?, title_source = ? " - "WHERE id = ? AND title IS ? AND title_source IS ?", - ( - title, - source if title else None, - session_id, - current["title"], - current["title_source"], - ), - ) - return cursor.rowcount - - rowcount = self._execute_write(_do) - return rowcount > 0 - - def set_session_title(self, session_id: str, title: str) -> bool: - """Set or update a session's title on the user's behalf. - - Returns True if session was found and title was set. - Raises ValueError if title is already in use by another session, - or if the title fails validation (too long, invalid characters). - Empty/whitespace-only strings are normalized to None (clearing the title). - - This records ``user`` provenance, so auto-titling will never replace - the result. Automatic callers must use :meth:`set_auto_title`. - """ - return self._set_session_title( - session_id, title, source=self.TITLE_SOURCE_USER - ) - - def set_auto_title(self, session_id: str, title: str, *, source: str) -> bool: - """Set an automatically generated title, honoring provenance precedence. - - Returns True when the title was written, False when a higher-authority - title already holds the row (nothing is modified in that case). - """ - if source not in (self.TITLE_SOURCE_DERIVED, self.TITLE_SOURCE_LLM): - raise ValueError(f"invalid automatic title source: {source!r}") - return self._set_session_title(session_id, title, source=source) - - def set_auto_title_if_empty(self, session_id: str, title: str) -> bool: - """Back-compat shim: set an LLM title only if nothing better exists. - - Retained because older callers (and third-party plugins) reference it - by name. New code should call :meth:`set_auto_title` with an explicit - source. - """ - return self.set_auto_title( - session_id, title, source=self.TITLE_SOURCE_LLM - ) - - def get_session_title(self, session_id: str) -> Optional[str]: - """Get the title for a session, or None.""" - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT title FROM sessions WHERE id = ?", (session_id,) - ) - row = cursor.fetchone() - return row["title"] if row else None - - def get_session_title_source(self, session_id: str) -> Optional[str]: - """Get the provenance of a session's title, or None when untitled.""" - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT title, title_source FROM sessions WHERE id = ?", - (session_id,), - ) - row = cursor.fetchone() - if not row or row["title"] is None: - return None - return row["title_source"] - - def set_session_title_source(self, session_id: str, source: str) -> bool: - """Overwrite a title's provenance without touching the title text. - - Used when a title is carried across a session boundary (compression - rotation) and the copy must keep the original's authority rather than - the authority of whichever setter performed the copy. - """ - if source not in self._TITLE_SOURCE_RANK: - raise ValueError(f"invalid title source: {source!r}") - - def _do(conn): - cursor = conn.execute( - "UPDATE sessions SET title_source = ? " - "WHERE id = ? AND title IS NOT NULL", - (source, session_id), - ) - return cursor.rowcount - - return self._execute_write(_do) > 0 def backfill_null_session_profiles(self, profile_name: str) -> int: """One-shot owner backfill for legacy pre-ownership session rows. @@ -11422,19 +4966,54 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) if not stamp: return 0 - def _do(conn): - cursor = conn.execute( - """UPDATE sessions - SET profile_name = ? - WHERE profile_name IS NULL OR TRIM(profile_name) = ''""", - (stamp,), - ) - rowcount = cursor.rowcount - if rowcount is None or rowcount < 0: - rowcount = conn.execute("SELECT changes()").fetchone()[0] - return rowcount + return int(self._write_rowcount( + """UPDATE sessions + SET profile_name = ? + WHERE profile_name IS NULL OR TRIM(profile_name) = ''""", + (stamp,), + ) or 0) - return int(self._execute_write(_do) or 0) + def _set_lineage_column(self, column: str, session_id: str, value: Any) -> bool: + """Set one ``sessions`` column across a whole compression lineage. + + Walks ancestors and descendants joined by ``end_reason='compression'`` + so the root and every continuation flip as a unit — Desktop projects + compression roots forward to their latest tip, and updating only the + displayed tip would let the untouched root resurrect it on refresh. + Returns True when at least one row changed. + """ + return self._write_rowcount( + f""" + WITH RECURSIVE + ancestors(id) AS ( + SELECT ? + UNION + SELECT parent.id + FROM ancestors a + JOIN sessions child ON child.id = a.id + JOIN sessions parent ON parent.id = child.parent_session_id + WHERE parent.end_reason = 'compression' + ), + descendants(id) AS ( + SELECT ? + UNION + SELECT child.id + FROM descendants d + JOIN sessions parent ON parent.id = d.id + JOIN sessions child ON child.parent_session_id = parent.id + WHERE parent.end_reason = 'compression' + ), + lineage(id) AS ( + SELECT id FROM ancestors + UNION + SELECT id FROM descendants + ) + UPDATE sessions + SET {column} = ? + WHERE id IN (SELECT id FROM lineage) + """, + (session_id, session_id, value), + ) > 0 def set_session_archived(self, session_id: str, archived: bool) -> bool: """Archive or unarchive a session. @@ -11446,45 +5025,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) displayed tip lets the still-unarchived root resurrect it on refresh. Returns True when at least one row was updated. """ - def _do(conn): - cursor = conn.execute( - """ - WITH RECURSIVE - ancestors(id) AS ( - SELECT ? - UNION - SELECT parent.id - FROM ancestors a - JOIN sessions child ON child.id = a.id - JOIN sessions parent ON parent.id = child.parent_session_id - WHERE parent.end_reason = 'compression' - ), - descendants(id) AS ( - SELECT ? - UNION - SELECT child.id - FROM descendants d - JOIN sessions parent ON parent.id = d.id - JOIN sessions child ON child.parent_session_id = parent.id - WHERE parent.end_reason = 'compression' - ), - lineage(id) AS ( - SELECT id FROM ancestors - UNION - SELECT id FROM descendants - ) - UPDATE sessions - SET archived = ? - WHERE id IN (SELECT id FROM lineage) - """, - (session_id, session_id, 1 if archived else 0), - ) - rowcount = cursor.rowcount - if rowcount is None or rowcount < 0: - rowcount = conn.execute("SELECT changes()").fetchone()[0] - return rowcount - rowcount = self._execute_write(_do) - return rowcount > 0 + return self._set_lineage_column('archived', session_id, 1 if archived else 0) # Accidental end reasons that recovery treats as resumable. Single source # of truth: hermes_state_common._RECOVERABLE_END_REASONS, interpolated @@ -11555,45 +5096,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) vice-versa) no matter which id the caller holds. Returns True when at least one row changed. """ - def _do(conn): - cursor = conn.execute( - """ - WITH RECURSIVE - ancestors(id) AS ( - SELECT ? - UNION - SELECT parent.id - FROM ancestors a - JOIN sessions child ON child.id = a.id - JOIN sessions parent ON parent.id = child.parent_session_id - WHERE parent.end_reason = 'compression' - ), - descendants(id) AS ( - SELECT ? - UNION - SELECT child.id - FROM descendants d - JOIN sessions parent ON parent.id = d.id - JOIN sessions child ON child.parent_session_id = parent.id - WHERE parent.end_reason = 'compression' - ), - lineage(id) AS ( - SELECT id FROM ancestors - UNION - SELECT id FROM descendants - ) - UPDATE sessions - SET pinned = ? - WHERE id IN (SELECT id FROM lineage) - """, - (session_id, session_id, 1 if pinned else 0), - ) - rowcount = cursor.rowcount - if rowcount is None or rowcount < 0: - rowcount = conn.execute("SELECT changes()").fetchone()[0] - return rowcount - rowcount = self._execute_write(_do) - return rowcount > 0 + return self._set_lineage_column('pinned', session_id, 1 if pinned else 0) def set_session_hidden(self, session_id: str, hidden: bool) -> bool: """Hide or unhide a session (and its whole compression lineage). @@ -11609,45 +5112,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) matter which id the caller holds. Returns True when at least one row changed. """ - def _do(conn): - cursor = conn.execute( - """ - WITH RECURSIVE - ancestors(id) AS ( - SELECT ? - UNION - SELECT parent.id - FROM ancestors a - JOIN sessions child ON child.id = a.id - JOIN sessions parent ON parent.id = child.parent_session_id - WHERE parent.end_reason = 'compression' - ), - descendants(id) AS ( - SELECT ? - UNION - SELECT child.id - FROM descendants d - JOIN sessions parent ON parent.id = d.id - JOIN sessions child ON child.parent_session_id = parent.id - WHERE parent.end_reason = 'compression' - ), - lineage(id) AS ( - SELECT id FROM ancestors - UNION - SELECT id FROM descendants - ) - UPDATE sessions - SET hidden = ? - WHERE id IN (SELECT id FROM lineage) - """, - (session_id, session_id, 1 if hidden else 0), - ) - rowcount = cursor.rowcount - if rowcount is None or rowcount < 0: - rowcount = conn.execute("SELECT changes()").fetchone()[0] - return rowcount - rowcount = self._execute_write(_do) - return rowcount > 0 + return self._set_lineage_column('hidden', session_id, 1 if hidden else 0) def set_session_read(self, session_id: str, read: bool = True) -> bool: """Mark a session read or unread (and its whole compression lineage). @@ -11669,45 +5134,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) tip clears the root (and vice-versa) no matter which id the caller holds. Returns True when at least one row changed. """ - def _do(conn): - cursor = conn.execute( - """ - WITH RECURSIVE - ancestors(id) AS ( - SELECT ? - UNION - SELECT parent.id - FROM ancestors a - JOIN sessions child ON child.id = a.id - JOIN sessions parent ON parent.id = child.parent_session_id - WHERE parent.end_reason = 'compression' - ), - descendants(id) AS ( - SELECT ? - UNION - SELECT child.id - FROM descendants d - JOIN sessions parent ON parent.id = d.id - JOIN sessions child ON child.parent_session_id = parent.id - WHERE parent.end_reason = 'compression' - ), - lineage(id) AS ( - SELECT id FROM ancestors - UNION - SELECT id FROM descendants - ) - UPDATE sessions - SET last_read_at = ? - WHERE id IN (SELECT id FROM lineage) - """, - (session_id, session_id, time.time() if read else 0.0), - ) - rowcount = cursor.rowcount - if rowcount is None or rowcount < 0: - rowcount = conn.execute("SELECT changes()").fetchone()[0] - return rowcount - rowcount = self._execute_write(_do) - return rowcount > 0 + return self._set_lineage_column('last_read_at', session_id, time.time() if read else 0.0) @staticmethod def session_unread(session_row: Dict[str, Any]) -> bool: @@ -11723,157 +5150,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) last_active = session_row.get("last_active") or session_row.get("started_at") return float(last_active or 0) > float(last_read) - def get_session_by_title(self, title: str) -> Optional[Dict[str, Any]]: - """Look up a session by exact title. Returns session dict or None.""" - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT s.*, " - "COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved " - "FROM sessions s " - "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash " - "WHERE s.title = ?", - (title,), - ) - row = cursor.fetchone() - return self._session_row_dict(row) if row else None - - def resolve_session_by_title(self, title: str) -> Optional[str]: - """Resolve a title to a session ID, preferring the latest in a lineage. - - If the exact title exists, returns that session's ID. - If not, searches for "title #N" variants and returns the latest one. - If the exact title exists AND numbered variants exist, returns the - latest numbered variant (the most recent continuation). - """ - # First try exact match - exact = self.get_session_by_title(title) - - # Also search for numbered variants: "title #2", "title #3", etc. - # Escape SQL LIKE wildcards (%, _) in the title to prevent false matches - escaped = _escape_like(title) - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT id, title, started_at FROM sessions " - "WHERE title LIKE ? ESCAPE '\\' ORDER BY started_at DESC", - (f"{escaped} #%",), - ) - numbered = cursor.fetchall() - - if numbered: - # Return the most recent numbered variant - return numbered[0]["id"] - elif exact: - return exact["id"] - return None - - def get_next_title_in_lineage(self, base_title: str) -> str: - """Generate the next title in a lineage (e.g., "my session" → "my session #2"). - - Strips any existing " #N" suffix to find the base name, then finds - the highest existing number and increments. - """ - # Strip existing #N suffix to find the true base - match = re.match(r'^(.*?) #(\d+)$', base_title) - if match: - base = match.group(1) - else: - base = base_title - - # Find all existing numbered variants - # Escape SQL LIKE wildcards (%, _) in the base to prevent false matches - escaped = _escape_like(base) - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT title FROM sessions WHERE title = ? OR title LIKE ? ESCAPE '\\'", - (base, f"{escaped} #%"), - ) - existing = [row["title"] for row in cursor.fetchall()] - - if not existing: - return base # No conflict, use the base name as-is - - # Find the highest number - max_num = 1 # The unnumbered original counts as #1 - for t in existing: - m = re.match(r'^.* #(\d+)$', t) - if m: - max_num = max(max_num, int(m.group(1))) - - return f"{base} #{max_num + 1}" - - def get_compression_chain(self, session_id: str) -> List[str]: - """Walk the compression-continuation chain forward and return every id. - - Root-first order, ending at the tip; ``[session_id]`` when no - continuation exists. ``get_compression_tip`` is this walk's last - element — kept as the single implementation so the two can never - disagree about what the chain is. - - A compression continuation is a child of a session whose - ``end_reason = 'compression'``. Older builds tried to distinguish - continuations from branches/subagents by requiring - ``child.started_at >= parent.ended_at``. That ordering is too brittle: - gateway + compression races can insert the real continuation row before - the parent row's ``ended_at`` is written, while a stale websocket later - creates/reuses a sibling that *does* satisfy the timestamp test. The - visible symptom is brutal: desktop resume follows the stale sibling and - the user's latest messages look "lost" even though they are persisted in - the real continuation chain. - - Instead, only follow children of compression-ended parents, exclude - explicit branch/delegate/tool children, and prefer children that are - themselves continuing the compression chain (``end_reason='compression'``) - or still live over stale closed siblings such as ``ws_orphan_reap``. - Returns the latest continuation tip, or the input id when no - continuation exists. - """ - current = session_id - chain = [current] if current else [] - seen = {current} if current else set() - # Bound the walk defensively — compression chains this deep are - # pathological and shouldn't happen in practice. 100 = plenty. - for _ in range(100): - with self._read_ctx() as conn: - cursor = conn.execute( - f""" - SELECT child.id - FROM sessions parent - JOIN sessions child ON child.parent_session_id = parent.id - WHERE parent.id = ? - AND parent.end_reason = 'compression' - AND json_extract(COALESCE(child.model_config, '{{}}'), '$._branched_from') IS NULL - AND json_extract(COALESCE(child.model_config, '{{}}'), '$._delegate_from') IS NULL - AND COALESCE(child.source, '') != 'tool' - ORDER BY - CASE - WHEN child.end_reason = 'compression' THEN 0 - WHEN child.ended_at IS NULL THEN 1 - ELSE 2 - END, - {_sql_session_last_active("child")} DESC, - child.started_at DESC, - child.id DESC - LIMIT 1 - """, - (current,), - ) - row = cursor.fetchone() - if row is None: - return chain - child_id = row["id"] - if not child_id or child_id in seen: - return chain - seen.add(child_id) - current = child_id - chain.append(child_id) - return chain - - def get_compression_tip(self, session_id: str) -> Optional[str]: - """The live tip of a compression-continuation chain (see - ``get_compression_chain`` for the walk's semantics). Returns the input - id when no continuation exists.""" - chain = self.get_compression_chain(session_id) - return chain[-1] if chain else session_id # Columns excluded from compact_rows projections: only the payload-heavy # blob no list consumer renders. Everything else — including gateway @@ -11886,34 +5162,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ) _session_compact_cols_sql: Optional[str] = None - def usage_totals(self, *, min_message_count: int = 1, include_archived: bool = False) -> Dict[str, float]: - """Tokens and spend across this store, as one aggregate. - - The sidebar shows a profile's totals beside a page of its sessions, so - summing the rows it happens to have loaded would report a fraction of - the truth and shrink as paging changed. SQLite adds the columns up over - every row instead, at the cost of one scan. - - Spend is the billed figure when the provider returned one and the - estimate otherwise — the same precedence a single row renders. - """ - where = ["parent_session_id IS NULL", "message_count >= ?"] - params: List[Any] = [min_message_count] - if not include_archived: - where.append("COALESCE(archived, 0) = 0") - - with self._read_ctx() as conn: - row = conn.execute( - f""" - SELECT COALESCE(SUM(COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0)), 0), - COALESCE(SUM(COALESCE(actual_cost_usd, estimated_cost_usd, 0)), 0) - FROM sessions - WHERE {' AND '.join(where)} - """, - params, - ).fetchone() - - return {"tokens": int(row[0] or 0), "cost_usd": float(row[1] or 0.0)} def list_sessions_rich( self, @@ -11936,59 +5184,28 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) session_key: str = None, include_hidden: bool = False, ) -> List[Dict[str, Any]]: - """List sessions with preview (first user message) and last active timestamp. + """List sessions with preview (first user message) and last_active, in one query. - Returns dicts with keys: id, source, model, title, started_at, ended_at, - message_count, preview (first 60 chars of first user message), - last_active (freshest of last_activity_at heartbeat and latest - message timestamp, else started_at). + ``last_active`` = freshest of the activity heartbeat and the latest + message timestamp, else ``started_at``. Implementation-detail children + (subagent runs, compression continuations) are excluded unless + ``include_children``; branch/reset children stay listable. - Uses a single query with correlated subqueries instead of N+2 queries. - - By default, child sessions that represent implementation details - (subagent runs, compression continuations) are excluded. User-visible - branch and reset children remain listable. Pass ``include_children=True`` - to include every child. - - With ``project_compression_tips=True`` (default), sessions that are - roots of compression chains are projected forward to their latest - continuation — one logical conversation = one list entry, showing the - live continuation's id/message_count/title/last_active. This prevents - compressed continuations from being invisible to users while keeping - delegate subagents and branches hidden. Pass ``False`` to return the - raw root rows (useful for admin/debug UIs). - - Pass ``order_by_last_active=True`` to sort by most-recent activity - instead of original conversation start time. For compression chains, - the "most-recent activity" is taken from the live tip (not the root), - so an old conversation that was compressed and continued recently - surfaces in the correct slot. Ordering is computed at SQL level via - a recursive CTE that walks compression-continuation edges, so LIMIT - and OFFSET still apply efficiently. - - ``search_query`` matches case-insensitive substrings against each - surfaced row's title and id (and, like ``id_query``, every title/id in - its forward compression chain). A punctuation-stripped variant is also - matched so e.g. ``an94`` finds ``AN-94``. Only honored in the - ``order_by_last_active`` path. - - Pass ``compact_rows=True`` for dashboard and picker callers that only - need lightweight metadata. This omits the ``system_prompt`` blob from - the SELECT so SQLite never copies it out of the B-tree page — a - significant I/O saving on large databases where the blob routinely - runs to tens of kilobytes per row. - - Pass ``include_pinned=True`` to back-fill any conversation carrying the - durable ``pinned`` flag that the LIMIT/OFFSET window left out. A pin is - a "this must always be reachable" statement, so a pinned conversation - aging past the requested page is a bug, not a paging outcome — the - desktop sidebar would render an empty Pinned section. Back-filled rows - obey the same filters (source, archived, min_message_count) as the - page: an archived or filtered-out conversation stays out. - - Pass ``session_key`` to restrict results to one stable gateway - conversation scope (DM, group, channel, or thread, including the - configured per-user isolation policy). + ``project_compression_tips`` (default) surfaces each compression chain + as ONE entry showing the live tip's id/message_count/title/last_active; + ``False`` returns raw root rows (admin/debug UIs). + ``order_by_last_active`` sorts by the chain TIP's activity via a + recursive CTE, so LIMIT/OFFSET stay cheap and a recently continued old + conversation lands in the right slot. ``search_query`` (that path + only) substring-matches title/id across the forward chain, plus a + punctuation-stripped variant (``an94`` finds ``AN-94``). + ``compact_rows`` omits the ``system_prompt`` blob from the SELECT so + SQLite never copies tens of KB per row out of the B-tree page. + ``include_pinned`` back-fills pinned conversations the page window left + out (a pin means "always reachable"; the desktop sidebar would + otherwise render an empty Pinned section) — they still obey the + source/archived/min_message_count filters. ``session_key`` restricts to + one gateway conversation scope. """ # Rows carry token/cost totals — drain queued deltas first so # listings (sidebar, /resume, dashboards) show exact counters. @@ -12186,9 +5403,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) LIMIT ? OFFSET ? """ params.extend([limit, offset]) - with self._read_ctx() as conn: - cursor = conn.execute(query, params) - rows = cursor.fetchall() + rows = self._read_all(query, params) sessions = [] for row in rows: s = self._session_row_dict(row) @@ -12227,9 +5442,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) {pinned_where} ORDER BY s.started_at DESC """ - with self._read_ctx() as conn: - pinned_cursor = conn.execute(pinned_query, base_where_params) - pinned_rows = pinned_cursor.fetchall() + pinned_rows = self._read_all(pinned_query, base_where_params) for row in pinned_rows: s = self._session_row_dict(row) if s["id"] in seen_ids: @@ -12343,8 +5556,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) GROUP BY session_id ) latest ON m.id = latest.max_id """ - with self._read_ctx() as conn: - rows = conn.execute(query, ids).fetchall() + rows = self._read_all(query, ids) for row in rows: statuses[row["session_id"]] = classify_session_status( role=row["role"], @@ -12363,1633 +5575,12 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # cannot collide with real user content. _CONTENT_JSON_PREFIX = "\x00json:" - @classmethod - def _encode_content(cls, content: Any) -> Any: - """Serialize structured (list/dict) message content for sqlite. - - sqlite3 can only bind ``str``, ``bytes``, ``int``, ``float``, and ``None`` - to query parameters. Multimodal messages have ``content`` as a list of - parts (``[{"type": "text", ...}, {"type": "image_url", ...}]``), which - raises ``ProgrammingError: Error binding parameter N: type 'list' is - not supported`` when bound directly. - - Returns the value unchanged when it's already a safe scalar, or a - sentinel-prefixed JSON string for lists/dicts. Paired with - :meth:`_decode_content` on read. - """ - if isinstance(content, str): - # Lone UTF-16 surrogates reach here inside tool results scraped - # from the web/social platforms (the same input that crashed the - # guardrail hasher). The proactive sanitizer upstream only cleans - # the *api_messages* copy, and the recovery sanitizer only runs - # after the API call itself raises — which it no longer does — so - # the canonical history keeps them and this write is where they - # land. Left raw, sqlite3 raises UnicodeEncodeError, the flush is - # abandoned, and the session silently stops persisting for the - # rest of its life. Scrub so persistence never fails. - return _sanitize_surrogates(content) - if content is None or isinstance(content, (bytes, int, float)): - return content - try: - # json.dumps defaults to ensure_ascii=True, which escapes any - # surrogate as \udXXX — already safe to bind. - return cls._CONTENT_JSON_PREFIX + json.dumps(content) - except (TypeError, ValueError): - # Last-resort fallback: stringify so persistence never fails. - return _sanitize_surrogates(str(content)) - - @classmethod - def _decode_content(cls, content: Any) -> Any: - """Reverse :meth:`_encode_content`; returns scalars unchanged.""" - if isinstance(content, str) and content.startswith(cls._CONTENT_JSON_PREFIX): - try: - return json.loads(content[len(cls._CONTENT_JSON_PREFIX):]) - except (json.JSONDecodeError, TypeError): - logger.warning( - "Failed to decode JSON-encoded message content; " - "returning raw string" - ) - return content - return content - - @staticmethod - def _encode_display_metadata(display_metadata: Any) -> Optional[str]: - """Serialize ``display_metadata`` for its TEXT column without double-encoding. - - Import/replace paths can hand us an already-serialized JSON string (the - same hazard ``tool_calls`` guards against above). ``json.dumps`` on that - string would store a quoted JSON string, and the single ``json.loads`` - on read then yields a ``str`` instead of a dict. - """ - if not display_metadata: - return None - if isinstance(display_metadata, str): - try: - parsed = json.loads(display_metadata) - except (json.JSONDecodeError, TypeError): - logger.warning("Ignoring non-JSON display metadata on write") - return None - if not isinstance(parsed, dict): - logger.warning("Ignoring non-object display metadata on write") - return None - return json.dumps(parsed) - if isinstance(display_metadata, dict): - return json.dumps(display_metadata) - logger.warning( - "Ignoring unexpected display metadata type on write: %s", - type(display_metadata).__name__, - ) - return None - - def _check_transcript_write_guards( - self, - conn, - session_id: str, - compression_lock_holder: Optional[str], - turn_lease_holder: Optional[str] = None, - turn_lease_ttl_seconds: float = 300.0, - reject_active_turn_lease: bool = False, - reject_active_compression_lock: bool = False, - allow_closed_compression_parent: bool = False, - ) -> None: - """Transcript-write admission checks, run INSIDE the write txn. - - Shared by :meth:`append_message` and :meth:`append_messages_batch` so - the two writers can never diverge on these correctness invariants - (this guard has already needed targeted fixes — see the #74478 patience - note below). User-initiated transcript mutations may opt in to rejecting - an active unowned turn lease in that same transaction. - """ - # NOTE (#75316 redesign): appends do NOT check compression_locks. - # The lock's job is to stop two COMPRESSIONS colliding, not to fence - # ordinary transcript writes. Concurrent appends during a compression - # are safe by construction: archive_and_compact() commits against a - # watermark captured at compression start and clones every row that - # arrived after it back into the live transcript, in the same write - # transaction. Blocking appends here was the root cause of a whole - # symptom family — turns dying as session_persistence_failed while a - # slow provider summary held the lease (#74568, #77386), including - # stale locks from dead PIDs blocking writes for the full TTL. - # Destructive user mutations are different: a compressor that already - # captured its watermark can otherwise publish the pre-rewind snapshot - # after the mutation and resurrect the removed turn. Keep that narrow - # fence opt-in so ordinary appends retain the watermark behavior. - if reject_active_compression_lock: - active_lock = conn.execute( - "SELECT holder, expires_at FROM compression_locks " - "WHERE session_id = ?", - (session_id,), - ).fetchone() - if active_lock is not None: - current_holder = active_lock["holder"] - if ( - float(active_lock["expires_at"]) <= time.time() - or _compression_lock_holder_process_is_dead(current_holder) - ): - conn.execute( - "DELETE FROM compression_locks " - "WHERE session_id = ? AND holder = ?", - (session_id, current_holder), - ) - elif current_holder != compression_lock_holder: - raise SessionCompressionInProgressError( - f"Session {session_id!r} is being compressed by another writer" - ) - if turn_lease_holder or reject_active_turn_lease: - conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) - lease = conn.execute( - "SELECT holder, expires_at FROM session_turn_leases " - "WHERE conversation_id = ?", - (conversation_id,), - ).fetchone() - now = time.time() - if turn_lease_holder: - if lease is None or lease["holder"] != turn_lease_holder: - raise SessionTurnLeaseLostError( - f"Session turn lease lost; refusing transcript write " - f"for {session_id!r}" - ) - if float(lease["expires_at"]) <= now: - # Expiry makes the row reclaimable; it does not prove that a - # takeover occurred. BEGIN IMMEDIATE serializes this renewal - # with acquisition, so a still-matching owner can recover from - # a starved refresher without weakening the foreign-holder fence. - conn.execute( - "UPDATE session_turn_leases SET expires_at = ? " - "WHERE conversation_id = ? AND holder = ?", - ( - now + max(0.1, float(turn_lease_ttl_seconds)), - conversation_id, - turn_lease_holder, - ), - ) - elif lease is not None: - current_holder = lease["holder"] - if ( - float(lease["expires_at"]) <= now - or _compression_lock_holder_process_is_dead(current_holder) - ): - # Match acquisition semantics: an expired or provably dead - # owner is reclaimable. Deleting it inside this BEGIN IMMEDIATE - # transaction also fences a stale late flush after the mutation. - conn.execute( - "DELETE FROM session_turn_leases " - "WHERE conversation_id = ? AND holder = ?", - (conversation_id, current_holder), - ) - else: - raise SessionTurnLeaseLostError( - f"Session has an active turn lease; refusing transcript " - f"mutation for {session_id!r}" - ) - session = conn.execute( - "SELECT ended_at, end_reason FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if ( - session is not None - and session["ended_at"] is not None - and session["end_reason"] == "compression" - and not allow_closed_compression_parent - ): - raise CompressionSessionClosedError(session_id) - - @staticmethod - def _decode_display_metadata(raw: Any) -> Optional[Dict[str, Any]]: - """Decode a ``display_metadata`` column into the dict every reader expects. - - Every message read path must go through this. Returning the raw TEXT - instead reaches the desktop as a string, where ``'task_count' in meta`` - throws and fails the whole resume. Rows written before the encode guard - landed are double-encoded, so unwrap a second layer when we find one. - """ - if raw is None: - return None - try: - meta = json.loads(raw) if isinstance(raw, str) else raw - if isinstance(meta, str): - meta = json.loads(meta) - except (json.JSONDecodeError, TypeError): - logger.warning("Ignoring invalid display metadata on message row") - return None - if not isinstance(meta, dict): - logger.warning("Ignoring non-object display metadata on message row") - return None - return meta - - @staticmethod - def _reasoning_json_text(value: Any) -> Optional[str]: - """Serialize a structured reasoning field for its TEXT column. - - ``reasoning_details`` / ``codex_reasoning_items`` / ``codex_message_items`` - arrive as list/dict structures from the live runtime, but callers that - round-trip stored rows — ``get_messages`` straight into - ``replace_messages``, e.g. the POST /api/sessions/{id}/fork handler — - hand back the raw TEXT these columns already hold, because - ``get_messages`` only deserializes ``content`` and ``tool_calls``. - Re-dumping that TEXT double-encodes it, and the forked session's next - ``get_messages_as_conversation`` json.loads then yields the inner - string instead of the original list, so every reasoning-replay consumer - (all of which check ``isinstance(..., list)``) silently drops it. - Strings are therefore stored as-is; structures are dumped. - """ - if not value: - return None - if isinstance(value, str): - return value - return json.dumps(value) - - def append_message( - self, - session_id: str, - role: str, - content: str = None, - tool_name: str = None, - tool_calls: Any = None, - tool_call_id: str = None, - token_count: int = None, - finish_reason: str = None, - reasoning: str = None, - reasoning_content: str = None, - reasoning_details: Any = None, - codex_reasoning_items: Any = None, - codex_message_items: Any = None, - platform_message_id: str = None, - observed: bool = False, - effect_disposition: Optional[str] = None, - _compressed_summary: bool = False, - timestamp: Any = None, - api_content: Optional[str] = None, - display_kind: Optional[str] = None, - display_metadata: Optional[Dict[str, Any]] = None, - compression_lock_holder: Optional[str] = None, - turn_lease_holder: Optional[str] = None, - turn_lease_ttl_seconds: float = 300.0, - ) -> int: - """ - Append a message to a session. Returns the message row ID. - - Also increments the session's message_count (and tool_call_count - if role is 'tool' or tool_calls is present). - - ``platform_message_id`` is the external messaging platform's own - message ID (e.g. Telegram update_id, Yuanbao msg_id). It is - independent of the SQLite autoincrement primary key and is used by - platform-specific flows like yuanbao's recall guard to redact a - message by its platform-side identifier. - - ``api_content`` is the exact content string sent to the API for this - message when it differs from ``content`` (ephemeral memory/plugin - injections, persist overrides). It is a byte-fidelity sidecar for - prompt-cache-stable replay — stored as sent, except lone surrogates - (which sqlite3 cannot bind and which the conversation loop scrubs - from every outgoing payload anyway, so the scrubbed form IS the - wire bytes). - """ - # Display metadata is presentation-only and never changes the model - # context role/content replayed to providers. - display_metadata_json = self._encode_display_metadata(display_metadata) - # Serialize structured fields to JSON before entering the write txn - reasoning_details_json = self._reasoning_json_text(reasoning_details) - codex_items_json = self._reasoning_json_text(codex_reasoning_items) - codex_message_items_json = self._reasoning_json_text(codex_message_items) - # tool_calls may arrive as a Python list (from the live agent) or - # as a JSON string (from import/export). Parse first to avoid - # double-encoding. - if isinstance(tool_calls, str): - try: - tool_calls = json.loads(tool_calls) - except (json.JSONDecodeError, TypeError): - tool_calls = [] - tool_calls_json = json.dumps(tool_calls) if tool_calls else None - # Multimodal content (list of parts) must be JSON-encoded: sqlite3 - # cannot bind list/dict parameters directly. - stored_content = self._encode_content(content) - - message_timestamp = time.time() - if timestamp is not None: - try: - if hasattr(timestamp, "timestamp"): - message_timestamp = float(timestamp.timestamp()) - else: - message_timestamp = float(timestamp) - except (TypeError, ValueError): - logger.debug("Ignoring invalid explicit message timestamp: %r", timestamp) - - # Pre-compute tool call count - num_tool_calls = 0 - if tool_calls is not None: - num_tool_calls = len(tool_calls) if isinstance(tool_calls, list) else 1 - - def _do(conn): - self._check_transcript_write_guards( - conn, - session_id, - compression_lock_holder, - turn_lease_holder=turn_lease_holder, - turn_lease_ttl_seconds=turn_lease_ttl_seconds, - ) - cursor = conn.execute( - """INSERT INTO messages (session_id, role, content, tool_call_id, - tool_calls, tool_name, effect_disposition, timestamp, token_count, finish_reason, - reasoning, reasoning_content, reasoning_details, codex_reasoning_items, - codex_message_items, platform_message_id, observed, _compressed_summary, active, api_content, display_kind, display_metadata) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - session_id, - role, - stored_content, - tool_call_id, - tool_calls_json, - _scrub_surrogates(tool_name), - effect_disposition, - message_timestamp, - token_count, - finish_reason, - _scrub_surrogates(reasoning), - _scrub_surrogates(reasoning_content), - reasoning_details_json, - codex_items_json, - codex_message_items_json, - platform_message_id, - 1 if observed else 0, - 1 if _compressed_summary else 0, - 1, - _scrub_surrogates(api_content) if isinstance(api_content, str) else None, - _scrub_surrogates(display_kind) if isinstance(display_kind, str) else None, - display_metadata_json, - ), - ) - msg_id = cursor.lastrowid - - # Update counters - if num_tool_calls > 0: - conn.execute( - """UPDATE sessions SET message_count = message_count + 1, - tool_call_count = tool_call_count + ? WHERE id = ?""", - (num_tool_calls, session_id), - ) - else: - conn.execute( - "UPDATE sessions SET message_count = message_count + 1 WHERE id = ?", - (session_id,), - ) - return msg_id - - # Transcript append is THE critical write: its failure aborts the - # user's turn (session_persistence_failed). Use the long patience so - # a sibling process legitimately holding the write lock for seconds - # (VACUUM, TRUNCATE checkpoint at close, an older pre-bounded-merge - # process's FTS optimize) can't destroy a healthy turn (#74478). - return self._execute_write( - _do, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S - ) - - def append_messages_batch( - self, - session_id: str, - messages: List[Dict[str, Any]], - compression_lock_holder: Optional[str] = None, - turn_lease_holder: Optional[str] = None, - chunk_rows: Optional[int] = None, - turn_lease_ttl_seconds: float = 300.0, - ) -> int: - """Append multiple messages atomically in ONE write transaction. - - ``messages`` is a list of dicts in the same shape - :meth:`_insert_message_rows` already consumes for replace/compact/ - import (role, content, tool_name, tool_calls, tool_call_id, - finish_reason, reasoning*, codex_*, timestamp, api_content, - display_kind, display_metadata, ...). Reusing that helper keeps ONE - row-serialization path for every multi-row writer. - - A turn-boundary flush writes the whole turn (user + assistant + tool - rows, typically 3-8 messages) as one BEGIN IMMEDIATE / commit pair - instead of one transaction (and, off WAL, one fsync) per row. - - Atomicity contract: all rows land or none do (the caller re-flushes - unstamped messages on the next attempt). The same admission guards - as :meth:`append_message` run once for the batch — same session, - same instant. - - ``chunk_rows`` bounds the transaction size for LARGE copies (branch - seeds can be thousands of rows; measured: 10k rows ≈ 2.4s inside one - BEGIN IMMEDIATE because the FTS triggers run per row, which would - monopolize the write lock and starve concurrent writers). When set, - the batch commits in chunks of at most that many rows — same - recovery semantics as the old per-row loops (a mid-copy failure - leaves a partial seed), just with bounded lock holds. A turn flush - never needs it. Returns the inserted row count. - """ - if not messages: - return 0 - - if chunk_rows is not None and len(messages) > chunk_rows: - inserted_total = 0 - for start in range(0, len(messages), chunk_rows): - inserted_total += self.append_messages_batch( - session_id, - messages[start:start + chunk_rows], - compression_lock_holder=compression_lock_holder, - turn_lease_holder=turn_lease_holder, - turn_lease_ttl_seconds=turn_lease_ttl_seconds, - ) - return inserted_total - - def _do(conn): - self._check_transcript_write_guards( - conn, - session_id, - compression_lock_holder, - turn_lease_holder=turn_lease_holder, - turn_lease_ttl_seconds=turn_lease_ttl_seconds, - ) - from agent.transcript_repair import resolve_and_repair_transcript_batch - - inserted_rows = resolve_and_repair_transcript_batch( - conn, - session_id, - messages, - encode_content_fn=self._encode_content, - decode_content_fn=self._decode_content, - ) - inserted = 0 - tool_calls_total = 0 - if inserted_rows: - inserted, tool_calls_total = self._insert_message_rows( - conn, session_id, inserted_rows - ) - - # One aggregated counter update for the newly inserted rows. - if tool_calls_total > 0: - conn.execute( - """UPDATE sessions SET message_count = message_count + ?, - tool_call_count = tool_call_count + ? WHERE id = ?""", - (inserted, tool_calls_total, session_id), - ) - elif inserted > 0: - conn.execute( - "UPDATE sessions SET message_count = message_count + ? WHERE id = ?", - (inserted, session_id), - ) - return inserted - - # Same criticality as append_message: this IS the turn's transcript. - return self._execute_write( - _do, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S - ) - - def set_latest_matching_message_display_kind( - self, session_id: str, *, role: str, content: str, display_kind: str, - display_metadata: Optional[Dict[str, Any]] = None, - ) -> bool: - """Stamp presentation metadata on this turn's freshly persisted row. - - The model still receives ``role`` and ``content`` unchanged. Gateway and - CLI synthetic inputs call this immediately after their serial turn has - flushed, preserving producer provenance without classifying by content - during transcript rendering. - """ - if not session_id or not content or not display_kind: - return False - - def _do(conn): - row = conn.execute( - "SELECT id FROM messages WHERE session_id = ? AND role = ? " - "AND content = ? AND active = 1 ORDER BY id DESC LIMIT 1", - (session_id, role, self._encode_content(content)), - ).fetchone() - if row is None: - return False - conn.execute( - "UPDATE messages SET display_kind = ?, display_metadata = ? WHERE id = ?", - ( - _scrub_surrogates(display_kind), - self._encode_display_metadata(display_metadata), - row[0], - ), - ) - return True - - return bool(self._execute_write(_do)) #: Key under which message reactions live inside ``display_metadata``. #: Reactions share the existing per-message JSON column rather than a side #: table so they survive rewind/compaction row rewrites with the row itself. REACTIONS_METADATA_KEY = "reactions" - def set_message_reaction( - self, - session_id: str, - message_row_id: int, - emoji: Optional[str], - *, - author: str = "user", - ) -> Optional[List[Dict[str, Any]]]: - """Set (or with ``emoji=None`` clear) *author*'s reaction on one message. - - iOS Tapback semantics: one reaction per author per message. Re-sending - the same emoji clears it, a different emoji replaces it. Returns the - message's full reaction list after the write, or ``None`` when the row - doesn't exist or isn't part of *session_id*. - """ - if not session_id or message_row_id is None: - return None - - def _do(conn): - row = conn.execute( - "SELECT display_metadata FROM messages WHERE id = ? AND session_id = ?", - (message_row_id, session_id), - ).fetchone() - if row is None: - return None - - meta = self._decode_display_metadata(row[0]) or {} - existing = meta.get(self.REACTIONS_METADATA_KEY) - reactions = [ - r - for r in (existing if isinstance(existing, list) else []) - if isinstance(r, dict) and r.get("author") != author - ] - previous = next( - ( - r - for r in (existing if isinstance(existing, list) else []) - if isinstance(r, dict) and r.get("author") == author - ), - None, - ) - # Tapping the live reaction again retracts it. - toggling_off = ( - emoji is not None and previous is not None and previous.get("emoji") == emoji - ) - if emoji and not toggling_off: - reactions.append( - {"emoji": _scrub_surrogates(emoji), "author": author, "at": time.time()} - ) - - if reactions: - meta[self.REACTIONS_METADATA_KEY] = reactions - else: - meta.pop(self.REACTIONS_METADATA_KEY, None) - - conn.execute( - "UPDATE messages SET display_metadata = ? WHERE id = ?", - (self._encode_display_metadata(meta) if meta else None, message_row_id), - ) - return reactions - - return self._execute_write(_do) - - def get_message_reactions( - self, session_id: str, message_row_id: int - ) -> List[Dict[str, Any]]: - """Return the reaction list persisted on one message row (never ``None``).""" - if not session_id or message_row_id is None: - return [] - - with self._read_ctx() as conn: - row = conn.execute( - "SELECT display_metadata FROM messages WHERE id = ? AND session_id = ?", - (message_row_id, session_id), - ).fetchone() - - if row is None: - return [] - - meta = self._decode_display_metadata(row[0]) or {} - reactions = meta.get(self.REACTIONS_METADATA_KEY) - - return [r for r in reactions if isinstance(r, dict)] if isinstance(reactions, list) else [] - - def take_unseen_reactions( - self, session_id: str, *, author: str = "user" - ) -> List[Dict[str, Any]]: - """Return *author*'s not-yet-surfaced reactions and mark them seen. - - Powers the cache-safe model-context path: reactions are announced on the - NEXT user turn (never by rewriting the message that was reacted to), and - the ``seen`` stamp guarantees each one is announced exactly once. - """ - if not session_id: - return [] - - def _do(conn): - rows = conn.execute( - "SELECT id, role, content, display_metadata FROM messages " - "WHERE session_id = ? AND active = 1 AND display_metadata IS NOT NULL " - "ORDER BY id", - (session_id,), - ).fetchall() - - pending = [] - for row in rows: - meta = self._decode_display_metadata(row["display_metadata"]) - if not meta: - continue - reactions = meta.get(self.REACTIONS_METADATA_KEY) - if not isinstance(reactions, list): - continue - - changed = False - for reaction in reactions: - if ( - not isinstance(reaction, dict) - or reaction.get("author") != author - or reaction.get("seen") - ): - continue - reaction["seen"] = True - changed = True - content = self._decode_content(row["content"]) - pending.append( - { - "row_id": row["id"], - "role": row["role"], - "emoji": reaction.get("emoji") or "", - "text": content if isinstance(content, str) else "", - } - ) - - if changed: - conn.execute( - "UPDATE messages SET display_metadata = ? WHERE id = ?", - (self._encode_display_metadata(meta), row["id"]), - ) - - return pending - - return self._execute_write(_do) or [] - - def latest_message_row_id( - self, session_id: str, *, role: str = "user", offset: int = 0, require_text: bool = True - ) -> Optional[int]: - """Row id of the most recent active message with *role*, or ``None``. - - Two callers, same need — "the message I mean, without an id": the agent - defaulting to the turn that triggered it, and the desktop reacting to a - live message that hasn't round-tripped through a resume yet. - ``offset`` steps to earlier turns (1 = the one before the latest) so a - reaction can land retroactively — "two messages ago" is how the caller - thinks about it. - - ``require_text`` (default) skips rows with no plain-text content — - tool-call-only assistant turns and attachment stubs don't render as - bubbles, so "the latest message" as a HUMAN means it must never - resolve to one (a reaction landing on an invisible row looks dropped, - and its annotation quotes an empty string). - """ - if not session_id or role not in {"user", "assistant"} or offset < 0: - return None - - text_filter = ( - "AND content IS NOT NULL AND TRIM(content) != '' " if require_text else "" - ) - - with self._read_ctx() as conn: - row = conn.execute( - "SELECT id FROM messages WHERE session_id = ? AND role = ? " - f"AND active = 1 {text_filter}ORDER BY id DESC LIMIT 1 OFFSET ?", - (session_id, role, int(offset)), - ).fetchone() - - return row[0] if row else None - - def latest_user_message_row_id(self, session_id: str) -> Optional[int]: - """Row id of the most recent active user message, or ``None``. - - The agent's default reaction target: "the message that triggered me", - so the model never has to thread row ids through a tool call (mirrors - the photon adapter's ``_record_last_inbound``). - """ - return self.latest_message_row_id(session_id, role="user") - - def get_message_role(self, session_id: str, row_id: int) -> Optional[str]: - """Role of the active message at *row_id* in *session_id*, or ``None``. - - Lets a reaction event carry the target's role so a renderer can match - a live message that doesn't know its durable row id yet. - """ - if not session_id: - return None - - with self._read_ctx() as conn: - row = conn.execute( - "SELECT role FROM messages WHERE id = ? AND session_id = ? AND active = 1", - (int(row_id), session_id), - ).fetchone() - - return row[0] if row else None - - def _insert_message_rows(self, conn, session_id: str, messages: List[Dict[str, Any]]) -> tuple[int, int]: - """Insert *messages* as fresh active rows for *session_id*. - - Shared by :meth:`replace_messages` (delete-then-insert) and - :meth:`archive_and_compact` (soft-archive-then-insert). Runs inside the - caller's write transaction (takes the live ``conn``). Returns - ``(inserted_count, tool_call_count)``. Does NOT touch sessions.* counters - — the caller owns that, since the two flows reconcile counts differently. - """ - now_ts = time.time() - inserted = 0 - tool_calls_total = 0 - for msg in messages: - role = msg.get("role", "unknown") - tool_calls = msg.get("tool_calls") - message_timestamp = now_ts - if msg.get("timestamp") is not None: - try: - ts_value = msg.get("timestamp") - if hasattr(ts_value, "timestamp"): - message_timestamp = float(ts_value.timestamp()) - else: - message_timestamp = float(ts_value) - except (TypeError, ValueError): - logger.debug("Ignoring invalid explicit message timestamp: %r", msg.get("timestamp")) - reasoning_details = msg.get("reasoning_details") if role == "assistant" else None - codex_reasoning_items = ( - msg.get("codex_reasoning_items") if role == "assistant" else None - ) - codex_message_items = ( - msg.get("codex_message_items") if role == "assistant" else None - ) - reasoning_details_json = self._reasoning_json_text(reasoning_details) - codex_items_json = self._reasoning_json_text(codex_reasoning_items) - codex_message_items_json = self._reasoning_json_text(codex_message_items) - # tool_calls may arrive as a Python list (from the live agent) - # or as a JSON string (from import_sessions / export_session, - # which store it as TEXT). json.dumps on an already-serialized - # string double-encodes it, so parse first. - if isinstance(tool_calls, str): - try: - tool_calls = json.loads(tool_calls) - except (json.JSONDecodeError, TypeError): - tool_calls = [] - tool_calls_json = json.dumps(tool_calls) if tool_calls else None - # Accept either `platform_message_id` (new explicit name) or - # `message_id` (yuanbao's existing convention on message dicts). - platform_msg_id = ( - msg.get("platform_message_id") or msg.get("message_id") - ) - - api_content = msg.get("api_content") - - cur = conn.execute( - """INSERT INTO messages (session_id, role, content, tool_call_id, - tool_calls, tool_name, effect_disposition, timestamp, token_count, finish_reason, - reasoning, reasoning_content, reasoning_details, codex_reasoning_items, - codex_message_items, platform_message_id, observed, _compressed_summary, active, api_content, display_kind, display_metadata) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - session_id, - role, - self._encode_content(msg.get("content")), - msg.get("tool_call_id"), - tool_calls_json, - _scrub_surrogates(msg.get("tool_name")), - msg.get("effect_disposition"), - message_timestamp, - msg.get("token_count"), - msg.get("finish_reason"), - _scrub_surrogates(msg.get("reasoning")) if role == "assistant" else None, - _scrub_surrogates(msg.get("reasoning_content")) if role == "assistant" else None, - reasoning_details_json, - codex_items_json, - codex_message_items_json, - platform_msg_id, - 1 if msg.get("observed") else 0, - 1 if msg.get("_compressed_summary") else 0, - 1, - _scrub_surrogates(api_content) if isinstance(api_content, str) else None, - _scrub_surrogates(msg.get("display_kind")) if isinstance(msg.get("display_kind"), str) else None, - self._encode_display_metadata(msg.get("display_metadata")), - ), - ) - if isinstance(msg, dict) and cur.lastrowid is not None: - msg["_row_id"] = cur.lastrowid - inserted += 1 - if tool_calls is not None: - tool_calls_total += ( - len(tool_calls) if isinstance(tool_calls, list) else 1 - ) - now_ts = max(now_ts + 1e-6, message_timestamp + 1e-6) - return inserted, tool_calls_total - - def replace_messages( - self, - session_id: str, - messages: List[Dict[str, Any]], - active_only: bool = False, - archive_dropped: bool = False, - reject_active_turn_lease: bool = False, - ) -> None: - """Atomically replace the stored messages for a session. - - Used by transcript-rewrite flows such as /retry, /undo, and /compress. - The delete + reinsert sequence must commit as one transaction so a - mid-rewrite failure does not leave SQLite with a partial transcript. - - DESTRUCTIVE by default: every row for the session is DELETEd (and drops - out of the FTS index). For compaction that must preserve the - pre-compaction transcript under the same id, use - :meth:`archive_and_compact` instead. - - Pass ``active_only=True`` to replace ONLY the live (``active = 1``) rows, - leaving soft-archived rows (``active = 0`` — e.g. the ``compacted = 1`` - turns that :meth:`archive_and_compact` keeps on disk for #38763 - durability, or rewind/undo rows) untouched. Callers that share a session - id with an agent already running in-place compaction must use this so a - full-history rewrite doesn't wipe the rows the agent deliberately - archived. ``message_count``/``tool_call_count`` then track the live set, - matching :meth:`archive_and_compact`. - - Pass ``archive_dropped=True`` to SOFT-archive the live rows instead of - DELETEing them: the replaced turns stay on disk with ``active = 0``, - ``compacted = 0`` — the same "the user took it back" marking - :meth:`rewind_to_message` applies — and stay readable via - :meth:`get_messages` with ``include_inactive=True``. This is the mode a - rewind/edit/regenerate must use: those flows overwrite a transcript the - user may not have meant to drop, and a plain DELETE also evicts the rows - from the FTS index, leaving nothing to recover from (#82756). It implies - active-only handling — already-archived rows are never touched — so - ``active_only`` is redundant with it. The rewritten set is inserted as - fresh active rows exactly as in the destructive path, so the live view - is identical either way; only the durability of the dropped turns - differs. - - Pass ``reject_active_turn_lease=True`` for user-initiated rewrites that - do not already own the cross-process turn lease. The lease check and - transcript mutation then share one write transaction, so a second - process cannot archive or replace a turn that is still being produced. - """ - - active_clause = " AND active = 1" if active_only else "" - - def _do(conn): - if reject_active_turn_lease: - self._check_transcript_write_guards( - conn, - session_id, - None, - reject_active_turn_lease=True, - reject_active_compression_lock=True, - ) - else: - session = conn.execute( - "SELECT ended_at, end_reason FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if ( - session is not None - and session["ended_at"] is not None - and session["end_reason"] == "compression" - ): - raise CompressionSessionClosedError(session_id) - if archive_dropped: - # Content-preserving UPDATE: the rows keep their FTS entries - # (the messages_fts triggers fire on INSERT / DELETE / UPDATE - # of content columns, not on `active`), so the replaced turns - # stay readable via get_messages(include_inactive=True) and - # searchable with include_inactive=True after the rewrite. - conn.execute( - "UPDATE messages SET active = 0 " - "WHERE session_id = ? AND active = 1", - (session_id,), - ) - else: - conn.execute( - f"DELETE FROM messages WHERE session_id = ?{active_clause}", - (session_id,), - ) - conn.execute( - "UPDATE sessions SET message_count = 0, tool_call_count = 0 WHERE id = ?", - (session_id,), - ) - total_messages, total_tool_calls = self._insert_message_rows( - conn, session_id, messages - ) - conn.execute( - "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", - (total_messages, total_tool_calls, session_id), - ) - - self._execute_write(_do) - - def has_archived_messages(self, session_id: str) -> bool: - """Return True if the session has any soft-archived (``active = 0``) rows. - - Cheap existence probe — does not load rows. NOTE: production rewrite - paths no longer branch on this (they pass ``active_only=True`` - unconditionally — a probe can fail open or race a concurrent - ``archive_and_compact``, #80216); kept for tests and diagnostics. - """ - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT 1 FROM messages WHERE session_id = ? AND active = 0 LIMIT 1", - (session_id,), - ) - return cursor.fetchone() is not None - - def get_active_message_watermark(self, session_id: str) -> int: - """MAX(id) of the session's active rows — the compression watermark. - - Captured at compression START (before the slow provider summary call). - Every active row with ``id > watermark`` at commit time arrived - concurrently and must survive the compaction verbatim. Returns 0 for - an empty/unknown session. - """ - if not session_id: - return 0 - with self._read_ctx() as conn: - row = conn.execute( - "SELECT COALESCE(MAX(id), 0) FROM messages " - "WHERE session_id = ? AND active = 1", - (session_id,), - ).fetchone() - return int(row[0]) if row else 0 - - def archive_and_compact( - self, - session_id: str, - compacted_messages: List[Dict[str, Any]], - model_config_patch: Optional[Dict[str, Any]] = None, - watermark: Optional[int] = None, - lock_holder: Optional[str] = None, - tail_count: int = 0, - ) -> int: - """Non-destructive in-place compaction for a single durable session id. - - Soft-archives the active messages (``active = 0``) and inserts - *compacted_messages* as fresh active rows — atomically, in one write - transaction. The conversation keeps ONE session id for life (#38763) - WITHOUT destroying history: - - - The live-context load (:meth:`get_messages_as_conversation`, - :meth:`get_messages`) filters ``active = 1`` by default, so the model - reloads ONLY the compacted set. - - The archived pre-compaction turns stay on disk (active=0) and stay - DISCOVERABLE: they are marked compacted=1, and search_messages() - includes compacted=1 rows by default — so session_search still finds - them, unlike rewind/undo rows (active=0, compacted=0) which stay - hidden. They remain in the FTS index (the messages_fts* triggers - index on INSERT / drop on DELETE and don't key on active/compacted; - flipping to active=0 is a content-preserving UPDATE) and are - recoverable via get_messages(..., include_inactive=True). - - Concurrent-append safety (#75316): when *watermark* is provided (the - value of :meth:`get_active_message_watermark` captured at compression - START), rows that arrived during the slow provider summary call - (``id > watermark``) are NOT summarized away. They are re-sequenced - after the compacted set by a pure-SQL column clone (every column - except ``id`` — content, api_content, platform_message_id, token - counts, reasoning sidecars all survive byte-exact, and the FTS - triggers index the clones naturally), and the originals are archived. - NOTE: re-sequencing assigns the tail rows fresh ids; consumers that - reference durable row ids re-resolve by content (see 3e8ab0610). - ``watermark=None`` preserves the historical archive-everything - behavior. - - Commit-fence safety: when *lock_holder* is provided, the commit - verifies INSIDE the transaction that the compression lock is still - held by that holder and unexpired — a compression whose lease was - reclaimed (crash cleanup, TTL expiry, competing writer) fails the - commit instead of clobbering the winner's transcript. - - *tail_count* (default 0) names how many of the LAST rows of - *compacted_messages* are the verbatim carried-forward tail the - compressor protected rather than summarized (#86366). Those rows' - ORIGINALS — which this call archives as a side effect of the blanket - soft-archive — are superseded byte-identical duplicates, not - "summarized away" content, so they are stamped rewind-style - (``active=0, compacted=0``, hidden from search_messages) instead of - ``compacted=1``. Without this the tail originals satisfy the recall - filter alongside their live clones and session_search returns every - carried-forward message once per compaction. Callers that cannot know - their tail shape keep the historical archive-everything behavior. - - ``message_count`` is set to the ACTIVE count after commit, matching - what the live load returns. ``model_config_patch`` is merged into the - session's JSON config in the same transaction; a ``None`` value - removes that key. Returns the new active count. - """ - - def _do(conn): - if lock_holder is not None: - lock_row = conn.execute( - "SELECT holder, expires_at FROM compression_locks " - "WHERE session_id = ?", - (session_id,), - ).fetchone() - if ( - lock_row is None - or lock_row["holder"] != lock_holder - or float(lock_row["expires_at"]) <= time.time() - ): - raise SessionCompressionInProgressError( - f"Compression lease for {session_id!r} lost before " - "commit; refusing to publish a stale compaction" - ) - - patched_model_config = None - if model_config_patch is not None: - # on_missing="raise": a prune/compaction must not commit - # against a vanished session row (the compressor's caller - # converts the raised error into a safe keep-the-original - # no-op), unlike the flag setters which tolerate missing rows. - patched_model_config = self._merge_model_config_json( - conn, session_id, model_config_patch, on_missing="raise" - ) - - # Concurrent tail: active rows that arrived after the watermark. - # Snapshot their ids and tool_calls now — the clone below needs a - # stable id list, and the tool-call count keeps sessions.* honest. - tail_ids: list[int] = [] - tail_tool_calls = 0 - if watermark is not None: - for row in conn.execute( - "SELECT id, tool_calls FROM messages " - "WHERE session_id = ? AND active = 1 AND id > ? " - "ORDER BY id", - (session_id, int(watermark)), - ).fetchall(): - tail_ids.append(int(row["id"])) - raw = row["tool_calls"] - if raw: - try: - parsed = json.loads(raw) if isinstance(raw, str) else raw - tail_tool_calls += len(parsed) if isinstance(parsed, list) else 0 - except (TypeError, ValueError): - pass - - # Soft-archive the live turns: active=0 hides them from the live - # context load, compacted=1 marks them as "summarized away" (vs - # rewind/undo's active=0+compacted=0, which means "user took it - # back"). search_messages includes compacted=1 rows by default so - # the pre-compaction transcript stays discoverable; live-context - # loads (active=1 only) still exclude them. Tail originals whose - # verbatim clones ride inside *compacted_messages* (tail_count) - # are superseded duplicates instead (#86366): they get the - # rewind-style flags so they stop matching the recall filter. - # Rewind-target ids: the originals of the carried-forward tail - # rows (tail_count), captured BEFORE any flag flips. Named apart - # from the watermark `tail_ids` below on purpose — the two are - # different sets (#86366): rewind targets sit AT/BELOW the - # watermark (the compressor only saw rows up to it), while - # `tail_ids` are concurrent appends ABOVE it. Without the bound, - # a concurrent append would steal a LIMIT slot and leave a real - # carried-forward original stamped compacted=1. - rewind_tail_ids: Optional[list[int]] = None - if tail_count > 0: - if watermark is not None: - tail_rows = conn.execute( - "SELECT id FROM messages " - "WHERE session_id = ? AND active = 1 AND id <= ? " - "ORDER BY id DESC LIMIT ?", - (session_id, int(watermark), int(tail_count)), - ).fetchall() - else: - tail_rows = conn.execute( - "SELECT id FROM messages " - "WHERE session_id = ? AND active = 1 ORDER BY id DESC LIMIT ?", - (session_id, int(tail_count)), - ).fetchall() - rewind_tail_ids = [int(row["id"]) for row in tail_rows] - - # The watermark clone below re-inserts `tail_ids` rows byte-exact - # as live rows — their originals are the SAME superseded-duplicate - # class as the carried-forward tail (#86366), so they take the - # rewind flags too instead of double-matching the recall filter. - rewind_ids = [*(rewind_tail_ids or []), *tail_ids] - - if rewind_ids: - placeholders = ",".join("?" for _ in rewind_ids) - conn.execute( - "UPDATE messages SET active = 0, compacted = 0 " - f"WHERE session_id = ? AND id IN ({placeholders})", - [session_id, *rewind_ids], - ) - conn.execute( - "UPDATE messages SET active = 0, compacted = 1 " - "WHERE session_id = ? AND active = 1 " - f"AND id NOT IN ({placeholders})", - [session_id, *rewind_ids], - ) - else: - conn.execute( - "UPDATE messages SET active = 0, compacted = 1 " - "WHERE session_id = ? AND active = 1", - (session_id,), - ) - inserted, tool_calls_total = self._insert_message_rows( - conn, session_id, compacted_messages - ) - - if tail_ids: - # Re-sequence the concurrent tail after the compacted set via - # a pure-SQL column clone: no decode/re-encode round trip, no - # field drift — new id, active=1, compacted=0, all else exact. - placeholders = ",".join("?" for _ in tail_ids) - clone_cols = [ - c for c in self._message_column_names(conn) - if c not in ("id", "active", "compacted") - ] - col_list = ", ".join(clone_cols) - conn.execute( - f"INSERT INTO messages ({col_list}, active, compacted) " - f"SELECT {col_list}, 1, 0 FROM messages " - f"WHERE id IN ({placeholders}) ORDER BY id", - tail_ids, - ) - inserted += len(tail_ids) - tool_calls_total += tail_tool_calls - - # message_count / tool_call_count reflect the LIVE (active) set — - # the archived rows are still on disk but not part of the live count. - if model_config_patch is None: - conn.execute( - "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", - (inserted, tool_calls_total, session_id), - ) - else: - conn.execute( - "UPDATE sessions SET message_count = ?, tool_call_count = ?, " - "model_config = ? WHERE id = ?", - (inserted, tool_calls_total, patched_model_config, session_id), - ) - return inserted - - return self._execute_write(_do) - - def _message_column_names(self, conn) -> List[str]: - """Column names of the messages table, cached per-connection era.""" - cached = getattr(self, "_message_columns_cache", None) - if cached: - return cached - cols = [r[1] for r in conn.execute("PRAGMA table_info(messages)").fetchall()] - self._message_columns_cache = cols - return cols - - def set_latest_user_api_content( - self, session_id: str, content: Any, api_content: str - ) -> int: - """Backfill the ``api_content`` sidecar onto the newest ACTIVE user row. - - In-place preflight compaction (:meth:`archive_and_compact`) inserts the - current turn's user row BEFORE the turn prologue composes the - prefetch/plugin sidecar, and the subsequent crash persist identity-skips - every compacted dict — without this backfill the stamped sidecar would - never land in the DB and any reload would replay clean content, - re-introducing the prompt-cache divergence the sidecar exists to close. - - The ``content`` match is a defensive guard: if the newest active user - row is not the message the caller stamped (racing rewrite, unexpected - tail shape), nothing is written. Returns the number of rows updated - (0 or 1). - """ - encoded = self._encode_content(content) - - def _do(conn): - cursor = conn.execute( - "UPDATE messages SET api_content = ? WHERE id = (" - "SELECT id FROM messages " - "WHERE session_id = ? AND role = 'user' AND active = 1 " - "ORDER BY id DESC LIMIT 1" - ") AND content IS ?", - (_scrub_surrogates(api_content), session_id, encoded), - ) - return cursor.rowcount - - return self._execute_write(_do) - - def _dedupe_display_generations(self, rows): - """Collapse compaction generations so each message appears once. - - Compaction epochs copy the protected tail into each new generation, so - one logical message can exist as several rows (identical - role/content/timestamp) with different ``active`` flags and ids. A - display read must surface each exactly once: prefer the live row, then - the newest generation. - - This is the ONE definition shared by every display projection — - :meth:`get_messages` (REST), :meth:`get_resume_conversations` and - :meth:`get_ancestor_display_prefix` (gateway resume), and - :meth:`get_messages_as_conversation` (warm-session payload) — so the - surfaces cannot disagree about the same transcript. *rows* must already - be ordered by ``id``; the returned list keeps that order. - """ - seen: Dict[Tuple[Any, ...], Any] = {} - for row in rows: - dedupe_content = row["content"] - if row["role"] == "user": - from agent.context_compressor import split_user_originated_turn - - candidate = { - "role": "user", - "content": self._decode_content(row["content"]), - "display_kind": row["display_kind"], - "display_metadata": self._decode_display_metadata( - row["display_metadata"] - ), - } - handoff, live_view = split_user_originated_turn(candidate) - if handoff is not None and live_view is not None: - dedupe_content = self._encode_content(live_view.get("content")) - # Tool fields participate in the dedupe key: compaction copies them - # verbatim, so identical tool messages across generations still - # collapse, while distinct tool calls that happen to share - # role/content/timestamp are never merged. - key = ( - row["role"], - dedupe_content, - row["timestamp"], - row["tool_call_id"], - row["tool_calls"], - row["tool_name"], - ) - cur = seen.get(key) - if cur is None or (row["active"], row["id"]) > (cur["active"], cur["id"]): - seen[key] = row - return sorted(seen.values(), key=lambda r: r["id"]) - - def get_messages( - self, - session_id: str, - include_inactive: bool = False, - include_compacted: bool = False, - limit: Optional[int] = None, - offset: int = 0, - latest: bool = False, - after_id: Optional[int] = None, - ) -> List[Dict[str, Any]]: - """Load messages for a session in insertion order. - - By default only active messages are returned. Pass - ``include_inactive=True`` to load soft-deleted rows (e.g. for - audit / debug views of rewound history). See - :meth:`rewind_to_message` for the soft-delete mechanic. - - Pass ``include_compacted=True`` to additionally load rows preserved - by in-place context compaction (``active=0, compacted=1``). Those are - durable display history, not soft-deleted rows — a user-visible - transcript read must not drop them, or earlier turns silently become - unreachable once the UI exhausts its active-only window. Soft-deleted - Undo/Rewind rows (``active=0, compacted=0``) stay excluded; use - ``include_inactive`` for those. - - Ordered by AUTOINCREMENT id (true insertion order) rather than - timestamp — see c03acca50 for the WSL2 clock-regression rationale. - - When ``limit`` is provided, returns at most ``limit`` messages - starting from ``offset`` (0-based, in insertion order). Enables - pagination for the API endpoint to avoid loading entire transcripts. - With ``latest=True``, the offset is measured back from the newest - message and the selected page is still returned in chronological - order. ``offset`` alone (without ``limit``) also pages — SQLite - requires a LIMIT clause for OFFSET, so it's emitted as ``LIMIT -1`` - (unbounded). - - ``after_id`` enables keyset pagination (``id > after_id``): O(1) - page seeks on huge transcripts where OFFSET degrades to O(n) per - page. Ascending order only (incompatible with ``latest``/``offset``). - """ - if after_id is not None and (latest or offset): - raise ValueError("after_id is incompatible with latest/offset paging") - if after_id is not None and include_compacted: - raise ValueError("after_id is incompatible with include_compacted (deduped display reads use offset paging)") - if include_inactive: - # Audit / debug reads: every row, including soft-deleted. - active_clause = "" - elif include_compacted: - # Display history: active rows plus rows preserved by in-place - # compaction (active=0, compacted=1), but never soft-deleted - # Undo/Rewind rows (active=0, compacted=0). - active_clause = " AND (active = 1 OR compacted = 1)" - else: - active_clause = " AND active = 1" - keyset_clause = " AND id > ?" if after_id is not None else "" - sql = ( - "SELECT * FROM messages WHERE session_id = ?" - f"{active_clause}{keyset_clause} ORDER BY id {'DESC' if latest else 'ASC'}" - ) - params: list = [session_id] - if after_id is not None: - params.append(after_id) - if include_compacted: - # Read the full display set (a session's rows are bounded; the - # UI-level 500-row cap lives in the endpoint, not here), dedupe - # generations, then apply paging. - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT * FROM messages WHERE session_id = ?" + active_clause - + " ORDER BY id ASC", - [session_id], - ) - all_rows = cursor.fetchall() - rows = self._dedupe_display_generations(all_rows) - if latest: - rows = rows[::-1] - rows = rows[offset:] - if limit is not None: - rows = rows[:limit] - if latest: - rows = rows[::-1] - else: - if limit is not None or offset: - # SQLite's OFFSET requires LIMIT; -1 means "no limit". - sql += " LIMIT ? OFFSET ?" - params.extend([-1 if limit is None else limit, offset]) - with self._read_ctx() as conn: - cursor = conn.execute(sql, params) - rows = cursor.fetchall() - if latest: - rows.reverse() - result = [] - for row in rows: - msg = dict(row) - if msg.pop("_compressed_summary", 0): - msg["_compressed_summary"] = True - if "content" in msg: - msg["content"] = self._decode_content(msg["content"]) - if msg.get("tool_calls"): - try: - msg["tool_calls"] = json.loads(msg["tool_calls"]) - except (json.JSONDecodeError, TypeError): - logger.warning("Failed to deserialize tool_calls in get_messages, falling back to []") - msg["tool_calls"] = [] - if msg.get("display_metadata") is not None: - msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"]) - result.append(msg) - return result - - def find_pr_url_messages(self, session_ids: List[str]) -> List[Dict[str, Any]]: - """Tool results in these sessions that mention a GitHub PR url. - - A candidate scan, deliberately loose: it hands back every tool result - containing ``/pull/`` and leaves the caller to decide which ones make a - claim (see the desktop's PR recovery, which only accepts an output that - is a bare PR url — the signature of ``gh pr create``). Ordered - oldest-first per session so the caller can take the last match. - """ - found: List[Dict[str, Any]] = [] - ids = [s for s in session_ids if s] - for start in range(0, len(ids), 900): # SQLite's bound-variable ceiling. - chunk = ids[start : start + 900] - placeholders = ",".join("?" * len(chunk)) - with self._read_ctx() as conn: - rows = conn.execute( - f"""SELECT session_id, content FROM messages - WHERE session_id IN ({placeholders}) - AND role = 'tool' AND content LIKE '%/pull/%' - ORDER BY id ASC""", - chunk, - ).fetchall() - found.extend({"session_id": row[0], "content": row[1]} for row in rows) - return found - - def get_messages_around( - self, - session_id: str, - around_message_id: int, - window: int = 5, - ) -> Dict[str, Any]: - """Load a window of messages anchored on a specific message id. - - Returns a dict with: - - ``window``: up to ``window`` messages before the anchor, the anchor - itself, and up to ``window`` messages after, ordered by id ascending. - - ``messages_before``: count of messages strictly before the anchor - still in the session (== window unless we hit the start). - - ``messages_after``: count of messages strictly after the anchor - still in the session (== window unless we hit the end). - - Used by ``session_search`` for both the discovery shape (anchored on the - FTS5 match) and the scroll shape (anchored on any message id). The - ``messages_before`` / ``messages_after`` counts let the caller detect - session boundaries: when either is less than ``window``, the agent has - reached one end of the session. - - Returns an empty window when ``around_message_id`` is not a real id in - ``session_id`` — callers decide how to surface that. - """ - if window < 0: - window = 0 - with self._read_ctx() as conn: - # Confirm the anchor exists in this session. - anchor_exists = conn.execute( - "SELECT 1 FROM messages WHERE id = ? AND session_id = ? LIMIT 1", - (around_message_id, session_id), - ).fetchone() - if not anchor_exists: - return {"window": [], "messages_before": 0, "messages_after": 0} - - # Two queries: anchor + before (DESC, take window+1), and after - # (ASC, take window). Final order is id ASC. - before_rows = conn.execute( - "SELECT * FROM messages " - "WHERE session_id = ? AND id <= ? " - "ORDER BY id DESC LIMIT ?", - (session_id, around_message_id, window + 1), - ).fetchall() - after_rows = conn.execute( - "SELECT * FROM messages " - "WHERE session_id = ? AND id > ? " - "ORDER BY id ASC LIMIT ?", - (session_id, around_message_id, window), - ).fetchall() - - # before_rows is DESC; reverse so it's ASC, then concatenate after_rows. - rows = list(reversed(before_rows)) + list(after_rows) - result = [] - for row in rows: - msg = dict(row) - if "content" in msg: - msg["content"] = self._decode_content(msg["content"]) - if msg.get("tool_calls"): - try: - msg["tool_calls"] = json.loads(msg["tool_calls"]) - except (json.JSONDecodeError, TypeError): - logger.warning( - "Failed to deserialize tool_calls in get_messages_around, falling back to []" - ) - msg["tool_calls"] = [] - if msg.get("display_metadata") is not None: - msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"]) - result.append(msg) - - # before_rows includes the anchor itself; subtract 1 for the count of - # messages strictly before the anchor in the returned slice. - messages_before = max(0, len(before_rows) - 1) - messages_after = len(after_rows) - return { - "window": result, - "messages_before": messages_before, - "messages_after": messages_after, - } - - def resolve_resume_session_id(self, session_id: str) -> str: - """Redirect a resume target to the descendant session that holds the messages. - - Context compression ends the current session and forks a new child session - (linked via ``parent_session_id``). The flush cursor is reset, so the - child is where new messages actually land — the parent ends up with - ``message_count = 0`` rows unless messages had already been flushed to - it before compression. See #15000. - - This helper walks ``parent_session_id`` forward from ``session_id`` and - returns the descendant in the chain that has the **most recent** messages. - Unlike the original logic, it does NOT short-circuit when the starting - session already has messages — a descendant that was created by - compression may hold the continuation content and should be preferred - by the WebUI and gateway for ``--resume`` and session loading. - - If no descendant (including the starting session) has any messages, - the original ``session_id`` is returned unchanged. - - The chain is always walked via the child whose ``started_at`` is - latest; that matches the single-chain shape that compression creates. - A depth cap (32) guards against accidental loops in malformed data. - """ - if not session_id: - return session_id - - # Follow the compression-continuation chain forward to the live tip - # FIRST. Auto-compression ends the current session and forks a - # continuation child, but a long-lived parent keeps its own flushed - # message rows — so the empty-head walk below never redirects it, and - # resuming the parent id reloads the pre-compression transcript while - # the turns generated *after* compression (and their responses) sit in - # the continuation. ``get_compression_tip`` is lineage-aware: it only - # follows children whose parent ended with ``end_reason='compression'`` - # (created after the parent was ended), so delegation / branch children - # never hijack the resume. This is the fix for the desktop "I came back - # and the reply isn't there" report on large sessions. - try: - tip = self.get_compression_tip(session_id) - except Exception: - tip = session_id - if tip and tip != session_id: - session_id = tip - - with self._read_ctx() as conn: - current = session_id - seen = {current} - best = None # tracks the last (deepest) node with messages - - for _ in range(32): - # Check if the current node has messages. - try: - row = conn.execute( - "SELECT 1 FROM messages WHERE session_id = ? LIMIT 1", - (current,), - ).fetchone() - except Exception: - return session_id - if row is not None: - best = current - - # Walk to the most-recently-started child — but skip explicit - # branch (`_branched_from`), delegate/subagent (`_delegate_from`), - # reset-continuation (`_reset_from` or the legacy same-key - # heuristic — a post-reset conversation must never be reached - # by resuming the parent the user reset away), and tool - # children. They also carry a ``parent_session_id`` yet - # are NOT compression continuations; following them would hijack - # the resume target to an unrelated session (e.g. a subagent - # run). This mirrors the child-exclusion in ``get_compression_tip``. - try: - child_row = conn.execute( - "SELECT id FROM sessions AS child " - "WHERE child.parent_session_id = ? " - " AND json_extract(COALESCE(child.model_config, '{}'), '$._branched_from') IS NULL " - " AND json_extract(COALESCE(child.model_config, '{}'), '$._delegate_from') IS NULL " - " AND json_extract(COALESCE(child.model_config, '{}'), '$._reset_from') IS NULL " - f" AND NOT {_legacy_reset_child_sql('child', _RESET_END_REASONS_SQL)} " - " AND COALESCE(child.source, '') != 'tool' " - "ORDER BY child.started_at DESC, child.id DESC LIMIT 1", - (current,), - ).fetchone() - except Exception: - return session_id - if child_row is None: - break - child_id = child_row["id"] if hasattr(child_row, "keys") else child_row[0] - if not child_id or child_id in seen: - break - seen.add(child_id) - current = child_id - - return best if best is not None else session_id - - def get_messages_as_conversation( - self, - session_id: str, - include_ancestors: bool = False, - include_inactive: bool = False, - repair_alternation: bool = False, - include_row_ids: bool = False, - include_compacted: bool = False, - ) -> List[Dict[str, Any]]: - """ - Load messages in the OpenAI conversation format (role + content dicts). - Used by the gateway to restore conversation history. - - By default only active messages are returned. Pass - ``include_inactive=True`` to load soft-deleted (rewound) rows - as well. See :meth:`rewind_to_message`. - - ``include_compacted=True`` additionally loads rows preserved by - in-place compaction (``active=0, compacted=1``), deduped by - :meth:`_dedupe_display_generations`. DISPLAY reads want this; the - model-fed restore must NOT pass it, or a resumed session regrows the - very history compaction just summarized away. - - ``repair_alternation=True`` runs ``repair_message_sequence`` over the - loaded list before returning it. Callers that restore a session for - LIVE REPLAY should pass it: a durable alternation violation (e.g. a - ``user;user`` pair left by a turn that persisted no assistant row) - otherwise re-triggers the pre-request defensive repair on every - single request for the rest of the session's life — the repair - mutates only the per-request list, never the stored transcript. - Inspection/export consumers keep the default and see the transcript - verbatim. - """ - session_ids = [session_id] - if include_ancestors and not self._is_explicit_branch_session(session_id): - session_ids = self._session_lineage_root_to_tip(session_id) - - if include_inactive: - active_clause = "" - elif include_compacted: - active_clause = " AND (active = 1 OR compacted = 1)" - else: - active_clause = " AND active = 1" - with self._read_ctx() as conn: - placeholders = ",".join("?" for _ in session_ids) - rows = conn.execute( - f"SELECT {self._CONVERSATION_ROW_COLUMNS} " - f"FROM messages WHERE session_id IN ({placeholders})" - # Order by AUTOINCREMENT id (true insertion order), NOT timestamp: - # append_message stamps rows with time.time(), which is not - # monotonic (WSL2, NTP steps, VM/laptop sleep resume). A later - # row can carry an earlier timestamp than its predecessor, and - # ORDER BY timestamp would then sort an assistant tool_calls row - # after its tool response, breaking tool-call/response adjacency - # and triggering an HTTP 400 on replay. This matches get_messages - # — see c03acca50 for the original fix. - f"{active_clause} ORDER BY id", - tuple(session_ids), - ).fetchall() - - if include_compacted: - rows = self._dedupe_display_generations(rows) - - return self._rows_to_conversation( - rows, - session_id=session_id, - include_ancestors=include_ancestors, - repair_alternation=repair_alternation, - include_row_ids=include_row_ids, - ) # Columns every conversation projection decodes. Shared by # get_messages_as_conversation and get_resume_conversations so a single @@ -14004,367 +5595,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) "api_content, display_kind, display_metadata" ) - def _rows_to_conversation( - self, - rows, - *, - session_id: str, - include_ancestors: bool, - repair_alternation: bool, - include_row_ids: bool = False, - include_summary_markers: bool = False, - ) -> List[Dict[str, Any]]: - """Decode fetched message rows into the OpenAI conversation format. - - Extracted from get_messages_as_conversation so get_resume_conversations - can build the model-fed and display views from one SELECT. ``rows`` must - already be ordered by ``id`` (insertion order) and filtered to the - desired session set / active state by the caller. - """ - messages = [] - # Watermark rotation column-clones concurrent tail rows into the child - # after the new summary, so the copies need not be adjacent. Index the - # exact durable clone identity while decoding instead of rescanning the - # whole accumulated lineage for every user row. - exact_user_clones: Dict[Tuple[Any, str], Dict[str, Any]] = {} - for row in rows: - content = self._decode_content(row["content"]) - if row["role"] in {"user", "assistant"} and isinstance(content, str): - content = sanitize_context(content).strip() - msg = {"role": row["role"], "content": content} - # Born durable (#92231): this dict is materialized FROM a durable - # row, so stamp the persistence marker at the source instead of - # relying on every restore caller to thread the loaded list back - # through a flush as ``conversation_history=`` — any - # identity-losing handoff (compression's durable-snapshot - # adoption, incremental persists with no history arg) would - # otherwise re-append the ENTIRE transcript on flush. - # Underscore-prefixed like ``_row_id``: every transport strips it - # before the wire, and compression's assembly copies deliberately - # strip it so rotated child handoffs still flush (see - # _fresh_compaction_message_copy). - msg[_DB_PERSISTED_MARKER_KEY] = True - # Durable per-message identity for surfaces that need to address a - # specific row later (desktop reactions). OPT-IN: only the gateway - # asks for it — every other consumer (ACP restore, export, - # inspection) gets the transcript in its historical shape. - # Underscore-prefixed so every transport's convert_messages() - # strips it before the wire. - if include_row_ids and row["id"] is not None: - msg["_row_id"] = row["id"] - # api_content is the byte-fidelity sidecar: the exact string sent - # to the API when it differed from the clean content. Returned - # VERBATIM — no sanitize_context, no strip — because the replay - # path substitutes it for content to keep the provider prompt - # cache prefix byte-stable across turns. Cleaning it here would - # re-introduce the divergence it exists to remove. - if row["api_content"]: - msg["api_content"] = row["api_content"] - if row["display_kind"]: - msg["display_kind"] = row["display_kind"] - if row["display_metadata"]: - decoded = self._decode_display_metadata(row["display_metadata"]) - if decoded is not None: - msg["display_metadata"] = decoded - if include_summary_markers and row["_compressed_summary"]: - msg["_compressed_summary"] = True - if row["timestamp"]: - msg["timestamp"] = row["timestamp"] - if row["tool_call_id"]: - msg["tool_call_id"] = row["tool_call_id"] - if row["tool_name"]: - msg["tool_name"] = row["tool_name"] - if row["effect_disposition"]: - msg["effect_disposition"] = row["effect_disposition"] - if row["tool_calls"]: - try: - msg["tool_calls"] = json.loads(row["tool_calls"]) - except (json.JSONDecodeError, TypeError): - logger.warning("Failed to deserialize tool_calls in conversation replay, falling back to []") - msg["tool_calls"] = [] - # Surface the platform-side message id (e.g. yuanbao msg_id, - # telegram update_id) so platform-specific flows like recall - # can match by external identifier instead of having to fall - # back to content-match heuristics. Exposed as ``message_id`` - # for backward compatibility with the JSONL transcript shape. - if row["platform_message_id"]: - msg["message_id"] = row["platform_message_id"] - if row["observed"]: - msg["observed"] = True - # Restore reasoning fields on assistant messages so providers - # that replay reasoning (OpenRouter, OpenAI, Nous) receive - # coherent multi-turn reasoning context. - if row["role"] == "assistant": - if row["finish_reason"]: - msg["finish_reason"] = row["finish_reason"] - if row["reasoning"]: - msg["reasoning"] = row["reasoning"] - if row["reasoning_content"] is not None: - msg["reasoning_content"] = row["reasoning_content"] - if row["reasoning_details"]: - try: - msg["reasoning_details"] = json.loads(row["reasoning_details"]) - except (json.JSONDecodeError, TypeError): - logger.warning("Failed to deserialize reasoning_details, falling back to None") - msg["reasoning_details"] = None - if row["codex_reasoning_items"]: - try: - msg["codex_reasoning_items"] = json.loads(row["codex_reasoning_items"]) - except (json.JSONDecodeError, TypeError): - logger.warning("Failed to deserialize codex_reasoning_items, falling back to None") - msg["codex_reasoning_items"] = None - if row["codex_message_items"]: - try: - msg["codex_message_items"] = json.loads(row["codex_message_items"]) - except (json.JSONDecodeError, TypeError): - logger.warning("Failed to deserialize codex_message_items, falling back to None") - msg["codex_message_items"] = None - if include_ancestors: - canonical_content, _is_composite = ( - self._canonical_replayed_user_content(msg) - ) - exact_clone_key = self._exact_replayed_user_clone_key( - msg.get("timestamp"), canonical_content - ) - previous_exact = ( - exact_user_clones.get(exact_clone_key) - if exact_clone_key is not None - else None - ) - duplicate = None - if previous_exact is not None: - previous_index = next( - ( - index - for index, candidate in enumerate(messages) - if candidate is previous_exact - ), - None, - ) - if previous_index is not None: - duplicate = (previous_index, True) - if duplicate is None: - duplicate = self._find_duplicate_replayed_user_message( - messages, msg - ) - if duplicate is not None: - duplicate_index, prefer_current = duplicate - if prefer_current: - # A rotated compression child can carry the same live - # ask as the parent row plus the only surviving summary - # scaffold. Keep the child carrier (and its durable row - # id), not the simpler ancestor copy. - messages.pop(duplicate_index) - else: - continue - messages.append(msg) - if include_ancestors and exact_clone_key is not None: - exact_user_clones[exact_clone_key] = msg - # DEFENSE-IN-DEPTH against background-review session pollution: a forked - # skill/memory review that (in older builds, before the _persist_disabled - # fix) shared the parent's session_id wrote its harness turn into this - # real session. The harness is a user/system message instructing the - # agent to "Review the conversation above and update the skill library / - # save to memory" under a hard tool restriction; re-loading it as live - # history makes the agent adopt the curator role and refuse the user's - # actual task. Strip any such harness message AND the curator-mode - # assistant reply immediately following it, so a polluted session - # resumes clean even if stray rows exist. - messages = _strip_background_review_harness(messages) - # DEFENSE-IN-DEPTH against #78148: before that fix, a bare tool-call - # marker (e.g. "[memory]") could get cached as a fallback and - # persisted as if it were the model's real answer. Sessions written - # before the fix can still carry those rows — clear the stray - # content on load so replaying history doesn't re-teach the model - # to keep emitting the marker. No-op for unaffected sessions. - messages = _strip_stale_tool_call_markers(messages) - if repair_alternation and messages: - # Lazy import: hermes_state already depends on agent.* (see - # sanitize_context above), but keep this optional path from - # widening the import surface at module load. - from agent.agent_runtime_helpers import repair_message_sequence - - repaired = repair_message_sequence(None, messages) - if repaired: - logger.info( - "Repaired %d message-alternation violation(s) while " - "restoring session %s — durable transcript kept them, " - "see repair_message_sequence", - repaired, - session_id, - ) - return messages - - def get_resume_conversations( - self, session_id: str - ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: - """Return ``(model_history, display_history)`` for a session resume in ONE SELECT. - - ``session.resume`` needs two projections of the same lineage: - - - ``model_history`` — the tip session's active rows, alternation-repaired - (the live-replay working conversation). Equivalent to - ``get_messages_as_conversation(session_id, repair_alternation=True)``. - - ``display_history`` — the full compression lineage (ancestors → tip), - verbatim, with replayed-user dedup. Explicit ``/branch`` sessions are - excluded from this lineage because their own rows already contain the - copied transcript; including the live parent's rows would let messages - written to the original after the fork leak into the branch. - - The display projection also includes rows preserved by IN-PLACE - compaction (``active=0, compacted=1``), deduped by - :meth:`_dedupe_display_generations`. Without them a compacted - conversation resumes showing only its summary plus the carried-forward - tail — the user's own turns read as deleted even though every row is - still on disk, and the REST transcript read (which has always included - them) disagreed with this one about the same session (#92080). - - The display fetch already reads a superset of the model fetch (the tip - rows are part of the lineage), so serving both from one lineage SELECT - halves the resume's DB work versus two separate calls, with byte-identical - output (see test_get_resume_conversations_matches_separate_reads). - """ - session_ids = self._resume_lineage_ids(session_id) - with self._read_ctx() as conn: - placeholders = ",".join("?" for _ in session_ids) - rows = conn.execute( - f"SELECT session_id, {self._CONVERSATION_ROW_COLUMNS} " - f"FROM messages WHERE session_id IN ({placeholders}) " - # Compaction-archived rows (active=0, compacted=1) are display - # history; Undo/Rewind rows (active=0, compacted=0) are not. - "AND (active = 1 OR compacted = 1) " - # ORDER BY id (insertion order) — see get_messages_as_conversation - # for why timestamp ordering is unsafe. - "ORDER BY id", - tuple(session_ids), - ).fetchall() - - # Tip rows are exactly the model-fed set (get_messages_as_conversation - # with session_ids=[session_id]); filtering the lineage fetch preserves - # their relative id order. The model projection stays active-only — it - # is the compressed working context and must not regrow the history - # compaction just summarized away. - tip_rows = [r for r in rows if r["session_id"] == session_id and r["active"]] - model_history = self._rows_to_conversation( - tip_rows, - session_id=session_id, - include_ancestors=False, - repair_alternation=True, - include_row_ids=True, - # Pre-compress checkpointing: the resumed model history must keep - # the summary marker so checkpoint providers can exclude derivative - # summaries after a process restart (marker survives restart). - include_summary_markers=True, - ) - display_history = self._rows_to_conversation( - self._dedupe_display_generations(rows), - session_id=session_id, - include_ancestors=True, - repair_alternation=False, - include_row_ids=True, - ) - return model_history, display_history - - def _resume_lineage_ids(self, session_id: str) -> List[str]: - """Session ids a full (display) resume materializes for *session_id*. - - Compression continuations need their ended ancestors' rows for the - display transcript; an explicit ``/branch`` copy already owns its - transcript, so its lineage is itself alone. This is the ONE definition - shared by the resume readers (``get_resume_conversations``, - ``get_ancestor_display_prefix``) and the resume guard - (``assert_resume_safe`` / ``get_resume_message_count``) — the guard must - count exactly the rows a resume would load, never a superset. - """ - if self._is_explicit_branch_session(session_id): - return [session_id] - return self._session_lineage_root_to_tip(session_id) - - def get_resume_message_count( - self, session_id: str, *, tip_only: bool = False - ) -> int: - """Count the rows that a resume would materialize. - - ``tip_only=True`` counts the tip segment's ACTIVE rows — the set a - model-history restore loads (``get_messages_as_conversation`` without - ancestors, or the deferred Desktop resume that pages the display - transcript over REST and never materializes the ancestor prefix in - memory). - - Otherwise this counts the full-lineage DISPLAY set — active rows plus - the compaction-archived rows ``get_resume_conversations`` now loads - for the transcript. Counting only active rows here would let a - heavily-compacted conversation pass a limit sized for a handful of - live rows and then materialize tens of thousands. - """ - session_ids = [session_id] if tip_only else self._resume_lineage_ids(session_id) - active_clause = "active = 1" if tip_only else "(active = 1 OR compacted = 1)" - placeholders = ",".join("?" for _ in session_ids) - with self._read_ctx() as conn: - row = conn.execute( - f"SELECT COUNT(*) FROM messages " - f"WHERE session_id IN ({placeholders}) AND {active_clause}", - tuple(session_ids), - ).fetchone() - return int(row[0] if row else 0) - - def assert_resume_safe( - self, - session_id: str, - max_messages: Optional[int] = None, - *, - tip_only: bool = False, - ) -> int: - """Return resume row count or reject a transcript too large to load. - - ``max_messages=None`` resolves the limit from config - (``sessions.max_resume_messages``); 0 disables the guard and returns - the (bounded) count without raising. - - ``tip_only=True`` bounds only the tip segment's ACTIVE rows, for - callers that never materialize the ancestor lineage or the - compaction archive in memory (tip-only model restore, deferred - Desktop resume whose display history is REST-paginated). A - heavily-compressed conversation — 85 compaction segments and ~29k - lineage rows behind a ~700-row tip — is exactly the shape compression - is supposed to produce; counting its whole lineage against a limit - sized for in-memory materialization rejected the healthiest sessions - (Desktop Bot Chat stuck on "Waking up…" with code 4130) while the - process would only ever have held the tip. - - The full (non-``tip_only``) bound counts the DISPLAY set — active plus - compaction-archived rows — because that is what - ``get_resume_conversations`` materializes for the transcript. - """ - if max_messages is None: - max_messages = resolved_max_resume_messages() - if max_messages < 0: - raise ValueError("max_messages must be non-negative") - if max_messages == 0: - # Guard disabled by config — skip counting entirely. Every live - # caller invokes this for its raise side effect and ignores the - # return value, and an unbounded lineage COUNT here would do the - # exact pathological work the disable exists to avoid. - return 0 - session_ids = [session_id] if tip_only else self._resume_lineage_ids(session_id) - active_clause = "active = 1" if tip_only else "(active = 1 OR compacted = 1)" - placeholders = ",".join("?" for _ in session_ids) - with self._read_ctx() as conn: - row = conn.execute( - "SELECT COUNT(*) FROM (" - f"SELECT 1 FROM messages WHERE session_id IN ({placeholders}) " - f"AND {active_clause} LIMIT ?" - ")", - (*session_ids, max_messages + 1), - ).fetchone() - message_count = int(row[0] if row else 0) - if message_count > max_messages: - raise SessionResumeTooLargeError( - message_count, - max_messages, - scope="in its tip segment" if tip_only else "across its lineage", - ) - return message_count def assert_export_safe( self, @@ -14390,75 +5620,17 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # this for its raise side effect only (and skip calling it # entirely when the limit is 0). return 0 - with self._read_ctx() as conn: - row = conn.execute( - "SELECT COUNT(*) FROM (" - "SELECT 1 FROM messages WHERE session_id = ? AND active = 1 LIMIT ?" - ")", - (session_id, max_messages + 1), - ).fetchone() + row = self._read_one( + "SELECT COUNT(*) FROM (" + "SELECT 1 FROM messages WHERE session_id = ? AND active = 1 LIMIT ?" + ")", + (session_id, max_messages + 1), + ) message_count = int(row[0] if row else 0) if message_count > max_messages: raise SessionExportTooLargeError(session_id, message_count, max_messages) return message_count - def get_ancestor_display_prefix(self, session_id: str) -> List[Dict[str, Any]]: - """Return the ancestor-only display messages for a session lineage. - - These are messages from parent/grandparent sessions (compression - ancestors) that appear in the display transcript but NOT in the - tip session's model-fed history. Used by ``session.resume`` to - build the ``display_history_prefix`` that ``_live_session_payload`` - prepends to the live model history. - - Previously the prefix was calculated as - ``display_history[:len(display) - len(raw)]``, but that overcounts - when ``repair_message_sequence`` removes messages from the MIDDLE - of the tip history (e.g. verification candidates collapsed by the - consecutive-assistant merge) — the length difference includes both - ancestor messages AND repair-removed tip messages, but the slice - only captures the first N display messages (which are tip messages - when there are no ancestors), causing duplication. This method - returns ONLY the genuine ancestor messages, identified by - ``session_id != tip_session_id``. (#65919) - """ - session_ids = self._resume_lineage_ids(session_id) - if len(session_ids) <= 1: - return [] - with self._read_ctx() as conn: - placeholders = ",".join("?" for _ in session_ids) - rows = conn.execute( - f"SELECT session_id, {self._CONVERSATION_ROW_COLUMNS} " - f"FROM messages WHERE session_id IN ({placeholders}) " - # Display read: compaction-archived rows included, Undo/Rewind - # rows excluded (see get_resume_conversations). - "AND (active = 1 OR compacted = 1) " - "ORDER BY id", - tuple(session_ids), - ).fetchall() - rows = self._dedupe_display_generations(rows) - ancestor_ids = { - int(row["id"]) - for row in rows - if row["session_id"] != session_id and row["id"] is not None - } - if not ancestor_ids: - return [] - lineage = self._rows_to_conversation( - rows, - session_id=session_id, - include_ancestors=True, - repair_alternation=False, - include_row_ids=True, - ) - prefix: List[Dict[str, Any]] = [] - for message in lineage: - if message.get("_row_id") not in ancestor_ids: - continue - projected = message.copy() - projected.pop("_row_id", None) - prefix.append(projected) - return prefix def _is_explicit_branch_session(self, session_id: str) -> bool: """Return whether *session_id* is a copied user-facing branch. @@ -14471,11 +5643,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) """ if not session_id: return False - with self._read_ctx() as conn: - row = conn.execute( - "SELECT model_config FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() + row = self._read_one( + "SELECT model_config FROM sessions WHERE id = ?", + (session_id,), + ) if row is None: return False raw_config = row["model_config"] if hasattr(row, "keys") else row[0] @@ -14487,19 +5658,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return False return isinstance(config, dict) and bool(config.get("_branched_from")) - def get_conversation_root(self, session_id: str) -> str: - """Return the ROOT id of *session_id*'s lineage chain. - - The root is the stable "conversation id": context compression - rotates ``session_id`` to a new segment linked via - ``parent_session_id``, and delegate subagents hang off their - parent the same way. Walking to the root gives every segment of - one user-facing conversation (and its delegation tree) a single - identifier — used for Nous Portal ``conversation=`` usage tagging. - Returns *session_id* unchanged when it has no recorded parent. - """ - chain = self._session_lineage_root_to_tip(session_id) - return (chain[0] if chain and chain[0] else session_id) def _session_lineage_root_to_tip(self, session_id: str) -> List[str]: if not session_id: @@ -14523,324 +5681,11 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) current = row["parent_session_id"] if hasattr(row, "keys") else row[0] return list(reversed(chain)) or [session_id] - @staticmethod - def _canonical_replayed_user_content( - msg: Dict[str, Any], - ) -> Tuple[Any, bool]: - """Return canonical live content and whether *msg* is composite.""" - if msg.get("role") != "user": - return None, False - - from agent.context_compressor import split_user_originated_turn - - handoff, live_view = split_user_originated_turn(msg) - is_composite = handoff is not None and live_view is not None - return ( - live_view.get("content") - if is_composite and live_view is not None - else msg.get("content"), - is_composite, - ) - - @staticmethod - def _exact_replayed_user_clone_key( - timestamp: Any, content: Any - ) -> Optional[Tuple[Any, str]]: - """Return a hashable key for a column-exact rotation clone.""" - if timestamp is None or content in (None, "", []): - return None - try: - encoded = json.dumps( - content, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - ) - except (TypeError, ValueError): - return None - return timestamp, encoded - - @staticmethod - def _find_duplicate_replayed_user_message( - messages: List[Dict[str, Any]], msg: Dict[str, Any] - ) -> Optional[Tuple[int, bool]]: - """Return an adjacent replay duplicate and whether *msg* must win. - - Compression rotation may persist the current ask once in the parent - and again inside a composite child carrier. Compare the canonical live - payload for that carrier, while retaining the historical exact-string - dedupe for ordinary replayed users. The child carrier wins because it - owns both the current durable row identity and the retained scaffold. - """ - if msg.get("role") != "user": - return None - - content, prefer_current = SessionDB._canonical_replayed_user_content(msg) - if content in (None, "", []): - return None - - for index in range(len(messages) - 1, -1, -1): - prev = messages[index] - if prev.get("role") == "user": - prev_content, prev_is_composite = ( - SessionDB._canonical_replayed_user_content(prev) - ) - if prev_content == content and ( - prefer_current - or prev_is_composite - or isinstance(content, str) - ): - return index, prefer_current - if prev.get("role") == "assistant" and (prev.get("content") or prev.get("tool_calls")): - return None - return None - - @staticmethod - def _is_duplicate_replayed_user_message( - messages: List[Dict[str, Any]], msg: Dict[str, Any] - ) -> bool: - return SessionDB._find_duplicate_replayed_user_message(messages, msg) is not None # ========================================================================= # Rewind (soft-delete) — see /rewind slash command + issue #21910 # ========================================================================= - def get_active_message_ids(self, session_id: str) -> List[int]: - """Return the ordered physical ids pinned by rewind CAS checks. - - Conversation projections intentionally omit legacy background-review - harness rows. Destructive rewinds must nevertheless pin every active - physical row so the caller snapshot matches the transaction-local - comparison in :meth:`rewind_to_message`. - """ - with self._read_ctx() as conn: - rows = conn.execute( - "SELECT id FROM messages " - "WHERE session_id = ? AND active = 1 ORDER BY id", - (session_id,), - ).fetchall() - return [int(row[0]) for row in rows] - - @staticmethod - def _active_transcript_counts(conn, session_id: str) -> tuple[int, int]: - """Return active message/tool-call counts inside the caller's txn.""" - rows = conn.execute( - "SELECT tool_calls FROM messages " - "WHERE session_id = ? AND active = 1", - (session_id,), - ).fetchall() - tool_call_count = 0 - for row in rows: - raw = row[0] - if not raw: - continue - try: - decoded = json.loads(raw) if isinstance(raw, str) else raw - except (json.JSONDecodeError, TypeError): - continue - if isinstance(decoded, list): - tool_call_count += len(decoded) - elif decoded: - tool_call_count += 1 - return len(rows), tool_call_count - - def rewind_to_message( - self, - session_id: str, - target_message_id: int, - *, - preserve_compaction_handoff: bool = False, - expected_active_ids: Optional[List[int]] = None, - expected_target_content: Any = None, - ) -> Dict[str, Any]: - """Soft-delete all messages with id >= ``target_message_id`` in *session_id*. - - The target message itself becomes inactive as well so the caller - can pre-fill it as the next user prompt without it appearing - twice in the replayed transcript. Rewound rows are kept on - disk with ``active=0`` for audit / forensic inspection — use - :meth:`get_messages` with ``include_inactive=True`` to see them. - - Returns a dict:: - - { - "rewound_count": int, # number of rows newly flipped to active=0 - "target_message": dict, # full row dict of the target - "new_head_id": int|None # id of the last still-active row, or None - } - - Raises ``ValueError`` if the target message does not exist in - *session_id* or if its role is not ``"user"``. With - ``preserve_compaction_handoff=True``, a composite summary carrier is - split inside the same write transaction: its original row is archived - and its canonical hidden handoff scaffold is inserted as the new head. - That opt-in result also contains ``replacement_message_id``. - - ``expected_active_ids`` optionally pins the ordered active row set. - ``expected_target_content`` additionally pins the selected canonical - live-user payload. Both checks run inside the write transaction before - any row or counter mutation. Presentation-only metadata changes (for - example Desktop reactions) deliberately do not invalidate a rewind. - A live cross-process turn lease always refuses the rewind; expired or - provably dead holders are reclaimed inside the mutation transaction. - - Always increments ``sessions.rewind_count`` — even when the - target is already inactive — so the counter accurately reflects - the number of rewind operations performed against the session. - Idempotent on the ``active`` flag: re-rewinding past the same - target is a no-op on row state but still bumps the counter. - """ - - def _do(conn): - # Rewind changes the active transcript and must honor the same - # compression/closed-parent and cross-process turn guards as - # append writers. - self._check_transcript_write_guards( - conn, - session_id, - None, - reject_active_turn_lease=True, - reject_active_compression_lock=True, - ) - - if expected_active_ids is not None: - active_rows = conn.execute( - "SELECT id FROM messages " - "WHERE session_id = ? AND active = 1 ORDER BY id", - (session_id,), - ).fetchall() - active_ids = [int(active_row[0]) for active_row in active_rows] - if active_ids != expected_active_ids: - raise RuntimeError( - "active transcript changed before the rewind could be persisted" - ) - - row = conn.execute( - "SELECT * FROM messages WHERE id = ? AND session_id = ?", - (target_message_id, session_id), - ).fetchone() - if row is None: - raise ValueError( - f"message {target_message_id} not found in session {session_id}" - ) - target_row = dict(row) - if target_row.get("role") != "user": - raise ValueError( - f"rewind target must be a 'user' message (got role=" - f"{target_row.get('role')!r}, id={target_message_id})" - ) - - replacement_message_id: Optional[int] = None - replacement: Optional[Dict[str, Any]] = None - if preserve_compaction_handoff or expected_target_content is not None: - if not target_row.get("active"): - raise ValueError("rewind target is not active") - from agent.context_compressor import split_user_originated_turn - - split_target = target_row.copy() - split_target["content"] = self._decode_content( - split_target.get("content") - ) - split_target["display_metadata"] = self._decode_display_metadata( - split_target.get("display_metadata") - ) - handoff, live_view = split_user_originated_turn(split_target) - if live_view is None: - raise ValueError("rewind target is not a user-originated turn") - live_content = live_view.get("content") - if isinstance(live_content, str): - live_content = sanitize_context(live_content).strip() - if ( - expected_target_content is not None - and live_content != expected_target_content - ): - raise RuntimeError( - "rewind target changed before it could be persisted" - ) - if preserve_compaction_handoff and handoff is None: - raise ValueError( - "preserve_compaction_handoff requires an active composite carrier" - ) - replacement = handoff if preserve_compaction_handoff else None - - cursor = conn.execute( - "SELECT id FROM messages " - "WHERE session_id = ? AND id >= ? AND active = 1", - (session_id, target_message_id), - ) - ids = [r[0] for r in cursor.fetchall()] - if ids: - placeholders = ",".join("?" for _ in ids) - conn.execute( - f"UPDATE messages SET active = 0 WHERE id IN ({placeholders})", - ids, - ) - if replacement is not None: - self._insert_message_rows(conn, session_id, [replacement]) - inserted = conn.execute("SELECT last_insert_rowid()").fetchone() - replacement_message_id = int(inserted[0]) - conn.execute( - "UPDATE sessions SET rewind_count = COALESCE(rewind_count, 0) + 1 " - "WHERE id = ?", - (session_id,), - ) - message_count, tool_call_count = self._active_transcript_counts( - conn, session_id - ) - conn.execute( - "UPDATE sessions SET message_count = ?, tool_call_count = ? " - "WHERE id = ?", - (message_count, tool_call_count, session_id), - ) - head_row = conn.execute( - "SELECT MAX(id) FROM messages WHERE session_id = ? AND active = 1", - (session_id,), - ).fetchone() - new_head_id = ( - head_row[0] if head_row and head_row[0] is not None else None - ) - return target_row, ids, new_head_id, replacement_message_id - - target_row, rewound, new_head_id, replacement_message_id = ( - self._execute_write(_do) - ) - - # Decode content for callers (prefill the prompt buffer) without a - # second fallible database operation after the transaction commits. - target_row["content"] = self._decode_content(target_row.get("content")) - - result = { - "rewound_count": len(rewound), - "target_message": target_row, - "new_head_id": new_head_id, - } - if preserve_compaction_handoff: - result["replacement_message_id"] = replacement_message_id - return result - - def restore_rewound(self, session_id: str, since_message_id: int) -> int: - """Mark inactive messages with id >= *since_message_id* active again. - - Returns the number of rows flipped back to ``active=1``. - Intended for undo-of-rewind and test cleanup; not wired to a - slash command in v1. - """ - def _do(conn): - cursor = conn.execute( - "SELECT id FROM messages " - "WHERE session_id = ? AND id >= ? AND active = 0", - (session_id, since_message_id), - ) - ids = [r[0] for r in cursor.fetchall()] - if ids: - placeholders = ",".join("?" for _ in ids) - conn.execute( - f"UPDATE messages SET active = 1 WHERE id IN ({placeholders})", - ids, - ) - return len(ids) - - return self._execute_write(_do) # ========================================================================= # Search @@ -14882,14 +5727,12 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) params.extend(ws_params) where_sql = f" WHERE {' AND '.join(where_clauses)}" if where_clauses else "" params.extend([limit, offset]) - with self._read_ctx() as conn: - cursor = conn.execute( - f"{select_with_last_active}" - f"{where_sql} " - "ORDER BY last_active DESC, s.started_at DESC, s.id DESC LIMIT ? OFFSET ?", - params, - ) - return [self._session_row_dict(row) for row in cursor.fetchall()] + return [self._session_row_dict(row) for row in self._read_all( + f"{select_with_last_active}" + f"{where_sql} " + "ORDER BY last_active DESC, s.started_at DESC, s.id DESC LIMIT ? OFFSET ?", + params, + )] # ========================================================================= # Utility @@ -14952,9 +5795,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) where_sql = f" WHERE {' AND '.join(where_clauses)}" if where_clauses else "" - with self._read_ctx() as conn: - cursor = conn.execute(f"SELECT COUNT(*) FROM sessions s{where_sql}", params) - return cursor.fetchone()[0] + return self._read_one(f"SELECT COUNT(*) FROM sessions s{where_sql}", params)[0] def session_count_ge(self, n: int = 1) -> bool: """Check if at least N sessions exist (archived included). @@ -14967,9 +5808,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) Use this instead of ``session_count() >= n`` when the exact count is irrelevant. """ - with self._read_ctx() as conn: - cursor = conn.execute("SELECT 1 FROM sessions LIMIT ?", (n,)) - rows = cursor.fetchall() + rows = self._read_all("SELECT 1 FROM sessions LIMIT ?", (n,)) return len(rows) >= n def session_count_by_source( @@ -15017,79 +5856,11 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ).fetchall() return {str(row["source"]): int(row["count"] or 0) for row in rows} - def message_count(self, session_id: str = None) -> int: - """Count messages, optionally for a specific session.""" - with self._read_ctx() as conn: - if session_id: - cursor = conn.execute( - "SELECT COUNT(*) FROM messages WHERE session_id = ?", (session_id,) - ) - else: - cursor = conn.execute("SELECT COUNT(*) FROM messages") - return cursor.fetchone()[0] - - def has_platform_message_id( - self, session_id: str, platform_message_id: str - ) -> bool: - """Check if a message with the given platform_message_id exists. - - Uses the idx_messages_platform_msg_id partial index for efficient - lookup. Used by the gateway's transient-failure dedupe guard (#47237) - to skip re-persisting a user message that was already saved on a - prior retry of the same inbound platform message. - """ - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT 1 FROM messages " - "WHERE session_id = ? AND platform_message_id = ? LIMIT 1", - (session_id, platform_message_id), - ) - return cursor.fetchone() is not None # ========================================================================= # Export and cleanup # ========================================================================= - def _is_explicit_fork_child_row(self, session: Dict[str, Any]) -> bool: - """True when ``session`` is a branch, delegate, or tool child of its parent. - - Markers only count as a fork when they point at ``parent_session_id``. - Compression copies ``model_config`` onto the continuation - (``publish_compression_child`` callers pass - ``agent._session_init_model_config``), so a delegate's continuation - carries ``_delegate_from=``. Presence-only - matching would treat that real continuation as a fork — the same - misclassification ``_NON_CONTINUATION_CHILD_FILTER_SQL`` already - avoids by binding both markers to the queried parent. - """ - if session.get("source") == "tool": - return True - raw = session.get("model_config") - if not raw: - return False - try: - cfg = json.loads(raw) if isinstance(raw, str) else raw - except (TypeError, json.JSONDecodeError): - return False - if not isinstance(cfg, dict): - return False - parent_id = session.get("parent_session_id") - branched = cfg.get("_branched_from") - delegated = cfg.get("_delegate_from") - if parent_id: - return branched == parent_id or delegated == parent_id - return branched is not None or delegated is not None - - def is_explicit_fork_child(self, session_id: str) -> bool: - """True when ``session_id`` is a /branch, delegate, or tool child row. - - Read-only public view of :meth:`_is_explicit_fork_child_row` for - callers that must respect the fork boundary without re-implementing - its marker rules (``agent/prompt_cache_scope.py`` keeps a declared - conversation key from crossing it). A missing row is not a fork. - """ - session = self.get_session(session_id) - return bool(session and self._is_explicit_fork_child_row(session)) def declared_scope_identity(self, session_id: str) -> Tuple[bool, str]: """Fork verdict and recorded ``source`` for *session_id*, in ONE read. @@ -15115,124 +5886,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) str(session.get("source") or "").strip(), ) - def latest_conversation_boundary( - self, session_key: str, source: str - ) -> Optional[int]: - """How many conversation boundaries this routing peer has crossed. - - A boundary is a row this peer ended at an intentional conversation - break — the ``_RESET_END_REASONS`` set (``/new``, ``/switch``, idle, - daily, suspended, resume_pending_expired). That is the same fence - :meth:`find_latest_gateway_session_for_peer` refuses to reach behind, - so the two agree on where one conversation stops and the next begins - and cannot drift. - - The peer is ``(session_key, source)``, the SAME identity tuple recovery - uses — never the key alone. ``X-Hermes-Session-Key`` accepts any - authenticated caller-supplied string, so an API conversation may - legally carry the same key as a Telegram row in one database; keying - on the string alone would let a ``/new`` on that unrelated row rotate - this conversation's affinity identity while recovery correctly refuses - to cross the same line. - - Returns the count, or ``None`` when this peer has never been reset. - - The value comes from ``conversation_generations``, which - :meth:`_bump_conversation_generation` advances inside the transaction - that writes each boundary — NOT from an aggregate over the session - rows. An aggregate cannot prove non-reuse: ``delete_session()`` - orphans children and deletes the row, and bulk prune selects ended - rows, so ``COUNT``/``MAX`` over boundaries can return a pair it already - emitted and hand a new conversation a retired affinity identity. It is - also wall-clock-free, so a backwards NTP correction cannot reorder it. - - Databases upgraded mid-conversation start at no generation and take - their first one from the next boundary written; a conversation that - reset before the upgrade shares its predecessor's scope once, which - costs a warm prompt-cache bucket and never crosses an identity. - - These rows are never garbage-collected, by design: dropping one resets - the peer to "no generation", so its next boundary writes ``1`` again - and re-issues a scope a retired conversation already used — the ABA - this counter exists to prevent. See the schema comment in - ``hermes_state_common.py``. - """ - if not session_key or not source: - return None - with self._read_ctx() as conn: - row = conn.execute( - "SELECT generation FROM conversation_generations " - "WHERE source = ? AND session_key = ?", - (source, session_key), - ).fetchone() - if row is None or row["generation"] is None: - return None - generation = int(row["generation"]) - return generation if generation > 0 else None - - def _is_compression_child_row(self, child: Dict[str, Any]) -> bool: - parent_id = child.get("parent_session_id") - if not parent_id or self._is_explicit_fork_child_row(child): - return False - parent = self.get_session(parent_id) - return bool(parent and parent.get("end_reason") == "compression") - - def get_compression_lineage(self, session_id: str) -> List[str]: - """Return compression ancestors through tip in chronological order.""" - session = self.get_session(session_id) - if not session or self._is_explicit_fork_child_row(session): - return [session_id] if session else [] - - root = session - ancestors = {root["id"]} - while self._is_compression_child_row(root): - parent = self.get_session(root["parent_session_id"]) - if not parent or parent["id"] in ancestors: - break - root = parent - ancestors.add(root["id"]) - - lineage = [root["id"]] - seen = {root["id"]} - current = root - while current.get("end_reason") == "compression": - with self._read_ctx() as conn: - rows = conn.execute( - """ - SELECT * FROM sessions - WHERE parent_session_id = ? - ORDER BY started_at ASC - """, - (current["id"],), - ).fetchall() - next_child = None - for row in rows: - candidate = dict(row) - if self._is_compression_child_row(candidate): - next_child = candidate - break - if not next_child or next_child["id"] in seen: - break - lineage.append(next_child["id"]) - seen.add(next_child["id"]) - current = next_child - if current["id"] == session_id: - # Continue to include later compression tips only when the - # requested session itself was compacted. - continue - return lineage if session_id in lineage else [session_id] - - def clear_messages(self, session_id: str) -> None: - """Delete all messages for a session and reset its counters.""" - def _do(conn): - conn.execute( - "DELETE FROM messages WHERE session_id = ?", (session_id,) - ) - conn.execute( - "UPDATE sessions SET message_count = 0, tool_call_count = 0 WHERE id = ?", - (session_id,), - ) - self._execute_write(_do) @staticmethod def _remove_session_files(sessions_dir: Optional[Path], session_id: str) -> None: @@ -15507,11 +6160,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) to clean up, and pre-populate the confirm dialog with the actual count. """ - with self._read_ctx() as conn: - cursor = conn.execute( - f"SELECT COUNT(*) FROM sessions WHERE {self._EMPTY_SESSION_WHERE}" - ) - return cursor.fetchone()[0] + return self._read_one(f"SELECT COUNT(*) FROM sessions WHERE {self._EMPTY_SESSION_WHERE}")[0] def delete_empty_sessions( self, @@ -15579,254 +6228,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._remove_session_files(sessions_dir, sid) return count - @staticmethod - def _prune_filter_where( - *, - last_active_before: Optional[float] = None, - last_active_after: Optional[float] = None, - started_before: Optional[float] = None, - started_after: Optional[float] = None, - source: Optional[str] = None, - title_like: Optional[str] = None, - end_reason: Optional[str] = None, - cwd_prefix: Optional[str] = None, - min_messages: Optional[int] = None, - max_messages: Optional[int] = None, - archived: Optional[bool] = None, - model_like: Optional[str] = None, - provider: Optional[str] = None, - user_id: Optional[str] = None, - chat_id: Optional[str] = None, - chat_type: Optional[str] = None, - branch_like: Optional[str] = None, - min_tokens: Optional[int] = None, - max_tokens: Optional[int] = None, - min_cost: Optional[float] = None, - max_cost: Optional[float] = None, - min_tool_calls: Optional[int] = None, - max_tool_calls: Optional[int] = None, - include_pinned: bool = False, - ) -> Tuple[str, list]: - """Build the shared WHERE clause for bulk prune/archive selection. - - All filters AND together. Only ended sessions are ever candidates - (``ended_at IS NOT NULL``) so a live session is never selected. - ``archived`` is a tri-state: ``None`` = both, ``True`` = only - archived rows, ``False`` = only unarchived rows. - - String matching conventions: ``model_like`` / ``branch_like`` / - ``title_like`` are case-insensitive substring matches (model slugs - and branch names vary in prefix format); ``provider`` / ``user_id`` - / ``chat_id`` / ``chat_type`` / ``source`` / ``end_reason`` are - exact (case-insensitive for provider). Token bounds apply to - ``input_tokens + output_tokens``; cost bounds apply to - ``COALESCE(actual_cost_usd, estimated_cost_usd)``. - - The clause references the ``s`` table alias — callers must select - ``FROM sessions s``. - """ - clauses = ["s.ended_at IS NOT NULL"] - params: list = [] - if last_active_before is not None: - clauses.append( - """COALESCE( - (SELECT MAX(m.timestamp) FROM messages m - WHERE m.session_id = s.id), - s.started_at - ) < ?""" - ) - params.append(last_active_before) - # An automatic orphan sweep closes a stale open row so the user can - # still recover it. Age those rows from the sweep, not from their old - # activity, or the next prune pass can delete them immediately. - clauses.append( - "(COALESCE(s.end_reason, '') != 'startup_orphan_reap' " - "OR s.ended_at < ?)" - ) - params.append(last_active_before) - if last_active_after is not None: - clauses.append( - """COALESCE( - (SELECT MAX(m.timestamp) FROM messages m - WHERE m.session_id = s.id), - s.started_at - ) >= ?""" - ) - params.append(last_active_after) - if started_before is not None: - clauses.append("s.started_at < ?") - params.append(started_before) - if started_after is not None: - clauses.append("s.started_at >= ?") - params.append(started_after) - if source: - clauses.append("s.source = ?") - params.append(source) - if title_like: - clauses.append("LOWER(COALESCE(s.title, '')) LIKE ? ESCAPE '\\'") - params.append(f"%{_escape_like(title_like.lower())}%") - if end_reason: - clauses.append("s.end_reason = ?") - params.append(end_reason) - if cwd_prefix: - clause, clause_params = _cwd_prefix_clause(cwd_prefix) - clauses.append(clause) - params.extend(clause_params) - if min_messages is not None: - clauses.append("s.message_count >= ?") - params.append(min_messages) - if max_messages is not None: - clauses.append("s.message_count <= ?") - params.append(max_messages) - if model_like: - clauses.append("LOWER(COALESCE(s.model, '')) LIKE ? ESCAPE '\\'") - params.append(f"%{_escape_like(model_like.lower())}%") - if provider: - clauses.append("LOWER(COALESCE(s.billing_provider, '')) = ?") - params.append(provider.lower()) - if user_id: - clauses.append("s.user_id = ?") - params.append(user_id) - if chat_id: - clauses.append("s.chat_id = ?") - params.append(chat_id) - if chat_type: - clauses.append("s.chat_type = ?") - params.append(chat_type) - if branch_like: - clauses.append("LOWER(COALESCE(s.git_branch, '')) LIKE ? ESCAPE '\\'") - params.append(f"%{_escape_like(branch_like.lower())}%") - if min_tokens is not None: - clauses.append( - "(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) >= ?" - ) - params.append(min_tokens) - if max_tokens is not None: - clauses.append( - "(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) <= ?" - ) - params.append(max_tokens) - if min_cost is not None: - clauses.append( - "COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) >= ?" - ) - params.append(min_cost) - if max_cost is not None: - clauses.append( - "COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) <= ?" - ) - params.append(max_cost) - if min_tool_calls is not None: - clauses.append("COALESCE(s.tool_call_count, 0) >= ?") - params.append(min_tool_calls) - if max_tool_calls is not None: - clauses.append("COALESCE(s.tool_call_count, 0) <= ?") - params.append(max_tool_calls) - if archived is True: - clauses.append("s.archived = 1") - elif archived is False: - clauses.append("s.archived = 0") - # Pinned sessions are a durable "keep" flag (exempt from the stale - # auto-archive sweep). Bulk prune/delete/archive must honor that too: - # exclude pinned rows unless the caller explicitly opts in. Without - # this, `sessions prune`/`delete`/`archive` with a filter silently - # destroyed pinned conversations (round-3 QA SES-01, data loss). - if not include_pinned: - clauses.append("COALESCE(s.pinned, 0) = 0") - return " AND ".join(clauses), params - - @staticmethod - def _apply_prune_age_filter( - older_than_days: Optional[float], filters: Dict[str, Any] - ) -> None: - """Translate the legacy age window into the shared activity filter.""" - if ( - filters.get("last_active_before") is None - and filters.get("started_before") is None - and older_than_days is not None - ): - filters["last_active_before"] = time.time() - ( - older_than_days * 86400 - ) - - def list_prune_candidates( - self, - older_than_days: Optional[float] = None, - source: str = None, - **filters, - ) -> List[Dict[str, Any]]: - """Return the sessions a matching :meth:`prune_sessions` / - :meth:`archive_sessions` call would touch, without modifying anything. - - Backs ``--dry-run`` and pre-confirmation counts. Accepts the same - keyword filters as :meth:`_prune_filter_where` (unknown names raise - ``TypeError`` there). Rows are ordered oldest-first and carry - ``id, source, title, model, started_at, last_active, ended_at, - message_count, archived``. ``older_than_days`` is an inactivity - threshold: it uses the latest message timestamp, falling back to - ``started_at`` for sessions without messages. - """ - self._apply_prune_age_filter(older_than_days, filters) - where, params = self._prune_filter_where(source=source, **filters) - with self._read_ctx() as conn: - cursor = conn.execute( - f"""SELECT s.id, s.source, s.title, s.model, s.started_at, - COALESCE( - (SELECT MAX(m.timestamp) FROM messages m - WHERE m.session_id = s.id), - s.started_at - ) AS last_active, - s.ended_at, s.message_count, s.archived - FROM sessions s WHERE {where} - ORDER BY last_active ASC, s.started_at ASC""", - params, - ) - return [dict(row) for row in cursor.fetchall()] - - def count_prune_matches( - self, - older_than_days: Optional[float] = None, - source: str = None, - **filters, - ) -> int: - """Count sessions a matching prune/archive would touch. - - Same filter surface as :meth:`list_prune_candidates` (including the - ``include_pinned`` tri-state), but returns only a count. Used by the - CLI to report how many pinned sessions are being spared. - """ - self._apply_prune_age_filter(older_than_days, filters) - where, params = self._prune_filter_where(source=source, **filters) - with self._read_ctx() as conn: - cursor = conn.execute( - f"SELECT COUNT(*) FROM sessions s WHERE {where}", params - ) - return int(cursor.fetchone()[0]) - - def count_open_prune_matches( - self, - older_than_days: Optional[float] = None, - source: str = None, - **filters, - ) -> int: - """Count open sessions excluded from a matching bulk prune. - - This applies every normal prune filter, but inverts only the - ``ended_at`` safety guard. It is visibility-only: callers can explain - why an otherwise matching session was skipped without making live - sessions eligible for destructive pruning. - """ - self._apply_prune_age_filter(older_than_days, filters) - where, params = self._prune_filter_where(source=source, **filters) - ended_guard = "s.ended_at IS NOT NULL" - if not where.startswith(ended_guard): - raise RuntimeError("prune filter lost its ended-session safety guard") - open_where = f"s.ended_at IS NULL{where[len(ended_guard):]}" - with self._read_ctx() as conn: - cursor = conn.execute( - f"SELECT COUNT(*) FROM sessions s WHERE {open_where}", params - ) - return int(cursor.fetchone()[0]) def archive_sessions( self, @@ -15854,256 +6255,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self.set_session_archived(row["id"], True) return len(rows) - def archive_stale_sessions( - self, idle_days: float, *, exclude_pinned: bool = True - ) -> int: - """Archive every session untouched for at least ``idle_days`` days. - - "Touched" is the freshest of ``last_activity_at`` and the latest - message timestamp (else ``started_at``) — i.e. real recency, not - creation time — so a session - created long ago but active yesterday is spared, while an old - abandoned one (even a still-open one) is swept. Unlike - :meth:`archive_sessions`, this method can also archive unended - sessions. - - Guards: - * ``pinned = 0`` when ``exclude_pinned`` (the Desktop "keep" flag). - * ``archived = 0`` so repeat runs are idempotent no-ops. - * only lineage *tips* / standalone rows are candidates - (``end_reason <> 'compression'``); a stale tip archives its whole - chain via :meth:`set_session_archived`, so we never resurrect an - active conversation by matching an old compressed-away root whose - live continuation is recent. - - Returns the number of sessions archived. Never raises for an empty or - non-positive ``idle_days`` — it simply archives nothing. - """ - if idle_days is None or idle_days < 0: - return 0 - cutoff = time.time() - float(idle_days) * 86400.0 - pin_clause = "AND s.pinned = 0" if exclude_pinned else "" - with self._read_ctx() as conn: - rows = conn.execute( - f""" - SELECT s.id FROM sessions s - WHERE s.archived = 0 - AND COALESCE(s.end_reason, '') <> 'compression' - {pin_clause} - AND {_sql_session_last_active("s")} < ? - ORDER BY s.started_at ASC - """, - (cutoff,), - ).fetchall() - ids = [(r["id"] if isinstance(r, sqlite3.Row) else r[0]) for r in rows] - for sid in ids: - self.set_session_archived(sid, True) - return len(ids) - - def prune_sessions( - self, - older_than_days: Optional[float] = 90, - source: str = None, - sessions_dir: Optional[Path] = None, - exclude_active_write_guards: bool = False, - **filters, - ) -> int: - """Delete sessions matching the filters. Returns count deleted. - - By default, delete ended sessions inactive for - ``older_than_days`` days, optionally restricted to ``source``. - Activity is the latest message timestamp, falling back to - ``started_at`` for sessions without messages. Additional keyword - filters AND together — the full set is defined by - :meth:`_prune_filter_where`: - - * ``last_active_before`` / ``last_active_after`` — epoch bounds on - the latest message timestamp (falling back to ``started_at``). - * ``started_before`` / ``started_after`` — epoch bounds on - ``started_at``. An explicit ``started_before`` overrides the - default ``older_than_days`` inactivity cutoff; pass - ``older_than_days=None`` for no implicit upper age bound. - * ``title_like`` / ``model_like`` / ``branch_like`` — - case-insensitive substring matches. - * ``end_reason`` / ``provider`` / ``user_id`` / ``chat_id`` / - ``chat_type`` — exact matches (provider case-insensitive, against - ``billing_provider``). - * ``cwd_prefix`` — session cwd equals or is under this path. - * ``min_messages`` / ``max_messages`` — bounds on message_count. - * ``min_tokens`` / ``max_tokens`` — bounds on input+output tokens. - * ``min_cost`` / ``max_cost`` — bounds on USD cost - (actual, falling back to estimated). - * ``min_tool_calls`` / ``max_tool_calls`` — bounds on tool_call_count. - * ``archived`` — tri-state: None = both (default), True = only - archived, False = only unarchived. - - Only prunes ended sessions (not active ones). Child sessions outside - the prune window are orphaned (parent_session_id set to NULL) rather - than cascade-deleted. When *sessions_dir* is provided, also removes - on-disk transcript files (``.json`` / ``.jsonl`` / - ``request_dump_*``) for every pruned session, outside the DB - transaction. - - ``exclude_active_write_guards`` is for destructive automatic - maintenance: rows protected by a live turn lease or compression lock - are skipped, while expired or provably dead holders are reclaimed and - fenced in the same write transaction. - """ - self._apply_prune_age_filter(older_than_days, filters) - where, where_params = self._prune_filter_where(source=source, **filters) - removed_ids: list[str] = [] - - def _do(conn): - cursor = conn.execute( - f"SELECT s.id FROM sessions s WHERE {where}", where_params - ) - session_ids = {row["id"] for row in cursor.fetchall()} - - if exclude_active_write_guards: - protected = set() - for sid in session_ids: - try: - self._check_transcript_write_guards( - conn, - sid, - compression_lock_holder=None, - turn_lease_holder=None, - reject_active_turn_lease=True, - reject_active_compression_lock=True, - allow_closed_compression_parent=True, - ) - except ( - SessionCompressionInProgressError, - SessionTurnLeaseLostError, - ): - protected.add(sid) - session_ids.difference_update(protected) - - if not session_ids: - return 0 - - # Orphan any sessions whose parent is about to be deleted - placeholders = ",".join("?" * len(session_ids)) - conn.execute( - f"UPDATE sessions SET parent_session_id = NULL " - f"WHERE parent_session_id IN ({placeholders})", - list(session_ids), - ) - - for sid in session_ids: - conn.execute("DELETE FROM messages WHERE session_id = ?", (sid,)) - conn.execute("DELETE FROM sessions WHERE id = ?", (sid,)) - removed_ids.append(sid) - self._delete_unreferenced_system_prompts(conn) - return len(session_ids) - - count = self._execute_write(_do) - # Clean up on-disk files outside the DB transaction - for sid in removed_ids: - self._remove_session_files(sessions_dir, sid) - return count - - def purge_stale_tool_call_markers( - self, *, dry_run: bool = False, backup: bool = True - ) -> Dict[str, Any]: - """Permanently clear bare tool-call marker content (e.g. "[memory]") - left in the ``messages`` table by sessions persisted before the - #78148 fix in ``agent.conversation_loop``. - - ``_strip_stale_tool_call_markers`` already repairs this in memory on - every session load (see ``_rows_to_conversation``), so running this - is optional — but for long-lived sessions the same rows get - re-scanned and re-repaired on every resume, which is wasted work - and keeps the contaminated bytes sitting in the DB (and in any - downstream cache/backup snapshot of it) indefinitely. This rewrites - the affected rows once, in place. - - Only the ``content`` column is touched — ``role``, ``tool_calls``, - and every other column on the row are left exactly as they are, so - provider tool_call/tool_result pairing is unaffected. - - Unlike the in-memory repair, this UPDATE is permanent and can't be - undone from within the DB. Since ``backup`` defaults to True, a - timestamped full snapshot is taken via ``VACUUM INTO`` (safe against - a live connection, unlike the raw-copy ``_backup_db_file`` used for - malformed-schema repair) before any row is touched — mirroring - ``repair_state_db_schema``'s backup-by-default convention for - destructive state.db operations. No snapshot is taken when there is - nothing to change. - - With ``dry_run=True``, reports the affected row count/ids without - writing or backing up (read-only, no write lock taken). - - Returns ``{"dry_run": bool, "rows_affected": int, "row_ids": [...], - "backup_path": str|None}``. - """ - - def _find_affected(conn) -> List[int]: - cursor = conn.execute( - "SELECT id, content FROM messages " - "WHERE role = 'assistant' AND tool_calls IS NOT NULL AND tool_calls != ''" - ) - affected: List[int] = [] - for row in cursor.fetchall(): - content = row["content"] - if isinstance(content, str) and _STALE_TOOL_CALL_MARKER_RE.fullmatch(content.strip()): - affected.append(row["id"]) - return affected - - with self._read_ctx() as conn: - affected_ids = _find_affected(conn) - - if dry_run: - return { - "dry_run": True, - "rows_affected": len(affected_ids), - "row_ids": affected_ids, - "backup_path": None, - } - - if not affected_ids: - return { - "dry_run": False, - "rows_affected": 0, - "row_ids": [], - "backup_path": None, - } - - backup_path: Optional[str] = None - if backup: - import datetime - - stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - dest = self.db_path.with_name( - f"{self.db_path.name}.pre-clean-markers-backup-{stamp}" - ) - with self._lock: - self._conn.execute("VACUUM INTO ?", (str(dest),)) - backup_path = str(dest) - logger.info("Backed up state.db to %s before clean-markers write", backup_path) - - def _do(conn): - ids = _find_affected(conn) - if ids: - placeholders = ",".join("?" * len(ids)) - conn.execute( - f"UPDATE messages SET content = '' WHERE id IN ({placeholders})", - ids, - ) - return ids - - affected_ids = self._execute_write(_do) - if affected_ids: - logger.info( - "Permanently cleared %d stale tool-call marker row(s) in state.db (#78148)", - len(affected_ids), - ) - return { - "dry_run": False, - "rows_affected": len(affected_ids), - "row_ids": affected_ids, - "backup_path": backup_path, - } # ── Meta key/value (for scheduler bookkeeping) ── @@ -16121,7 +6272,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ).fetchone() if row is None: return None - return row["value"] if isinstance(row, sqlite3.Row) else row[0] + return row[0] def set_meta( self, key: str, value: str, *, cursor: Optional[sqlite3.Cursor] = None @@ -16142,13 +6293,11 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ) return - def _do(conn): - conn.execute( - "INSERT INTO state_meta (key, value) VALUES (?, ?) " - "ON CONFLICT(key) DO UPDATE SET value = excluded.value", - (key, value), - ) - self._execute_write(_do) + self._write_sql( + "INSERT INTO state_meta (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (key, value), + ) def retag_kanban_worker_sessions(self, workspaces_root: str) -> int: """Retag legacy kanban worker rows from ``cli`` to ``kanban``. @@ -16196,567 +6345,12 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) if not prefix: return [] escaped = prefix.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - with self._read_ctx() as conn: - rows = conn.execute( - "SELECT key, value FROM state_meta WHERE key LIKE ? ESCAPE '\\'", - (escaped + "%",), - ).fetchall() + rows = self._read_all( + "SELECT key, value FROM state_meta WHERE key LIKE ? ESCAPE '\\'", + (escaped + "%",), + ) return [(row[0], row[1]) for row in rows] - def apply_telegram_topic_migration(self) -> None: - """Create Telegram DM topic-mode tables on explicit /topic opt-in. - - This migration is deliberately not part of automatic SessionDB startup - reconciliation. Operators must be able to upgrade Hermes, keep the old - Telegram bot behavior running, and only mutate topic-mode state when the - user executes /topic to opt into the feature. - - Schema versions: - v1 — initial shape (no ON DELETE CASCADE on session_id FK) - v2 — session_id FK gets ON DELETE CASCADE so session pruning - automatically clears bindings. - v3 — ``profile_name`` dimension on both tables so multiplexed - gateways (shared ``state.db``) isolate topic mode/bindings - per Hermes profile (issue #76423). - """ - # (table, column list, DDL body). ``profile_name`` leads the primary - # key so multiplexed profiles sharing one state.db never collide on a - # private chat_id (which is the user id, identical across bots). - tables = ( - ( - "telegram_dm_topic_mode", - "profile_name, chat_id, user_id, enabled, activated_at, updated_at, " - "has_topics_enabled, allows_users_to_create_topics, " - "capability_checked_at, intro_message_id, pinned_message_id", - """ - profile_name TEXT NOT NULL DEFAULT 'default', - chat_id TEXT NOT NULL, - user_id TEXT NOT NULL, - enabled INTEGER NOT NULL DEFAULT 1, - activated_at REAL NOT NULL, - updated_at REAL NOT NULL, - has_topics_enabled INTEGER, - allows_users_to_create_topics INTEGER, - capability_checked_at REAL, - intro_message_id TEXT, - pinned_message_id TEXT, - PRIMARY KEY (profile_name, chat_id) - """, - ), - ( - "telegram_dm_topic_bindings", - "profile_name, chat_id, thread_id, user_id, session_key, " - "session_id, managed_mode, linked_at, updated_at", - """ - profile_name TEXT NOT NULL DEFAULT 'default', - chat_id TEXT NOT NULL, - thread_id TEXT NOT NULL, - user_id TEXT NOT NULL, - session_key TEXT NOT NULL, - session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, - managed_mode TEXT NOT NULL DEFAULT 'auto', - linked_at REAL NOT NULL, - updated_at REAL NOT NULL, - PRIMARY KEY (profile_name, chat_id, thread_id) - """, - ), - ) - - def _do(conn): - for table, columns, ddl in tables: - # Fresh installs get the v3 shape immediately. - conn.execute(f"CREATE TABLE IF NOT EXISTS {table} ({ddl})") - have = {row[1] for row in conn.execute(f"PRAGMA table_info('{table}')")} - if "profile_name" in have: - continue - # Pre-profile shape (v1 or v2) → v3. SQLite can't ALTER a - # primary key (or a foreign key), so rebuild; this also - # supplies the v2 ON DELETE CASCADE for v1 bindings tables. - # Legacy rows land in the "default" namespace only — never - # replicated across profiles. - legacy_columns = columns.replace("profile_name, ", "", 1) - conn.executescript( - f""" - CREATE TABLE {table}_new ({ddl}); - INSERT INTO {table}_new ({columns}) - SELECT 'default', {legacy_columns} FROM {table}; - DROP TABLE {table}; - ALTER TABLE {table}_new RENAME TO {table}; - """ - ) - - # Indexes after any rebuild so they always target the v3 shape - # (a legacy table lacking profile_name can't take the user index). - conn.executescript( - """ - CREATE UNIQUE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_session - ON telegram_dm_topic_bindings(session_id); - - CREATE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_user - ON telegram_dm_topic_bindings(profile_name, user_id, chat_id); - """ - ) - - conn.execute( - "INSERT INTO state_meta (key, value) VALUES (?, ?) " - "ON CONFLICT(key) DO UPDATE SET value = excluded.value", - ("telegram_dm_topic_schema_version", "3"), - ) - self._execute_write(_do) - - def enable_telegram_topic_mode( - self, - *, - chat_id: str, - user_id: str, - profile_name: str = "default", - has_topics_enabled: Optional[bool] = None, - allows_users_to_create_topics: Optional[bool] = None, - ) -> None: - """Enable Telegram DM topic mode for one private chat/user. - - This method intentionally owns the explicit topic migration. Ordinary - SessionDB startup must not create these side tables. - - ``profile_name`` namespaces rows under a shared multiplex ``state.db`` - (issue #76423). Callers handling a multiplexed event must pass the - routed profile from ``source.profile``, not the process-global active - profile. - """ - self.apply_telegram_topic_migration() - now = time.time() - profile_name = _normalize_telegram_topic_profile_name(profile_name) - - def _to_int(value: Optional[bool]) -> Optional[int]: - if value is None: - return None - return 1 if value else 0 - - def _do(conn): - conn.execute( - """ - INSERT INTO telegram_dm_topic_mode ( - profile_name, chat_id, user_id, enabled, activated_at, updated_at, - has_topics_enabled, allows_users_to_create_topics, - capability_checked_at - ) VALUES (?, ?, ?, 1, ?, ?, ?, ?, ?) - ON CONFLICT(profile_name, chat_id) DO UPDATE SET - user_id = excluded.user_id, - enabled = 1, - updated_at = excluded.updated_at, - has_topics_enabled = excluded.has_topics_enabled, - allows_users_to_create_topics = excluded.allows_users_to_create_topics, - capability_checked_at = excluded.capability_checked_at - """, - ( - profile_name, - str(chat_id), - str(user_id), - now, - now, - _to_int(has_topics_enabled), - _to_int(allows_users_to_create_topics), - now, - ), - ) - self._execute_write(_do) - - def disable_telegram_topic_mode( - self, - *, - chat_id: str, - profile_name: str = "default", - clear_bindings: bool = True, - ) -> None: - """Disable Telegram DM topic mode for one private chat. - - When ``clear_bindings`` is True (default) the (chat_id, thread_id) - bindings for this chat are also cleared so re-enabling later - starts from a clean slate. Set to False if the operator wants to - preserve bindings for a later re-enable. - - Never creates the topic-mode tables from scratch; if they don't - exist there is nothing to disable and the call is a no-op. - """ - profile_name = _normalize_telegram_topic_profile_name(profile_name) - - def _do(conn): - try: - conn.execute( - "UPDATE telegram_dm_topic_mode SET enabled = 0, updated_at = ? " - "WHERE profile_name = ? AND chat_id = ?", - (time.time(), profile_name, str(chat_id)), - ) - if clear_bindings: - conn.execute( - "DELETE FROM telegram_dm_topic_bindings " - "WHERE profile_name = ? AND chat_id = ?", - (profile_name, str(chat_id)), - ) - except sqlite3.OperationalError: - # Tables don't exist yet — nothing to disable. - return - self._execute_write(_do) - - def is_telegram_topic_mode_enabled( - self, - *, - chat_id: str, - user_id: str, - profile_name: str = "default", - ) -> bool: - """Return whether Telegram DM topic mode is enabled for this chat/user.""" - profile_name = _normalize_telegram_topic_profile_name(profile_name) - with self._read_ctx() as conn: - try: - row = conn.execute( - """ - SELECT enabled FROM telegram_dm_topic_mode - WHERE profile_name = ? AND chat_id = ? AND user_id = ? - """, - (profile_name, str(chat_id), str(user_id)), - ).fetchone() - except sqlite3.OperationalError: - return False - if row is None: - return False - enabled = row["enabled"] if isinstance(row, sqlite3.Row) else row[0] - return bool(enabled) - - def get_telegram_topic_binding( - self, - *, - chat_id: str, - thread_id: str, - profile_name: str = "default", - ) -> Optional[Dict[str, Any]]: - """Return the session binding for a Telegram DM topic, if present.""" - profile_name = _normalize_telegram_topic_profile_name(profile_name) - with self._read_ctx() as conn: - try: - row = conn.execute( - """ - SELECT * FROM telegram_dm_topic_bindings - WHERE profile_name = ? AND chat_id = ? AND thread_id = ? - """, - (profile_name, str(chat_id), str(thread_id)), - ).fetchone() - except sqlite3.OperationalError: - return None - return dict(row) if row else None - - def list_telegram_topic_bindings_for_chat( - self, - *, - chat_id: str, - profile_name: str = "default", - ) -> List[Dict[str, Any]]: - """All Telegram DM topic bindings for one chat, newest first. - - Read-only; returns [] if the bindings table doesn't exist yet - (does not trigger the topic-mode migration). - """ - profile_name = _normalize_telegram_topic_profile_name(profile_name) - with self._read_ctx() as conn: - try: - rows = conn.execute( - "SELECT * FROM telegram_dm_topic_bindings " - "WHERE profile_name = ? AND chat_id = ? " - "ORDER BY updated_at DESC", - (profile_name, str(chat_id)), - ).fetchall() - except sqlite3.OperationalError: - return [] - return [dict(row) for row in rows] - - def get_telegram_topic_binding_by_session( - self, - *, - session_id: str, - ) -> Optional[Dict[str, Any]]: - """Return the Telegram DM topic binding for a given session_id, if present. - - Uses the UNIQUE INDEX on telegram_dm_topic_bindings(session_id) for an - efficient reverse lookup. Returns None when the session has no binding or - the table does not exist yet. - """ - with self._read_ctx() as conn: - try: - row = conn.execute( - """ - SELECT * FROM telegram_dm_topic_bindings - WHERE session_id = ? - """, - (str(session_id),), - ).fetchone() - except sqlite3.OperationalError: - return None - return dict(row) if row else None - - def delete_telegram_topic_binding( - self, - *, - chat_id: str, - thread_id: str, - profile_name: str = "default", - ) -> int: - """Remove the binding row for a single (chat, thread) pair. - - Called when the Telegram Bot API confirms a topic was deleted - externally (``Thread not found`` after the same-thread retry - already failed). Without this prune, the stale row keeps - living in ``telegram_dm_topic_bindings`` and the - recovery logic in ``gateway.run._recover_telegram_topic_thread_id`` - cheerfully redirects future inbound messages to the deleted - topic, causing tool progress, approvals, and replies to land - in the wrong place. Issue #31501. - - When this prune removes the chat's *last* remaining binding, - the chat's row in ``telegram_dm_topic_mode`` is also flipped to - ``enabled = 0`` in the same transaction. Otherwise the chat - would be left in topic mode with zero lanes — and - ``gateway.run._recover_telegram_topic_thread_id`` keeps treating - the chat as topic-enabled, lobby messages keep hunting for a - binding that no longer exists, and a user who disabled topics in - the Telegram client (rather than via ``/topic off``) stays stuck - until the next send happens to fail. Clearing the flag makes - recovery fully stand down once the dead topics are gone. - - Returns the number of binding rows deleted (0 when the binding - was already absent or the topic-mode tables haven't been - migrated yet — both are silent no-ops; we never raise from - a cleanup hot path). - """ - chat_id = str(chat_id) - thread_id = str(thread_id) - profile_name = _normalize_telegram_topic_profile_name(profile_name) - deleted = {"count": 0} - - def _do(conn): - try: - cursor = conn.execute( - """ - DELETE FROM telegram_dm_topic_bindings - WHERE profile_name = ? AND chat_id = ? AND thread_id = ? - """, - (profile_name, chat_id, thread_id), - ) - deleted["count"] = cursor.rowcount or 0 - except sqlite3.OperationalError: - # Tables don't exist yet — nothing to prune. - deleted["count"] = 0 - return - if not deleted["count"]: - return - # If that was the chat's last binding, disable topic mode for - # the chat so recovery stops steering lobby messages at a now - # empty lane set. Same transaction → no read-after-prune race. - try: - remaining = conn.execute( - """ - SELECT 1 FROM telegram_dm_topic_bindings - WHERE profile_name = ? AND chat_id = ? LIMIT 1 - """, - (profile_name, chat_id), - ).fetchone() - if remaining is None: - conn.execute( - "UPDATE telegram_dm_topic_mode " - "SET enabled = 0, updated_at = ? " - "WHERE profile_name = ? AND chat_id = ?", - (time.time(), profile_name, chat_id), - ) - except sqlite3.OperationalError: - # telegram_dm_topic_mode absent — binding prune still stands. - pass - - self._execute_write(_do) - return deleted["count"] - - def bind_telegram_topic( - self, - *, - chat_id: str, - thread_id: str, - user_id: str, - session_key: str, - session_id: str, - managed_mode: str = "auto", - profile_name: str = "default", - ) -> None: - """Bind one Telegram DM topic thread to one Hermes session. - - A Hermes session may only be linked to one Telegram topic in MVP. - Rebinding the same topic to the same session is idempotent; trying to - link the same session to a different topic raises ValueError. - """ - self.apply_telegram_topic_migration() - now = time.time() - chat_id = str(chat_id) - thread_id = str(thread_id) - user_id = str(user_id) - session_key = str(session_key) - session_id = str(session_id) - profile_name = _normalize_telegram_topic_profile_name(profile_name) - - def _do(conn): - existing_session = conn.execute( - """ - SELECT profile_name, chat_id, thread_id - FROM telegram_dm_topic_bindings - WHERE session_id = ? - """, - (session_id,), - ).fetchone() - if existing_session is not None: - if isinstance(existing_session, sqlite3.Row): - linked_profile = existing_session["profile_name"] - linked_chat = existing_session["chat_id"] - linked_thread = existing_session["thread_id"] - else: - linked_profile, linked_chat, linked_thread = existing_session - if ( - str(linked_profile) != profile_name - or str(linked_chat) != chat_id - or str(linked_thread) != thread_id - ): - raise ValueError("session is already linked to another Telegram topic") - - conn.execute( - """ - INSERT INTO telegram_dm_topic_bindings ( - profile_name, chat_id, thread_id, user_id, session_key, session_id, - managed_mode, linked_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(profile_name, chat_id, thread_id) DO UPDATE SET - user_id = excluded.user_id, - session_key = excluded.session_key, - session_id = excluded.session_id, - managed_mode = excluded.managed_mode, - updated_at = excluded.updated_at - """, - ( - profile_name, - chat_id, - thread_id, - user_id, - session_key, - session_id, - managed_mode, - now, - now, - ), - ) - self._execute_write(_do) - - def is_telegram_session_linked_to_topic(self, *, session_id: str) -> bool: - """Return True if a Hermes session is already bound to any Telegram DM topic. - - Read-only: does NOT trigger the telegram-topic migration. If the - topic-mode tables have not been created yet (i.e. nobody has run - ``/topic`` in this profile), the session is by definition unbound - and we return False. - """ - with self._read_ctx() as conn: - try: - row = conn.execute( - """ - SELECT 1 FROM telegram_dm_topic_bindings - WHERE session_id = ? - LIMIT 1 - """, - (str(session_id),), - ).fetchone() - except sqlite3.OperationalError: - return False - return row is not None - - def list_unlinked_telegram_sessions_for_user( - self, - *, - chat_id: str, - user_id: str, - profile_name: str = "default", - limit: int = 10, - ) -> List[Dict[str, Any]]: - """List previous Telegram sessions for this user that are not bound to a topic. - - Read-only: does NOT trigger the telegram-topic migration. If the - topic-mode tables are absent, fall back to a simpler query that - just returns this user's Telegram sessions — there can't be any - bindings yet. - - Scoped by ``profile_name`` so multiplexed profiles do not surface - each other's unlinked sessions (issue #76423). - """ - profile_name = _normalize_telegram_topic_profile_name(profile_name) - # sessions.profile_name is NULL/empty for legacy rows → treat as default. - profile_clause = "AND COALESCE(NULLIF(TRIM(s.profile_name), ''), 'default') = ?" - with self._read_ctx() as conn: - try: - rows = conn.execute( - f""" - SELECT s.*, - COALESCE(sp.prompt, s.system_prompt) - AS _system_prompt_resolved, - COALESCE( - (SELECT {_PREVIEW_RAW_SELECT} - FROM messages m - WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL - AND {_PREVIEW_ELIGIBLE_SQL} - ORDER BY m.timestamp, m.id LIMIT 1), - '' - ) AS _preview_raw, - {_sql_session_last_active("s")} AS last_active - FROM sessions s - LEFT JOIN system_prompts sp - ON sp.hash = s.system_prompt_hash - WHERE s.source = 'telegram' - AND s.user_id = ? - {profile_clause} - AND NOT EXISTS ( - SELECT 1 FROM telegram_dm_topic_bindings b - WHERE b.session_id = s.id - ) - ORDER BY last_active DESC, s.started_at DESC - LIMIT ? - """, - (str(user_id), profile_name, int(limit)), - ).fetchall() - except sqlite3.OperationalError: - # telegram_dm_topic_bindings doesn't exist yet — no bindings - # means every telegram session for this user is "unlinked". - rows = conn.execute( - f""" - SELECT s.*, - COALESCE(sp.prompt, s.system_prompt) - AS _system_prompt_resolved, - COALESCE( - (SELECT {_PREVIEW_RAW_SELECT} - FROM messages m - WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL - AND {_PREVIEW_ELIGIBLE_SQL} - ORDER BY m.timestamp, m.id LIMIT 1), - '' - ) AS _preview_raw, - {_sql_session_last_active("s")} AS last_active - FROM sessions s - LEFT JOIN system_prompts sp - ON sp.hash = s.system_prompt_hash - WHERE s.source = 'telegram' - AND s.user_id = ? - ORDER BY last_active DESC, s.started_at DESC - LIMIT ? - """, - (str(user_id), int(limit)), - ).fetchall() - - sessions: List[Dict[str, Any]] = [] - for row in rows: - session = self._session_row_dict(row) - session["preview"] = _shape_preview(session.pop("_preview_raw", "")) - sessions.append(session) - return sessions # ── Space reclamation ── @@ -16766,269 +6360,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # is present — so we probe each before touching it (see optimize_fts). _FTS_TABLES = ("messages_fts", "messages_fts_trigram", "messages_fts_cjk") - def logical_size_bytes(self) -> Optional[int]: - """Database size in bytes as SQLite itself accounts for it. - - ``page_count * page_size`` — the size the main DB file will have once - the WAL is checkpointed back into it. - - Prefer this over ``os.path.getsize(db_path)`` when reporting the effect - of a VACUUM. In WAL mode a VACUUM's rewrite lands in the ``-wal`` file, - and the checkpoint that folds it back is refused while any other - connection (a live gateway) holds a read-mark. Until that happens the - main file on disk still carries its pre-VACUUM size and keeps growing, - so a stat()-based before/after delta understates the win and can go - negative — the "reclaimed -3820.1 MB" report on a database that had - actually shrunk 60%. - - Returns None if the pragmas cannot be read. - """ - try: - with self._read_ctx() as conn: - if self._conn is None: - return None - page_count = conn.execute("PRAGMA page_count").fetchone()[0] - page_size = conn.execute("PRAGMA page_size").fetchone()[0] - return int(page_count) * int(page_size) - except Exception as exc: - logger.debug("Could not read logical DB size: %s", exc) - return None - - def _freelist_ratio(self) -> Optional[float]: - """Fraction of database pages that are on the freelist (reclaimable). - - ``PRAGMA freelist_count / PRAGMA page_count`` read over the existing - connection (never a byte-level probe of the live file — see - ``sqlite_safe_read``). This is what VACUUM would actually give back; - it is the gate :meth:`maybe_auto_prune_and_vacuum` uses to decide - whether a full rewrite pays off (#54189). - - Returns None if the pragmas cannot be read (callers treat that as - "unknown" and fall back to the time throttle alone). - """ - try: - with self._read_ctx() as conn: - if self._conn is None: - return None - page_count = int(conn.execute("PRAGMA page_count").fetchone()[0]) - freelist = int(conn.execute("PRAGMA freelist_count").fetchone()[0]) - if page_count <= 0: - return 0.0 - return freelist / page_count - except Exception as exc: - logger.debug("Could not read freelist ratio: %s", exc) - return None - - def vacuum(self) -> int: - """Run VACUUM to reclaim disk space after large deletes. - - SQLite does not shrink the database file when rows are deleted — - freed pages just get reused on the next insert. After a prune that - removed hundreds of sessions, the file stays bloated unless we - explicitly VACUUM. - - VACUUM rewrites the entire DB, so it's expensive (seconds per - 100MB) and cannot run inside a transaction. It also acquires an - exclusive lock, so callers must ensure no other writers are - active. Safe to call at startup before the gateway/CLI starts - serving traffic. - - FTS5 segments are merged first via :meth:`optimize_fts` so the - subsequent VACUUM reclaims the pages freed by the merge. This is a - layout-only optimization — search results are unchanged. - - Returns the number of FTS indexes that were optimized (0 if the - merge step failed or no FTS tables exist). - """ - # Merge FTS5 segments before VACUUM so the freed pages are returned - # to the OS in the same pass. optimize_fts() manages its own lock. - optimized = 0 - try: - optimized = self.optimize_fts() - except Exception as exc: - logger.warning("FTS optimize before VACUUM failed: %s", exc) - # VACUUM cannot be executed inside a transaction. - with self._lock: - # Best-effort WAL checkpoint first, then VACUUM. PASSIVE, not - # TRUNCATE: a manual `hermes sessions vacuum` runs in a transient - # CLI process, and a TRUNCATE reset here would race a live gateway - # writer and tear B-tree pages (#45383). VACUUM folds the WAL back - # itself; journal_size_limit bounds the file. - try: - self._conn.execute("PRAGMA wal_checkpoint(PASSIVE)") - except Exception as exc: - logger.debug("WAL checkpoint (PASSIVE) before VACUUM failed: %s", exc) - self._conn.execute("VACUUM") - # ...and again afterwards. VACUUM rewrites every page THROUGH the - # WAL, so the pre-VACUUM checkpoint above does nothing for the - # slack VACUUM itself creates: on a 3.0 GB database it left a - # 3.07 GB state.db-wal behind, so `sessions optimize` reported - # "reclaimed -11.2 MB" while actually consuming 3 GB of disk and - # filling the host to 100%. Truncating here is what makes the - # command a net win instead of a net loss on large databases. - try: - self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") - except Exception as exc: - logger.debug("WAL checkpoint (TRUNCATE) after VACUUM failed: %s", exc) - # TRUNCATE may replace the WAL inode; adopt the post-VACUUM - # sidecars so the write-path generation guard does not halt a - # healthy exclusive maintenance connection. - self._record_db_file_identity() - return optimized - - def maybe_auto_prune_and_vacuum( - self, - retention_days: int = 90, - min_interval_hours: int = 24, - vacuum: bool = True, - sessions_dir: Optional[Path] = None, - min_vacuum_interval_days: int = 30, - min_vacuum_freelist_ratio: float = AUTO_VACUUM_MIN_FREELIST_RATIO, - ) -> Dict[str, Any]: - """Idempotent auto-maintenance: prune inactive sessions + optional VACUUM. - - Records the last run timestamp in state_meta so subsequent calls - within ``min_interval_hours`` no-op. VACUUM has its own, typically - longer, throttle controlled by ``min_vacuum_interval_days`` so routine - pruning does not repeatedly rewrite the database, and is additionally - gated on the reclaimable fraction of the file: it only runs when - ``PRAGMA freelist_count / PRAGMA page_count`` exceeds - ``min_vacuum_freelist_ratio`` (default - :data:`AUTO_VACUUM_MIN_FREELIST_RATIO`, 25%), so pruning a few small - sessions on a dense multi-GB database never triggers a full rewrite - (#54189). Designed to be called once at startup from long-lived - entrypoints (CLI, gateway, cron scheduler). - - When *sessions_dir* is provided, on-disk transcript files - (``.json`` / ``.jsonl`` / ``request_dump_*``) for pruned sessions - are removed as part of the same sweep (issue #3015). - - Stale-open reconciliation (#54189): several state-owned producers - (cron, kanban workers, subagents, one-shot CLI runs) never set - ``ended_at`` when their process dies, and ``prune_sessions`` only - deletes ended rows — so retention was a no-op exactly where growth - concentrates. After pruning, this pass closes open rows from - :attr:`_AUTO_PRUNE_STALE_OPEN_SOURCES` whose activity is older than - ``retention_days`` (``end_reason='startup_orphan_reap'``). Closed rows - stay resumable and are aged from their close, so they get one more - full retention window before a later pass deletes them. Messaging - and UI sources are never touched here. - - Never raises. On any failure, logs a warning and returns a dict - with ``"error"`` set. - - Returns a dict with keys: - - ``"skipped"`` (bool) — true if within min_interval_hours of last run - - ``"pruned"`` (int) — number of sessions deleted - - ``"closed"`` (int) — stale open state-owned sessions marked ended - - ``"vacuumed"`` (bool) — true if VACUUM ran - - ``"freelist_ratio"`` (float|None) — reclaimable fraction measured - when a VACUUM was considered (absent when it was not) - - ``"error"`` (str, optional) — present only on failure - """ - result: Dict[str, Any] = { - "skipped": False, - "pruned": 0, - "closed": 0, - "vacuumed": False, - } - maintenance_lock = _try_acquire_auto_maintenance_lock(self.db_path) - if maintenance_lock is None: - result["skipped"] = True - return result - try: - # Skip if another process/call did maintenance recently. - last_raw = self.get_meta("last_auto_prune") - now = time.time() - if last_raw: - try: - last_ts = float(last_raw) - if now - last_ts < min_interval_hours * 3600: - result["skipped"] = True - return result - except (TypeError, ValueError): - pass # corrupt meta; treat as no prior run - - # Delete only sessions that were already explicitly closed. A - # startup orphan discovered by this pass is closed *after* pruning, - # preserving a full retention window in which it can be resumed. - pruned = self.prune_sessions( - older_than_days=retention_days, - sessions_dir=sessions_dir, - exclude_active_write_guards=True, - ) - result["pruned"] = pruned - - # Reap stale state-owned rows only. Runtime-owned messaging sources - # are intentionally outside this automatic destructive scope. - closed = self.sweep_orphaned_sessions( - max_idle_seconds=float(retention_days) * 86400.0, - sources=self._AUTO_PRUNE_STALE_OPEN_SOURCES, - exclude_pinned=True, - # These sources are owned by state.db lifecycles, not by the - # dashboard/TUI gateway heartbeats used by startup recovery. - respect_gateway_heartbeats=False, - ) - result["closed"] = len(closed) - # Only VACUUM if we actually freed rows, no more often than once - # every min_vacuum_interval_days, AND only when the rewrite pays - # off: the reclaimable fraction of the file (freelist_count / - # page_count) must exceed AUTO_VACUUM_MIN_FREELIST_RATIO (#54189). - # A large prune (e.g. the first one to cross retention_days on a - # DB with tens of thousands of rows) can free enough pages that - # pruned > 0 fires on every subsequent startup even though a - # VACUUM already ran recently; and pruning one tiny session on a - # dense multi-GB DB would otherwise rewrite the whole file to - # reclaim a few MB. VACUUM on this DB's size (FTS5 shadow tables) - # is not cheap -- it holds an exclusive lock for the full rewrite. - # The time throttle says "not too often"; the ratio gate says - # "only when it pays off". Both must pass. - last_vacuum_raw = self.get_meta("last_vacuum") - vacuum_due = True - if last_vacuum_raw: - try: - vacuum_due = (now - float(last_vacuum_raw)) >= min_vacuum_interval_days * 86400 - except (TypeError, ValueError): - vacuum_due = True - if vacuum and pruned > 0 and vacuum_due: - ratio = self._freelist_ratio() - result["freelist_ratio"] = ratio - if ratio is None or ratio > min_vacuum_freelist_ratio: - try: - self.vacuum() - result["vacuumed"] = True - self.set_meta("last_vacuum", str(now)) - except Exception as exc: - logger.warning("state.db VACUUM failed: %s", exc) - else: - logger.debug( - "state.db auto-maintenance: skipping VACUUM, only " - "%.1f%% of pages reclaimable (threshold %.0f%%)", - ratio * 100.0, - min_vacuum_freelist_ratio * 100.0, - ) - - # Record the attempt even if pruned == 0, so we don't retry - # every startup within the min_interval_hours window. - self.set_meta("last_auto_prune", str(now)) - - if closed or pruned > 0: - logger.info( - "state.db auto-maintenance: closed %d stale open session(s), " - "pruned %d session(s) inactive for %d days%s", - len(closed), - pruned, - retention_days, - " + VACUUM" if result["vacuumed"] else "", - ) - except Exception as exc: - # Maintenance must never block startup. Log and return error marker. - logger.warning("state.db auto-maintenance failed: %s", exc) - result["error"] = str(exc) - finally: - _release_auto_maintenance_lock(maintenance_lock) - - return result def maybe_auto_archive( self, @@ -17095,172 +6426,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # The CLI writes "pending" then poll-waits for terminal state. The gateway # watcher transitions pending→running→{completed,failed}. - def request_handoff(self, session_id: str, platform: str) -> bool: - """Mark a session as pending handoff to the given platform. - - Returns True if the row was found and not already in flight; False if - the session is already in a non-terminal handoff state. - """ - def _do(conn): - cur = conn.execute( - "UPDATE sessions " - "SET handoff_state = 'pending', " - " handoff_platform = ?, " - " handoff_error = NULL " - "WHERE id = ? AND (handoff_state IS NULL " - " OR handoff_state IN ('completed', 'failed'))", - (platform, session_id), - ) - return cur.rowcount > 0 - return self._execute_write(_do) - - def get_handoff_state(self, session_id: str) -> Optional[Dict[str, Any]]: - """Read the current handoff state for a session. - - Returns ``{"state", "platform", "error"}`` or None if the session has - no handoff record. - """ - try: - with self._read_ctx() as conn: - row = conn.execute( - "SELECT handoff_state, handoff_platform, handoff_error " - "FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if not row: - return None - return { - "state": row["handoff_state"], - "platform": row["handoff_platform"], - "error": row["handoff_error"], - } - except Exception: - return None - - def list_pending_handoffs(self) -> List[Dict[str, Any]]: - """Return all sessions in handoff_state='pending', oldest first. - - Used by the gateway's handoff watcher. - """ - try: - with self._read_ctx() as conn: - rows = conn.execute( - "SELECT s.*, " - "COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved " - "FROM sessions s " - "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash " - "WHERE s.handoff_state = 'pending' " - "ORDER BY s.started_at ASC" - ).fetchall() - return [self._session_row_dict(r) for r in rows] - except Exception: - return [] - - def claim_handoff(self, session_id: str) -> bool: - """Atomically transition pending → running. Returns True if claimed.""" - def _do(conn): - cur = conn.execute( - "UPDATE sessions SET handoff_state = 'running' " - "WHERE id = ? AND handoff_state = 'pending'", - (session_id,), - ) - return cur.rowcount > 0 - return self._execute_write(_do) - - def complete_handoff(self, session_id: str) -> None: - """Mark a handoff as completed.""" - def _do(conn): - conn.execute( - "UPDATE sessions SET handoff_state = 'completed', " - "handoff_error = NULL WHERE id = ?", - (session_id,), - ) - self._execute_write(_do) - - def fail_handoff( - self, - session_id: str, - error: str, - *, - only_states: Optional[Tuple[str, ...]] = None, - ) -> bool: - """Mark a handoff as failed and record the reason. - - ``only_states`` makes the write a compare-and-swap: the row is only - failed when its current ``handoff_state`` is in the given tuple. - Waiters that give up (CLI 60s poll, Desktop bounded poll) MUST pass - ``only_states=("pending",)`` — once the gateway watcher has claimed - the row (``running``) it owns the terminal state, and a waiter-side - unconditional fail races the dispatch: the gateway later overwrites - ``failed`` → ``completed`` while the user was already told the - gateway is down (split-brain — the handoff actually delivered and - ``switch_session`` re-pointed the session). - - The gateway watcher itself fails its OWN claimed row unconditionally - (no ``only_states``) — it is the owner while the row is ``running``. - - Returns True when a row was transitioned to ``failed``. - """ - def _do(conn): - if only_states: - placeholders = ", ".join("?" for _ in only_states) - cur = conn.execute( - "UPDATE sessions SET handoff_state = 'failed', " - f"handoff_error = ? WHERE id = ? AND handoff_state IN ({placeholders})", - (error[:500], session_id, *only_states), - ) - else: - cur = conn.execute( - "UPDATE sessions SET handoff_state = 'failed', " - "handoff_error = ? WHERE id = ?", - (error[:500], session_id), - ) - return cur.rowcount > 0 - return bool(self._execute_write(_do)) - - def reclaim_stale_running_handoffs(self, error: str) -> List[str]: - """Fail every handoff stuck in ``running``. Returns the ids reclaimed. - - Only the gateway's watcher ever sets ``running``, and it does so for - the duration of a single in-process dispatch. So any row still in - ``running`` when a watcher starts up belongs to a PREVIOUS gateway - that died mid-dispatch (crash, kill, machine reboot). - - Such a row is unrecoverable *and* poisonous: ``request_handoff`` only - accepts a new request when the state is NULL/``completed``/``failed``, - so a stranded ``running`` row makes that session permanently unable to - hand off again — with no error surfaced anywhere. - - Failing (rather than re-queueing as ``pending``) is deliberate: the - dead gateway may have already switched the session key and dispatched - the synthetic turn before dying, so a blind retry risks double - delivery. The user's CLI has long since timed out; the right outcome - is a clean terminal state they can retry from explicitly. - """ - def _do(conn): - cur = conn.execute( - "SELECT id FROM sessions WHERE handoff_state = 'running'" - ) - ids = [r[0] for r in cur.fetchall()] - if ids: - conn.execute( - "UPDATE sessions SET handoff_state = 'failed', " - "handoff_error = ? WHERE handoff_state = 'running'", - (error[:500],), - ) - return ids - try: - return self._execute_write(_do) or [] - except Exception: - # Swallow but never silently: a persistently failing reclaim - # leaves poisonous 'running' rows in place (sessions that can - # never hand off again), so the operator needs a trace of it. - logger.warning( - "reclaim_stale_running_handoffs failed; stranded 'running' " - "handoff rows (if any) were left in place", exc_info=True, - ) - return [] - class AsyncSessionDB: """Async door onto SessionDB: offloads each call via asyncio.to_thread so a blocking SQLite call never freezes the event loop. Generic forwarder — the audit confirms no method returns a live cursor/generator.""" diff --git a/hermes_state_common.py b/hermes_state_common.py index af12d322d3..88faf2e3ba 100644 --- a/hermes_state_common.py +++ b/hermes_state_common.py @@ -1,9 +1,7 @@ -"""Shared module-level constants for the SessionDB family of modules. +"""Shared constants and helpers for the SessionDB family of modules. -Extracted verbatim from hermes_state.py so the SessionDB mixin modules -(hermes_state_search / hermes_state_schema / hermes_state_portability) can -reference them without importing hermes_state (which would be a cycle). -hermes_state re-imports every name here for backward compatibility. +Lives outside hermes_state so the mixin modules can import it without a +cycle; hermes_state re-exports every name for backward compatibility. """ import contextlib @@ -29,16 +27,11 @@ from agent.context_compressor import ( ) -# Session preview = the head of the first user message, shown wherever a -# session has no title (sidebar rows, pickers, exports, the desktop's -# `sessionTitle` fallback). -# -# A /skill invocation expands into a message that embeds the whole skill body, -# so the plain head of it previews the SKILL's opening prose as if the user had -# written it. Scaffolded rows therefore carry a wider excerpt so -# ``_shape_preview`` can hand it to ``describe_skill_invocation`` and recover -# ``/work — fix the title leak``: the whole message while it stays under the -# budget, and head + tail (where the typed instruction lands) once it doesn't. +# Session preview = head of the first user message, shown wherever a session +# has no title. A /skill invocation embeds the whole skill body, so its plain +# head would preview the SKILL's prose; scaffolded rows carry a wider excerpt +# (whole message under budget, else head + tail where the typed instruction +# lands) so ``_shape_preview`` can recover ``/work — fix the title leak``. _PREVIEW_HEAD_CHARS = 63 @@ -49,13 +42,9 @@ _PREVIEW_MAX_CHARS = 60 def escape_like(text: str) -> str: - """Escape SQL LIKE wildcards so operator/session-derived text matches - literally. Pair with ``ESCAPE '\\'`` in the clause. - - ``%`` and ``_`` are wildcards to LIKE, and ``_`` in particular is common - in the values these patterns run against (branch names, session titles, - filesystem paths). A match documented as substring/prefix must not - silently widen. + """Escape LIKE wildcards (``%``, ``_``) so derived text matches literally; + pair with ``ESCAPE '\\'``. ``_`` is common in branch names, titles and + paths, and a documented substring/prefix match must not silently widen. """ return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") @@ -90,10 +79,9 @@ def _sql_starts_with(expression: str, prefixes: tuple[str, ...]) -> str: return "(" + " OR ".join(checks) + ")" -# Current and historical long-form prefixes share this complete introduction; -# their stale-item guidance diverges only after it. Matching the whole intro -# avoids treating an ordinary user message that merely starts with the short -# bracketed label as a compaction carrier. +# Current and legacy long-form prefixes share this whole introduction; matching +# all of it keeps an ordinary message that merely starts with the bracketed +# label from counting as a compaction carrier. _PREVIEW_LONG_FORM_PREFIX = SUMMARY_PREFIX.split("Do NOT answer", 1)[0] _PREVIEW_SUMMARY_PREFIXES = ( _PREVIEW_LONG_FORM_PREFIX, @@ -127,9 +115,8 @@ _PREVIEW_FORCE_USER_REMAINDER_SQL = ( f" + {len(_SUMMARY_END_MARKER)})" ) -# Session preview subqueries select their first eligible user-authored content. -# Pure compaction rows are ineligible; force-user-leading and merged carriers -# remain eligible only when authentic content survives the wire boundary. +# Pure compaction rows are ineligible for previews; force-user-leading and +# merged carriers are eligible only when authentic content survives. _PREVIEW_ELIGIBLE_SQL = ( f"((NOT {_PREVIEW_STANDALONE_SUMMARY_SQL} AND NOT {_PREVIEW_MERGED_SUMMARY_SQL})" f" OR ({_PREVIEW_STANDALONE_SUMMARY_SQL}" @@ -140,10 +127,8 @@ _PREVIEW_ELIGIBLE_SQL = ( ) -# The shared ``_preview_raw`` SELECT expression, interpolated by every listing -# query. A scaffolded row gets a wider excerpt: the whole message while it fits -# the budget, else head + tail (where the typed instruction lands) spliced -# around SKILL_EXCERPT_JOINT. +# Shared ``_preview_raw`` SELECT expression for every listing query (scaffolded +# rows: head + tail spliced around SKILL_EXCERPT_JOINT when over budget). _PREVIEW_RAW_SELECT = ( f"CASE WHEN {_PREVIEW_STANDALONE_SUMMARY_SQL}" f" THEN {_PREVIEW_FORCE_USER_REMAINDER_SQL}" @@ -173,8 +158,17 @@ def _shape_preview(raw: Any) -> str: return text -# A child session counts as a /branch (kept visible, never cascade-deleted) if -# it carries the stable marker OR the legacy end_reason heuristic holds. +# Correlated ``_preview_raw`` column for a ``sessions s`` row. +_PREVIEW_RAW_SUBQUERY_SQL = ( + f"COALESCE((SELECT {_PREVIEW_RAW_SELECT} FROM messages m" + f" WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL" + f" AND {_PREVIEW_ELIGIBLE_SQL}" + f" ORDER BY m.timestamp, m.id LIMIT 1), '') AS _preview_raw" +) + + +# A /branch child (kept visible, never cascade-deleted): stable marker OR the +# legacy end_reason heuristic. _BRANCH_CHILD_SQL = ( "json_extract(COALESCE({a}.model_config, '{{}}'), '$._branched_from') IS NOT NULL" " OR EXISTS (SELECT 1 FROM sessions p" @@ -193,12 +187,10 @@ _COMPRESSION_CHILD_SQL = ( _RESET_END_REASONS = ( "session_reset", - # switch_session() never creates a child row, but pre-marker DBs can hold - # legacy reset children whose parent later ended with 'session_switch' - # (resumed then switched away before reopen-time stamping existed). Also - # keeps this set identical to the recovery fence in - # find_latest_gateway_session_for_peer, which interpolates - # _RESET_END_REASONS_SQL so the two cannot drift. + # switch_session() creates no child row, but pre-marker DBs hold legacy + # reset children whose parent later ended 'session_switch'. Also keeps + # this set identical to the recovery fence in + # find_latest_gateway_session_for_peer (which interpolates the SQL form). "session_switch", "idle", "daily", @@ -207,34 +199,25 @@ _RESET_END_REASONS = ( ) _RESET_END_REASONS_SQL = ", ".join(f"'{reason}'" for reason in _RESET_END_REASONS) -# Accidental end reasons that recovery treats as resumable (see -# docs/session-lifecycle.md "recoverable accidental reasons"). Interpolated -# into the recovery SQL below AND exposed as SessionDB.RECOVERABLE_END_REASONS -# so the tuple is the single source of truth — literals cannot drift. +# Accidental end reasons recovery treats as resumable (docs/session-lifecycle.md). +# Single source of truth: interpolated into recovery SQL AND exposed as +# SessionDB.RECOVERABLE_END_REASONS. _RECOVERABLE_END_REASONS = ( "agent_close", "ws_orphan_reap", - # A stale sentinel-parked runtime quietly superseded by a fresh - # session.resume of the same stored session (no reclaimed broadcast); - # the stored session stays resumable like any accidental end. + # Stale sentinel-parked runtime superseded by a fresh session.resume. "superseded_by_resume", - # Startup sweep of rows orphaned by a dead gateway process (#65194): - # the in-process ws-orphan grace timer died with the process, so the - # row was closed at the next boot instead. Same accident class as - # ws_orphan_reap — kept distinct for forensics — and equally resumable. + # Startup sweep of rows orphaned by a dead gateway process: same accident + # class as ws_orphan_reap, kept distinct for forensics. "startup_orphan_reap", ) _RECOVERABLE_END_REASONS_SQL = ", ".join(f"'{reason}'" for reason in _RECOVERABLE_END_REASONS) -# End reasons written by AUTOMATIC infrastructure cleanup (server shutdown, -# orphan reapers, idle/LRU eviction) rather than by a deliberate conversation -# boundary (compression, session_reset, session_switch, explicit user close). -# An automatic stamp records "some runtime went away", NOT "this conversation -# ended" — so a writer that can prove the conversation is still live (e.g. an -# active compression rotation holding the lease, #88197) may treat the stamp -# as stale and clear it. Superset of the recoverable set: those are already -# resumable accidents; the extra TUI reasons are the same accident class but -# were historically only known to tui_gateway's _AUTOMATIC_SESSION_END_REASONS. +# End reasons written by AUTOMATIC cleanup (shutdown, orphan reapers, idle/LRU +# eviction), not by a deliberate conversation boundary. Such a stamp means +# "some runtime went away", NOT "this conversation ended", so a writer that can +# prove liveness (e.g. a compression rotation holding the lease) may clear it. +# Superset of the recoverable set plus the TUI gateway's automatic reasons. _AUTOMATIC_END_REASONS = frozenset(_RECOVERABLE_END_REASONS) | { "tui_shutdown", "ws_disconnect", @@ -244,23 +227,17 @@ _AUTOMATIC_END_REASONS = frozenset(_RECOVERABLE_END_REASONS) | { def is_automatic_end_reason(reason) -> bool: - """True when *reason* is an automatic-cleanup end stamp (see above). - - Single owner of the "accidental vs deliberate end" predicate — every - compression-liveness site must call this instead of re-implementing the - reason taxonomy (#88197, never-patch-predicates). - """ + """True when *reason* is an automatic-cleanup end stamp (see above). Single + owner of the accidental-vs-deliberate predicate; compression-liveness + sites must call this rather than re-implement the taxonomy.""" return isinstance(reason, str) and reason in _AUTOMATIC_END_REASONS def _legacy_reset_child_sql(alias: str, reasons_sql: str) -> str: - """Pre-marker reset-continuation heuristic. - - A child is a legacy reset continuation when it rides its parent's exact - non-empty routing key and the parent ended at a reset boundary. Shared by - the listing predicate (``_RESET_CHILD_SQL``) and ``reopen_session()``'s - marker-stamping UPDATE so the two sites cannot drift; ``reasons_sql`` is - either the literal ``_RESET_END_REASONS_SQL`` or a bound-placeholder list. + """Pre-marker reset-continuation heuristic: child rides its parent's exact + non-empty routing key and the parent ended at a reset boundary. Shared by + ``_RESET_CHILD_SQL`` and ``reopen_session()``'s marker-stamping UPDATE so + the two cannot drift; ``reasons_sql`` is a literal or placeholder list. """ return ( f"EXISTS (SELECT 1 FROM sessions p" @@ -272,19 +249,17 @@ def _legacy_reset_child_sql(alias: str, reasons_sql: str) -> str: ) -# A reset starts a separate user-visible conversation even though gateway rows -# retain parent_session_id for durable lineage. New rows carry the stable -# marker; the same-key fallback recovers rows written before the marker existed. -# Requiring the exact non-empty routing key keeps ordinary child/subagent rows -# out even when their parent is later reset. +# A reset starts a separate user-visible conversation though rows keep +# parent_session_id for lineage. Stable marker, or the same-key fallback for +# pre-marker rows (the exact-key requirement keeps subagent children out). _RESET_CHILD_SQL = ( "json_extract(COALESCE({a}.model_config, '{{}}'), '$._reset_from') IS NOT NULL" " OR " + _legacy_reset_child_sql("{a}", _RESET_END_REASONS_SQL) ) -# Rows that surface in pickers: roots + branch/reset children. Subagent runs -# and compression continuations stay hidden. +# Picker-visible rows: roots + branch/reset children (not subagent runs or +# compression continuations). _LISTABLE_CHILD_SQL = ( f"(s.parent_session_id IS NULL OR {_BRANCH_CHILD_SQL.format(a='s')}" f" OR {_RESET_CHILD_SQL.format(a='s')})" @@ -305,14 +280,9 @@ def _ephemeral_child_sql(alias: str = "s") -> str: def _sql_session_last_active(alias: str = "s") -> str: - """SQL expression for session recency used by list/status surfaces. - - Freshest of ``last_activity_at`` (mid-turn agent activity heartbeat) and - the latest message timestamp, then fall back to ``started_at``. - - Must not prefer a stale heartbeat over a newer message: durable - heartbeats are rate-limited (~60s), so after a turn writes messages - ``last_activity_at`` can lag ``MAX(messages.timestamp)``. + """Session recency: freshest of ``last_activity_at`` and the latest message + timestamp, else ``started_at``. Heartbeats are rate-limited (~60s) so + ``last_activity_at`` can lag a newer message; never prefer it alone. """ msg_max = ( f"(SELECT MAX(_act_m.timestamp) FROM messages _act_m " @@ -356,24 +326,38 @@ def _sql_session_last_active_by_id(session_id_expr: str) -> str: SCHEMA_VERSION = 28 +# Auto-maintenance VACUUMs only when at least this fraction of pages is on the +# freelist; below it a full rewrite costs more I/O than it returns. +AUTO_VACUUM_MIN_FREELIST_RATIO = 0.25 -# FTS storage-layout version, tracked INDEPENDENTLY of SCHEMA_VERSION in the -# state_meta key ``fts_storage_version``. The main schema version advances -# freely on open (so future migrations always land); the FTS *layout* only -# reaches the current version when a DB is either born fresh or explicitly -# optimized via ``hermes sessions optimize-storage``. A legacy DB sits at -# layout 0 (marker absent) with a working inline index until the user opts in. -# 1 = v23 external-content layout (content/tool_name/tool_calls, -# tool-row-excluded trigram) + +# FTS storage layout, tracked INDEPENDENTLY of SCHEMA_VERSION (state_meta +# ``fts_storage_version``): the schema version advances freely on open, but +# the FTS layout only changes when a DB is born fresh or explicitly optimized +# via ``hermes sessions optimize-storage``. Legacy DBs sit at 0 (marker +# absent) with a working inline index. 1 = v23 external-content layout. FTS_STORAGE_VERSION = 1 -# Cap on user-controlled FTS5 query input before regex/sanitizer processing. -# Search queries do not need to be arbitrarily large, and bounding them keeps -# sanitizer/runtime behavior predictable under adversarial input. +# Cap on user-controlled FTS5 query input before sanitizer processing. MAX_FTS5_QUERY_CHARS = 2_048 +# ── Helpers shared by SessionDB, its mixins and the registry ────────────── + +def stat_db_file_identity(path) -> "tuple[int, int] | None": + """``(st_dev, st_ino)`` for *path*, or None. st_ino=0 (Windows, some + network FS) would false-positive every replaced-file check, so it counts + as unknown.""" + try: + st = os.stat(path) + except OSError: + return None + if not st.st_dev or not st.st_ino: + return None + return (st.st_dev, st.st_ino) + + _FTS_TRIGGERS = ( "messages_fts_insert", "messages_fts_delete", @@ -628,10 +612,8 @@ CREATE INDEX IF NOT EXISTS idx_async_delegations_delivery """ -# Indexes that reference columns added in later schema versions must be -# created AFTER _reconcile_columns() has had a chance to ADD them on -# existing databases. SCHEMA_SQL above is run by sqlite executescript -# which would otherwise fail on legacy DBs ("no such column: active"). +# Indexes on columns added in later schema versions must run AFTER +# _reconcile_columns() adds them, or executescript fails on legacy DBs. DEFERRED_INDEX_SQL = """ CREATE INDEX IF NOT EXISTS idx_messages_session_active ON messages(session_id, active, timestamp); @@ -648,25 +630,17 @@ CREATE INDEX IF NOT EXISTS idx_sessions_system_prompt_hash """ -# ── Deferred FTS rebuild bookkeeping (schema v23) ── -# While a background index rebuild is pending, two state_meta keys define -# which message rows are currently IN the FTS indexes: +# ── Deferred FTS rebuild bookkeeping ── +# While a background rebuild is pending, two state_meta keys define which rows +# are IN the FTS indexes: H = fts_rebuild_high_water (MAX(messages.id) when the +# old indexes were dropped), P = fts_rebuild_progress (highest backfilled id). +# A row is indexed iff id <= P OR id > H (AUTOINCREMENT ids, so post-drop rows +# are indexed live by the insert triggers); rows in (P, H] are not. # -# fts_rebuild_high_water H — MAX(messages.id) at the moment the old -# indexes were dropped -# fts_rebuild_progress P — highest id the chunked backfill has indexed -# -# A row is indexed iff id <= P (backfilled) OR id > H (inserted after -# the drop; ids are AUTOINCREMENT so new rows are always > H and the insert -# triggers index them live). Rows in (P, H] are not yet indexed. -# -# Every trigger below gates on that same predicate: firing an FTS5 -# external-content 'delete' for a row that is NOT in the index corrupts the -# index, and skipping it for a row that IS indexed leaves a stale entry. -# When no rebuild is pending both keys are absent and COALESCE turns the -# predicate into a tautology (id > -1 OR id <= -1), i.e. normal operation. -# The two state_meta PK probes per write are negligible next to the FTS -# insert itself. +# Every trigger gates on that predicate: an FTS5 external-content 'delete' for +# a row NOT in the index corrupts it, and skipping one for an indexed row +# leaves a stale entry. With no rebuild pending both keys are absent and +# COALESCE makes the predicate a tautology. FTS_SQL = """ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( content, @@ -717,20 +691,12 @@ END; """ -# Trigram FTS5 table for CJK substring search. The default unicode61 -# tokenizer splits CJK characters into individual tokens, breaking phrase -# matching. The trigram tokenizer creates overlapping 3-byte sequences so -# substring queries work natively for any script (CJK, Thai, etc.). -# -# The trigram index is the most expensive index in state.db (~2.6x the size -# of the text it covers), and ``role='tool'`` rows are ~90% of message bytes -# while being almost entirely machine noise (base64 payloads, file dumps, -# delegation transcripts). The index therefore reads through -# ``messages_fts_trigram_src``, a view that excludes tool rows — they stay -# fully stored in ``messages`` and fully searchable via the standard -# ``messages_fts`` index; they just don't get trigram (CJK substring) -# treatment. ``search_messages`` routes CJK queries that filter on -# ``role='tool'`` to the LIKE fallback for the same reason. +# Trigram FTS5 table for CJK substring search (unicode61 splits CJK into single +# tokens, breaking phrase matching). The trigram index is ~2.6x the text it +# covers and ``role='tool'`` rows are ~90% of message bytes of machine noise, +# so it reads through the ``messages_fts_trigram_src`` view, which excludes +# tool rows; those remain searchable via ``messages_fts``, and +# ``search_messages`` routes CJK queries filtered on role='tool' to LIKE. FTS_TRIGRAM_SQL = """ CREATE VIEW IF NOT EXISTS messages_fts_trigram_src AS SELECT id, role, content, tool_name, tool_calls @@ -796,18 +762,15 @@ _FTS_CJK_TRIGGERS = ( ) -# state_meta breadcrumb set when a tokenizer-less process had to drop the -# cjk triggers to keep message writes alive: rows written from that moment -# on are missing from the cjk index, so it must not serve reads until +# Set when a tokenizer-less process dropped the cjk triggers to keep writes +# alive: the cjk index is missing rows and must not serve reads until # `hermes sessions optimize-storage` rebuilds it on a capable host. FTS_CJK_STALE_KEY = "fts_cjk_stale" -# Durable breadcrumb for a base/trigram FTS index that was detached from the -# canonical messages table after runtime corruption. While present, startup -# must rebuild the complete index before reinstalling sync triggers: rows may -# have been written while those triggers were absent, so merely recreating -# them would preserve an unknown index gap. +# Set when a base/trigram FTS index was detached after runtime corruption. +# While present, startup must rebuild the complete index before reinstalling +# sync triggers: rows written while they were absent leave an unknown gap. FTS_STALE_KEY = "fts_stale" # Durable diagnostic for stale FTS recovery blocked across process restarts. @@ -815,16 +778,11 @@ FTS_REBUILD_DEFERRAL_KEY = "fts_rebuild_deferral" # ── Legacy (v22 / inline-content) FTS DDL ────────────────────────────── -# Used ONLY to keep an existing pre-v23 install's search working and its -# triggers repairable UNTIL the user opts into `hermes db optimize`. This is -# the exact inline shape v11..v22 shipped: each virtual table stores its own -# copy of ``content || tool_name || tool_calls`` and the trigram table indexes -# every row (including role='tool'). We never CREATE these on a fresh install — -# fresh installs are born on the v23 external-content schema above. These -# constants exist so a legacy DB is never accidentally handed the v23 DDL -# (which would create the external-content trigram source VIEW and leave the -# DB in a mixed, broken state). `optimize_fts_storage()` is what migrates a -# legacy DB to the v23 shape. +# Used ONLY to keep a pre-v23 install's search working and its triggers +# repairable until `optimize_fts_storage()` migrates it: inline copies of +# content || tool_name || tool_calls, trigram over every row. Never created +# on a fresh install. Handing a legacy DB the v23 DDL would create the +# external-content trigram VIEW and leave it in a mixed, broken state. LEGACY_FTS_SQL = """ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( content @@ -882,39 +840,25 @@ END; # ── Cross-process full-FTS-rebuild admission (single authority) ────────────── # -# Several independent Hermes processes routinely share one state.db (gateway -# service, the Desktop app's `hermes serve` backend, interactive CLI sessions, -# the TUI slash worker). A full structural FTS rebuild — the FTS5 'rebuild' -# command or the drop/recreate script in `_recover_stale_fts` — must only ever -# run in ONE of them at a time: two concurrent rebuilds collide on write and -# have structurally corrupted state.db in production (PR #93200; the -# 2026-08-15 / 2026-08-23 incidents and issues #89293 / #90950). +# Several Hermes processes share one state.db. A full structural FTS rebuild +# (FTS5 'rebuild' or the drop/recreate in `_recover_stale_fts`) must run in +# ONE of them at a time: concurrent rebuilds have structurally corrupted +# state.db in production. This is the single admission authority for +# `rebuild_fts()`, `_rebuild_fts_indexes()` and `_recover_stale_fts()`. The +# chunked backfill (`fts_rebuild_step`) is deliberately NOT routed through it: +# it claims progress under SQLite transaction authority and is multi-process. # -# This is the single admission authority for every full structural rebuild -# entry point: `SessionSearchMixin.rebuild_fts()`, -# `SessionSchemaMixin._rebuild_fts_indexes()` (via `_init_schema`), and -# `SessionSchemaMixin._recover_stale_fts()`. The chunked deferred backfill -# (`fts_rebuild_step`) is deliberately NOT routed through it — it claims -# progress under `_execute_write`'s SQLite transaction authority and is -# intentionally multi-process. +# Semantics mirror `hermes_state._cross_process_repair_lock`: portable (msvcrt +# on Windows, flock elsewhere), bounded wait, FAIL CLOSED. The kernel drops +# the lock when the holder dies UNLESS a forked child inherited the fd (flock +# rides the open file description), which holds it forever; so the holder's +# pid + start time are recorded under the lock and a provably-dead holder's +# lock is broken by unlinking and retaking on a fresh inode. Indeterminate +# liveness still defers. Lives here because the mixins cannot import +# hermes_state (cycle). # -# Semantics mirror `hermes_state._cross_process_repair_lock` (the schema- -# surgery authority): portable (msvcrt on Windows, flock elsewhere), bounded -# wait, and FAIL CLOSED — a caller that cannot acquire the lock must NOT -# rebuild. The kernel drops both lock types when the holder dies — UNLESS a -# forked child inherited the lock fd (flock rides the open file description, -# which fork() duplicates), in which case the orphaned descriptor holds the -# lock forever (issue #100108). `_acquire_db_flock` therefore records the -# holder's pid + start time under the lock and, when the recorded holder is -# provably dead, breaks the orphaned lock by unlinking and retaking it on a -# fresh inode; indeterminate liveness still defers. It lives here (not -# hermes_state) because the search/schema mixins cannot import hermes_state -# (cycle). -# -# The lock file is `.fts_rebuild.lock`, distinct from `.repair.lock`: -# schema surgery runs on an EXCLUSIVE offline connection and can legitimately -# take minutes in VACUUM, while runtime rebuilds run on live connections. The -# timeout is sized for a full 'rebuild' of both indexes on a large DB. +# `.fts_rebuild.lock` is distinct from `.repair.lock`: schema surgery +# runs on an EXCLUSIVE offline connection and may take minutes in VACUUM. logger = logging.getLogger("hermes_state") @@ -922,31 +866,23 @@ _FTS_REBUILD_LOCK_TIMEOUT_SECONDS = 120.0 _FTS_REBUILD_LOCK_POLL_SECONDS = 0.1 _IS_WINDOWS = sys.platform == "win32" -# Post-break re-acquire budget: once a provably-orphaned lock has been broken -# the fresh inode is uncontended (or contended only by live processes), so a -# short bounded wait suffices — never re-enter the full timeout. +# Post-break re-acquire budget: the fresh inode is contended only by live +# processes, so a short wait suffices — never re-enter the full timeout. _LOCK_BREAK_REACQUIRE_SECONDS = 5.0 -# errno set for "another process holds this advisory lock". flock() reports -# contention as EWOULDBLOCK/EAGAIN; msvcrt.locking() as EACCES (and EDEADLK -# when its internal retry gives up). Anything else — ESTALE on a dropped NFS -# handle, ENOTSUP/ENOLCK on a filesystem without advisory locks, EIO — is a -# persistent environment failure that no amount of polling turns into an -# acquire. Treating every OSError as contention made such a failure look -# like a live holder and burned the full 120s admission timeout on every -# attempt (#100108, PR #100130). +# "Another process holds the lock": flock → EWOULDBLOCK/EAGAIN, msvcrt.locking +# → EACCES (EDEADLK when its retry gives up). Anything else (ESTALE, ENOTSUP, +# ENOLCK, EIO) is a persistent environment failure that polling cannot fix; +# treating it as contention burned the full timeout on every attempt. _LOCK_CONTENTION_ERRNOS = {errno.EAGAIN, errno.EACCES, errno.EWOULDBLOCK} if hasattr(errno, "EDEADLK"): _LOCK_CONTENTION_ERRNOS.add(errno.EDEADLK) def is_advisory_lock_contention(exc: BaseException) -> bool: - """True when *exc* means another process holds the advisory lock. - - False for every other ``OSError`` (ESTALE, ENOTSUP, ENOLCK, EIO, ...): - callers must fail closed IMMEDIATELY rather than poll to the deadline, - because retrying cannot succeed and the wait only stalls the caller. - """ + """True when *exc* means another process holds the advisory lock. For + any other ``OSError`` callers must fail closed IMMEDIATELY: retrying + cannot succeed and polling only stalls the caller.""" if isinstance(exc, BlockingIOError): return True if not isinstance(exc, OSError): @@ -955,13 +891,9 @@ def is_advisory_lock_contention(exc: BaseException) -> bool: def _proc_start_ticks(pid: int): - """Kernel start time of *pid* in clock ticks, or None when unknowable. - - Field 22 of ``/proc//stat`` (``starttime``) uniquely identifies a - process together with its PID: a recycled PID gets a different start - time. Returns None off Linux or on any read/parse failure — callers must - treat None as "unknowable" and FAIL CLOSED. - """ + """Kernel start time of *pid* (field 22 of ``/proc//stat``), which + with the PID uniquely identifies a process; None off Linux or on any + failure — callers must treat None as unknowable and FAIL CLOSED.""" try: with open(f"/proc/{pid}/stat", "rb") as fh: stat = fh.read() @@ -988,12 +920,8 @@ def _read_lock_holder_record(handle): def _write_lock_holder_record(handle) -> None: - """Record this process as the lock holder (advisory, best effort). - - Written under the flock so contenders that time out can tell an - orphaned-fd holder (recorded process dead, flock inherited by a forked - child — issue #100108) from a live wedged holder. - """ + """Record this process as holder (best effort), written under the flock so + timed-out contenders can tell an orphaned-fd holder from a live wedged one.""" try: record = { "pid": os.getpid(), @@ -1009,12 +937,8 @@ def _write_lock_holder_record(handle) -> None: def _clear_lock_holder_record(handle) -> None: - """Erase holder metadata before a normal release. - - Guarantees that a surviving record always describes an ABNORMAL exit - (holder died without releasing), which is the only condition under which - a contender may break the lock. - """ + """Erase holder metadata before a normal release, so a surviving record + always means an ABNORMAL exit — the only condition allowing a break.""" try: handle.seek(0) handle.truncate() @@ -1025,12 +949,8 @@ def _clear_lock_holder_record(handle) -> None: def _lock_holder_provably_dead(record) -> bool: """True ONLY when the recorded holder is provably dead or PID-recycled. - - Any indeterminate state (no record, malformed record, PID owned by - another user, /proc unavailable, start-time unknowable) returns False — - the caller must FAIL CLOSED and defer, never break a possibly-live - holder's lock. - """ + Anything indeterminate (no/malformed record, PID owned by another user, + /proc unavailable) is False — the caller must FAIL CLOSED and defer.""" if not isinstance(record, dict): return False try: @@ -1052,34 +972,27 @@ def _lock_holder_provably_dead(record) -> bool: current_ticks = _proc_start_ticks(pid) if current_ticks is None: return False - # Same PID, different kernel start time: the recorded holder is dead and - # its PID was recycled by an unrelated process. + # Same PID, different start time: recycled by an unrelated process. return current_ticks != recorded_ticks def _acquire_db_flock(lock_path, handle, timeout_seconds, poll_seconds, description): """Bounded POSIX flock acquire with orphaned-holder staleness break. - Returns ``(acquired, handle)``; *handle* may have been re-opened (the - caller owns closing whichever handle comes back). *acquired* is True on - success, False when a holder kept the lock past the deadline, and None - when a non-contention ``OSError`` (ESTALE/ENOTSUP/EIO) made acquisition - impossible — already logged here; callers treat None as "not acquired" - without emitting the held-by-another-process warning. + Returns ``(acquired, handle)``; *handle* may have been re-opened and the + caller closes whichever comes back. *acquired* is True, False (a holder + kept the lock past the deadline), or None (non-contention ``OSError``, + already logged; callers treat it as not acquired without the + held-by-another-process warning). - Why breaking exists at all (issue #100108): ``flock`` belongs to the open - file DESCRIPTION, which ``fork()`` duplicates into every child. A holder - that forks (multiprocessing worker, daemonized helper) and then dies - leaves the flock held by a child that will never release it — the - kernel's holder-death release never triggers, and every contender defers - forever. The recorded-holder liveness check distinguishes exactly that - case: the process that ACQUIRED is provably dead (so its critical section - died with it), yet the flock is still held. Only then is the lock file - unlinked and retaken on a fresh inode; the orphan's flock stays on the - old unlinked inode where it blocks nobody. Every successful acquire - verifies its inode still names *lock_path*, so a racer that locked a dead - inode retries instead of running concurrently with the breaker. - Indeterminate liveness always defers (fail closed). + ``flock`` belongs to the open file DESCRIPTION, which ``fork()`` + duplicates: a holder that forks then dies leaves the lock held forever by + a child that never releases. When the process that ACQUIRED is provably + dead (its critical section died with it) yet the flock is held, the file + is unlinked and retaken on a fresh inode; the orphan's flock stays on the + old inode blocking nobody. Every successful acquire verifies its inode + still names *lock_path*, so a racer that locked a dead inode retries + instead of running alongside the breaker. Indeterminate liveness defers. """ import fcntl @@ -1090,9 +1003,7 @@ def _acquire_db_flock(lock_path, handle, timeout_seconds, poll_seconds, descript fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) except (BlockingIOError, OSError) as exc: if not is_advisory_lock_contention(exc): - # ESTALE / ENOTSUP / EIO: not a holder, and polling cannot - # fix it. Defer NOW instead of pretending a live process - # held the lock for the whole timeout (#100108). + # Not a holder and polling cannot fix it: defer NOW. logger.warning( "Could not acquire %s %s (%s) — deferring rather than " "waiting out the %.0fs holder timeout on a " @@ -1135,9 +1046,8 @@ def _acquire_db_flock(lock_path, handle, timeout_seconds, poll_seconds, descript broke_lock = True deadline = time.monotonic() + _LOCK_BREAK_REACQUIRE_SECONDS continue - # flock acquired — verify the path still names our inode: a breaker - # may have unlinked/replaced the file while we waited, and a lock on - # a dead inode excludes nobody. + # Verify the path still names our inode: a breaker may have replaced + # the file while we waited, and a lock on a dead inode excludes nobody. try: fd_stat = os.fstat(handle.fileno()) path_stat = os.stat(lock_path) @@ -1178,21 +1088,13 @@ def _describe_lock_holder(record) -> str: def fts_rebuild_admission(db_path, *, timeout_seconds=None): """Serialize full structural FTS rebuilds on *db_path* across processes. - Yields True when this process holds the rebuild authority, False when the - bounded acquire timed out or the lock file could not be opened at all. A - caller that gets False must NOT perform a full rebuild — proceeding is - exactly the concurrent-rebuild interleaving this lock exists to prevent - (fail closed). The deferred/stale breadcrumb machinery already guarantees - a skipped rebuild is retried later. + Yields True when this process holds the authority, False when the bounded + acquire timed out or the lock file could not be opened. On False the + caller must NOT rebuild (fail closed); the stale breadcrumb guarantees a + retry. ``db_path`` None (in-memory DB) yields True. - ``db_path`` may be a str or Path; None (in-memory DB / tests without a - file path) yields True — a private in-memory DB has no cross-process - surface. - - *timeout_seconds* defaults to ``_FTS_REBUILD_LOCK_TIMEOUT_SECONDS``. - Opportunistic in-process retries (``retry_deferred_fts_recovery``) pass - ``0`` so a live holder never stalls a long-lived writer for two minutes; - the orphaned-holder break still applies on the single attempt. + Opportunistic in-process retries pass ``timeout_seconds=0`` so a live + holder never stalls a long-lived writer; the orphan break still applies. """ if db_path is None: yield True @@ -1206,16 +1108,12 @@ def fts_rebuild_admission(db_path, *, timeout_seconds=None): try: handle = open(lock_path, "a+b") except OSError as exc: - # Fail closed, exactly as a timed-out acquire does. A lock file we - # cannot even open means the filesystem is out of space, inodes or - # descriptors — and a sibling process that opened ITS handle before - # the disk filled is still holding the authority and rebuilding. - # Yielding True here handed every process on a full disk a concurrent - # structural rebuild of the same live state.db with no cross-process - # authority at all: the disk-full trigger and the re-corruption on - # every multi-writer boot in #100368. Deferring costs nothing that - # was reachable anyway — the breadcrumb retries, and on a read-only - # directory the rebuild's own writes could not have committed either. + # Fail closed like a timed-out acquire. An unopenable lock file means + # the FS is out of space/inodes/descriptors, and a sibling that opened + # its handle earlier may still be rebuilding; yielding True here gave + # every process on a full disk a concurrent rebuild of the same DB. + # Deferring costs nothing: the breadcrumb retries, and the rebuild's + # own writes could not have committed either. logger.warning( "Could not open FTS rebuild lock %s (%s) — deferring this rebuild " "rather than running it without cross-process authority.", @@ -1257,14 +1155,13 @@ def fts_rebuild_admission(db_path, *, timeout_seconds=None): "FTS rebuild lock", ) if acquired is None: - # Non-contention failure: already logged with the real errno; - # a "held by another process" line here would be a lie. + # Already logged with the real errno; "held by another process" + # would be a lie. acquired = False elif not acquired: record = None if _IS_WINDOWS else _read_lock_holder_record(handle) if timeout <= 0: - # Non-blocking probe from an in-process retry: a busy lock - # is expected and will be tried again, so keep it quiet. + # Non-blocking probe from an in-process retry: keep it quiet. logger.info( "FTS rebuild lock %s is busy — deferring this retry " "(the stale-FTS breadcrumb keeps it retryable). " diff --git a/hermes_state_compression.py b/hermes_state_compression.py new file mode 100644 index 0000000000..cb91823f39 --- /dev/null +++ b/hermes_state_compression.py @@ -0,0 +1,1030 @@ +"""Compression lineage, cooldown/streak counters, locks and turn leases for SessionDB. + +Mixin bound onto ``SessionDB`` via the MRO, built on its ``_read_ctx`` / +``_execute_write`` / ``_write_sql`` / ``_read_one`` primitives.""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +import time +from typing import Any, Dict, List, Optional + +from hermes_state_common import _sql_session_last_active, is_automatic_end_reason + +# Log-record parity with the origin module (caplog tests pin "hermes_state"). +logger = logging.getLogger("hermes_state") + + +class SessionCompressionMixin: + """Compression lineage, cooldown/streak counters, locks and turn leases.""" + + def find_live_compression_child( + self, parent_session_id: str + ) -> Optional[Dict[str, Any]]: + """Return the unique live direct child of a compression-ended session. + + A stale agent whose parent was rotated elsewhere may recover only when the + lineage names exactly one live continuation; more than one fails closed + rather than guessing which transcript owns later messages.""" + if not parent_session_id: + return None + with self._read_ctx() as conn: + parent = conn.execute( + "SELECT ended_at, end_reason FROM sessions WHERE id = ?", + (parent_session_id,), + ).fetchone() + if ( + parent is None + or parent["ended_at"] is None + or parent["end_reason"] != "compression" + ): + return None + rows = conn.execute( + """ + SELECT s.*, + COALESCE(sp.prompt, s.system_prompt) + AS _system_prompt_resolved + FROM sessions s + LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash + WHERE s.parent_session_id = ? + AND s.ended_at IS NULL + """ + + self._NON_CONTINUATION_CHILD_FILTER_SQL.format(alias="s.") + + """ + ORDER BY s.started_at ASC + LIMIT 2 + """, + (parent_session_id, parent_session_id, parent_session_id), + ).fetchall() + return self._session_row_dict(rows[0]) if len(rows) == 1 else None + + def reopen_orphaned_compression_session(self, session_id: str) -> bool: + """Reopen a compression parent only when no continuation was published. + + Publication is atomic now, but older builds could leave a closed parent + after an interrupted handoff. Conservative by design: an active lease or + any canonical child means another path owns the lineage — fail closed.""" + if not session_id: + return False + + def _do(conn): + parent = conn.execute( + "SELECT ended_at, end_reason FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + if ( + parent is None + or parent["ended_at"] is None + or parent["end_reason"] != "compression" + ): + return False + + # Any non-branch/non-delegate/non-tool child is a continuation, ended + # or not; reopening past it could give one lineage a second live head. + child = conn.execute( + """ + SELECT 1 + FROM sessions + WHERE parent_session_id = ? + """ + + self._NON_CONTINUATION_CHILD_FILTER_SQL.format(alias="") + + """ + LIMIT 1 + """, + (session_id, session_id, session_id), + ).fetchone() + if child is not None: + return False + + # refresh_compression_lock() lets an owner revive its own expired + # row, so reclaim it inside this write transaction before reopening: + # refresh-first makes the lease active and aborts recovery; + # recovery-first deletes the holder so a later refresh can't resurrect it. + now = time.time() + lock_row = conn.execute( + "SELECT holder, expires_at FROM compression_locks " + "WHERE session_id = ?", + (session_id,), + ).fetchone() + if lock_row is not None: + expires_at = lock_row["expires_at"] + if expires_at is None or float(expires_at) >= now: + return False + deleted = conn.execute( + "DELETE FROM compression_locks " + "WHERE session_id = ? AND holder = ? AND expires_at = ?", + (session_id, lock_row["holder"], expires_at), + ) + if deleted.rowcount != 1: + return False + + updated = conn.execute( + "UPDATE sessions SET ended_at = NULL, end_reason = NULL " + "WHERE id = ? AND ended_at IS NOT NULL " + "AND end_reason = 'compression'", + (session_id,), + ) + # rowcount==1 is guaranteed by the parent SELECT in this same BEGIN + # IMMEDIATE transaction. If a False return is ever added past this + # point, raise instead: _execute_write commits the lease DELETE above unless _do raises. + return updated.rowcount == 1 + + return bool(self._execute_write(_do)) + + def publish_compression_child( + self, + *, + parent_session_id: str, + child_session_id: str, + source: str, + messages: List[Dict[str, Any]], + model: str = None, + model_config: Dict[str, Any] = None, + system_prompt: str = None, + cwd: str = None, + profile_name: str = None, + compression_lock_holder: str = None, + require_compression_lease: bool = True, + require_lease_refresh: bool = False, + lease_ttl_seconds: float = 300.0, + watermark: Optional[int] = None, + watermark_ceiling: Optional[int] = None, + ) -> None: + """Atomically close a parent and publish its durable compression child. + + Closure, child row, and handoff commit in one transaction: readers see + the live parent or a complete child, never an ended parent with a + missing/empty child. + + *watermark* (parent's ``get_active_message_watermark`` at compression + start): parent rows with ``id > watermark`` — appends landed during the + slow summary call — are column-cloned into the child AFTER the handoff + so they survive rotation. *watermark_ceiling* bounds the clone: the + rotation path flushes its OWN transcript to the parent just before + publishing and those rows are already in the handoff, so the caller + captures ``MAX(id)`` right BEFORE that flush and only + ``(watermark, watermark_ceiling]`` is foreign tail. ``None`` = unbounded. + + *require_lease_refresh* + *compression_lock_holder* refreshes the lease + on the same ``conn`` before the expiry check (no TOCTOU window), so a + refresher that died on transient DB errors gets one last chance.""" + from hermes_state import CompressionSessionBusyError + def _do(conn): + if require_lease_refresh and compression_lock_holder: + conn.execute( + "UPDATE compression_locks SET expires_at = ? " + "WHERE session_id = ? AND holder = ?", + (time.time() + lease_ttl_seconds, parent_session_id, + compression_lock_holder), + ) + lock_row = conn.execute( + "SELECT holder, expires_at FROM compression_locks WHERE session_id = ?", + (parent_session_id,), + ).fetchone() + if require_compression_lease and ( + lock_row is None + or not compression_lock_holder + or lock_row["holder"] != compression_lock_holder + or float(lock_row["expires_at"]) <= time.time() + ): + raise CompressionSessionBusyError( + f"Compression lease lost before publication: {parent_session_id}" + ) + parent = conn.execute( + """SELECT ended_at, end_reason, cwd, git_branch, git_repo_root, + user_id, session_key, chat_id, chat_type, + thread_id, display_name, origin_json, profile_name + FROM sessions WHERE id = ?""", + (parent_session_id,), + ).fetchone() + if parent is None: + raise RuntimeError(f"Compression parent not found: {parent_session_id}") + if parent["ended_at"] is not None: + # An ended stamp from AUTOMATIC cleanup (tui_shutdown, ws_disconnect, + # orphan reap, idle/LRU evict) is stale by construction — this lease + # holder is still continuing the conversation. Left alone it wedges + # rotation forever (every attempt aborts here; each pre-publish flush + # re-grows the parent until the provider rejects it). Clear it; the + # closure UPDATE below re-stamps end_reason='compression'. Deliberate + # boundaries (compression, session_reset, explicit close) still fail + # closed — another path owns the lineage. + if is_automatic_end_reason(parent["end_reason"]): + conn.execute( + "UPDATE sessions SET ended_at = NULL, end_reason = NULL " + "WHERE id = ?", + (parent_session_id,), + ) + else: + raise RuntimeError( + f"Compression parent already ended: {parent_session_id}" + ) + if not messages: + raise RuntimeError("Compression child handoff must not be empty") + system_prompt_hash = self._store_system_prompt(conn, system_prompt) + + conn.execute( + """INSERT INTO sessions ( + id, source, model, model_config, system_prompt, + system_prompt_hash, + parent_session_id, cwd, git_branch, git_repo_root, + profile_name, user_id, session_key, chat_id, chat_type, + thread_id, display_name, origin_json, started_at + ) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + child_session_id, + source, + model, + json.dumps(model_config) if model_config else None, + system_prompt_hash, + parent_session_id, + cwd or parent["cwd"], + parent["git_branch"], + parent["git_repo_root"], + # Same contract as _insert_session_row's compression-fork backfill: + # child stays on the parent's profile and keeps gateway routing/ + # origin columns so peer recovery works after a boundary crash. No + # owner on either side (legacy NULL parent) → stamp this store's + # profile so the child doesn't extend the unowned lineage. + profile_name + or parent["profile_name"] + or self._own_profile_name(), + parent["user_id"], + parent["session_key"], + parent["chat_id"], + parent["chat_type"], + parent["thread_id"], + parent["display_name"], + parent["origin_json"], + time.time(), + ), + ) + total_messages, total_tool_calls = self._insert_message_rows( + conn, child_session_id, messages + ) + if watermark is not None: + # Clone the parent's concurrent tail (see docstring) into the + # child after the handoff: column-exact except id/session_id; + # originals stay in the closed parent for lineage recovery. + _ceiling_clause = "" + _params: list = [parent_session_id, int(watermark)] + if watermark_ceiling is not None: + _ceiling_clause = " AND id <= ?" + _params.append(int(watermark_ceiling)) + tail_rows = conn.execute( + "SELECT id, tool_calls FROM messages " + "WHERE session_id = ? AND active = 1 AND id > ?" + f"{_ceiling_clause} ORDER BY id", + _params, + ).fetchall() + if tail_rows: + tail_ids = [int(r["id"]) for r in tail_rows] + placeholders = ",".join("?" for _ in tail_ids) + clone_cols = [ + c for c in self._message_column_names(conn) + if c not in ("id", "session_id", "active", "compacted") + ] + col_list = ", ".join(clone_cols) + conn.execute( + f"INSERT INTO messages ({col_list}, session_id, active, compacted) " + f"SELECT {col_list}, ?, 1, 0 FROM messages " + f"WHERE id IN ({placeholders}) ORDER BY id", + [child_session_id, *tail_ids], + ) + total_messages += len(tail_ids) + for r in tail_rows: + raw = r["tool_calls"] + if raw: + try: + parsed = json.loads(raw) if isinstance(raw, str) else raw + total_tool_calls += len(parsed) if isinstance(parsed, list) else 0 + except (TypeError, ValueError): + pass + conn.execute( + "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", + (total_messages, total_tool_calls, child_session_id), + ) + updated = conn.execute( + "UPDATE sessions SET ended_at = ?, end_reason = 'compression' " + "WHERE id = ? AND ended_at IS NULL", + (time.time(), parent_session_id), + ) + if updated.rowcount != 1: + raise RuntimeError( + f"Compression parent changed during publication: {parent_session_id}" + ) + + self._execute_write(_do) + + def record_compression_failure_cooldown( + self, + session_id: str, + cooldown_until: float, + error: Optional[str] = None, + ) -> None: + """Persist the active compression-failure cooldown for a session.""" + if not session_id: + return + + try: + # Merge-max with any longer live deadline so a later shorter write + # can't reopen the thrash window; error always takes the latest diagnostic. + self._write_sql( + "UPDATE sessions SET compression_failure_cooldown_until = CASE " + "WHEN compression_failure_cooldown_until IS NOT NULL " + " AND compression_failure_cooldown_until > ? " + "THEN compression_failure_cooldown_until ELSE ? END, " + "compression_failure_error = ? WHERE id = ?", + (cooldown_until, cooldown_until, error, session_id), + ) + except sqlite3.Error as exc: + logger.warning( + "record_compression_failure_cooldown(%s) failed: %s", + session_id, exc, + ) + + def get_compression_failure_cooldown( + self, + session_id: str, + ) -> Optional[Dict[str, Any]]: + """Return the active compression-failure cooldown for ``session_id``.""" + if not session_id: + return None + now = time.time() + row = self._read_one( + "SELECT compression_failure_cooldown_until, compression_failure_error " + "FROM sessions WHERE id = ?", + (session_id,), + ) + if row is None: + return None + cooldown_until = row[0] + if cooldown_until is None: + return None + cooldown_until = float(cooldown_until) + if cooldown_until <= now: + return None + error = row[1] + return { + "cooldown_until": cooldown_until, + "remaining_seconds": cooldown_until - now, + "error": error, + } + + def get_compression_failure_cooldown_row( + self, + session_id: str, + ) -> Dict[str, Any]: + """Exact stored cooldown columns, no expiry filtering. Compression + cancellation uses this under its session lease so rollback preserves an + expired, partially-null, or absent row exactly instead of coercing it + through the active-cooldown API.""" + if not session_id: + return {"session_exists": False, "cooldown_until": None, "error": None} + row = self._read_one( + "SELECT compression_failure_cooldown_until, compression_failure_error " + "FROM sessions WHERE id = ?", + (session_id,), + ) + if row is None: + return {"session_exists": False, "cooldown_until": None, "error": None} + cooldown_until = row[0] + error = row[1] + return { + "session_exists": True, + "cooldown_until": ( + float(cooldown_until) if cooldown_until is not None else None + ), + "error": error, + } + + def restore_compression_failure_cooldown_row( + self, + session_id: str, + snapshot: Dict[str, Any], + ) -> None: + """Restore and verify an exact cooldown-row snapshot. Unlike record/clear, + this rollback API propagates write and verification failures: cancellation + must not be reported mutation-free when compensation failed.""" + expected_exists = bool(snapshot.get("session_exists", False)) + if not expected_exists: + actual = self.get_compression_failure_cooldown_row(session_id) + if actual.get("session_exists", False): + raise RuntimeError( + "cannot restore absent compression cooldown row: session now exists" + ) + return + + deadline = snapshot.get("cooldown_until") + error = snapshot.get("error") + + def _do(conn): + cursor = conn.execute( + "UPDATE sessions SET compression_failure_cooldown_until = ?, " + "compression_failure_error = ? WHERE id = ?", + (deadline, error, session_id), + ) + if cursor.rowcount != 1: + raise RuntimeError( + f"compression cooldown rollback session missing: {session_id}" + ) + + self._execute_write(_do) + actual = self.get_compression_failure_cooldown_row(session_id) + expected = { + "session_exists": True, + "cooldown_until": float(deadline) if deadline is not None else None, + "error": error, + } + if actual != expected: + raise RuntimeError( + f"compression cooldown rollback verification failed: " + f"expected={expected!r}, actual={actual!r}" + ) + + def clear_compression_failure_cooldown(self, session_id: str) -> None: + """Clear any persisted compression-failure cooldown for a session.""" + if not session_id: + return + + try: + self._write_sql( + "UPDATE sessions SET compression_failure_cooldown_until = NULL, " + "compression_failure_error = NULL WHERE id = ?", + (session_id,), + ) + except sqlite3.Error as exc: + logger.warning( + "clear_compression_failure_cooldown(%s) failed: %s", + session_id, exc, + ) + + def _read_session_number(self, column: str, session_id: str, cast: type, zero: Any) -> Any: + """Read one numeric ``sessions`` column clamped at ``zero``; a missing + session, NULL, or unparsable value also reads as ``zero``.""" + if not session_id: + return zero + row = self._read_one( + f"SELECT {column} FROM sessions WHERE id = ?", (session_id,) + ) + if row is None: + return zero + try: + return max(zero, cast(row[0] or zero)) + except (TypeError, ValueError): + return zero + + def get_compression_fallback_streak(self, session_id: str) -> int: + """Return the persisted deterministic-fallback streak.""" + return self._read_session_number("compression_fallback_streak", session_id, int, 0) + + def set_compression_fallback_streak(self, session_id: str, streak: int) -> None: + """Persist the deterministic-fallback streak for one session.""" + if session_id: + self._write_sql( + "UPDATE sessions SET compression_fallback_streak = ? WHERE id = ?", + (max(0, int(streak)), session_id), + ) + + def get_compression_ineffective_count(self, session_id: str) -> int: + """Persisted ineffective-compaction strike count: the durable half of + the built-in compressor's anti-thrash guard, so a fresh compressor bound + to a resumed session inherits an armed/tripped guard across restarts.""" + return self._read_session_number("compression_ineffective_count", session_id, int, 0) + + def set_compression_ineffective_count(self, session_id: str, count: int) -> None: + """Persist the ineffective-compaction strike count for one session.""" + if session_id: + self._write_sql( + "UPDATE sessions SET compression_ineffective_count = ? WHERE id = ?", + (max(0, int(count)), session_id), + ) + + def get_compression_recovery_deadline(self, session_id: str) -> float: + """Persisted anti-thrash recovery deadline (epoch; ``0.0`` = not armed). + Durable because the gateway rebuilds the compressor every turn / cache + eviction: a process-local deadline restarted on each rebuild, so a + tripped session never earned its probe.""" + return self._read_session_number("compression_recovery_deadline", session_id, float, 0.0) + + def set_compression_recovery_deadline(self, session_id: str, deadline: float) -> None: + """Persist the anti-thrash recovery deadline; ``0`` / ``None`` disarms it.""" + if not session_id: + return + try: + normalized = max(0.0, float(deadline or 0.0)) + except (TypeError, ValueError): + normalized = 0.0 + stored = normalized if normalized > 0.0 else None + + self._write_sql( + "UPDATE sessions SET compression_recovery_deadline = ? WHERE id = ?", + (stored, session_id), + ) + + def refresh_compression_lock( + self, + session_id: str, + holder: str, + ttl_seconds: float = 300.0, + ) -> bool: + """Extend the compression lock lease if ``holder`` still owns it. + + Ownership is decided by ``holder`` alone, deliberately NOT ``expires_at``: + a live owner whose refresher stalled past its TTL (GC pause, loaded CI + runner, slow write escaping ``_execute_write``'s retry budget) must be + able to revive its still-unclaimed row. Requiring ``expires_at >= now`` + made such a stall permanent — every later refresh matched 0 rows and the + owner kept compressing/rotating with no lease, exactly the window in + which a competing path can fork the lineage. + + It cannot resurrect a lock someone else took: SQLite serialises writes, + so :meth:`try_acquire_compression_lock`'s reclaim (DELETE-expired + + INSERT-or-IGNORE) never interleaves with this UPDATE. Reclaim-first + replaces ``holder`` and this matches nothing; refresh-first pushes + ``expires_at`` forward and the reclaimer's DELETE matches nothing.""" + if not session_id or not holder: + return False + now = time.time() + expires_at = now + ttl_seconds + + try: + return self._write_rowcount( + "UPDATE compression_locks SET expires_at = ? " + "WHERE session_id = ? AND holder = ?", + (expires_at, session_id, holder), + ) > 0 + except sqlite3.Error as exc: + logger.warning( + "refresh_compression_lock(%s) failed: %s", + session_id, exc, + ) + return False + + def try_acquire_compression_lock( + self, + session_id: str, + holder: str, + ttl_seconds: float = 300.0, + ) -> bool: + """Try to atomically acquire the compression lock for ``session_id``. + + ``True``: caller owns the lock and must :meth:`release_compression_lock`. + ``False``: another holder owns a live lock and the caller MUST NOT + compress — its rotation would race the holder's and split the lineage. + Expired locks and structured holders whose local ``pid=`` is dead are + reclaimed transparently, so a gateway killed mid-compression doesn't + stall its replacement for the full TTL. Single-transaction DELETE-expired + + INSERT-or-IGNORE + SELECT-to-confirm; SQLite serialises writes, so it's atomic.""" + from hermes_state import _compression_lock_holder_process_is_dead + if not session_id: + return False + now = time.time() + expires_at = now + ttl_seconds + + def _do(conn): + reclaimed_holder = None + row = conn.execute( + "SELECT holder, expires_at FROM compression_locks " + "WHERE session_id = ?", + (session_id,), + ).fetchone() + if row is not None: + current_holder = ( + row[0] + ) + current_expires_at = ( + row[1] + ) + if ( + current_expires_at < now + or _compression_lock_holder_process_is_dead(current_holder) + ): + conn.execute( + "DELETE FROM compression_locks " + "WHERE session_id = ? AND holder = ?", + (session_id, current_holder), + ) + reclaimed_holder = current_holder + # INSERT OR IGNORE gives no rowcount signal — verify ownership via SELECT. + conn.execute( + "INSERT OR IGNORE INTO compression_locks " + "(session_id, holder, acquired_at, expires_at) " + "VALUES (?, ?, ?, ?)", + (session_id, holder, now, expires_at), + ) + row = conn.execute( + "SELECT holder FROM compression_locks WHERE session_id = ?", + (session_id,), + ).fetchone() + acquired = row is not None and ( + row[0] + ) == holder + return acquired, reclaimed_holder + + try: + acquired, reclaimed_holder = self._execute_write(_do) + if reclaimed_holder: + logger.warning( + "Reclaimed stale compression lock for session=%s " + "(holder=%s)", + session_id, + reclaimed_holder, + ) + return bool(acquired) + except sqlite3.Error as exc: + logger.warning( + "try_acquire_compression_lock(%s) failed: %s", + session_id, exc, + ) + # False makes the caller skip compression — the safe behaviour + # when the lock subsystem is broken. + return False + + def release_compression_lock(self, session_id: str, holder: str) -> None: + """Release the compression lock for ``session_id`` iff we own it. Idempotent + when the lock is gone or reclaimed; the ``holder`` check stops a late + compressor clobbering someone else's fresh lock.""" + if not session_id: + return + + try: + self._write_sql( + "DELETE FROM compression_locks " + "WHERE session_id = ? AND holder = ?", + (session_id, holder), + ) + except sqlite3.Error as exc: + logger.warning( + "release_compression_lock(%s) failed: %s", + session_id, exc, + ) + + def _session_turn_lease_key_on_conn(self, conn, session_id: str) -> str: + """Walk compression parents on ``conn`` to the conversation lease key. + + Must share the connection of the lease INSERT/UPDATE/DELETE: a failed + ``get_session`` must not yield a child id the write then persists + (refresh would walk to the parent and fail-close). Markers bind to + ``parent_session_id`` (as in ``_NON_CONTINUATION_CHILD_FILTER_SQL``). + Lock errors propagate so ``_execute_write`` / ``acquire_session_turn_lease`` can retry.""" + if not session_id: + return session_id + + def _row(sid: str): + row = conn.execute( + "SELECT id, parent_session_id, source, model_config, end_reason " + "FROM sessions WHERE id = ?", + (sid,), + ).fetchone() + return dict(row) if row else None + + current = _row(session_id) + seen = {session_id} + while current: + parent_id = current.get("parent_session_id") + if ( + not parent_id + or parent_id in seen + or self._is_explicit_fork_child_row(current) + ): + break + parent = _row(parent_id) + if not parent or parent.get("end_reason") != "compression": + break + seen.add(parent_id) + current = parent + return str(current.get("id") or session_id) if current else session_id + + def _session_turn_lease_key(self, session_id: str) -> str: + """Return the stable serialization key for every compression segment. + + Acquire/refresh/release resolve this inside their write transaction; this + is for tests/diagnostics. It does not swallow lock errors — a swallowed + walk plus a later successful write was the fail-open that replayed the + post-rotation refresh miss.""" + if not session_id: + return session_id + with self._read_ctx() as conn: + return self._session_turn_lease_key_on_conn(conn, session_id) + + def try_acquire_session_turn_lease( + self, + session_id: str, + holder: str, + *, + ttl_seconds: float = 300.0, + patience_s: Optional[float] = None, + ) -> bool: + """Atomically acquire the cross-process turn lease for a conversation. + Compression rotates a session into child segments, so the durable key is + the lineage root, not the current segment id. The walk, the INSERT, and + reclaim of expired or dead-local-PID leases share one write transaction.""" + from hermes_state import _compression_lock_holder_process_is_dead + if not session_id or not holder: + return False + now = time.time() + expires_at = now + max(0.1, float(ttl_seconds)) + + def _do(conn): + conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) + row = conn.execute( + "SELECT holder, expires_at FROM session_turn_leases " + "WHERE conversation_id = ?", + (conversation_id,), + ).fetchone() + if row is not None: + current_holder = row["holder"] + if ( + float(row["expires_at"]) <= now + or _compression_lock_holder_process_is_dead(current_holder) + ): + conn.execute( + "DELETE FROM session_turn_leases " + "WHERE conversation_id = ? AND holder = ?", + (conversation_id, current_holder), + ) + conn.execute( + "INSERT OR IGNORE INTO session_turn_leases " + "(conversation_id, holder, acquired_at, expires_at) " + "VALUES (?, ?, ?, ?)", + (conversation_id, holder, now, expires_at), + ) + owner = conn.execute( + "SELECT holder FROM session_turn_leases WHERE conversation_id = ?", + (conversation_id,), + ).fetchone() + return owner is not None and owner["holder"] == holder + + return bool(self._execute_write(_do, patience_s=patience_s)) + + def acquire_session_turn_lease( + self, + session_id: str, + holder: str, + *, + ttl_seconds: float = 300.0, + wait_seconds: float = 1800.0, + poll_interval_seconds: float = 1.0, + on_wait=None, + wait_notice_interval_seconds: float = 15.0, + should_abort=None, + acquire_patience_s: float = 0.5, + ) -> bool: + """Wait for a cross-process turn lease without holding a SQLite lock. + + ``on_wait(elapsed)`` is best-effort: called when the first attempt fails + (elapsed ~0) and about every ``wait_notice_interval_seconds`` after, so + UIs can show another process holds the conversation. ``should_abort()`` + True (e.g. ``/stop``) returns False at once, not after ``wait_seconds``.""" + from hermes_state import classify_persistence_error + deadline = time.monotonic() + max(0.0, float(wait_seconds)) + wait_started = None + last_notice_at = None + notice_every = max(0.0, float(wait_notice_interval_seconds)) + while True: + if should_abort is not None: + try: + if should_abort(): + return False + except Exception: + logger.debug( + "session turn lease should_abort callback failed", + exc_info=True, + ) + try: + if self.try_acquire_session_turn_lease( + session_id, + holder, + ttl_seconds=ttl_seconds, + patience_s=acquire_patience_s, + ): + return True + except sqlite3.Error as exc: + # Long holder transactions (compression publish, large flushes) + # can exhaust one write-patience budget; keep polling until + # wait_seconds or should_abort. + if classify_persistence_error(exc) != "locked": + raise + now = time.monotonic() + remaining = deadline - now + if remaining <= 0: + return False + if wait_started is None: + wait_started = now + if on_wait is not None and ( + last_notice_at is None + or notice_every == 0.0 + or (now - last_notice_at) >= notice_every + ): + try: + on_wait(max(0.0, now - wait_started)) + except Exception: + logger.debug( + "session turn lease on_wait callback failed", + exc_info=True, + ) + last_notice_at = now + time.sleep(min(max(0.01, float(poll_interval_seconds)), remaining)) + + def refresh_session_turn_lease( + self, + session_id: str, + holder: str, + *, + ttl_seconds: float = 300.0, + ) -> bool: + """Extend a turn lease only while ``holder`` still owns it.""" + if not session_id or not holder: + return False + expires_at = time.time() + max(0.1, float(ttl_seconds)) + + def _do(conn): + conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) + cursor = conn.execute( + "UPDATE session_turn_leases SET expires_at = ? " + "WHERE conversation_id = ? AND holder = ?", + (expires_at, conversation_id, holder), + ) + return cursor.rowcount > 0 + + return bool(self._execute_write(_do)) + + def release_session_turn_lease(self, session_id: str, holder: str) -> None: + """Release a turn lease iff ``holder`` still owns it; idempotent.""" + if not session_id or not holder: + return + + def _do(conn): + conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) + conn.execute( + "DELETE FROM session_turn_leases " + "WHERE conversation_id = ? AND holder = ?", + (conversation_id, holder), + ) + + self._execute_write(_do) + + def get_compression_lock_holder(self, session_id: str) -> Optional[str]: + """Return the current (non-expired) holder for ``session_id``, or None. + Diagnostic only — not part of the locking protocol.""" + if not session_id: + return None + now = time.time() + row = self._read_one( + "SELECT holder FROM compression_locks " + "WHERE session_id = ? AND expires_at >= ?", + (session_id, now), + ) + if row is None: + return None + return row[0] + + def finalize_orphaned_compression_sessions(self) -> int: + """Mark orphaned compression continuations (parent ended by compression; + child has messages, no end_reason/ended_at, api_call_count=0) as + ``orphaned_compression``. Non-destructive: messages are preserved.""" + cutoff = time.time() - 604800 # 7 days + + def _do(conn): + now = time.time() + result = conn.execute( + """ + UPDATE sessions + SET ended_at = ?, + end_reason = 'orphaned_compression' + WHERE api_call_count = 0 + AND end_reason IS NULL + AND ended_at IS NULL + AND started_at < ? + AND parent_session_id IS NOT NULL + AND EXISTS ( + SELECT 1 FROM sessions p + WHERE p.id = sessions.parent_session_id + AND p.end_reason = 'compression' + AND p.ended_at IS NOT NULL + ) + AND EXISTS ( + SELECT 1 FROM messages m + WHERE m.session_id = sessions.id + ) + """, + (now, cutoff), + ) + return result.rowcount + + return self._execute_write(_do) or 0 + + def get_compression_chain(self, session_id: str) -> List[str]: + """Walk the compression-continuation chain forward and return every id. + + Root-first, ending at the tip; ``[session_id]`` when no continuation + exists. ``get_compression_tip`` is this walk's last element — one + implementation so the two can never disagree. + + A continuation is a child of a session with ``end_reason='compression'``. + Older builds also required ``child.started_at >= parent.ended_at``; + too brittle — gateway + compression races can insert the real + continuation before the parent's ``ended_at`` is written while a stale + websocket later creates a sibling that passes the timestamp test, so + desktop resume followed the sibling and recent messages looked "lost". + Instead: follow only children of compression-ended parents, exclude + explicit branch/delegate/tool children, and prefer children that continue + the chain (``end_reason='compression'``) or are still live over stale + closed siblings such as ``ws_orphan_reap``.""" + current = session_id + chain = [current] if current else [] + seen = {current} if current else set() + # Defensive bound; chains this deep are pathological. + for _ in range(100): + with self._read_ctx() as conn: + cursor = conn.execute( + f""" + SELECT child.id + FROM sessions parent + JOIN sessions child ON child.parent_session_id = parent.id + WHERE parent.id = ? + AND parent.end_reason = 'compression' + AND json_extract(COALESCE(child.model_config, '{{}}'), '$._branched_from') IS NULL + AND json_extract(COALESCE(child.model_config, '{{}}'), '$._delegate_from') IS NULL + AND COALESCE(child.source, '') != 'tool' + ORDER BY + CASE + WHEN child.end_reason = 'compression' THEN 0 + WHEN child.ended_at IS NULL THEN 1 + ELSE 2 + END, + {_sql_session_last_active("child")} DESC, + child.started_at DESC, + child.id DESC + LIMIT 1 + """, + (current,), + ) + row = cursor.fetchone() + if row is None: + return chain + child_id = row["id"] + if not child_id or child_id in seen: + return chain + seen.add(child_id) + current = child_id + chain.append(child_id) + return chain + + def get_compression_tip(self, session_id: str) -> Optional[str]: + """Live tip of a compression chain (walk semantics: ``get_compression_chain``); + the input id when no continuation exists.""" + chain = self.get_compression_chain(session_id) + return chain[-1] if chain else session_id + + def _is_compression_child_row(self, child: Dict[str, Any]) -> bool: + parent_id = child.get("parent_session_id") + if not parent_id or self._is_explicit_fork_child_row(child): + return False + parent = self.get_session(parent_id) + return bool(parent and parent.get("end_reason") == "compression") + + def get_compression_lineage(self, session_id: str) -> List[str]: + """Return compression ancestors through tip in chronological order.""" + session = self.get_session(session_id) + if not session or self._is_explicit_fork_child_row(session): + return [session_id] if session else [] + + root = session + ancestors = {root["id"]} + while self._is_compression_child_row(root): + parent = self.get_session(root["parent_session_id"]) + if not parent or parent["id"] in ancestors: + break + root = parent + ancestors.add(root["id"]) + + lineage = [root["id"]] + seen = {root["id"]} + current = root + while current.get("end_reason") == "compression": + rows = self._read_all( + """ + SELECT * FROM sessions + WHERE parent_session_id = ? + ORDER BY started_at ASC + """, + (current["id"],), + ) + next_child = None + for row in rows: + candidate = dict(row) + if self._is_compression_child_row(candidate): + next_child = candidate + break + if not next_child or next_child["id"] in seen: + break + lineage.append(next_child["id"]) + seen.add(next_child["id"]) + current = next_child + if current["id"] == session_id: + # Later tips are included only when the requested session itself was compacted. + continue + return lineage if session_id in lineage else [session_id] diff --git a/hermes_state_dbfile.py b/hermes_state_dbfile.py new file mode 100644 index 0000000000..ba0af9d837 --- /dev/null +++ b/hermes_state_dbfile.py @@ -0,0 +1,674 @@ +"""state.db file-level health helpers. + +Split out of ``hermes_state.py``: header probes (application_id / zeroed-file +detection), deleted-WAL-sidecar holder scans, quarantine of zeroed or +lock-poisoned databases, ``collect_state_db_stats`` and holder-process +classification. Every name is re-imported into ``hermes_state`` so +``hermes_state.`` keeps resolving — and tests that monkeypatch it keep +intercepting, because intra-module calls to patched helpers go through a +lazy ``from hermes_state import ...`` at call time. +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import os +import sqlite3 +import struct +import sys +import threading +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple + +from hermes_state_common import ( + FTS_REBUILD_DEFERRAL_KEY, + stat_db_file_identity as _stat_db_file_identity, +) + +# Log-record parity with the origin module (caplog tests pin "hermes_state"). +logger = logging.getLogger("hermes_state") + + +# _read_sqlite_application_id runs on EVERY write via _raise_if_db_replaced, +# against the LIVE state.db. A bare open()/read()/close() there is the +# howtocorrupt §2.2 bug: close() cancels every POSIX advisory lock this +# process holds on the file — measured on Linux/SQLite 3.53.1, one probe call +# drops the WAL-mode DMS shared lock the writer connection holds on state.db +# (see hermes_cli/sqlite_safe_read.py for the module built around this rule). +# With the DMS lock gone, a fresh opener in another process can treat this +# writer as dead and rerun WAL-index recovery underneath it. +# +# The probe therefore reads through a per-path fd cached for the life of the +# process: opening an fd never cancels locks (only close() does), and +# os.pread takes no shared file position. When the path is re-pointed at a +# new inode (the very replacement this probe exists to detect), the stale fd +# is RETIRED, never closed — closing it would cancel the live connection's +# locks on the old file, the exact bug being avoided. Replacement events are +# rare and halt writes anyway, so the leak is bounded. +_HEADER_PROBE_LOCK = threading.Lock() + + +_HEADER_PROBE_FDS: "dict[str, tuple[int, int, int]]" = {} # key -> (fd, dev, ino) + + +_RETIRED_HEADER_PROBE_FDS: "list[int]" = [] # intentionally never closed + + +def _pread_db_header(db_path: Path, length: int) -> "Optional[bytes]": + """Lock-safe raw header read of a possibly-live SQLite database. + + POSIX: pread from a cached, never-closed fd (rebound when the path names + a new inode). Windows: plain read — advisory-lock cancellation is a + POSIX-only hazard and msvcrt locks do not share the failure mode. + """ + from hermes_state import _IS_WINDOWS + if _IS_WINDOWS: + try: + with db_path.open("rb") as handle: + return handle.read(length) + except OSError: + return None + key = str(db_path) + try: + st = os.stat(db_path) + except OSError: + return None + with _HEADER_PROBE_LOCK: + cached = _HEADER_PROBE_FDS.get(key) + if cached is not None and (cached[1], cached[2]) != (st.st_dev, st.st_ino): + # Path re-pointed at a new file. Retire (never close) the old fd. + _RETIRED_HEADER_PROBE_FDS.append(cached[0]) + cached = None + del _HEADER_PROBE_FDS[key] + if cached is None: + try: + fd = os.open(db_path, os.O_RDONLY) + except OSError: + return None + try: + fst = os.fstat(fd) + except OSError: + _RETIRED_HEADER_PROBE_FDS.append(fd) + return None + cached = (fd, fst.st_dev, fst.st_ino) + _HEADER_PROBE_FDS[key] = cached + try: + return os.pread(cached[0], length, 0) + except OSError: + return None + + +def _read_sqlite_application_id(db_path: Path) -> "Optional[int]": + """Read application_id from the SQLite header without opening a connection. + + Safe against live databases: routed through :func:`_pread_db_header`, + which never issues a ``close()`` that would cancel this process's POSIX + locks on the file (howtocorrupt §2.2). + """ + from hermes_state import _STATE_DB_APPLICATION_ID_OFFSET + header = _pread_db_header(db_path, _STATE_DB_APPLICATION_ID_OFFSET + 4) + if header is None: + return None + if len(header) < _STATE_DB_APPLICATION_ID_OFFSET + 4: + return None + if header[:16] != b"SQLite format 3\x00": + return None + return int( + struct.unpack( + ">I", + header[_STATE_DB_APPLICATION_ID_OFFSET:_STATE_DB_APPLICATION_ID_OFFSET + 4], + )[0] + ) + + +def _stat_sqlite_sidecar_identity(db_path: Path) -> Dict[str, tuple]: + """Snapshot ``(st_dev, st_ino)`` for existing WAL/SHM sidecars.""" + identities: Dict[str, tuple] = {} + base = os.fspath(db_path) + for suffix in ("-wal", "-shm"): + ident = _stat_db_file_identity(Path(base + suffix)) + if ident is not None: + identities[suffix] = ident + return identities + + +def _canonical_sqlite_path(path: str) -> str: + """Normalize a /proc fd target, stripping the Linux `` (deleted)`` suffix.""" + return os.path.normcase(os.path.abspath(path.removesuffix(" (deleted)"))) + + +def _watched_sqlite_sidecar_paths(db_path) -> Set[str]: + base = os.path.abspath(os.fspath(db_path)) + return { + _canonical_sqlite_path(base + "-wal"), + _canonical_sqlite_path(base + "-shm"), + } + + +def iter_deleted_sqlite_sidecar_holders(db_path) -> List[Tuple[int, str]]: + """Return processes holding an unlinked ``state.db-wal`` / ``-shm``. + + Linux-only (``/proc//fd`` readlink). Windows and other hosts + return ``[]`` — Windows cannot unlink a sidecar another process still + holds, and macOS does not use the `` (deleted)`` suffix. + + The scan includes this process: on the SessionDB open/write refuse + path, the in-process writer that still holds the orphan inode is the + one that must not mint a replacement WAL (and must stop committing). + ``_foreign_state_db_holders`` keeps skipping this PID for FTS + maintenance so a process does not block its own optional repair. + """ + if not sys.platform.startswith("linux"): + return [] + + holders: List[Tuple[int, str]] = [] + watched = _watched_sqlite_sidecar_paths(db_path) + try: + for pid_str in os.listdir("/proc"): + if not pid_str.isdigit(): + continue + pid = int(pid_str) + fd_dir = f"/proc/{pid}/fd" + try: + fds = os.listdir(fd_dir) + except OSError: + continue + for fd in fds: + try: + target = os.readlink(f"{fd_dir}/{fd}") + except OSError: + continue + if " (deleted)" not in target: + continue + if _canonical_sqlite_path(target) in watched: + holders.append((pid, target)) + except Exception as exc: + logger.debug("deleted-WAL holder scan failed for %s: %s", db_path, exc) + return holders + return holders + + +def refuse_deleted_wal_generation(db_path) -> None: + """Raise if any process holds a deleted WAL/SHM generation for *db_path*. + + Called *before* ``sqlite3.connect`` so a second opener cannot mint a + replacement WAL inode while a live writer still holds the orphan. + """ + from hermes_state import DeletedWalGenerationError, _DELETED_WAL_GENERATION_MSG + holders = iter_deleted_sqlite_sidecar_holders(db_path) + if not holders: + return + logger.error(_DELETED_WAL_GENERATION_MSG) + raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG) + + +def _connect_tracked_db(path, tracking_path=None, **kwargs): + """``sqlite3.connect`` that registers the open fd for lock-safety. + + While a connection is live, byte-level probes of the same file are + refused: an ``open()``/``close()`` cancels every POSIX advisory lock this + process holds on it -- including a running VACUUM's EXCLUSIVE lock. + Released automatically on ``close()``. + + The ONLY tolerated fallback is the helper being absent entirely + (scaffold/embed installs that ship hermes_state without hermes_cli). A + real connection failure must propagate: silently retrying an *untracked* + connect would disable the guard for the lifetime of that connection, + which is precisely the failure mode this module exists to prevent. + """ + try: + from hermes_cli.sqlite_safe_read import connect_tracked + except ImportError: + logger.debug( + "hermes_cli.sqlite_safe_read unavailable; opening %s untracked " + "(byte-probe guard inactive in this install)", + path, + ) + return sqlite3.connect(str(path), **kwargs) + + # Open through THIS module's sqlite3.connect so callers (and tests) that + # patch hermes_state.sqlite3.connect keep control of connection creation; + # the helper still owns tracking. + return connect_tracked( + path, + tracking_path=tracking_path, + connect_fn=sqlite3.connect, + **kwargs, + ) + + +def is_zeroed_state_db( + path: Path, *, probe_bytes: int = 100, force: bool = False +) -> bool: + """Detect the #68474/#97568 zeroed state.db signature (0-byte or NUL header). + + Byte-level probe, so it is only safe BEFORE any connection to *path* + exists in this process: ``close()`` cancels every POSIX advisory lock the + process holds on the file, which can pull the EXCLUSIVE lock out from + under a running VACUUM and corrupt the database. The read is routed + through ``read_header_bytes_preopen``, which refuses (returning False + here) once a connection is live. Pass ``force=True`` only for offline + files -- quarantined copies, snapshots, archives. + + Prefer ``hermes_cli.backup.is_zeroed_sqlite_file`` when available; this + local copy keeps SessionDB openable without importing the CLI package + in constrained embed paths. + """ + try: + from hermes_cli.backup import is_zeroed_sqlite_file + + return is_zeroed_sqlite_file(path, probe_bytes=probe_bytes, force=force) + except Exception: + pass + try: + if not path.is_file(): + # Special files (FIFO, device, socket) are never "zeroed", and + # probing a FIFO would block until a writer appears. + return False + size = path.stat().st_size + except OSError: + return False + if size < 0: + return False + from hermes_cli.sqlite_safe_read import has_live_connection, read_header_bytes_preopen + + if not force and has_live_connection(path): + return False + + head = read_header_bytes_preopen( + path, length=max(16, probe_bytes), force=force + ) + if head is None: + return False + if len(head) == 0: + return True + if head.startswith(b"SQLite format 3"): + return False + return all(byte == 0 for byte in head) + + +@contextlib.contextmanager +def quarantine_cross_process_lock(path: Path, timeout: float = 5.0): + """Acquire the cross-process lock for path.quarantine.lock.""" + import platform + + lock_path = path.with_name(path.name + ".quarantine.lock") + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = lock_path.open("a+b") + acquired = False + try: + deadline = time.monotonic() + timeout + if platform.system() == "Windows": + import msvcrt + + while True: + try: + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + acquired = True + break + except OSError: + if time.monotonic() >= deadline: + break + time.sleep(0.020) + else: + import fcntl + + while True: + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + acquired = True + break + except (BlockingIOError, OSError): + if time.monotonic() >= deadline: + break + time.sleep(0.020) + yield acquired + finally: + try: + if acquired: + if platform.system() == "Windows": + import msvcrt + + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + except (OSError, AttributeError): + pass + finally: + handle.close() + + +def quarantine_zeroed_state_db( + path: Path, *, already_locked: bool = False +) -> Optional[Path]: + """Move a zeroed state.db aside (preserve bytes) and return quarantine path. + + Uses a cross-process lock (``#68805``) so two concurrent startups cannot + race: the first process moves the zeroed file and the second re-checks + under the lock, finding the file already gone (or a fresh DB in its place) + instead of clobbering the quarantine. + """ + def _do_quarantine(): + if not path.exists(): + logger.info( + "quarantine_zeroed_state_db: %s already moved by another process", + path, + ) + return None + if not is_zeroed_state_db(path): + logger.info( + "quarantine_zeroed_state_db: %s is no longer zeroed (another " + "process quarantined it and a fresh DB was created)", + path, + ) + return None + + try: + ts = time.strftime("%Y%m%d-%H%M%S") + except Exception: + ts = "unknown" + dest = path.with_name( + f"{path.name}.zeroed-{ts}-{os.getpid()}.bak" + ) + n = 0 + while dest.exists(): + n += 1 + dest = path.with_name( + f"{path.name}.zeroed-{ts}-{os.getpid()}-{n}.bak" + ) + try: + path.rename(dest) + except OSError as exc: + logger.error("Failed to quarantine zeroed %s: %s", path, exc) + return None + for suffix in ("-wal", "-shm"): + side = Path(str(path) + suffix) + if side.exists(): + try: + side.rename(Path(str(dest) + suffix)) + except OSError: + pass + return dest + + if already_locked: + return _do_quarantine() + + with quarantine_cross_process_lock(path) as acquired: + if not acquired: + logger.error( + "quarantine lock for %s not acquired within 5s — refusing to " + "quarantine without the cross-process lock. The zeroed file " + "is left in place. If sessions fail to load, restore from " + "state-snapshots via `hermes snapshot list` / " + "`hermes snapshot restore `.", + path, + ) + return None + return _do_quarantine() + + +def collect_state_db_stats(db_path: Path) -> Dict[str, Any]: + """Best-effort, strictly read-only stats snapshot of a state.db file. + + Opens the database with ``mode=ro`` (URI) and a short timeout so it can + run against a *live* database held by a gateway without ever taking a + write lock or mutating the file. Every field is collected independently: + a failed pragma/SELECT yields ``None`` for that field, and the helper + itself never raises. + + Deliberately does NOT instantiate :class:`SessionDB` — its constructor + runs schema DDL (migrations, FTS table creation), which is exactly the + kind of write a diagnostics probe must never perform. + + Returned keys (all present, any may be None on failure): + + - ``page_count``, ``page_size``, ``freelist_count`` — PRAGMA values + - ``logical_size_bytes`` — page_count * page_size (post-checkpoint size) + - ``wal_size_bytes`` — stat() of ``-wal`` (0 when absent) + - ``journal_mode`` — PRAGMA journal_mode string + - ``messages`` / ``sessions`` — row counts + - ``fts_tables`` — dict of {table_name: bool} presence for + messages_fts / messages_fts_trigram / messages_fts_cjk + - ``fts_storage_version`` — int from state_meta, None when the marker is + absent (legacy pre-v23 inline layout) + - ``fts_rebuild_pending`` — True when the deferred v23 backfill has not + finished (high_water present and progress < high_water) + - ``fts_rebuild_high_water`` / ``fts_rebuild_progress`` — raw ints + - ``fts_rebuild_deferral`` — durable blocked-repair diagnostic, when present + """ + from hermes_state import _connect_tracked_db + stats: Dict[str, Any] = { + "page_count": None, + "page_size": None, + "freelist_count": None, + "logical_size_bytes": None, + "wal_size_bytes": None, + "journal_mode": None, + "messages": None, + "sessions": None, + "fts_tables": None, + "fts_storage_version": None, + "fts_rebuild_pending": None, + "fts_rebuild_high_water": None, + "fts_rebuild_progress": None, + "fts_rebuild_deferral": None, + } + + # WAL sidecar size needs no connection at all. + try: + wal_path = Path(str(db_path) + "-wal") + stats["wal_size_bytes"] = wal_path.stat().st_size if wal_path.exists() else 0 + except OSError: + pass + + conn = None + try: + # mode=ro refuses to create the file and refuses every write; a + # short timeout keeps doctor snappy when a writer holds the lock. + # Route through the tracked connect so byte-probe helpers + # (read_header_bytes_preopen) see this connection and refuse raw + # opens that could cancel our POSIX locks mid-read. + conn = _connect_tracked_db( + f"file:{Path(db_path)}?mode=ro", + tracking_path=Path(db_path), + uri=True, + timeout=2.0, + ) + except Exception as exc: + logger.debug("collect_state_db_stats: cannot open %s read-only: %s", + db_path, exc) + return stats + + def _scalar(sql: str) -> Any: + try: + row = conn.execute(sql).fetchone() + return row[0] if row else None + except Exception: + return None + + try: + pc = _scalar("PRAGMA page_count") + ps = _scalar("PRAGMA page_size") + stats["page_count"] = int(pc) if pc is not None else None + stats["page_size"] = int(ps) if ps is not None else None + if stats["page_count"] is not None and stats["page_size"] is not None: + stats["logical_size_bytes"] = stats["page_count"] * stats["page_size"] + + fl = _scalar("PRAGMA freelist_count") + stats["freelist_count"] = int(fl) if fl is not None else None + + jm = _scalar("PRAGMA journal_mode") + stats["journal_mode"] = str(jm) if jm is not None else None + + msgs = _scalar("SELECT COUNT(*) FROM messages") + stats["messages"] = int(msgs) if msgs is not None else None + sess = _scalar("SELECT COUNT(*) FROM sessions") + stats["sessions"] = int(sess) if sess is not None else None + + # FTS table presence via sqlite_master (never SELECTs from the + # virtual tables themselves — a corrupt index must not fail stats). + try: + names = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' " + "AND name IN (?, ?, ?)", + ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"), + ).fetchall() + } + stats["fts_tables"] = { + t: (t in names) + for t in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk") + } + except Exception: + pass + + # Raw state_meta reads — cheap, and independent of SessionDB. + def _meta_int(key: str) -> Optional[int]: + try: + row = conn.execute( + "SELECT value FROM state_meta WHERE key = ?", (key,) + ).fetchone() + return int(row[0]) if row and row[0] is not None else None + except Exception: + return None + + stats["fts_storage_version"] = _meta_int("fts_storage_version") + high_water = _meta_int("fts_rebuild_high_water") + progress = _meta_int("fts_rebuild_progress") + stats["fts_rebuild_high_water"] = high_water + stats["fts_rebuild_progress"] = progress + if high_water is None: + stats["fts_rebuild_pending"] = False + else: + stats["fts_rebuild_pending"] = (progress or 0) < high_water + try: + row = conn.execute( + "SELECT value FROM state_meta WHERE key = ? LIMIT 1", + (FTS_REBUILD_DEFERRAL_KEY,), + ).fetchone() + if row: + parsed = json.loads(row[0]) + if isinstance(parsed, dict): + stats["fts_rebuild_deferral"] = parsed + except Exception: + pass + finally: + try: + conn.close() + except Exception: + pass + + return stats + + +def count_db_holders(db_path: Path) -> Optional[int]: + """Best-effort count of processes holding ``db_path`` open (Linux only). + + Scans ``/proc/*/fd`` symlinks for the resolved database path. Returns + the number of distinct PIDs with the file open, or ``None`` on any + error or on non-Linux platforms. Never raises; no lsof dependency. + Unreadable per-process fd dirs (other users' processes without root) + are silently skipped, so the count is a lower bound. + """ + try: + if not sys.platform.startswith("linux"): + return None + target = os.path.realpath(str(db_path)) + holders = 0 + for pid in os.listdir("/proc"): + if not pid.isdigit(): + continue + fd_dir = f"/proc/{pid}/fd" + try: + fds = os.listdir(fd_dir) + except OSError: + continue # process gone or not ours + for fd in fds: + try: + if os.readlink(f"{fd_dir}/{fd}") == target: + holders += 1 + break # one hit per PID + except OSError: + continue + return holders + except Exception: + return None + + +def _is_inactive_orphan_desktop_holder( + *, + ppid: int, + age_seconds: float, + min_age_seconds: float, + ephemeral_backend: bool, + connection_statuses: List[str], +) -> bool: + """Pure safety predicate for the narrow Desktop holder reap.""" + return ( + ppid in (0, 1) + and age_seconds >= min_age_seconds + and ephemeral_backend + and "ESTABLISHED" not in connection_statuses + ) + + +def _concrete_state_db_holder_pids( + db_path: Path, holders: List[Tuple[int, str]] +) -> List[int]: + """Return unique PIDs proven to hold this DB or one of its sidecars.""" + canonical_db = os.path.normcase(os.path.abspath(os.fspath(db_path))) + watched = { + canonical_db, + canonical_db + "-wal", + canonical_db + "-shm", + } + pids: List[int] = [] + seen = set() + for pid, path in holders: + canonical_path = os.path.normcase( + os.path.abspath(path.removesuffix(" (deleted)")) + ) + if pid <= 0 or pid in seen or canonical_path not in watched: + continue + seen.add(pid) + pids.append(pid) + return pids + + +def _read_proc_cmdline(pid: int) -> Optional[str]: + """Read /proc//cmdline, world-readable even when fd table is not. + + Returns the cmdline as a space-joined string, or None when unreadable + (process exited, or hidepid mount). + """ + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + raw = f.read() + if not raw: + return None + return raw.replace(b"\x00", b" ").decode("utf-8", "replace").strip() + except OSError: + return None + + +_HERMES_CMDLINE_MARKERS = ("hermes_cli.main", "hermes_cli/main", "hermes serve", + "hermes-agent", "hermes gateway", "hermes chat") + + +def _looks_like_hermes(cmdline: str) -> bool: + """Heuristic: does this cmdline look like a Hermes process? + + Used to decide whether an uninspectable process (fd table unreadable + due to different user) should be treated as a potential state.db holder. + We only flag processes that look like Hermes, not every system daemon. + """ + lower = cmdline.lower() + return any(marker in lower for marker in _HERMES_CMDLINE_MARKERS) diff --git a/hermes_state_gateway.py b/hermes_state_gateway.py new file mode 100644 index 0000000000..f74b7dcce0 --- /dev/null +++ b/hermes_state_gateway.py @@ -0,0 +1,1005 @@ +"""Gateway-facing SessionDB persistence: routing index, peers, orphans, heartbeats, handoffs. + +Mixin bound onto ``SessionDB`` via the MRO; built on its ``_read_ctx`` / +``_execute_write`` / ``_write_sql`` / ``_read_all`` primitives. +""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +import sys +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple + +from hermes_state_common import ( + _RECOVERABLE_END_REASONS_SQL, + _RESET_END_REASONS_SQL, + _sql_session_last_active, +) + +# Log-record parity with the origin module (caplog tests pin "hermes_state"). +logger = logging.getLogger("hermes_state") + + +class SessionGatewayMixin: + """Routing index, session peers/orphans, hygiene streaks, heartbeats, handoffs.""" + + def _reap_inactive_orphan_desktop_holders( + self, holders: List[Tuple[int, str]], *, min_age_seconds: float + ) -> List[int]: + """Terminate old PPID-1 Desktop ephemeral backends with no client. + + Fails closed: anything whose parent, age, argv, or network connections + cannot be proved safe remains a repair-blocking holder. + """ + from hermes_state import _concrete_state_db_holder_pids, _is_inactive_orphan_desktop_holder, psutil + if not sys.platform.startswith("linux") or psutil is None: + return [] + try: + from hermes_cli.dashboard_procs import _is_ephemeral_port_zero_backend + except Exception: + return [] + + now = time.time() + candidates = [] + for pid in _concrete_state_db_holder_pids(self.db_path, holders): + try: + process = psutil.Process(pid) + statuses = [ + conn.status for conn in process.net_connections(kind="inet") + ] + if not _is_inactive_orphan_desktop_holder( + ppid=process.ppid(), + age_seconds=now - process.create_time(), + min_age_seconds=min_age_seconds, + ephemeral_backend=_is_ephemeral_port_zero_backend(process.cmdline()), + connection_statuses=statuses, + ): + continue + except Exception: + continue + candidates.append(process) + + signalled: List[int] = [] + for process in candidates: + try: + process.terminate() + signalled.append(process.pid) + except (psutil.Error, OSError): + continue + if not signalled: + return [] + + try: + _gone, alive = psutil.wait_procs(candidates, timeout=1.5) + except Exception: + alive = [] + for process in alive: + try: + process.kill() + except (psutil.Error, OSError): + continue + if alive: + try: + psutil.wait_procs(alive, timeout=1.5) + except Exception: + pass + return signalled + + def record_gateway_session_peer( + self, + session_id: str, + *, + source: str, + user_id: str = None, + session_key: str = None, + chat_id: str = None, + chat_type: str = None, + thread_id: str = None, + display_name: str = None, + origin_json: str = None, + include_compression_ancestors: bool = False, + ) -> None: + """Persist the gateway routing peer for an existing session row. + + ``display_name`` / ``origin_json`` let consumers (mcp_serve, mirror, + channel directory) read routing data from state.db instead of + sessions.json; ``None`` leaves the existing value untouched. + ``include_compression_ancestors`` keeps a compression lineage on one + routing peer when an explicit resume moves its tip to another lane; + normal per-turn refreshes update only the supplied row. + + Self-healing: a missing target row (deferred ``create_session`` write, + or crash between routing publication and row creation) is INSERTed + with full identity rather than silently no-opped, so a gateway row can + never be first-created by an identity-less lazy writer + (``update_token_counts``) and stay unroutable forever. + """ + if not session_id or not session_key: + return + + def _do(conn): + lineage_cte = "" + target_clause = "WHERE id = ?" + query_params = [] + if include_compression_ancestors: + lineage_cte = """ + WITH RECURSIVE compression_lineage(id) AS ( + SELECT ? + UNION + SELECT parent.id + FROM compression_lineage lineage + JOIN sessions child ON child.id = lineage.id + JOIN sessions parent ON parent.id = child.parent_session_id + WHERE parent.end_reason = 'compression' + AND json_extract( + COALESCE(child.model_config, '{}'), + '$._branched_from' + ) IS NULL + AND json_extract( + COALESCE(child.model_config, '{}'), + '$._delegate_from' + ) IS NULL + AND COALESCE(child.source, '') != 'tool' + ) + """ + target_clause = "WHERE id IN (SELECT id FROM compression_lineage)" + query_params.append(session_id) + query_params.extend( + ( + session_key, + source, + user_id, + chat_id, + chat_type, + thread_id, + display_name, + origin_json, + ) + ) + if not include_compression_ancestors: + query_params.append(session_id) + conn.execute( + f"""{lineage_cte} + UPDATE sessions + SET session_key = ?, source = ?, user_id = ?, chat_id = ?, + chat_type = ?, thread_id = ?, + display_name = COALESCE(?, display_name), + origin_json = COALESCE(?, origin_json) + {target_clause}""", + query_params, + ) + # Self-heal: the UPDATE silently no-ops on a missing row — insert it + # with full identity so the session is durably routable. + if not include_compression_ancestors: + cur = conn.execute( + "SELECT 1 FROM sessions WHERE id = ? LIMIT 1", (session_id,) + ) + if cur.fetchone() is None: + conn.execute( + """INSERT INTO sessions ( + id, source, user_id, session_key, chat_id, + chat_type, thread_id, display_name, origin_json, + profile_name, started_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + session_key = COALESCE(sessions.session_key, excluded.session_key), + chat_id = COALESCE(sessions.chat_id, excluded.chat_id), + chat_type = COALESCE(sessions.chat_type, excluded.chat_type), + thread_id = COALESCE(sessions.thread_id, excluded.thread_id), + display_name = COALESCE(sessions.display_name, excluded.display_name), + origin_json = COALESCE(sessions.origin_json, excluded.origin_json)""", + ( + session_id, + source, + user_id, + session_key, + chat_id, + chat_type, + thread_id, + display_name, + origin_json, + # Same ownership stamp as _insert_session_row: an + # unowned (NULL) row vanishes from profile-keyed consumers. + self._own_profile_name(), + time.time(), + ), + ) + + self._execute_write(_do) + + def save_gateway_routing_entry( + self, session_key: str, entry_json: str, *, scope: str = "" + ) -> None: + """Upsert one gateway routing entry (session_key -> SessionEntry JSON). + + ``gateway_routing`` durably replaces sessions.json. ``scope`` namespaces + the index per sessions_dir so two stores never share routing state. + """ + if not session_key or not entry_json: + return + + self._write_sql( + """INSERT INTO gateway_routing (scope, session_key, entry_json, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(scope, session_key) DO UPDATE SET + entry_json = excluded.entry_json, + updated_at = excluded.updated_at""", + (scope, session_key, entry_json, time.time()), + ) + + def replace_gateway_routing_entries( + self, entries: Dict[str, str], *, scope: str = "" + ) -> None: + """Atomically replace the routing index for *scope* with *entries*. + + Full-rewrite semantics: keys absent from *entries* are removed. One + write transaction; other scopes untouched. + """ + now = time.time() + + def _do(conn): + conn.execute("DELETE FROM gateway_routing WHERE scope = ?", (scope,)) + if entries: + conn.executemany( + "INSERT INTO gateway_routing (scope, session_key, entry_json, updated_at) " + "VALUES (?, ?, ?, ?)", + [(scope, k, v, now) for k, v in entries.items() if k and v], + ) + + self._execute_write(_do) + + def load_gateway_routing_entries(self, *, scope: str = "") -> Dict[str, str]: + """Load routing entries for *scope* as {session_key: entry_json}.""" + rows = self._read_all( + "SELECT session_key, entry_json FROM gateway_routing WHERE scope = ?", + (scope,), + ) + return {r["session_key"]: r["entry_json"] for r in rows} + + def list_never_active_keyed_sessions( + self, *, older_than_days: float + ) -> List[Dict[str, Any]]: + """Keyed gateway rows that were opened and then never used at all. + + Keyed, still-open rows with no evidence of a single turn (no messages, + tokens, tool/API calls, activity, or title): a leaked test fixture or a + chat routed but never answered. Safe to drop — no transcript to lose, + and the gateway mints a fresh session on the next inbound message. + Needs its own selector because ``bulk prune``/``archive`` are pinned to + ``ended_at IS NOT NULL`` (never pick a live session), which excludes + every never-closed row. ``pinned``/``archived`` = explicit keep intent. + """ + cutoff = time.time() - (float(older_than_days) * 86400.0) + rows = self._read_all( + """ + SELECT s.id, s.session_key, s.source, s.chat_id, + s.chat_type, s.user_id, s.started_at + FROM sessions s + WHERE s.session_key IS NOT NULL + AND s.ended_at IS NULL + AND s.title IS NULL + AND s.last_activity_at IS NULL + AND COALESCE(s.message_count, 0) = 0 + AND COALESCE(s.tool_call_count, 0) = 0 + AND COALESCE(s.api_call_count, 0) = 0 + AND COALESCE(s.input_tokens, 0) = 0 + AND COALESCE(s.output_tokens, 0) = 0 + AND COALESCE(s.pinned, 0) = 0 + AND COALESCE(s.archived, 0) = 0 + AND s.started_at IS NOT NULL + AND s.started_at < ? + AND NOT EXISTS ( + SELECT 1 FROM messages m WHERE m.session_id = s.id + ) + ORDER BY s.started_at + """, + (cutoff,), + ) + return [dict(r) for r in rows] + + def _delete_routing_entries_for_sessions(self, session_ids: Set[str]) -> int: + """Drop ``gateway_routing`` rows pointing at any of *session_ids*. + + The target id lives only inside ``entry_json``, so matching is done in + Python over all scopes. + """ + if not session_ids: + return 0 + rows = self._read_all("SELECT scope, session_key, entry_json FROM gateway_routing") + doomed: List[Tuple[str, str]] = [] + for row in rows: + try: + entry = json.loads(row["entry_json"] or "{}") + except Exception: + continue + if isinstance(entry, dict) and entry.get("session_id") in session_ids: + doomed.append((row["scope"], row["session_key"])) + if not doomed: + return 0 + + self._write_sql( + "DELETE FROM gateway_routing WHERE scope = ? AND session_key = ?", + doomed, + many=True, + ) + return len(doomed) + + def prune_never_active_keyed_sessions( + self, + *, + older_than_days: float, + sessions_dir: Optional[Path] = None, + ) -> Tuple[int, int]: + """Delete never-active keyed rows and the routing entries naming them. + + Returns ``(sessions_deleted, routing_entries_deleted)``. Routing + entries go first: a stale entry outliving its target would have the + gateway resume a nonexistent session id. Deletion goes through + :meth:`delete_session` so the delegate cascade, FTS bookkeeping and + transcript cleanup stay owned by one implementation. + """ + candidates = self.list_never_active_keyed_sessions( + older_than_days=older_than_days + ) + if not candidates: + return (0, 0) + ids = {str(row["id"]) for row in candidates} + routing_deleted = self._delete_routing_entries_for_sessions(ids) + deleted = 0 + for session_id in ids: + if self.delete_session(session_id, sessions_dir=sessions_dir): + deleted += 1 + return (deleted, routing_deleted) + + def list_gateway_sessions( + self, + *, + platform: Optional[str] = None, + active_only: bool = True, + ) -> List[Dict[str, Any]]: + """List gateway sessions (rows with a session_key): newest row per key, + one live mapping per routing key. ``platform`` filters on ``source``.""" + # Full rows carry token/cost totals — drain queued async accounting + # deltas so consumers see exact counters. + self.flush_token_counts() + query = f""" + SELECT sessions.*, + COALESCE(sp.prompt, sessions.system_prompt) + AS _system_prompt_resolved, + {_sql_session_last_active("sessions")} AS last_active + FROM sessions + LEFT JOIN system_prompts sp + ON sp.hash = sessions.system_prompt_hash + WHERE session_key IS NOT NULL + AND started_at = ( + SELECT MAX(s2.started_at) FROM sessions s2 + WHERE s2.session_key = sessions.session_key + ) + """ + params: list = [] + if platform: + query += " AND LOWER(source) = LOWER(?)" + params.append(platform) + if active_only: + query += " AND ended_at IS NULL" + query += " ORDER BY last_active DESC" + rows = self._read_all(query, params) + return [self._session_row_dict(r) for r in rows] + + def find_latest_gateway_session_for_peer( + self, + *, + source: str, + user_id: Optional[str] = None, + session_key: Optional[str] = None, + chat_id: Optional[str] = None, + chat_type: Optional[str] = None, + thread_id: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Find the latest recoverable gateway session for a routing peer. + + ``sessions.json`` is the fast index but can be missing or pruned; the + durable ``session_key`` on the row rebuilds the mapping exactly. Rows + ended only by the old ``agent_close`` bug or a mistaken TUI + ``ws_orphan_reap`` are recoverable; explicit boundaries (/new, /resume + switches, compression splits) are not. + + Ordering: rank by ``last_activity_at`` (falling back to ``started_at``) + — ``started_at`` alone resurrected days-old zombie rows over the live + conversation. Rows with messages win, but an empty keyed row is still + returned rather than ``None``: ``None`` mints a brand-new session id, + worse than resuming an empty-but-correctly-keyed row (whose transcript + may live under a compression child). + + Reset fence: an intentional boundary (``session_reset`` or any + non-recoverable end_reason) must block fallback to an *older* row for + the same peer, or the has-messages ranking could reach behind a /new + and restore the exact context the user reset — so a candidate is + rejected when a peer boundary row ended *after* its last activity. + """ + if not session_key: + return None + with self._read_ctx() as conn: + row = conn.execute( + f""" + SELECT s.*, + COALESCE(sp.prompt, s.system_prompt) + AS _system_prompt_resolved, + (COALESCE(s.message_count, 0) > 0 OR EXISTS ( + SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1 + )) AS _has_messages + FROM sessions s + LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash + WHERE s.session_key = ? + AND s.source = ? + AND (s.ended_at IS NULL OR s.end_reason IN ({_RECOVERABLE_END_REASONS_SQL})) + AND NOT EXISTS ( + SELECT 1 FROM sessions b + WHERE b.session_key = s.session_key + AND b.source = s.source + AND b.ended_at IS NOT NULL + AND b.end_reason IN ({_RESET_END_REASONS_SQL}) + AND b.ended_at + > COALESCE(s.last_activity_at, s.started_at) + ) + ORDER BY _has_messages DESC, + COALESCE(s.last_activity_at, s.started_at) DESC + LIMIT 1 + """, + (session_key, source), + ).fetchone() + if row is not None: + return self._session_row_dict(row) + + # Conservative fallback for a temporarily-missing exact key: still + # require the complete peer tuple so we never cross chats/threads/users. + if chat_id is None or chat_type is None: + return None + # Profile fence: a Telegram DM's peer tuple is identical for every + # bot (chat_id == user_id, no thread), so a sibling profile's legacy + # row would otherwise be adopted. A row is ours when profile_name is + # the owner or NULL (legacy rows this store minted); stores outside + # the profile tree derive no owner and stay unfenced. + owner = self._own_profile_name() + row = conn.execute( + f""" + SELECT s.*, + COALESCE(sp.prompt, s.system_prompt) + AS _system_prompt_resolved, + (COALESCE(s.message_count, 0) > 0 OR EXISTS ( + SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1 + )) AS _has_messages + FROM sessions s + LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash + WHERE s.source = ? + AND COALESCE(s.user_id, '') = COALESCE(?, '') + AND COALESCE(s.chat_id, '') = COALESCE(?, '') + AND COALESCE(s.chat_type, '') = COALESCE(?, '') + AND COALESCE(s.thread_id, '') = COALESCE(?, '') + AND (? IS NULL OR COALESCE(s.profile_name, ?) = ?) + AND (s.ended_at IS NULL OR s.end_reason IN ({_RECOVERABLE_END_REASONS_SQL})) + AND (COALESCE(s.message_count, 0) > 0 OR EXISTS ( + SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1 + )) + AND NOT EXISTS ( + SELECT 1 FROM sessions b + WHERE b.source = s.source + AND COALESCE(b.user_id, '') = COALESCE(s.user_id, '') + AND COALESCE(b.chat_id, '') = COALESCE(s.chat_id, '') + AND COALESCE(b.chat_type, '') = COALESCE(s.chat_type, '') + AND COALESCE(b.thread_id, '') = COALESCE(s.thread_id, '') + AND b.ended_at IS NOT NULL + AND b.end_reason IN ({_RESET_END_REASONS_SQL}) + AND b.ended_at + > COALESCE(s.last_activity_at, s.started_at) + ) + ORDER BY COALESCE(s.last_activity_at, s.started_at) DESC + LIMIT 1 + """, + (source, user_id, chat_id, chat_type, thread_id, owner, owner, owner), + ).fetchone() + return self._session_row_dict(row) if row else None + + def find_orphaned_gateway_sessions( + self, *, max_gap_s: Optional[float] = None + ) -> List[Dict[str, Any]]: + """Report message-bearing session rows that lost their routing identity. + + A candidate orphan has messages but no ``session_key``; it is + *adoptable* only when exactly one keyed predecessor can be named: + + * ``lineage`` — ``parent_session_id`` points at a keyed row of the + same source (a recorded fact; no time window). + * ``contiguity`` — exactly one keyed row of the same source (and + compatible ``user_id``) fell quiet within *max_gap_s* of the + orphan's start, and is older than the orphan's own last activity. + + Ambiguity is reported ``adoptable=False`` with a reason, never guessed: + mis-adopting splices one person's conversation into another's chat. + Branch/delegate/tool rows are excluded — unkeyed by design, not damage. + """ + gap = ( + self._ORPHAN_ADOPTION_MAX_GAP_S + if max_gap_s is None + else float(max_gap_s) + ) + orphan_active = _sql_session_last_active("o") + donor_active = _sql_session_last_active("d") + donor_columns = ( + "d.id, d.session_key, d.chat_id, d.chat_type, d.thread_id, " + "d.user_id, d.origin_json, d.display_name, d.end_reason" + ) + records: List[Dict[str, Any]] = [] + + with self._read_ctx() as conn: + orphans = conn.execute( + f""" + SELECT o.id, o.source, o.user_id, o.started_at, + o.parent_session_id, + {orphan_active} AS last_active, + (SELECT COUNT(*) FROM messages m + WHERE m.session_id = o.id) AS message_count + FROM sessions o + WHERE o.session_key IS NULL + AND EXISTS (SELECT 1 FROM messages m + WHERE m.session_id = o.id) + AND COALESCE(o.source, '') != 'tool' + AND json_extract(COALESCE(o.model_config, '{{}}'), + '$._branched_from') IS NULL + AND json_extract(COALESCE(o.model_config, '{{}}'), + '$._delegate_from') IS NULL + ORDER BY o.started_at ASC + """ + ).fetchall() + + for orphan in orphans: + donor = None + evidence = "" + reason = "" + + if orphan["parent_session_id"]: + evidence = "lineage" + donor = conn.execute( + f""" + SELECT {donor_columns} + FROM sessions d + WHERE d.id = ? + AND d.session_key IS NOT NULL + AND COALESCE(d.source, '') = COALESCE(?, '') + """, + (orphan["parent_session_id"], orphan["source"]), + ).fetchone() + if donor is None: + reason = ( + "parent session carries no gateway identity of " + "this source" + ) + else: + evidence = "contiguity" + candidates = conn.execute( + f""" + SELECT {donor_columns}, {donor_active} AS last_active + FROM sessions d + WHERE d.session_key IS NOT NULL + AND d.id != ? + AND COALESCE(d.source, '') = COALESCE(?, '') + AND (COALESCE(d.user_id, '') = '' + OR COALESCE(?, '') = '' + OR d.user_id = ?) + AND {donor_active} BETWEEN ? AND ? + AND {donor_active} < ? + ORDER BY last_active DESC + LIMIT 2 + """, + ( + orphan["id"], + orphan["source"], + orphan["user_id"], + orphan["user_id"], + (orphan["started_at"] or 0) - gap, + (orphan["started_at"] or 0) + gap, + orphan["last_active"], + ), + ).fetchall() + if not candidates: + reason = ( + f"no keyed predecessor fell quiet within {gap:.0f}s " + "of this session's start" + ) + elif len(candidates) > 1: + reason = ( + "ambiguous: more than one keyed predecessor " + "matches this window" + ) + else: + donor = candidates[0] + + records.append( + { + "orphan_id": orphan["id"], + "source": orphan["source"], + "message_count": orphan["message_count"], + "started_at": orphan["started_at"], + "last_active": orphan["last_active"], + "donor_id": donor["id"] if donor else None, + "session_key": donor["session_key"] if donor else None, + "evidence": evidence if donor else "", + "adoptable": donor is not None, + "reason": reason, + } + ) + + # Two unkeyed successors claiming one predecessor: at most one continues + # that chat, and nothing here says which. + contested = { + r["donor_id"] + for r in records + if r["adoptable"] + and sum(1 for x in records if x["donor_id"] == r["donor_id"]) > 1 + } + for record in records: + if record["donor_id"] in contested: + record["adoptable"] = False + record["reason"] = ( + "ambiguous: more than one unkeyed session claims this " + "predecessor" + ) + return records + + def adopt_orphaned_gateway_session( + self, orphan_id: str, donor_id: str + ) -> bool: + """Stamp *orphan_id* with *donor_id*'s routing identity, retire *donor_id*. + + Re-verifies the pair inside the write transaction so a concurrent + gateway that healed either row makes this a no-op, not a conflicting + write. Non-NULL orphan columns are preserved. True when applied. + """ + if not orphan_id or not donor_id or orphan_id == donor_id: + return False + + def _do(conn): + donor = conn.execute( + "SELECT session_key, chat_id, chat_type, thread_id, user_id, " + "origin_json, display_name, source FROM sessions WHERE id = ?", + (donor_id,), + ).fetchone() + orphan = conn.execute( + "SELECT session_key, source FROM sessions WHERE id = ?", + (orphan_id,), + ).fetchone() + if donor is None or orphan is None: + return False + if not donor["session_key"] or orphan["session_key"]: + return False + if (donor["source"] or "") != (orphan["source"] or ""): + return False + + conn.execute( + """UPDATE sessions + SET session_key = ?, + chat_id = COALESCE(chat_id, ?), + chat_type = COALESCE(chat_type, ?), + thread_id = COALESCE(thread_id, ?), + user_id = COALESCE(user_id, ?), + origin_json = COALESCE(origin_json, ?), + display_name = COALESCE(display_name, ?), + parent_session_id = COALESCE(parent_session_id, ?) + WHERE id = ? AND session_key IS NULL""", + ( + donor["session_key"], + donor["chat_id"], + donor["chat_type"], + donor["thread_id"], + donor["user_id"], + donor["origin_json"], + donor["display_name"], + donor_id, + orphan_id, + ), + ) + # Retire the predecessor under a reason recovery does NOT treat as + # resumable — 'agent_close'/'ws_orphan_reap' would keep it in the + # running and the newly keyed orphan could lose the chat again. + conn.execute( + "UPDATE sessions SET ended_at = COALESCE(ended_at, ?), " + "end_reason = 'superseded_by_repair' WHERE id = ?", + (time.time(), donor_id), + ) + return True + + return self._execute_write(_do) + + def increment_hygiene_failure_streak(self, session_key: str) -> int: + """Atomically increment the session-hygiene failure streak for one chat.""" + if not session_key: + return 1 + result = [] + + def _do(conn): + conn.execute( + """INSERT INTO gateway_hygiene_state (session_key, failure_streak) + VALUES (?, 1) + ON CONFLICT(session_key) DO UPDATE SET + failure_streak = gateway_hygiene_state.failure_streak + 1""", + (session_key,), + ) + row = conn.execute( + "SELECT failure_streak FROM gateway_hygiene_state WHERE session_key = ?", + (session_key,), + ).fetchone() + result.append(int(row[0])) + + self._execute_write(_do) + return result[0] + + def reset_hygiene_failure_streak(self, session_key: str) -> None: + """Clear the persisted session-hygiene failure streak for one chat.""" + if not session_key: + return + + self._write_sql( + "DELETE FROM gateway_hygiene_state WHERE session_key = ?", + (session_key,), + ) + + @staticmethod + def session_gateway_runtime(session_meta: Optional[Dict[str, Any]]) -> Dict[str, Any]: + """Read the persisted runtime route off a session row dict. + + Accepts ``get_session``'s dict (``model_config`` as JSON string) or a + parsed dict. Precedence: nested ``gateway_runtime`` (gateway sync / CLI + ``/model``), then top-level ``provider``/``base_url``/``api_mode`` (TUI + ``_runtime_model_config``), then ``billing_provider`` so sessions that + never ran ``/model`` still restore the provider that served them. + Empty dict on parse failure — resume uses ambient config. + """ + from hermes_state import _BARE_BILLING_PROVIDERS + raw = (session_meta or {}).get("model_config") + if isinstance(raw, str): + try: + raw = json.loads(raw) + except Exception: + raw = {} + if not isinstance(raw, dict): + raw = {} + runtime = raw.get("gateway_runtime") + if isinstance(runtime, dict) and runtime.get("provider"): + # Filter None: the persist path writes or-None to trigger deletion + # in the top-level merge, but gateway_runtime is replaced whole + # (not deep-merged), so None values survive here. + return {k: v for k, v in runtime.items() if v is not None} + top_level = { + key: raw.get(key) + for key in ("provider", "base_url", "api_mode") + if raw.get(key) + } + if top_level: + return top_level + # Last resort: billing_provider, COALESCE-written on the first accounted + # API call — the only durable record for sessions that never ran /model. + # Bare buckets ("auto"/"custom") are not routable identities; filter + # them so resume falls back to the ambient config default. + billing_provider = str( + (session_meta or {}).get("billing_provider") or "" + ).strip() + if ( + billing_provider + and billing_provider.lower() not in _BARE_BILLING_PROVIDERS + ): + return {"provider": billing_provider} + return {k: v for k, v in (runtime or {}).items() if v is not None} if isinstance(runtime, dict) else {} + + def register_backend_heartbeat( + self, + *, + backend_id: str, + pid: int, + started_at: float, + last_heartbeat: Optional[float] = None, + profile: str = "", + host: str = "", + ) -> None: + """Upsert this backend's liveness row. + + ``backend_id`` MUST be stable for the process lifetime (e.g. + ``f"{profile}@{host}:{pid}"``) so a respawn cannot inherit a dead + predecessor's heartbeat and protect stale rows. ``started_at`` is when + THIS process started, not first-refresh wall clock, so a backend whose + previous run died is not mistaken for a freshly-spawned sibling. + """ + if not backend_id: + return + ts = time.time() if last_heartbeat is None else float(last_heartbeat) + self._write_sql( + "INSERT INTO gateway_heartbeats" + " (backend_id, pid, started_at, last_heartbeat, profile, host)" + " VALUES (?, ?, ?, ?, ?, ?)" + " ON CONFLICT(backend_id) DO UPDATE SET" + " pid = excluded.pid," + " started_at = excluded.started_at," + " last_heartbeat = excluded.last_heartbeat," + " profile = excluded.profile," + " host = excluded.host", + (str(backend_id), int(pid), float(started_at), ts, + str(profile), str(host)), + ) + + def clear_backend_heartbeat(self, backend_id: str) -> bool: + """Remove this backend's heartbeat row (from ``atexit``); True if removed. + A crashed backend's row is reclaimed later by ``prune_stale_heartbeats``.""" + if not backend_id: + return False + return self._write_rowcount( + "DELETE FROM gateway_heartbeats WHERE backend_id = ?", + (str(backend_id),), + ) > 0 + + def prune_stale_heartbeats(self, *, max_age_seconds: float) -> List[str]: + """Drop heartbeat rows older than the staleness window; return removed + backend ids. Safe from any process — only stale rows are touched.""" + if max_age_seconds <= 0: + return [] + cutoff = time.time() - max_age_seconds + def _do(conn): + cur = conn.execute( + "DELETE FROM gateway_heartbeats WHERE last_heartbeat < ?" + " RETURNING backend_id", + (cutoff,), + ) + return [str(r[0]) for r in cur.fetchall()] + return list(self._execute_write(_do) or []) + + def list_backend_heartbeats(self) -> List[Dict[str, Any]]: + """Snapshot of every backend heartbeat (diagnostics/tests); fields mirror the table.""" + rows = self._read_all( + "SELECT backend_id, pid, started_at, last_heartbeat," + " profile, host FROM gateway_heartbeats" + " ORDER BY last_heartbeat DESC", + ) + out: List[Dict[str, Any]] = [] + for r in rows: + if isinstance(r, sqlite3.Row): + out.append({k: r[k] for k in r.keys()}) + else: + out.append({ + "backend_id": r[0], "pid": r[1], "started_at": r[2], + "last_heartbeat": r[3], "profile": r[4], "host": r[5], + }) + return out + + def request_handoff(self, session_id: str, platform: str) -> bool: + """Mark a session pending handoff to *platform*; False if a handoff is already in flight.""" + return self._write_rowcount( + "UPDATE sessions " + "SET handoff_state = 'pending', " + " handoff_platform = ?, " + " handoff_error = NULL " + "WHERE id = ? AND (handoff_state IS NULL " + " OR handoff_state IN ('completed', 'failed'))", + (platform, session_id), + ) > 0 + + def get_handoff_state(self, session_id: str) -> Optional[Dict[str, Any]]: + """Return ``{"state", "platform", "error"}`` or None if the session has no handoff record.""" + try: + row = self._read_one( + "SELECT handoff_state, handoff_platform, handoff_error " + "FROM sessions WHERE id = ?", + (session_id,), + ) + if not row: + return None + return { + "state": row["handoff_state"], + "platform": row["handoff_platform"], + "error": row["handoff_error"], + } + except Exception: + return None + + def list_pending_handoffs(self) -> List[Dict[str, Any]]: + """All sessions in handoff_state='pending', oldest first (gateway handoff watcher).""" + try: + rows = self._read_all( + "SELECT s.*, " + "COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved " + "FROM sessions s " + "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash " + "WHERE s.handoff_state = 'pending' " + "ORDER BY s.started_at ASC", + ) + return [self._session_row_dict(r) for r in rows] + except Exception: + return [] + + def claim_handoff(self, session_id: str) -> bool: + """Atomically transition pending → running. Returns True if claimed.""" + return self._write_rowcount( + "UPDATE sessions SET handoff_state = 'running' " + "WHERE id = ? AND handoff_state = 'pending'", + (session_id,), + ) > 0 + + def complete_handoff(self, session_id: str) -> None: + """Mark a handoff as completed.""" + self._write_sql( + "UPDATE sessions SET handoff_state = 'completed', " + "handoff_error = NULL WHERE id = ?", + (session_id,), + ) + + def fail_handoff( + self, + session_id: str, + error: str, + *, + only_states: Optional[Tuple[str, ...]] = None, + ) -> bool: + """Mark a handoff failed and record the reason; True when a row transitioned. + + ``only_states`` makes the write a compare-and-swap on ``handoff_state``. + Waiters that give up (CLI 60s poll, Desktop bounded poll) MUST pass + ``only_states=("pending",)``: once the gateway watcher has claimed the + row (``running``) it owns the terminal state, and an unconditional + waiter-side fail races the dispatch — the gateway later overwrites + ``failed`` → ``completed`` after the user was told the gateway is down + (split-brain: the handoff delivered and ``switch_session`` re-pointed + the session). The watcher fails its OWN claimed row unconditionally. + """ + def _do(conn): + if only_states: + placeholders = ", ".join("?" for _ in only_states) + cur = conn.execute( + "UPDATE sessions SET handoff_state = 'failed', " + f"handoff_error = ? WHERE id = ? AND handoff_state IN ({placeholders})", + (error[:500], session_id, *only_states), + ) + else: + cur = conn.execute( + "UPDATE sessions SET handoff_state = 'failed', " + "handoff_error = ? WHERE id = ?", + (error[:500], session_id), + ) + return cur.rowcount > 0 + return bool(self._execute_write(_do)) + + def reclaim_stale_running_handoffs(self, error: str) -> List[str]: + """Fail every handoff stuck in ``running``. Returns the ids reclaimed. + + Only the gateway watcher sets ``running``, and only for one in-process + dispatch — so a ``running`` row at watcher startup belongs to a PREVIOUS + gateway that died mid-dispatch. It is poisonous: ``request_handoff`` + only accepts NULL/``completed``/``failed``, so the session could never + hand off again, with no error surfaced. Failing rather than re-queueing + is deliberate: the dead gateway may already have switched the session + key and dispatched the synthetic turn, so a blind retry risks double + delivery; a clean terminal state the user can retry from is right. + """ + def _do(conn): + cur = conn.execute( + "SELECT id FROM sessions WHERE handoff_state = 'running'" + ) + ids = [r[0] for r in cur.fetchall()] + if ids: + conn.execute( + "UPDATE sessions SET handoff_state = 'failed', " + "handoff_error = ? WHERE handoff_state = 'running'", + (error[:500],), + ) + return ids + try: + return self._execute_write(_do) or [] + except Exception: + # Swallow but never silently: a persistently failing reclaim leaves + # poisonous 'running' rows in place, so the operator needs a trace. + logger.warning( + "reclaim_stale_running_handoffs failed; stranded 'running' " + "handoff rows (if any) were left in place", exc_info=True, + ) + return [] diff --git a/hermes_state_maintenance.py b/hermes_state_maintenance.py new file mode 100644 index 0000000000..89829b9191 --- /dev/null +++ b/hermes_state_maintenance.py @@ -0,0 +1,699 @@ +"""Retention pruning, stale-session archiving and VACUUM policy mixin for +SessionDB.""" + +from __future__ import annotations + +import logging +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from hermes_state_common import ( + AUTO_VACUUM_MIN_FREELIST_RATIO, + _sql_session_last_active, + escape_like as _escape_like, +) + +# caplog tests pin the "hermes_state" logger name. +logger = logging.getLogger("hermes_state") + + +class SessionMaintenanceMixin: + """Retention pruning, stale-session archiving and VACUUM policy for SessionDB.""" + + def prune_empty_ghost_sessions(self, sessions_dir: "Optional[Path]" = None) -> int: + """Remove empty TUI ghost sessions (no messages, no title, >24hr old).""" + cutoff = time.time() - 86400 + + def _do(conn): + rows = conn.execute(""" + SELECT id FROM sessions + WHERE source = 'tui' + AND title IS NULL + AND ended_at IS NOT NULL + AND started_at < ? + AND NOT EXISTS ( + SELECT 1 FROM messages WHERE messages.session_id = sessions.id + ) + """, (cutoff,)).fetchall() + ids = [r[0] for r in rows] + if ids: + placeholders = ",".join("?" * len(ids)) + conn.execute( + f"DELETE FROM sessions WHERE id IN ({placeholders})", ids + ) + self._delete_unreferenced_system_prompts(conn) + return ids + + removed_ids = self._execute_write(_do) or [] + if sessions_dir and removed_ids: + for sid in removed_ids: + self._remove_session_files(sessions_dir, sid) + return len(removed_ids) + + def sweep_orphaned_sessions( + self, + *, + max_idle_seconds: float, + sources: Tuple[str, ...] = ("tui", "desktop", "subagent"), + exclude_ids: Tuple[str, ...] = (), + exclude_pinned: bool = False, + heartbeat_staleness_seconds: Optional[float] = None, + heartbeat_ownership_grace_seconds: Optional[float] = None, + respect_gateway_heartbeats: bool = True, + ) -> List[str]: + """Close session rows orphaned by a dead gateway process. + + The TUI/desktop gateway reaps disconnected sessions with an in-process + grace timer; a restart destroys the timer and leaves ``ended_at IS + NULL`` forever. This closes rows for ``sources`` whose ``started_at`` + AND canonical last activity (newest of ``last_activity_at`` and the + newest message, else ``started_at``) are both older than + ``max_idle_seconds``, with ``end_reason='startup_orphan_reap'``. The + separate ``started_at`` predicate protects fresh compression/branch + children whose copied activity is old. + + Only pass sources whose lifecycle the caller owns — never messaging + platforms like ``telegram`` (ending those triggers a routing loop). + ``exclude_ids`` spares rows this process still holds in memory. + Non-destructive: messages are kept and the row stays resumable; + first-reason-wins via ``ended_at IS NULL``. + + Cross-backend liveness: with ``respect_gateway_heartbeats``, a row is + reaped only when stale AND no live backend (heartbeat within + ``heartbeat_staleness_seconds``, default ``2 * max_idle_seconds``) could + own it, where backend B owns session S if ``B.started_at <= S.started_at + + heartbeat_ownership_grace_seconds`` (default = staleness). The grace + covers a migrating backend whose sessions predate its first heartbeat + but is bounded so a PID-reuse respawn cannot protect rows forever. + Disable the gate only for sources owned by state.db itself. + + SELECT, live-lease validation and UPDATE run in one ``BEGIN IMMEDIATE`` + transaction. Active turn leases / compression locks spare the row; + expired guards are removed so their former owner is fenced. + """ + from hermes_state import SessionCompressionInProgressError, SessionTurnLeaseLostError + srcs = tuple(s for s in sources if s) + if max_idle_seconds <= 0 or not srcs: + return [] + hb_staleness = ( + heartbeat_staleness_seconds + if heartbeat_staleness_seconds and heartbeat_staleness_seconds > 0 + else max_idle_seconds * 2 + ) + hb_grace = ( + heartbeat_ownership_grace_seconds + if heartbeat_ownership_grace_seconds is not None + and heartbeat_ownership_grace_seconds >= 0 + else hb_staleness + ) + now = time.time() + cutoff = now - max_idle_seconds + hb_cutoff = now - hb_staleness + placeholders = ",".join("?" for _ in srcs) + staleness = ( + f"started_at < ? AND {_sql_session_last_active('sessions')} < ?" + ) + pin_scope = " AND COALESCE(pinned, 0) = 0" if exclude_pinned else "" + heartbeat_params: Tuple[float, ...] = () + orphan_predicate = staleness + if respect_gateway_heartbeats: + orphan_predicate += ( + " AND NOT EXISTS (" + "SELECT 1 FROM gateway_heartbeats h" + " WHERE h.last_heartbeat >= ?" + " AND h.started_at <= sessions.started_at + ?" + ")" + ) + heartbeat_params = (hb_cutoff, hb_grace) + + def _do(conn): + rows = conn.execute( + f"SELECT id FROM sessions WHERE ended_at IS NULL" + f" AND source IN ({placeholders}){pin_scope}" + f" AND {orphan_predicate}", + (*srcs, cutoff, cutoff, *heartbeat_params), + ).fetchall() + excluded = {str(x) for x in exclude_ids if x} + victims = [] + for row in rows: + sid = str(row["id"]) + if sid in excluded: + continue + try: + self._check_transcript_write_guards( + conn, + sid, + compression_lock_holder=None, + turn_lease_holder=None, + reject_active_turn_lease=True, + reject_active_compression_lock=True, + ) + except ( + SessionCompressionInProgressError, + SessionTurnLeaseLostError, + ): + continue + victims.append(sid) + if not victims: + return [] + closed_at = time.time() + marks = ",".join("?" for _ in victims) + # Re-apply every predicate under the write lock. + conn.execute( + f"UPDATE sessions SET ended_at = ?, end_reason = 'startup_orphan_reap'" + f" WHERE id IN ({marks}) AND ended_at IS NULL" + f" AND source IN ({placeholders}){pin_scope}" + f" AND {orphan_predicate}", + ( + closed_at, + *victims, + *srcs, + cutoff, + cutoff, + *heartbeat_params, + ), + ) + return victims + + return self._execute_write(_do) or [] + + @staticmethod + def _prune_filter_where( + *, + last_active_before: Optional[float] = None, + last_active_after: Optional[float] = None, + started_before: Optional[float] = None, + started_after: Optional[float] = None, + source: Optional[str] = None, + title_like: Optional[str] = None, + end_reason: Optional[str] = None, + cwd_prefix: Optional[str] = None, + min_messages: Optional[int] = None, + max_messages: Optional[int] = None, + archived: Optional[bool] = None, + model_like: Optional[str] = None, + provider: Optional[str] = None, + user_id: Optional[str] = None, + chat_id: Optional[str] = None, + chat_type: Optional[str] = None, + branch_like: Optional[str] = None, + min_tokens: Optional[int] = None, + max_tokens: Optional[int] = None, + min_cost: Optional[float] = None, + max_cost: Optional[float] = None, + min_tool_calls: Optional[int] = None, + max_tool_calls: Optional[int] = None, + include_pinned: bool = False, + ) -> Tuple[str, list]: + """Shared WHERE clause for bulk prune/archive selection (alias ``s``). + + Filters AND together; only ended sessions are ever candidates. + ``archived`` is tri-state (None = both). ``*_like`` filters are + case-insensitive substrings; the rest are exact (provider + case-insensitive). Token bounds use input+output; cost bounds use + ``COALESCE(actual_cost_usd, estimated_cost_usd)``. + """ + from hermes_state import _cwd_prefix_clause + clauses = ["s.ended_at IS NOT NULL"] + params: list = [] + if last_active_before is not None: + clauses.append( + """COALESCE( + (SELECT MAX(m.timestamp) FROM messages m + WHERE m.session_id = s.id), + s.started_at + ) < ?""" + ) + params.append(last_active_before) + # Orphan-swept rows age from the sweep, not their old activity, or + # the next prune pass deletes them before the user can recover. + clauses.append( + "(COALESCE(s.end_reason, '') != 'startup_orphan_reap' " + "OR s.ended_at < ?)" + ) + params.append(last_active_before) + if last_active_after is not None: + clauses.append( + """COALESCE( + (SELECT MAX(m.timestamp) FROM messages m + WHERE m.session_id = s.id), + s.started_at + ) >= ?""" + ) + params.append(last_active_after) + if started_before is not None: + clauses.append("s.started_at < ?") + params.append(started_before) + if started_after is not None: + clauses.append("s.started_at >= ?") + params.append(started_after) + if source: + clauses.append("s.source = ?") + params.append(source) + if title_like: + clauses.append("LOWER(COALESCE(s.title, '')) LIKE ? ESCAPE '\\'") + params.append(f"%{_escape_like(title_like.lower())}%") + if end_reason: + clauses.append("s.end_reason = ?") + params.append(end_reason) + if cwd_prefix: + clause, clause_params = _cwd_prefix_clause(cwd_prefix) + clauses.append(clause) + params.extend(clause_params) + if min_messages is not None: + clauses.append("s.message_count >= ?") + params.append(min_messages) + if max_messages is not None: + clauses.append("s.message_count <= ?") + params.append(max_messages) + if model_like: + clauses.append("LOWER(COALESCE(s.model, '')) LIKE ? ESCAPE '\\'") + params.append(f"%{_escape_like(model_like.lower())}%") + if provider: + clauses.append("LOWER(COALESCE(s.billing_provider, '')) = ?") + params.append(provider.lower()) + if user_id: + clauses.append("s.user_id = ?") + params.append(user_id) + if chat_id: + clauses.append("s.chat_id = ?") + params.append(chat_id) + if chat_type: + clauses.append("s.chat_type = ?") + params.append(chat_type) + if branch_like: + clauses.append("LOWER(COALESCE(s.git_branch, '')) LIKE ? ESCAPE '\\'") + params.append(f"%{_escape_like(branch_like.lower())}%") + if min_tokens is not None: + clauses.append( + "(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) >= ?" + ) + params.append(min_tokens) + if max_tokens is not None: + clauses.append( + "(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) <= ?" + ) + params.append(max_tokens) + if min_cost is not None: + clauses.append( + "COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) >= ?" + ) + params.append(min_cost) + if max_cost is not None: + clauses.append( + "COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) <= ?" + ) + params.append(max_cost) + if min_tool_calls is not None: + clauses.append("COALESCE(s.tool_call_count, 0) >= ?") + params.append(min_tool_calls) + if max_tool_calls is not None: + clauses.append("COALESCE(s.tool_call_count, 0) <= ?") + params.append(max_tool_calls) + if archived is True: + clauses.append("s.archived = 1") + elif archived is False: + clauses.append("s.archived = 0") + # Pinned is a durable "keep" flag: bulk prune/delete/archive exclude + # pinned rows unless the caller explicitly opts in. + if not include_pinned: + clauses.append("COALESCE(s.pinned, 0) = 0") + return " AND ".join(clauses), params + + @staticmethod + def _apply_prune_age_filter( + older_than_days: Optional[float], filters: Dict[str, Any] + ) -> None: + """Translate the legacy age window into the shared activity filter.""" + if ( + filters.get("last_active_before") is None + and filters.get("started_before") is None + and older_than_days is not None + ): + filters["last_active_before"] = time.time() - ( + older_than_days * 86400 + ) + + def list_prune_candidates( + self, + older_than_days: Optional[float] = None, + source: str = None, + **filters, + ) -> List[Dict[str, Any]]: + """Sessions a matching prune/archive would touch (dry-run), oldest + first. Same filters as :meth:`_prune_filter_where`; ``older_than_days`` + is an inactivity threshold (latest message, else ``started_at``).""" + self._apply_prune_age_filter(older_than_days, filters) + where, params = self._prune_filter_where(source=source, **filters) + with self._read_ctx() as conn: + cursor = conn.execute( + f"""SELECT s.id, s.source, s.title, s.model, s.started_at, + COALESCE( + (SELECT MAX(m.timestamp) FROM messages m + WHERE m.session_id = s.id), + s.started_at + ) AS last_active, + s.ended_at, s.message_count, s.archived + FROM sessions s WHERE {where} + ORDER BY last_active ASC, s.started_at ASC""", + params, + ) + return [dict(row) for row in cursor.fetchall()] + + def count_prune_matches( + self, + older_than_days: Optional[float] = None, + source: str = None, + **filters, + ) -> int: + """Count-only variant of :meth:`list_prune_candidates` (the CLI uses it + to report how many pinned sessions are spared).""" + self._apply_prune_age_filter(older_than_days, filters) + where, params = self._prune_filter_where(source=source, **filters) + with self._read_ctx() as conn: + cursor = conn.execute( + f"SELECT COUNT(*) FROM sessions s WHERE {where}", params + ) + return int(cursor.fetchone()[0]) + + def count_open_prune_matches( + self, + older_than_days: Optional[float] = None, + source: str = None, + **filters, + ) -> int: + """Count open sessions a matching prune skips: every normal filter with + only the ``ended_at`` guard inverted. Visibility-only; live sessions + never become prune-eligible.""" + self._apply_prune_age_filter(older_than_days, filters) + where, params = self._prune_filter_where(source=source, **filters) + ended_guard = "s.ended_at IS NOT NULL" + if not where.startswith(ended_guard): + raise RuntimeError("prune filter lost its ended-session safety guard") + open_where = f"s.ended_at IS NULL{where[len(ended_guard):]}" + with self._read_ctx() as conn: + cursor = conn.execute( + f"SELECT COUNT(*) FROM sessions s WHERE {open_where}", params + ) + return int(cursor.fetchone()[0]) + + def archive_stale_sessions( + self, idle_days: float, *, exclude_pinned: bool = True + ) -> int: + """Archive every session untouched for ``idle_days`` (real recency: + freshest of ``last_activity_at`` / latest message / ``started_at``). + Unlike :meth:`archive_sessions`, this can archive unended sessions. + + Guards: ``pinned = 0`` when ``exclude_pinned``; ``archived = 0`` so + repeats are no-ops; only lineage tips (``end_reason <> 'compression'``) + are candidates — a stale tip archives its chain via + :meth:`set_session_archived`, so an old compressed-away root with a + recent continuation is never matched. Returns the count archived. + """ + if idle_days is None or idle_days < 0: + return 0 + cutoff = time.time() - float(idle_days) * 86400.0 + pin_clause = "AND s.pinned = 0" if exclude_pinned else "" + rows = self._read_all( + f""" + SELECT s.id FROM sessions s + WHERE s.archived = 0 + AND COALESCE(s.end_reason, '') <> 'compression' + {pin_clause} + AND {_sql_session_last_active("s")} < ? + ORDER BY s.started_at ASC + """, + (cutoff,), + ) + ids = [r[0] for r in rows] + for sid in ids: + self.set_session_archived(sid, True) + return len(ids) + + def prune_sessions( + self, + older_than_days: Optional[float] = 90, + source: str = None, + sessions_dir: Optional[Path] = None, + exclude_active_write_guards: bool = False, + **filters, + ) -> int: + """Delete ended sessions matching the filters; returns the count. + + Default: inactive for ``older_than_days`` (latest message, else + ``started_at``), optionally by ``source``. Extra keyword filters are + those of :meth:`_prune_filter_where`; an explicit ``started_before`` / + ``last_active_before`` overrides the ``older_than_days`` cutoff + (pass ``older_than_days=None`` for no implicit age bound). + + Children outside the window are orphaned (parent NULLed), not cascade- + deleted. With *sessions_dir*, on-disk transcript files are removed + outside the DB transaction. ``exclude_active_write_guards`` (automatic + maintenance) skips rows under a live turn lease or compression lock, + while expired/dead holders are reclaimed and fenced in the same write. + """ + from hermes_state import SessionCompressionInProgressError, SessionTurnLeaseLostError + self._apply_prune_age_filter(older_than_days, filters) + where, where_params = self._prune_filter_where(source=source, **filters) + removed_ids: list[str] = [] + + def _do(conn): + cursor = conn.execute( + f"SELECT s.id FROM sessions s WHERE {where}", where_params + ) + session_ids = {row["id"] for row in cursor.fetchall()} + + if exclude_active_write_guards: + protected = set() + for sid in session_ids: + try: + self._check_transcript_write_guards( + conn, + sid, + compression_lock_holder=None, + turn_lease_holder=None, + reject_active_turn_lease=True, + reject_active_compression_lock=True, + allow_closed_compression_parent=True, + ) + except ( + SessionCompressionInProgressError, + SessionTurnLeaseLostError, + ): + protected.add(sid) + session_ids.difference_update(protected) + + if not session_ids: + return 0 + + placeholders = ",".join("?" * len(session_ids)) + conn.execute( + f"UPDATE sessions SET parent_session_id = NULL " + f"WHERE parent_session_id IN ({placeholders})", + list(session_ids), + ) + + for sid in session_ids: + conn.execute("DELETE FROM messages WHERE session_id = ?", (sid,)) + conn.execute("DELETE FROM sessions WHERE id = ?", (sid,)) + removed_ids.append(sid) + self._delete_unreferenced_system_prompts(conn) + return len(session_ids) + + count = self._execute_write(_do) + for sid in removed_ids: + self._remove_session_files(sessions_dir, sid) + return count + + def logical_size_bytes(self) -> Optional[int]: + """``page_count * page_size``: the main-file size once the WAL is + checkpointed back in. Prefer over ``os.path.getsize`` when reporting a + VACUUM: in WAL mode the rewrite lands in ``-wal`` and the checkpoint is + refused while another connection holds a read-mark, so a stat() delta + understates the win and can go negative. None if pragmas fail. + """ + try: + with self._read_ctx() as conn: + if self._conn is None: + return None + page_count = conn.execute("PRAGMA page_count").fetchone()[0] + page_size = conn.execute("PRAGMA page_size").fetchone()[0] + return int(page_count) * int(page_size) + except Exception as exc: + logger.debug("Could not read logical DB size: %s", exc) + return None + + def _freelist_ratio(self) -> Optional[float]: + """Reclaimable fraction (``freelist_count / page_count``) over the + existing connection — never a byte-level probe of the live file. Gates + VACUUM in :meth:`maybe_auto_prune_and_vacuum`. None if pragmas fail + (callers then fall back to the time throttle alone). + """ + try: + with self._read_ctx() as conn: + if self._conn is None: + return None + page_count = int(conn.execute("PRAGMA page_count").fetchone()[0]) + freelist = int(conn.execute("PRAGMA freelist_count").fetchone()[0]) + if page_count <= 0: + return 0.0 + return freelist / page_count + except Exception as exc: + logger.debug("Could not read freelist ratio: %s", exc) + return None + + def vacuum(self) -> int: + """VACUUM to reclaim space after large deletes (SQLite never shrinks + the file on its own). + + Rewrites the whole DB, cannot run inside a transaction, and takes an + exclusive lock — callers must ensure no other writers are active (safe + at startup before serving traffic). FTS5 segments are merged first via + :meth:`optimize_fts` so the VACUUM reclaims those pages too. Returns + the number of FTS indexes optimized (0 on merge failure / no FTS). + """ + # optimize_fts() manages its own lock. + optimized = 0 + try: + optimized = self.optimize_fts() + except Exception as exc: + logger.warning("FTS optimize before VACUUM failed: %s", exc) + # VACUUM cannot be executed inside a transaction. + with self._lock: + # PASSIVE, not TRUNCATE: a manual `hermes sessions vacuum` runs in + # a transient CLI process, and a TRUNCATE reset here would race a + # live gateway writer and tear B-tree pages. + try: + self._conn.execute("PRAGMA wal_checkpoint(PASSIVE)") + except Exception as exc: + logger.debug("WAL checkpoint (PASSIVE) before VACUUM failed: %s", exc) + self._conn.execute("VACUUM") + # VACUUM rewrites every page THROUGH the WAL; without this TRUNCATE + # a 3 GB database leaves a 3 GB -wal behind and the command is a + # net loss on disk. + try: + self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + except Exception as exc: + logger.debug("WAL checkpoint (TRUNCATE) after VACUUM failed: %s", exc) + # TRUNCATE may replace the WAL inode; adopt the new sidecars so the + # write-path generation guard does not halt this connection. + self._record_db_file_identity() + return optimized + + def maybe_auto_prune_and_vacuum( + self, + retention_days: int = 90, + min_interval_hours: int = 24, + vacuum: bool = True, + sessions_dir: Optional[Path] = None, + min_vacuum_interval_days: int = 30, + min_vacuum_freelist_ratio: float = AUTO_VACUUM_MIN_FREELIST_RATIO, + ) -> Dict[str, Any]: + """Idempotent startup auto-maintenance: prune inactive sessions, + reap stale open state-owned rows, optional VACUUM. Never raises. + + Runs at most once per ``min_interval_hours`` (state_meta). VACUUM has + its own ``min_vacuum_interval_days`` throttle and additionally requires + ``freelist_count / page_count`` > ``min_vacuum_freelist_ratio`` so a + small prune on a dense multi-GB database never triggers a full rewrite. + With *sessions_dir*, pruned transcripts are removed from disk too. + + Stale-open reconciliation: cron/kanban/subagent/one-shot CLI rows never + set ``ended_at`` when their process dies, and prune only deletes ended + rows. After pruning, open rows from :attr:`_AUTO_PRUNE_STALE_OPEN_SOURCES` + older than ``retention_days`` are closed (``startup_orphan_reap``); they + stay resumable and age from their close, so they get one more full + retention window. Messaging and UI sources are never touched. + + Returns ``{"skipped", "pruned", "closed", "vacuumed"}`` plus + ``"freelist_ratio"`` when a VACUUM was considered and ``"error"`` on + failure. + """ + from hermes_state import _release_auto_maintenance_lock, _try_acquire_auto_maintenance_lock + result: Dict[str, Any] = { + "skipped": False, + "pruned": 0, + "closed": 0, + "vacuumed": False, + } + maintenance_lock = _try_acquire_auto_maintenance_lock(self.db_path) + if maintenance_lock is None: + result["skipped"] = True + return result + try: + last_raw = self.get_meta("last_auto_prune") + now = time.time() + if last_raw: + try: + last_ts = float(last_raw) + if now - last_ts < min_interval_hours * 3600: + result["skipped"] = True + return result + except (TypeError, ValueError): + pass # corrupt meta; treat as no prior run + + # Prune first: orphans closed below get a full retention window. + pruned = self.prune_sessions( + older_than_days=retention_days, + sessions_dir=sessions_dir, + exclude_active_write_guards=True, + ) + result["pruned"] = pruned + + closed = self.sweep_orphaned_sessions( + max_idle_seconds=float(retention_days) * 86400.0, + sources=self._AUTO_PRUNE_STALE_OPEN_SOURCES, + exclude_pinned=True, + # State-owned lifecycles, not gateway heartbeats. + respect_gateway_heartbeats=False, + ) + result["closed"] = len(closed) + # VACUUM only if rows were freed, the time throttle passed ("not + # too often") AND the freelist ratio passed ("only when it pays + # off") — it holds an exclusive lock for a full rewrite. + last_vacuum_raw = self.get_meta("last_vacuum") + vacuum_due = True + if last_vacuum_raw: + try: + vacuum_due = (now - float(last_vacuum_raw)) >= min_vacuum_interval_days * 86400 + except (TypeError, ValueError): + vacuum_due = True + if vacuum and pruned > 0 and vacuum_due: + ratio = self._freelist_ratio() + result["freelist_ratio"] = ratio + if ratio is None or ratio > min_vacuum_freelist_ratio: + try: + self.vacuum() + result["vacuumed"] = True + self.set_meta("last_vacuum", str(now)) + except Exception as exc: + logger.warning("state.db VACUUM failed: %s", exc) + else: + logger.debug( + "state.db auto-maintenance: skipping VACUUM, only " + "%.1f%% of pages reclaimable (threshold %.0f%%)", + ratio * 100.0, + min_vacuum_freelist_ratio * 100.0, + ) + + # Record even when pruned == 0 so the throttle holds. + self.set_meta("last_auto_prune", str(now)) + + if closed or pruned > 0: + logger.info( + "state.db auto-maintenance: closed %d stale open session(s), " + "pruned %d session(s) inactive for %d days%s", + len(closed), + pruned, + retention_days, + " + VACUUM" if result["vacuumed"] else "", + ) + except Exception as exc: + # Maintenance must never block startup. + logger.warning("state.db auto-maintenance failed: %s", exc) + result["error"] = str(exc) + finally: + _release_auto_maintenance_lock(maintenance_lock) + + return result diff --git a/hermes_state_messages.py b/hermes_state_messages.py new file mode 100644 index 0000000000..4e5091d46c --- /dev/null +++ b/hermes_state_messages.py @@ -0,0 +1,2642 @@ +"""Transcript persistence for SessionDB. + +Mixin split out of ``hermes_state.py``; bound onto ``SessionDB`` via the MRO +and built on its ``_read_ctx`` / ``_execute_write`` / ``_write_sql`` / +``_read_one`` / ``_read_all`` primitives. Covers message append / replace / +rewind, reactions, resume-conversation assembly and replayed-user-message +duplicate detection. +""" + +from __future__ import annotations + +import json +import logging +import time +from typing import Any, Dict, List, Optional, Tuple + +from agent.context_compressor import _DB_PERSISTED_MARKER as _DB_PERSISTED_MARKER_KEY +from agent.memory_manager import sanitize_context +from agent.message_sanitization import _sanitize_surrogates +from hermes_state_common import ( + _RESET_END_REASONS, + _RESET_END_REASONS_SQL, + _legacy_reset_child_sql, +) + +# Log-record parity with the origin module (caplog tests pin "hermes_state"). +logger = logging.getLogger("hermes_state") + + +class SessionMessagesMixin: + """Message append/replace/rewind, reactions, resume conversations, replay dedupe.""" + + def _bump_conversation_generation(self, conn, session_id: str, end_reason: str) -> None: + """Advance this peer's conversation generation past a boundary. + + Called inside the transaction that writes the boundary, so the + generation and the ``end_reason`` that caused it commit together. + + Only ``_RESET_END_REASONS`` count: ``compression`` continues one + conversation, and an accidental close is not a replacement. Rows with + no ``session_key`` have no routing peer to advance. + + The counter deliberately does NOT read the session rows. An aggregate + over them (COUNT/MAX of boundaries) can return a pair it already + emitted once ``delete_session()`` or bulk pruning removes an ended row, + which would hand a new conversation a retired affinity identity. This + value only ever increments, so a generation is never reused for a peer + even if every row behind it is gone. + """ + if end_reason not in _RESET_END_REASONS: + return + row = conn.execute( + "SELECT source, session_key FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + if row is None: + return + source = str(row["source"] or "").strip() + session_key = str(row["session_key"] or "").strip() + if not source or not session_key: + return + conn.execute( + """ + INSERT INTO conversation_generations (source, session_key, generation) + VALUES (?, ?, 1) + ON CONFLICT(source, session_key) DO UPDATE + SET generation = conversation_generations.generation + 1 + """, + (source, session_key), + ) + + @classmethod + def _encode_content(cls, content: Any) -> Any: + """Serialize structured (list/dict) message content for sqlite. + + sqlite3 can only bind ``str``, ``bytes``, ``int``, ``float``, and ``None`` + to query parameters. Multimodal messages have ``content`` as a list of + parts (``[{"type": "text", ...}, {"type": "image_url", ...}]``), which + raises ``ProgrammingError: Error binding parameter N: type 'list' is + not supported`` when bound directly. + + Returns the value unchanged when it's already a safe scalar, or a + sentinel-prefixed JSON string for lists/dicts. Paired with + :meth:`_decode_content` on read. + """ + if isinstance(content, str): + # Lone UTF-16 surrogates reach here inside tool results scraped + # from the web/social platforms (the same input that crashed the + # guardrail hasher). The proactive sanitizer upstream only cleans + # the *api_messages* copy, and the recovery sanitizer only runs + # after the API call itself raises — which it no longer does — so + # the canonical history keeps them and this write is where they + # land. Left raw, sqlite3 raises UnicodeEncodeError, the flush is + # abandoned, and the session silently stops persisting for the + # rest of its life. Scrub so persistence never fails. + return _sanitize_surrogates(content) + if content is None or isinstance(content, (bytes, int, float)): + return content + try: + # json.dumps defaults to ensure_ascii=True, which escapes any + # surrogate as \udXXX — already safe to bind. + return cls._CONTENT_JSON_PREFIX + json.dumps(content) + except (TypeError, ValueError): + # Last-resort fallback: stringify so persistence never fails. + return _sanitize_surrogates(str(content)) + + @classmethod + def _decode_content(cls, content: Any) -> Any: + """Reverse :meth:`_encode_content`; returns scalars unchanged.""" + if isinstance(content, str) and content.startswith(cls._CONTENT_JSON_PREFIX): + try: + return json.loads(content[len(cls._CONTENT_JSON_PREFIX):]) + except (json.JSONDecodeError, TypeError): + logger.warning( + "Failed to decode JSON-encoded message content; " + "returning raw string" + ) + return content + return content + + @staticmethod + def _encode_display_metadata(display_metadata: Any) -> Optional[str]: + """Serialize ``display_metadata`` for its TEXT column without double-encoding. + + Import/replace paths can hand us an already-serialized JSON string (the + same hazard ``tool_calls`` guards against above). ``json.dumps`` on that + string would store a quoted JSON string, and the single ``json.loads`` + on read then yields a ``str`` instead of a dict. + """ + if not display_metadata: + return None + if isinstance(display_metadata, str): + try: + parsed = json.loads(display_metadata) + except (json.JSONDecodeError, TypeError): + logger.warning("Ignoring non-JSON display metadata on write") + return None + if not isinstance(parsed, dict): + logger.warning("Ignoring non-object display metadata on write") + return None + return json.dumps(parsed) + if isinstance(display_metadata, dict): + return json.dumps(display_metadata) + logger.warning( + "Ignoring unexpected display metadata type on write: %s", + type(display_metadata).__name__, + ) + return None + + def _check_transcript_write_guards( + self, + conn, + session_id: str, + compression_lock_holder: Optional[str], + turn_lease_holder: Optional[str] = None, + turn_lease_ttl_seconds: float = 300.0, + reject_active_turn_lease: bool = False, + reject_active_compression_lock: bool = False, + allow_closed_compression_parent: bool = False, + ) -> None: + """Transcript-write admission checks, run INSIDE the write txn. + + Shared by :meth:`append_message` and :meth:`append_messages_batch` so + the two writers can never diverge on these correctness invariants + (this guard has already needed targeted fixes — see the #74478 patience + note below). User-initiated transcript mutations may opt in to rejecting + an active unowned turn lease in that same transaction. + """ + from hermes_state import CompressionSessionClosedError, SessionCompressionInProgressError, SessionTurnLeaseLostError, _compression_lock_holder_process_is_dead + # NOTE (#75316 redesign): appends do NOT check compression_locks. + # The lock's job is to stop two COMPRESSIONS colliding, not to fence + # ordinary transcript writes. Concurrent appends during a compression + # are safe by construction: archive_and_compact() commits against a + # watermark captured at compression start and clones every row that + # arrived after it back into the live transcript, in the same write + # transaction. Blocking appends here was the root cause of a whole + # symptom family — turns dying as session_persistence_failed while a + # slow provider summary held the lease (#74568, #77386), including + # stale locks from dead PIDs blocking writes for the full TTL. + # Destructive user mutations are different: a compressor that already + # captured its watermark can otherwise publish the pre-rewind snapshot + # after the mutation and resurrect the removed turn. Keep that narrow + # fence opt-in so ordinary appends retain the watermark behavior. + if reject_active_compression_lock: + active_lock = conn.execute( + "SELECT holder, expires_at FROM compression_locks " + "WHERE session_id = ?", + (session_id,), + ).fetchone() + if active_lock is not None: + current_holder = active_lock["holder"] + if ( + float(active_lock["expires_at"]) <= time.time() + or _compression_lock_holder_process_is_dead(current_holder) + ): + conn.execute( + "DELETE FROM compression_locks " + "WHERE session_id = ? AND holder = ?", + (session_id, current_holder), + ) + elif current_holder != compression_lock_holder: + raise SessionCompressionInProgressError( + f"Session {session_id!r} is being compressed by another writer" + ) + if turn_lease_holder or reject_active_turn_lease: + conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) + lease = conn.execute( + "SELECT holder, expires_at FROM session_turn_leases " + "WHERE conversation_id = ?", + (conversation_id,), + ).fetchone() + now = time.time() + if turn_lease_holder: + if lease is None or lease["holder"] != turn_lease_holder: + raise SessionTurnLeaseLostError( + f"Session turn lease lost; refusing transcript write " + f"for {session_id!r}" + ) + if float(lease["expires_at"]) <= now: + # Expiry makes the row reclaimable; it does not prove that a + # takeover occurred. BEGIN IMMEDIATE serializes this renewal + # with acquisition, so a still-matching owner can recover from + # a starved refresher without weakening the foreign-holder fence. + conn.execute( + "UPDATE session_turn_leases SET expires_at = ? " + "WHERE conversation_id = ? AND holder = ?", + ( + now + max(0.1, float(turn_lease_ttl_seconds)), + conversation_id, + turn_lease_holder, + ), + ) + elif lease is not None: + current_holder = lease["holder"] + if ( + float(lease["expires_at"]) <= now + or _compression_lock_holder_process_is_dead(current_holder) + ): + # Match acquisition semantics: an expired or provably dead + # owner is reclaimable. Deleting it inside this BEGIN IMMEDIATE + # transaction also fences a stale late flush after the mutation. + conn.execute( + "DELETE FROM session_turn_leases " + "WHERE conversation_id = ? AND holder = ?", + (conversation_id, current_holder), + ) + else: + raise SessionTurnLeaseLostError( + f"Session has an active turn lease; refusing transcript " + f"mutation for {session_id!r}" + ) + session = conn.execute( + "SELECT ended_at, end_reason FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + if ( + session is not None + and session["ended_at"] is not None + and session["end_reason"] == "compression" + and not allow_closed_compression_parent + ): + raise CompressionSessionClosedError(session_id) + + @staticmethod + def _decode_display_metadata(raw: Any) -> Optional[Dict[str, Any]]: + """Decode a ``display_metadata`` column into the dict every reader expects. + + Every message read path must go through this. Returning the raw TEXT + instead reaches the desktop as a string, where ``'task_count' in meta`` + throws and fails the whole resume. Rows written before the encode guard + landed are double-encoded, so unwrap a second layer when we find one. + """ + if raw is None: + return None + try: + meta = json.loads(raw) if isinstance(raw, str) else raw + if isinstance(meta, str): + meta = json.loads(meta) + except (json.JSONDecodeError, TypeError): + logger.warning("Ignoring invalid display metadata on message row") + return None + if not isinstance(meta, dict): + logger.warning("Ignoring non-object display metadata on message row") + return None + return meta + + @staticmethod + def _reasoning_json_text(value: Any) -> Optional[str]: + """Serialize a structured reasoning field for its TEXT column. + + ``reasoning_details`` / ``codex_reasoning_items`` / ``codex_message_items`` + arrive as list/dict structures from the live runtime, but callers that + round-trip stored rows — ``get_messages`` straight into + ``replace_messages``, e.g. the POST /api/sessions/{id}/fork handler — + hand back the raw TEXT these columns already hold, because + ``get_messages`` only deserializes ``content`` and ``tool_calls``. + Re-dumping that TEXT double-encodes it, and the forked session's next + ``get_messages_as_conversation`` json.loads then yields the inner + string instead of the original list, so every reasoning-replay consumer + (all of which check ``isinstance(..., list)``) silently drops it. + Strings are therefore stored as-is; structures are dumped. + """ + if not value: + return None + if isinstance(value, str): + return value + return json.dumps(value) + + def append_message( + self, + session_id: str, + role: str, + content: str = None, + tool_name: str = None, + tool_calls: Any = None, + tool_call_id: str = None, + token_count: int = None, + finish_reason: str = None, + reasoning: str = None, + reasoning_content: str = None, + reasoning_details: Any = None, + codex_reasoning_items: Any = None, + codex_message_items: Any = None, + platform_message_id: str = None, + observed: bool = False, + effect_disposition: Optional[str] = None, + _compressed_summary: bool = False, + timestamp: Any = None, + api_content: Optional[str] = None, + display_kind: Optional[str] = None, + display_metadata: Optional[Dict[str, Any]] = None, + compression_lock_holder: Optional[str] = None, + turn_lease_holder: Optional[str] = None, + turn_lease_ttl_seconds: float = 300.0, + ) -> int: + """ + Append a message to a session. Returns the message row ID. + + Also increments the session's message_count (and tool_call_count + if role is 'tool' or tool_calls is present). + + ``platform_message_id`` is the external messaging platform's own + message ID (e.g. Telegram update_id, Yuanbao msg_id). It is + independent of the SQLite autoincrement primary key and is used by + platform-specific flows like yuanbao's recall guard to redact a + message by its platform-side identifier. + + ``api_content`` is the exact content string sent to the API for this + message when it differs from ``content`` (ephemeral memory/plugin + injections, persist overrides). It is a byte-fidelity sidecar for + prompt-cache-stable replay — stored as sent, except lone surrogates + (which sqlite3 cannot bind and which the conversation loop scrubs + from every outgoing payload anyway, so the scrubbed form IS the + wire bytes). + """ + from hermes_state import _scrub_surrogates + # Display metadata is presentation-only and never changes the model + # context role/content replayed to providers. + display_metadata_json = self._encode_display_metadata(display_metadata) + # Serialize structured fields to JSON before entering the write txn + reasoning_details_json = self._reasoning_json_text(reasoning_details) + codex_items_json = self._reasoning_json_text(codex_reasoning_items) + codex_message_items_json = self._reasoning_json_text(codex_message_items) + # tool_calls may arrive as a Python list (from the live agent) or + # as a JSON string (from import/export). Parse first to avoid + # double-encoding. + if isinstance(tool_calls, str): + try: + tool_calls = json.loads(tool_calls) + except (json.JSONDecodeError, TypeError): + tool_calls = [] + tool_calls_json = json.dumps(tool_calls) if tool_calls else None + # Multimodal content (list of parts) must be JSON-encoded: sqlite3 + # cannot bind list/dict parameters directly. + stored_content = self._encode_content(content) + + message_timestamp = time.time() + if timestamp is not None: + try: + if hasattr(timestamp, "timestamp"): + message_timestamp = float(timestamp.timestamp()) + else: + message_timestamp = float(timestamp) + except (TypeError, ValueError): + logger.debug("Ignoring invalid explicit message timestamp: %r", timestamp) + + # Pre-compute tool call count + num_tool_calls = 0 + if tool_calls is not None: + num_tool_calls = len(tool_calls) if isinstance(tool_calls, list) else 1 + + def _do(conn): + self._check_transcript_write_guards( + conn, + session_id, + compression_lock_holder, + turn_lease_holder=turn_lease_holder, + turn_lease_ttl_seconds=turn_lease_ttl_seconds, + ) + cursor = conn.execute( + """INSERT INTO messages (session_id, role, content, tool_call_id, + tool_calls, tool_name, effect_disposition, timestamp, token_count, finish_reason, + reasoning, reasoning_content, reasoning_details, codex_reasoning_items, + codex_message_items, platform_message_id, observed, _compressed_summary, active, api_content, display_kind, display_metadata) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + session_id, + role, + stored_content, + tool_call_id, + tool_calls_json, + _scrub_surrogates(tool_name), + effect_disposition, + message_timestamp, + token_count, + finish_reason, + _scrub_surrogates(reasoning), + _scrub_surrogates(reasoning_content), + reasoning_details_json, + codex_items_json, + codex_message_items_json, + platform_message_id, + 1 if observed else 0, + 1 if _compressed_summary else 0, + 1, + _scrub_surrogates(api_content) if isinstance(api_content, str) else None, + _scrub_surrogates(display_kind) if isinstance(display_kind, str) else None, + display_metadata_json, + ), + ) + msg_id = cursor.lastrowid + + # Update counters + if num_tool_calls > 0: + conn.execute( + """UPDATE sessions SET message_count = message_count + 1, + tool_call_count = tool_call_count + ? WHERE id = ?""", + (num_tool_calls, session_id), + ) + else: + conn.execute( + "UPDATE sessions SET message_count = message_count + 1 WHERE id = ?", + (session_id,), + ) + return msg_id + + # Transcript append is THE critical write: its failure aborts the + # user's turn (session_persistence_failed). Use the long patience so + # a sibling process legitimately holding the write lock for seconds + # (VACUUM, TRUNCATE checkpoint at close, an older pre-bounded-merge + # process's FTS optimize) can't destroy a healthy turn (#74478). + return self._execute_write( + _do, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S + ) + + def append_messages_batch( + self, + session_id: str, + messages: List[Dict[str, Any]], + compression_lock_holder: Optional[str] = None, + turn_lease_holder: Optional[str] = None, + chunk_rows: Optional[int] = None, + turn_lease_ttl_seconds: float = 300.0, + ) -> int: + """Append multiple messages atomically in ONE write transaction. + + ``messages`` is a list of dicts in the same shape + :meth:`_insert_message_rows` already consumes for replace/compact/ + import (role, content, tool_name, tool_calls, tool_call_id, + finish_reason, reasoning*, codex_*, timestamp, api_content, + display_kind, display_metadata, ...). Reusing that helper keeps ONE + row-serialization path for every multi-row writer. + + A turn-boundary flush writes the whole turn (user + assistant + tool + rows, typically 3-8 messages) as one BEGIN IMMEDIATE / commit pair + instead of one transaction (and, off WAL, one fsync) per row. + + Atomicity contract: all rows land or none do (the caller re-flushes + unstamped messages on the next attempt). The same admission guards + as :meth:`append_message` run once for the batch — same session, + same instant. + + ``chunk_rows`` bounds the transaction size for LARGE copies (branch + seeds can be thousands of rows; measured: 10k rows ≈ 2.4s inside one + BEGIN IMMEDIATE because the FTS triggers run per row, which would + monopolize the write lock and starve concurrent writers). When set, + the batch commits in chunks of at most that many rows — same + recovery semantics as the old per-row loops (a mid-copy failure + leaves a partial seed), just with bounded lock holds. A turn flush + never needs it. Returns the inserted row count. + """ + if not messages: + return 0 + + if chunk_rows is not None and len(messages) > chunk_rows: + inserted_total = 0 + for start in range(0, len(messages), chunk_rows): + inserted_total += self.append_messages_batch( + session_id, + messages[start:start + chunk_rows], + compression_lock_holder=compression_lock_holder, + turn_lease_holder=turn_lease_holder, + turn_lease_ttl_seconds=turn_lease_ttl_seconds, + ) + return inserted_total + + def _do(conn): + self._check_transcript_write_guards( + conn, + session_id, + compression_lock_holder, + turn_lease_holder=turn_lease_holder, + turn_lease_ttl_seconds=turn_lease_ttl_seconds, + ) + from agent.transcript_repair import resolve_and_repair_transcript_batch + + inserted_rows = resolve_and_repair_transcript_batch( + conn, + session_id, + messages, + encode_content_fn=self._encode_content, + decode_content_fn=self._decode_content, + ) + inserted = 0 + tool_calls_total = 0 + if inserted_rows: + inserted, tool_calls_total = self._insert_message_rows( + conn, session_id, inserted_rows + ) + + # One aggregated counter update for the newly inserted rows. + if tool_calls_total > 0: + conn.execute( + """UPDATE sessions SET message_count = message_count + ?, + tool_call_count = tool_call_count + ? WHERE id = ?""", + (inserted, tool_calls_total, session_id), + ) + elif inserted > 0: + conn.execute( + "UPDATE sessions SET message_count = message_count + ? WHERE id = ?", + (inserted, session_id), + ) + return inserted + + # Same criticality as append_message: this IS the turn's transcript. + return self._execute_write( + _do, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S + ) + + def set_latest_matching_message_display_kind( + self, session_id: str, *, role: str, content: str, display_kind: str, + display_metadata: Optional[Dict[str, Any]] = None, + ) -> bool: + """Stamp presentation metadata on this turn's freshly persisted row. + + The model still receives ``role`` and ``content`` unchanged. Gateway and + CLI synthetic inputs call this immediately after their serial turn has + flushed, preserving producer provenance without classifying by content + during transcript rendering. + """ + from hermes_state import _scrub_surrogates + if not session_id or not content or not display_kind: + return False + + def _do(conn): + row = conn.execute( + "SELECT id FROM messages WHERE session_id = ? AND role = ? " + "AND content = ? AND active = 1 ORDER BY id DESC LIMIT 1", + (session_id, role, self._encode_content(content)), + ).fetchone() + if row is None: + return False + conn.execute( + "UPDATE messages SET display_kind = ?, display_metadata = ? WHERE id = ?", + ( + _scrub_surrogates(display_kind), + self._encode_display_metadata(display_metadata), + row[0], + ), + ) + return True + + return bool(self._execute_write(_do)) + + def set_message_reaction( + self, + session_id: str, + message_row_id: int, + emoji: Optional[str], + *, + author: str = "user", + ) -> Optional[List[Dict[str, Any]]]: + """Set (or with ``emoji=None`` clear) *author*'s reaction on one message. + + iOS Tapback semantics: one reaction per author per message. Re-sending + the same emoji clears it, a different emoji replaces it. Returns the + message's full reaction list after the write, or ``None`` when the row + doesn't exist or isn't part of *session_id*. + """ + from hermes_state import _scrub_surrogates + if not session_id or message_row_id is None: + return None + + def _do(conn): + row = conn.execute( + "SELECT display_metadata FROM messages WHERE id = ? AND session_id = ?", + (message_row_id, session_id), + ).fetchone() + if row is None: + return None + + meta = self._decode_display_metadata(row[0]) or {} + existing = meta.get(self.REACTIONS_METADATA_KEY) + reactions = [ + r + for r in (existing if isinstance(existing, list) else []) + if isinstance(r, dict) and r.get("author") != author + ] + previous = next( + ( + r + for r in (existing if isinstance(existing, list) else []) + if isinstance(r, dict) and r.get("author") == author + ), + None, + ) + # Tapping the live reaction again retracts it. + toggling_off = ( + emoji is not None and previous is not None and previous.get("emoji") == emoji + ) + if emoji and not toggling_off: + reactions.append( + {"emoji": _scrub_surrogates(emoji), "author": author, "at": time.time()} + ) + + if reactions: + meta[self.REACTIONS_METADATA_KEY] = reactions + else: + meta.pop(self.REACTIONS_METADATA_KEY, None) + + conn.execute( + "UPDATE messages SET display_metadata = ? WHERE id = ?", + (self._encode_display_metadata(meta) if meta else None, message_row_id), + ) + return reactions + + return self._execute_write(_do) + + def get_message_reactions( + self, session_id: str, message_row_id: int + ) -> List[Dict[str, Any]]: + """Return the reaction list persisted on one message row (never ``None``).""" + if not session_id or message_row_id is None: + return [] + + row = self._read_one( + "SELECT display_metadata FROM messages WHERE id = ? AND session_id = ?", + (message_row_id, session_id), + ) + + if row is None: + return [] + + meta = self._decode_display_metadata(row[0]) or {} + reactions = meta.get(self.REACTIONS_METADATA_KEY) + + return [r for r in reactions if isinstance(r, dict)] if isinstance(reactions, list) else [] + + def take_unseen_reactions( + self, session_id: str, *, author: str = "user" + ) -> List[Dict[str, Any]]: + """Return *author*'s not-yet-surfaced reactions and mark them seen. + + Powers the cache-safe model-context path: reactions are announced on the + NEXT user turn (never by rewriting the message that was reacted to), and + the ``seen`` stamp guarantees each one is announced exactly once. + """ + if not session_id: + return [] + + def _do(conn): + rows = conn.execute( + "SELECT id, role, content, display_metadata FROM messages " + "WHERE session_id = ? AND active = 1 AND display_metadata IS NOT NULL " + "ORDER BY id", + (session_id,), + ).fetchall() + + pending = [] + for row in rows: + meta = self._decode_display_metadata(row["display_metadata"]) + if not meta: + continue + reactions = meta.get(self.REACTIONS_METADATA_KEY) + if not isinstance(reactions, list): + continue + + changed = False + for reaction in reactions: + if ( + not isinstance(reaction, dict) + or reaction.get("author") != author + or reaction.get("seen") + ): + continue + reaction["seen"] = True + changed = True + content = self._decode_content(row["content"]) + pending.append( + { + "row_id": row["id"], + "role": row["role"], + "emoji": reaction.get("emoji") or "", + "text": content if isinstance(content, str) else "", + } + ) + + if changed: + conn.execute( + "UPDATE messages SET display_metadata = ? WHERE id = ?", + (self._encode_display_metadata(meta), row["id"]), + ) + + return pending + + return self._execute_write(_do) or [] + + def latest_message_row_id( + self, session_id: str, *, role: str = "user", offset: int = 0, require_text: bool = True + ) -> Optional[int]: + """Row id of the most recent active message with *role*, or ``None``. + + Two callers, same need — "the message I mean, without an id": the agent + defaulting to the turn that triggered it, and the desktop reacting to a + live message that hasn't round-tripped through a resume yet. + ``offset`` steps to earlier turns (1 = the one before the latest) so a + reaction can land retroactively — "two messages ago" is how the caller + thinks about it. + + ``require_text`` (default) skips rows with no plain-text content — + tool-call-only assistant turns and attachment stubs don't render as + bubbles, so "the latest message" as a HUMAN means it must never + resolve to one (a reaction landing on an invisible row looks dropped, + and its annotation quotes an empty string). + """ + if not session_id or role not in {"user", "assistant"} or offset < 0: + return None + + text_filter = ( + "AND content IS NOT NULL AND TRIM(content) != '' " if require_text else "" + ) + + row = self._read_one( + "SELECT id FROM messages WHERE session_id = ? AND role = ? " + f"AND active = 1 {text_filter}ORDER BY id DESC LIMIT 1 OFFSET ?", + (session_id, role, int(offset)), + ) + + return row[0] if row else None + + def latest_user_message_row_id(self, session_id: str) -> Optional[int]: + """Row id of the most recent active user message, or ``None``. + + The agent's default reaction target: "the message that triggered me", + so the model never has to thread row ids through a tool call (mirrors + the photon adapter's ``_record_last_inbound``). + """ + return self.latest_message_row_id(session_id, role="user") + + def get_message_role(self, session_id: str, row_id: int) -> Optional[str]: + """Role of the active message at *row_id* in *session_id*, or ``None``. + + Lets a reaction event carry the target's role so a renderer can match + a live message that doesn't know its durable row id yet. + """ + if not session_id: + return None + + row = self._read_one( + "SELECT role FROM messages WHERE id = ? AND session_id = ? AND active = 1", + (int(row_id), session_id), + ) + + return row[0] if row else None + + def _insert_message_rows(self, conn, session_id: str, messages: List[Dict[str, Any]]) -> tuple[int, int]: + """Insert *messages* as fresh active rows for *session_id*. + + Shared by :meth:`replace_messages` (delete-then-insert) and + :meth:`archive_and_compact` (soft-archive-then-insert). Runs inside the + caller's write transaction (takes the live ``conn``). Returns + ``(inserted_count, tool_call_count)``. Does NOT touch sessions.* counters + — the caller owns that, since the two flows reconcile counts differently. + """ + from hermes_state import _scrub_surrogates + now_ts = time.time() + inserted = 0 + tool_calls_total = 0 + for msg in messages: + role = msg.get("role", "unknown") + tool_calls = msg.get("tool_calls") + message_timestamp = now_ts + if msg.get("timestamp") is not None: + try: + ts_value = msg.get("timestamp") + if hasattr(ts_value, "timestamp"): + message_timestamp = float(ts_value.timestamp()) + else: + message_timestamp = float(ts_value) + except (TypeError, ValueError): + logger.debug("Ignoring invalid explicit message timestamp: %r", msg.get("timestamp")) + reasoning_details = msg.get("reasoning_details") if role == "assistant" else None + codex_reasoning_items = ( + msg.get("codex_reasoning_items") if role == "assistant" else None + ) + codex_message_items = ( + msg.get("codex_message_items") if role == "assistant" else None + ) + reasoning_details_json = self._reasoning_json_text(reasoning_details) + codex_items_json = self._reasoning_json_text(codex_reasoning_items) + codex_message_items_json = self._reasoning_json_text(codex_message_items) + # tool_calls may arrive as a Python list (from the live agent) + # or as a JSON string (from import_sessions / export_session, + # which store it as TEXT). json.dumps on an already-serialized + # string double-encodes it, so parse first. + if isinstance(tool_calls, str): + try: + tool_calls = json.loads(tool_calls) + except (json.JSONDecodeError, TypeError): + tool_calls = [] + tool_calls_json = json.dumps(tool_calls) if tool_calls else None + # Accept either `platform_message_id` (new explicit name) or + # `message_id` (yuanbao's existing convention on message dicts). + platform_msg_id = ( + msg.get("platform_message_id") or msg.get("message_id") + ) + + api_content = msg.get("api_content") + + cur = conn.execute( + """INSERT INTO messages (session_id, role, content, tool_call_id, + tool_calls, tool_name, effect_disposition, timestamp, token_count, finish_reason, + reasoning, reasoning_content, reasoning_details, codex_reasoning_items, + codex_message_items, platform_message_id, observed, _compressed_summary, active, api_content, display_kind, display_metadata) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + session_id, + role, + self._encode_content(msg.get("content")), + msg.get("tool_call_id"), + tool_calls_json, + _scrub_surrogates(msg.get("tool_name")), + msg.get("effect_disposition"), + message_timestamp, + msg.get("token_count"), + msg.get("finish_reason"), + _scrub_surrogates(msg.get("reasoning")) if role == "assistant" else None, + _scrub_surrogates(msg.get("reasoning_content")) if role == "assistant" else None, + reasoning_details_json, + codex_items_json, + codex_message_items_json, + platform_msg_id, + 1 if msg.get("observed") else 0, + 1 if msg.get("_compressed_summary") else 0, + 1, + _scrub_surrogates(api_content) if isinstance(api_content, str) else None, + _scrub_surrogates(msg.get("display_kind")) if isinstance(msg.get("display_kind"), str) else None, + self._encode_display_metadata(msg.get("display_metadata")), + ), + ) + if isinstance(msg, dict) and cur.lastrowid is not None: + msg["_row_id"] = cur.lastrowid + inserted += 1 + if tool_calls is not None: + tool_calls_total += ( + len(tool_calls) if isinstance(tool_calls, list) else 1 + ) + now_ts = max(now_ts + 1e-6, message_timestamp + 1e-6) + return inserted, tool_calls_total + + def replace_messages( + self, + session_id: str, + messages: List[Dict[str, Any]], + active_only: bool = False, + archive_dropped: bool = False, + reject_active_turn_lease: bool = False, + ) -> None: + """Atomically replace the stored messages for a session. + + Used by transcript-rewrite flows such as /retry, /undo, and /compress. + The delete + reinsert sequence must commit as one transaction so a + mid-rewrite failure does not leave SQLite with a partial transcript. + + DESTRUCTIVE by default: every row for the session is DELETEd (and drops + out of the FTS index). For compaction that must preserve the + pre-compaction transcript under the same id, use + :meth:`archive_and_compact` instead. + + Pass ``active_only=True`` to replace ONLY the live (``active = 1``) rows, + leaving soft-archived rows (``active = 0`` — e.g. the ``compacted = 1`` + turns that :meth:`archive_and_compact` keeps on disk for #38763 + durability, or rewind/undo rows) untouched. Callers that share a session + id with an agent already running in-place compaction must use this so a + full-history rewrite doesn't wipe the rows the agent deliberately + archived. ``message_count``/``tool_call_count`` then track the live set, + matching :meth:`archive_and_compact`. + + Pass ``archive_dropped=True`` to SOFT-archive the live rows instead of + DELETEing them: the replaced turns stay on disk with ``active = 0``, + ``compacted = 0`` — the same "the user took it back" marking + :meth:`rewind_to_message` applies — and stay readable via + :meth:`get_messages` with ``include_inactive=True``. This is the mode a + rewind/edit/regenerate must use: those flows overwrite a transcript the + user may not have meant to drop, and a plain DELETE also evicts the rows + from the FTS index, leaving nothing to recover from (#82756). It implies + active-only handling — already-archived rows are never touched — so + ``active_only`` is redundant with it. The rewritten set is inserted as + fresh active rows exactly as in the destructive path, so the live view + is identical either way; only the durability of the dropped turns + differs. + + Pass ``reject_active_turn_lease=True`` for user-initiated rewrites that + do not already own the cross-process turn lease. The lease check and + transcript mutation then share one write transaction, so a second + process cannot archive or replace a turn that is still being produced. + """ + from hermes_state import CompressionSessionClosedError + + active_clause = " AND active = 1" if active_only else "" + + def _do(conn): + if reject_active_turn_lease: + self._check_transcript_write_guards( + conn, + session_id, + None, + reject_active_turn_lease=True, + reject_active_compression_lock=True, + ) + else: + session = conn.execute( + "SELECT ended_at, end_reason FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + if ( + session is not None + and session["ended_at"] is not None + and session["end_reason"] == "compression" + ): + raise CompressionSessionClosedError(session_id) + if archive_dropped: + # Content-preserving UPDATE: the rows keep their FTS entries + # (the messages_fts triggers fire on INSERT / DELETE / UPDATE + # of content columns, not on `active`), so the replaced turns + # stay readable via get_messages(include_inactive=True) and + # searchable with include_inactive=True after the rewrite. + conn.execute( + "UPDATE messages SET active = 0 " + "WHERE session_id = ? AND active = 1", + (session_id,), + ) + else: + conn.execute( + f"DELETE FROM messages WHERE session_id = ?{active_clause}", + (session_id,), + ) + conn.execute( + "UPDATE sessions SET message_count = 0, tool_call_count = 0 WHERE id = ?", + (session_id,), + ) + total_messages, total_tool_calls = self._insert_message_rows( + conn, session_id, messages + ) + conn.execute( + "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", + (total_messages, total_tool_calls, session_id), + ) + + self._execute_write(_do) + + def has_archived_messages(self, session_id: str) -> bool: + """Return True if the session has any soft-archived (``active = 0``) rows. + + Cheap existence probe — does not load rows. NOTE: production rewrite + paths no longer branch on this (they pass ``active_only=True`` + unconditionally — a probe can fail open or race a concurrent + ``archive_and_compact``, #80216); kept for tests and diagnostics. + """ + return self._read_one( + "SELECT 1 FROM messages WHERE session_id = ? AND active = 0 LIMIT 1", + (session_id,), + ) is not None + + def get_active_message_watermark(self, session_id: str) -> int: + """MAX(id) of the session's active rows — the compression watermark. + + Captured at compression START (before the slow provider summary call). + Every active row with ``id > watermark`` at commit time arrived + concurrently and must survive the compaction verbatim. Returns 0 for + an empty/unknown session. + """ + if not session_id: + return 0 + row = self._read_one( + "SELECT COALESCE(MAX(id), 0) FROM messages " + "WHERE session_id = ? AND active = 1", + (session_id,), + ) + return int(row[0]) if row else 0 + + def archive_and_compact( + self, + session_id: str, + compacted_messages: List[Dict[str, Any]], + model_config_patch: Optional[Dict[str, Any]] = None, + watermark: Optional[int] = None, + lock_holder: Optional[str] = None, + tail_count: int = 0, + ) -> int: + """Non-destructive in-place compaction for a single durable session id. + + Soft-archives the active messages (``active = 0``) and inserts + *compacted_messages* as fresh active rows — atomically, in one write + transaction. The conversation keeps ONE session id for life (#38763) + WITHOUT destroying history: + + - The live-context load (:meth:`get_messages_as_conversation`, + :meth:`get_messages`) filters ``active = 1`` by default, so the model + reloads ONLY the compacted set. + - The archived pre-compaction turns stay on disk (active=0) and stay + DISCOVERABLE: they are marked compacted=1, and search_messages() + includes compacted=1 rows by default — so session_search still finds + them, unlike rewind/undo rows (active=0, compacted=0) which stay + hidden. They remain in the FTS index (the messages_fts* triggers + index on INSERT / drop on DELETE and don't key on active/compacted; + flipping to active=0 is a content-preserving UPDATE) and are + recoverable via get_messages(..., include_inactive=True). + + Concurrent-append safety (#75316): when *watermark* is provided (the + value of :meth:`get_active_message_watermark` captured at compression + START), rows that arrived during the slow provider summary call + (``id > watermark``) are NOT summarized away. They are re-sequenced + after the compacted set by a pure-SQL column clone (every column + except ``id`` — content, api_content, platform_message_id, token + counts, reasoning sidecars all survive byte-exact, and the FTS + triggers index the clones naturally), and the originals are archived. + NOTE: re-sequencing assigns the tail rows fresh ids; consumers that + reference durable row ids re-resolve by content (see 3e8ab0610). + ``watermark=None`` preserves the historical archive-everything + behavior. + + Commit-fence safety: when *lock_holder* is provided, the commit + verifies INSIDE the transaction that the compression lock is still + held by that holder and unexpired — a compression whose lease was + reclaimed (crash cleanup, TTL expiry, competing writer) fails the + commit instead of clobbering the winner's transcript. + + *tail_count* (default 0) names how many of the LAST rows of + *compacted_messages* are the verbatim carried-forward tail the + compressor protected rather than summarized (#86366). Those rows' + ORIGINALS — which this call archives as a side effect of the blanket + soft-archive — are superseded byte-identical duplicates, not + "summarized away" content, so they are stamped rewind-style + (``active=0, compacted=0``, hidden from search_messages) instead of + ``compacted=1``. Without this the tail originals satisfy the recall + filter alongside their live clones and session_search returns every + carried-forward message once per compaction. Callers that cannot know + their tail shape keep the historical archive-everything behavior. + + ``message_count`` is set to the ACTIVE count after commit, matching + what the live load returns. ``model_config_patch`` is merged into the + session's JSON config in the same transaction; a ``None`` value + removes that key. Returns the new active count. + """ + from hermes_state import SessionCompressionInProgressError + + def _do(conn): + if lock_holder is not None: + lock_row = conn.execute( + "SELECT holder, expires_at FROM compression_locks " + "WHERE session_id = ?", + (session_id,), + ).fetchone() + if ( + lock_row is None + or lock_row["holder"] != lock_holder + or float(lock_row["expires_at"]) <= time.time() + ): + raise SessionCompressionInProgressError( + f"Compression lease for {session_id!r} lost before " + "commit; refusing to publish a stale compaction" + ) + + patched_model_config = None + if model_config_patch is not None: + # on_missing="raise": a prune/compaction must not commit + # against a vanished session row (the compressor's caller + # converts the raised error into a safe keep-the-original + # no-op), unlike the flag setters which tolerate missing rows. + patched_model_config = self._merge_model_config_json( + conn, session_id, model_config_patch, on_missing="raise" + ) + + # Concurrent tail: active rows that arrived after the watermark. + # Snapshot their ids and tool_calls now — the clone below needs a + # stable id list, and the tool-call count keeps sessions.* honest. + tail_ids: list[int] = [] + tail_tool_calls = 0 + if watermark is not None: + for row in conn.execute( + "SELECT id, tool_calls FROM messages " + "WHERE session_id = ? AND active = 1 AND id > ? " + "ORDER BY id", + (session_id, int(watermark)), + ).fetchall(): + tail_ids.append(int(row["id"])) + raw = row["tool_calls"] + if raw: + try: + parsed = json.loads(raw) if isinstance(raw, str) else raw + tail_tool_calls += len(parsed) if isinstance(parsed, list) else 0 + except (TypeError, ValueError): + pass + + # Soft-archive the live turns: active=0 hides them from the live + # context load, compacted=1 marks them as "summarized away" (vs + # rewind/undo's active=0+compacted=0, which means "user took it + # back"). search_messages includes compacted=1 rows by default so + # the pre-compaction transcript stays discoverable; live-context + # loads (active=1 only) still exclude them. Tail originals whose + # verbatim clones ride inside *compacted_messages* (tail_count) + # are superseded duplicates instead (#86366): they get the + # rewind-style flags so they stop matching the recall filter. + # Rewind-target ids: the originals of the carried-forward tail + # rows (tail_count), captured BEFORE any flag flips. Named apart + # from the watermark `tail_ids` below on purpose — the two are + # different sets (#86366): rewind targets sit AT/BELOW the + # watermark (the compressor only saw rows up to it), while + # `tail_ids` are concurrent appends ABOVE it. Without the bound, + # a concurrent append would steal a LIMIT slot and leave a real + # carried-forward original stamped compacted=1. + rewind_tail_ids: Optional[list[int]] = None + if tail_count > 0: + if watermark is not None: + tail_rows = conn.execute( + "SELECT id FROM messages " + "WHERE session_id = ? AND active = 1 AND id <= ? " + "ORDER BY id DESC LIMIT ?", + (session_id, int(watermark), int(tail_count)), + ).fetchall() + else: + tail_rows = conn.execute( + "SELECT id FROM messages " + "WHERE session_id = ? AND active = 1 ORDER BY id DESC LIMIT ?", + (session_id, int(tail_count)), + ).fetchall() + rewind_tail_ids = [int(row["id"]) for row in tail_rows] + + # The watermark clone below re-inserts `tail_ids` rows byte-exact + # as live rows — their originals are the SAME superseded-duplicate + # class as the carried-forward tail (#86366), so they take the + # rewind flags too instead of double-matching the recall filter. + rewind_ids = [*(rewind_tail_ids or []), *tail_ids] + + if rewind_ids: + placeholders = ",".join("?" for _ in rewind_ids) + conn.execute( + "UPDATE messages SET active = 0, compacted = 0 " + f"WHERE session_id = ? AND id IN ({placeholders})", + [session_id, *rewind_ids], + ) + conn.execute( + "UPDATE messages SET active = 0, compacted = 1 " + "WHERE session_id = ? AND active = 1 " + f"AND id NOT IN ({placeholders})", + [session_id, *rewind_ids], + ) + else: + conn.execute( + "UPDATE messages SET active = 0, compacted = 1 " + "WHERE session_id = ? AND active = 1", + (session_id,), + ) + inserted, tool_calls_total = self._insert_message_rows( + conn, session_id, compacted_messages + ) + + if tail_ids: + # Re-sequence the concurrent tail after the compacted set via + # a pure-SQL column clone: no decode/re-encode round trip, no + # field drift — new id, active=1, compacted=0, all else exact. + placeholders = ",".join("?" for _ in tail_ids) + clone_cols = [ + c for c in self._message_column_names(conn) + if c not in ("id", "active", "compacted") + ] + col_list = ", ".join(clone_cols) + conn.execute( + f"INSERT INTO messages ({col_list}, active, compacted) " + f"SELECT {col_list}, 1, 0 FROM messages " + f"WHERE id IN ({placeholders}) ORDER BY id", + tail_ids, + ) + inserted += len(tail_ids) + tool_calls_total += tail_tool_calls + + # message_count / tool_call_count reflect the LIVE (active) set — + # the archived rows are still on disk but not part of the live count. + if model_config_patch is None: + conn.execute( + "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", + (inserted, tool_calls_total, session_id), + ) + else: + conn.execute( + "UPDATE sessions SET message_count = ?, tool_call_count = ?, " + "model_config = ? WHERE id = ?", + (inserted, tool_calls_total, patched_model_config, session_id), + ) + return inserted + + return self._execute_write(_do) + + def _message_column_names(self, conn) -> List[str]: + """Column names of the messages table, cached per-connection era.""" + cached = getattr(self, "_message_columns_cache", None) + if cached: + return cached + cols = [r[1] for r in conn.execute("PRAGMA table_info(messages)").fetchall()] + self._message_columns_cache = cols + return cols + + def set_latest_user_api_content( + self, session_id: str, content: Any, api_content: str + ) -> int: + """Backfill the ``api_content`` sidecar onto the newest ACTIVE user row. + + In-place preflight compaction (:meth:`archive_and_compact`) inserts the + current turn's user row BEFORE the turn prologue composes the + prefetch/plugin sidecar, and the subsequent crash persist identity-skips + every compacted dict — without this backfill the stamped sidecar would + never land in the DB and any reload would replay clean content, + re-introducing the prompt-cache divergence the sidecar exists to close. + + The ``content`` match is a defensive guard: if the newest active user + row is not the message the caller stamped (racing rewrite, unexpected + tail shape), nothing is written. Returns the number of rows updated + (0 or 1). + """ + from hermes_state import _scrub_surrogates + encoded = self._encode_content(content) + + return self._write_rowcount( + "UPDATE messages SET api_content = ? WHERE id = (" + "SELECT id FROM messages " + "WHERE session_id = ? AND role = 'user' AND active = 1 " + "ORDER BY id DESC LIMIT 1" + ") AND content IS ?", + (_scrub_surrogates(api_content), session_id, encoded), + ) + + def _dedupe_display_generations(self, rows): + """Collapse compaction generations so each message appears once. + + Compaction epochs copy the protected tail into each new generation, so + one logical message can exist as several rows (identical + role/content/timestamp) with different ``active`` flags and ids. A + display read must surface each exactly once: prefer the live row, then + the newest generation. + + This is the ONE definition shared by every display projection — + :meth:`get_messages` (REST), :meth:`get_resume_conversations` and + :meth:`get_ancestor_display_prefix` (gateway resume), and + :meth:`get_messages_as_conversation` (warm-session payload) — so the + surfaces cannot disagree about the same transcript. *rows* must already + be ordered by ``id``; the returned list keeps that order. + """ + seen: Dict[Tuple[Any, ...], Any] = {} + for row in rows: + dedupe_content = row["content"] + if row["role"] == "user": + from agent.context_compressor import split_user_originated_turn + + candidate = { + "role": "user", + "content": self._decode_content(row["content"]), + "display_kind": row["display_kind"], + "display_metadata": self._decode_display_metadata( + row["display_metadata"] + ), + } + handoff, live_view = split_user_originated_turn(candidate) + if handoff is not None and live_view is not None: + dedupe_content = self._encode_content(live_view.get("content")) + # Tool fields participate in the dedupe key: compaction copies them + # verbatim, so identical tool messages across generations still + # collapse, while distinct tool calls that happen to share + # role/content/timestamp are never merged. + key = ( + row["role"], + dedupe_content, + row["timestamp"], + row["tool_call_id"], + row["tool_calls"], + row["tool_name"], + ) + cur = seen.get(key) + if cur is None or (row["active"], row["id"]) > (cur["active"], cur["id"]): + seen[key] = row + return sorted(seen.values(), key=lambda r: r["id"]) + + def get_messages( + self, + session_id: str, + include_inactive: bool = False, + include_compacted: bool = False, + limit: Optional[int] = None, + offset: int = 0, + latest: bool = False, + after_id: Optional[int] = None, + ) -> List[Dict[str, Any]]: + """Load messages for a session in insertion order. + + By default only active messages are returned. Pass + ``include_inactive=True`` to load soft-deleted rows (e.g. for + audit / debug views of rewound history). See + :meth:`rewind_to_message` for the soft-delete mechanic. + + Pass ``include_compacted=True`` to additionally load rows preserved + by in-place context compaction (``active=0, compacted=1``). Those are + durable display history, not soft-deleted rows — a user-visible + transcript read must not drop them, or earlier turns silently become + unreachable once the UI exhausts its active-only window. Soft-deleted + Undo/Rewind rows (``active=0, compacted=0``) stay excluded; use + ``include_inactive`` for those. + + Ordered by AUTOINCREMENT id (true insertion order) rather than + timestamp — see c03acca50 for the WSL2 clock-regression rationale. + + When ``limit`` is provided, returns at most ``limit`` messages + starting from ``offset`` (0-based, in insertion order). Enables + pagination for the API endpoint to avoid loading entire transcripts. + With ``latest=True``, the offset is measured back from the newest + message and the selected page is still returned in chronological + order. ``offset`` alone (without ``limit``) also pages — SQLite + requires a LIMIT clause for OFFSET, so it's emitted as ``LIMIT -1`` + (unbounded). + + ``after_id`` enables keyset pagination (``id > after_id``): O(1) + page seeks on huge transcripts where OFFSET degrades to O(n) per + page. Ascending order only (incompatible with ``latest``/``offset``). + """ + if after_id is not None and (latest or offset): + raise ValueError("after_id is incompatible with latest/offset paging") + if after_id is not None and include_compacted: + raise ValueError("after_id is incompatible with include_compacted (deduped display reads use offset paging)") + if include_inactive: + # Audit / debug reads: every row, including soft-deleted. + active_clause = "" + elif include_compacted: + # Display history: active rows plus rows preserved by in-place + # compaction (active=0, compacted=1), but never soft-deleted + # Undo/Rewind rows (active=0, compacted=0). + active_clause = " AND (active = 1 OR compacted = 1)" + else: + active_clause = " AND active = 1" + keyset_clause = " AND id > ?" if after_id is not None else "" + sql = ( + "SELECT * FROM messages WHERE session_id = ?" + f"{active_clause}{keyset_clause} ORDER BY id {'DESC' if latest else 'ASC'}" + ) + params: list = [session_id] + if after_id is not None: + params.append(after_id) + if include_compacted: + # Read the full display set (a session's rows are bounded; the + # UI-level 500-row cap lives in the endpoint, not here), dedupe + # generations, then apply paging. + all_rows = self._read_all( + "SELECT * FROM messages WHERE session_id = ?" + active_clause + + " ORDER BY id ASC", + [session_id], + ) + rows = self._dedupe_display_generations(all_rows) + if latest: + rows = rows[::-1] + rows = rows[offset:] + if limit is not None: + rows = rows[:limit] + if latest: + rows = rows[::-1] + else: + if limit is not None or offset: + # SQLite's OFFSET requires LIMIT; -1 means "no limit". + sql += " LIMIT ? OFFSET ?" + params.extend([-1 if limit is None else limit, offset]) + rows = self._read_all(sql, params) + if latest: + rows.reverse() + result = [] + for row in rows: + msg = dict(row) + if msg.pop("_compressed_summary", 0): + msg["_compressed_summary"] = True + if "content" in msg: + msg["content"] = self._decode_content(msg["content"]) + if msg.get("tool_calls"): + try: + msg["tool_calls"] = json.loads(msg["tool_calls"]) + except (json.JSONDecodeError, TypeError): + logger.warning("Failed to deserialize tool_calls in get_messages, falling back to []") + msg["tool_calls"] = [] + if msg.get("display_metadata") is not None: + msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"]) + result.append(msg) + return result + + def find_pr_url_messages(self, session_ids: List[str]) -> List[Dict[str, Any]]: + """Tool results in these sessions that mention a GitHub PR url. + + A candidate scan, deliberately loose: it hands back every tool result + containing ``/pull/`` and leaves the caller to decide which ones make a + claim (see the desktop's PR recovery, which only accepts an output that + is a bare PR url — the signature of ``gh pr create``). Ordered + oldest-first per session so the caller can take the last match. + """ + found: List[Dict[str, Any]] = [] + ids = [s for s in session_ids if s] + for start in range(0, len(ids), 900): # SQLite's bound-variable ceiling. + chunk = ids[start : start + 900] + placeholders = ",".join("?" * len(chunk)) + rows = self._read_all( + f"""SELECT session_id, content FROM messages + WHERE session_id IN ({placeholders}) + AND role = 'tool' AND content LIKE '%/pull/%' + ORDER BY id ASC""", + chunk, + ) + found.extend({"session_id": row[0], "content": row[1]} for row in rows) + return found + + def get_messages_around( + self, + session_id: str, + around_message_id: int, + window: int = 5, + ) -> Dict[str, Any]: + """Load a window of messages anchored on a specific message id. + + Returns a dict with: + - ``window``: up to ``window`` messages before the anchor, the anchor + itself, and up to ``window`` messages after, ordered by id ascending. + - ``messages_before``: count of messages strictly before the anchor + still in the session (== window unless we hit the start). + - ``messages_after``: count of messages strictly after the anchor + still in the session (== window unless we hit the end). + + Used by ``session_search`` for both the discovery shape (anchored on the + FTS5 match) and the scroll shape (anchored on any message id). The + ``messages_before`` / ``messages_after`` counts let the caller detect + session boundaries: when either is less than ``window``, the agent has + reached one end of the session. + + Returns an empty window when ``around_message_id`` is not a real id in + ``session_id`` — callers decide how to surface that. + """ + if window < 0: + window = 0 + with self._read_ctx() as conn: + # Confirm the anchor exists in this session. + anchor_exists = conn.execute( + "SELECT 1 FROM messages WHERE id = ? AND session_id = ? LIMIT 1", + (around_message_id, session_id), + ).fetchone() + if not anchor_exists: + return {"window": [], "messages_before": 0, "messages_after": 0} + + # Two queries: anchor + before (DESC, take window+1), and after + # (ASC, take window). Final order is id ASC. + before_rows = conn.execute( + "SELECT * FROM messages " + "WHERE session_id = ? AND id <= ? " + "ORDER BY id DESC LIMIT ?", + (session_id, around_message_id, window + 1), + ).fetchall() + after_rows = conn.execute( + "SELECT * FROM messages " + "WHERE session_id = ? AND id > ? " + "ORDER BY id ASC LIMIT ?", + (session_id, around_message_id, window), + ).fetchall() + + # before_rows is DESC; reverse so it's ASC, then concatenate after_rows. + rows = list(reversed(before_rows)) + list(after_rows) + result = [] + for row in rows: + msg = dict(row) + if "content" in msg: + msg["content"] = self._decode_content(msg["content"]) + if msg.get("tool_calls"): + try: + msg["tool_calls"] = json.loads(msg["tool_calls"]) + except (json.JSONDecodeError, TypeError): + logger.warning( + "Failed to deserialize tool_calls in get_messages_around, falling back to []" + ) + msg["tool_calls"] = [] + if msg.get("display_metadata") is not None: + msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"]) + result.append(msg) + + # before_rows includes the anchor itself; subtract 1 for the count of + # messages strictly before the anchor in the returned slice. + messages_before = max(0, len(before_rows) - 1) + messages_after = len(after_rows) + return { + "window": result, + "messages_before": messages_before, + "messages_after": messages_after, + } + + def resolve_resume_session_id(self, session_id: str) -> str: + """Redirect a resume target to the descendant session that holds the messages. + + Context compression ends the current session and forks a new child session + (linked via ``parent_session_id``). The flush cursor is reset, so the + child is where new messages actually land — the parent ends up with + ``message_count = 0`` rows unless messages had already been flushed to + it before compression. See #15000. + + This helper walks ``parent_session_id`` forward from ``session_id`` and + returns the descendant in the chain that has the **most recent** messages. + Unlike the original logic, it does NOT short-circuit when the starting + session already has messages — a descendant that was created by + compression may hold the continuation content and should be preferred + by the WebUI and gateway for ``--resume`` and session loading. + + If no descendant (including the starting session) has any messages, + the original ``session_id`` is returned unchanged. + + The chain is always walked via the child whose ``started_at`` is + latest; that matches the single-chain shape that compression creates. + A depth cap (32) guards against accidental loops in malformed data. + """ + if not session_id: + return session_id + + # Follow the compression-continuation chain forward to the live tip + # FIRST. Auto-compression ends the current session and forks a + # continuation child, but a long-lived parent keeps its own flushed + # message rows — so the empty-head walk below never redirects it, and + # resuming the parent id reloads the pre-compression transcript while + # the turns generated *after* compression (and their responses) sit in + # the continuation. ``get_compression_tip`` is lineage-aware: it only + # follows children whose parent ended with ``end_reason='compression'`` + # (created after the parent was ended), so delegation / branch children + # never hijack the resume. This is the fix for the desktop "I came back + # and the reply isn't there" report on large sessions. + try: + tip = self.get_compression_tip(session_id) + except Exception: + tip = session_id + if tip and tip != session_id: + session_id = tip + + with self._read_ctx() as conn: + current = session_id + seen = {current} + best = None # tracks the last (deepest) node with messages + + for _ in range(32): + # Check if the current node has messages. + try: + row = conn.execute( + "SELECT 1 FROM messages WHERE session_id = ? LIMIT 1", + (current,), + ).fetchone() + except Exception: + return session_id + if row is not None: + best = current + + # Walk to the most-recently-started child — but skip explicit + # branch (`_branched_from`), delegate/subagent (`_delegate_from`), + # reset-continuation (`_reset_from` or the legacy same-key + # heuristic — a post-reset conversation must never be reached + # by resuming the parent the user reset away), and tool + # children. They also carry a ``parent_session_id`` yet + # are NOT compression continuations; following them would hijack + # the resume target to an unrelated session (e.g. a subagent + # run). This mirrors the child-exclusion in ``get_compression_tip``. + try: + child_row = conn.execute( + "SELECT id FROM sessions AS child " + "WHERE child.parent_session_id = ? " + " AND json_extract(COALESCE(child.model_config, '{}'), '$._branched_from') IS NULL " + " AND json_extract(COALESCE(child.model_config, '{}'), '$._delegate_from') IS NULL " + " AND json_extract(COALESCE(child.model_config, '{}'), '$._reset_from') IS NULL " + f" AND NOT {_legacy_reset_child_sql('child', _RESET_END_REASONS_SQL)} " + " AND COALESCE(child.source, '') != 'tool' " + "ORDER BY child.started_at DESC, child.id DESC LIMIT 1", + (current,), + ).fetchone() + except Exception: + return session_id + if child_row is None: + break + child_id = child_row["id"] if hasattr(child_row, "keys") else child_row[0] + if not child_id or child_id in seen: + break + seen.add(child_id) + current = child_id + + return best if best is not None else session_id + + def get_messages_as_conversation( + self, + session_id: str, + include_ancestors: bool = False, + include_inactive: bool = False, + repair_alternation: bool = False, + include_row_ids: bool = False, + include_compacted: bool = False, + ) -> List[Dict[str, Any]]: + """ + Load messages in the OpenAI conversation format (role + content dicts). + Used by the gateway to restore conversation history. + + By default only active messages are returned. Pass + ``include_inactive=True`` to load soft-deleted (rewound) rows + as well. See :meth:`rewind_to_message`. + + ``include_compacted=True`` additionally loads rows preserved by + in-place compaction (``active=0, compacted=1``), deduped by + :meth:`_dedupe_display_generations`. DISPLAY reads want this; the + model-fed restore must NOT pass it, or a resumed session regrows the + very history compaction just summarized away. + + ``repair_alternation=True`` runs ``repair_message_sequence`` over the + loaded list before returning it. Callers that restore a session for + LIVE REPLAY should pass it: a durable alternation violation (e.g. a + ``user;user`` pair left by a turn that persisted no assistant row) + otherwise re-triggers the pre-request defensive repair on every + single request for the rest of the session's life — the repair + mutates only the per-request list, never the stored transcript. + Inspection/export consumers keep the default and see the transcript + verbatim. + """ + session_ids = [session_id] + if include_ancestors and not self._is_explicit_branch_session(session_id): + session_ids = self._session_lineage_root_to_tip(session_id) + + if include_inactive: + active_clause = "" + elif include_compacted: + active_clause = " AND (active = 1 OR compacted = 1)" + else: + active_clause = " AND active = 1" + with self._read_ctx() as conn: + placeholders = ",".join("?" for _ in session_ids) + rows = conn.execute( + f"SELECT {self._CONVERSATION_ROW_COLUMNS} " + f"FROM messages WHERE session_id IN ({placeholders})" + # Order by AUTOINCREMENT id (true insertion order), NOT timestamp: + # append_message stamps rows with time.time(), which is not + # monotonic (WSL2, NTP steps, VM/laptop sleep resume). A later + # row can carry an earlier timestamp than its predecessor, and + # ORDER BY timestamp would then sort an assistant tool_calls row + # after its tool response, breaking tool-call/response adjacency + # and triggering an HTTP 400 on replay. This matches get_messages + # — see c03acca50 for the original fix. + f"{active_clause} ORDER BY id", + tuple(session_ids), + ).fetchall() + + if include_compacted: + rows = self._dedupe_display_generations(rows) + + return self._rows_to_conversation( + rows, + session_id=session_id, + include_ancestors=include_ancestors, + repair_alternation=repair_alternation, + include_row_ids=include_row_ids, + ) + + def _rows_to_conversation( + self, + rows, + *, + session_id: str, + include_ancestors: bool, + repair_alternation: bool, + include_row_ids: bool = False, + include_summary_markers: bool = False, + ) -> List[Dict[str, Any]]: + """Decode fetched message rows into the OpenAI conversation format. + + Extracted from get_messages_as_conversation so get_resume_conversations + can build the model-fed and display views from one SELECT. ``rows`` must + already be ordered by ``id`` (insertion order) and filtered to the + desired session set / active state by the caller. + """ + from hermes_state import _strip_background_review_harness, _strip_stale_tool_call_markers + messages = [] + # Watermark rotation column-clones concurrent tail rows into the child + # after the new summary, so the copies need not be adjacent. Index the + # exact durable clone identity while decoding instead of rescanning the + # whole accumulated lineage for every user row. + exact_user_clones: Dict[Tuple[Any, str], Dict[str, Any]] = {} + for row in rows: + content = self._decode_content(row["content"]) + if row["role"] in {"user", "assistant"} and isinstance(content, str): + content = sanitize_context(content).strip() + msg = {"role": row["role"], "content": content} + # Born durable (#92231): this dict is materialized FROM a durable + # row, so stamp the persistence marker at the source instead of + # relying on every restore caller to thread the loaded list back + # through a flush as ``conversation_history=`` — any + # identity-losing handoff (compression's durable-snapshot + # adoption, incremental persists with no history arg) would + # otherwise re-append the ENTIRE transcript on flush. + # Underscore-prefixed like ``_row_id``: every transport strips it + # before the wire, and compression's assembly copies deliberately + # strip it so rotated child handoffs still flush (see + # _fresh_compaction_message_copy). + msg[_DB_PERSISTED_MARKER_KEY] = True + # Durable per-message identity for surfaces that need to address a + # specific row later (desktop reactions). OPT-IN: only the gateway + # asks for it — every other consumer (ACP restore, export, + # inspection) gets the transcript in its historical shape. + # Underscore-prefixed so every transport's convert_messages() + # strips it before the wire. + if include_row_ids and row["id"] is not None: + msg["_row_id"] = row["id"] + # api_content is the byte-fidelity sidecar: the exact string sent + # to the API when it differed from the clean content. Returned + # VERBATIM — no sanitize_context, no strip — because the replay + # path substitutes it for content to keep the provider prompt + # cache prefix byte-stable across turns. Cleaning it here would + # re-introduce the divergence it exists to remove. + if row["api_content"]: + msg["api_content"] = row["api_content"] + if row["display_kind"]: + msg["display_kind"] = row["display_kind"] + if row["display_metadata"]: + decoded = self._decode_display_metadata(row["display_metadata"]) + if decoded is not None: + msg["display_metadata"] = decoded + if include_summary_markers and row["_compressed_summary"]: + msg["_compressed_summary"] = True + if row["timestamp"]: + msg["timestamp"] = row["timestamp"] + if row["tool_call_id"]: + msg["tool_call_id"] = row["tool_call_id"] + if row["tool_name"]: + msg["tool_name"] = row["tool_name"] + if row["effect_disposition"]: + msg["effect_disposition"] = row["effect_disposition"] + if row["tool_calls"]: + try: + msg["tool_calls"] = json.loads(row["tool_calls"]) + except (json.JSONDecodeError, TypeError): + logger.warning("Failed to deserialize tool_calls in conversation replay, falling back to []") + msg["tool_calls"] = [] + # Surface the platform-side message id (e.g. yuanbao msg_id, + # telegram update_id) so platform-specific flows like recall + # can match by external identifier instead of having to fall + # back to content-match heuristics. Exposed as ``message_id`` + # for backward compatibility with the JSONL transcript shape. + if row["platform_message_id"]: + msg["message_id"] = row["platform_message_id"] + if row["observed"]: + msg["observed"] = True + # Restore reasoning fields on assistant messages so providers + # that replay reasoning (OpenRouter, OpenAI, Nous) receive + # coherent multi-turn reasoning context. + if row["role"] == "assistant": + if row["finish_reason"]: + msg["finish_reason"] = row["finish_reason"] + if row["reasoning"]: + msg["reasoning"] = row["reasoning"] + if row["reasoning_content"] is not None: + msg["reasoning_content"] = row["reasoning_content"] + if row["reasoning_details"]: + try: + msg["reasoning_details"] = json.loads(row["reasoning_details"]) + except (json.JSONDecodeError, TypeError): + logger.warning("Failed to deserialize reasoning_details, falling back to None") + msg["reasoning_details"] = None + if row["codex_reasoning_items"]: + try: + msg["codex_reasoning_items"] = json.loads(row["codex_reasoning_items"]) + except (json.JSONDecodeError, TypeError): + logger.warning("Failed to deserialize codex_reasoning_items, falling back to None") + msg["codex_reasoning_items"] = None + if row["codex_message_items"]: + try: + msg["codex_message_items"] = json.loads(row["codex_message_items"]) + except (json.JSONDecodeError, TypeError): + logger.warning("Failed to deserialize codex_message_items, falling back to None") + msg["codex_message_items"] = None + if include_ancestors: + canonical_content, _is_composite = ( + self._canonical_replayed_user_content(msg) + ) + exact_clone_key = self._exact_replayed_user_clone_key( + msg.get("timestamp"), canonical_content + ) + previous_exact = ( + exact_user_clones.get(exact_clone_key) + if exact_clone_key is not None + else None + ) + duplicate = None + if previous_exact is not None: + previous_index = next( + ( + index + for index, candidate in enumerate(messages) + if candidate is previous_exact + ), + None, + ) + if previous_index is not None: + duplicate = (previous_index, True) + if duplicate is None: + duplicate = self._find_duplicate_replayed_user_message( + messages, msg + ) + if duplicate is not None: + duplicate_index, prefer_current = duplicate + if prefer_current: + # A rotated compression child can carry the same live + # ask as the parent row plus the only surviving summary + # scaffold. Keep the child carrier (and its durable row + # id), not the simpler ancestor copy. + messages.pop(duplicate_index) + else: + continue + messages.append(msg) + if include_ancestors and exact_clone_key is not None: + exact_user_clones[exact_clone_key] = msg + # DEFENSE-IN-DEPTH against background-review session pollution: a forked + # skill/memory review that (in older builds, before the _persist_disabled + # fix) shared the parent's session_id wrote its harness turn into this + # real session. The harness is a user/system message instructing the + # agent to "Review the conversation above and update the skill library / + # save to memory" under a hard tool restriction; re-loading it as live + # history makes the agent adopt the curator role and refuse the user's + # actual task. Strip any such harness message AND the curator-mode + # assistant reply immediately following it, so a polluted session + # resumes clean even if stray rows exist. + messages = _strip_background_review_harness(messages) + # DEFENSE-IN-DEPTH against #78148: before that fix, a bare tool-call + # marker (e.g. "[memory]") could get cached as a fallback and + # persisted as if it were the model's real answer. Sessions written + # before the fix can still carry those rows — clear the stray + # content on load so replaying history doesn't re-teach the model + # to keep emitting the marker. No-op for unaffected sessions. + messages = _strip_stale_tool_call_markers(messages) + if repair_alternation and messages: + # Lazy import: hermes_state already depends on agent.* (see + # sanitize_context above), but keep this optional path from + # widening the import surface at module load. + from agent.agent_runtime_helpers import repair_message_sequence + + repaired = repair_message_sequence(None, messages) + if repaired: + logger.info( + "Repaired %d message-alternation violation(s) while " + "restoring session %s — durable transcript kept them, " + "see repair_message_sequence", + repaired, + session_id, + ) + return messages + + def get_resume_conversations( + self, session_id: str + ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + """Return ``(model_history, display_history)`` for a session resume in ONE SELECT. + + ``session.resume`` needs two projections of the same lineage: + + - ``model_history`` — the tip session's active rows, alternation-repaired + (the live-replay working conversation). Equivalent to + ``get_messages_as_conversation(session_id, repair_alternation=True)``. + - ``display_history`` — the full compression lineage (ancestors → tip), + verbatim, with replayed-user dedup. Explicit ``/branch`` sessions are + excluded from this lineage because their own rows already contain the + copied transcript; including the live parent's rows would let messages + written to the original after the fork leak into the branch. + + The display projection also includes rows preserved by IN-PLACE + compaction (``active=0, compacted=1``), deduped by + :meth:`_dedupe_display_generations`. Without them a compacted + conversation resumes showing only its summary plus the carried-forward + tail — the user's own turns read as deleted even though every row is + still on disk, and the REST transcript read (which has always included + them) disagreed with this one about the same session (#92080). + + The display fetch already reads a superset of the model fetch (the tip + rows are part of the lineage), so serving both from one lineage SELECT + halves the resume's DB work versus two separate calls, with byte-identical + output (see test_get_resume_conversations_matches_separate_reads). + """ + session_ids = self._resume_lineage_ids(session_id) + with self._read_ctx() as conn: + placeholders = ",".join("?" for _ in session_ids) + rows = conn.execute( + f"SELECT session_id, {self._CONVERSATION_ROW_COLUMNS} " + f"FROM messages WHERE session_id IN ({placeholders}) " + # Compaction-archived rows (active=0, compacted=1) are display + # history; Undo/Rewind rows (active=0, compacted=0) are not. + "AND (active = 1 OR compacted = 1) " + # ORDER BY id (insertion order) — see get_messages_as_conversation + # for why timestamp ordering is unsafe. + "ORDER BY id", + tuple(session_ids), + ).fetchall() + + # Tip rows are exactly the model-fed set (get_messages_as_conversation + # with session_ids=[session_id]); filtering the lineage fetch preserves + # their relative id order. The model projection stays active-only — it + # is the compressed working context and must not regrow the history + # compaction just summarized away. + tip_rows = [r for r in rows if r["session_id"] == session_id and r["active"]] + model_history = self._rows_to_conversation( + tip_rows, + session_id=session_id, + include_ancestors=False, + repair_alternation=True, + include_row_ids=True, + # Pre-compress checkpointing: the resumed model history must keep + # the summary marker so checkpoint providers can exclude derivative + # summaries after a process restart (marker survives restart). + include_summary_markers=True, + ) + display_history = self._rows_to_conversation( + self._dedupe_display_generations(rows), + session_id=session_id, + include_ancestors=True, + repair_alternation=False, + include_row_ids=True, + ) + return model_history, display_history + + def _resume_lineage_ids(self, session_id: str) -> List[str]: + """Session ids a full (display) resume materializes for *session_id*. + + Compression continuations need their ended ancestors' rows for the + display transcript; an explicit ``/branch`` copy already owns its + transcript, so its lineage is itself alone. This is the ONE definition + shared by the resume readers (``get_resume_conversations``, + ``get_ancestor_display_prefix``) and the resume guard + (``assert_resume_safe`` / ``get_resume_message_count``) — the guard must + count exactly the rows a resume would load, never a superset. + """ + if self._is_explicit_branch_session(session_id): + return [session_id] + return self._session_lineage_root_to_tip(session_id) + + def get_resume_message_count( + self, session_id: str, *, tip_only: bool = False + ) -> int: + """Count the rows that a resume would materialize. + + ``tip_only=True`` counts the tip segment's ACTIVE rows — the set a + model-history restore loads (``get_messages_as_conversation`` without + ancestors, or the deferred Desktop resume that pages the display + transcript over REST and never materializes the ancestor prefix in + memory). + + Otherwise this counts the full-lineage DISPLAY set — active rows plus + the compaction-archived rows ``get_resume_conversations`` now loads + for the transcript. Counting only active rows here would let a + heavily-compacted conversation pass a limit sized for a handful of + live rows and then materialize tens of thousands. + """ + session_ids = [session_id] if tip_only else self._resume_lineage_ids(session_id) + active_clause = "active = 1" if tip_only else "(active = 1 OR compacted = 1)" + placeholders = ",".join("?" for _ in session_ids) + row = self._read_one( + f"SELECT COUNT(*) FROM messages " + f"WHERE session_id IN ({placeholders}) AND {active_clause}", + tuple(session_ids), + ) + return int(row[0] if row else 0) + + def assert_resume_safe( + self, + session_id: str, + max_messages: Optional[int] = None, + *, + tip_only: bool = False, + ) -> int: + """Return resume row count or reject a transcript too large to load. + + ``max_messages=None`` resolves the limit from config + (``sessions.max_resume_messages``); 0 disables the guard and returns + the (bounded) count without raising. + + ``tip_only=True`` bounds only the tip segment's ACTIVE rows, for + callers that never materialize the ancestor lineage or the + compaction archive in memory (tip-only model restore, deferred + Desktop resume whose display history is REST-paginated). A + heavily-compressed conversation — 85 compaction segments and ~29k + lineage rows behind a ~700-row tip — is exactly the shape compression + is supposed to produce; counting its whole lineage against a limit + sized for in-memory materialization rejected the healthiest sessions + (Desktop Bot Chat stuck on "Waking up…" with code 4130) while the + process would only ever have held the tip. + + The full (non-``tip_only``) bound counts the DISPLAY set — active plus + compaction-archived rows — because that is what + ``get_resume_conversations`` materializes for the transcript. + """ + from hermes_state import SessionResumeTooLargeError, resolved_max_resume_messages + if max_messages is None: + max_messages = resolved_max_resume_messages() + if max_messages < 0: + raise ValueError("max_messages must be non-negative") + if max_messages == 0: + # Guard disabled by config — skip counting entirely. Every live + # caller invokes this for its raise side effect and ignores the + # return value, and an unbounded lineage COUNT here would do the + # exact pathological work the disable exists to avoid. + return 0 + session_ids = [session_id] if tip_only else self._resume_lineage_ids(session_id) + active_clause = "active = 1" if tip_only else "(active = 1 OR compacted = 1)" + placeholders = ",".join("?" for _ in session_ids) + row = self._read_one( + "SELECT COUNT(*) FROM (" + f"SELECT 1 FROM messages WHERE session_id IN ({placeholders}) " + f"AND {active_clause} LIMIT ?" + ")", + (*session_ids, max_messages + 1), + ) + message_count = int(row[0] if row else 0) + if message_count > max_messages: + raise SessionResumeTooLargeError( + message_count, + max_messages, + scope="in its tip segment" if tip_only else "across its lineage", + ) + return message_count + + def get_ancestor_display_prefix(self, session_id: str) -> List[Dict[str, Any]]: + """Return the ancestor-only display messages for a session lineage. + + These are messages from parent/grandparent sessions (compression + ancestors) that appear in the display transcript but NOT in the + tip session's model-fed history. Used by ``session.resume`` to + build the ``display_history_prefix`` that ``_live_session_payload`` + prepends to the live model history. + + Previously the prefix was calculated as + ``display_history[:len(display) - len(raw)]``, but that overcounts + when ``repair_message_sequence`` removes messages from the MIDDLE + of the tip history (e.g. verification candidates collapsed by the + consecutive-assistant merge) — the length difference includes both + ancestor messages AND repair-removed tip messages, but the slice + only captures the first N display messages (which are tip messages + when there are no ancestors), causing duplication. This method + returns ONLY the genuine ancestor messages, identified by + ``session_id != tip_session_id``. (#65919) + """ + session_ids = self._resume_lineage_ids(session_id) + if len(session_ids) <= 1: + return [] + with self._read_ctx() as conn: + placeholders = ",".join("?" for _ in session_ids) + rows = conn.execute( + f"SELECT session_id, {self._CONVERSATION_ROW_COLUMNS} " + f"FROM messages WHERE session_id IN ({placeholders}) " + # Display read: compaction-archived rows included, Undo/Rewind + # rows excluded (see get_resume_conversations). + "AND (active = 1 OR compacted = 1) " + "ORDER BY id", + tuple(session_ids), + ).fetchall() + rows = self._dedupe_display_generations(rows) + ancestor_ids = { + int(row["id"]) + for row in rows + if row["session_id"] != session_id and row["id"] is not None + } + if not ancestor_ids: + return [] + lineage = self._rows_to_conversation( + rows, + session_id=session_id, + include_ancestors=True, + repair_alternation=False, + include_row_ids=True, + ) + prefix: List[Dict[str, Any]] = [] + for message in lineage: + if message.get("_row_id") not in ancestor_ids: + continue + projected = message.copy() + projected.pop("_row_id", None) + prefix.append(projected) + return prefix + + def get_conversation_root(self, session_id: str) -> str: + """Return the ROOT id of *session_id*'s lineage chain. + + The root is the stable "conversation id": context compression + rotates ``session_id`` to a new segment linked via + ``parent_session_id``, and delegate subagents hang off their + parent the same way. Walking to the root gives every segment of + one user-facing conversation (and its delegation tree) a single + identifier — used for Nous Portal ``conversation=`` usage tagging. + Returns *session_id* unchanged when it has no recorded parent. + """ + chain = self._session_lineage_root_to_tip(session_id) + return (chain[0] if chain and chain[0] else session_id) + + @staticmethod + def _canonical_replayed_user_content( + msg: Dict[str, Any], + ) -> Tuple[Any, bool]: + """Return canonical live content and whether *msg* is composite.""" + if msg.get("role") != "user": + return None, False + + from agent.context_compressor import split_user_originated_turn + + handoff, live_view = split_user_originated_turn(msg) + is_composite = handoff is not None and live_view is not None + return ( + live_view.get("content") + if is_composite and live_view is not None + else msg.get("content"), + is_composite, + ) + + @staticmethod + def _exact_replayed_user_clone_key( + timestamp: Any, content: Any + ) -> Optional[Tuple[Any, str]]: + """Return a hashable key for a column-exact rotation clone.""" + if timestamp is None or content in (None, "", []): + return None + try: + encoded = json.dumps( + content, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + except (TypeError, ValueError): + return None + return timestamp, encoded + + @staticmethod + def _find_duplicate_replayed_user_message( + messages: List[Dict[str, Any]], msg: Dict[str, Any] + ) -> Optional[Tuple[int, bool]]: + """Return an adjacent replay duplicate and whether *msg* must win. + + Compression rotation may persist the current ask once in the parent + and again inside a composite child carrier. Compare the canonical live + payload for that carrier, while retaining the historical exact-string + dedupe for ordinary replayed users. The child carrier wins because it + owns both the current durable row identity and the retained scaffold. + """ + from hermes_state import SessionDB + if msg.get("role") != "user": + return None + + content, prefer_current = SessionDB._canonical_replayed_user_content(msg) + if content in (None, "", []): + return None + + for index in range(len(messages) - 1, -1, -1): + prev = messages[index] + if prev.get("role") == "user": + prev_content, prev_is_composite = ( + SessionDB._canonical_replayed_user_content(prev) + ) + if prev_content == content and ( + prefer_current + or prev_is_composite + or isinstance(content, str) + ): + return index, prefer_current + if prev.get("role") == "assistant" and (prev.get("content") or prev.get("tool_calls")): + return None + return None + + def get_active_message_ids(self, session_id: str) -> List[int]: + """Return the ordered physical ids pinned by rewind CAS checks. + + Conversation projections intentionally omit legacy background-review + harness rows. Destructive rewinds must nevertheless pin every active + physical row so the caller snapshot matches the transaction-local + comparison in :meth:`rewind_to_message`. + """ + rows = self._read_all( + "SELECT id FROM messages " + "WHERE session_id = ? AND active = 1 ORDER BY id", + (session_id,), + ) + return [int(row[0]) for row in rows] + + @staticmethod + def _active_transcript_counts(conn, session_id: str) -> tuple[int, int]: + """Return active message/tool-call counts inside the caller's txn.""" + rows = conn.execute( + "SELECT tool_calls FROM messages " + "WHERE session_id = ? AND active = 1", + (session_id,), + ).fetchall() + tool_call_count = 0 + for row in rows: + raw = row[0] + if not raw: + continue + try: + decoded = json.loads(raw) if isinstance(raw, str) else raw + except (json.JSONDecodeError, TypeError): + continue + if isinstance(decoded, list): + tool_call_count += len(decoded) + elif decoded: + tool_call_count += 1 + return len(rows), tool_call_count + + def rewind_to_message( + self, + session_id: str, + target_message_id: int, + *, + preserve_compaction_handoff: bool = False, + expected_active_ids: Optional[List[int]] = None, + expected_target_content: Any = None, + ) -> Dict[str, Any]: + """Soft-delete all messages with id >= ``target_message_id`` in *session_id*. + + The target message itself becomes inactive as well so the caller + can pre-fill it as the next user prompt without it appearing + twice in the replayed transcript. Rewound rows are kept on + disk with ``active=0`` for audit / forensic inspection — use + :meth:`get_messages` with ``include_inactive=True`` to see them. + + Returns a dict:: + + { + "rewound_count": int, # number of rows newly flipped to active=0 + "target_message": dict, # full row dict of the target + "new_head_id": int|None # id of the last still-active row, or None + } + + Raises ``ValueError`` if the target message does not exist in + *session_id* or if its role is not ``"user"``. With + ``preserve_compaction_handoff=True``, a composite summary carrier is + split inside the same write transaction: its original row is archived + and its canonical hidden handoff scaffold is inserted as the new head. + That opt-in result also contains ``replacement_message_id``. + + ``expected_active_ids`` optionally pins the ordered active row set. + ``expected_target_content`` additionally pins the selected canonical + live-user payload. Both checks run inside the write transaction before + any row or counter mutation. Presentation-only metadata changes (for + example Desktop reactions) deliberately do not invalidate a rewind. + A live cross-process turn lease always refuses the rewind; expired or + provably dead holders are reclaimed inside the mutation transaction. + + Always increments ``sessions.rewind_count`` — even when the + target is already inactive — so the counter accurately reflects + the number of rewind operations performed against the session. + Idempotent on the ``active`` flag: re-rewinding past the same + target is a no-op on row state but still bumps the counter. + """ + + def _do(conn): + # Rewind changes the active transcript and must honor the same + # compression/closed-parent and cross-process turn guards as + # append writers. + self._check_transcript_write_guards( + conn, + session_id, + None, + reject_active_turn_lease=True, + reject_active_compression_lock=True, + ) + + if expected_active_ids is not None: + active_rows = conn.execute( + "SELECT id FROM messages " + "WHERE session_id = ? AND active = 1 ORDER BY id", + (session_id,), + ).fetchall() + active_ids = [int(active_row[0]) for active_row in active_rows] + if active_ids != expected_active_ids: + raise RuntimeError( + "active transcript changed before the rewind could be persisted" + ) + + row = conn.execute( + "SELECT * FROM messages WHERE id = ? AND session_id = ?", + (target_message_id, session_id), + ).fetchone() + if row is None: + raise ValueError( + f"message {target_message_id} not found in session {session_id}" + ) + target_row = dict(row) + if target_row.get("role") != "user": + raise ValueError( + f"rewind target must be a 'user' message (got role=" + f"{target_row.get('role')!r}, id={target_message_id})" + ) + + replacement_message_id: Optional[int] = None + replacement: Optional[Dict[str, Any]] = None + if preserve_compaction_handoff or expected_target_content is not None: + if not target_row.get("active"): + raise ValueError("rewind target is not active") + from agent.context_compressor import split_user_originated_turn + + split_target = target_row.copy() + split_target["content"] = self._decode_content( + split_target.get("content") + ) + split_target["display_metadata"] = self._decode_display_metadata( + split_target.get("display_metadata") + ) + handoff, live_view = split_user_originated_turn(split_target) + if live_view is None: + raise ValueError("rewind target is not a user-originated turn") + live_content = live_view.get("content") + if isinstance(live_content, str): + live_content = sanitize_context(live_content).strip() + if ( + expected_target_content is not None + and live_content != expected_target_content + ): + raise RuntimeError( + "rewind target changed before it could be persisted" + ) + if preserve_compaction_handoff and handoff is None: + raise ValueError( + "preserve_compaction_handoff requires an active composite carrier" + ) + replacement = handoff if preserve_compaction_handoff else None + + cursor = conn.execute( + "SELECT id FROM messages " + "WHERE session_id = ? AND id >= ? AND active = 1", + (session_id, target_message_id), + ) + ids = [r[0] for r in cursor.fetchall()] + if ids: + placeholders = ",".join("?" for _ in ids) + conn.execute( + f"UPDATE messages SET active = 0 WHERE id IN ({placeholders})", + ids, + ) + if replacement is not None: + self._insert_message_rows(conn, session_id, [replacement]) + inserted = conn.execute("SELECT last_insert_rowid()").fetchone() + replacement_message_id = int(inserted[0]) + conn.execute( + "UPDATE sessions SET rewind_count = COALESCE(rewind_count, 0) + 1 " + "WHERE id = ?", + (session_id,), + ) + message_count, tool_call_count = self._active_transcript_counts( + conn, session_id + ) + conn.execute( + "UPDATE sessions SET message_count = ?, tool_call_count = ? " + "WHERE id = ?", + (message_count, tool_call_count, session_id), + ) + head_row = conn.execute( + "SELECT MAX(id) FROM messages WHERE session_id = ? AND active = 1", + (session_id,), + ).fetchone() + new_head_id = ( + head_row[0] if head_row and head_row[0] is not None else None + ) + return target_row, ids, new_head_id, replacement_message_id + + target_row, rewound, new_head_id, replacement_message_id = ( + self._execute_write(_do) + ) + + # Decode content for callers (prefill the prompt buffer) without a + # second fallible database operation after the transaction commits. + target_row["content"] = self._decode_content(target_row.get("content")) + + result = { + "rewound_count": len(rewound), + "target_message": target_row, + "new_head_id": new_head_id, + } + if preserve_compaction_handoff: + result["replacement_message_id"] = replacement_message_id + return result + + def message_count(self, session_id: str = None) -> int: + """Count messages, optionally for a specific session.""" + with self._read_ctx() as conn: + if session_id: + cursor = conn.execute( + "SELECT COUNT(*) FROM messages WHERE session_id = ?", (session_id,) + ) + else: + cursor = conn.execute("SELECT COUNT(*) FROM messages") + return cursor.fetchone()[0] + + def has_platform_message_id( + self, session_id: str, platform_message_id: str + ) -> bool: + """Check if a message with the given platform_message_id exists. + + Uses the idx_messages_platform_msg_id partial index for efficient + lookup. Used by the gateway's transient-failure dedupe guard (#47237) + to skip re-persisting a user message that was already saved on a + prior retry of the same inbound platform message. + """ + return self._read_one( + "SELECT 1 FROM messages " + "WHERE session_id = ? AND platform_message_id = ? LIMIT 1", + (session_id, platform_message_id), + ) is not None + + def _is_explicit_fork_child_row(self, session: Dict[str, Any]) -> bool: + """True when ``session`` is a branch, delegate, or tool child of its parent. + + Markers only count as a fork when they point at ``parent_session_id``. + Compression copies ``model_config`` onto the continuation + (``publish_compression_child`` callers pass + ``agent._session_init_model_config``), so a delegate's continuation + carries ``_delegate_from=``. Presence-only + matching would treat that real continuation as a fork — the same + misclassification ``_NON_CONTINUATION_CHILD_FILTER_SQL`` already + avoids by binding both markers to the queried parent. + """ + if session.get("source") == "tool": + return True + raw = session.get("model_config") + if not raw: + return False + try: + cfg = json.loads(raw) if isinstance(raw, str) else raw + except (TypeError, json.JSONDecodeError): + return False + if not isinstance(cfg, dict): + return False + parent_id = session.get("parent_session_id") + branched = cfg.get("_branched_from") + delegated = cfg.get("_delegate_from") + if parent_id: + return branched == parent_id or delegated == parent_id + return branched is not None or delegated is not None + + def is_explicit_fork_child(self, session_id: str) -> bool: + """True when ``session_id`` is a /branch, delegate, or tool child row. + + Read-only public view of :meth:`_is_explicit_fork_child_row` for + callers that must respect the fork boundary without re-implementing + its marker rules (``agent/prompt_cache_scope.py`` keeps a declared + conversation key from crossing it). A missing row is not a fork. + """ + session = self.get_session(session_id) + return bool(session and self._is_explicit_fork_child_row(session)) + + def latest_conversation_boundary( + self, session_key: str, source: str + ) -> Optional[int]: + """How many conversation boundaries this routing peer has crossed. + + A boundary is a row this peer ended at an intentional conversation + break — the ``_RESET_END_REASONS`` set (``/new``, ``/switch``, idle, + daily, suspended, resume_pending_expired). That is the same fence + :meth:`find_latest_gateway_session_for_peer` refuses to reach behind, + so the two agree on where one conversation stops and the next begins + and cannot drift. + + The peer is ``(session_key, source)``, the SAME identity tuple recovery + uses — never the key alone. ``X-Hermes-Session-Key`` accepts any + authenticated caller-supplied string, so an API conversation may + legally carry the same key as a Telegram row in one database; keying + on the string alone would let a ``/new`` on that unrelated row rotate + this conversation's affinity identity while recovery correctly refuses + to cross the same line. + + Returns the count, or ``None`` when this peer has never been reset. + + The value comes from ``conversation_generations``, which + :meth:`_bump_conversation_generation` advances inside the transaction + that writes each boundary — NOT from an aggregate over the session + rows. An aggregate cannot prove non-reuse: ``delete_session()`` + orphans children and deletes the row, and bulk prune selects ended + rows, so ``COUNT``/``MAX`` over boundaries can return a pair it already + emitted and hand a new conversation a retired affinity identity. It is + also wall-clock-free, so a backwards NTP correction cannot reorder it. + + Databases upgraded mid-conversation start at no generation and take + their first one from the next boundary written; a conversation that + reset before the upgrade shares its predecessor's scope once, which + costs a warm prompt-cache bucket and never crosses an identity. + + These rows are never garbage-collected, by design: dropping one resets + the peer to "no generation", so its next boundary writes ``1`` again + and re-issues a scope a retired conversation already used — the ABA + this counter exists to prevent. See the schema comment in + ``hermes_state_common.py``. + """ + if not session_key or not source: + return None + row = self._read_one( + "SELECT generation FROM conversation_generations " + "WHERE source = ? AND session_key = ?", + (source, session_key), + ) + if row is None or row["generation"] is None: + return None + generation = int(row["generation"]) + return generation if generation > 0 else None + + def clear_messages(self, session_id: str) -> None: + """Delete all messages for a session and reset its counters.""" + def _do(conn): + conn.execute( + "DELETE FROM messages WHERE session_id = ?", (session_id,) + ) + conn.execute( + "UPDATE sessions SET message_count = 0, tool_call_count = 0 WHERE id = ?", + (session_id,), + ) + self._execute_write(_do) + + def purge_stale_tool_call_markers( + self, *, dry_run: bool = False, backup: bool = True + ) -> Dict[str, Any]: + """Permanently clear bare tool-call marker content (e.g. "[memory]") + left in the ``messages`` table by sessions persisted before the + #78148 fix in ``agent.conversation_loop``. + + ``_strip_stale_tool_call_markers`` already repairs this in memory on + every session load (see ``_rows_to_conversation``), so running this + is optional — but for long-lived sessions the same rows get + re-scanned and re-repaired on every resume, which is wasted work + and keeps the contaminated bytes sitting in the DB (and in any + downstream cache/backup snapshot of it) indefinitely. This rewrites + the affected rows once, in place. + + Only the ``content`` column is touched — ``role``, ``tool_calls``, + and every other column on the row are left exactly as they are, so + provider tool_call/tool_result pairing is unaffected. + + Unlike the in-memory repair, this UPDATE is permanent and can't be + undone from within the DB. Since ``backup`` defaults to True, a + timestamped full snapshot is taken via ``VACUUM INTO`` (safe against + a live connection, unlike the raw-copy ``_backup_db_file`` used for + malformed-schema repair) before any row is touched — mirroring + ``repair_state_db_schema``'s backup-by-default convention for + destructive state.db operations. No snapshot is taken when there is + nothing to change. + + With ``dry_run=True``, reports the affected row count/ids without + writing or backing up (read-only, no write lock taken). + + Returns ``{"dry_run": bool, "rows_affected": int, "row_ids": [...], + "backup_path": str|None}``. + """ + from hermes_state import _STALE_TOOL_CALL_MARKER_RE + + def _find_affected(conn) -> List[int]: + cursor = conn.execute( + "SELECT id, content FROM messages " + "WHERE role = 'assistant' AND tool_calls IS NOT NULL AND tool_calls != ''" + ) + affected: List[int] = [] + for row in cursor.fetchall(): + content = row["content"] + if isinstance(content, str) and _STALE_TOOL_CALL_MARKER_RE.fullmatch(content.strip()): + affected.append(row["id"]) + return affected + + with self._read_ctx() as conn: + affected_ids = _find_affected(conn) + + if dry_run: + return { + "dry_run": True, + "rows_affected": len(affected_ids), + "row_ids": affected_ids, + "backup_path": None, + } + + if not affected_ids: + return { + "dry_run": False, + "rows_affected": 0, + "row_ids": [], + "backup_path": None, + } + + backup_path: Optional[str] = None + if backup: + import datetime + + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + dest = self.db_path.with_name( + f"{self.db_path.name}.pre-clean-markers-backup-{stamp}" + ) + with self._lock: + self._conn.execute("VACUUM INTO ?", (str(dest),)) + backup_path = str(dest) + logger.info("Backed up state.db to %s before clean-markers write", backup_path) + + def _do(conn): + ids = _find_affected(conn) + if ids: + placeholders = ",".join("?" * len(ids)) + conn.execute( + f"UPDATE messages SET content = '' WHERE id IN ({placeholders})", + ids, + ) + return ids + + affected_ids = self._execute_write(_do) + if affected_ids: + logger.info( + "Permanently cleared %d stale tool-call marker row(s) in state.db (#78148)", + len(affected_ids), + ) + return { + "dry_run": False, + "rows_affected": len(affected_ids), + "row_ids": affected_ids, + "backup_path": backup_path, + } diff --git a/hermes_state_portability.py b/hermes_state_portability.py index ca8df96809..2f740786ff 100644 --- a/hermes_state_portability.py +++ b/hermes_state_portability.py @@ -1,11 +1,9 @@ """Session listing/rich rows, export, and import (portability) for SessionDB. -Mixin contract: this is a plain mixin class consumed by -``hermes_state.SessionDB``. It defines no ``__init__`` and no state of its -own; methods access the host's attributes (``self._conn``, ``self.db_path``, -``self._execute_write`` and other SessionDB methods) established by -``SessionDB.__init__``. It must never import hermes_state (cycle) — shared -module-level constants live in hermes_state_common. +Plain mixin consumed by ``hermes_state.SessionDB``: no ``__init__``, no state +of its own; methods use host attributes established by ``SessionDB.__init__``. +Must never import hermes_state (cycle) — shared constants live in +hermes_state_common. """ import logging @@ -16,14 +14,12 @@ from typing import Any, Dict, List, Optional from agent.skill_commands import SKILL_SCAFFOLD_SQL_LIKE from hermes_state_common import ( SCHEMA_SQL, - _PREVIEW_ELIGIBLE_SQL, - _PREVIEW_RAW_SELECT, + _PREVIEW_RAW_SUBQUERY_SQL, _shape_preview, _sql_session_last_active, ) -# Moved methods logged under the "hermes_state" logger before the split; -# keep that logger identity so log filtering/capture behavior is unchanged. +# Keep the pre-split logger identity so log filtering/capture is unchanged. logger = logging.getLogger("hermes_state") @@ -32,9 +28,8 @@ class SessionPortabilityMixin: @classmethod def _compact_session_cols(cls) -> str: - """SELECT list for compact_rows: every ``sessions`` column declared in - SCHEMA_SQL except prompt storage internals, aliased with the ``s`` - prefix used by list_sessions_rich/_get_session_rich_row queries.""" + """``s.``-prefixed SELECT list of every SCHEMA_SQL ``sessions`` column + except prompt storage internals (the compact_rows projection).""" if cls._session_compact_cols_sql is None: declared = cls._parse_schema_columns(SCHEMA_SQL)["sessions"] cls._session_compact_cols_sql = ", ".join( @@ -43,13 +38,19 @@ class SessionPortabilityMixin: ) return cls._session_compact_cols_sql + @classmethod + def _rich_row(cls, row) -> Dict[str, Any]: + """Session row dict with ``_preview_raw`` shaped into ``preview``.""" + s = cls._session_row_dict(row) + s["preview"] = _shape_preview(s.pop("_preview_raw", "")) + return s + def distinct_session_cwds(self, include_archived: bool = False) -> List[Dict[str, Any]]: """Distinct non-empty session cwds with usage stats, for repo discovery. - Aggregates across ALL session history (not a single page), so the desktop - can surface every git repo the user has worked in — not just the repos - that happen to be in the currently-loaded recents. Children/branches - count: a worktree session is still a real workspace signal. + Aggregates across ALL history (not one page) so every repo the user + worked in surfaces. Children/branches count: a worktree session is a + real workspace signal. """ where = "cwd IS NOT NULL AND TRIM(cwd) != ''" if not include_archived: @@ -77,41 +78,24 @@ class SessionPortabilityMixin: ) -> List[Dict[str, Any]]: """List the run sessions produced by a single cron job, newest first. - Cron runs are flat, independent sessions whose id is - ``cron_{job_id}_{timestamp}`` (see ``cron/scheduler.run_job``). They are - never compression roots and never branch, so this deliberately skips the - ``list_sessions_rich`` recursive compression-chain CTE / leading-wildcard - ``id_query`` path — that path seeds from *every* ``source='cron'`` row in - the DB and only filters to one job's runs after the scan, so it scales - with the whole cron pile (a heavy history makes the desktop run-history - endpoint time out before it eventually populates). + Cron runs are flat sessions with id ``cron_{job_id}_{timestamp}``; they + never compress or branch, so this skips ``list_sessions_rich``'s + compression-chain CTE / leading-wildcard ``id_query`` path, which seeds + from EVERY ``source='cron'`` row and scales with the whole cron pile. + Instead: a ``[prefix, prefix_hi)`` index range scan on id, filtered to + ``source='cron'``, so work scales with the requested window. - Instead this binds to one job with a ``[prefix, prefix_hi)`` range over - the id (an index range scan, not a ``%...%`` substring), filters - ``source='cron'``, and orders by ``started_at DESC``. Work scales with - the requested window, not the total cron history. - - Returns the same enriched row shape as ``list_sessions_rich`` (adds - ``preview`` + ``last_active``) so callers can reuse it. + Returns the ``list_sessions_rich`` row shape (``preview`` + ``last_active``). """ prefix = f"cron_{job_id}_" - # Half-open upper bound for an index range scan: increment the final - # byte of the prefix so the range covers exactly the ids that start - # with ``prefix`` and nothing else. ``prefix`` always ends in '_', but - # compute it generically rather than hardcoding the successor char. + # Half-open upper bound: bump the final byte so the range covers exactly + # the ids starting with ``prefix``. prefix_hi = prefix[:-1] + chr(ord(prefix[-1]) + 1) query = f""" SELECT s.*, COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved, - COALESCE( - (SELECT {_PREVIEW_RAW_SELECT} - FROM messages m - WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL - AND {_PREVIEW_ELIGIBLE_SQL} - ORDER BY m.timestamp, m.id LIMIT 1), - '' - ) AS _preview_raw, + {_PREVIEW_RAW_SUBQUERY_SQL}, {_sql_session_last_active("s")} AS last_active FROM sessions s LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash @@ -120,27 +104,12 @@ class SessionPortabilityMixin: LIMIT ? OFFSET ? """ with self._lock: - cursor = self._conn.execute(query, (prefix, prefix_hi, limit, offset)) - rows = cursor.fetchall() - - runs: List[Dict[str, Any]] = [] - for row in rows: - s = self._session_row_dict(row) - s["preview"] = _shape_preview(s.pop("_preview_raw", "")) - runs.append(s) - return runs + rows = self._conn.execute(query, (prefix, prefix_hi, limit, offset)).fetchall() + return [self._rich_row(row) for row in rows] def _get_session_rich_row(self, session_id: str, compact_rows: bool = False) -> Optional[Dict[str, Any]]: - """Fetch a single session with the same enriched columns as - ``list_sessions_rich`` (preview + last_active). Returns None if the - session doesn't exist. - - Pass ``compact_rows=True`` to omit the ``system_prompt`` blob (see - ``list_sessions_rich`` for details). - - Thin wrapper over ``_get_session_rich_rows_batch`` so the enriched - SELECT lives in exactly one place. - """ + """One session with the ``list_sessions_rich`` enriched columns, or + None. ``compact_rows=True`` omits the ``system_prompt`` blob.""" return self._get_session_rich_rows_batch( [session_id], compact_rows=compact_rows ).get(session_id) @@ -148,23 +117,15 @@ class SessionPortabilityMixin: def _get_session_rich_rows_batch( self, session_ids, compact_rows: bool = False ) -> Dict[str, Dict[str, Any]]: - """Fetch multiple sessions with the same enriched columns as - ``_get_session_rich_row``, in a single query. - - Used by ``list_sessions_rich``'s compression-tip projection to resolve - every tip row for a page in one round trip instead of one query per - compression-root row. Returns a dict keyed by session id; ids that - don't exist are simply absent from the result (same as - ``_get_session_rich_row`` returning ``None`` for them). + """Enriched rows for many sessions in one query, keyed by id; missing + ids are simply absent. Resolves a page of compression tips in one + round trip instead of one query per root row. """ ids = [sid for sid in session_ids if sid] if not ids: return {} - # Old SQLite builds cap bound variables at 999 - # (SQLITE_MAX_VARIABLE_NUMBER); large pages (limit=10000 callers - # exist) could exceed it. Chunk the IN list so the helper is safe at - # any page size — this is the single choke point for the enriched - # multi-row fetch, so the bound lives here, not at call sites. + # Old SQLite caps bound variables at 999 (SQLITE_MAX_VARIABLE_NUMBER); + # limit=10000 callers exist. Chunk here — the single choke point. _CHUNK = 900 if len(ids) > _CHUNK: result: Dict[str, Dict[str, Any]] = {} @@ -189,45 +150,26 @@ class SessionPortabilityMixin: ) query = f""" SELECT {_sel}{prompt_select}, - COALESCE( - (SELECT {_PREVIEW_RAW_SELECT} - FROM messages m - WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL - AND {_PREVIEW_ELIGIBLE_SQL} - ORDER BY m.timestamp, m.id LIMIT 1), - '' - ) AS _preview_raw, + {_PREVIEW_RAW_SUBQUERY_SQL}, {_sql_session_last_active("s")} AS last_active FROM sessions s {prompt_join} WHERE s.id IN ({placeholders}) """ with self._lock: - cursor = self._conn.execute(query, ids) - rows = cursor.fetchall() - result: Dict[str, Dict[str, Any]] = {} - for row in rows: - s = self._session_row_dict(row) - s["preview"] = _shape_preview(s.pop("_preview_raw", "")) - result[s["id"]] = s - return result + rows = self._conn.execute(query, ids).fetchall() + return {s["id"]: s for s in map(self._rich_row, rows)} def get_session_rich_row(self, session_id: str, compact_rows: bool = False) -> Optional[Dict[str, Any]]: - """Public wrapper for :meth:`_get_session_rich_row`. - - Exposes the single-session enriched row (same columns as - ``list_sessions_rich``: preview + last_active) for callers outside - this module, e.g. the web server's session-search hydration. - """ + """Public wrapper for :meth:`_get_session_rich_row` (web server hydration).""" return self._get_session_rich_row(session_id, compact_rows=compact_rows) def list_skill_scaffolded_sessions(self, limit: int = 200) -> List[Dict[str, Any]]: """Titled sessions whose first user turn was a ``/skill`` invocation. - Those titles were generated from the expanded message, which embeds the - whole skill body — so they describe the skill rather than the request. - Returns ``id``, ``title``, and the full first-turn ``content`` so a - caller can re-derive what the user typed. Newest first. + Their titles were generated from the expanded skill body, so they + describe the skill, not the request. Returns ``id``, ``title`` and the + first-turn ``content`` so callers can re-derive what was typed. Newest first. """ with self._lock: rows = self._conn.execute( @@ -248,63 +190,36 @@ class SessionPortabilityMixin: ).fetchall() return [dict(row) for row in rows] - def get_first_assistant_text(self, session_id: str) -> str: - """The session's first assistant reply as plain text ('' when none). - - Pairs with :meth:`list_skill_scaffolded_sessions` so a re-title can feed - the titler the same (request, reply) shape the live path uses. - """ - with self._lock: - row = self._conn.execute( - "SELECT content FROM messages " - "WHERE session_id = ? AND role = 'assistant' AND content IS NOT NULL " - "ORDER BY timestamp, id LIMIT 1", - (session_id,), - ).fetchone() - if not row: - return "" - decoded = self._decode_content(row["content"]) - return decoded if isinstance(decoded, str) else "" - def export_session(self, session_id: str) -> Optional[Dict[str, Any]]: """Export a single session with all its messages as a dict.""" session = self.get_session(session_id) if not session: return None - messages = self.get_messages(session_id) - return {**session, "messages": messages} + return {**session, "messages": self.get_messages(session_id)} def export_session_lineage(self, session_id: str) -> Optional[Dict[str, Any]]: """Export a compression lineage as one logical session dict.""" lineage_ids = self.get_compression_lineage(session_id) if not lineage_ids: return None - segments = [] - for sid in lineage_ids: - segment = self.export_session(sid) - if segment: - segments.append(segment) + segments = [seg for seg in map(self.export_session, lineage_ids) if seg] if not segments: return None - base = dict(segments[-1]) - total_messages = sum(len(seg.get("messages") or []) for seg in segments) - base["segments"] = segments - base["lineage_session_ids"] = [seg["id"] for seg in segments] - base["message_count"] = total_messages - base["messages"] = [msg for seg in segments for msg in (seg.get("messages") or [])] - return base + messages = [msg for seg in segments for msg in (seg.get("messages") or [])] + return { + **segments[-1], + "segments": segments, + "lineage_session_ids": [seg["id"] for seg in segments], + "message_count": len(messages), + "messages": messages, + } def export_all(self, source: str = None) -> List[Dict[str, Any]]: - """ - Export all sessions (with messages) as a list of dicts. - Suitable for writing to a JSONL file for backup/analysis. - """ - sessions = self.search_sessions(source=source, limit=100000) - results = [] - for session in sessions: - messages = self.get_messages(session["id"]) - results.append({**session, "messages": messages}) - return results + """Export all sessions (with messages) as dicts, e.g. for JSONL backup.""" + return [ + {**session, "messages": self.get_messages(session["id"])} + for session in self.search_sessions(source=source, limit=100000) + ] def adopt_session_lineage_from( self, @@ -313,52 +228,36 @@ class SessionPortabilityMixin: *, retire_donor: bool = True, ) -> Dict[str, Any]: - """Adopt *session_id*'s full compression lineage from *donor_db* into - this store. + """Adopt *session_id*'s full compression lineage from *donor_db*. - The stranded-bot-session heal (#93091 follow-up to #93296): before the - desktop routed session RPCs by their target session, a profile bot's - turns executed on whichever backend held window focus — usually the - default one — so the bot's canonical session rows and messages - accumulated in the DEFAULT profile's state.db. Once routing was fixed, - the profile backend correctly received the RPCs but had no such - session, so the same chat 4001'd for the opposite reason. This method - moves the conversation to where routing now looks for it. + Stranded-bot-session heal: before the desktop routed session RPCs by + target session, a profile bot's rows accumulated in the DEFAULT + profile's state.db; this moves the conversation to where routing now + looks. Pure composition: ``donor_db.export_session_lineage()`` -> + ``self.import_sessions()`` — routing/handoff/activity fields reset, + already-present ids skipped (idempotent re-adoption). - Composition of existing primitives (no new import/export machinery): - ``donor_db.export_session_lineage()`` -> ``self.import_sessions()``. - Import semantics apply unchanged: gateway routing, handoff, and live - activity fields are reset; already-present ids are skipped - (idempotent re-adoption after a partial run). + With ``retire_donor`` and a complete adoption, donor rows are ARCHIVED + (never deleted) with ``end_reason='adopted_by_profile'``. That + end_reason is deliberately NOT in the recoverable set + (agent_close/ws_orphan_reap): resurrection must not undo an adoption. - When ``retire_donor`` is True and at least one segment was imported - (or every segment already exists here), the donor rows are ARCHIVED — - never deleted — with ``end_reason='adopted_by_profile'`` so the - default profile's list stops advertising a conversation that now - lives elsewhere, while the bytes stay recoverable. The archive is - deliberately NOT in the recoverable set (agent_close/ws_orphan_reap): - canonical-lookup resurrection must not undo an adoption. - - Returns the ``import_sessions`` result dict, plus ``adopted`` (bool) - and ``donor_retired`` (bool — True only when EVERY segment's - retirement actually applied). + Returns the ``import_sessions`` dict plus ``adopted`` and + ``donor_retired`` (True only when EVERY segment's retirement applied). """ payload = donor_db.export_session_lineage(session_id) if not payload: return { - "ok": False, - "adopted": False, - "donor_retired": False, + "ok": False, "adopted": False, "donor_retired": False, "error": f"session {session_id!r} not found in donor store", } segments = payload.get("segments") or [payload] - # Divergence guard: a segment we are about to SKIP (already present - # here) may have kept accumulating messages in the donor store after - # a partial earlier adoption. Retiring it would strand those newer - # messages behind a non-recoverable archive. Compare counts up front - # and refuse to retire (still adopt/import) when the donor is ahead. + # Divergence guard: a segment we will SKIP (already here) may have kept + # growing in the donor after a partial adoption; retiring it would strand + # those messages behind a non-recoverable archive. Still import, but + # refuse to retire when the donor is ahead. donor_ahead = False for seg in segments: seg_id = seg.get("id") @@ -394,15 +293,11 @@ class SessionPortabilityMixin: if not seg_id: continue try: - # TOCTOU close-out: the guard above compared EXPORT-TIME - # counts, but another backend can append donor messages - # between export and this loop. Re-read both stores right - # before stamping; a donor-ahead signal here skips the - # stamp so growth never lands behind a non-recoverable - # archive. (Count comparison cannot see equal-count - # CONTENT divergence — e.g. a donor rewind+rewrite; that - # residual case is accepted: bytes stay in the donor - # store either way, only reachability differs.) + # TOCTOU close-out: the guard above used EXPORT-TIME counts; + # re-read both stores right before stamping so donor growth + # never lands behind a non-recoverable archive. (Equal-count + # CONTENT divergence is accepted: bytes stay in the donor + # either way, only reachability differs.) donor_now = len(donor_db.get_messages(seg_id)) local_now = len(self.get_messages(seg_id)) if donor_now > local_now: @@ -414,17 +309,15 @@ class SessionPortabilityMixin: seg_id, donor_now, local_now, ) continue - # First end_reason wins in end_session(); reopen first so - # the adoption boundary is stamped even on ended segments - # (e.g. 'compression' parents). + # First end_reason wins in end_session(); reopen so the + # adoption boundary is stamped even on ended segments. donor_db.reopen_session(seg_id) donor_db.end_session(seg_id, "adopted_by_profile") donor_db.set_session_archived(seg_id, True) except Exception: - # Best-effort by design: a retirement failure must not - # fail the adoption (the profile copy is already whole; - # a later resume retries retirement idempotently). But - # never claim success we didn't have. + # Best-effort: a retirement failure must not fail the adoption + # (a later resume retries idempotently) — but never claim + # success we didn't have. retire_ok = False logger.warning( "failed to retire donor segment %s after adoption", @@ -507,21 +400,16 @@ class SessionPortabilityMixin: def import_sessions(self, sessions: List[Dict[str, Any]]) -> Dict[str, Any]: """Import sessions exported by :meth:`export_session` or ``export_all``. - Existing session IDs are skipped. Imported child sessions keep their - parent only when that parent already exists or is included in the same - import payload; otherwise the child is detached so partial imports don't - fail foreign-key validation. Gateway routing, handoff, rewind, and other - live runtime state are intentionally reset: this restores conversation - history, not ownership of a live channel or process. + Existing ids are skipped. A child keeps its parent only when the parent + exists or is in the same payload; otherwise it is detached so partial + imports pass FK validation. Gateway routing, handoff, rewind and other + live runtime state are reset: this restores history, not ownership of + a live channel or process. - Activity contract (#76354 review S4): export INCLUDES the live - activity fields (``last_activity_at`` / ``last_activity_description`` - / ``last_activity_provenance``) because they are part of the durable - row, but import deliberately RESETS them to NULL. Resurrecting a - stale "working ..." label on a machine where no agent is running - would fabricate activity the watchdog and session listings act on. - This asymmetry is intentional and covered by regression - (tests/gateway/test_watchdog_review_76354.py::test_s4_export_includes_activity_import_resets_it). + Activity contract: export INCLUDES ``last_activity_*`` (durable row + fields) but import RESETS them to NULL — resurrecting a stale + "working ..." label would fabricate activity the watchdog and listings + act on. Intentional asymmetry, pinned by regression test. """ if not isinstance(sessions, list): raise ValueError("sessions must be a list") @@ -536,32 +424,14 @@ class SessionPortabilityMixin: total_messages = 0 total_bytes = 0 session_text_fields = ( - "source", - "user_id", - "model", - "system_prompt", - "end_reason", - "cwd", - "git_branch", - "git_repo_root", - "billing_provider", - "billing_base_url", - "billing_mode", - "cost_status", - "cost_source", - "pricing_version", - "title", + "source", "user_id", "model", "system_prompt", "end_reason", "cwd", + "git_branch", "git_repo_root", "billing_provider", "billing_base_url", + "billing_mode", "cost_status", "cost_source", "pricing_version", "title", ) + # ``role`` is validated separately below (non-empty string). message_text_fields = ( - "role", - "tool_call_id", - "tool_name", - "effect_disposition", - "finish_reason", - "reasoning", - "reasoning_content", - "platform_message_id", - "message_id", + "tool_call_id", "tool_name", "effect_disposition", "finish_reason", + "reasoning", "reasoning_content", "platform_message_id", "message_id", ) for index, raw in enumerate(sessions): @@ -580,43 +450,23 @@ class SessionPortabilityMixin: errors.append(self._import_error(index, session_id, "messages must be a list")) continue if len(messages) > self._IMPORT_MAX_MESSAGES_PER_SESSION: - errors.append( - self._import_error( - index, - session_id, - "messages exceeds the per-session import limit", - ) - ) + errors.append(self._import_error(index, session_id, "messages exceeds the per-session import limit")) continue if any(not isinstance(msg, dict) for msg in messages): - errors.append( - self._import_error( - index, - session_id, - "messages must contain only objects", - ) - ) + errors.append(self._import_error(index, session_id, "messages must contain only objects")) continue try: - session_bytes = len( - json.dumps(raw, ensure_ascii=False, separators=(",", ":")).encode("utf-8") - ) + session_bytes = len(json.dumps(raw, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) except (TypeError, ValueError): - errors.append( - self._import_error(index, session_id, "session must be JSON serializable") - ) + errors.append(self._import_error(index, session_id, "session must be JSON serializable")) continue if session_bytes > self._IMPORT_MAX_SESSION_BYTES: - errors.append( - self._import_error(index, session_id, "session exceeds the import size limit") - ) + errors.append(self._import_error(index, session_id, "session exceeds the import size limit")) continue total_bytes += session_bytes if total_bytes > self._IMPORT_MAX_TOTAL_BYTES: - errors.append( - self._import_error(index, session_id, "import exceeds the total size limit") - ) + errors.append(self._import_error(index, session_id, "import exceeds the total size limit")) continue try: @@ -640,8 +490,6 @@ class SessionPortabilityMixin: if not isinstance(role, str) or not role: raise ValueError(f"messages[{message_index}].role must be a non-empty string") for field in message_text_fields: - if field == "role": - continue clean_message[field] = self._import_text_or_none( clean_message.get(field), field ) @@ -655,13 +503,7 @@ class SessionPortabilityMixin: total_messages += len(clean_messages) if total_messages > self._IMPORT_MAX_TOTAL_MESSAGES: - errors.append( - self._import_error( - index, - session_id, - "messages exceeds the total import limit", - ) - ) + errors.append(self._import_error(index, session_id, "messages exceeds the total import limit")) continue seen_ids.add(session_id) normalized.append( @@ -669,13 +511,7 @@ class SessionPortabilityMixin: ) if errors: - return { - "ok": False, - "imported": 0, - "skipped": 0, - "detached": 0, - "errors": errors, - } + return {"ok": False, "imported": 0, "skipped": 0, "detached": 0, "errors": errors} def _do(conn): imported_ids: List[str] = [] @@ -738,27 +574,17 @@ class SessionPortabilityMixin: "end_reason": raw.get("end_reason"), "input_tokens": self._int_or_default(raw.get("input_tokens")), "output_tokens": self._int_or_default(raw.get("output_tokens")), - "cache_read_tokens": self._int_or_default( - raw.get("cache_read_tokens") - ), - "cache_write_tokens": self._int_or_default( - raw.get("cache_write_tokens") - ), - "reasoning_tokens": self._int_or_default( - raw.get("reasoning_tokens") - ), + "cache_read_tokens": self._int_or_default(raw.get("cache_read_tokens")), + "cache_write_tokens": self._int_or_default(raw.get("cache_write_tokens")), + "reasoning_tokens": self._int_or_default(raw.get("reasoning_tokens")), "cwd": raw.get("cwd"), "git_branch": raw.get("git_branch"), "git_repo_root": raw.get("git_repo_root"), "billing_provider": raw.get("billing_provider"), "billing_base_url": raw.get("billing_base_url"), "billing_mode": raw.get("billing_mode"), - "estimated_cost_usd": self._float_or_none( - raw.get("estimated_cost_usd") - ), - "actual_cost_usd": self._float_or_none( - raw.get("actual_cost_usd") - ), + "estimated_cost_usd": self._float_or_none(raw.get("estimated_cost_usd")), + "actual_cost_usd": self._float_or_none(raw.get("actual_cost_usd")), "cost_status": raw.get("cost_status"), "cost_source": raw.get("cost_source"), "pricing_version": raw.get("pricing_version"), @@ -771,18 +597,12 @@ class SessionPortabilityMixin: sanitized_messages: List[Dict[str, Any]] = [] for msg in messages: clean = dict(msg) - for key in ( - "reasoning_details", - "codex_reasoning_items", - "codex_message_items", - ): + for key in ("reasoning_details", "codex_reasoning_items", "codex_message_items"): clean[key] = self._reasoning_json_value(clean.get(key)) sanitized_messages.append(clean) total_messages, total_tool_calls = self._insert_message_rows( - conn, - session_id, - sanitized_messages, + conn, session_id, sanitized_messages ) conn.execute( "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", @@ -826,9 +646,8 @@ class SessionPortabilityMixin: (parent_id, session_id), ) else: - # Drop only the closing edge. Later entries can still attach - # to this now-root session, preserving the acyclic portion - # of a malformed imported lineage. + # Drop only the closing edge; later entries can still attach + # to this now-root session. parent_by_child.pop(session_id, None) detached += 1 diff --git a/hermes_state_registry.py b/hermes_state_registry.py index 0f3bcedc20..70d83cd34e 100644 --- a/hermes_state_registry.py +++ b/hermes_state_registry.py @@ -1,40 +1,31 @@ -"""Process-wide shared SessionDB registry (#90837). +"""Process-wide shared SessionDB registry. -A gateway process opens state.db from many call sites — the runner's -``AsyncSessionDB``, the ``SessionStore`` per-path cache, per-agent lazy -recall (``run_agent._get_session_db_for_recall``), per-job cron opens, -and per-message opens in mirror / channel_directory / slash_commands / -shutdown_flush / session_search / react_to_message. Each bare -``SessionDB()`` mints its own writer connection, ``self._lock``, -close-time WAL checkpoint, and async token-writer thread. With N -independent writer connections on one WAL file, mutual exclusion relies -only on SQLite's WAL write lock plus each instance's busy_timeout retry -ladder — and one connection's close-time checkpoint can race another's -growth, producing the lost/reordered-page-write signature reported -across 11+ incidents (#90837). - -This module owns that boundary: one shared ``SessionDB`` per resolved -path per process, refcounted, with generation-aware retirement when the -underlying file is replaced (snapshot restore, recovery swap). +A gateway process opens state.db from many call sites (runner, SessionStore, +per-agent recall, cron, per-message helpers). Each bare ``SessionDB()`` mints +its own writer connection, lock, close-time WAL checkpoint and token-writer +thread; N independent writers on one WAL file rely only on SQLite's write lock +plus busy_timeout, and one connection's close-time checkpoint can race another's +growth (lost/reordered-page-write corruption). This module owns that boundary: +one shared ``SessionDB`` per resolved path per process, refcounted, with +generation-aware retirement when the file is replaced (snapshot restore, +recovery swap). Lifecycle rules: -- ``acquire(path)`` returns the current generation for *path*, - incrementing its refcount. Same path ⇒ same instance ⇒ same writer - connection. -- ``close()`` on a shared instance is a NO-OP. The registry — not any - individual caller — owns the connection lifecycle, so one caller's - ``close()`` can never tear down a writer other callers still hold. -- ``release(db)`` decrements the generation *db was acquired from* - (object-keyed, not pathname-keyed, so an inode replacement cannot - strand a still-owned generation). The final release of a retired - generation tears it down. -- On inode change, the old generation is RETIRED — never lent again — - but stays alive until its existing holders release. If a replacement - open fails, the registry is left WITHOUT a path entry (never a closed - stale object), so the next acquire retries fresh. -- All teardown happens OUTSIDE the registry lock: a final release's - WAL checkpoint must never stall acquisition for every state.db. +- ``acquire(path)`` returns the current generation for *path* and bumps its + refcount. Same path ⇒ same instance ⇒ same writer connection. +- ``close()`` on a shared instance is a NO-OP: the registry, not any caller, + owns the connection lifecycle, so one caller can never tear down a writer + others still hold. +- ``release(db)`` decrements the generation *db was acquired from* (object- + keyed, not pathname-keyed, so an inode replacement cannot strand a + still-owned generation). The final release of a retired generation tears + it down. +- On inode change the old generation is RETIRED (never lent again) but stays + alive until its holders release. If the replacement open fails the registry + keeps NO path entry (never a closed stale object) so the next acquire retries. +- All teardown happens OUTSIDE the registry lock: a final release's WAL + checkpoint must never stall acquisition for every state.db. """ from __future__ import annotations @@ -44,34 +35,14 @@ import threading from pathlib import Path from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +from hermes_state_common import stat_db_file_identity as _stat_db_file_identity + if TYPE_CHECKING: # pragma: no cover - import cycle guard, typed only from hermes_state import SessionDB logger = logging.getLogger(__name__) -def _stat_db_file_identity(path: Path) -> Optional[Tuple[int, int]]: - """Return ``(st_dev, st_ino)`` for *path*, or None when unavailable. - - Mirrors the hermes_state helper of the same name; kept local so this - module has no import-time dependency on hermes_state (which imports - this module — the cycle is resolved by deferring SessionDB lookup - to call time). - """ - import os - - try: - st = os.stat(path) - except OSError: - return None - # Windows volumes (and some network FS) report st_ino=0; a (0, 0) - # identity would false-positive every check. Skip the inode half of - # the guard there. - if not st.st_dev or not st.st_ino: - return None - return (st.st_dev, st.st_ino) - - class _Generation: """One shared SessionDB generation: instance, refcount, file identity.""" @@ -85,16 +56,13 @@ class _Generation: _lock = threading.Lock() -# path → live generation (never retired). A retired generation leaves -# this table immediately on retirement and lives on in _retired until -# its last holder releases. +# path → live generation. Retired generations move to _retired (keyed by +# id(db)) until their last holder releases. _generations: Dict[Path, _Generation] = {} -# Object-keyed retired generations still draining holders. -_retired: Dict[int, _Generation] = {} # id(db) → generation -# Paths whose next generation is currently being constructed. Construction -# stays outside _lock because schema reconciliation can take seconds, but peers -# for the SAME file must wait: otherwise every cold caller opens a writable -# SQLite connection before the registry chooses one winner. +_retired: Dict[int, _Generation] = {} +# Paths whose next generation is being constructed. Construction runs outside +# _lock (schema reconciliation can take seconds), but peers for the SAME file +# must wait or every cold caller opens its own writer before a winner is chosen. _opening: Dict[Path, threading.Event] = {} @@ -120,20 +88,13 @@ def _teardown(db: "SessionDB") -> None: def acquire(db_path: Optional[Path] = None) -> "SessionDB": """Return the shared SessionDB for *db_path*, incrementing its refcount. - The same resolved path always returns the same ``SessionDB`` instance - within one process, so all long-lived in-process callers share one - writer connection, one ``self._lock``, and one token-writer thread. + If the file was replaced (different inode) since the generation opened — + ``hermes sessions recover``, snapshot restore — that generation is RETIRED + but stays alive for its holders, and a fresh one is opened in its place. - If the underlying file was replaced (different inode) since the - shared generation was opened — e.g. by ``hermes sessions recover`` or - a snapshot restore — the current generation is RETIRED (never lent - again) but stays alive for its existing holders, and a fresh - generation is opened in its place. - - Raises whatever ``SessionDB.__init__`` raises (malformed, locked, - etc.). On a replacement-open failure the registry holds NO entry for - the path, so the next acquire retries fresh rather than handing out - a closed stale object. + Raises whatever ``SessionDB.__init__`` raises. On a replacement-open + failure the registry holds NO entry for the path, so the next acquire + retries fresh instead of receiving a closed stale object. """ from hermes_state import _default_db_path @@ -153,9 +114,8 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB": and generation.identity is not None and current != generation.identity ): - # File replaced: retire the live generation (its - # holders keep it until they release) and elect one - # caller to construct the replacement below. + # File replaced: retire, then elect one caller to open + # the replacement below. _retire_generation_locked(path, generation) else: generation.refcount += 1 @@ -167,13 +127,12 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB": _opening[path] = opening break - # Another caller is constructing this path. Do not hold the global - # registry lock while waiting: unrelated databases continue opening. - # A failed opener signals too, so one waiter can retry as the successor. + # Another caller is constructing this path; wait without holding the + # global lock. A failed opener signals too, so a waiter can retry. opening.wait() - # Open a fresh generation OUTSIDE the lock. The per-path opening marker - # prevents redundant writer connections without serialising other files. + # Open OUTSIDE the lock; the per-path marker prevents redundant writers + # without serialising other files. try: db = _open_session_db(path) db._shared_registry_owned = True @@ -188,8 +147,7 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB": with _lock: existing = _generations.get(path) if existing is not None: - # Defensive: a generation may have been installed by explicit - # registry manipulation while this open was in flight. + # Defensive: installed by explicit registry manipulation mid-open. existing.refcount += 1 winner = existing.db else: @@ -206,9 +164,8 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB": def _retire_generation_locked(path: Path, generation: _Generation) -> None: """Retire *generation* so it is never lent again (caller holds _lock). - The instance stays alive — its holders still own references — and is - tracked in ``_retired`` keyed by ``id(db)`` so their releases find - the right generation even after the path maps to a new one. + It stays alive for its holders, tracked in ``_retired`` by ``id(db)`` so + their releases find it even after the path maps to a new generation. """ generation.retired = True if _generations.get(path) is generation: @@ -219,15 +176,11 @@ def _retire_generation_locked(path: Path, generation: _Generation) -> None: def release(db: "SessionDB") -> bool: """Decrement the refcount of a shared SessionDB. - Returns ``True`` if *db* was a shared instance and its refcount was - decremented; ``False`` if *db* is not registry-managed (caller owns - its own close()). The final release of a generation tears it down — - OUTSIDE the registry lock, so a close-time WAL checkpoint never - stalls acquisition for every state.db in the process. - - Object-keyed lookup means an inode replacement cannot strand a - still-owned generation: holders of the old generation release into - the retired record, not into whatever the path currently names. + Returns ``True`` if *db* was shared; ``False`` if it is not registry-managed + (caller owns close()). The final release tears the generation down OUTSIDE + the registry lock so a close-time WAL checkpoint never stalls acquisition. + Lookup is object-keyed, so holders of an old generation release into its + retired record, not into whatever the path currently names. """ if db is None: return False @@ -244,8 +197,7 @@ def release(db: "SessionDB") -> bool: return False generation = _generations.get(path) if generation is None or generation.db is not db: - # Not a shared instance (caller used SessionDB() - # directly) — nothing to do; the caller owns close(). + # Not shared (bare SessionDB()); the caller owns close(). return False generation.refcount -= 1 needs_teardown = generation.refcount <= 0 @@ -259,21 +211,17 @@ def release(db: "SessionDB") -> bool: _generations.pop(Path(path), None) except (TypeError, ValueError): pass - # Teardown OUTSIDE the lock: it stops the token writer, checkpoints - # the WAL, and drains the read pool — none of which may hold up - # acquisition for every other state.db in the process. + # Teardown OUTSIDE the lock: stopping the token writer, WAL checkpoint and + # read-pool drain must not block acquisition for every other state.db. if needs_teardown: _teardown(db) return True def close_all() -> int: - """Close every shared SessionDB in this process, regardless of refcount. + """Close every shared SessionDB regardless of refcount; returns the count. - Called at gateway shutdown (after all agents and cron jobs have - finished) to release every WAL write lock and drain every - token-writer thread cleanly. Returns the number of instances - closed. Idempotent. + For gateway shutdown, after all agents and cron jobs finished. Idempotent. """ closed = 0 with _lock: @@ -282,7 +230,6 @@ def close_all() -> int: _retired.clear() for generation in generations: generation.retired = True - # Teardown outside the lock, one generation at a time. for generation in generations: _teardown(generation.db) closed += 1 @@ -290,12 +237,11 @@ def close_all() -> int: def live_shared_session_dbs() -> List["SessionDB"]: - """Snapshot of every live (non-retired) shared SessionDB in this process. + """Snapshot of every live (non-retired) shared SessionDB. - For periodic in-process maintenance (the gateway housekeeping tick's - deferred-FTS retry). Refcounts are NOT touched: the caller only invokes - a method on an instance that some holder already keeps alive; a - concurrent final release closes it and the callee sees ``_conn is None``. + For in-process maintenance (housekeeping deferred-FTS retry). Refcounts + are NOT touched: a concurrent final release may close an instance, in + which case the callee sees ``_conn is None``. """ with _lock: return [g.db for g in _generations.values() if not g.retired] @@ -314,9 +260,7 @@ def stats() -> Dict[str, int]: } -# ── Backwards-compatible aliases (hermes_state re-exports) ── -# Kept so call sites and tests can import either from hermes_state -# (the historical path) or from this module directly. +# ── Backwards-compatible aliases (hermes_state re-exports them) ── def get_shared_session_db(db_path: Optional[Path] = None) -> "SessionDB": return acquire(db_path) @@ -333,10 +277,8 @@ def close_shared_session_dbs() -> int: def release_or_close(db: "SessionDB") -> None: """Release a shared instance, or close it when it is not registry-managed. - The one-line cleanup for call sites that previously did a plain - ``db.close()``: shared instances return their refcount to the - registry (the registry owns the lifecycle), anything else — read-only - opens, CLI one-shots, test fakes — falls back to a direct close. + Drop-in for a plain ``db.close()``: read-only opens, CLI one-shots and + test fakes fall back to a direct close. """ if not release(db): try: diff --git a/hermes_state_repair.py b/hermes_state_repair.py new file mode 100644 index 0000000000..ba10d93376 --- /dev/null +++ b/hermes_state_repair.py @@ -0,0 +1,1639 @@ +"""state.db repair, backup and writability preflight (split from hermes_state). + +Every name is re-imported into ``hermes_state``; intra-module calls to +patchable helpers go through a lazy ``from hermes_state import ...`` at call +time so monkeypatches there still intercept. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import json +import logging +import os +import sqlite3 +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from hermes_constants import get_hermes_home +from hermes_startup_watchdog import report_startup_progress +from hermes_state_common import ( + _acquire_db_flock, + _clear_lock_holder_record, + _describe_lock_holder, + _read_lock_holder_record, + is_advisory_lock_contention, +) + +# Log-record parity with the origin module (caplog tests pin "hermes_state"). +logger = logging.getLogger("hermes_state") + + +def _claim_repair_attempt(db_path: Path) -> bool: + """Claim the one-shot per-process repair attempt for *db_path*. + + True for the first caller, False afterwards: bounds the repair/reopen loop + and stops concurrent callers racing surgery on one file. + """ + from hermes_state import _repair_attempt_lock, _repair_attempted_paths + key = str(db_path) + with _repair_attempt_lock: + if key in _repair_attempted_paths: + return False + _repair_attempted_paths.add(key) + return True + + +_REPAIR_LOCK_POLL_SECONDS = 0.1 + + +# Snapshot copies are data transfer, not inter-process locking: bound them +# separately at 10 MiB/s, with the historical two-minute floor. +_REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND = 10 * 1024 * 1024 + + +@contextlib.contextmanager +def _cross_process_repair_lock(db_path: Path): + """Serialize state.db schema surgery across processes. + + Yields True when this process holds the repair lock for *db_path*, False + when the bounded acquire timed out or the lock file could not be opened. + Unlike the kanban init lock (idempotent critical section), running surgery + unlocked IS the unsafe interleaving this prevents: a caller that gets + False must NOT do surgery. + + ``flock`` because the kernel drops it when the holder dies (a pidfile + would wedge every future repair); a forked child that inherited the fd is + the exception, so the acquire records the holder's pid + start time and + breaks the lock when that holder is provably dead (``_acquire_db_flock``). + The acquire is bounded because a *live* repairer can sit in ``VACUUM`` + for minutes, and an unbounded wait would hang the caller's open silently. + """ + from hermes_state import _IS_WINDOWS, _REPAIR_LOCK_TIMEOUT_SECONDS + lock_path = db_path.with_name(db_path.name + ".repair.lock") + try: + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = lock_path.open("a+b") + except OSError as exc: + # Fail closed, like a timed-out acquire. An unopenable lock file means + # out of space/inodes/descriptors — and a sibling that opened ITS + # handle before the disk filled may still be inside surgery; yielding + # True here once let two processes run surgery on the same live + # state.db. Callers handle False by re-probing. + logger.warning( + "Could not open state.db repair lock %s (%s) — skipping schema " + "surgery rather than running it without cross-process authority.", + lock_path, exc, + ) + yield False + return + + acquired = False + try: + if _IS_WINDOWS: + deadline = time.monotonic() + _REPAIR_LOCK_TIMEOUT_SECONDS + while True: + try: + import msvcrt + + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + acquired = True + break + except (BlockingIOError, OSError) as exc: + if not is_advisory_lock_contention(exc): + logger.warning( + "Could not acquire state.db repair lock %s (%s) — " + "skipping schema surgery on a non-contention error.", + lock_path, exc, + ) + acquired = None + break + if time.monotonic() >= deadline: + break + time.sleep(_REPAIR_LOCK_POLL_SECONDS) + else: + acquired, handle = _acquire_db_flock( + str(lock_path), + handle, + _REPAIR_LOCK_TIMEOUT_SECONDS, + _REPAIR_LOCK_POLL_SECONDS, + "state.db repair lock", + ) + if acquired is None: + # Non-contention failure already logged with its errno. + acquired = False + elif not acquired: + record = None if _IS_WINDOWS else _read_lock_holder_record(handle) + logger.warning( + "state.db repair lock %s held by another process for more " + "than %.0fs — skipping schema surgery in this process to " + "avoid racing the repairer. Recorded holder: %s.", + lock_path, _REPAIR_LOCK_TIMEOUT_SECONDS, + _describe_lock_holder(record), + ) + yield acquired + finally: + try: + if acquired: + if _IS_WINDOWS: + import msvcrt + + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + _clear_lock_holder_record(handle) + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + except OSError: # pragma: no cover - best effort release + pass + finally: + handle.close() + + +def _try_acquire_auto_maintenance_lock(db_path: Path) -> Optional[Any]: + """Non-blocking cross-process lock for one auto-maintenance pass. + + Advisory lock the kernel releases if the holder exits. A caller that cannot + acquire it must skip the pass: otherwise two startups both pass the interval + check and the second prunes a row the first has only just closed recoverably. + """ + from hermes_state import _IS_WINDOWS + lock_path = db_path.with_name(db_path.name + ".auto-maintenance.lock") + try: + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = lock_path.open("a+b") + except OSError as exc: + logger.warning( + "Could not open state.db auto-maintenance lock %s (%s) — skipping " + "automatic maintenance.", + lock_path, + exc, + ) + return None + + try: + if _IS_WINDOWS: + import msvcrt + + handle.seek(0) + msvcrt.locking( # type: ignore[attr-defined] + handle.fileno(), msvcrt.LK_NBLCK, 1 # type: ignore[attr-defined] + ) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except (BlockingIOError, OSError): + handle.close() + return None + return handle + + +def _release_auto_maintenance_lock(handle: Any) -> None: + """Release a handle returned by :func:`_try_acquire_auto_maintenance_lock`.""" + from hermes_state import _IS_WINDOWS + try: + if _IS_WINDOWS: + import msvcrt + + handle.seek(0) + msvcrt.locking( # type: ignore[attr-defined] + handle.fileno(), msvcrt.LK_UNLCK, 1 # type: ignore[attr-defined] + ) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + except OSError: # pragma: no cover - best effort release + pass + finally: + handle.close() + + +def _bump_schema_cookie(conn: sqlite3.Connection) -> None: + """Increment the schema cookie after direct ``sqlite_master`` surgery. + + Ordinary DDL bumps this counter and other connections compare it before + running a prepared statement — that is how they discard a cached schema. + Editing ``sqlite_master`` under ``writable_schema=ON`` does NOT bump it, + so live connections elsewhere keep compiling against the old schema (e.g. + firing triggers into ``messages_fts*`` shadow tables that no longer + exist). Best-effort, never raises: a failed bump leaves the status quo. + """ + try: + current = conn.execute("PRAGMA schema_version").fetchone()[0] + # Wrap within SQLite's 32-bit signed range; peers compare for equality. + conn.execute(f"PRAGMA schema_version={(int(current) + 1) & 0x7FFFFFFF}") + except (sqlite3.DatabaseError, TypeError, IndexError) as exc: + logger.warning("Could not bump state.db schema cookie: %s", exc) + + +_MAX_PERSISTENT_REPAIR_ATTEMPTS = 3 + + +_MAX_MALFORMED_BACKUPS = 3 + + +# Sidecars copied alongside a damaged DB and pruned with it. ``-journal`` +# matters because rollback-journal (DELETE) mode — Hermes's fallback on +# NFS/SMB/FUSE/ZFS and WAL-reset-vulnerable SQLite builds — leaves a hot +# journal whenever a transaction was open; without it the forensic copy +# cannot be rolled back to a consistent state by hand. +_DB_SIDECAR_SUFFIXES = ("-wal", "-shm", "-journal") + + +# Head/tail bytes sampled by ``_db_fingerprint``: changes on any genuine +# repair/truncation/restore while staying O(1) on a multi-GB file. +_FINGERPRINT_SAMPLE_BYTES = 65536 + + +# Header ranges that move on ordinary commits rather than on repair, masked +# out of the content sample: file change counter (24-27) and version-valid-for +# (92-95). In DELETE mode a commit writes the main file directly and a +# malformed-SCHEMA DB still accepts writes, so without the mask any live write +# re-keys the ledger and the repair budget resets to 1 forever. (WAL mode +# routes commits to the -wal sidecar; masking is harmless there.) The page-1 +# sqlite_master b-tree — what repair identity depends on — sits after byte 100 +# and stays in the sample. +_FINGERPRINT_VOLATILE_HEADER_RANGES = ((24, 28), (92, 96)) + + +def _mask_volatile_header(head: bytes) -> bytes: + """Zero the commit-counter fields so ordinary writes don't re-key the ledger.""" + if len(head) < 96: + return head + buf = bytearray(head) + for start, end in _FINGERPRINT_VOLATILE_HEADER_RANGES: + buf[start:end] = b"\x00" * (end - start) + return bytes(buf) + + +# Free-space headroom for the pre-repair forensic backup: a full raw copy of +# the damaged DB plus sidecars, so a repair loop on a large state.db is a disk +# amplifier (one incident wrote ~98MB every ~10s until the volume was nearly +# full). Proportional, not a flat floor: an absolute multi-GB reserve would +# refuse backups that fit on small container/VM volumes, and since a refused +# backup is a HARD STOP that would turn "repair loops" into "repair never +# runs" there. Require the copy plus a small slice of the volume, with a +# modest floor. +_REPAIR_BACKUP_MIN_FREE_BYTES = 256 * 1024 * 1024 # 256 MiB absolute floor + + +_REPAIR_BACKUP_FREE_FRACTION = 0.02 # plus 2% of the volume + + +def _repair_backup_headroom_bytes(total_bytes: int) -> int: + """Free space required *beyond* the copy itself, for a volume of *total_bytes*.""" + return max( + _REPAIR_BACKUP_MIN_FREE_BYTES, + int(total_bytes * _REPAIR_BACKUP_FREE_FRACTION), + ) + + +def _repair_scratch_space_error(db_path: Path) -> Optional[str]: + """Return an error unless snapshot, VACUUM and promotion can fit safely.""" + import shutil + + try: + main_bytes = db_path.stat().st_size + snapshot_bytes = main_bytes + for suffix in _DB_SIDECAR_SUFFIXES: + sidecar = db_path.with_name(db_path.name + suffix) + if sidecar.exists(): + snapshot_bytes += sidecar.stat().st_size + usage = shutil.disk_usage(db_path.parent) + headroom = _repair_backup_headroom_bytes(usage.total) + # Strategy 2 runs VACUUM on the staged DB, which SQLite documents may + # need up to 2x the database size in extra space; the same reserve then + # covers transactional promotion into the live DB. + required = snapshot_bytes + (2 * snapshot_bytes) + headroom + if usage.free >= required: + return None + return ( + f"only {usage.free / 1e9:.2f}GB free on {db_path.parent}; the " + f"repair snapshot needs up to {snapshot_bytes / 1e9:.2f}GB, " + f"VACUUM may need another {(2 * snapshot_bytes) / 1e9:.2f}GB, and " + f"{headroom / 1e9:.2f}GB must remain as headroom. Free disk space, " + "then retry." + ) + except OSError as exc: + return ( + f"could not determine free space on {db_path.parent} ({exc}); " + "refusing the repair snapshot rather than risk filling the volume" + ) + + +def _repair_snapshot_timeout_seconds(source_path: Path) -> float: + """Bound one SQLite snapshot by source size, including live sidecars. + + A WAL can hold committed rows not yet in the main file; count it so a + healthy large-database copy is not cut off by the repair-lock timeout. + """ + from hermes_state import _REPAIR_LOCK_TIMEOUT_SECONDS, _REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND + source_bytes = 0 + for suffix in ("", *_DB_SIDECAR_SUFFIXES): + candidate = ( + source_path + if not suffix + else source_path.with_name(source_path.name + suffix) + ) + try: + source_bytes += candidate.stat().st_size + except FileNotFoundError: + continue + return max( + _REPAIR_LOCK_TIMEOUT_SECONDS, + source_bytes / _REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND, + ) + + +def _repair_failure_consumes_attempt(exc: BaseException) -> bool: + """Whether a pre-strategy SQLite failure proves deterministic corruption. + + Lock contention, timeouts, disk-full, I/O and filesystem failures are + environmental — a retry may succeed, so they must not burn the repair + ledger. Only SQLite's corruption/image result codes prove deterministic + damage, even when SQLite cannot stage a snapshot far enough to run a + named strategy. + """ + if not isinstance(exc, sqlite3.DatabaseError): + return False + error_code = getattr(exc, "sqlite_errorcode", None) + if isinstance(error_code, int): + # Extended result codes keep the primary code in the low byte. + primary_code = error_code & 0xFF + return primary_code in (sqlite3.SQLITE_CORRUPT, sqlite3.SQLITE_NOTADB) + + # Older sqlite3 without result-code attributes: narrow message match only, + # never turning generic "disk is full"/"readonly" into permanent failures. + message = str(exc).lower() + return ( + "file is not a database" in message + or "database disk image is malformed" in message + ) + + +def _repair_ledger_path(db_path: Path) -> Path: + return db_path.with_name(db_path.name + ".repair-attempts.json") + + +def _db_fingerprint(db_path: Path) -> "Optional[str]": + """Cheap identity for a damaged DB file: size + a bounded content sample. + + Deliberately EXCLUDES mtime: the malformed-schema class still accepts + writes, so live writers, WAL checkpoints and the strategies themselves move + mtime between passes; keyed on mtime, every pass looked like a NEW file, + the attempt counter reset to 1 forever and each pass wrote another + full-size forensic copy. Hashing a multi-GB file on every open is the cost + this ledger exists to avoid, so sample the head/tail slices any real + repair, truncation or restore necessarily changes. + + Runs under ``offline_file_access``: ``close()`` on ANY raw descriptor + cancels every POSIX advisory lock this process holds on the file, including + a peer connection's RESERVED lock (``hermes_cli.sqlite_safe_read`` rule 1), + and a live peer is the expected case here (this runs BEFORE + ``_backup_db_file``'s ``has_live_connection`` guard). Returns ``None`` + ("identity unavailable") when that makes the read unsafe. Callers MUST NOT + substitute a differently-shaped key: the ledger compares keys for equality, + so alternating shapes never matches and the unbounded loop returns; the + ledger helpers keep the recorded key instead. + """ + try: + st = db_path.stat() + try: + from hermes_cli.sqlite_safe_read import ( + LiveConnectionError, + offline_file_access, + ) + except ImportError: + # Scaffold/embed installs ship hermes_state without hermes_cli; no + # tracked connections exist there, so the raw read is safe. + @contextmanager + def offline_file_access(_path, **_kw): + yield + + class LiveConnectionError(Exception): + pass + + try: + with offline_file_access(db_path, what="fingerprint"): + with open(db_path, "rb") as fh: + head = fh.read(_FINGERPRINT_SAMPLE_BYTES) + if st.st_size > _FINGERPRINT_SAMPLE_BYTES: + fh.seek(max(0, st.st_size - _FINGERPRINT_SAMPLE_BYTES)) + tail = fh.read(_FINGERPRINT_SAMPLE_BYTES) + else: + tail = b"" + except LiveConnectionError: + return None + digest = hashlib.sha256(_mask_volatile_header(head) + tail).hexdigest()[:32] + return f"{st.st_size}:{digest}" + except OSError: + return None + + +def _backup_content_identity(db_path: Path) -> "Optional[str]": + """Recovery-image identity for forensic-backup dedupe: whole file + sidecars. + + A DIFFERENT equivalence relation from :func:`_db_fingerprint`; never + conflate them. The fingerprint answers "same repair epoch?" and masks + commit counters / samples only head+tail so an ordinary write does not mint + a fresh repair budget. A live writer can commit rows into an *interior* + page while preserving size and the first/last 64 KiB, so two materially + different recovery images share one fingerprint; reusing a backup on that + basis hands the operator a snapshot predating real user data. A forensic + copy must claim byte identity, so this digests the ENTIRE main file plus + every present sidecar (the WAL can hold uncheckpointed committed + frames). The O(n) read is cheaper than the O(n) write it avoids. Runs under + ``offline_file_access`` (same POSIX-lock reason as ``_db_fingerprint``); + ``None`` when a live connection makes the read unsafe — the caller then + takes a fresh backup, never a false reuse. + """ + try: + from hermes_cli.sqlite_safe_read import ( + LiveConnectionError, + offline_file_access, + ) + except ImportError: + @contextmanager + def offline_file_access(_path, **_kw): + yield + + class LiveConnectionError(Exception): + pass + + def _hash_whole(path: Path, hasher: "Any") -> None: + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + hasher.update(chunk) + + try: + hasher = hashlib.sha256() + with offline_file_access(db_path, what="backup-identity"): + # Length-delimit every member (main file included) so the + # concatenation is prefix-free; otherwise a main-file tail could + # coincide with a main+sidecar split and dedupe two images together. + hasher.update(f"\0main:{db_path.stat().st_size}\0".encode()) + _hash_whole(db_path, hasher) + for suffix in _DB_SIDECAR_SUFFIXES: + sidecar = db_path.with_name(db_path.name + suffix) + if sidecar.exists(): + hasher.update(f"\0{suffix}:{sidecar.stat().st_size}\0".encode()) + _hash_whole(sidecar, hasher) + return hasher.hexdigest() + except LiveConnectionError: + return None + except OSError: + return None + + +def _read_repair_ledger(db_path: Path) -> "Dict[str, Any]": + try: + raw = json.loads(_repair_ledger_path(db_path).read_text(encoding="utf-8")) + if isinstance(raw, dict): + return raw + except (OSError, ValueError): + pass + return {} + + +def _persistent_repair_attempts_exhausted(db_path: Path) -> bool: + """Whether *db_path* has already burned its cross-restart repair budget. + + True only when the ledger records ``_MAX_PERSISTENT_REPAIR_ATTEMPTS`` + failures against the CURRENT fingerprint. Never raises; a missing/corrupt + ledger or unstatable DB reads as "not exhausted" (the in-process claim and + cross-process lock still bound one run). When a live connection makes the + fingerprint unavailable, fall back to the SIZE the ledger recorded — + otherwise a peer connection hides an exhausted budget on every pass. + """ + ledger = _read_repair_ledger(db_path) + recorded = ledger.get("fingerprint") + fp = _db_fingerprint(db_path) + if fp is None: + # Size is the one key component that needs no raw read. + try: + size_prefix = f"{db_path.stat().st_size}:" + except OSError: + return False + if not isinstance(recorded, str) or not recorded.startswith(size_prefix): + return False + elif recorded != fp: + return False + return int(ledger.get("failed_attempts", 0)) >= _MAX_PERSISTENT_REPAIR_ATTEMPTS + + +def _persistent_repair_exhausted_error(db_path: Path) -> str: + """The stable operator-facing diagnostic for an exhausted repair budget.""" + return ( + f"automatic repair has already failed " + f"{_MAX_PERSISTENT_REPAIR_ATTEMPTS} times on this exact file — " + "the corruption is beyond the schema/FTS repair strategies " + "(likely b-tree page damage). Manual recovery required: restore " + f"a backup, or salvage with `sqlite3 {db_path} \".recover\"`. " + f"Delete {_repair_ledger_path(db_path).name} to force another " + "automatic attempt." + ) + + +def _record_repair_outcome( + db_path: Path, *, repaired: bool, fingerprint: "Optional[str]" = None +) -> None: + """Update the persistent attempt ledger after a repair pass. Never raises. + + Defaults to the post-attempt fingerprint (what the NEXT exhaustion probe + observes). When a live connection makes it unavailable, keep the recorded + key and still increment — dropping the pass would let a peer connection + reset the budget every time. Never write a differently shaped key. + """ + ledger_path = _repair_ledger_path(db_path) + try: + if repaired: + ledger_path.unlink(missing_ok=True) + return + ledger = _read_repair_ledger(db_path) + recorded = ledger.get("fingerprint") + fp = fingerprint if fingerprint is not None else _db_fingerprint(db_path) + if fp is None: + if not isinstance(recorded, str): + # No prior key to extend and no safe way to mint one; the + # in-process claim and cross-process lock still bound this run. + return + fp = recorded + attempts = ( + int(ledger.get("failed_attempts", 0)) + 1 if recorded == fp else 1 + ) + import datetime + + ledger_path.write_text( + json.dumps( + { + "fingerprint": fp, + "failed_attempts": attempts, + "last_attempt": datetime.datetime.now().isoformat( + timespec="seconds" + ), + } + ), + encoding="utf-8", + ) + except Exception as exc: # pragma: no cover - best effort + logger.warning("Could not update state.db repair ledger: %s", exc) + + +def _existing_malformed_backups(db_path: Path) -> "List[Path]": + """Timestamped forensic backups of *db_path*, newest first.""" + prefix = f"{db_path.name}.malformed-backup-" + try: + found = [ + p + for p in db_path.parent.iterdir() + if p.name.startswith(prefix) + and not p.name.endswith(_DB_SIDECAR_SUFFIXES) + ] + except OSError: + return [] + return sorted(found, key=lambda p: p.name, reverse=True) + + +def _prune_malformed_backups(db_path: Path, keep: int = _MAX_MALFORMED_BACKUPS) -> None: + """Delete all but the *keep* newest forensic backups (and sidecars).""" + for stale in _existing_malformed_backups(db_path)[keep:]: + for victim in ( + stale, + *(stale.with_name(stale.name + suffix) for suffix in _DB_SIDECAR_SUFFIXES), + ): + try: + victim.unlink(missing_ok=True) + except OSError as exc: # pragma: no cover - best effort + logger.warning("Could not prune stale DB backup %s: %s", victim, exc) + + +def _backup_db_file(db_path: Path) -> "Tuple[Optional[Path], Optional[str]]": + """Raw-copy a (possibly malformed) DB plus sidecars to a timestamped backup. + + Raw bytes on purpose: the DB won't open cleanly, so preserve them exactly + for forensics / manual restore. Returns ``(backup_path, None)`` or + ``(None, reason)``; the repair path treats a refused backup as a HARD STOP + because the forensic bundle is the recovery path when every strategy fails. + Refuses while a connection to this DB is live in the process: reading the + file would ``close()`` a descriptor and cancel that connection's POSIX + advisory locks (see ``hermes_cli.sqlite_safe_read``) — a real case, since + one SessionDB can enter repair while the gateway holds others. + """ + import datetime + import shutil + + try: + from hermes_cli.sqlite_safe_read import has_live_connection + except ImportError: + has_live_connection = None # type: ignore[assignment] + + if has_live_connection is not None and has_live_connection(db_path): + reason = ( + f"a connection to {db_path} is still open in this process; " + "raw-copying it would cancel that connection's POSIX advisory " + "locks. Close all SessionDB handles first." + ) + logger.error("Refusing to raw-copy %s for backup: %s", db_path, reason) + return None, reason + + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + backup_path = db_path.with_name(f"{db_path.name}.malformed-backup-{stamp}") + # Same-second collision (two damaged states within one second) must not + # overwrite the earlier forensic copy. + seq = 1 + while backup_path.exists(): + backup_path = db_path.with_name( + f"{db_path.name}.malformed-backup-{stamp}_{seq}" + ) + seq += 1 + try: + # Sweep staging debris from an earlier interrupted pass BEFORE the + # dedupe: leftover staging is byte-identical to the damaged DB, so + # dedupe would hand it back as a legitimate backup. Also sweeps the + # pre-merge ``.incomplete`` spelling, which prefix-matches as a backup, + # sorts NEWEST and would otherwise survive prune forever. + for pattern in ( + f"{db_path.name}.backup-staging-*", + f"{db_path.name}.malformed-backup-*.incomplete*", + ): + for old in db_path.parent.glob(pattern): + try: + old.unlink(missing_ok=True) + except OSError: # pragma: no cover - best effort + pass + # Dedupe: a repair loop used to copy the SAME damaged bytes on every + # restart (~900MB a pass, 89GB over 11 days). If the newest existing + # backup is byte-identical to the current recovery image, reuse it. + # Match on ``_backup_content_identity`` — NOT mtime (the malformed- + # SCHEMA class still accepts writes, so mtime missed every pass) and + # NOT ``_db_fingerprint`` (see its docstring: an interior-page write + # changes the recovery image without changing the fingerprint). + try: + # Only hash the source when there is a candidate to compare + # against; hashing a multi-GB source right before copying it is + # pure waste on the common first-corruption pass. + existing_backups = _existing_malformed_backups(db_path)[:1] + if existing_backups: + src_id = _backup_content_identity(db_path) + for existing in existing_backups: + if src_id is not None and _backup_content_identity(existing) == src_id: + logger.info( + "Reusing existing forensic backup %s (identical to the " + "damaged DB).", existing, + ) + return existing, None + except OSError: + pass + # Disk guard: a full raw copy on a nearly-full volume (which a + # preceding repair loop may itself have caused) can finish off the + # disk and every process on the machine. Refuse while there is room to. + try: + need = db_path.stat().st_size + for suffix in _DB_SIDECAR_SUFFIXES: + sidecar = db_path.with_name(db_path.name + suffix) + if sidecar.exists(): + need += sidecar.stat().st_size + usage = shutil.disk_usage(db_path.parent) + headroom = _repair_backup_headroom_bytes(usage.total) + if usage.free - need < headroom: + reason = ( + f"only {usage.free / 1e9:.2f}GB free on {db_path.parent}; " + f"copying the damaged DB needs {need / 1e9:.2f}GB and must " + f"leave {headroom / 1e9:.2f}GB headroom. Free disk space, " + f"then retry (or recover manually with `sqlite3 {db_path} " + '".recover"`).' + ) + logger.error("Refusing forensic backup of %s: %s", db_path, reason) + return None, reason + except OSError as exc: + # Fail CLOSED: the nearly-full volume this guard exists for is + # exactly where stat()/disk_usage() is most likely to fail, and + # proceeding would take the copy that finishes off the disk. Repair + # then waits (HARD STOP) for a human to free space — the safe side. + reason = ( + f"could not determine free space on {db_path.parent} ({exc}); " + "refusing the forensic copy rather than risk filling the " + f"volume. Free disk space, then retry (or recover manually " + f'with `sqlite3 {db_path} ".recover"`).' + ) + logger.error("Refusing forensic backup of %s: %s", db_path, reason) + return None, reason + # Copy to a staging name OUTSIDE the ``.malformed-backup-`` prefix and + # rename into place only once every copy succeeded. A staging name + # inside the prefix (e.g. ``…-.incomplete``) counts as a backup, + # sorts NEWEST (so prune kept partials and deleted intact copies), and + # dedupe could return it as ``backup_path`` — passing the hard stop with + # no real forensic copy on disk. + staging = db_path.with_name(f"{db_path.name}.backup-staging-{stamp}") + # (staging_src, final_dst) pairs. PUBLICATION ORDER MATTERS: the main + # DB name is the bundle's commit marker (what + # ``_existing_malformed_backups`` counts), so sidecars go FIRST and the + # main DB LAST; a failure partway then never leaves a countable main + # backup over a missing sidecar that would pass the hard stop and dedupe. + staged_sidecars: "List[Tuple[Path, Path, Path]]" = [] + for suffix in _DB_SIDECAR_SUFFIXES: + sidecar = db_path.with_name(db_path.name + suffix) + if sidecar.exists(): + side_staging = staging.with_name(staging.name + suffix) + side_dst = backup_path.with_name(backup_path.name + suffix) + staged_sidecars.append((sidecar, side_staging, side_dst)) + main_pair = (staging, backup_path) + published: "List[Path]" = [] + all_staging_srcs = [staging] + [s for _src, s, _d in staged_sidecars] + try: + shutil.copy2(db_path, staging) + for sidecar, side_staging, _side_dst in staged_sidecars: + shutil.copy2(sidecar, side_staging) + publish_order = [ + (s, d) for _src, s, d in staged_sidecars + ] + [main_pair] + for src, dst in publish_order: + os.replace(src, dst) + published.append(dst) + except Exception: + # Roll back unpublished staging files AND anything already promoted, + # so a failure after the main os.replace leaves no official backup_path. + for src in all_staging_srcs: + try: + src.unlink(missing_ok=True) + except OSError: + pass + for dst in published: + try: + dst.unlink(missing_ok=True) + except OSError: + pass + try: + staging.unlink(missing_ok=True) + except OSError: + pass + raise + _prune_malformed_backups(db_path) + return backup_path, None + except Exception as exc: # pragma: no cover - best effort + logger.warning("Could not back up malformed DB %s: %s", db_path, exc) + return None, f"backup copy failed: {exc}" + + +def preflight_db_writability( + db_path: Path, + *, + db_label: str = "state.db", +) -> None: + """Refuse-or-repair read-only DB files BEFORE the first connection opens. + + A stray read-only ``state.db`` / ``-wal`` / ``-shm`` (sudo run, restored + backup, copied dotfiles) otherwise surfaces as an opaque "attempt to write + a readonly database" deep inside ``_init_schema``, and the obvious wrong + "fix" (deleting the ``-wal``) silently loses committed transactions. + Repairs with ``chmod u+rw`` only inside the Hermes home tree (Hermes owns + those files, and ``chmod`` fails on files the user doesn't own, which + bounds the repair exactly); otherwise fails fast naming the exact file and + ``chmod`` command. Never deletes or truncates a WAL sidecar — once + writable, the normal open path checkpoints its committed frames. + ``:memory:`` and ``file:`` URIs are skipped. Shared with ``kanban_db``. + """ + raw = str(db_path) + if raw == ":memory:" or raw.startswith("file:"): + return + + try: + home: Optional[Path] = Path(get_hermes_home()).resolve() + except Exception: # pragma: no cover - defensive + home = None + + def _in_repair_scope(p: Path) -> bool: + if home is None: + return False + try: + return p.resolve().is_relative_to(home) + except (OSError, ValueError): + return False + + def _ensure_writable(p: Path, *, is_dir: bool = False) -> None: + import stat as _stat + + if os.access(p, os.R_OK | os.W_OK): + return + if _in_repair_scope(p): + try: + add = _stat.S_IRUSR | _stat.S_IWUSR | (_stat.S_IXUSR if is_dir else 0) + os.chmod(p, p.stat().st_mode | add) + except OSError: + pass + if os.access(p, os.R_OK | os.W_OK): + logger.info( + "%s preflight: repaired read-only %s (chmod u+rw%s)", + db_label, + p, + "x" if is_dir else "", + ) + return + kind = "directory" if is_dir else "file" + wal_note = ( + " Do NOT delete the -wal file — it contains committed data that " + "will be merged into the database once it is writable." + if p.name.endswith("-wal") + else "" + ) + raise sqlite3.OperationalError( + f"{db_label} is not writable: {kind} {p} is read-only for this " + f"user. Hermes needs read-write access to open the database. " + f"Fix with: chmod u+rw{'x' if is_dir else ''} '{p}'" + f" (files owned by another user may need sudo/chown).{wal_note}" + ) + + parent = db_path.parent + if parent.is_dir(): + # SQLite needs a writable directory in every journal mode (WAL/SHM + # sidecars, or the rollback journal in DELETE mode). + _ensure_writable(parent, is_dir=True) + + for suffix in ("", "-wal", "-shm"): + p = db_path.with_name(db_path.name + suffix) if suffix else db_path + if p.is_file(): + _ensure_writable(p) + + +def _connect_repair_durable( + db_path: Path, *, timeout: float = 5.0 +) -> sqlite3.Connection: + """``sqlite3.connect`` for the repair/probe paths, with macOS write barriers. + + These paths open ``state.db`` directly (not via ``SessionDB`` / + :func:`apply_wal_with_fallback`), so they inherited ``synchronous=NORMAL`` + and no ``checkpoint_fullfsync`` — on Darwin, where ``fsync()`` guarantees + neither data-on-platter nor ordering, an interrupted rewrite leaves + half-written b-tree pages, and ``REINDEX``/``VACUUM``/``writable_schema`` + surgery rewrite nearly every page. Autocommit (``isolation_level=None``) + is preserved: DDL and ``VACUUM`` are illegal inside an implicit + transaction. Barriers are best-effort by necessity: SQLite loads the schema + before any statement, so on a malformed schema even ``PRAGMA + synchronous=FULL`` raises — and a malformed DB is this helper's input. + Whole-file rewrites call :func:`_reapply_durability_barriers` once the + schema parses again. + """ + conn = sqlite3.connect(str(db_path), timeout=timeout, isolation_level=None) + _reapply_durability_barriers(conn) + return conn + + +def _reapply_durability_barriers(conn: sqlite3.Connection) -> bool: + """Best-effort (re)application of the macOS write barriers. Never raises. + + True when the pragmas were accepted. Call before ``VACUUM``/``REINDEX`` + once the schema parses: a connection opened on a malformed schema could + not take them at open time. + """ + from hermes_state import _apply_macos_checkpoint_barrier, _enforce_macos_synchronous_full + try: + _apply_macos_checkpoint_barrier(conn) + _enforce_macos_synchronous_full(conn) + return True + except sqlite3.DatabaseError: + # Schema still unparseable — pragmas cannot be set yet. + return False + except Exception: + return False + + +def apply_durability_barriers(conn: sqlite3.Connection) -> bool: + """Apply state-store durability barriers without changing journal mode. + + Public entry point for secondary users of ``state.db`` that must inherit + its owner's journal mode. Also applies the configured + ``database.synchronous`` level, a per-connection pragma that otherwise + only rides on the journal-mode setup path guests must not run. + """ + from hermes_state import _apply_synchronous_pragma + ok = _reapply_durability_barriers(conn) + try: + # Local import: avoids a circular import with hermes_cli.config. + from hermes_cli.config import cfg_get, load_config_readonly + + cfg = load_config_readonly() + raw_synchronous = cfg_get(cfg, "database", "synchronous", default=None) + if raw_synchronous is not None: + _apply_synchronous_pragma( + conn, raw_synchronous, db_label="state.db (guest)" + ) + except Exception: + pass + return ok + + +@contextmanager +def _exclusive_repair_db_guard(db_path: Path): + """Yield one live connection that excludes writers for repair surgery. + + ``locking_mode=EXCLUSIVE`` retains file-level exclusion after the short + ``BEGIN EXCLUSIVE`` is rolled back. The rollback is essential: + ``Connection.backup`` uses this connection as *source* and later as the + promotion *destination*, both of which require it transaction-free. It + stays open across the whole snapshot -> strategies -> promotion window, so + no other writer can commit a change promotion would overwrite. Existing + readers make acquisition fail rather than being disturbed: repair fails + closed unless this process owns the whole window. + """ + guard: Optional[sqlite3.Connection] = None + try: + # The cross-process repair lock already serializes repairers. Do not + # wait behind an ordinary application connection: a partial repair is + # less safe than an explicit "stop the gateway and retry". + guard = _connect_repair_durable(db_path, timeout=0.0) + guard.execute("PRAGMA locking_mode=EXCLUSIVE") + guard.execute("BEGIN EXCLUSIVE") + guard.execute("ROLLBACK") + except (sqlite3.Error, OSError) as exc: + if guard is not None: + try: + guard.execute("PRAGMA locking_mode=NORMAL") + except Exception: + pass + guard.close() + yield None, exc + return + + try: + yield guard, None + finally: + try: + # Release the exclusive locks before close; also keeps a close-time + # checkpoint from being mistaken for a repair write by callers that + # immediately reopen state.db. + guard.execute("PRAGMA locking_mode=NORMAL") + except Exception: + pass + guard.close() + + +def _copy_database_snapshot( + source_path: Path, + destination_path: Path, + *, + source_connection: Optional[sqlite3.Connection] = None, + destination_connection: Optional[sqlite3.Connection] = None, +) -> None: + """Copy one complete SQLite snapshot without replacing either file inode. + + The online backup API folds committed WAL frames into the source snapshot + and writes the destination in one transaction (rolled back if interrupted), + so ``state.db`` is never swapped out from under handles that refer to it. + """ + # Compute the deadline before opening an owned source connection: a + # sidecar vanishing mid-stat must not leak a just-opened descriptor. + deadline_seconds = _repair_snapshot_timeout_seconds(source_path) + deadline = time.monotonic() + deadline_seconds + source = source_connection or _connect_repair_durable(source_path) + destination = destination_connection + own_source = source_connection is None + own_destination = destination_connection is None + + def _check_deadline(_status: int, _remaining: int, _total: int) -> None: + if time.monotonic() >= deadline: + raise TimeoutError( + "timed out copying SQLite repair snapshot after " + f"{deadline_seconds:.0f}s" + ) + + try: + if destination is None: + destination = _connect_repair_durable(destination_path) + elif destination.in_transaction: + # sqlite3_backup needs a transaction-free destination; the exclusive + # guard retains exclusion via locking_mode, not a transaction. + raise sqlite3.ProgrammingError( + "SQLite repair backup destination has an active transaction" + ) + source.backup( + destination, + pages=256, + progress=_check_deadline, + sleep=_REPAIR_LOCK_POLL_SECONDS, + ) + finally: + if own_destination and destination is not None: + destination.close() + if own_source: + source.close() + + +def _db_opens_cleanly(db_path: Path) -> Optional[str]: + """Probe a DB on a fresh connection. Returns None if healthy, else a reason. + + Runs the first statement that trips the malformed-schema parse (``PRAGMA + journal_mode``), ``integrity_check``, a ``sessions`` read, FTS5 MATCH + probes, and a rolled-back ``messages`` write — so FTS5 index corruption, + which leaves reads and ``integrity_check`` passing while every ``INSERT + INTO messages`` fails through the FTS triggers, is reported as unhealthy. + """ + from hermes_state import SessionDB, load_fts5_cjk_extension + conn = _connect_repair_durable(db_path) + try: + # Best-effort tokenizer load: messages_fts_cjk needs cjk_unicode61 + # before any statement (incl. the trigger-driven write probe) can touch + # it. Without it this probe sees the DB as a tokenizer-less SessionDB + # would (which drops the cjk triggers), so tokenizer absence must never + # classify as corruption. + load_fts5_cjk_extension(conn) + conn.execute("PRAGMA journal_mode").fetchone() + rows = conn.execute("PRAGMA integrity_check").fetchall() + problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != "ok"] + if problems: + return "; ".join(problems[:3]) + conn.execute("SELECT COUNT(*) FROM sessions").fetchone() + + # FTS5 read probe. The write probe below misses partial shadow-table + # corruption where MATCH / snippet / rank raise DatabaseError("database + # disk image is malformed"), silently breaking session_search and + # /resume title resolution while check-only reports healthy. + for fts_table in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"): + try: + # Trigram backs title resolution, so probe it too. MATCH '""' + # (empty phrase) parses, scans zero rows and exercises the + # shadow-table read path; FTS5 rejects MATCH '' outright. + conn.execute( + f"SELECT 1 FROM {fts_table} WHERE {fts_table} MATCH '\"\"' LIMIT 1" + ).fetchone() + except sqlite3.OperationalError as exc: + # Canonical capability classifier: on builds without fts5 a + # legacy messages_fts table may exist and MATCH raises "no such + # module: fts5"; treating that as corruption would send the DB + # into repair, whose final fallback deletes the messages_fts% + # schema. Covers "no such tokenizer: trigram" too. + if SessionDB._is_fts5_unavailable_error(exc): + continue + msg = str(exc).lower() + if "no such table" in msg or "no such column" in msg: + # FTS5 not built yet (brand new file mid-init). + continue + return f"fts5 read probe failed on {fts_table}: {exc}" + except sqlite3.DatabaseError as exc: + # Partial shadow-table damage: MATCH raises though the table parses. + return f"fts5 read probe failed on {fts_table}: {exc}" + + # FTS write probe: drive a row through the messages_fts* triggers in a + # transaction that is always rolled back. Missing messages/sessions + # tables (brand new file mid-init) mean "not yet populated", not corruption. + probe_session_id = f"_hermes_fts_health_probe_{time.time_ns()}" + try: + conn.execute("BEGIN IMMEDIATE") + conn.execute( + "INSERT INTO sessions (id, source, started_at) VALUES (?, ?, ?)", + (probe_session_id, "_health_probe", time.time()), + ) + conn.execute( + "INSERT INTO messages (session_id, role, content, timestamp) " + "VALUES (?, ?, ?, ?)", + (probe_session_id, "user", "_fts_health_probe", time.time()), + ) + conn.execute("ROLLBACK") + except sqlite3.OperationalError as exc: + # Missing tables / FTS disabled — not the corruption class we probe. + try: + conn.execute("ROLLBACK") + except sqlite3.Error: + pass + msg = str(exc).lower() + if "no such table" in msg or "no such column" in msg: + return None + if "no such tokenizer: cjk_unicode61" in msg: + # This process couldn't load the cjk extension while the DB + # carries the cjk index — capability gap, not corruption. A + # tokenizer-less SessionDB self-heals by dropping the triggers. + return None + return str(exc) + return None + except sqlite3.DatabaseError as exc: + return str(exc) + finally: + conn.close() + + +def _live_writer_holds_db(db_path: Path) -> bool: + """True when a connection outside this call still holds ``db_path`` open. + + Asks SQLite for what a repair needs and a live holder cannot grant: + ``PRAGMA locking_mode=EXCLUSIVE`` then ``BEGIN IMMEDIATE``. In WAL mode + that needs exclusive locks on the WAL index, so any other open connection + fails it with SQLITE_BUSY; neither statement parses the schema, so it + works on malformed DBs. Fails **open** (False) on anything but a positive + busy/locked signal — refusing to repair a DB nobody holds would strand the + self-heal path. + + Scope: WAL mode only. In ``journal_mode=DELETE`` (Hermes's fallback on + WAL-reset-vulnerable builds and NFS/SMB) a held reader takes only SHARED + and this returns False; repair is then serialised only by the cross-process + repairer lock. Broadening to DELETE mode is a follow-up. + """ + probe = None + try: + probe = _connect_repair_durable(db_path, timeout=0.0) + probe.execute("PRAGMA locking_mode=EXCLUSIVE") + probe.execute("BEGIN IMMEDIATE") + probe.execute("ROLLBACK") + return False + except sqlite3.OperationalError as exc: + lowered = str(exc).lower() + return "locked" in lowered or "busy" in lowered + except sqlite3.DatabaseError: + # Malformed/unreadable: no evidence of a live holder either way. + return False + except Exception: + return False + finally: + if probe is not None: + try: + # Drop exclusive mode before close so the probe never leaves + # the file pinned. + probe.execute("PRAGMA locking_mode=NORMAL") + except Exception: + pass + try: + probe.close() + except Exception: + pass + + +def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, Any]: + """Repair a state.db whose ``sqlite_master`` is malformed or whose FTS + indexes reject writes. + + Two corruption classes: malformed schema / "duplicate object definition" + (even ``PRAGMA`` fails), and FTS write-corruption (base tables read fine, + ``integrity_check`` passes, writes fail through ``messages_fts*`` + triggers). Least-destructive first: (1) rebuild FTS in place via FTS5 + ``'rebuild'``; (2) de-duplicate ``sqlite_master`` (lowest rowid per + ``type``/``name``), FTS preserved; (3) drop the FTS schema + ``VACUUM``, + rebuilt on the next ``SessionDB()`` open. Canonical rows are never + modified by a failed attempt: strategies run on a complete SQLite snapshot + and a successful result is copied back transactionally. A raw backup is + taken first unless ``backup=False``. Surgery is serialised across + processes (:func:`_cross_process_repair_lock`): the gateway, Desktop + backend and CLI all open the same file, and concurrent ``writable_schema`` + surgery is itself a corruption source. + + Returns ``{repaired: bool, strategy: str|None, backup_path: str|None, + error: str|None}``. + """ + from hermes_state import _cross_process_repair_lock, _db_opens_cleanly, _live_writer_holds_db, _persistent_repair_attempts_exhausted, _probe_journal_mode_for_repair, _record_repair_outcome, _repair_state_db_schema_locked + report: Dict[str, Any] = { + "repaired": False, + "strategy": None, + "backup_path": None, + "error": None, + } + + # Startup-watchdog progress lease: repair is I/O-bound (near-zero CPU), + # which the watchdog's CPU fallback would misread as a parked deadlock. A + # single lease (clamped to _MAX_LEASE_S=900) is deliberate: up to that much + # zombie time on a wedged repair beats per-chunk renewal complexity. + report_startup_progress(900.0, phase="state_db_repair") + + db_path = Path(db_path) + if not db_path.exists(): + report["error"] = f"{db_path} does not exist" + return report + + # Cross-restart attempt cap: the in-memory claim bounds one process, but a + # class the strategies cannot heal (b-tree page damage) used to re-run the + # whole surgery, with a fresh forensic backup, on EVERY restart. After + # _MAX_PERSISTENT_REPAIR_ATTEMPTS failures on the same file, stop. + if _persistent_repair_attempts_exhausted(db_path): + report["error"] = _persistent_repair_exhausted_error(db_path) + logger.error("state.db repair skipped: %s", report["error"]) + return report + + result = report + with _cross_process_repair_lock(db_path) as holding_lock: + if not holding_lock: + # Another process is inside its critical section, or the lock file + # could not be opened. It may have healed the file already (long + # VACUUM after a successful strategy), so re-probe before failing. + if _db_opens_cleanly(db_path) is None: + report["repaired"] = True + report["strategy"] = "repaired_by_other_process" + else: + report["error"] = ( + "could not obtain the state.db repair lock (held by " + "another process, or the lock file was unopenable); " + "skipped schema surgery to avoid racing a concurrent " + "repairer" + ) + else: + # Recheck exhaustion after acquisition: a queued repairer can have + # recorded the final failure while this process waited, and this + # process must not start a fourth attempt. + if _persistent_repair_attempts_exhausted(db_path): + report["error"] = _persistent_repair_exhausted_error(db_path) + logger.error("state.db repair skipped: %s", report["error"]) + # WAL-holder preflight: fail-closed for active readers before a + # forensic backup is taken. Not the race defence — the exclusive + # guard in the locked routine excludes writers through promotion and + # rejects DELETE-mode readers this probe cannot see. + elif _live_writer_holds_db(db_path): + report["error"] = ( + "a live writer still holds state.db; skipped schema surgery " + "to avoid tearing b-tree pages under a concurrent writer. " + "Stop the gateway (hermes gateway stop) and retry." + ) + logger.error("state.db repair skipped: %s", report["error"]) + else: + # Probe the journal mode BEFORE surgery: a rebuilt file comes + # back in the default (delete) mode and nothing else records + # the flip (see _restore_journal_mode_after_repair). The probe + # may fail on a damaged file; then database.journal_mode is + # the restore target. + before_mode = _probe_journal_mode_for_repair(db_path) + result = _repair_state_db_schema_locked( + db_path, backup=backup, report=report + ) + if result.get("repaired"): + result["journal_mode_before"] = before_mode + _restore_journal_mode_after_repair(db_path, before_mode) + # Environmental aborts happen before a strategy mutates the + # snapshot; they are retriable, not proof a strategy was exhausted. + # Keep that private marker out of the public report. The ledger + # update stays under the same cross-process lock as surgery so two + # repairers cannot lose each other's updates; a queued loser must + # not record at all. + attempted = bool(result.pop("_repair_attempted", False)) + if attempted or result.get("repaired"): + _record_repair_outcome( + db_path, repaired=bool(result.get("repaired")) + ) + return result + + +def _probe_journal_mode_for_repair(db_path: Path) -> Optional[str]: + """Best-effort journal-mode probe for a (possibly malformed) DB file. + + Returns ``wal``/``delete``, or ``None`` when the file cannot be opened or + probed (malformed header, concurrent opener's locks — both expected on + the repair path); callers then fall back to ``database.journal_mode``. + """ + from hermes_state import _on_disk_journal_mode + try: + conn = _connect_repair_durable(db_path) + try: + return _on_disk_journal_mode(conn) + finally: + conn.close() + except (sqlite3.Error, OSError): + return None + + +def _restore_journal_mode_after_repair(db_path: Path, before_mode: Optional[str]) -> None: + """Re-apply the journal mode after schema surgery. + + A rebuilt SQLite file comes back in the default (delete) mode; without + this, a corruption event silently moves a WAL store out of WAL (the + open-time WAL-reset gate never sees a flip made inside repair). Routed + through :func:`apply_wal_with_fallback`, not a direct pragma, so it + inherits the vulnerable-SQLite WAL-reset gate (a rebuilt file IS a new + database; on a vulnerable runtime the gate deliberately keeps DELETE, and + "could not reach WAL" is expected there), the macOS-NFS silent-refusal + handling, and the WAL companions (size limit, checkpoint barrier, + synchronous=FULL). ``before_mode`` (None if unprobeable) is only for the + log comparison; the target comes from ``database.journal_mode``. + Best-effort: the repair already succeeded, so failures log at WARNING. + """ + from hermes_state import apply_wal_with_fallback + try: + conn = _connect_repair_durable(db_path) + try: + after = apply_wal_with_fallback(conn, db_label=db_path.name) + finally: + conn.close() + if before_mode and after != before_mode: + logger.warning( + "state.db repair changed journal_mode %r -> %r " + "(pre-surgery probe %r; restore resolved through " + "apply_wal_with_fallback per database.journal_mode and the " + "WAL-reset gate)", + before_mode, after, before_mode, + ) + except (sqlite3.Error, OSError) as exc: + logger.warning( + "state.db repair at %s: post-surgery journal-mode restore " + "failed (%s); verify with PRAGMA journal_mode on the next open", + db_path, exc, + ) + + +def _repair_state_db_schema_locked( + db_path: Path, *, backup: bool, report: Dict[str, Any] +) -> Dict[str, Any]: + """Repair strategies for :func:`repair_state_db_schema`. + + Caller must hold the cross-process repair lock for *db_path*. Strategies + run on a SCRATCH COPY; the result is copied back through SQLite's + transactional backup API only once proven to open cleanly, so a failed + repair cannot modify or lose committed canonical data. (A WAL checkpoint + of already-committed frames on guard release is not a repair mutation.) + + WHY not in place: Strategy 2 ends in ``VACUUM``, which rebuilds the file + from the schema SQLite can still parse. When the damage IS in the schema + b-tree — the ``malformed database schema ()`` class handled here — every + table hanging off the unreadable part is silently dropped, the probe then + correctly reports STILL malformed, and repair returned ``repaired=False`` + having destroyed what it was asked to save. The forensic backup does not + close this (nothing reads it back). Not mutating the original is the + property that holds without a human in the loop. + """ + from hermes_state import _backup_db_file, _copy_database_snapshot, _db_opens_cleanly, _repair_scratch_space_error, _run_repair_strategies, _unlink_db_triple + scratch = db_path.with_name(f"{db_path.name}.repair-scratch") + cleanup_error = _unlink_db_triple(scratch) + if cleanup_error is not None: + report["error"] = ( + "could not remove a stale repair snapshot before probing state.db: " + f"{cleanup_error}" + ) + logger.error("state.db repair aborted: %s", report["error"]) + return report + + # Re-probe under the lock: a process we queued behind may have just + # repaired the file; redoing surgery would undo its work (the + # repair/re-corrupt cascade this lock exists to break). + if _db_opens_cleanly(db_path) is None: + report["repaired"] = True + report["strategy"] = "already_healthy" + return report + + if backup: + bpath, backup_error = _backup_db_file(db_path) + report["backup_path"] = str(bpath) if bpath else None + if bpath is None: + # HARD STOP: the forensic image is still required when corruption + # defeats every strategy, even though strategies run on a snapshot. + report["error"] = ( + "pre-repair backup refused; aborting schema repair to avoid " + f"mutating the only copy of the damaged DB: {backup_error}" + ) + logger.error("state.db repair aborted: %s", report["error"]) + return report + + # The forensic copy deliberately precedes this guard: its raw-copy safety + # checks inspect real live holders and would be poisoned by our exclusive + # connection. Everything affecting the repair image or live promotion + # happens only after writer exclusion is held. + with _exclusive_repair_db_guard(db_path) as (live_guard, guard_error): + if live_guard is None: + report["error"] = ( + "could not acquire exclusive state.db repair ownership; " + "skipped schema surgery to avoid overwriting a concurrent " + f"writer. Stop the gateway and retry: {guard_error}" + ) + if guard_error is not None and _repair_failure_consumes_attempt( + guard_error + ): + report["_repair_attempted"] = True + logger.error("state.db repair skipped: %s", report["error"]) + return report + + space_error = _repair_scratch_space_error(db_path) + if space_error is not None: + report["error"] = space_error + logger.error("state.db repair aborted: %s", report["error"]) + return report + + try: + # Reuse live_guard rather than a second source connection: the + # guard owns the exclusion, and a second connection could be + # blocked by our own EXCLUSIVE lock on some SQLite builds. + _copy_database_snapshot( + db_path, scratch, source_connection=live_guard + ) + except (OSError, sqlite3.Error, TimeoutError) as exc: + report["error"] = ( + f"could not stage a complete SQLite repair snapshot of {db_path}: {exc}" + ) + if _repair_failure_consumes_attempt(exc): + report["_repair_attempted"] = True + logger.error("state.db repair aborted: %s", report["error"]) + _unlink_db_triple(scratch) + return report + + try: + # Private marker consumed by the outer wrapper: a strategy failure + # consumes the persistent budget, but a later promotion failure is + # classified separately (disk/I/O/permission/lock = environmental). + report["_repair_attempted"] = True + _run_repair_strategies(scratch, report) + if report.get("repaired"): + try: + # Do not os.replace the live DB: Windows rejects replacement + # under open handles and POSIX would leave those handles on + # the old inode. The guard that staged the live image + # receives the promotion, keeping writer exclusion throughout. + _copy_database_snapshot( + scratch, + db_path, + destination_connection=live_guard, + ) + except (OSError, sqlite3.Error, TimeoutError) as exc: + report["repaired"] = False + report["strategy"] = None + report["_repair_attempted"] = _repair_failure_consumes_attempt( + exc + ) + report["error"] = ( + "repaired snapshot could not be promoted transactionally: " + f"{exc}" + ) + logger.error("state.db repair promotion failed: %s", exc) + else: + logger.warning( + "state.db repaired via '%s' and promoted transactionally: %s", + report.get("strategy"), + db_path, + ) + if not report.get("repaired"): + # Logged HERE, not in the strategies: they see the scratch copy, + # and the one message a human acts on must not name a path + # that no longer exists by the time they read it. + logger.error( + "state.db schema repair could not recover %s automatically " + "(no committed canonical data was modified or lost; backup: %s); " + "manual restore from backup may be required.", + db_path, + report["backup_path"], + ) + return report + finally: + # Never leave a half-repaired file beside the DB for a later probe + # or human to mistake for the real thing. + cleanup_error = _unlink_db_triple(scratch) + if cleanup_error is not None: + logger.warning( + "Could not remove state.db repair snapshot after repair: %s", + cleanup_error, + ) + + +def _unlink_db_triple(path: Path) -> Optional[str]: + """Remove *path* and every SQLite sidecar; return any cleanup failure.""" + from hermes_state import _IS_WINDOWS + failures: List[str] = [] + for suffix in ("", *_DB_SIDECAR_SUFFIXES): + victim = path if not suffix else path.with_name(path.name + suffix) + for attempt in range(10): + try: + victim.unlink() + break + except FileNotFoundError: + break + except PermissionError as exc: + # Windows may retain a just-closed SQLite handle for a few + # scheduler ticks; bounded retry. A later open still fails + # safely if the handle truly remains live. + if _IS_WINDOWS and attempt < 9: + time.sleep(0.05) + continue + failures.append(f"{victim}: {exc}") + break + except OSError as exc: + failures.append(f"{victim}: {exc}") + break + return "; ".join(failures) or None + + +def _run_repair_strategies( + db_path: Path, report: Dict[str, Any] +) -> Dict[str, Any]: + """Escalating repair attempts, applied to *db_path* IN PLACE. + + Every strategy mutates its argument, so this is only ever called by + :func:`_repair_state_db_schema_locked` on a scratch copy nothing else + holds open — never on the user's database. + """ + from hermes_state import _db_opens_cleanly, load_fts5_cjk_extension + # ── Strategy 0: rebuild FTS indexes in place (FTS write-corruption) ── + # FTS5 'rebuild' rewrites the index from the content table: the + # least-destructive fix for an index that rejects writes while reads work. + try: + conn = _connect_repair_durable(db_path) + try: + # The cjk index can only be rebuilt with its tokenizer loaded + # (best-effort; a tokenizer-less host skips it below). + load_fts5_cjk_extension(conn) + for table_name in ( + "messages_fts", "messages_fts_trigram", "messages_fts_cjk" + ): + try: + conn.execute( + f"INSERT INTO {table_name}({table_name}) VALUES('rebuild')" + ) + except sqlite3.OperationalError: + # Table absent (FTS disabled / trigram off / cjk not present + # or tokenizer unavailable). + continue + finally: + conn.close() + if _db_opens_cleanly(db_path) is None: + report["repaired"] = True + report["strategy"] = "rebuild_fts" + logger.warning( + "state.db FTS indexes rebuilt in place (schema preserved): %s", + db_path, + ) + return report + except sqlite3.DatabaseError as exc: + logger.warning("state.db FTS in-place rebuild pass failed: %s", exc) + + # ── Strategy 0.5: rebuild stale B-tree indexes ── + # integrity_check reports "wrong # of entries in index" when a B-tree index + # drifts from its base table; REINDEX rewrites it from the canonical rows. + try: + conn = _connect_repair_durable(db_path) + try: + # REINDEX rewrites every index b-tree; take the barriers now that + # the schema parses, in case the open-time attempt was refused. + _reapply_durability_barriers(conn) + conn.execute("REINDEX") + conn.commit() + finally: + conn.close() + if _db_opens_cleanly(db_path) is None: + report["repaired"] = True + report["strategy"] = "reindex_btree" + logger.warning( + "state.db B-tree indexes rebuilt via REINDEX: %s", db_path + ) + return report + except sqlite3.DatabaseError as exc: + logger.warning("state.db REINDEX pass failed: %s", exc) + + # ── Strategy 1: de-duplicate sqlite_master (keeps FTS index) ── + try: + conn = _connect_repair_durable(db_path) + try: + conn.execute("PRAGMA writable_schema=ON") + dupes = conn.execute( + "SELECT type, name, COUNT(*) AS c, MIN(rowid) AS keep " + "FROM sqlite_master GROUP BY type, name HAVING c > 1" + ).fetchall() + for type_, name, _count, keep in dupes: + conn.execute( + "DELETE FROM sqlite_master " + "WHERE type IS ? AND name IS ? AND rowid <> ?", + (type_, name, keep), + ) + if dupes: + _bump_schema_cookie(conn) + conn.execute("PRAGMA writable_schema=OFF") + conn.commit() + finally: + conn.close() + if _db_opens_cleanly(db_path) is None: + report["repaired"] = True + report["strategy"] = "dedup_schema" + logger.warning( + "state.db schema repaired by de-duplicating sqlite_master " + "(FTS index preserved): %s", db_path + ) + return report + except sqlite3.DatabaseError as exc: + logger.warning("state.db dedup repair pass failed: %s", exc) + + # ── Strategy 2: drop all FTS schema, VACUUM, rebuild on next open ── + # The destructive one, and why this path runs on a scratch copy: on a + # damaged schema b-tree VACUUM silently drops every table hanging off the + # unreadable part (see _repair_state_db_schema_locked). + try: + conn = _connect_repair_durable(db_path) + try: + conn.execute("PRAGMA writable_schema=ON") + conn.execute("DELETE FROM sqlite_master WHERE name LIKE 'messages_fts%'") + _bump_schema_cookie(conn) + conn.execute("PRAGMA writable_schema=OFF") + conn.commit() + # The schema parses now, so the barriers can finally stick — and + # VACUUM rewrites the entire file, the worst operation to lose halfway. + _reapply_durability_barriers(conn) + conn.execute("VACUUM") + finally: + conn.close() + reason = _db_opens_cleanly(db_path) + if reason is None: + report["repaired"] = True + report["strategy"] = "drop_fts_rebuild" + logger.warning( + "state.db schema repaired by dropping FTS schema; indexes " + "will rebuild from messages on next open: %s", db_path + ) + return report + report["error"] = reason + except sqlite3.DatabaseError as exc: + report["error"] = str(exc) + + # The "could not recover" log lives in the caller: it must name the user's + # database, not the scratch copy. + return report diff --git a/hermes_state_schema.py b/hermes_state_schema.py index 01801a4870..0130fd434f 100644 --- a/hermes_state_schema.py +++ b/hermes_state_schema.py @@ -1,20 +1,21 @@ """Schema creation, column reconciliation, and FTS DDL management for SessionDB. -Mixin contract: this is a plain mixin class consumed by -``hermes_state.SessionDB``. It defines no ``__init__`` and no state of its -own; methods access the host's attributes (``self._conn``, ``self.db_path``, -``self._execute_write`` and other SessionDB methods) established by -``SessionDB.__init__``. It must never import hermes_state (cycle) — shared -module-level constants live in hermes_state_common. +Plain mixin consumed by ``hermes_state.SessionDB``: no ``__init__``, no state +of its own; methods use host attributes established by ``SessionDB.__init__``. +Must never import hermes_state (cycle) — shared constants live in +hermes_state_common. """ import datetime +import hashlib import logging import json +import os import sqlite3 +import tempfile import time import uuid -from typing import Dict, Optional, Sequence +from typing import Dict, List, Optional, Sequence from hermes_constants import get_hermes_home @@ -37,36 +38,29 @@ from hermes_state_common import ( fts_rebuild_admission, ) -# Moved methods logged under the "hermes_state" logger before the split; -# keep that logger identity so log filtering/capture behavior is unchanged. +# Keep the pre-split logger identity so log filtering/capture is unchanged. logger = logging.getLogger("hermes_state") _FTS_HOLDER_ESCALATE_ATTEMPTS = 3 _FTS_HOLDER_ESCALATE_SECONDS = 60.0 -# Minimum spacing between in-process retries of a deferred stale-FTS rebuild -# (``retry_deferred_fts_recovery``). The startup open already paid the full -# admission wait once; later retries are non-blocking probes on this cadence -# so a live holder never stalls a long-lived writer. +# In-process retry cadence for a deferred stale-FTS rebuild +# (``retry_deferred_fts_recovery``): startup paid the full admission wait once; +# later retries are non-blocking probes so a live holder never stalls a +# long-lived writer. Each failed retry doubles the spacing up to the cap, so a +# permanent holder costs one deferral warning per hour, not per minute. _FTS_STALE_RETRY_SECONDS = 60.0 -# Each failed retry doubles the spacing up to this cap, so a holder that never -# goes away (a second long-lived writer) costs one deferral warning per hour, -# not one per minute. A successful rebuild clears the stale state entirely. _FTS_STALE_RETRY_MAX_SECONDS = 3600.0 -# Cache for schema_read_probe_statements() — parsing SCHEMA_SQL spins up an -# in-memory SQLite database, so derive the statements once per process. +# schema_read_probe_statements() cache — deriving it parses SCHEMA_SQL in an +# in-memory SQLite database, so do it once per process. _READ_PROBE_STATEMENTS: Optional[tuple] = None -# _FTS_TRIGGERS is the full canonical set, but its two halves have different -# availability: the trigram triggers are declared ONLY by FTS_TRIGRAM_SQL / -# LEGACY_FTS_TRIGRAM_SQL, whose CREATE VIRTUAL TABLE needs the trigram -# tokenizer (SQLite >= 3.34). On a build without it, _ensure_fts_schema -# soft-fails that DDL, so those three triggers can never exist and any check -# for "all six are present" is permanently unsatisfiable. Split the set so a -# trigger's absence is only ever measured against the DDL that can create it. -# The two subsets are exhaustive and disjoint by construction (base is the -# complement of trigram); test_fts_trigger_subsets_match_the_ddl pins them -# against the DDL those triggers actually come from. +# The trigram triggers come ONLY from FTS_TRIGRAM_SQL / LEGACY_FTS_TRIGRAM_SQL, +# whose CREATE VIRTUAL TABLE needs the trigram tokenizer (SQLite >= 3.34); +# without it _ensure_fts_schema soft-fails that DDL and "all six present" is +# permanently unsatisfiable. Split the set so a trigger's absence is only +# measured against the DDL that can create it. Exhaustive and disjoint by +# construction; pinned by test_fts_trigger_subsets_match_the_ddl. _FTS_TRIGRAM_TRIGGERS = tuple(n for n in _FTS_TRIGGERS if "_trigram_" in n) _FTS_BASE_TRIGGERS = tuple(n for n in _FTS_TRIGGERS if n not in _FTS_TRIGRAM_TRIGGERS) @@ -76,25 +70,17 @@ def schema_read_probe_statements() -> tuple: Read-only opens skip ``_reconcile_columns()`` by design (no DDL against another profile's live DB), so a store created before a schema addition - keeps 500ing on read paths until something opens it writable. Callers - that heal on staleness (see ``_open_session_db_at_path`` in - ``hermes_cli/web_server.py``) run these probes right after a read-only - open: any missing table raises "no such table" and any missing column - raises "no such column", both at prepare time. + keeps failing on read paths until something opens it writable. Healing + callers (``_open_session_db_at_path`` in the web server) run these probes + after a read-only open: a missing table/column raises at prepare time. - Derived from SCHEMA_SQL — the same source of truth the writable - reconciler diffs against — so a column added there is covered here - automatically. A hand-maintained probe list went stale within days of - shipping (it never learned ``sessions.last_activity_at``, so the sidebar - served an empty session list after `hermes update` until the user's - first message forced a writable open). - - Each statement is ``LIMIT 0``: column resolution happens at prepare - time, so the probe reads zero rows. Column references are qualified - with the table name — an unqualified double-quoted identifier that - fails to resolve silently degrades to a string literal (SQLite's - double-quoted-string misfeature), which would make the probe pass on - exactly the stale store it exists to catch. + Derived from SCHEMA_SQL so a column added there is covered automatically + (a hand-maintained list went stale within days). Each statement is + ``LIMIT 0`` (resolution at prepare time, zero rows read). Column + references are table-qualified: an unqualified double-quoted identifier + that fails to resolve silently degrades to a string literal (SQLite + misfeature), which would make the probe pass on exactly the stale store + it exists to catch. """ global _READ_PROBE_STATEMENTS if _READ_PROBE_STATEMENTS is None: @@ -120,15 +106,11 @@ class SessionSchemaMixin: def _dedupe_legacy_system_prompts(self, cursor: sqlite3.Cursor) -> None: """Move inline prompt snapshots into the shared content-addressed table. - Contention-safe by design: a ``database is locked`` (or any other - ``OperationalError``) mid-loop returns instead of raising. Partial - migration is safe — the legacy ``system_prompt`` column is kept as a - read fallback for unmigrated rows, and the next schema init picks up - the remainder. Letting the error propagate aborted schema init - entirely, left the version below 25, and made every subsequent - ``SessionDB.__init__`` re-enter this migration against the same - contended DB (enterprise field report, 2026-08-14: gateway watchdog - crash loop). + Contention-safe: any ``OperationalError`` mid-loop returns instead of + raising. Partial migration is safe — the legacy ``system_prompt`` + column stays a read fallback and the next schema init resumes. + Propagating the error aborted schema init, left the version below + 25, and re-entered this migration on every open (gateway crash loop). """ try: rows = cursor.execute( @@ -138,9 +120,7 @@ class SessionSchemaMixin: except sqlite3.OperationalError: return - for row in rows: - session_id = row["id"] if isinstance(row, sqlite3.Row) else row[0] - prompt = row["system_prompt"] if isinstance(row, sqlite3.Row) else row[1] + for session_id, prompt in rows: try: prompt_hash = self._store_system_prompt(cursor, prompt) cursor.execute( @@ -182,15 +162,10 @@ class SessionSchemaMixin: cursor: sqlite3.Cursor, names: Sequence[str] = _FTS_TRIGGERS, ) -> int: - """Count how many of *names* currently exist as triggers. - - Defaults to the full canonical set so existing callers are unchanged; - callers that need to know whether one HALF of the set is intact pass - _FTS_BASE_TRIGGERS or _FTS_TRIGRAM_TRIGGERS. - """ + """Count how many of *names* currently exist as triggers (pass + _FTS_BASE_TRIGGERS / _FTS_TRIGRAM_TRIGGERS to check one half).""" if not names: - # "name IN ()" is a syntax error in SQLite, and nothing can be - # missing from an empty set anyway. + # "name IN ()" is a SQLite syntax error. return 0 placeholders = ",".join("?" for _ in names) row = cursor.execute( @@ -198,8 +173,7 @@ class SessionSchemaMixin: f"WHERE type = 'trigger' AND name IN ({placeholders})", tuple(names), ).fetchone() - return int(row[0] if not isinstance(row, sqlite3.Row) else row[0]) - + return int(row[0]) @staticmethod def _fts_update_trigger_needs_narrowing(sql: Optional[str]) -> bool: @@ -217,26 +191,20 @@ class SessionSchemaMixin: def _migrate_broad_fts_update_triggers(self, cursor: sqlite3.Cursor) -> int: """Replace broad AFTER UPDATE FTS triggers with AFTER UPDATE OF variants. - ``CREATE TRIGGER IF NOT EXISTS`` will not replace an existing broad - trigger, so installs that already created ``AFTER UPDATE ON messages`` - would keep firing on every messages row touch (status/compaction - writes included). Inspect ``sqlite_master``, drop any still-broad - UPDATE triggers, and re-apply the current DDL constants. - - No FTS rebuild: content correctness was already gated by WHEN clauses - on modern installs; OF only skips unnecessary trigger evaluation. - - Returns the number of triggers dropped (0 when already converged). + ``CREATE TRIGGER IF NOT EXISTS`` never replaces an existing broad + trigger, so it would keep firing on every messages row touch. Drop + still-broad UPDATE triggers and re-apply the current DDL. No FTS + rebuild: correctness was already gated by WHEN clauses; OF only skips + unnecessary trigger evaluation. Returns the number dropped. """ - # CJK is a v23-only surface. Decide the layout before selecting - # destructive candidates so the legacy branch never drops a trigger - # it does not recreate. + # CJK is v23-only. Decide the layout before selecting destructive + # candidates so the legacy branch never drops a trigger it won't recreate. legacy_layout = self._db_has_legacy_inline_fts(cursor) update_names = ( "messages_fts_update", "messages_fts_trigram_update", ) - if not legacy_layout and hasattr(self, "_ensure_fts_cjk_schema"): + if not legacy_layout: update_names += ("messages_fts_cjk_update",) placeholders = ", ".join("?" for _ in update_names) rows = cursor.execute( @@ -244,12 +212,10 @@ class SessionSchemaMixin: f"WHERE type = 'trigger' AND name IN ({placeholders})", update_names, ).fetchall() - to_drop = [] - for row in rows: - name = row[0] if not isinstance(row, sqlite3.Row) else row["name"] - sql = row[1] if not isinstance(row, sqlite3.Row) else row["sql"] - if self._fts_update_trigger_needs_narrowing(sql): - to_drop.append(name) + to_drop = [ + name for name, sql in rows + if self._fts_update_trigger_needs_narrowing(sql) + ] if not to_drop: return 0 @@ -270,12 +236,11 @@ class SessionSchemaMixin: self._ensure_fts_schema( cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL ) - # CJK triggers live on the host SessionDB; only recreate one that - # this migration actually dropped. ``_ensure_fts_cjk_schema`` is - # documented never-raises and soft-fails OperationalError by - # clearing availability — raise-path handling alone is not - # enough. After ensure, require a narrowed CJK UPDATE trigger or - # durable quarantine (stale breadcrumb + unavailable). + # Only recreate the CJK trigger this migration actually dropped. + # ``_ensure_fts_cjk_schema`` soft-fails OperationalError by clearing + # availability (never raises), so after ensure require a narrowed + # CJK UPDATE trigger or durable quarantine (stale breadcrumb + + # unavailable). if "messages_fts_cjk_update" in to_drop: try: self._ensure_fts_cjk_schema(cursor) @@ -306,20 +271,14 @@ class SessionSchemaMixin: "WHERE type = 'trigger' AND name = ?", ("messages_fts_cjk_update",), ).fetchone() - if not row: - return False - sql = row[0] if not isinstance(row, sqlite3.Row) else row["sql"] - return not self._fts_update_trigger_needs_narrowing(sql) + return bool(row) and not self._fts_update_trigger_needs_narrowing(row[0]) def _quarantine_cjk_after_update_of_migration( self, cursor: sqlite3.Cursor ) -> None: - """Fail-closed after dropping CJK UPDATE during OF migration. - - Clears availability, persists ``fts_cjk_stale``, and drops any - residual broad/partial CJK UPDATE trigger so a later open cannot - ``CREATE TRIGGER IF NOT EXISTS`` a gap without rebuild. - """ + """Fail closed after dropping the CJK UPDATE trigger mid-migration: + clear availability, persist ``fts_cjk_stale``, drop any residual CJK + UPDATE trigger so a later open cannot IF-NOT-EXISTS over a gap.""" self._fts_cjk_available = False try: self.set_meta(FTS_CJK_STALE_KEY, "1", cursor=cursor) @@ -343,18 +302,15 @@ class SessionSchemaMixin: *, include_trigram: bool = True, ) -> None: - # Both FTS tables are external-content (v23+): the special 'rebuild' - # command wipes the inverted index and repopulates it from the - # content source (messages for the standard index, the tool-row- - # excluding messages_fts_trigram_src view for the trigram index). + # v23+ external-content tables: 'rebuild' repopulates the inverted + # index from the content source (messages / messages_fts_trigram_src). cursor.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')") if include_trigram: cursor.execute( "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('rebuild')" ) - # 'rebuild' indexes EVERY row, so any deferred-backfill markers are - # now satisfied — clear them, otherwise the background worker would - # re-insert rows the rebuild already covered (duplicate entries). + # 'rebuild' indexes EVERY row: clear deferred-backfill markers or the + # worker would re-insert rows already covered (duplicates). cursor.execute( "DELETE FROM state_meta WHERE key IN " "('fts_rebuild_high_water', 'fts_rebuild_progress')" @@ -366,49 +322,37 @@ class SessionSchemaMixin: *, include_trigram: bool = True, ) -> None: - """Rebuild the LEGACY inline FTS indexes (pre-v23) from messages. + """Rebuild the LEGACY inline (pre-v23) FTS indexes from messages. - Used only to repair a legacy DB whose triggers degraded under an - earlier no-FTS5 runtime. Inline tables have no external-content - 'rebuild' source, so we DELETE + reinsert the concatenated content - the legacy triggers produced. Never touches the v23 shape. + Inline tables have no external-content 'rebuild' source, so DELETE + + reinsert the concatenated content the legacy triggers produced. + Never touches the v23 shape. """ - cursor.execute("DELETE FROM messages_fts") - cursor.execute( - "INSERT INTO messages_fts(rowid, content) " - "SELECT id, " - "COALESCE(content, '') || ' ' || " - "COALESCE(tool_name, '') || ' ' || " - "COALESCE(tool_calls, '') " - "FROM messages" - ) - if not include_trigram: - return - cursor.execute("DELETE FROM messages_fts_trigram") - cursor.execute( - "INSERT INTO messages_fts_trigram(rowid, content) " - "SELECT id, " - "COALESCE(content, '') || ' ' || " - "COALESCE(tool_name, '') || ' ' || " - "COALESCE(tool_calls, '') " - "FROM messages" - ) + tables = ("messages_fts", "messages_fts_trigram") if include_trigram else ("messages_fts",) + for tbl in tables: + cursor.execute(f"DELETE FROM {tbl}") + cursor.execute( + f"INSERT INTO {tbl}(rowid, content) " + "SELECT id, " + "COALESCE(content, '') || ' ' || " + "COALESCE(tool_name, '') || ' ' || " + "COALESCE(tool_calls, '') " + "FROM messages" + ) def _fts_table_probe(self, cursor: sqlite3.Cursor, table_name: str) -> Optional[bool]: try: cursor.execute(f"SELECT * FROM {table_name} LIMIT 0") return True except (sqlite3.OperationalError, UnicodeDecodeError) as exc: - # UnicodeDecodeError can occur when FTS shadow tables or content - # columns hold invalid UTF-8 bytes. On some Python/SQLite builds - # it surfaces as a bare UnicodeDecodeError (ValueError subclass, - # not sqlite3.Error); on others as OperationalError("Could not - # decode to UTF-8 column ..."). Catch both so the probe never - # kills the connection or raises to writable-init/recovery flows. + # Invalid UTF-8 in FTS content surfaces as a bare UnicodeDecodeError + # (not sqlite3.Error) on some builds and as OperationalError("Could + # not decode to UTF-8 ...") on others; catch both so the probe never + # raises into writable-init/recovery flows. if isinstance(exc, sqlite3.OperationalError): if self._is_fts5_unavailable_error(exc): - # Only disable FTS entirely when the whole module is missing. - # A missing trigram tokenizer only affects trigram searches. + # A missing trigram tokenizer only affects trigram search; + # only a missing FTS5 module disables FTS entirely. if self._is_trigram_unavailable_error(exc): self._warn_trigram_unavailable(exc) else: @@ -416,13 +360,11 @@ class SessionSchemaMixin: return None if "no such table" in str(exc).lower(): return False - # Re-raise any other OperationalError (e.g. malformed schema, - # corrupt vtable that isn't a decode error). + # Anything else (malformed schema, corrupt vtable) re-raises. if "decode to utf-8" not in str(exc).lower(): raise - # Swallow: decode error means the index is degraded but the - # store remains accessible. Writable init / recovery will - # schedule a rebuild or degrade to LIKE. + # Decode error: index degraded, store accessible; writable init / + # recovery schedules a rebuild or degrades to LIKE. logger.warning( "%s probe encountered invalid UTF-8 in FTS content; " "search may return incomplete results until FTS is rebuilt: %s", @@ -449,8 +391,7 @@ class SessionSchemaMixin: (FTS_REBUILD_DEFERRAL_KEY,), ).fetchone() if row: - raw = row["value"] if isinstance(row, sqlite3.Row) else row[0] - parsed = json.loads(raw) + parsed = json.loads(row[0]) if isinstance(parsed, dict): record = parsed except (sqlite3.Error, TypeError, ValueError, json.JSONDecodeError): @@ -513,13 +454,10 @@ class SessionSchemaMixin: attempts, ) return False - # Full structural rebuild: admit through the single cross-process - # authority (fail closed). Losing the race means another process is - # already performing this exact recovery; the stale breadcrumb stays - # set, so this process simply keeps FTS detached and retries later. - with fts_rebuild_admission( - getattr(self, "db_path", None), timeout_seconds=timeout_seconds - ) as admitted: + # Full structural rebuild: admit through the cross-process authority + # (fail closed). Losing the race means another process is doing this + # exact recovery; the breadcrumb stays set and we retry later. + with fts_rebuild_admission(self.db_path, timeout_seconds=timeout_seconds) as admitted: if not admitted: logger.warning( "Deferred stale state.db FTS rebuild: another process " @@ -532,24 +470,18 @@ class SessionSchemaMixin: def retry_deferred_fts_recovery(self) -> bool: """Retry a deferred stale-FTS rebuild on this open SessionDB. - ``_recover_stale_fts`` runs at open and fails closed when foreign - holders or the rebuild lock are busy, leaving ``_fts_stale`` set and - search on the LIKE fallback. Live write/search paths must never start - a full rebuild (#97940), so on a short-lived CLI that deferral is - cleared by the next process open — but a gateway opens state.db - once and stays up for days, so "next open" never came (#100108). - This is the in-process retry: bounded backoff from - ``_FTS_STALE_RETRY_SECONDS`` doubling to ``_FTS_STALE_RETRY_MAX_SECONDS``, - non-blocking admission (``timeout=0``) so a live holder is skipped and - tried again later, no new thread — the caller is an existing periodic - tick (gateway housekeeping). + ``_recover_stale_fts`` fails closed at open when holders or the rebuild + lock are busy, leaving ``_fts_stale`` set and search on LIKE. Live + write/search paths must never start a full rebuild, and a gateway + opens state.db once for days, so "next open" never comes. This is the + in-process retry from the gateway housekeeping tick: bounded backoff + (``_FTS_STALE_RETRY_SECONDS`` doubling to the max), non-blocking + admission (``timeout=0``), no new thread. Returns True only when the index was rebuilt and sync triggers restored. Never raises. """ - if not getattr(self, "_fts_stale", False): - return False - if getattr(self, "read_only", False) or getattr(self, "_conn", None) is None: + if not self._fts_stale or self.read_only or self._conn is None: return False now = time.monotonic() if now < getattr(self, "_fts_stale_retry_after", 0.0): @@ -596,8 +528,8 @@ class SessionSchemaMixin: try: trigram_status = self._fts_table_probe(cursor, "messages_fts_trigram") except (sqlite3.DatabaseError, UnicodeDecodeError): - # A corrupt vtable may fail even a LIMIT 0 probe. It still needs - # to be included in the drop-and-recreate recovery below. + # A corrupt vtable may fail even a LIMIT 0 probe; it must still be + # included in the drop-and-recreate below. trigram_status = True include_trigram = trigram_status is True @@ -665,8 +597,8 @@ class SessionSchemaMixin: self._conn.rollback() except sqlite3.Error: pass - # Stale indexes must remain detached even on SQLite builds whose - # DDL transaction behavior differs. + # Stale indexes must stay detached even on SQLite builds whose DDL + # transaction behavior differs. self._drop_all_fts_triggers(cursor) self._conn.commit() logger.error( @@ -687,34 +619,22 @@ class SessionSchemaMixin: @staticmethod def _parse_schema_columns(schema_sql: str) -> Dict[str, Dict[str, str]]: - """Extract expected columns per table from SCHEMA_SQL. + """Expected columns per table, parsed from SCHEMA_SQL. - Uses an in-memory SQLite database to parse the SQL — SQLite itself - handles all syntax (DEFAULT expressions with commas, inline - REFERENCES, CHECK constraints, etc.) so there are zero regex - edge cases. The in-memory DB is opened, the schema DDL is - executed, and PRAGMA table_info extracts the column metadata. - - Adding a column to SCHEMA_SQL is all that's needed; the - reconciliation loop picks it up automatically. - - The parse result is memoized on disk keyed by a hash of the DDL: - executing SCHEMA_SQL (FTS5 virtual tables included) in the scratch - DB costs ~85ms on every startup, but the output is a pure function - of the DDL text, which only changes when the shipped code changes. - Reconciliation itself (diffing the LIVE database) still runs every - startup — only the reference-side parse is cached. A corrupt or - stale cache degrades to recomputation. + Executes the DDL in an in-memory SQLite database and reads PRAGMA + table_info, so SQLite handles every syntax edge case (no regex). + The result is memoized on disk keyed by a hash of the DDL (~85ms per + startup otherwise; a pure function of the DDL text). Only the + reference-side parse is cached — diffing the LIVE database still runs + every startup. A corrupt or stale cache degrades to recomputation. """ - import hashlib as _hashlib - import json as _json - cache_path = None - schema_hash = _hashlib.sha256(schema_sql.encode("utf-8")).hexdigest() + schema_hash = hashlib.sha256(schema_sql.encode("utf-8")).hexdigest() try: - from hermes_constants import get_hermes_home - cache_path = get_hermes_home() / "cache" / "schema_columns.json" - blob = _json.loads(cache_path.read_text(encoding="utf-8")) + # Late import: resolves a test-patched hermes_constants.get_hermes_home. + from hermes_constants import get_hermes_home as _home + cache_path = _home() / "cache" / "schema_columns.json" + blob = json.loads(cache_path.read_text(encoding="utf-8")) if ( isinstance(blob, dict) and blob.get("schema_hash") == schema_hash @@ -739,15 +659,9 @@ class SessionSchemaMixin: "WHERE type='table' AND name NOT LIKE 'sqlite_%'" ).fetchall(): cols: Dict[str, str] = {} - for row in ref.execute( + for _cid, col_name, col_type, notnull, default, pk in ref.execute( f'PRAGMA table_info("{tbl}")' ).fetchall(): - # row: (cid, name, type, notnull, dflt_value, pk) - col_name = row[1] - col_type = row[2] or "" - notnull = row[3] - default = row[4] - pk = row[5] # Reconstruct the type expression for ALTER TABLE ADD COLUMN parts = [col_type] if col_type else [] if notnull and not pk: @@ -761,48 +675,33 @@ class SessionSchemaMixin: if cache_path is not None: try: - import os as _os - import tempfile as _tempfile cache_path.parent.mkdir(parents=True, exist_ok=True) - fd, tmp = _tempfile.mkstemp( + fd, tmp = tempfile.mkstemp( dir=str(cache_path.parent), prefix=".schema_columns." ) - with _os.fdopen(fd, "w", encoding="utf-8") as fh: - _json.dump( - {"schema_hash": schema_hash, "tables": table_columns}, fh - ) - _os.replace(tmp, cache_path) + with os.fdopen(fd, "w", encoding="utf-8") as fh: + json.dump({"schema_hash": schema_hash, "tables": table_columns}, fh) + os.replace(tmp, cache_path) except Exception: pass # cache write is best-effort return table_columns def _reconcile_columns(self, cursor: sqlite3.Cursor) -> None: - """Ensure live tables have every column declared in SCHEMA_SQL. + """ADD every SCHEMA_SQL column missing from the live tables. - Follows the Beets/sqlite-utils pattern: the CREATE TABLE definition - in SCHEMA_SQL is the single source of truth for the desired schema. - On every startup this method diffs the live columns (via PRAGMA - table_info) against the declared columns, and ADDs any that are - missing. - - This makes column additions a declarative operation — just add - the column to SCHEMA_SQL and it appears on the next startup. - Version-gated migration blocks are no longer needed for ADD COLUMN. + Beets/sqlite-utils pattern: SCHEMA_SQL is the single source of truth; + column additions are declarative and need no version-gated migration. """ expected = self._parse_schema_columns(SCHEMA_SQL) for table_name, declared_cols in expected.items(): - # Get current columns from the live table try: rows = cursor.execute( f'PRAGMA table_info("{table_name}")' ).fetchall() except sqlite3.OperationalError: continue # Table doesn't exist yet (shouldn't happen after executescript) - live_cols = set() - for row in rows: - # PRAGMA table_info returns (cid, name, type, notnull, dflt_value, pk) - name = row[1] if isinstance(row, (tuple, list)) else row["name"] - live_cols.add(name) + # PRAGMA table_info rows: (cid, name, type, notnull, dflt_value, pk) + live_cols = {row[1] for row in rows} for col_name, col_type in declared_cols.items(): if col_name not in live_cols: @@ -814,78 +713,49 @@ class SessionSchemaMixin: except sqlite3.OperationalError as exc: message = str(exc).lower() if "duplicate column" in message: - # Expected: a sibling process won the race to ADD - # this column between our PRAGMA diff and the - # ALTER. The store ends up correct either way. + # A sibling process won the ADD race; store is correct. logger.debug( "reconcile %s.%s: %s", table_name, col_name, exc, ) continue if "locked" in message or "busy" in message: - # Lock contention (e.g. an orphaned sibling - # backend holding the write lock, #79531). This - # used to be swallowed at DEBUG, leaving the - # store half-reconciled: startup "succeeded" and - # every session-list read then failed with - # "no such column" until an unrelated writable - # open. Re-raise instead so the open-time lock - # patience in _connect_and_init_with_lock_patience - # retries the WHOLE init (executescript is - # idempotent CREATE IF NOT EXISTS) with jittered - # backoff rather than serving a stale schema. + # Lock contention: swallowing it left the store + # half-reconciled ("no such column" on every read). + # Re-raise so _connect_and_init_with_lock_patience + # retries the WHOLE init (idempotent) with backoff. raise - # Anything else ("Cannot add a NOT NULL column with - # default value NULL", ...) is a schema mistake that - # permanently strands the store behind SCHEMA_SQL — - # be loud, don't bury it at DEBUG. + # Anything else is a schema mistake that permanently + # strands the store behind SCHEMA_SQL — be loud. logger.warning( "reconcile %s.%s failed; store remains behind " "SCHEMA_SQL: %s", table_name, col_name, exc, ) + @staticmethod + def _live_pk_columns(cursor: sqlite3.Cursor, table: str) -> Optional[List[str]]: + """PRIMARY KEY column names of *table* in key order; None when the + table is missing or has no columns (SCHEMA_SQL creates it correctly).""" + try: + rows = cursor.execute(f'PRAGMA table_info("{table}")').fetchall() + except sqlite3.OperationalError: + return None + if not rows: + return None + # row: (cid, name, type, notnull, dflt_value, pk) + return [r[1] for r in sorted((r for r in rows if r[5]), key=lambda r: r[5])] + def _heal_gateway_routing_pk(self, cursor: sqlite3.Cursor) -> None: """Rebuild ``gateway_routing`` when its PRIMARY KEY predates scoping. - Early builds of the routing-index migration (#59203) created the - table with ``session_key TEXT PRIMARY KEY`` and no ``scope`` column. - ``_reconcile_columns()`` ADDs the missing ``scope`` column on those - databases, but SQLite cannot ALTER a primary key, so the shipped - composite ``PRIMARY KEY (scope, session_key)`` never lands. On such - tables every write path is broken: - - * ``save_gateway_routing_entry`` fails with "ON CONFLICT clause does - not match any PRIMARY KEY or UNIQUE constraint" (its upsert targets - the composite key), and - * ``replace_gateway_routing_entries`` fails with "UNIQUE constraint - failed: gateway_routing.session_key" whenever the same session_key - exists under a different scope — the exact isolation the composite - key exists to provide. - - Each failed save logs a warning and falls back to sessions.json, - so a legacy-shaped table produces endless per-save warning spam. - Rebuild it once, preserving rows. On a session_key collision across - scopes (possible while the PK was wrong) the newest row wins. + Early builds used ``session_key TEXT PRIMARY KEY``; the reconciler ADDs + ``scope`` but SQLite cannot ALTER a PK, so the composite key never + lands and every routing write fails (ON CONFLICT mismatch / UNIQUE + violation across scopes) with per-save warning spam. Rebuild once, + preserving rows; on a cross-scope session_key collision the newest + row wins. """ - try: - rows = cursor.execute( - 'PRAGMA table_info("gateway_routing")' - ).fetchall() - except sqlite3.OperationalError: - return - if not rows: - return - - def _col(row, idx, name): - return row[idx] if isinstance(row, (tuple, list)) else row[name] - - pk_cols = [ - _col(r, 1, "name") - for r in sorted( - (r for r in rows if _col(r, 5, "pk")), - key=lambda r: _col(r, 5, "pk"), - ) - ] - if pk_cols == ["scope", "session_key"]: + pk_cols = self._live_pk_columns(cursor, "gateway_routing") + if pk_cols is None or pk_cols == ["scope", "session_key"]: return logger.info( @@ -905,9 +775,7 @@ class SessionSchemaMixin: PRIMARY KEY (scope, session_key) )""" ) - # INSERT OR REPLACE + updated_at ordering: if the broken PK ever let - # two scopes race over one session_key, keep the newest row per - # (scope, session_key) pair. + # INSERT OR REPLACE in updated_at order: newest row per key wins. cursor.execute( "INSERT OR REPLACE INTO gateway_routing " "(scope, session_key, entry_json, updated_at) " @@ -919,40 +787,15 @@ class SessionSchemaMixin: def _heal_session_model_usage_pk(self, cursor: sqlite3.Cursor) -> None: """Rebuild ``session_model_usage`` when its PRIMARY KEY lacks ``task``. - Installs whose ``state.db`` reached ``schema_version >= 22`` before - the ``task`` dimension was added carry a 5-column PRIMARY KEY - ``(session_id, model, billing_provider, billing_base_url, - billing_mode)``. ``_reconcile_columns()`` ADDs the ``task`` column - as a bare nullable, but SQLite cannot ALTER a primary key, so the - shipped composite 6-column key never lands. The version-gated v22 - rebuild is unreachable on those installs (``current_version < 22`` - is already false), so every upsert in ``_record_model_usage()`` - fails with "ON CONFLICT clause does not match any PRIMARY KEY or - UNIQUE constraint" — aborting the enclosing write transaction and - silently zeroing all token *and* cost accounting (#73823). - - Idempotent; runs unconditionally on every open, same pattern as - :meth:`_heal_gateway_routing_pk` above. On healthy databases the - PRAGMA check short-circuits and this is a no-op. + Installs already at v22+ when ``task`` landed carry the 5-column PK; + the reconciler ADDs ``task`` as a bare nullable but SQLite cannot + ALTER a PK, and the version-gated v22 rebuild is unreachable there. + Every ``_record_model_usage()`` upsert then fails (ON CONFLICT + mismatch), aborting the write transaction and silently zeroing token + and cost accounting. Idempotent; no-op on healthy databases. """ - try: - rows = cursor.execute( - 'PRAGMA table_info("session_model_usage")' - ).fetchall() - except sqlite3.OperationalError: - return - if not rows: - # Table doesn't exist yet — SCHEMA_SQL creates it correctly. - return - - def _col(row, idx, name): - return row[idx] if isinstance(row, (tuple, list)) else row[name] - - pk_cols = { - _col(r, 1, "name") for r in rows if _col(r, 5, "pk") - } - if "task" in pk_cols: - # task is already in the PK — healthy. + pk_cols = self._live_pk_columns(cursor, "session_model_usage") + if pk_cols is None or "task" in pk_cols: return logger.info( @@ -960,15 +803,10 @@ class SessionSchemaMixin: "rebuilding with composite 6-column key", sorted(pk_cols), ) - # FK-off window: the connection enables PRAGMA foreign_keys=ON - # before _init_schema runs, and session_model_usage.session_id - # REFERENCES sessions(id). INSERT OR IGNORE does NOT suppress - # foreign-key violations (OR IGNORE only covers uniqueness/NOT - # NULL conflicts), so an orphaned usage row — possible after a - # partial prune while accounting was broken — would abort the - # whole rebuild. Disable FK enforcement for the copy and restore - # it afterwards. PRAGMA foreign_keys is a no-op inside a - # transaction, which is fine here: _init_schema runs on an + # FK-off window: INSERT OR IGNORE does NOT suppress foreign-key + # violations, so an orphaned usage row (partial prune while accounting + # was broken) would abort the whole rebuild. PRAGMA foreign_keys is a + # no-op inside a transaction — fine here, _init_schema runs on an # isolation_level=None connection with no transaction open. cursor.execute("PRAGMA foreign_keys=OFF") try: @@ -999,10 +837,8 @@ class SessionSchemaMixin: PRIMARY KEY (session_id, model, billing_provider, billing_base_url, billing_mode, task) )""" ) - # OR IGNORE: while the PK was wrong the reconciler may have left - # ``task`` NULL on old rows; COALESCE to '' can theoretically - # collide with a genuine ''-task row — keep the first, drop the - # duplicate rather than fail the heal. + # OR IGNORE: COALESCE(task, '') on legacy NULL rows can collide + # with a genuine ''-task row — keep the first rather than fail. cursor.execute( """INSERT OR IGNORE INTO session_model_usage ( session_id, model, billing_provider, billing_base_url, @@ -1037,57 +873,34 @@ class SessionSchemaMixin: cursor.execute("PRAGMA foreign_keys=ON") def _init_schema(self): - """Create tables and FTS if they don't exist, reconcile columns. + """Create tables and FTS if missing, reconcile columns, run data migrations. - Schema management follows the declarative reconciliation pattern - (Beets, sqlite-utils): SCHEMA_SQL is the single source of truth. - On existing databases, _reconcile_columns() diffs live columns - against SCHEMA_SQL and ADDs any missing ones. This eliminates - the version-gated migration chain for column additions, making - it impossible for reordered or inserted migrations to skip columns. - - The schema_version table is retained for future data migrations - (transforming existing rows) which cannot be handled declaratively. + SCHEMA_SQL is the single source of truth: column additions are + declarative via _reconcile_columns(), so reordered migrations can + never skip a column. schema_version remains for data migrations + (row transforms) that cannot be expressed declaratively. """ - # Declare a startup-watchdog progress lease before potentially long - # synchronous work: on multi-GB state.db files the reconciliation + - # version-gated data migrations below are legitimately slow and can - # be I/O-bound (near-zero CPU), which the watchdog's CPU fallback - # would misread as a parked deadlock (OOF-298 / PR #89750). - # Single lease is deliberate: this is the one pre-loop phase that can - # legitimately exceed the 300s default deadline (multi-GB DBs), and - # the lease is clamped to _MAX_LEASE_S=900. Honest worst case: a - # genuinely wedged DB init delays supervisor respawn by up to the - # lease duration. Per-chunk renewal would shrink that, but adds - # complexity to the migration loops for a rare failure mode. + # Startup-watchdog progress lease: on multi-GB state.db files the + # reconciliation + data migrations are legitimately slow and I/O-bound + # (near-zero CPU), which the watchdog's CPU fallback would misread as + # a parked deadlock. Single lease is deliberate (clamped to + # _MAX_LEASE_S=900): a genuinely wedged init delays supervisor respawn + # by up to the lease; per-chunk renewal isn't worth the complexity. report_startup_progress(600.0, phase="state_db_init_schema") cursor = self._conn.cursor() cursor.executescript(SCHEMA_SQL) - # ── Declarative column reconciliation ────────────────────────── - # Diff live tables against SCHEMA_SQL and ADD any missing columns. - # This is idempotent and self-healing: even if a version-gated - # migration was skipped (e.g. due to version renumbering), the - # column gets created here. + # Idempotent, self-healing column reconciliation, then the two + # table-shape repairs ADD COLUMN cannot express (PK rebuilds). self._reconcile_columns(cursor) - - # Rebuild gateway_routing if it still carries the pre-scope PRIMARY - # KEY (session_key alone). ADD COLUMN cannot fix a PK, so this is - # the one table-shape repair reconciliation can't express. self._heal_gateway_routing_pk(cursor) - - # Rebuild session_model_usage if its PRIMARY KEY lacks the ``task`` - # column (5-column PK on installs already at v22+ when the column - # landed — the version-gated rebuild is unreachable there, #73823). - # Same PK-rebuild constraint as gateway_routing above. self._heal_session_model_usage_pk(cursor) - # Indexes that reference reconciler-added columns must be created - # AFTER _reconcile_columns runs — declaring them in SCHEMA_SQL - # makes the initial executescript fail on legacy DBs (the index's - # WHERE clause references a column that doesn't exist yet). + # Indexes referencing reconciler-added columns must be created AFTER + # _reconcile_columns — in SCHEMA_SQL the initial executescript would + # fail on legacy DBs (WHERE references a not-yet-existing column). try: cursor.execute( "CREATE INDEX IF NOT EXISTS idx_messages_platform_msg_id " @@ -1097,20 +910,14 @@ class SessionSchemaMixin: except sqlite3.OperationalError as exc: logger.debug("idx_messages_platform_msg_id create skipped: %s", exc) - # Deferred indexes that reference the reconciler-added ``active`` - # column (idx_messages_session_active) — same ordering constraint. + # Same ordering constraint (idx_messages_session_active on ``active``). cursor.executescript(DEFERRED_INDEX_SQL) - # Heal NULL ``active`` rows unconditionally on every startup. - # On real-world DBs the reconciler-added ``active`` column can lack - # its NOT NULL DEFAULT 1 (older reconciler builds reconstructed the - # type without the default — see #51646: PRAGMA shows - # (17,'active','INTEGER',0,None,0) in the wild), so INSERTs that - # omitted the column wrote NULL and the ``WHERE active = 1`` - # transcript loaders hid the whole history. The INSERTs now set - # active=1 explicitly; this idempotent repair un-hides rows written - # before the fix. It was previously gated at ``current_version < - # 12`` which never re-ran for already-v12+ databases. + # Heal NULL ``active`` rows on every startup: older reconciler builds + # added ``active`` without its NOT NULL DEFAULT 1, so INSERTs omitting + # it wrote NULL and ``WHERE active = 1`` loaders hid whole histories. + # Unconditional because a ``current_version < 12`` gate never re-ran + # for already-v12+ databases. try: cursor.execute( "UPDATE messages SET active = 1 WHERE active IS NULL" @@ -1125,28 +932,23 @@ class SessionSchemaMixin: (FTS_STALE_KEY,), ).fetchone() is not None if self._fts_stale: - # A prior process deliberately detached FTS after corruption. - # Keep every FTS writer detached until a full rebuild succeeds. + # A prior process detached FTS after corruption; keep every FTS + # writer detached until a full rebuild succeeds. self._drop_all_fts_triggers(cursor) if not fts5_available: - # Existing FTS triggers can still fire on messages INSERT/UPDATE - # even though the current sqlite runtime cannot read the virtual - # tables they target. Drop only the triggers so core persistence - # continues; if a future runtime has FTS5, _ensure_fts_schema() - # recreates them. + # Existing FTS triggers would still fire on messages writes even + # though this runtime cannot read their targets. Drop only the + # triggers so persistence continues; a future FTS5 runtime's + # _ensure_fts_schema() recreates them. self._drop_fts_triggers(cursor) - # ── Schema version bookkeeping ───────────────────────────────── - # Bump to current so future data migrations (if any) can gate on - # version. No version-gated column additions remain. - cursor.execute("SELECT version FROM schema_version LIMIT 1") - row = cursor.fetchone() + row = cursor.execute("SELECT version FROM schema_version LIMIT 1").fetchone() if row is None: cursor.execute( "INSERT INTO schema_version (version) VALUES (?)", (SCHEMA_VERSION,), ) - # Record store provenance on creation so fresh vs wiped stores are distinguishable (#97568) + # Store provenance so fresh vs wiped stores are distinguishable. now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat() instance_id = str(uuid.uuid4()) cursor.executemany( @@ -1158,27 +960,16 @@ class SessionSchemaMixin: ) else: - current_version = row["version"] if isinstance(row, sqlite3.Row) else row[0] - # Renew the progress lease: the version-gated chain below can - # rewrite whole tables (PK rebuilds, backfills) on large DBs. - # Same deliberate single-lease trade-off as _init_schema: honest - # worst case is up to the lease duration of zombie time on a - # wedged migration, accepted over per-chunk renewal complexity. + current_version = row[0] + # Renew the lease: the version-gated chain can rewrite whole tables + # on large DBs (same single-lease trade-off as above). report_startup_progress(600.0, phase="state_db_data_migrations") - # Data migrations that can't be expressed declaratively (row - # backfills, index changes tied to a specific version step) stay - # in a version-gated chain. Column additions are handled by - # _reconcile_columns() above and no longer need entries here. + # Version-gated chain for data migrations only (row backfills, + # version-specific index changes); column additions never belong here. if current_version < 10 and SCHEMA_VERSION == 10: - # v10: trigram FTS5 table for CJK/substring search. The - # virtual table + triggers are created unconditionally via - # FTS_TRIGRAM_SQL below, but existing rows need a one-time - # backfill into the FTS index. - # - # Only run this when v10 itself is the target schema. Current - # v11+ code drops and rebuilds both FTS tables below, so doing - # the v10-only trigram backfill first only burns startup time - # and WAL space before v11 throws the work away. + # v10: one-time trigram backfill. Only when v10 itself is the + # target: v11+ drops and rebuilds both FTS tables, so the + # backfill would only burn startup time and WAL space. if fts5_available: _fts_trigram_exists = self._fts_table_probe( cursor, "messages_fts_trigram" @@ -1197,20 +988,11 @@ class SessionSchemaMixin: fts_migrations_complete = False else: fts_migrations_complete = False - if current_version < 11 and SCHEMA_VERSION < 23: - # v11 (SUPERSEDED by v23): re-index FTS5 tables to cover - # tool_name + tool_calls in inline mode (#16751). v23 drops - # and rebuilds both FTS tables in external-content form, so - # running the v11 inline backfill first would only burn - # startup time and WAL space before v23 throws the work - # away — and its inline INSERT shape no longer matches the - # current external-content FTS_SQL anyway. Kept only for - # source archaeology; unreachable while SCHEMA_VERSION >= 23. - pass + # (v11 inline FTS re-index was superseded by v23 and removed.) if current_version < 16: # v16: tag delegate subagent rows so pickers stay clean after - # parent deletes that used to orphan them (parent_session_id → NULL). - # The shared predicate excludes user-visible reset children. + # parent deletes orphan them. The shared predicate excludes + # user-visible reset children. try: cursor.execute( "UPDATE sessions SET model_config = json_set(" @@ -1235,28 +1017,17 @@ class SessionSchemaMixin: except sqlite3.OperationalError: pass if current_version < 18: - # v18: gateway metadata consolidation (#9006). Backfill - # display_name / origin_json / expiry_finalized from - # sessions.json so pre-migration gateway sessions are - # discoverable from state.db without the JSON index. + # v18: backfill gateway metadata from sessions.json. Best-effort: + # missing metadata just means consumers fall back to + # sessions.json until the gateway rewrites those rows. try: self._backfill_gateway_metadata_from_sessions_json(cursor) except Exception as exc: - # Backfill is best-effort: sessions.json may be absent, - # corrupted, or partially stale. Missing metadata simply - # means consumers fall back to sessions.json for those - # rows until the gateway rewrites them. logger.debug("v18 gateway metadata backfill skipped: %s", exc) if current_version < 20: - # v20: per-model usage attribution (issue #51607). Going - # forward update_token_counts() records each API call into - # session_model_usage keyed by the live model, but existing - # sessions only have their aggregate totals on the sessions - # row. Seed one usage row per historical session from those - # aggregates so insights reads uniformly from the new table. - # INSERT OR IGNORE keeps it idempotent: if newer code already - # wrote a (session_id, model, provider) row for a session, the - # PK conflict skips the stale aggregate rather than doubling it. + # v20: seed one session_model_usage row per historical session + # from the sessions aggregates. INSERT OR IGNORE: a row newer + # code already wrote wins over the stale aggregate. try: cursor.execute( """INSERT OR IGNORE INTO session_model_usage ( @@ -1292,15 +1063,10 @@ class SessionSchemaMixin: except sqlite3.OperationalError: pass if current_version < 22: - # v22: task-dimension usage attribution (issue #23270). - # session_model_usage gains a ``task`` column ('' = main agent - # loop; 'vision'/'compression'/'title_generation'/... = - # auxiliary calls) so aux model spend is visible in analytics. - # The column participates in the PRIMARY KEY and SQLite cannot - # ALTER a PK, so rebuild the table. The reconciler will have - # already ADDed the plain column on legacy DBs (harmless); - # the rebuild bakes it into the PK properly. Existing rows are - # main-loop accounting by definition → task=''. + # v22: ``task`` joins the session_model_usage PRIMARY KEY ('' = + # main loop; 'vision'/'compression'/... = aux calls). SQLite + # cannot ALTER a PK, so rebuild; existing rows are main-loop + # accounting → task=''. try: legacy_pk = cursor.execute( "SELECT COUNT(*) FROM pragma_table_info('session_model_usage') " @@ -1357,57 +1123,34 @@ class SessionSchemaMixin: ) except sqlite3.OperationalError as exc: logger.debug("v22 session_model_usage rebuild skipped: %s", exc) - if current_version < 23: - # v23: FTS storage redesign (issues #22478, #43690, #55233). - # The v11 inline-mode FTS tables each store a full private - # copy of every message (content || tool_name || tool_calls), - # and the trigram index additionally covers role='tool' rows - # (~90% of message bytes: base64 payloads, file dumps) at - # ~2.6x amplification — together ~75% of state.db on heavy - # installs (observed: 18.9 GB of a 25 GB DB). - # - # OPT-IN, NOT AUTOMATIC. The transition (demote old vtables → - # new external-content schema → backfill → teardown → VACUUM) - # is disk-heavy (transient ~2x file size to fully reclaim via - # VACUUM) and long (~1-2h background on a 25 GB DB). Doing it - # silently on every big user's next open — with a completeness - # guarantee that depends on the process staying alive long - # enough — is the wrong default. So on an EXISTING install we - # touch nothing here: the v22 inline FTS keeps working exactly - # as before, and we only record a flag advertising that the - # optimization is available. `hermes sessions optimize-storage` - # performs the whole transition as one deliberate, disk-checked, - # progress-reported foreground operation. - # - # DECOUPLED VERSIONING. Crucially, this does NOT hold back the - # main schema_version. The FTS storage LAYOUT is tracked by an - # independent `fts_storage_version` marker (see - # _fts_storage_version / SETTLE below), so schema_version - # advances to SCHEMA_VERSION here like every other migration — - # future v24+ migrations land automatically for legacy-FTS - # users too. Only the FTS *layout* waits for opt-in. - if fts5_available and self._db_has_legacy_inline_fts(cursor): - self.set_meta("fts_optimize_available", "1", cursor=cursor) + # v23: FTS storage redesign (external-content tables; inline v11 + # tables were ~75% of state.db on heavy installs). OPT-IN, NOT + # AUTOMATIC: the transition is disk-heavy (~2x transient) and long + # (hours on a 25 GB DB), so an existing install only gets a flag + # advertising it; `hermes sessions optimize-storage` performs it as + # a deliberate foreground operation. DECOUPLED VERSIONING: the FTS + # layout is tracked by the independent `fts_storage_version` + # marker, so schema_version still advances here and future + # migrations land for legacy-FTS users too. + if ( + current_version < 23 + and fts5_available + and self._db_has_legacy_inline_fts(cursor) + ): + self.set_meta("fts_optimize_available", "1", cursor=cursor) if current_version < 25: - # v25: de-duplicate per-session system prompt snapshots into - # a shared content-addressed table. Keep the old column as a - # read fallback for partially migrated or externally written - # rows, but clear migrated rows so future writes do not keep - # one large prompt copy per session. + # v25: de-duplicate system prompt snapshots into the shared + # content-addressed table; the old column stays a read fallback + # for partially migrated or externally written rows. self._dedupe_legacy_system_prompts(cursor) - # The FTS storage layout is versioned independently of the main - # schema (see the v23 note above). Stamp the current layout so the - # main version can always advance: a fresh/optimized DB is at - # FTS_STORAGE_VERSION; a legacy DB is left at whatever it had - # (absent/0) until `optimize-storage` runs. An INTERRUPTED - # optimize (legacy vtables already demoted, but rebuild markers - # or demoted trash tables still present, or an empty external - # index against non-empty messages) is NOT stamped either — - # the marker is the source of truth for "fully optimized", and - # `fts_optimize_available()` keeps offering the resume until the - # transition actually completes. + # Stamp the FTS layout version (fresh/optimized DBs) so the main + # version can always advance; a legacy DB keeps its absent/0 marker + # until optimize-storage runs. An INTERRUPTED optimize (rebuild + # markers, trash tables, or an empty external index against + # non-empty messages) is NOT stamped: the marker is the source of + # truth for "fully optimized" and keeps the resume offer alive. if ( fts5_available and not self._db_has_legacy_inline_fts(cursor) @@ -1422,12 +1165,10 @@ class SessionSchemaMixin: "fts_storage_version", str(FTS_STORAGE_VERSION), cursor=cursor ) - # Advance schema_version to current for ALL non-FTS-layout - # migrations. This is deliberately NOT gated on the FTS opt-in — - # holding the whole version back would block every future schema - # migration for a user who never optimizes. FTS5 being unavailable - # is the one case we skip (we can't have created the current FTS - # objects, so claiming the current schema would be a lie). + # Advance schema_version — deliberately NOT gated on the FTS opt-in + # (that would block every future migration for a user who never + # optimizes). FTS5 unavailable is the one skip: we can't have + # created the current FTS objects, so claiming current would lie. if ( current_version < SCHEMA_VERSION and fts_migrations_complete @@ -1438,9 +1179,8 @@ class SessionSchemaMixin: (SCHEMA_VERSION,), ) - # Unique title index — always ensure it exists. Older databases may - # contain duplicate aliases from before the constraint was enforced; - # preserve every session while letting the newest one retain the alias. + # Unique title index. Older DBs may hold duplicate aliases from before + # the constraint; keep every session, the newest retains the alias. title_index_sql = ( "CREATE UNIQUE INDEX IF NOT EXISTS idx_sessions_title_unique " "ON sessions(title) WHERE title IS NOT NULL" @@ -1448,8 +1188,7 @@ class SessionSchemaMixin: try: cursor.execute(title_index_sql) except sqlite3.IntegrityError: - # The index is an optimization — its creation must never abort - # opening the database, so the repair itself is also guarded. + # The index must never abort opening the DB; guard the repair too. try: cursor.execute( """UPDATE sessions AS older @@ -1475,30 +1214,28 @@ class SessionSchemaMixin: pass # Index already exists if fts5_available: - # FTS5 setup. Run the DDL even when the virtual table exists so - # CREATE TRIGGER IF NOT EXISTS repairs trigger-only degradation from - # an earlier no-FTS5 runtime. - # - # OPT-IN v23 boundary: a legacy v22 install (inline-content FTS, - # not yet opted into `hermes db optimize`) must keep its EXISTING - # inline schema + triggers. Running the v23 external-content DDL - # here would create the trigram source VIEW and leave the DB in a - # mixed inline/external state. So for a legacy DB we only ensure - # its inline triggers exist (via the legacy DDL), and skip the - # v23 view/external tables entirely. Fresh installs and opted-in - # DBs have no legacy inline FTS, so they get the v23 DDL. + # Run the FTS DDL even when the vtable exists so CREATE TRIGGER IF + # NOT EXISTS repairs trigger-only degradation from a no-FTS5 runtime. + # OPT-IN v23 boundary: a legacy v22 inline install must keep its + # inline schema + triggers — the v23 external-content DDL would + # create the trigram source VIEW and leave a mixed state — so it + # gets the legacy DDL only; fresh/opted-in DBs get v23. legacy_fts = self._db_has_legacy_inline_fts(cursor) if self._fts_stale: if self._recover_stale_fts(cursor, legacy=legacy_fts): - # CJK was detached alongside the corrupt base indexes and - # has its own stale marker. Its existing ensure path keeps - # it offline until its dedicated rebuild. + # CJK was detached alongside the base indexes and has its + # own stale marker; its ensure path decides when it returns. self._ensure_fts_cjk_schema(cursor) else: self._fts_enabled = False self._trigram_available = False self._fts_cjk_available = False - elif legacy_fts: + else: + base_sql, trigram_sql, rebuild = ( + (LEGACY_FTS_SQL, LEGACY_FTS_TRIGRAM_SQL, self._rebuild_legacy_fts_indexes) + if legacy_fts + else (FTS_SQL, FTS_TRIGRAM_SQL, self._rebuild_fts_indexes) + ) # Measure BEFORE the DDL below runs, so these describe the # pre-repair state. Whether the trigram half is even # creatable is only known AFTER _ensure_fts_schema, which is @@ -1512,11 +1249,14 @@ class SessionSchemaMixin: < len(_FTS_TRIGRAM_TRIGGERS) ) self._fts_enabled = self._ensure_fts_schema( - cursor, "messages_fts", LEGACY_FTS_SQL + cursor, "messages_fts", base_sql ) if self._fts_enabled: + # Trigram FTS5 for CJK/substring search is optional + # relative to the main table; if it cannot be created, + # CJK search falls back to LIKE. trigram_enabled = self._ensure_fts_schema( - cursor, "messages_fts_trigram", LEGACY_FTS_TRIGRAM_SQL + cursor, "messages_fts_trigram", trigram_sql ) self._trigram_available = trigram_enabled if base_triggers_missing or ( @@ -1524,49 +1264,16 @@ class SessionSchemaMixin: ): self._run_admitted_startup_rebuild( cursor, - lambda: self._rebuild_legacy_fts_indexes( - cursor, include_trigram=trigram_enabled - ), + lambda: rebuild(cursor, include_trigram=trigram_enabled), ) - else: - # Same split as the legacy branch above, same reason. - base_triggers_missing = ( - self._fts_trigger_count(cursor, _FTS_BASE_TRIGGERS) - < len(_FTS_BASE_TRIGGERS) - ) - trigram_triggers_missing = ( - self._fts_trigger_count(cursor, _FTS_TRIGRAM_TRIGGERS) - < len(_FTS_TRIGRAM_TRIGGERS) - ) - self._fts_enabled = self._ensure_fts_schema( - cursor, "messages_fts", FTS_SQL - ) - - # Trigram FTS5 for CJK/substring search. This is optional - # relative to the main FTS table; if it cannot be created, - # CJK search falls back to LIKE. - if self._fts_enabled: - trigram_enabled = self._ensure_fts_schema( - cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL - ) - self._trigram_available = trigram_enabled - if base_triggers_missing or ( - trigram_enabled and trigram_triggers_missing - ): - self._run_admitted_startup_rebuild( - cursor, - lambda: self._rebuild_fts_indexes( - cursor, - include_trigram=trigram_enabled, - ), - ) - # CJK-bigram index (cjk_unicode61). Strictly additive to - # the surfaces above and gated on the loadable tokenizer: - self._ensure_fts_cjk_schema(cursor) + if not legacy_fts: + # CJK-bigram index (cjk_unicode61): strictly additive + # and gated on the loadable tokenizer. + self._ensure_fts_cjk_schema(cursor) # Replace any pre-existing broad AFTER UPDATE triggers with # AFTER UPDATE OF variants. IF NOT EXISTS cannot rewrite them. - if getattr(self, "_fts_enabled", False): + if self._fts_enabled: self._migrate_broad_fts_update_triggers(cursor) self._conn.commit() @@ -1574,24 +1281,18 @@ class SessionSchemaMixin: def _run_admitted_startup_rebuild(self, cursor, rebuild_fn) -> None: """Run a full trigger-repair FTS rebuild under cross-process admission. - ``_init_schema`` reaches here when the sync triggers were missing and - the DDL just recreated them, so the index has a gap of unknown extent - and must be rebuilt in full. Two processes opening the same DB after - an update commonly hit this path simultaneously — the exact + Reached when the sync triggers were missing and the DDL just recreated + them: the index has a gap of unknown extent. Two processes opening the + same DB after an update commonly hit this simultaneously — the concurrent-rebuild interleaving that structurally corrupted state.db - in production (PR #93200) — so the rebuild admits through - ``fts_rebuild_admission`` and FAILS CLOSED. - - On deferral (another process holds the rebuild authority) the - just-repaired triggers are dropped again and the durable stale - breadcrumb is persisted, mirroring ``_enter_fts_fail_open``'s - ordering contract: triggers must never be live over an index with an - unrebuilt gap. FTS stays detached for this instance; the winner's - rebuild — or ``retry_deferred_fts_recovery`` from the gateway - housekeeping tick, or ``_recover_stale_fts`` at the next startup — restores - the index and triggers atomically. + in production — so this admits through ``fts_rebuild_admission`` and + FAILS CLOSED. On deferral the just-repaired triggers are dropped again + and the stale breadcrumb persisted (``_enter_fts_fail_open``'s + ordering contract: triggers must never be live over an unrebuilt gap); + the winner's rebuild, ``retry_deferred_fts_recovery``, or + ``_recover_stale_fts`` at next startup restores index and triggers. """ - with fts_rebuild_admission(getattr(self, "db_path", None)) as admitted: + with fts_rebuild_admission(self.db_path) as admitted: if admitted: rebuild_fn() return @@ -1615,12 +1316,7 @@ class SessionSchemaMixin: self, cursor: sqlite3.Cursor ) -> None: """One-time v18 backfill of gateway metadata from sessions.json. - - Existing gateway sessions predate the display_name / origin_json / - expiry_finalized columns; copy what sessions.json knows so consumers - can switch to state.db without losing pre-migration sessions. - Only fills NULL columns — never overwrites data written by newer code. - """ + Only fills NULL columns — never overwrites data written by newer code.""" sessions_file = get_hermes_home() / "sessions" / "sessions.json" if not sessions_file.exists(): return diff --git a/hermes_state_search.py b/hermes_state_search.py index 3dafeebc4a..7abfe3801f 100644 --- a/hermes_state_search.py +++ b/hermes_state_search.py @@ -1,11 +1,9 @@ """Full-text / trigram / CJK message search and FTS maintenance for SessionDB. -Mixin contract: this is a plain mixin class consumed by -``hermes_state.SessionDB``. It defines no ``__init__`` and no state of its -own; methods access the host's attributes (``self._conn``, ``self.db_path``, -``self._execute_write`` and other SessionDB methods) established by -``SessionDB.__init__``. It must never import hermes_state (cycle) — shared -module-level constants live in hermes_state_common. +Plain mixin consumed by ``hermes_state.SessionDB``: no ``__init__``, no state +of its own; methods use host attributes established by ``SessionDB.__init__``. +Must never import hermes_state (cycle) — shared constants live in +hermes_state_common. """ import logging @@ -30,22 +28,78 @@ from hermes_state_common import ( fts_rebuild_admission, ) -# Moved methods logged under the "hermes_state" logger before the split; -# keep that logger identity so log filtering/capture behavior is unchanged. +# Keep the pre-split logger identity so log filtering/capture is unchanged. logger = logging.getLogger("hermes_state") # Characters FTS5's query grammar rejects outside a quoted phrase. Anything -# missing from this set reaches MATCH raw and raises, which the execute site -# swallows into zero results — the failure this strip step exists to prevent. -# Assembled through re.escape so the backslash cannot be eaten as a regex -# escape inside the class (it was, while the set was written as a literal). -# -# ``%`` is deliberately excluded: a CJK query falls back to a LIKE search that -# needs it preserved as a literal (that path escapes wildcards itself), so -# stripping it here widened those queries onto unrelated rows. +# missing here reaches MATCH raw and raises, which the execute site swallows +# into zero results. Assembled through re.escape so the backslash cannot be +# eaten as a regex escape. ``%`` is deliberately excluded: the CJK LIKE +# fallback needs it as a literal (that path escapes wildcards itself). _FTS5_SPECIAL_CHARS = '+{}():"^@/#&|~[]<>,;!?$=\\\'' _FTS5_SPECIAL_RE = re.compile(f"[{re.escape(_FTS5_SPECIAL_CHARS)}]") +_FTS_OPERATORS = frozenset({"AND", "OR", "NOT"}) + +# Column list shared by every search route (snippet + metadata, never content). +_SEARCH_SELECT_TAIL = ( + "m.timestamp, m.tool_name, s.source, s.model, s.started_at AS session_started" +) +_LIKE_SNIPPET_SQL = "substr(m.content, max(1, instr(m.content, ?) - 40), 120) AS snippet" +_LIKE_ANY_COLUMN_SQL = ( + "(m.content LIKE ? ESCAPE '\\' OR m.tool_name LIKE ? ESCAPE '\\' " + "OR m.tool_calls LIKE ? ESCAPE '\\')" +) + + +def _meta_row(conn, key: str) -> Optional[sqlite3.Row]: + """Point-read one ``state_meta`` row (``None`` when absent).""" + return conn.execute( + "SELECT value FROM state_meta WHERE key = ?", (key,) + ).fetchone() + + +def _delete_meta(conn, *keys: str) -> None: + conn.execute( + f"DELETE FROM state_meta WHERE key IN ({','.join('?' for _ in keys)})", keys + ) + + +def _quote_fts_tokens(raw_query: str) -> str: + """Quote each non-operator token (neutralising FTS5 special characters) + while preserving AND/OR/NOT.""" + return " ".join( + tok if tok.upper() in _FTS_OPERATORS else '"' + tok.replace('"', '""') + '"' + for tok in raw_query.split() + ) + + +def _search_filter_clauses( + where: List[str], + params: list, + *, + include_inactive: bool, + source_filter: Optional[List[str]], + exclude_sources: Optional[List[str]], + role_filter: Optional[List[str]], +) -> None: + """Append the visibility/source/role predicates every search route shares. + + Live rows (active=1) AND compaction-archived rows (compacted=1) are + discoverable; only rewind/undo rows (active=0, compacted=0) are hidden. + """ + if not include_inactive: + where.append("(m.active = 1 OR m.compacted = 1)") + if source_filter is not None: + where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") + params.extend(source_filter) + if exclude_sources is not None: + where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") + params.extend(exclude_sources) + if role_filter: + where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") + params.extend(role_filter) + class SessionSearchMixin: """See module docstring — mixin for SessionDB (Search cluster).""" @@ -89,35 +143,31 @@ class SessionSearchMixin: max_pages=self._FTS_MERGE_MAX_PAGES_PER_INDEX ) except Exception as exc: # noqa: BLE001 - post-commit maintenance - # The canonical write is already committed before this cadence - # runs. No maintenance failure — including the bare SystemError - # the CPython sqlite3 layer can raise under cross-thread errmsg - # scrambling — may escape and make the caller replay an - # ambiguous, possibly-durable write (#90734, #85079). + # The canonical write is already committed. No maintenance failure + # — including the bare SystemError CPython's sqlite3 layer can raise + # under cross-thread errmsg scrambling — may escape and make the + # caller replay an ambiguous, possibly-durable write. logger.warning("FTS incremental merge failed after commit: %s", exc) def fts_rebuild_status(self) -> Optional[Dict[str, Any]]: - """Return deferred-rebuild progress, or None when no rebuild pending. + """Deferred-rebuild progress ``{"pending", "total", "indexed", + "percent"}``, or None when no rebuild is pending. - Shape: {"pending": True, "total": , - "indexed": , "percent": <0-100 int>}. - Consumed by search_messages() notes and by status surfaces - (dashboard/desktop can poll this to render a progress indicator). - - Reads state_meta directly via _read_ctx instead of calling - get_meta() (which takes self._lock) so search_messages doesn't - block on the writer lock when checking rebuild status. + Reads state_meta via the pooled reader rather than get_meta() (which + takes self._lock) so search_messages never blocks on the writer lock. """ - with self._read_ctx() as conn: - row = conn.execute( - "SELECT key, value FROM state_meta WHERE key IN (?, ?)", - ("fts_rebuild_high_water", "fts_rebuild_progress"), - ).fetchall() - meta = {r["key"]: r["value"] for r in row} - high_water = meta.get("fts_rebuild_high_water") + return self._rebuild_status("fts_rebuild") + + def _rebuild_status(self, prefix: str) -> Optional[Dict[str, Any]]: + rows = self._read_all( + "SELECT key, value FROM state_meta WHERE key IN (?, ?)", + (f"{prefix}_high_water", f"{prefix}_progress"), + ) + meta = {r["key"]: r["value"] for r in rows} + high_water = meta.get(f"{prefix}_high_water") if high_water is None: return None - progress = int(meta.get("fts_rebuild_progress") or 0) + progress = int(meta.get(f"{prefix}_progress") or 0) total = int(high_water) if total <= 0: return None @@ -127,69 +177,53 @@ class SessionSearchMixin: def _fts_rebuild_finish(self) -> None: """Finalize the deferred rebuild: boundary sweep + clear markers. - The sweep is cheap insurance against any write that slipped through - the migration-boundary instant (between high_water capture and - trigger activation): re-index any row near the boundary that the - index is missing. docsize has one row per indexed doc, so the - anti-join is exact and runs on a narrow id range. - - The trigram half of the sweep is gated on ``self._trigram_available`` - for the same reason ``fts_rebuild_step()`` gates its backfill INSERT: - when the SQLite build has no trigram tokenizer (or the table was - never created), an unconditional INSERT raises ``no such table`` - and aborts the whole rebuild — taking ``optimize_fts_storage()`` - down with it. + The sweep is cheap insurance against a write that slipped through the + migration-boundary instant (between high_water capture and trigger + activation). The trigram half is gated on ``_trigram_available``: + without the tokenizer/table an unconditional INSERT raises ``no such + table`` and aborts the whole rebuild (and optimize_fts_storage()). """ - include_trigram = self._trigram_available - - def _do(conn): - hw_row = conn.execute( - "SELECT value FROM state_meta WHERE key = 'fts_rebuild_high_water'" - ).fetchone() - if hw_row is not None: - hw = int(hw_row[0]) - # Sweep a generous window around the boundary. - lo, hi = hw - 1000, hw + 1000 - conn.execute( - "INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) " - "SELECT m.id, m.content, m.tool_name, m.tool_calls " - "FROM messages m " - "WHERE m.id > ? AND m.id <= ? " - "AND NOT EXISTS (SELECT 1 FROM messages_fts_docsize d WHERE d.id = m.id)", - (lo, hi), - ) - if include_trigram: - conn.execute( - "INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) " - "SELECT m.id, m.content, m.tool_name, m.tool_calls " - "FROM messages m " - "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' " - "AND NOT EXISTS (SELECT 1 FROM messages_fts_trigram_docsize d WHERE d.id = m.id)", - (lo, hi), - ) - conn.execute( - "DELETE FROM state_meta WHERE key IN " - "('fts_rebuild_high_water', 'fts_rebuild_progress')" - ) - self._execute_write(_do) + sweeps = [self._BOUNDARY_SWEEP_SQL.format(table="messages_fts", extra="")] + if self._trigram_available: + sweeps.append(self._BOUNDARY_SWEEP_SQL.format( + table="messages_fts_trigram", extra="AND m.role <> 'tool' " + )) + self._rebuild_finish("fts_rebuild", sweeps) logger.info("Deferred FTS rebuild complete — all messages indexed.") + # Re-index rows in an id window the index is missing. docsize has one row + # per indexed doc, so the anti-join is exact. + _BOUNDARY_SWEEP_SQL = ( + "INSERT INTO {table}(rowid, content, tool_name, tool_calls) " + "SELECT m.id, m.content, m.tool_name, m.tool_calls " + "FROM messages m " + "WHERE m.id > ? AND m.id <= ? {extra}" + "AND NOT EXISTS (SELECT 1 FROM {table}_docsize d WHERE d.id = m.id)" + ) + + def _rebuild_finish(self, prefix: str, sweep_sqls: List[str]) -> None: + """Sweep a generous window around the high-water boundary, then clear + the ``{prefix}_high_water`` / ``{prefix}_progress`` markers.""" + def _do(conn): + hw_row = _meta_row(conn, f"{prefix}_high_water") + if hw_row is not None: + hw = int(hw_row[0]) + for sql in sweep_sqls: + conn.execute(sql, (hw - 1000, hw + 1000)) + _delete_meta(conn, f"{prefix}_high_water", f"{prefix}_progress") + self._execute_write(_do) + def _fts_teardown_trash_step(self) -> bool: - """Tear down one chunk of a demoted v22 FTS shadow table. + """Tear down one chunk of a demoted v22 FTS shadow table; True while + work remains. - The trash tables are PLAIN tables (their vtable parent was demoted - away during the migration), so chunked DELETE + final DROP involve - no FTS5 machinery at all. Returns True while teardown work remains. - - Single-column-key trash tables (the common shape — FTS shadow - tables carry a rowid/integer PK) are drained with a high-water - marker mirroring :meth:`fts_rebuild_step`: each chunk deletes only - rows after the previously-drained key, so the per-chunk scan is - bounded instead of re-scanning from the start of the table every - chunk (O(n²) total on large trash tables, #79324). Compound-key - trash tables (multi-column PK) cannot use a scalar high-water - comparison, so they keep the legacy chunked ``LIMIT`` delete — - those shadow tables are small by construction. + Trash tables are PLAIN tables (their vtable parent was demoted), so + chunked DELETE + final DROP involve no FTS5 machinery. Integer + single-column-key tables are drained with a high-water marker so each + chunk's scan is bounded (re-scanning from the start was O(n²) on + large tables). Compound-key tables cannot use a scalar high-water + comparison and keep the chunked ``LIMIT`` delete — they are small by + construction. """ with self._lock: trash = [ @@ -214,18 +248,13 @@ class SessionSearchMixin: key = ", ".join(pk_cols) if pk_cols else "rowid" if len(pk_cols) == 1 and (not pk_info or pk_info[0][1] == "INTEGER"): - # High-water drain: delete only rows past the marker key. - # The marker is read/written inside the same BEGIN IMMEDIATE - # transaction as the DELETE, so concurrent callers claim - # disjoint key ranges instead of re-deleting. Only integer - # PKs can anchor a numeric high-water comparison — the FTS - # config shadow table (TEXT pk like 'version') falls back to - # the legacy chunked delete below. + # High-water drain. The marker is read/written in the same + # BEGIN IMMEDIATE as the DELETE, so concurrent callers claim + # disjoint key ranges. Only integer PKs can anchor the numeric + # comparison — the config shadow table (TEXT pk) falls through + # to the chunked delete below. marker_key = f"fts_teardown_{tbl}_progress" - row = conn.execute( - "SELECT value FROM state_meta WHERE key = ?", - (marker_key,), - ).fetchone() + row = _meta_row(conn, marker_key) high_water = int(row[0]) if row is not None else 0 # Claim the chunk's upper bound: the LAST row of the @@ -238,9 +267,7 @@ class SessionSearchMixin: if not upper_rows: # Drained — the DROP is cheap now. conn.execute(f"DROP TABLE IF EXISTS {tbl}") - conn.execute( - "DELETE FROM state_meta WHERE key = ?", (marker_key,) - ) + _delete_meta(conn, marker_key) logger.info("Old FTS shadow table %s torn down.", tbl) return True @@ -250,17 +277,11 @@ class SessionSearchMixin: (high_water, upper), ) if cur.rowcount > 0: - conn.execute( - "INSERT INTO state_meta (key, value) VALUES (?, ?) " - "ON CONFLICT(key) DO UPDATE SET value = excluded.value", - (marker_key, str(upper)), - ) + self.set_meta(marker_key, str(upper), cursor=conn) return True - # Compound-key or rowid trash table: legacy chunked delete. - # These shadow tables are small, so the quadratic re-scan is - # not a concern (#79324 keeps the high-water path for the big - # single-key tables). + # Compound-key or rowid trash table: chunked delete (small tables, + # quadratic re-scan is not a concern). cur = conn.execute( f"DELETE FROM {tbl} WHERE ({key}) IN " f"(SELECT {key} FROM {tbl} LIMIT {self._FTS_REBUILD_CHUNK_ROWS})" @@ -278,29 +299,44 @@ class SessionSearchMixin: return True def fts_rebuild_step(self) -> bool: - """Backfill one chunk of the deferred FTS rebuild. - - Returns True when more work remains, False when the rebuild is - complete (or none is pending). Safe to call from any process at any - time; chunks are claimed atomically inside the write transaction, so - concurrent callers interleave instead of duplicating rows. - """ + """Backfill one chunk of the deferred FTS rebuild; True while work + remains. Safe from any process: chunks are claimed atomically inside + the write transaction, so concurrent callers interleave instead of + duplicating rows.""" if not self._fts_enabled: return False - high_water_raw = self.get_meta("fts_rebuild_high_water") + inserts = [self._CHUNK_INSERT_SQL.format(table="messages_fts", extra="")] + if self._trigram_available: + inserts.append(self._CHUNK_INSERT_SQL.format( + table="messages_fts_trigram", extra=" AND role <> 'tool'" + )) + return self._rebuild_step( + "fts_rebuild", inserts, + fail_msg="FTS rebuild chunk failed (will retry): %s", + finish=self._fts_rebuild_finish, + ) + + _CHUNK_INSERT_SQL = ( + "INSERT INTO {table}(rowid, content, tool_name, tool_calls) " + "SELECT id, content, tool_name, tool_calls FROM messages " + "WHERE id > ? AND id <= ?{extra}" + ) + + def _rebuild_step( + self, prefix: str, insert_sqls: List[str], *, fail_msg: str, finish + ) -> bool: + """Shared chunk engine for the base and CJK deferred backfills.""" + high_water_raw = self.get_meta(f"{prefix}_high_water") if high_water_raw is None: return False high_water = int(high_water_raw) - include_trigram = self._trigram_available chunk = self._FTS_REBUILD_CHUNK_ROWS def _do(conn): # Re-read progress inside the write transaction (BEGIN IMMEDIATE # is already held by _execute_write) — this is the claim: two # workers can't read the same progress value concurrently. - row = conn.execute( - "SELECT value FROM state_meta WHERE key = 'fts_rebuild_progress'" - ).fetchone() + row = _meta_row(conn, f"{prefix}_progress") if row is None: return False # finished (or cleared) by another process progress = int(row[0]) @@ -310,179 +346,87 @@ class SessionSearchMixin: # The chunk upper bound is an id, not a row count, so gaps from # deleted rows don't shrink chunks below the claimed range. upper = min(progress + chunk, high_water) - conn.execute( - "INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) " - "SELECT id, content, tool_name, tool_calls FROM messages " - "WHERE id > ? AND id <= ?", - (progress, upper), - ) - if include_trigram: - conn.execute( - "INSERT INTO messages_fts_trigram" - "(rowid, content, tool_name, tool_calls) " - "SELECT id, content, tool_name, tool_calls FROM messages " - "WHERE id > ? AND id <= ? AND role <> 'tool'", - (progress, upper), - ) + for sql in insert_sqls: + conn.execute(sql, (progress, upper)) # Publish progress in the same transaction as the rows it # covers — crash-atomic: either both land or neither does. conn.execute( - "UPDATE state_meta SET value = ? " - "WHERE key = 'fts_rebuild_progress'", - (str(upper),), + "UPDATE state_meta SET value = ? WHERE key = ?", + (str(upper), f"{prefix}_progress"), ) return upper < high_water try: more = self._execute_write(_do) except sqlite3.OperationalError as exc: - logger.debug("FTS rebuild chunk failed (will retry): %s", exc) + logger.debug(fail_msg, exc) return True # transient (lock contention) — caller retries if more is False: - status = self.fts_rebuild_status() + status = self._rebuild_status(prefix) if status is not None and status["indexed"] >= status["total"]: - self._fts_rebuild_finish() + finish() return False return bool(more) def fts_cjk_rebuild_status(self) -> Optional[Dict[str, Any]]: """CJK-index backfill progress, or None when none is pending.""" - with self._read_ctx() as conn: - row = conn.execute( - "SELECT key, value FROM state_meta WHERE key IN (?, ?)", - ("fts_cjk_rebuild_high_water", "fts_cjk_rebuild_progress"), - ).fetchall() - meta = {r["key"]: r["value"] for r in row} - high_water = meta.get("fts_cjk_rebuild_high_water") - if high_water is None: - return None - progress = int(meta.get("fts_cjk_rebuild_progress") or 0) - total = int(high_water) - if total <= 0: - return None - pct = min(100, int(100 * progress / total)) - return {"pending": True, "total": total, "indexed": progress, "percent": pct} + return self._rebuild_status("fts_cjk_rebuild") def fts_cjk_rebuild_step(self) -> bool: """Backfill one chunk of the CJK index. True while work remains.""" if not self._fts_enabled or not self._fts_cjk_loaded: return False - high_water_raw = self.get_meta("fts_cjk_rebuild_high_water") - if high_water_raw is None: - return False - high_water = int(high_water_raw) - chunk = self._FTS_REBUILD_CHUNK_ROWS - - def _do(conn): - row = conn.execute( - "SELECT value FROM state_meta " - "WHERE key = 'fts_cjk_rebuild_progress'" - ).fetchone() - if row is None: - return False # finished (or cleared) by another process - progress = int(row[0]) - if progress >= high_water: - return False - upper = min(progress + chunk, high_water) - conn.execute( - "INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) " - "SELECT id, content, tool_name, tool_calls FROM messages " - "WHERE id > ? AND id <= ? AND role <> 'tool'", - (progress, upper), - ) - conn.execute( - "UPDATE state_meta SET value = ? " - "WHERE key = 'fts_cjk_rebuild_progress'", - (str(upper),), - ) - return upper < high_water - - try: - more = self._execute_write(_do) - except sqlite3.OperationalError as exc: - logger.debug("CJK FTS rebuild chunk failed (will retry): %s", exc) - return True - if more is False: - status = self.fts_cjk_rebuild_status() - if status is not None and status["indexed"] >= status["total"]: - self._fts_cjk_rebuild_finish() - return False - return bool(more) + return self._rebuild_step( + "fts_cjk_rebuild", + [self._CHUNK_INSERT_SQL.format( + table="messages_fts_cjk", extra=" AND role <> 'tool'" + )], + fail_msg="CJK FTS rebuild chunk failed (will retry): %s", + finish=self._fts_cjk_rebuild_finish, + ) def _fts_cjk_rebuild_finish(self) -> None: """Boundary sweep + clear the cjk markers; index becomes servable.""" - def _do(conn): - hw_row = conn.execute( - "SELECT value FROM state_meta " - "WHERE key = 'fts_cjk_rebuild_high_water'" - ).fetchone() - if hw_row is not None: - hw = int(hw_row[0]) - lo, hi = hw - 1000, hw + 1000 - conn.execute( - "INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) " - "SELECT m.id, m.content, m.tool_name, m.tool_calls " - "FROM messages m " - "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' " - "AND NOT EXISTS (SELECT 1 FROM messages_fts_cjk_docsize d WHERE d.id = m.id)", - (lo, hi), - ) - conn.execute( - "DELETE FROM state_meta WHERE key IN " - "('fts_cjk_rebuild_high_water', 'fts_cjk_rebuild_progress')" + self._rebuild_finish("fts_cjk_rebuild", [ + self._BOUNDARY_SWEEP_SQL.format( + table="messages_fts_cjk", extra="AND m.role <> 'tool' " ) - self._execute_write(_do) + ]) self._fts_cjk_available = True logger.info("CJK FTS index backfill complete — serving CJK search.") def _fts_cjk_reset_if_stale(self) -> None: - """Rebuild path for a stale cjk index (triggers were dropped). - - The gap's extent is unknown, so the only safe recovery is a from- - scratch rebuild: drop the table + triggers, clear the breadcrumb, - recreate via ``_ensure_fts_cjk_schema`` (which sets fresh backfill - markers on a populated DB). Called from ``optimize_fts_storage`` on - a tokenizer-capable host; no-op when not stale. - """ + """From-scratch rebuild of a stale cjk index (triggers were dropped, + gap extent unknown): drop table + triggers, clear the breadcrumb, + recreate via ``_ensure_fts_cjk_schema`` (fresh backfill markers on a + populated DB). No-op when not stale.""" if not self._fts_cjk_loaded: return def _do(conn): - stale = conn.execute( - "SELECT 1 FROM state_meta WHERE key = ?", - (FTS_CJK_STALE_KEY,), - ).fetchone() - if not stale: + if _meta_row(conn, FTS_CJK_STALE_KEY) is None: return False for trig in _FTS_CJK_TRIGGERS: conn.execute(f"DROP TRIGGER IF EXISTS {trig}") conn.execute("DROP TABLE IF EXISTS messages_fts_cjk") conn.execute("DROP VIEW IF EXISTS messages_fts_cjk_src") - conn.execute( - "DELETE FROM state_meta WHERE key IN " - f"('{FTS_CJK_STALE_KEY}', 'fts_cjk_rebuild_high_water', " - "'fts_cjk_rebuild_progress')" + _delete_meta( + conn, FTS_CJK_STALE_KEY, "fts_cjk_rebuild_high_water", "fts_cjk_rebuild_progress" ) return True was_stale = self._execute_write(_do) if was_stale: # Recreate outside the write transaction — _ensure_fts_cjk_schema - # uses executescript(), which implicitly commits any pending - # transaction and must not run inside _execute_write's BEGIN - # IMMEDIATE. Sets fresh backfill markers on a populated DB. + # uses executescript(), which implicitly commits and must not run + # inside _execute_write's BEGIN IMMEDIATE. with self._lock: self._ensure_fts_cjk_schema(self._conn) self._conn.commit() def _fts_external_index_empty_with_messages(self, conn) -> bool: - """True when the base FTS table exists but indexes nothing while - ``messages`` has rows. Caller must hold ``self._lock``. - - This is the post-demote empty-index shape: external-content FTS with - zero ``messages_fts_docsize`` rows against a non-empty messages table. - Healthy installs (and mid-backfill installs that still hold markers) - never match. - """ + """True when the base FTS table indexes nothing while ``messages`` has + rows (the post-demote empty-index shape). Caller holds ``self._lock``. + Healthy and mid-backfill installs never match.""" try: has_msg = conn.execute( "SELECT EXISTS(SELECT 1 FROM messages)" @@ -490,11 +434,9 @@ class SessionSearchMixin: if not has_msg: return False # docsize is the authoritative "is this rowid indexed" surface for - # external-content FTS5; probing the virtual table itself is - # not reliable across SQLite builds. EXISTS instead of COUNT(*): - # this runs on every writable open via the _init_schema stamp - # condition, and COUNT(*) is a full b-tree scan (~100ms on a - # 2M-row table) while EXISTS is O(1). + # external-content FTS5 (probing the vtable is unreliable across + # builds). EXISTS not COUNT(*): this runs on every writable open + # and COUNT(*) is a full b-tree scan. has_fts = conn.execute( "SELECT EXISTS(SELECT 1 FROM messages_fts_docsize)" ).fetchone()[0] @@ -504,11 +446,8 @@ class SessionSearchMixin: return False def _fts_index_known_empty(self, conn) -> bool: - """True when the base external-content index holds no rows. - - A missing table counts as empty: the schema ensure that follows - creates it fresh. - """ + """True when the base external-content index holds no rows; a missing + table counts as empty (the schema ensure that follows creates it).""" try: n = conn.execute( "SELECT COUNT(*) FROM messages_fts_docsize" @@ -518,18 +457,14 @@ class SessionSearchMixin: return True def _reset_fts_index_to_empty(self, conn) -> None: - """Delete every indexed row from the v23 external-content tables. + """Truncate the v23 external-content tables via FTS5 ``'delete-all'``. - Uses the FTS5 ``'delete-all'`` special command — the documented O(1) - truncate for external-content tables. A plain no-WHERE ``DELETE`` is - O(rows) on external-content FTS5 (each row's delete tokens are - regenerated from the content table; measured ~12µs/row, minutes on a - large index, while holding the write lock) and corrupts the index if - indexed rows have diverged from ``messages`` — precisely the broken- - bookkeeping shape this repair path handles. The backfill chunk worker - replays its whole selected id range with no anti-join, so a replay - from zero is only safe once the index is known empty — this is how a - partially indexed DB gets there. + A plain ``DELETE`` is O(rows) on external-content FTS5 (minutes on a + large index, holding the write lock) and corrupts the index when + indexed rows have diverged from ``messages`` — precisely the shape + this repair handles. The backfill worker replays its id range with no + anti-join, so a replay from zero is only safe once the index is known + empty; this is how a partially indexed DB gets there. """ for tbl in ("messages_fts", "messages_fts_trigram"): try: @@ -539,90 +474,49 @@ class SessionSearchMixin: def _seed_fts_rebuild_markers(self, conn, *, force: bool = False) -> int: """Write ``fts_rebuild_high_water`` / ``fts_rebuild_progress`` for a - full backfill. Returns the high-water id. + full backfill; returns the high-water id. - When ``force`` is False and high_water is already set, only repairs a - missing progress key (stuck no-op when high_water exists alone), and - only after the index is known empty: the chunk worker replays its - whole selected id range without an anti-join, so a partially indexed - DB is first reset to a known-empty surface rather than rebuilt from - zero on top of surviving rows. Caller must hold the write - transaction / lock as appropriate. + Without ``force`` and with high_water already set, only repairs a + missing progress key, resetting a partially indexed DB to a + known-empty surface first (the chunk worker replays without an + anti-join). Caller holds the write transaction. """ - existing_hw = conn.execute( - "SELECT value FROM state_meta WHERE key = 'fts_rebuild_high_water'" - ).fetchone() + existing_hw = _meta_row(conn, "fts_rebuild_high_water") if existing_hw is not None and not force: - hw = int(existing_hw[0]) - progress = conn.execute( - "SELECT value FROM state_meta WHERE key = 'fts_rebuild_progress'" - ).fetchone() - if progress is None: + if _meta_row(conn, "fts_rebuild_progress") is None: # high_water without progress: fts_rebuild_step treats missing # progress as "done by another process" and optimize would # no-op then stamp. Re-seed progress so the chunk loop runs. if not self._fts_index_known_empty(conn): self._reset_fts_index_to_empty(conn) - conn.execute( - "INSERT INTO state_meta (key, value) VALUES " - "('fts_rebuild_progress', '0') " - "ON CONFLICT(key) DO UPDATE SET value = excluded.value" - ) - return hw + self.set_meta("fts_rebuild_progress", "0", cursor=conn) + return int(existing_hw[0]) - hw = conn.execute( - "SELECT COALESCE(MAX(id), 0) FROM messages" - ).fetchone()[0] - for k, v in ( - ("fts_rebuild_high_water", str(hw)), - ("fts_rebuild_progress", "0"), - ): - conn.execute( - "INSERT INTO state_meta (key, value) VALUES (?, ?) " - "ON CONFLICT(key) DO UPDATE SET value = excluded.value", - (k, v), - ) + hw = conn.execute("SELECT COALESCE(MAX(id), 0) FROM messages").fetchone()[0] + self.set_meta("fts_rebuild_high_water", str(hw), cursor=conn) + self.set_meta("fts_rebuild_progress", "0", cursor=conn) return int(hw) def _repair_optimize_bookkeeping(self) -> None: """Heal interrupted demote/backfill bookkeeping before optimize runs. - Covers two post-#65798 failure classes: + 1. Empty external-content index with messages and no markers (demote + crash window / settle without backfill): seed a full backfill. + 2. high_water without progress: seed progress (resetting a partially + populated index first so the anti-join-free replay cannot + duplicate rows). - 1. Empty external-content index with messages present and no rebuild - markers (demote crash window after empty v23 tables landed but - before markers, or settle that stamped without backfill). Seed a - full backfill. - 2. ``fts_rebuild_high_water`` present without ``fts_rebuild_progress`` - (partial meta) — seed progress so the chunk loop is not a no-op, - resetting a partially populated index to a known-empty surface - first so the anti-join-free chunk replay cannot duplicate rows. - - Must not invent markers on a still-legacy inline DB: that would make - ``optimize_fts_storage`` skip demote (``legacy and not pending``) and - attempt v23-shaped INSERTs against the inline table forever. + Must not invent markers on a still-legacy inline DB: optimize would + then skip demote and attempt v23 INSERTs against the inline table forever. """ def _do(conn): - existing_hw = conn.execute( - "SELECT value FROM state_meta " - "WHERE key = 'fts_rebuild_high_water'" - ).fetchone() - - if existing_hw is not None: + if _meta_row(conn, "fts_rebuild_high_water") is not None: # Repair orphan high_water-without-progress only. Never # invent a fresh claim on a healthy complete index. - progress = conn.execute( - "SELECT 1 FROM state_meta " - "WHERE key = 'fts_rebuild_progress'" - ).fetchone() - if progress is None: + if _meta_row(conn, "fts_rebuild_progress") is None: if not self._fts_index_known_empty(conn): self._reset_fts_index_to_empty(conn) - conn.execute( - "INSERT INTO state_meta (key, value) VALUES " - "('fts_rebuild_progress', '0') " - "ON CONFLICT(key) DO UPDATE SET value = '0'" - ) + self.set_meta("fts_rebuild_progress", "0", cursor=conn) return # No markers. On a still-legacy DB demote owns marker creation. @@ -632,65 +526,48 @@ class SessionSearchMixin: # Non-legacy empty external index (demote crash window / premature # stamp): seed a full backfill claim. if self._fts_external_index_empty_with_messages(conn): - conn.execute( - "DELETE FROM state_meta WHERE key = 'fts_storage_version'" - ) + _delete_meta(conn, "fts_storage_version") self._seed_fts_rebuild_markers(conn, force=True) self._execute_write(_do) def fts_optimize_available(self) -> bool: - """True when `optimize_fts_storage()` has work to do: either this DB - is a legacy inline-FTS install that can be optimized to the v23 - external-content schema, or a previous optimize run was interrupted - (legacy vtables already demoted, but backfill markers and/or trash - tables remain) and re-running would resume it, or the CJK-bigram - index needs a backfill/rebuild on this tokenizer-capable host, or - a prior demote left an empty external-content index without markers - (healable on re-run). - False for fresh and fully-optimized installs (and when FTS5 is - unavailable).""" + """True when `optimize_fts_storage()` has work: legacy inline FTS to + migrate, an interrupted optimize to resume (markers/trash remain), a + CJK-bigram backfill/rebuild on this tokenizer-capable host, or an + empty external index left without markers. False for fresh and + fully-optimized installs and when FTS5 is unavailable.""" if not self._fts_enabled or self.read_only: return False with self._lock: if self._db_has_legacy_inline_fts(self._conn): return True - # Interrupted optimize: demotion already removed the legacy - # vtables (so the check above is False), but the transition is - # unfinished until the backfill markers are cleared and the - # demoted trash tables are torn down. Search stays complete - # through the gap supplement meanwhile; re-running resumes. - if self._conn.execute( - "SELECT 1 FROM state_meta " - "WHERE key = 'fts_rebuild_high_water' LIMIT 1" - ).fetchone(): + # Interrupted optimize: legacy vtables already demoted, but the + # transition is unfinished until markers clear and trash is gone. + if _meta_row(self._conn, "fts_rebuild_high_water") is not None: return True # CJK-bigram index work — only offerable when THIS process can # tokenize: a pending backfill (markers set at creation on a # populated DB) or a stale index awaiting a from-scratch rebuild. - if self._fts_cjk_loaded and self._conn.execute( - "SELECT 1 FROM state_meta WHERE key IN " - f"('fts_cjk_rebuild_high_water', '{FTS_CJK_STALE_KEY}') LIMIT 1" - ).fetchone(): + if self._fts_cjk_loaded and ( + _meta_row(self._conn, "fts_cjk_rebuild_high_water") is not None + or _meta_row(self._conn, FTS_CJK_STALE_KEY) is not None + ): return True if self._has_fts_trash(self._conn): return True - # Pre-fix crash window: empty external-content index with - # messages still present, no markers, no trash (teardown already - # finished or never needed). Re-run seeds markers and backfills. + # Crash window: empty external index, messages present, no markers, + # no trash. Re-run seeds markers and backfills. return self._fts_external_index_empty_with_messages(self._conn) def _demote_legacy_fts_to_trash(self) -> int: """Demote the legacy inline FTS vtables and stage their shadow tables - for chunked teardown. Returns MAX(messages.id) as the rebuild high - water. O(1) schema surgery — the heavy delete is deferred to the - chunked teardown, exactly as the validated auto path did. + for chunked teardown; returns MAX(messages.id) as the rebuild high + water. O(1) schema surgery — the heavy delete is deferred. - Markers are written in the same BEGIN IMMEDIATE as the demote, *before* - the empty v23 schema is created. Schema creation uses - ``executescript`` and therefore cannot run inside that transaction - (it issues an implicit COMMIT — see the CJK recreate path). Creating - the empty schema only after markers are durable closes the crash - window where trash + empty v23 tables exist with no backfill claim. + Markers are written in the same BEGIN IMMEDIATE as the demote, BEFORE + the empty v23 schema is created (``executescript`` implicitly COMMITs + and cannot run inside that transaction). This closes the crash window + where trash + empty v23 tables exist with no backfill claim. """ def _stage(conn): self._drop_fts_triggers(conn) @@ -717,23 +594,26 @@ class SessionSearchMixin: ] for sh in shadows: conn.execute(f"ALTER TABLE {sh} RENAME TO fts_v22_trash_{sh}") - # Claim the backfill *before* empty v23 tables exist. A crash - # between this commit and schema ensure still leaves markers, so - # optimize-storage resumes instead of tearing down trash and - # stamping an empty index as complete. + # Claim the backfill BEFORE empty v23 tables exist so a crash + # before schema ensure resumes instead of stamping an empty index. hw = self._seed_fts_rebuild_markers(conn, force=True) - conn.execute( - "DELETE FROM state_meta WHERE key = 'fts_optimize_available'" - ) + _delete_meta(conn, "fts_optimize_available") return hw hw = int(self._execute_write(_stage)) - # Create the empty v23 schema outside the write transaction — - # ``_ensure_fts_schema`` uses executescript(), which implicitly - # commits any pending transaction and must not run inside - # ``_execute_write``'s BEGIN IMMEDIATE (same rule as the CJK recreate - # path above). Markers are already durable. + # Outside the write transaction: ``_ensure_fts_schema`` uses + # executescript(), which implicitly commits. Markers are durable. + self._ensure_v23_fts_tables( + "failed to create v23 messages_fts during optimize-storage demote" + ) + return hw + + def _ensure_v23_fts_tables(self, failure_message: str) -> None: + """Ensure the v23 external-content base + trigram tables under the + lock (IF NOT EXISTS, cheap); raise *failure_message* without the base + table, since the backfill loop would otherwise retry "no such table" + forever.""" with self._lock: base_ok = self._ensure_fts_schema(self._conn, "messages_fts", FTS_SQL) trigram_ok = self._ensure_fts_schema( @@ -741,11 +621,8 @@ class SessionSearchMixin: ) self._trigram_available = bool(trigram_ok) if not base_ok: - raise sqlite3.OperationalError( - "failed to create v23 messages_fts during optimize-storage demote" - ) + raise sqlite3.OperationalError(failure_message) self._conn.commit() - return hw def optimize_fts_storage( self, @@ -754,30 +631,21 @@ class SessionSearchMixin: vacuum: bool = True, ) -> Dict[str, Any]: """Migrate a legacy v22 inline-FTS DB to the v23 external-content - schema, foreground and to completion. Safe to re-run: if a previous - attempt was interrupted it resumes from the progress marker. - - ``progress_cb`` receives {"phase", "percent", "indexed", "total"} - dicts for a CLI progress bar. Returns a summary dict. - - The trigram tokenizer being unavailable is not fatal — the base index - is still rebuilt (CJK falls back to LIKE), mirroring normal startup. - """ + schema, foreground and to completion; re-running resumes an + interrupted attempt. ``progress_cb`` receives {"phase", "percent", + "indexed", "total"}. A missing trigram tokenizer is not fatal (CJK + falls back to LIKE, as at startup).""" if not self._fts_enabled: return {"ok": False, "reason": "fts5_unavailable"} if self.read_only: return {"ok": False, "reason": "read_only"} - # Heal empty-index / orphan-marker bookkeeping from an interrupted - # demote *before* deciding whether to demote again. This re-seeds - # markers when trash was already staged (or torn down) without a - # backfill claim so the phases below actually run. + # Heal empty-index / orphan-marker bookkeeping BEFORE deciding whether + # to demote again, so the phases below actually run. self._repair_optimize_bookkeeping() - # Only demote if we're actually still on the legacy shape. If a prior - # run already demoted (markers/trash present), skip straight to - # finishing the backfill + teardown — this is what makes re-running - # after an interruption safe. + # Demote only when still on the legacy shape; a prior demote + # (markers/trash present) skips straight to backfill + teardown. with self._lock: legacy = self._db_has_legacy_inline_fts(self._conn) pending = self.get_meta("fts_rebuild_high_water") is not None @@ -786,31 +654,15 @@ class SessionSearchMixin: elif pending and not legacy: # Resume mid-demote: markers exist, empty v23 tables may still be # missing if the process died between the staged demote commit and - # schema ensure. Re-ensure is IF NOT EXISTS and cheap. - with self._lock: - base_ok = self._ensure_fts_schema( - self._conn, "messages_fts", FTS_SQL - ) - trigram_ok = self._ensure_fts_schema( - self._conn, "messages_fts_trigram", FTS_TRIGRAM_SQL - ) - self._trigram_available = bool(trigram_ok) - if not base_ok: - # Fail fast: without the base table the backfill loop - # below would retry "no such table" errors forever. - raise sqlite3.OperationalError( - "failed to re-create v23 messages_fts " - "on optimize-storage resume" - ) - self._conn.commit() + # schema ensure. + self._ensure_v23_fts_tables( + "failed to re-create v23 messages_fts on optimize-storage resume" + ) - # A stale CJK index (triggers dropped by a tokenizer-less process) - # can only be recovered from scratch — reset it now so the cjk - # backfill phase below rebuilds it. No-op without the tokenizer. + # A stale CJK index can only be recovered from scratch; reset it so + # the cjk backfill phase rebuilds it. Then ensure table + markers + # exist (a v23 DB gaining the cjk index for the first time). self._fts_cjk_reset_if_stale() - # An optimized v23 DB gaining the cjk index for the first time (no - # legacy work left, tokenizer newly installed): ensure the table + - # markers exist so the backfill phase has work to claim. if self._fts_cjk_loaded: with self._lock: self._ensure_fts_cjk_schema(self._conn) @@ -830,57 +682,40 @@ class SessionSearchMixin: }) def _pause(chunk_seconds: float) -> None: - """Inter-chunk throttle (see the chunk-engine note above). - - The chunk methods themselves never sleep, so this loop is the - single place the duty cycle is enforced: without it, back-to-back - BEGIN IMMEDIATE chunks starve any live gateway/CLI process - sharing the DB out of its lock retries (the measured ~85% - write-lock ownership that froze concurrent sessions). - """ + """Inter-chunk throttle — the single place the duty cycle is + enforced. Without it back-to-back BEGIN IMMEDIATE chunks starve a + live gateway/CLI sharing the DB out of its lock retries.""" time.sleep(max( self._FTS_REBUILD_MIN_PAUSE, chunk_seconds * self._FTS_REBUILD_DUTY_FACTOR, )) - # Phase 1: backfill (foreground, throttled between chunks so a live - # gateway sharing the DB stays responsive). - _emit("backfill") - while True: - _t0 = time.monotonic() - if not self.fts_rebuild_step(): - break - _emit("backfill") - _pause(time.monotonic() - _t0) - _emit("backfill") + def _drive(phase: str, step) -> None: + """Run *step* to completion, emitting progress and throttling + between chunks so a live gateway sharing the DB stays responsive.""" + while True: + _t0 = time.monotonic() + if not step(): + break + _emit(phase) + _pause(time.monotonic() - _t0) - # Phase 1b: backfill the CJK-bigram index (its own marker pair; a - # no-op when the tokenizer isn't loadable or nothing is pending). - while True: - _t0 = time.monotonic() - if not self.fts_cjk_rebuild_step(): - break - _emit("backfill") - _pause(time.monotonic() - _t0) + # Phase 1: base backfill. Phase 1b: CJK-bigram backfill (its own + # marker pair; no-op without the tokenizer or pending work). + _emit("backfill") + _drive("backfill", self.fts_rebuild_step) + _emit("backfill") + _drive("backfill", self.fts_cjk_rebuild_step) # Phase 2: tear down the demoted legacy shadow tables in chunks. _emit("teardown") - while True: - _t0 = time.monotonic() - if not self._fts_teardown_trash_step(): - break - _emit("teardown") - _pause(time.monotonic() - _t0) + _drive("teardown", self._fts_teardown_trash_step) # Refuse to stamp "optimized" while work remains or the base index is - # still empty against a non-empty messages table. Pre-fix code could - # tear down trash and settle after a no-op backfill when markers were - # missing — permanent search-index loss for historical rows. + # empty against non-empty messages (settling after a no-op backfill + # meant permanent search-index loss for historical rows). with self._lock: - still_pending = self._conn.execute( - "SELECT 1 FROM state_meta " - "WHERE key = 'fts_rebuild_high_water' LIMIT 1" - ).fetchone() is not None + still_pending = _meta_row(self._conn, "fts_rebuild_high_water") is not None still_trash = self._has_fts_trash(self._conn) empty_index = self._fts_external_index_empty_with_messages(self._conn) if still_pending or still_trash or empty_index: @@ -904,23 +739,15 @@ class SessionSearchMixin: self._conn.execute("VACUUM") vacuum_ok = True except sqlite3.OperationalError as exc: - # Most common cause: not enough free disk for VACUUM's temp - # copy. The optimization still succeeded; space just isn't - # reclaimed until a later VACUUM. Non-fatal. + # Usually no free disk for VACUUM's temp copy; the optimization + # still succeeded, space is reclaimed by a later VACUUM. logger.warning("VACUUM after FTS optimize failed: %s", exc) vacuum_ok = False - # Best-effort: fold the WAL back into the main file so the on-disk - # size settles now rather than at close(). NOTE this is REFUSED - # (SQLITE_BUSY) while any other connection holds a WAL read-mark — - # e.g. a live gateway sharing the DB — so it is not sufficient on - # its own. Callers must therefore NOT size the result by stat()ing - # the file; use :meth:`logical_size_bytes`, which is truthful - # immediately regardless of readers. - # PASSIVE, not TRUNCATE: optimize-storage runs from a transient CLI - # process; a TRUNCATE reset here would race a live gateway writer - # and tear B-tree pages (#45383). (The TRUNCATE was already refused - # SQLITE_BUSY while the gateway holds a read-mark, per the note - # above; PASSIVE removes the reset attempt entirely.) + # Best-effort WAL fold-back. REFUSED (SQLITE_BUSY) while another + # connection holds a WAL read-mark, so callers must NOT size the + # result by stat()ing the file — use :meth:`logical_size_bytes`. + # PASSIVE, not TRUNCATE: a TRUNCATE reset from this transient CLI + # process would race a live gateway writer and tear B-tree pages. try: with self._lock: self._conn.execute("PRAGMA wal_checkpoint(PASSIVE)") @@ -930,30 +757,21 @@ class SessionSearchMixin: exc, ) - # Phase 4: stamp the FTS storage layout as current, clear the "available" - # flag, and advance schema_version if it was somehow still behind (the - # main version normally advances on open now, but bump defensively so a - # DB opened only by pre-decoupling code still settles). The FTS-layout - # marker is the source of truth for "is this DB optimized". + # Phase 4: stamp the FTS layout (the source of truth for "optimized"), + # clear the "available" flag, and advance schema_version if a DB + # opened only by pre-decoupling code left it behind. def _settle(conn): # Re-check inside the write transaction so a concurrent writer # cannot race a stamp past incomplete work. Returns a refusal - # reason (stamping nothing) or None once the stamp is written. - if conn.execute( - "SELECT 1 FROM state_meta " - "WHERE key = 'fts_rebuild_high_water' LIMIT 1" - ).fetchone() is not None: + # reason (nothing stamped) or None. + if _meta_row(conn, "fts_rebuild_high_water") is not None: return "backfill_incomplete" if self._has_fts_trash(conn): return "teardown_incomplete" if self._fts_external_index_empty_with_messages(conn): return "backfill_incomplete" - conn.execute( - "INSERT INTO state_meta (key, value) VALUES ('fts_storage_version', ?) " - "ON CONFLICT(key) DO UPDATE SET value = excluded.value", - (str(FTS_STORAGE_VERSION),), - ) - conn.execute("DELETE FROM state_meta WHERE key = 'fts_optimize_available'") + self.set_meta("fts_storage_version", str(FTS_STORAGE_VERSION), cursor=conn) + _delete_meta(conn, "fts_optimize_available") conn.execute( "UPDATE schema_version SET version = ? WHERE version < ?", (SCHEMA_VERSION, SCHEMA_VERSION), @@ -961,10 +779,8 @@ class SessionSearchMixin: return None refusal = self._execute_write(_settle) if refusal is not None: - # A concurrent process re-seeded markers, left trash, or emptied - # the index between the pre-vacuum check above and this write - # transaction. Nothing was stamped. Report the failure instead of - # crashing the CLI with a traceback; a re-run can still settle. + # A concurrent process changed state since the pre-vacuum check; + # report instead of crashing the CLI — a re-run can still settle. logger.warning( "FTS storage optimization settle refused (%s)", refusal ) @@ -983,37 +799,22 @@ class SessionSearchMixin: bookend: int = 3, keep_roles: Optional[Tuple[str, ...]] = ("user", "assistant"), ) -> Dict[str, Any]: - """Return an anchored window plus session bookends. + """Anchored window (``get_messages_around``) plus session bookends. - Built on top of ``get_messages_around``. Three slices: + - ``window``: filtered to ``keep_roles``, EXCEPT the anchor itself is + always kept regardless of role. + - ``bookend_start`` / ``bookend_end``: first/last ``bookend`` messages + with ids strictly outside the window (empty when the window already + overlaps the head/tail). Empty-content rows (tool-call-only turns) + are skipped so they don't crowd out prose. - - ``window``: messages immediately surrounding the anchor. Filtered - to ``keep_roles`` (tool-response noise dropped by default), EXCEPT - the anchor itself is always preserved regardless of role. - - ``bookend_start``: first ``bookend`` user/assistant messages of the - session — but only those whose id is strictly before the window's - first message id. Empty when the window already overlaps the - session head. Empty-content messages (tool-call-only assistant - turns) are skipped so they don't crowd out actual prose openings. - - ``bookend_end``: last ``bookend`` user/assistant messages of the - session, same non-overlap rule at the tail. - - Bookends let an FTS5 hit anywhere in a long session yield the goal - (opening) and the resolution (closing) on a single call — without - loading the whole transcript. - - Returns ``{"window": [], "messages_before": 0, "messages_after": 0, - "bookend_start": [], "bookend_end": []}`` when the anchor isn't in - the session. - - ``keep_roles=None`` disables role filtering (raw window + raw - bookends). + Bookends let a hit anywhere in a long session yield the goal and the + resolution in one call. Empty slices + zero counts when the anchor + isn't in the session. ``keep_roles=None`` disables role filtering. """ if bookend < 0: bookend = 0 - # Reuse the primitive — handles anchor-existence, content decoding, - # tool_calls deserialisation, and boundary counts. primitive = self.get_messages_around( session_id, around_message_id, window=window ) @@ -1040,38 +841,28 @@ class SessionSearchMixin: window_min_id = window_rows[0]["id"] window_max_id = window_rows[-1]["id"] - # Fetch bookends only when there's room outside the window. SQL filters - # by id range, role, and non-empty content — tool-call-only assistant - # turns (content='' with tool_calls populated) are excluded so they - # don't crowd out actual prose openings/closings. bookend_start_rows: List[Any] = [] bookend_end_rows: List[Any] = [] if bookend > 0: + role_clause = "" + role_params: list = [] + if keep_roles is not None: + role_clause = f" AND role IN ({','.join('?' for _ in keep_roles)})" + role_params = list(keep_roles) + with self._read_ctx() as conn: - role_clause = "" - role_params: list = [] - if keep_roles is not None: - role_placeholders = ",".join("?" for _ in keep_roles) - role_clause = f" AND role IN ({role_placeholders})" - role_params = list(keep_roles) + def _bookend(op: str, boundary_id: int, order: str): + return conn.execute( + f"SELECT * FROM messages " + f"WHERE session_id = ? AND id {op} ?{role_clause} " + f"AND length(content) > 0 " + f"ORDER BY id {order} LIMIT ?", + (session_id, boundary_id, *role_params, bookend), + ).fetchall() - bookend_start_rows = conn.execute( - f"SELECT * FROM messages " - f"WHERE session_id = ? AND id < ?{role_clause} " - f"AND length(content) > 0 " - f"ORDER BY id ASC LIMIT ?", - (session_id, window_min_id, *role_params, bookend), - ).fetchall() - - bookend_end_rows = conn.execute( - f"SELECT * FROM messages " - f"WHERE session_id = ? AND id > ?{role_clause} " - f"AND length(content) > 0 " - f"ORDER BY id DESC LIMIT ?", - (session_id, window_max_id, *role_params, bookend), - ).fetchall() - # End rows came back DESC for the LIMIT cap; flip to ASC. - bookend_end_rows = list(reversed(bookend_end_rows)) + bookend_start_rows = _bookend("<", window_min_id, "ASC") + # End rows come back DESC for the LIMIT cap; flip to ASC. + bookend_end_rows = list(reversed(_bookend(">", window_max_id, "DESC"))) def _hydrate(row) -> Dict[str, Any]: msg = dict(row) @@ -1103,43 +894,30 @@ class SessionSearchMixin: limit: int = 20, include_inactive: bool = False, ) -> List[Dict[str, Any]]: - """Return the *limit* most-recent user messages, newest first. + """The *limit* most-recent real user turns, newest first, as + ``{id, timestamp, preview}`` (preview = first 80 chars, whitespace + collapsed). Used by /rewind and ``/undo [N]``. - Each entry is a dict with keys ``id``, ``timestamp``, ``preview``. - ``preview`` is the first 80 characters of the message content - (with line breaks collapsed to spaces). Used by the /rewind - slash command picker, CLI/TUI/gateway ``/undo [N]``, and any other - caller that needs real user-turn targets. - - Bookkeeping timeline rows (``display_kind`` set — e.g. model_switch, - async_delegation_complete, auto_continue, hidden) are excluded. They - are durable ``role='user'`` rows for the API transcript, but no client - counts them as user turns (desktop demotes them to system / drops them; - the CLI already uses ``not m.get("display_kind")``). Including them here - made ``/undo`` soft-delete from a marker instead of the last real turn — - same class of index skew as the prompt.submit ordinal bug. - - By default only active messages are returned. + Bookkeeping timeline rows (``display_kind`` set) are excluded: they + are durable ``role='user'`` rows but no client counts them as user + turns, and including them made ``/undo`` soft-delete from a marker + instead of the last real turn. Only active messages by default. """ active_clause = "" if include_inactive else " AND active = 1" - # Match CLI/desktop: only real user turns, not timeline bookkeeping. display_clause = " AND (display_kind IS NULL OR display_kind = '')" - # Legacy standalone compaction handoffs (persisted pre-#80622) are - # durable role='user' rows with NO display_kind — SQL can't see them, - # so fetch with headroom and drop them in the decode loop below. - # Without this, /undo N and rewind pair an in-memory count that - # excludes handoffs with a DB pick that includes them, soft-deleting - # the wrong turn. + # Legacy standalone compaction handoffs are role='user' rows with NO + # display_kind — SQL can't see them, so fetch with headroom and drop + # them in the decode loop; otherwise /undo N pairs an in-memory count + # that excludes handoffs with a DB pick that includes them. fetch_limit = int(limit) * 2 + 5 with self._lock: - cursor = self._conn.execute( + rows = self._conn.execute( "SELECT id, timestamp, content FROM messages " "WHERE session_id = ? AND role = 'user'" f"{active_clause}{display_clause} " "ORDER BY id DESC LIMIT ?", (session_id, fetch_limit), - ) - rows = cursor.fetchall() + ).fetchall() from agent.context_compressor import ContextCompressor @@ -1149,8 +927,7 @@ class SessionSearchMixin: break decoded = self._decode_content(row["content"]) if ContextCompressor._is_context_summary_content(decoded): - # Compaction handoff — never a user-originated turn (#80622). - continue + continue # compaction handoff — never a user-originated turn if isinstance(decoded, list): # Multimodal — flatten text parts. text_parts = [ @@ -1169,41 +946,21 @@ class SessionSearchMixin: preview = " ".join(preview.split()) # collapse whitespace if len(preview) > 80: preview = preview[:77] + "..." - result.append( - { - "id": row["id"], - "timestamp": row["timestamp"], - "preview": preview, - } - ) + result.append({"id": row["id"], "timestamp": row["timestamp"], "preview": preview}) return result @staticmethod def _sanitize_fts5_query(query: str) -> str: - """Sanitize user input for safe use in FTS5 MATCH queries. - - FTS5 has its own query syntax where characters like ``"``, ``(``, ``)``, - ``+``, ``*``, ``{``, ``}``, the column-filter operator ``:`` and bare - boolean operators (``AND``, ``OR``, ``NOT``) have special meaning. - Passing raw user input directly to MATCH can cause - ``sqlite3.OperationalError``. - - Strategy: - - Preserve properly paired quoted phrases (``"exact phrase"``) - - Strip unmatched FTS5-special characters that would cause errors - - Wrap unquoted hyphenated and dotted terms in quotes so FTS5 - matches them as exact phrases instead of splitting on the - hyphen/dot (e.g. ``chat-send``, ``P2.2``, ``my-app.config.ts``) - """ - # Cap user-controlled FTS input before any regex processing. Search - # queries do not need to be arbitrarily large, and bounding them keeps - # sanitizer/runtime behavior predictable under adversarial input. + """Sanitize user input for FTS5 MATCH (raw special characters raise + ``sqlite3.OperationalError``): preserve paired quoted phrases, strip + unmatched special characters, and quote hyphenated/dotted terms so + FTS5 matches them as phrases instead of splitting (``chat-send``, + ``P2.2``, ``my-app.config.ts``).""" + # Cap before any regex processing so adversarial input stays bounded. query = query[:MAX_FTS5_QUERY_CHARS] - # Step 1: Extract balanced double-quoted phrases and protect them - # from further processing via numbered placeholders. Do this with a - # single linear scan rather than a regex so pathological quote runs - # cannot induce backtracking. + # Step 1: protect balanced quoted phrases via numbered placeholders. + # Linear scan, not regex, so pathological quote runs cannot backtrack. _quoted_parts: list = [] pieces: list[str] = [] i = 0 @@ -1215,8 +972,7 @@ class SessionSearchMixin: continue end = query.find('"', i + 1) if end == -1: - # Unmatched quote: replace with whitespace like the old - # sanitizer's special-char stripping step. + # Unmatched quote: replace with whitespace. pieces.append(" ") i += 1 continue @@ -1226,46 +982,30 @@ class SessionSearchMixin: sanitized = "".join(pieces) - # Step 2: Strip remaining (unmatched) FTS5-special characters. ``:`` is - # FTS5's column-filter operator (``col:term``); since the FTS table has a - # single ``content`` column, an unquoted colon query like ``TODO: fix`` - # parses as ``column:term`` and raises "no such column" — swallowed at - # the execute site into zero results. Strip it like the others. - # The class below is every character FTS5's query grammar rejects - # outside a quoted phrase. Anything omitted here reaches MATCH raw and - # raises, which the execute site swallows into zero results — the - # failure mode this step exists to prevent. Measured against a real - # FTS5 table: ``it's``, ``gateway/run.py``, ``user@host``, ``a,b`` and - # ``50%`` all raised before the class was completed. + # Step 2: strip remaining FTS5-special characters (see + # _FTS5_SPECIAL_CHARS); e.g. an unquoted ``TODO: fix`` parses as + # ``column:term`` and raises "no such column". sanitized = _FTS5_SPECIAL_RE.sub(" ", sanitized) - # Step 2b: ``%`` is excluded from the class above only to protect the - # CJK LIKE-fallback path (LIKE treats % as a wildcard the fallback - # builds itself). A non-CJK query never reaches that fallback - # (``is_cjk`` gates it), so ``50%`` would sail into MATCH raw and - # raise like the rest. Strip it whenever the query has no CJK. + # Step 2b: ``%`` is only spared for the CJK LIKE fallback; a non-CJK + # query never reaches it, so ``50%`` would hit MATCH raw and raise. if "%" in sanitized and not SessionSearchMixin._contains_cjk(sanitized): sanitized = sanitized.replace("%", " ") - # Step 3: Collapse repeated * (e.g. "***") into a single one, - # and remove leading * (prefix-only needs at least one char before *) + # Step 3: collapse repeated * and drop leading * (prefix needs a char). sanitized = re.sub(r"\*+", "*", sanitized) sanitized = re.sub(r"(^|\s)\*", r"\1", sanitized) - # Step 4: Remove dangling boolean operators at start/end that would - # cause syntax errors (e.g. "hello AND" or "OR world") + # Step 4: drop dangling boolean operators at start/end (syntax errors). sanitized = re.sub(r"(?i)^(AND|OR|NOT)\b\s*", "", sanitized.strip()) sanitized = re.sub(r"(?i)\s+(AND|OR|NOT)\s*$", "", sanitized.strip()) - # Step 5: Wrap unquoted dotted and/or hyphenated terms in double - # quotes. FTS5's tokenizer splits on dots and hyphens, turning - # ``chat-send`` into ``chat AND send`` and ``P2.2`` into ``p2 AND 2``. - # Quoting preserves phrase semantics. A single pass avoids the - # double-quoting bug that would occur if dotted, hyphenated and underscored - # patterns were applied sequentially (e.g. ``my-app.config``). + # Step 5: quote dotted/hyphenated/underscored terms in ONE pass (the + # tokenizer splits on them; sequential passes double-quote + # ``my-app.config``). sanitized = re.sub(r"\b(\w+(?:[._-]\w+)+)\b", r'"\1"', sanitized) - # Step 6: Restore preserved quoted phrases + # Step 6: restore preserved quoted phrases. for i, quoted in enumerate(_quoted_parts): sanitized = sanitized.replace(f"\x00Q{i}\x00", quoted) @@ -1284,17 +1024,7 @@ class SessionSearchMixin: @staticmethod def _contains_cjk(text: str) -> bool: """Check if text contains CJK (Chinese, Japanese, Korean) characters.""" - for ch in text: - cp = ord(ch) - if (0x4E00 <= cp <= 0x9FFF or # CJK Unified Ideographs - 0x3400 <= cp <= 0x4DBF or # CJK Extension A - 0x20000 <= cp <= 0x2A6DF or # CJK Extension B - 0x3000 <= cp <= 0x303F or # CJK Symbols - 0x3040 <= cp <= 0x309F or # Hiragana - 0x30A0 <= cp <= 0x30FF or # Katakana - 0xAC00 <= cp <= 0xD7AF): # Hangul Syllables - return True - return False + return any(SessionSearchMixin._is_cjk_codepoint(ord(ch)) for ch in text) @classmethod def _count_cjk(cls, text: str) -> int: @@ -1303,12 +1033,9 @@ class SessionSearchMixin: @classmethod def _has_lone_cjk_run(cls, query: str) -> bool: - """True when any maximal CJK run in the query is a single char. - - The cjk-bigram index stores bigrams for runs >=2 chars and unigrams - only for isolated chars, so a 1-char CJK term can't match inside - longer runs there — those queries keep the LIKE substring route. - """ + """True when any maximal CJK run in the query is a single char: the + cjk-bigram index stores unigrams only for isolated chars, so such a + term can't match inside longer runs — those queries keep LIKE.""" run = 0 for ch in query: if cls._is_cjk_codepoint(ord(ch)): @@ -1321,21 +1048,25 @@ class SessionSearchMixin: @staticmethod def _trigram_eligible_tokens(query: str) -> bool: - """True when every non-operator token is long enough for the trigram - tokenizer to match (>=3 chars). - - The trigram tokenizer indexes overlapping 3-character sequences, so a - token shorter than 3 chars produces no trigrams and can never match. - With FTS5's implicit-AND between tokens, a single short token makes the - whole MATCH return nothing, so the trigram path is only worth taking - when every searchable token qualifies. - """ + """True when every non-operator token is >=3 chars: a shorter token + produces no trigrams, and with FTS5's implicit AND one such token + makes the whole MATCH return nothing.""" tokens = [ - t for t in query.strip('"').strip().split() - if t.upper() not in {"AND", "OR", "NOT"} + t for t in query.strip('"').strip().split() if t.upper() not in _FTS_OPERATORS ] return bool(tokens) and all(len(t) >= 3 for t in tokens) + @classmethod + def _has_short_cjk_token(cls, raw_query: str) -> bool: + """True when any non-operator CJK token has fewer than 3 CJK chars — + the trigram tokenizer needs >=3 per token, so such a query returns + nothing there and must take the LIKE route.""" + return any( + cls._count_cjk(t) < 3 + for t in raw_query.split() + if t.upper() not in _FTS_OPERATORS and cls._contains_cjk(t) + ) + def _run_trigram_search( self, raw_query: str, @@ -1349,63 +1080,19 @@ class SessionSearchMixin: limit: int = 20, offset: int = 0, ) -> Optional[List[Dict[str, Any]]]: - """Run a search against a substring-capable FTS index. - - ``table`` is ``messages_fts_trigram`` (default) or - ``messages_fts_cjk``. The trigram tokenizer indexes overlapping - 3-byte sequences, so it matches substrings regardless of word - boundaries — both CJK phrases the unicode61 tokenizer splits into - single characters and Latin runs the unicode61 tokenizer fuses onto - adjacent CJK (e.g. ``修改youer服务端``). The cjk-bigram tokenizer - splits Latin runs off adjacent CJK, giving the same recovery as an - exact ranked token match. Each non-operator token is quoted to - neutralise FTS5 special characters while boolean operators - (AND/OR/NOT) are preserved. - - Returns the matching rows, or ``None`` when the query cannot be - executed (e.g. the tokenizer is unavailable at runtime) so the - caller can fall back to another strategy. - """ - tokens = raw_query.split() - parts = [] - for tok in tokens: - if tok.upper() in {"AND", "OR", "NOT"}: - parts.append(tok) - else: - parts.append('"' + tok.replace('"', '""') + '"') - trigram_query = " ".join(parts) - tri_where = [f"{table} MATCH ?"] - tri_params: list = [trigram_query] - if not include_inactive: - tri_where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - tri_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - tri_params.extend(source_filter) - if exclude_sources is not None: - tri_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - tri_params.extend(exclude_sources) - if role_filter: - tri_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - tri_params.extend(role_filter) - tri_sql = f""" - SELECT - m.id, - m.session_id, - m.role, - snippet({table}, -1, '>>>', '<<<', '...', 40) AS snippet, - m.timestamp, - m.tool_name, - s.source, - s.model, - s.started_at AS session_started - FROM {table} - JOIN messages m ON m.id = {table}.rowid - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(tri_where)} - {order_by_sql} - LIMIT ? OFFSET ? - """ - tri_params.extend([limit, offset]) + """Search a substring-capable index (``messages_fts_trigram`` or + ``messages_fts_cjk``): trigram matches substrings regardless of word + boundaries (CJK phrases unicode61 splits, Latin runs it fuses onto + adjacent CJK like ``修改youer服务端``); cjk-bigram splits Latin runs + off CJK for an exact ranked match. Returns ``None`` when the query + cannot execute (e.g. tokenizer unavailable) so the caller can fall + back.""" + tri_sql, tri_params = self._fts_match_sql( + table, _quote_fts_tokens(raw_query), order_by_sql, + include_inactive=include_inactive, source_filter=source_filter, + exclude_sources=exclude_sources, role_filter=role_filter, + limit=limit, offset=offset, + ) with self._read_ctx() as conn: try: tri_cursor = conn.execute(tri_sql, tri_params) @@ -1414,6 +1101,41 @@ class SessionSearchMixin: return None return [dict(row) for row in tri_cursor.fetchall()] + @staticmethod + def _fts_match_sql( + table: str, + match_query: str, + order_by_sql: str, + *, + include_inactive: bool, + source_filter: Optional[List[str]], + exclude_sources: Optional[List[str]], + role_filter: Optional[List[str]], + limit: int, + offset: int, + ) -> Tuple[str, list]: + """MATCH query + params against one FTS5 index joined to messages/sessions.""" + where = [f"{table} MATCH ?"] + params: list = [match_query] + _search_filter_clauses( + where, params, include_inactive=include_inactive, + source_filter=source_filter, exclude_sources=exclude_sources, + role_filter=role_filter, + ) + params.extend([limit, offset]) + sql = f""" + SELECT m.id, m.session_id, m.role, + snippet({table}, -1, '>>>', '<<<', '...', 40) AS snippet, + {_SEARCH_SELECT_TAIL} + FROM {table} + JOIN messages m ON m.id = {table}.rowid + JOIN sessions s ON s.id = m.session_id + WHERE {' AND '.join(where)} + {order_by_sql} + LIMIT ? OFFSET ? + """ + return sql, params + def search_messages( self, query: str, @@ -1426,14 +1148,10 @@ class SessionSearchMixin: include_inactive: bool = False, fields: Optional[Collection[str]] = None, ) -> List[Dict[str, Any]]: - """Instrumented wrapper around :meth:`_search_messages_impl`. - - Logs one line per slow search with the routing path taken, so - production latency stays attributable per query shape (the 2026-07 - session_search investigation needed trace archaeology to discover - the LIKE full scans; this makes the next regression a grep). - Threshold: HERMES_SEARCH_SLOW_MS (default 1000; 0 logs every call). - """ + """Instrumented wrapper around :meth:`_search_messages_impl`: logs one + line per slow search with the routing path taken so latency stays + attributable per query shape. Threshold HERMES_SEARCH_SLOW_MS + (default 1000; 0 logs every call).""" started = time.time() rows = None try: @@ -1477,12 +1195,11 @@ class SessionSearchMixin: raw = sanitized.strip('"').strip() if self._fts_cjk_available and not self._has_lone_cjk_run(raw): return "fts_cjk" - tokens = [ - t for t in raw.split() - if t.upper() not in {"AND", "OR", "NOT"} and self._contains_cjk(t) - ] - short = any(self._count_cjk(t) < 3 for t in tokens) - if self._count_cjk(raw) >= 3 and not short and self._trigram_available: + if ( + self._count_cjk(raw) >= 3 + and not self._has_short_cjk_token(raw) + and self._trigram_available + ): return "trigram" return "like_scan" except Exception: @@ -1492,13 +1209,9 @@ class SessionSearchMixin: def _compile_like_boolean_query( query: str, ) -> Tuple[str, List[Any], Optional[str]]: - """Compile the supported FTS boolean subset into LIKE predicates. - - Terms within an OR group are ANDed by default, matching FTS5's - implicit conjunction. ``NOT`` negates the following term inside that - group instead of being discarded, so ``python NOT java`` becomes a - positive Python match plus a Java exclusion. - """ + """Compile the supported FTS boolean subset into LIKE predicates: + terms within an OR group are ANDed (FTS5's implicit conjunction) and + ``NOT`` negates the following term rather than being discarded.""" groups: List[List[Tuple[str, bool]]] = [[]] negate_next = False for raw_token in re.findall(r'"[^"]+"|\S+', query): @@ -1527,18 +1240,13 @@ class SessionSearchMixin: continue clauses: List[str] = [] for term, negated in group: - escaped = ( - term.replace("\\", "\\\\") - .replace("%", "\\%") - .replace("_", "\\_") - ) clause = ( "(COALESCE(m.content, '') LIKE ? ESCAPE '\\' OR " "COALESCE(m.tool_name, '') LIKE ? ESCAPE '\\' OR " "COALESCE(m.tool_calls, '') LIKE ? ESCAPE '\\')" ) clauses.append(f"NOT {clause}" if negated else clause) - params.extend([f"%{escaped}%"] * 3) + params.extend([f"%{_escape_like(term)}%"] * 3) if snippet_term is None and not negated: snippet_term = term compiled_groups.append(f"({' AND '.join(clauses)})") @@ -1563,52 +1271,45 @@ class SessionSearchMixin: return [] where = [f"({predicate})"] - if not include_inactive: - where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - params.extend(source_filter) - if exclude_sources is not None: - where.append( - f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})" - ) - params.extend(exclude_sources) - if role_filter: - where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - params.extend(role_filter) - + _search_filter_clauses( + where, params, include_inactive=include_inactive, + source_filter=source_filter, exclude_sources=exclude_sources, + role_filter=role_filter, + ) order = ( "ASC" if isinstance(sort, str) and sort.strip().lower() == "oldest" else "DESC" ) + return self._like_rows( + where, [snippet_term, *params, limit, offset], + order_by=f"ORDER BY m.timestamp {order}, m.id {order}", limit_sql="LIMIT ? OFFSET ?", + ) + + def _like_rows( + self, where: List[str], params: list, *, order_by: str, limit_sql: str + ) -> List[Dict[str, Any]]: + """Canonical-table LIKE scan; ``params[0]`` is the snippet anchor term.""" sql = f""" SELECT m.id, m.session_id, m.role, - substr(m.content, max(1, instr(m.content, ?) - 40), 120) AS snippet, - m.timestamp, m.tool_name, - s.source, s.model, s.started_at AS session_started + {_LIKE_SNIPPET_SQL}, + {_SEARCH_SELECT_TAIL} FROM messages m JOIN sessions s ON s.id = m.session_id WHERE {' AND '.join(where)} - ORDER BY m.timestamp {order}, m.id {order} - LIMIT ? OFFSET ? + {order_by} + {limit_sql} """ - with self._read_ctx() as conn: - rows = conn.execute( - sql, [snippet_term, *params, limit, offset] - ).fetchall() - return [dict(row) for row in rows] + return [dict(row) for row in self._read_all(sql, params)] def _refresh_fts_stale_state(self) -> None: """Observe fail-open initiated by another process sharing state.db.""" if self._fts_stale or not self._fts_enabled: return try: - with self._read_ctx() as conn: - stale = conn.execute( - "SELECT 1 FROM state_meta WHERE key = ? LIMIT 1", - (FTS_STALE_KEY,), - ).fetchone() + stale = self._read_one( + "SELECT 1 FROM state_meta WHERE key = ? LIMIT 1", (FTS_STALE_KEY,) + ) except sqlite3.Error: return if stale is not None: @@ -1622,13 +1323,9 @@ class SessionSearchMixin: matches: List[Dict[str, Any]], result_fields: Optional[Collection[str]] = None, ) -> List[Dict[str, Any]]: - """Attach neighboring messages and trim full content from results. - - Context (1 message before + after each match) is only loaded when - the selected result projection consumes it. Each query takes its - own fresh read transaction via _read_ctx, so we never hold a lock - across N sequential queries. - """ + """Attach neighboring messages (1 before + after, only when the + projection consumes ``context``) and trim full content. Each context + query takes its own read transaction, never a lock across N queries.""" context_matches = ( matches if result_fields is None or "context" in result_fields else () ) @@ -1691,12 +1388,8 @@ class SessionSearchMixin: except Exception: match["context"] = [] - # Full message content is never selected by any search route: every - # SELECT returns snippet + metadata only (saves I/O on multi-MB tool - # rows and the tokens a content column would cost downstream). The - # context query above re-fetches its 3-message window by id, so - # nothing reads content from the match rows themselves. The pop stays - # as a belt-and-braces guard for any future route that selects it. + # No search route selects full content (snippet + metadata only); the + # pop is a guard for any future route that does. for match in matches: match.pop("content", None) @@ -1720,36 +1413,19 @@ class SessionSearchMixin: include_inactive: bool = False, fields: Optional[Collection[str]] = None, ) -> List[Dict[str, Any]]: - """ - Full-text search across session messages using FTS5. + """FTS5 search across session messages (keywords, ``"phrases"``, + AND/OR/NOT, ``prefix*``). - Supports FTS5 query syntax: - - Simple keywords: "docker deployment" - - Phrases: '"exact phrase"' - - Boolean: "docker OR kubernetes", "python NOT java" - - Prefix: "deploy*" + Returns snippet + session metadata + 1-message context per hit; + ``fields`` selects a projection (context is only loaded when it + consumes it). ``sort``: None = BM25 rank only; "newest"/"oldest" = + timestamp then rank. The short-CJK LIKE fallback orders by timestamp + DESC and ignores ``sort``. - Returns matching messages with session metadata, content snippet, - and surrounding context (1 message before and after the match). - ``fields`` selects a result projection; omitting it preserves the - complete legacy result. Context is only loaded when that projection - consumes it. - - ``sort`` controls temporal ordering: - - ``None`` (default): FTS5 BM25 relevance only. Time-neutral. - - ``"newest"``: order by message timestamp DESC, then by rank. - - ``"oldest"``: order by message timestamp ASC, then by rank. - - The short-CJK LIKE fallback already orders by timestamp DESC and - ignores ``sort``. The trigram CJK path honours ``sort`` like the main - FTS5 path. - - Rewound (``active=0``, ``compacted=0``) rows are excluded by default — - the user took those back. Compaction-archived rows (``active=0``, - ``compacted=1``) ARE included by default: they were summarized away from - the live context but remain part of the conversation's record, so the - pre-compaction transcript stays discoverable after in-place compaction - (#38763). Pass ``include_inactive=True`` to search every row regardless. + Rewound rows (``active=0, compacted=0``) are excluded by default; + compaction-archived rows (``compacted=1``) ARE included so the + pre-compaction transcript stays discoverable. ``include_inactive`` + searches every row. """ result_fields = self._search_message_fields(fields) @@ -1760,27 +1436,21 @@ class SessionSearchMixin: if not query: return [] + filters = dict( + include_inactive=include_inactive, source_filter=source_filter, + exclude_sources=exclude_sources, role_filter=role_filter, + ) self._refresh_fts_stale_state() if self._fts_stale: matches = self._search_messages_like_fallback( - query, - source_filter=source_filter, - exclude_sources=exclude_sources, - role_filter=role_filter, - limit=limit, - offset=offset, - sort=sort, - include_inactive=include_inactive, - ) - return self._finalize_search_matches( - matches, result_fields=result_fields + query, limit=limit, offset=offset, sort=sort, **filters ) + return self._finalize_search_matches(matches, result_fields=result_fields) if not self._fts_enabled: return [] - # Normalise sort. Anything not in the allowed set falls back to None - # (FTS5 rank-only) so callers can pass through user input without - # validation. + # Normalise sort; anything unknown falls back to rank-only so callers + # can pass through user input. if isinstance(sort, str): sort_norm = sort.strip().lower() if sort_norm not in ("newest", "oldest"): @@ -1788,8 +1458,6 @@ class SessionSearchMixin: else: sort_norm = None - # ORDER BY shared across the main FTS5 path and trigram CJK path. - # With sort set, timestamp is primary and rank is the tiebreaker. if sort_norm == "newest": order_by_sql = "ORDER BY m.timestamp DESC, rank" elif sort_norm == "oldest": @@ -1797,157 +1465,44 @@ class SessionSearchMixin: else: order_by_sql = "ORDER BY rank" - # Build WHERE clauses dynamically - where_clauses = ["messages_fts MATCH ?"] - params: list = [query] - if not include_inactive: - # Live rows (active=1) AND compaction-archived rows (compacted=1) - # are discoverable; only rewind/undo rows (active=0, compacted=0) - # are hidden. See archive_and_compact() / #38763. - where_clauses.append("(m.active = 1 OR m.compacted = 1)") - - if source_filter is not None: - source_placeholders = ",".join("?" for _ in source_filter) - where_clauses.append(f"s.source IN ({source_placeholders})") - params.extend(source_filter) - - if exclude_sources is not None: - exclude_placeholders = ",".join("?" for _ in exclude_sources) - where_clauses.append(f"s.source NOT IN ({exclude_placeholders})") - params.extend(exclude_sources) - - if role_filter: - role_placeholders = ",".join("?" for _ in role_filter) - where_clauses.append(f"m.role IN ({role_placeholders})") - params.extend(role_filter) - - where_sql = " AND ".join(where_clauses) - params.extend([limit, offset]) - - sql = f""" - SELECT - m.id, - m.session_id, - m.role, - snippet(messages_fts, -1, '>>>', '<<<', '...', 40) AS snippet, - m.timestamp, - m.tool_name, - s.source, - s.model, - s.started_at AS session_started - FROM messages_fts - JOIN messages m ON m.id = messages_fts.rowid - JOIN sessions s ON s.id = m.session_id - WHERE {where_sql} - {order_by_sql} - LIMIT ? OFFSET ? - """ - - # CJK queries bypass the unicode61 FTS5 table. The default tokenizer - # splits CJK characters into individual tokens, so "大别山项目" becomes - # "大 AND 别 AND 山 AND 项 AND 目" — producing false positives and - # missing exact phrase matches. - # - # For queries with 3+ CJK characters, we use the trigram FTS5 table - # (indexed substring matching with ranking and snippets). For shorter - # CJK queries (1-2 chars), trigram can't match (it needs ≥9 UTF-8 - # bytes = 3 CJK chars), so we fall back to LIKE. + # CJK queries bypass the unicode61 table, whose tokenizer splits CJK + # into single characters ("大别山项目" -> "大 AND 别 AND ...": false + # positives, missed phrases). 3+ CJK chars -> trigram; shorter -> + # LIKE (trigram needs 9 UTF-8 bytes = 3 CJK chars). matches: List[Dict[str, Any]] = [] is_cjk = self._contains_cjk(query) if is_cjk: raw_query = query.strip('"').strip() - cjk_count = self._count_cjk(raw_query) - - # Per-token CJK length check (#20494): trigram needs >=3 CJK chars - # per token. A query like "广西 OR 桂林 OR 漓江" has cjk_count=6 - # (>=3) but each individual token is only 2 chars — trigram returns 0. - # Route to LIKE when any non-operator CJK token is <3 CJK chars. - _tokens_for_check = [ - t for t in raw_query.split() - if t.upper() not in {"AND", "OR", "NOT"} and self._contains_cjk(t) - ] - _any_short_cjk = any( - self._count_cjk(t) < 3 for t in _tokens_for_check - ) - _trigram_succeeded = False - # Tool rows are excluded from the trigram index (they're ~90% of - # message bytes and machine noise — see FTS_TRIGRAM_SQL). A CJK - # query explicitly filtering on role='tool' must therefore use - # the LIKE fallback, which scans the base table directly. + # Tool rows are excluded from the trigram/cjk indexes (see + # FTS_TRIGRAM_SQL), so a role='tool' CJK query must use LIKE. _wants_tool_rows = bool(role_filter) and "tool" in role_filter - # ── CJK-bigram route (messages_fts_cjk, cjk_unicode61) ────── - # When the bigram index is available it serves EVERY CJK query - # shape the legacy code split between trigram (>=3 chars/token) - # and LIKE full scans (1-2 char tokens) — the whole point of the - # index (PR #65544). Exceptions stay on the legacy routes: - # - role_filter=['tool'] queries (tool rows aren't in the cjk - # index, same exclusion as trigram), - # - queries containing a LONE 1-char CJK run: the index stores - # bigrams for runs >=2, so a single-char term can only match - # isolated chars — LIKE substring semantics are broader. + # CJK-bigram route: serves every CJK shape the legacy code split + # between trigram and LIKE full scans, except role='tool' queries + # and LONE 1-char CJK runs (the index stores bigrams for runs >=2, + # so a single-char term only matches isolated chars — LIKE is broader). if ( self._fts_cjk_available and not _wants_tool_rows and not self._has_lone_cjk_run(raw_query) ): - tokens = raw_query.split() - parts = [] - for tok in tokens: - if tok.upper() in {"AND", "OR", "NOT"}: - parts.append(tok) - else: - parts.append('"' + tok.replace('"', '""') + '"') - cjk_query = " ".join(parts) - cjk_where = ["messages_fts_cjk MATCH ?"] - cjk_params: list = [cjk_query] - if not include_inactive: - cjk_where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - cjk_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - cjk_params.extend(source_filter) - if exclude_sources is not None: - cjk_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - cjk_params.extend(exclude_sources) - if role_filter: - cjk_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - cjk_params.extend(role_filter) - cjk_sql = f""" - SELECT - m.id, - m.session_id, - m.role, - snippet(messages_fts_cjk, -1, '>>>', '<<<', '...', 40) AS snippet, - m.timestamp, - m.tool_name, - s.source, - s.model, - s.started_at AS session_started - FROM messages_fts_cjk - JOIN messages m ON m.id = messages_fts_cjk.rowid - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(cjk_where)} - {order_by_sql} - LIMIT ? OFFSET ? - """ - cjk_params.extend([limit, offset]) + cjk_sql, cjk_params = self._fts_match_sql( + "messages_fts_cjk", _quote_fts_tokens(raw_query), order_by_sql, + limit=limit, offset=offset, **filters, + ) try: - with self._read_ctx() as conn: - cjk_cursor = conn.execute(cjk_sql, cjk_params) - matches = [dict(row) for row in cjk_cursor.fetchall()] - _trigram_succeeded = True + matches = [dict(row) for row in self._read_all(cjk_sql, cjk_params)] + _trigram_succeeded = True except sqlite3.OperationalError: - # Tokenizer missing on this connection / query syntax — - # the trigram + LIKE routes below still answer. + # Tokenizer missing / query syntax — trigram + LIKE still answer. logger.debug( "messages_fts_cjk query failed; falling back to " "trigram/LIKE", exc_info=True, ) except sqlite3.DatabaseError as exc: - # A full-message rebuild is unbounded and holds the writer - # lock, so a live search never performs one. Detach the - # derived indexes and answer from canonical rows instead. + # A live search never performs the unbounded full rebuild: + # detach the derived indexes and answer from canonical rows. # Non-FTS corruption is not safe to reinterpret here. if not self._enter_fts_fail_open(exc): raise @@ -1957,69 +1512,29 @@ class SessionSearchMixin: exc, ) + # Per-token CJK length check: trigram needs >=3 CJK chars per + # token. "广西 OR 桂林 OR 漓江" has 6 CJK chars total but 2 per + # token — trigram returns 0, so such queries take LIKE. if ( not _trigram_succeeded - and cjk_count >= 3 - and not _any_short_cjk + and self._count_cjk(raw_query) >= 3 + and not self._has_short_cjk_token(raw_query) and self._trigram_available and not _wants_tool_rows ): - # Trigram FTS5 path — quote each non-operator token to handle - # FTS5 special chars (%, *, etc.) while preserving boolean - # operators (AND, OR, NOT) for multi-term queries. - tokens = raw_query.split() - parts = [] - for tok in tokens: - if tok.upper() in {"AND", "OR", "NOT"}: - parts.append(tok) - else: - parts.append('"' + tok.replace('"', '""') + '"') - trigram_query = " ".join(parts) - tri_where = ["messages_fts_trigram MATCH ?"] - tri_params: list = [trigram_query] - if not include_inactive: - tri_where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - tri_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - tri_params.extend(source_filter) - if exclude_sources is not None: - tri_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - tri_params.extend(exclude_sources) - if role_filter: - tri_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - tri_params.extend(role_filter) - tri_sql = f""" - SELECT - m.id, - m.session_id, - m.role, - snippet(messages_fts_trigram, -1, '>>>', '<<<', '...', 40) AS snippet, - m.timestamp, - m.tool_name, - s.source, - s.model, - s.started_at AS session_started - FROM messages_fts_trigram - JOIN messages m ON m.id = messages_fts_trigram.rowid - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(tri_where)} - {order_by_sql} - LIMIT ? OFFSET ? - """ - tri_params.extend([limit, offset]) + tri_sql, tri_params = self._fts_match_sql( + "messages_fts_trigram", _quote_fts_tokens(raw_query), order_by_sql, + limit=limit, offset=offset, **filters, + ) try: - with self._read_ctx() as conn: - tri_cursor = conn.execute(tri_sql, tri_params) - matches = [dict(row) for row in tri_cursor.fetchall()] - _trigram_succeeded = True + matches = [dict(row) for row in self._read_all(tri_sql, tri_params)] + _trigram_succeeded = True except sqlite3.OperationalError: # Trigram query failed at runtime — fall through to LIKE. pass except sqlite3.DatabaseError as exc: - # Preserve the same bounded recovery contract as the CJK - # and main FTS paths: detach derived indexes, then fall - # through to the canonical LIKE query. A non-FTS storage - # error remains fatal rather than being hidden as a miss. + # Same bounded recovery as the CJK/main paths; a non-FTS + # storage error stays fatal rather than hidden as a miss. if not self._enter_fts_fail_open(exc): raise logger.warning( @@ -2028,163 +1543,81 @@ class SessionSearchMixin: exc, ) if not _trigram_succeeded: - # Short / mixed CJK query, trigram unavailable, or trigram - # <3 CJK chars. Fall back to LIKE substring search. - # For multi-token OR queries (e.g. "广西 OR 桂林 OR 漓江"), - # build one LIKE condition per non-operator token so each term - # is matched independently (#20494). + # LIKE substring fallback; one clause per non-operator token so + # "广西 OR 桂林 OR 漓江" matches each term independently. non_op_tokens = [ - t for t in raw_query.split() - if t.upper() not in {"AND", "OR", "NOT"} + t for t in raw_query.split() if t.upper() not in _FTS_OPERATORS ] or [raw_query] - token_clauses = [] like_params: list = [] for tok in non_op_tokens: - esc = _escape_like(tok) - token_clauses.append( - "(m.content LIKE ? ESCAPE '\\' OR m.tool_name LIKE ? ESCAPE '\\' OR m.tool_calls LIKE ? ESCAPE '\\')" - ) - like_params += [f"%{esc}%", f"%{esc}%", f"%{esc}%"] - like_where = [f"({' OR '.join(token_clauses)})"] - if not include_inactive: - # Same visibility rule as the FTS5 paths: live rows and - # compaction-archived rows are discoverable; rewind/undo - # rows (active=0, compacted=0) are hidden (#38763). - like_where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - like_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - like_params.extend(source_filter) - if exclude_sources is not None: - like_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - like_params.extend(exclude_sources) - if role_filter: - like_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - like_params.extend(role_filter) - like_sql = f""" - SELECT m.id, m.session_id, m.role, - substr(m.content, - max(1, instr(m.content, ?) - 40), - 120) AS snippet, - m.timestamp, m.tool_name, - s.source, s.model, s.started_at AS session_started - FROM messages m - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(like_where)} - ORDER BY m.timestamp DESC - LIMIT ? OFFSET ? - """ - like_params.extend([limit, offset]) + like_params += [f"%{_escape_like(tok)}%"] * 3 + like_where = [ + f"({' OR '.join([_LIKE_ANY_COLUMN_SQL] * len(non_op_tokens))})" + ] + _search_filter_clauses(like_where, like_params, **filters) # instr() for snippet uses first search token - like_params = [non_op_tokens[0]] + like_params - with self._read_ctx() as conn: - like_cursor = conn.execute(like_sql, like_params) - matches = [dict(row) for row in like_cursor.fetchall()] + matches = self._like_rows( + like_where, [non_op_tokens[0], *like_params, limit, offset], + order_by="ORDER BY m.timestamp DESC", limit_sql="LIMIT ? OFFSET ?", + ) else: + sql, params = self._fts_match_sql( + "messages_fts", query, order_by_sql, limit=limit, offset=offset, **filters + ) try: - with self._read_ctx() as conn: - cursor = conn.execute(sql, params) - matches = [dict(row) for row in cursor.fetchall()] + matches = [dict(row) for row in self._read_all(sql, params)] except sqlite3.OperationalError: # FTS5 query syntax error despite sanitization — return empty return [] except sqlite3.DatabaseError as exc: - # A corrupt FTS index raises the malformed / "fts5: corrupt - # structure record" class on the MATCH read, the same class the - # write path handles (#66296). OperationalError (query syntax) - # is a subclass caught above; this arm is the corruption - # parent. Live search must remain bounded, so detach the - # derived indexes and answer from canonical message rows. The - # existing stale-open/repair paths retain rebuild ownership. + # Corruption parent class (OperationalError is caught above). + # Live search must stay bounded: detach the derived indexes and + # answer from canonical rows; repair paths own the rebuild. if not self._enter_fts_fail_open(exc): raise matches = self._search_messages_like_fallback( - query, - source_filter=source_filter, - exclude_sources=exclude_sources, - role_filter=role_filter, - limit=limit, - offset=offset, - sort=sort, - include_inactive=include_inactive, + query, limit=limit, offset=offset, sort=sort, **filters ) - # Deferred-rebuild supplement (schema v23): while the background - # backfill is pending, the FTS indexes only cover rows outside the - # (progress, high_water] gap. Top the results up with a bounded LIKE - # scan over just that id range so search never silently loses old - # messages mid-rebuild. The range shrinks as the backfill advances, - # so this cost decays to zero. The CJK LIKE-fallback path above - # already scans the whole base table and needs no supplement. + # Deferred-rebuild supplement: while the backfill is pending the FTS + # indexes miss the (progress, high_water] gap; top up with a bounded + # LIKE scan over that range so old messages never silently vanish + # mid-rebuild. The cost decays to zero as the backfill advances. rebuild_status = self.fts_rebuild_status() if rebuild_status is not None and len(matches) < limit: try: gap_matches = self._search_unindexed_gap( - query, - limit - len(matches), - include_inactive=include_inactive, - source_filter=source_filter, - exclude_sources=exclude_sources, - role_filter=role_filter, + query, limit - len(matches), **filters ) seen_ids = {m["id"] for m in matches} matches.extend(m for m in gap_matches if m["id"] not in seen_ids) except sqlite3.OperationalError as exc: logger.debug("Unindexed-gap supplement skipped: %s", exc) - # Pure-Latin queries run against the unicode61 ``messages_fts`` table, - # whose tokenizer does not insert a boundary between Latin letters and - # adjacent CJK characters: "修改youer服务端" is indexed as one token, - # so MATCH "youer" finds nothing even though the substring is present - # (#54242). When the exact-token search returns nothing, retry on the - # substring-capable indexes. Preference order: - # 1. messages_fts_cjk (when built): its tokenizer splits Latin runs - # off adjacent CJK, so "youer" is an exact ranked token match. - # 2. messages_fts_trigram: substring matching, needs >=3-char - # tokens (shorter tokens produce no trigrams). - # Gated on a zero-result miss so successful Latin searches keep their - # unicode61 ranking — strictly additive, never reorders existing - # hits. Trade-off on the trigram leg: any zero-result Latin query - # gains substring semantics (e.g. "cat" can then match - # "concatenate"). Genuinely absent terms still return []. Skipped for - # role_filter=['tool'] queries — both fallback indexes exclude tool - # rows (v23), so a retry could never add hits. + # unicode61 puts no boundary between Latin and adjacent CJK + # ("修改youer服务端" is one token, so MATCH "youer" misses). On a + # zero-result Latin miss retry the substring-capable indexes: cjk + # first (splits Latin off CJK: exact ranked match), then trigram + # (needs >=3-char tokens). Gated on a miss so successful searches keep + # their ranking; trade-off: "cat" may then match "concatenate". + # Skipped for role='tool' (both indexes exclude tool rows). if ( not matches and not is_cjk and not (bool(role_filter) and "tool" in role_filter) ): _fb_query = query.strip('"').strip() + fb_kwargs = dict(order_by_sql=order_by_sql, limit=limit, offset=offset, **filters) if self._fts_cjk_available: - cjk_fb = self._run_trigram_search( - _fb_query, - table="messages_fts_cjk", - order_by_sql=order_by_sql, - include_inactive=include_inactive, - source_filter=source_filter, - exclude_sources=exclude_sources, - role_filter=role_filter, - limit=limit, - offset=offset, - ) - if cjk_fb: - matches = cjk_fb + matches = self._run_trigram_search( + _fb_query, table="messages_fts_cjk", **fb_kwargs + ) or matches if ( not matches and self._trigram_available and self._trigram_eligible_tokens(query) ): - tri_matches = self._run_trigram_search( - _fb_query, - order_by_sql=order_by_sql, - include_inactive=include_inactive, - source_filter=source_filter, - exclude_sources=exclude_sources, - role_filter=role_filter, - limit=limit, - offset=offset, - ) - if tri_matches: - matches = tri_matches + matches = self._run_trigram_search(_fb_query, **fb_kwargs) or matches return self._finalize_search_matches(matches, result_fields=result_fields) @@ -2198,68 +1631,37 @@ class SessionSearchMixin: exclude_sources: Optional[List[str]] = None, role_filter: Optional[List[str]] = None, ) -> List[Dict[str, Any]]: - """LIKE-scan the rows the deferred rebuild hasn't indexed yet. - - Only touches ids in (fts_rebuild_progress, fts_rebuild_high_water] — - a range that shrinks to nothing as the backfill advances. The FTS - query is degraded to per-token substring terms (AND-joined; quoted - phrases kept whole), which is deliberately recall-over-precision: - temporary results beat silently missing ones mid-rebuild. - """ + """LIKE-scan ids in (fts_rebuild_progress, fts_rebuild_high_water] — + the rows the deferred rebuild hasn't indexed yet. The FTS query is + degraded to AND-joined substring terms (quoted phrases kept whole): + deliberately recall-over-precision mid-rebuild.""" status = self.fts_rebuild_status() if status is None or limit <= 0: return [] progress, high_water = status["indexed"], status["total"] - # Degrade the FTS query to LIKE terms: strip operators/wildcards, - # keep quoted phrases intact, AND the rest. terms: List[str] = [] for raw_tok in re.findall(r'"[^"]+"|\S+', fts_query): tok = raw_tok.strip('"').strip("*").strip() - if not tok or tok.upper() in {"AND", "OR", "NOT", "NEAR"}: - continue - terms.append(tok) + if tok and tok.upper() not in {"AND", "OR", "NOT", "NEAR"}: + terms.append(tok) if not terms: return [] where = ["m.id > ? AND m.id <= ?"] params: list = [progress, high_water] for term in terms: - esc = _escape_like(term) - where.append( - "(m.content LIKE ? ESCAPE '\\' OR m.tool_name LIKE ? ESCAPE '\\' " - "OR m.tool_calls LIKE ? ESCAPE '\\')" - ) - params += [f"%{esc}%"] * 3 - if not include_inactive: - where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - params.extend(source_filter) - if exclude_sources is not None: - where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - params.extend(exclude_sources) - if role_filter: - where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - params.extend(role_filter) - - sql = f""" - SELECT m.id, m.session_id, m.role, - substr(m.content, - max(1, instr(m.content, ?) - 40), - 120) AS snippet, - m.timestamp, m.tool_name, - s.source, s.model, s.started_at AS session_started - FROM messages m - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(where)} - ORDER BY m.timestamp DESC - LIMIT ? - """ - params = [terms[0]] + params + [limit] - with self._read_ctx() as conn: - rows = conn.execute(sql, params).fetchall() - return [dict(r) for r in rows] + where.append(_LIKE_ANY_COLUMN_SQL) + params += [f"%{_escape_like(term)}%"] * 3 + _search_filter_clauses( + where, params, include_inactive=include_inactive, + source_filter=source_filter, exclude_sources=exclude_sources, + role_filter=role_filter, + ) + return self._like_rows( + where, [terms[0], *params, limit], + order_by="ORDER BY m.timestamp DESC", limit_sql="LIMIT ?", + ) def search_sessions_by_id( self, @@ -2270,24 +1672,16 @@ class SessionSearchMixin: sources: List[str] = None, exclude_sources: List[str] = None, ) -> List[Dict[str, Any]]: - """Search surfaced sessions by exact/prefix/substring session id. - - Desktop search uses this alongside FTS message search so users can paste - a session id from logs, CLI output, or another Hermes surface and jump - straight to that conversation. Matching also checks ``_lineage_root_id`` - for projected compression-chain tips, so an old root id still resolves to - the live continuation row. - """ + """Search surfaced sessions by exact/prefix/substring session id (paste + an id from logs and jump to it). Also matches ``_lineage_root_id`` so + an old compression root id resolves to the live continuation row.""" needle = (query or "").strip().lower() if not needle or limit <= 0: return [] - # SQL-bounded: list_sessions_rich pushes the id LIKE filter into the - # query (matching the row's own id AND any id in its forward - # compression chain), so we only materialize matching rows instead of - # scanning every session. Fetch a small multiple of `limit` so the - # in-Python exact/prefix/substring ranking below has enough candidates - # to order, then truncate. + # list_sessions_rich pushes the id LIKE filter (own id + forward + # compression chain) into SQL; over-fetch so the in-Python + # exact/prefix/substring ranking has candidates, then truncate. candidates = self.list_sessions_rich( source=source, sources=sources, @@ -2320,32 +1714,17 @@ class SessionSearchMixin: self._conn.execute(f"SELECT 1 FROM {name} LIMIT 0") return True except sqlite3.DatabaseError: - # OperationalError ("no such table") or the broader - # DatabaseError class ("vtable constructor failed", raised when - # e.g. a required tokenizer is missing or the table is mid- - # teardown) — in every case the table is not queryable. + # "no such table", or "vtable constructor failed" (missing + # tokenizer / mid-teardown) — either way not queryable. return False def optimize_fts(self) -> int: - """Merge fragmented FTS5 b-tree segments into one per index. + """Merge fragmented FTS5 segments into one per index (``'optimize'``). - FTS5 indexes grow as a series of incremental segments — one per - ``INSERT`` batch driven by the message triggers. Over tens of - thousands of messages these segments accumulate, which both bloats - the ``*_data`` shadow tables and slows ``MATCH`` queries that must - scan every segment. The special ``'optimize'`` command rewrites each - index as a single merged segment. - - This is purely a maintenance operation — it changes neither search - results nor ``snippet()`` output, only on-disk layout and query - speed. It is complementary to VACUUM: ``optimize`` compacts the FTS - index internally, then VACUUM returns the freed pages to the OS. - - Skips any FTS table that does not exist (e.g. the trigram index when - disabled via ``HERMES_DISABLE_FTS_TRIGRAM`` or not yet created), so - it is safe to call unconditionally. - - Returns the number of FTS indexes that were optimized. + Pure maintenance: changes neither results nor ``snippet()`` output, + only layout and speed; complementary to VACUUM, which then returns + the freed pages. Skips absent tables, so it is safe unconditionally. + Returns the number of indexes optimized. """ optimized = 0 with self._lock: @@ -2353,8 +1732,6 @@ class SessionSearchMixin: if not self._fts_table_exists(tbl): continue try: - # The column name in the INSERT must match the table name - # for FTS5 special commands. self._conn.execute( f"INSERT INTO {tbl}({tbl}) VALUES('optimize')" ) @@ -2366,31 +1743,19 @@ class SessionSearchMixin: return optimized def rebuild_fts(self) -> int: - """Rebuild FTS5 indexes from the canonical ``messages`` table. + """Rebuild FTS5 indexes from ``messages`` (``'rebuild'``) — the + documented recovery for a corrupt index that rejects writes while + reads succeed. - Uses the FTS5 ``'rebuild'`` command, which rewrites the internal - b-tree segments from the content rows. This is the documented - recovery for a corrupt FTS index that rejects message writes while - reads still succeed (issue #50502). Unlike ``optimize_fts`` (which - merges existing segments), ``rebuild`` discards and recreates the - index data entirely. - - A full structural rebuild must never run concurrently in two - processes sharing one state.db — that interleaving has structurally - corrupted the database in production (PR #93200) — so this admits - through the cross-process ``fts_rebuild_admission`` authority and - FAILS CLOSED: if another process holds the rebuild lock beyond the - bounded wait, this call defers (returns 0) rather than racing it. - Callers already treat 0 as "rebuild made no progress" and fall back - to the stale-FTS breadcrumb path, which retries in-process from the - gateway housekeeping tick (``retry_deferred_fts_recovery``) and at - next startup. - - Safe to call when FTS tables don't exist (skips them). - Returns the number of FTS indexes that were rebuilt. + A full structural rebuild must never run concurrently in two processes + sharing one state.db (that interleaving corrupted production DBs), so + this admits through ``fts_rebuild_admission`` and FAILS CLOSED, + returning 0 on deferral; callers treat 0 as "no progress" and fall + back to the stale-FTS breadcrumb path. Skips absent tables. Returns + the number of indexes rebuilt. """ rebuilt = 0 - with fts_rebuild_admission(getattr(self, "db_path", None)) as admitted: + with fts_rebuild_admission(self.db_path) as admitted: if not admitted: logger.warning( "Deferred in-place FTS rebuild: another process holds " @@ -2419,34 +1784,23 @@ class SessionSearchMixin: ) -> int: """Run bounded FTS5 ``'merge'`` commands against each present index. - A positive merge rank tells SQLite to stop after approximately that - many output pages, so each command holds the write lock for - milliseconds regardless of index size — unlike ``'optimize'``, which - rewrites the whole index in one transaction (measured 9-18 s per - index on a 10 GB production DB, long enough to exhaust a competing - writer's entire lock-retry patience). - - Protocol (SQLite FTS5 §6.8-6.9): + A positive merge rank stops after ~that many output pages, so each + command holds the write lock for milliseconds regardless of index + size — unlike ``'optimize'`` (9-18 s per index on a 10 GB DB, enough + to exhaust a competing writer's lock-retry patience). - ``usermerge`` is lowered to its minimum of 2 (persisted in the - ``%_config`` shadow table, applied once per instance) so a - positive merge acts on ANY level holding >= 2 segments. With the - default of 4, levels below that threshold are never merged by a - positive-rank command and a fragmented index cannot converge. - - Up to *max_commands* merge commands run per index, stopping early - on the documented no-progress signal: the delta in - ``total_changes`` is < 2 (the command's own INSERT accounts - for 1 change; >= 2 means real merge work happened). + ``%_config`` shadow table, once per instance) so a positive merge + acts on ANY level with >= 2 segments; at the default 4 a fragmented + index cannot converge. + - Up to *max_commands* per index, stopping on the documented + no-progress signal: ``total_changes`` delta < 2 (the command's own + INSERT is 1 change). - Each command is its own implicit transaction (the connection runs - with ``isolation_level=None``), so the SQLite write lock is released - between commands and competing processes can interleave writes - mid-pass. Missing tables are valid schema variants (FTS variants are - optional, and ``optimize_fts_storage`` legitimately drops + backfills - these tables while writers keep running) and are skipped, mirroring - ``optimize_fts``. Other SQLite errors propagate to the caller. - - Returns the number of merge commands executed. + Each command is its own implicit transaction (``isolation_level=None``), + so competing processes interleave mid-pass. Missing tables are valid + variants (optimize_fts_storage drops + backfills them live) and are + skipped; other SQLite errors propagate. Returns commands executed. """ if isinstance(max_pages, bool) or not isinstance(max_pages, int): raise TypeError("max_pages must be an integer") @@ -2464,11 +1818,9 @@ class SessionSearchMixin: for tbl in self._FTS_TABLES: if not self._fts_table_exists(tbl): continue - # One-time (per instance) usermerge floor; the value is - # persisted in the index's config shadow table so future - # connections inherit it. Setting config is a metadata-only - # write — it never touches segment data. - if not getattr(self, "_fts_usermerge_floor_applied", False): + # One-time (per instance) usermerge floor; metadata-only write, + # persisted so future connections inherit it. + if not self._fts_usermerge_floor_applied: self._conn.execute( f"INSERT INTO {tbl}({tbl}, rank) " "VALUES('usermerge', 2)" diff --git a/hermes_state_telegram.py b/hermes_state_telegram.py new file mode 100644 index 0000000000..2c8dbf8501 --- /dev/null +++ b/hermes_state_telegram.py @@ -0,0 +1,529 @@ +"""Telegram DM topic-mode mixin for :class:`hermes_state.SessionDB`.""" + +from __future__ import annotations + +import logging +import sqlite3 +import time +from typing import Any, Dict, List, Optional + +from hermes_state_common import ( + _PREVIEW_ELIGIBLE_SQL, + _PREVIEW_RAW_SELECT, + _shape_preview, + _sql_session_last_active, +) + +# caplog tests pin the "hermes_state" logger name. +logger = logging.getLogger("hermes_state") + + +def _normalize_telegram_topic_profile_name(profile_name: Optional[str] = None) -> str: + """Empty/missing → ``"default"`` (single namespace for non-multiplexed + gateways). Multiplexed callers must pass the *routed* profile + (``source.profile``), never the process-global active profile.""" + name = str(profile_name or "").strip() + return name if name else "default" + + +class SessionTelegramTopicsMixin: + """Telegram DM topic-mode tables, bindings and lookups.""" + + def apply_telegram_topic_migration(self) -> None: + """Create Telegram DM topic-mode tables on explicit /topic opt-in. + + Deliberately NOT part of startup reconciliation: operators can upgrade + and keep the old bot behavior until a user runs /topic. + + Schema versions: v1 initial; v2 session_id FK ON DELETE CASCADE (pruning + clears bindings); v3 ``profile_name`` on both tables so multiplexed + gateways sharing one state.db isolate topic state per profile. + """ + # (table, column list, DDL body). profile_name leads the PK: a private + # chat_id is the user id, identical across bots sharing one state.db. + tables = ( + ( + "telegram_dm_topic_mode", + "profile_name, chat_id, user_id, enabled, activated_at, updated_at, " + "has_topics_enabled, allows_users_to_create_topics, " + "capability_checked_at, intro_message_id, pinned_message_id", + """ + profile_name TEXT NOT NULL DEFAULT 'default', + chat_id TEXT NOT NULL, + user_id TEXT NOT NULL, + enabled INTEGER NOT NULL DEFAULT 1, + activated_at REAL NOT NULL, + updated_at REAL NOT NULL, + has_topics_enabled INTEGER, + allows_users_to_create_topics INTEGER, + capability_checked_at REAL, + intro_message_id TEXT, + pinned_message_id TEXT, + PRIMARY KEY (profile_name, chat_id) + """, + ), + ( + "telegram_dm_topic_bindings", + "profile_name, chat_id, thread_id, user_id, session_key, " + "session_id, managed_mode, linked_at, updated_at", + """ + profile_name TEXT NOT NULL DEFAULT 'default', + chat_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + user_id TEXT NOT NULL, + session_key TEXT NOT NULL, + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + managed_mode TEXT NOT NULL DEFAULT 'auto', + linked_at REAL NOT NULL, + updated_at REAL NOT NULL, + PRIMARY KEY (profile_name, chat_id, thread_id) + """, + ), + ) + + def _do(conn): + for table, columns, ddl in tables: + conn.execute(f"CREATE TABLE IF NOT EXISTS {table} ({ddl})") + have = {row[1] for row in conn.execute(f"PRAGMA table_info('{table}')")} + if "profile_name" in have: + continue + # v1/v2 → v3. SQLite can't ALTER a PK or FK, so rebuild (this + # also supplies v2's ON DELETE CASCADE). Legacy rows land in + # "default" only, never replicated across profiles. + legacy_columns = columns.replace("profile_name, ", "", 1) + conn.executescript( + f""" + CREATE TABLE {table}_new ({ddl}); + INSERT INTO {table}_new ({columns}) + SELECT 'default', {legacy_columns} FROM {table}; + DROP TABLE {table}; + ALTER TABLE {table}_new RENAME TO {table}; + """ + ) + + # Indexes after any rebuild: the user index needs profile_name. + conn.executescript( + """ + CREATE UNIQUE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_session + ON telegram_dm_topic_bindings(session_id); + + CREATE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_user + ON telegram_dm_topic_bindings(profile_name, user_id, chat_id); + """ + ) + + conn.execute( + "INSERT INTO state_meta (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ("telegram_dm_topic_schema_version", "3"), + ) + self._execute_write(_do) + + def enable_telegram_topic_mode( + self, + *, + chat_id: str, + user_id: str, + profile_name: str = "default", + has_topics_enabled: Optional[bool] = None, + allows_users_to_create_topics: Optional[bool] = None, + ) -> None: + """Enable Telegram DM topic mode for one private chat/user. + + Owns the explicit topic migration; SessionDB startup must not create + these tables. Multiplexed callers pass the routed ``source.profile``, + not the process-global active profile. + """ + self.apply_telegram_topic_migration() + now = time.time() + profile_name = _normalize_telegram_topic_profile_name(profile_name) + + def _to_int(value: Optional[bool]) -> Optional[int]: + if value is None: + return None + return 1 if value else 0 + + self._write_sql( + """ + INSERT INTO telegram_dm_topic_mode ( + profile_name, chat_id, user_id, enabled, activated_at, updated_at, + has_topics_enabled, allows_users_to_create_topics, + capability_checked_at + ) VALUES (?, ?, ?, 1, ?, ?, ?, ?, ?) + ON CONFLICT(profile_name, chat_id) DO UPDATE SET + user_id = excluded.user_id, + enabled = 1, + updated_at = excluded.updated_at, + has_topics_enabled = excluded.has_topics_enabled, + allows_users_to_create_topics = excluded.allows_users_to_create_topics, + capability_checked_at = excluded.capability_checked_at + """, + ( + profile_name, + str(chat_id), + str(user_id), + now, + now, + _to_int(has_topics_enabled), + _to_int(allows_users_to_create_topics), + now, + ), + ) + + def disable_telegram_topic_mode( + self, + *, + chat_id: str, + profile_name: str = "default", + clear_bindings: bool = True, + ) -> None: + """Disable Telegram DM topic mode for one private chat. + + ``clear_bindings`` also drops the chat's bindings so a later re-enable + starts clean. Never creates the tables; absent tables are a no-op. + """ + profile_name = _normalize_telegram_topic_profile_name(profile_name) + + def _do(conn): + try: + conn.execute( + "UPDATE telegram_dm_topic_mode SET enabled = 0, updated_at = ? " + "WHERE profile_name = ? AND chat_id = ?", + (time.time(), profile_name, str(chat_id)), + ) + if clear_bindings: + conn.execute( + "DELETE FROM telegram_dm_topic_bindings " + "WHERE profile_name = ? AND chat_id = ?", + (profile_name, str(chat_id)), + ) + except sqlite3.OperationalError: + return + self._execute_write(_do) + + def is_telegram_topic_mode_enabled( + self, + *, + chat_id: str, + user_id: str, + profile_name: str = "default", + ) -> bool: + """Return whether Telegram DM topic mode is enabled for this chat/user.""" + profile_name = _normalize_telegram_topic_profile_name(profile_name) + with self._read_ctx() as conn: + try: + row = conn.execute( + """ + SELECT enabled FROM telegram_dm_topic_mode + WHERE profile_name = ? AND chat_id = ? AND user_id = ? + """, + (profile_name, str(chat_id), str(user_id)), + ).fetchone() + except sqlite3.OperationalError: + return False + if row is None: + return False + enabled = row[0] + return bool(enabled) + + def get_telegram_topic_binding( + self, + *, + chat_id: str, + thread_id: str, + profile_name: str = "default", + ) -> Optional[Dict[str, Any]]: + """Return the session binding for a Telegram DM topic, if present.""" + profile_name = _normalize_telegram_topic_profile_name(profile_name) + with self._read_ctx() as conn: + try: + row = conn.execute( + """ + SELECT * FROM telegram_dm_topic_bindings + WHERE profile_name = ? AND chat_id = ? AND thread_id = ? + """, + (profile_name, str(chat_id), str(thread_id)), + ).fetchone() + except sqlite3.OperationalError: + return None + return dict(row) if row else None + + def list_telegram_topic_bindings_for_chat( + self, + *, + chat_id: str, + profile_name: str = "default", + ) -> List[Dict[str, Any]]: + """All bindings for one chat, newest first. Read-only: [] when the + table is absent (never triggers the migration).""" + profile_name = _normalize_telegram_topic_profile_name(profile_name) + with self._read_ctx() as conn: + try: + rows = conn.execute( + "SELECT * FROM telegram_dm_topic_bindings " + "WHERE profile_name = ? AND chat_id = ? " + "ORDER BY updated_at DESC", + (profile_name, str(chat_id)), + ).fetchall() + except sqlite3.OperationalError: + return [] + return [dict(row) for row in rows] + + def get_telegram_topic_binding_by_session( + self, + *, + session_id: str, + ) -> Optional[Dict[str, Any]]: + """Reverse lookup via the UNIQUE INDEX on session_id; None when + unbound or the table is absent.""" + with self._read_ctx() as conn: + try: + row = conn.execute( + """ + SELECT * FROM telegram_dm_topic_bindings + WHERE session_id = ? + """, + (str(session_id),), + ).fetchone() + except sqlite3.OperationalError: + return None + return dict(row) if row else None + + def delete_telegram_topic_binding( + self, + *, + chat_id: str, + thread_id: str, + profile_name: str = "default", + ) -> int: + """Remove the binding row for one (chat, thread) pair. + + Called when the Bot API confirms a topic was deleted externally + (``Thread not found`` after the same-thread retry failed); otherwise + ``gateway.run._recover_telegram_topic_thread_id`` keeps redirecting + inbound messages to the dead topic. + + If this removes the chat's *last* binding, ``telegram_dm_topic_mode`` + is flipped to ``enabled = 0`` in the same transaction; otherwise the + chat stays in topic mode with zero lanes and a user who disabled topics + in the Telegram client (not via ``/topic off``) stays stuck. + + Returns the number of rows deleted; absent binding or unmigrated tables + are silent no-ops (never raise from a cleanup hot path). + """ + chat_id = str(chat_id) + thread_id = str(thread_id) + profile_name = _normalize_telegram_topic_profile_name(profile_name) + deleted = {"count": 0} + + def _do(conn): + try: + cursor = conn.execute( + """ + DELETE FROM telegram_dm_topic_bindings + WHERE profile_name = ? AND chat_id = ? AND thread_id = ? + """, + (profile_name, chat_id, thread_id), + ) + deleted["count"] = cursor.rowcount or 0 + except sqlite3.OperationalError: + deleted["count"] = 0 + return + if not deleted["count"]: + return + # Last binding gone → disable topic mode. Same transaction, so no + # read-after-prune race. + try: + remaining = conn.execute( + """ + SELECT 1 FROM telegram_dm_topic_bindings + WHERE profile_name = ? AND chat_id = ? LIMIT 1 + """, + (profile_name, chat_id), + ).fetchone() + if remaining is None: + conn.execute( + "UPDATE telegram_dm_topic_mode " + "SET enabled = 0, updated_at = ? " + "WHERE profile_name = ? AND chat_id = ?", + (time.time(), profile_name, chat_id), + ) + except sqlite3.OperationalError: + # telegram_dm_topic_mode absent — binding prune still stands. + pass + + self._execute_write(_do) + return deleted["count"] + + def bind_telegram_topic( + self, + *, + chat_id: str, + thread_id: str, + user_id: str, + session_key: str, + session_id: str, + managed_mode: str = "auto", + profile_name: str = "default", + ) -> None: + """Bind one Telegram DM topic thread to one Hermes session. + + A session may be linked to only one topic: rebinding the same pair is + idempotent; linking the session to a different topic raises ValueError. + """ + self.apply_telegram_topic_migration() + now = time.time() + chat_id = str(chat_id) + thread_id = str(thread_id) + user_id = str(user_id) + session_key = str(session_key) + session_id = str(session_id) + profile_name = _normalize_telegram_topic_profile_name(profile_name) + + def _do(conn): + existing_session = conn.execute( + """ + SELECT profile_name, chat_id, thread_id + FROM telegram_dm_topic_bindings + WHERE session_id = ? + """, + (session_id,), + ).fetchone() + if existing_session is not None: + if isinstance(existing_session, sqlite3.Row): + linked_profile = existing_session["profile_name"] + linked_chat = existing_session["chat_id"] + linked_thread = existing_session["thread_id"] + else: + linked_profile, linked_chat, linked_thread = existing_session + if ( + str(linked_profile) != profile_name + or str(linked_chat) != chat_id + or str(linked_thread) != thread_id + ): + raise ValueError("session is already linked to another Telegram topic") + + conn.execute( + """ + INSERT INTO telegram_dm_topic_bindings ( + profile_name, chat_id, thread_id, user_id, session_key, session_id, + managed_mode, linked_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(profile_name, chat_id, thread_id) DO UPDATE SET + user_id = excluded.user_id, + session_key = excluded.session_key, + session_id = excluded.session_id, + managed_mode = excluded.managed_mode, + updated_at = excluded.updated_at + """, + ( + profile_name, + chat_id, + thread_id, + user_id, + session_key, + session_id, + managed_mode, + now, + now, + ), + ) + self._execute_write(_do) + + def is_telegram_session_linked_to_topic(self, *, session_id: str) -> bool: + """True if the session is bound to any Telegram DM topic. Read-only: + absent tables (nobody ran ``/topic``) mean unbound → False.""" + with self._read_ctx() as conn: + try: + row = conn.execute( + """ + SELECT 1 FROM telegram_dm_topic_bindings + WHERE session_id = ? + LIMIT 1 + """, + (str(session_id),), + ).fetchone() + except sqlite3.OperationalError: + return False + return row is not None + + def list_unlinked_telegram_sessions_for_user( + self, + *, + chat_id: str, + user_id: str, + profile_name: str = "default", + limit: int = 10, + ) -> List[Dict[str, Any]]: + """This user's Telegram sessions not bound to a topic. + + Read-only: if the bindings table is absent, every session is unlinked + and a simpler query is used. Scoped by ``profile_name`` so multiplexed + profiles do not surface each other's sessions. + """ + profile_name = _normalize_telegram_topic_profile_name(profile_name) + # sessions.profile_name is NULL/empty for legacy rows → treat as default. + profile_clause = "AND COALESCE(NULLIF(TRIM(s.profile_name), ''), 'default') = ?" + with self._read_ctx() as conn: + try: + rows = conn.execute( + f""" + SELECT s.*, + COALESCE(sp.prompt, s.system_prompt) + AS _system_prompt_resolved, + COALESCE( + (SELECT {_PREVIEW_RAW_SELECT} + FROM messages m + WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL + AND {_PREVIEW_ELIGIBLE_SQL} + ORDER BY m.timestamp, m.id LIMIT 1), + '' + ) AS _preview_raw, + {_sql_session_last_active("s")} AS last_active + FROM sessions s + LEFT JOIN system_prompts sp + ON sp.hash = s.system_prompt_hash + WHERE s.source = 'telegram' + AND s.user_id = ? + {profile_clause} + AND NOT EXISTS ( + SELECT 1 FROM telegram_dm_topic_bindings b + WHERE b.session_id = s.id + ) + ORDER BY last_active DESC, s.started_at DESC + LIMIT ? + """, + (str(user_id), profile_name, int(limit)), + ).fetchall() + except sqlite3.OperationalError: + rows = conn.execute( + f""" + SELECT s.*, + COALESCE(sp.prompt, s.system_prompt) + AS _system_prompt_resolved, + COALESCE( + (SELECT {_PREVIEW_RAW_SELECT} + FROM messages m + WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL + AND {_PREVIEW_ELIGIBLE_SQL} + ORDER BY m.timestamp, m.id LIMIT 1), + '' + ) AS _preview_raw, + {_sql_session_last_active("s")} AS last_active + FROM sessions s + LEFT JOIN system_prompts sp + ON sp.hash = s.system_prompt_hash + WHERE s.source = 'telegram' + AND s.user_id = ? + ORDER BY last_active DESC, s.started_at DESC + LIMIT ? + """, + (str(user_id), int(limit)), + ).fetchall() + + sessions: List[Dict[str, Any]] = [] + for row in rows: + session = self._session_row_dict(row) + session["preview"] = _shape_preview(session.pop("_preview_raw", "")) + sessions.append(session) + return sessions diff --git a/hermes_state_titles.py b/hermes_state_titles.py new file mode 100644 index 0000000000..3d6dbea0f5 --- /dev/null +++ b/hermes_state_titles.py @@ -0,0 +1,305 @@ +"""Session title mixin for SessionDB: sanitizing, auto/user provenance +ranking, and lineage-aware lookups.""" + +from __future__ import annotations + +import logging +import re +from typing import Any, Dict, Optional + +from agent.message_sanitization import _sanitize_surrogates +from hermes_state_common import _COMPRESSION_CHILD_SQL, escape_like as _escape_like + +# caplog tests pin the "hermes_state" logger name. +logger = logging.getLogger("hermes_state") + + +class SessionTitlesMixin: + """Sanitizing, ranking auto/user titles, lineage-aware lookups.""" + + @classmethod + def _title_rank(cls, source: Optional[str]) -> int: + """Rank a stored title_source. + + NULL (pre-provenance rows) is indistinguishable from a manual ``/title`` + of that era, so it ranks as ``user``: auto-titling only ever fills + genuinely empty legacy titles. + """ + if source is None: + return cls._TITLE_SOURCE_RANK[cls.TITLE_SOURCE_USER] + return cls._TITLE_SOURCE_RANK.get(str(source), 0) + + @staticmethod + def sanitize_title(title: Optional[str]) -> Optional[str]: + """Strip control/zero-width/bidi chars, collapse whitespace, normalize + empty to None. Raises ValueError if longer than MAX_TITLE_LENGTH + after cleaning.""" + from hermes_state import SessionDB + if not title: + return None + + # Lone surrogates cannot be bound by sqlite3 (UnicodeEncodeError). + title = _sanitize_surrogates(title) + + # ASCII controls, keeping \t \n \r so the whitespace collapse below + # turns them into spaces. + cleaned = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', title) + + # Zero-width, bidi override, object-replacement, interlinear annotation. + cleaned = re.sub( + r'[\u200b-\u200f\u2028-\u202e\u2060-\u2069\ufeff\ufffc\ufff9-\ufffb]', + '', cleaned, + ) + + cleaned = re.sub(r'\s+', ' ', cleaned).strip() + + if not cleaned: + return None + + if len(cleaned) > SessionDB.MAX_TITLE_LENGTH: + raise ValueError( + f"Title too long ({len(cleaned)} chars, max {SessionDB.MAX_TITLE_LENGTH})" + ) + + return cleaned + + def _is_compression_ancestor( + self, conn, *, ancestor_id: str, descendant_id: str + ) -> bool: + """True if *ancestor_id* is a compression predecessor of *descendant_id*. + + Uses the canonical continuation edge ``_COMPRESSION_CHILD_SQL`` (parent + ended with ``end_reason = 'compression'`` and child started at/after its + ``ended_at``), which excludes delegate/branch children that also carry + ``parent_session_id``. One recursive CTE so the edge is defined once. + """ + if not ancestor_id or not descendant_id or ancestor_id == descendant_id: + return False + edge = _COMPRESSION_CHILD_SQL.format(a="child") + row = conn.execute( + f""" + WITH RECURSIVE ancestors(id) AS ( + SELECT ? + UNION + SELECT parent.id + FROM ancestors a + JOIN sessions child ON child.id = a.id + JOIN sessions parent ON parent.id = child.parent_session_id + WHERE {edge} + ) + SELECT 1 FROM ancestors WHERE id = ? AND id != ? LIMIT 1 + """, + (descendant_id, ancestor_id, descendant_id), + ).fetchone() + return row is not None + + def _set_session_title( + self, + session_id: str, + title: str, + *, + source: str, + ) -> bool: + """Write a title, enforcing provenance precedence. + + A ``user`` write always lands. ``derived``/``llm`` land only when the + row is untitled or holds strictly lower authority, so derived upgrades + to llm exactly once, nothing overwrites a user name, and re-running the + titler on an llm row is a no-op (stops sessions renaming themselves). + No writer may move a hidden canonical Bot Chat off its title. + + Read and write are one compare-and-swap in a single transaction, so a + manual ``/title`` racing an in-flight generation is not clobbered. + """ + title = self.sanitize_title(title) + is_user = source == self.TITLE_SOURCE_USER + new_rank = self._title_rank(source) if not is_user else None + + def _do(conn): + current = conn.execute( + "SELECT title, title_source, hidden FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + if current is None: + return 0 + # The canonical Bot Chat's NAME is its identity: Bot Mode resolves it + # by exact-title lookup on every open, so a rename orphans the whole + # conversation (next open mints an empty replacement and UNIQUE(title) + # blocks renaming back). Refuse here, the single write path every + # surface funnels through. Hidden is the discriminator: canonical + # chats are born hidden; a visible session merely named "Bot Chat" + # stays renameable. Provenance-blind so the auto-titler no-ops too. + if ( + (current["title"] or "") == self.CANONICAL_BOT_CHAT_TITLE + and bool(current["hidden"]) + and title != self.CANONICAL_BOT_CHAT_TITLE + ): + if is_user: + raise ValueError( + "This is the bot's canonical Bot Chat — its name is its " + "identity, and renaming it would orphan the conversation. " + "To start fresh, create a new bot instead." + ) + return 0 + if not is_user and current["title"] is not None: + if self._title_rank(current["title_source"]) >= new_rank: + return 0 + + if title: + cursor = conn.execute( + "SELECT id FROM sessions WHERE title = ? AND id != ?", + (title, session_id), + ) + conflict = cursor.fetchone() + if conflict: + conflict_id = conflict["id"] + # If the conflicting holder is a hidden compressed ancestor + # of this continuation, the user cannot free the title, so + # transfer it onto the tip. Uniqueness and lineage are kept. + if self._is_compression_ancestor( + conn, ancestor_id=conflict_id, descendant_id=session_id + ): + conn.execute( + "UPDATE sessions SET title = NULL WHERE id = ?", + (conflict_id,), + ) + else: + raise ValueError( + f"Title '{title}' is already in use by session {conflict_id}" + ) + # CAS on the values just read (``IS`` is NULL-safe): a concurrent + # write between the SELECT and here loses instead of being overwritten. + cursor = conn.execute( + "UPDATE sessions SET title = ?, title_source = ? " + "WHERE id = ? AND title IS ? AND title_source IS ?", + ( + title, + source if title else None, + session_id, + current["title"], + current["title_source"], + ), + ) + return cursor.rowcount + + rowcount = self._execute_write(_do) + return rowcount > 0 + + def set_session_title(self, session_id: str, title: str) -> bool: + """Set a title on the user's behalf (``user`` provenance; auto-titling + never replaces it). Empty clears the title. Raises ValueError on a + title conflict or validation failure. Automatic callers must use + :meth:`set_auto_title`.""" + return self._set_session_title( + session_id, title, source=self.TITLE_SOURCE_USER + ) + + def set_auto_title(self, session_id: str, title: str, *, source: str) -> bool: + """Set an automatic title; False (untouched) when a higher-authority + title already holds the row.""" + if source not in (self.TITLE_SOURCE_DERIVED, self.TITLE_SOURCE_LLM): + raise ValueError(f"invalid automatic title source: {source!r}") + return self._set_session_title(session_id, title, source=source) + + def set_auto_title_if_empty(self, session_id: str, title: str) -> bool: + """Back-compat shim (third-party plugins reference it by name); new + code calls :meth:`set_auto_title` with an explicit source.""" + return self.set_auto_title( + session_id, title, source=self.TITLE_SOURCE_LLM + ) + + def get_session_title(self, session_id: str) -> Optional[str]: + """Get the title for a session, or None.""" + with self._read_ctx() as conn: + cursor = conn.execute( + "SELECT title FROM sessions WHERE id = ?", (session_id,) + ) + row = cursor.fetchone() + return row["title"] if row else None + + def get_session_title_source(self, session_id: str) -> Optional[str]: + """Get the provenance of a session's title, or None when untitled.""" + with self._read_ctx() as conn: + cursor = conn.execute( + "SELECT title, title_source FROM sessions WHERE id = ?", + (session_id,), + ) + row = cursor.fetchone() + if not row or row["title"] is None: + return None + return row["title_source"] + + def set_session_title_source(self, session_id: str, source: str) -> bool: + """Overwrite a title's provenance without touching the text: a title + copied across a compression rotation keeps the original's authority.""" + if source not in self._TITLE_SOURCE_RANK: + raise ValueError(f"invalid title source: {source!r}") + + return self._write_rowcount( + "UPDATE sessions SET title_source = ? " + "WHERE id = ? AND title IS NOT NULL", + (source, session_id), + ) > 0 + + def get_session_by_title(self, title: str) -> Optional[Dict[str, Any]]: + """Look up a session by exact title. Returns session dict or None.""" + with self._read_ctx() as conn: + cursor = conn.execute( + "SELECT s.*, " + "COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved " + "FROM sessions s " + "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash " + "WHERE s.title = ?", + (title,), + ) + row = cursor.fetchone() + return self._session_row_dict(row) if row else None + + def resolve_session_by_title(self, title: str) -> Optional[str]: + """Resolve a title to a session ID, preferring the latest "title #N" + continuation over the exact match.""" + exact = self.get_session_by_title(title) + + # Escape LIKE wildcards so "%"/"_" in titles cannot false-match. + escaped = _escape_like(title) + with self._read_ctx() as conn: + cursor = conn.execute( + "SELECT id, title, started_at FROM sessions " + "WHERE title LIKE ? ESCAPE '\\' ORDER BY started_at DESC", + (f"{escaped} #%",), + ) + numbered = cursor.fetchall() + + if numbered: + return numbered[0]["id"] + elif exact: + return exact["id"] + return None + + def get_next_title_in_lineage(self, base_title: str) -> str: + """Next title in a lineage ("my session" → "my session #2"): strip any + " #N" suffix, then increment the highest existing number.""" + match = re.match(r'^(.*?) #(\d+)$', base_title) + if match: + base = match.group(1) + else: + base = base_title + + escaped = _escape_like(base) + with self._read_ctx() as conn: + cursor = conn.execute( + "SELECT title FROM sessions WHERE title = ? OR title LIKE ? ESCAPE '\\'", + (base, f"{escaped} #%"), + ) + existing = [row["title"] for row in cursor.fetchall()] + + if not existing: + return base + + max_num = 1 # The unnumbered original counts as #1 + for t in existing: + m = re.match(r'^.* #(\d+)$', t) + if m: + max_num = max(max_num, int(m.group(1))) + + return f"{base} #{max_num + 1}" diff --git a/hermes_state_usage.py b/hermes_state_usage.py new file mode 100644 index 0000000000..9abae52aa7 --- /dev/null +++ b/hermes_state_usage.py @@ -0,0 +1,597 @@ +"""Token/usage accounting mixin for SessionDB: the coalescing background +token writer, per-model usage rows, and billing-route columns. Writer thread +state lives on the SessionDB instance.""" + +from __future__ import annotations + +import atexit +import logging +import threading +import time +import weakref +from typing import Any, Dict, List, Optional, Tuple + +# caplog tests pin the "hermes_state" logger name. +logger = logging.getLogger("hermes_state") + + +class SessionUsageMixin: + """Coalesced token writer, per-model usage rows, billing route.""" + + def update_session_billing_route( + self, + session_id: str, + *, + provider: str, + base_url: str, + billing_mode: Optional[str] = None, + ) -> None: + """Unconditionally set the billing route (``update_token_counts`` only + COALESCE-fills NULLs) so the dashboard reflects the latest /model switch. + + Also nulls ``system_prompt`` so the cached snapshot (stale ``Model:`` / + ``Provider:`` header) is rebuilt, like ``update_session_model``. + """ + # Barrier against queued token deltas — see update_session_model. + self.flush_token_counts() + + def _do(conn): + conn.execute( + """UPDATE sessions SET + billing_provider = ?, + billing_base_url = ?, + billing_mode = COALESCE(?, billing_mode), + system_prompt = NULL, + system_prompt_hash = NULL + WHERE id = ?""", + (provider, base_url, billing_mode, session_id), + ) + self._delete_unreferenced_system_prompts(conn) + self._execute_write(_do) + + def queue_token_counts(self, session_id: str, **kwargs) -> None: + """Enqueue a token/cost delta for the background writer. + + Same kwargs and semantics as :meth:`update_token_counts`, applied + asynchronously; cheap enough for the turn thread. After close() has + stopped the writer, falls back to the synchronous path and may raise. + """ + with self._token_queue_cond: + thread = self._token_writer_thread + writer_stopped = self._token_writer_stop and ( + thread is None or not thread.is_alive() + ) + if not writer_stopped: + self._token_queue.append((session_id, kwargs)) + if thread is None or not thread.is_alive(): + # Daemon so exit never hangs on accounting; the atexit hook + # (registered once per instance) drains leftovers. Checking + # ``not is_alive()`` rather than ``is None`` respawns a writer + # that died from an unexpected escape, otherwise deltas + # would pile up until a reader's flush drained them. + thread = threading.Thread( + target=self._token_writer_loop, + name="session-db-token-writer", + daemon=True, + ) + self._token_writer_thread = thread + thread.start() + if self._token_atexit_hook is None: + self_ref = weakref.ref(self) + + def _drain_at_exit() -> None: + db = self_ref() + if db is not None: + db._drain_token_queue_at_exit() + + self._token_atexit_hook = _drain_at_exit + atexit.register(_drain_at_exit) + self._token_queue_cond.notify_all() + if writer_stopped: + # close() ran (a stop-flagged but live writer still accepts; its + # loop drains before exiting). Enqueueing now would drop the delta + # silently — no writer, atexit hook gone — so apply inline and let a + # closed-connection failure raise at the call site. + self.update_token_counts(session_id, **kwargs) + + def flush_token_counts(self, timeout: float = 5.0) -> bool: + """Block until every queued token delta has been applied. + + False on timeout (callers then read totals stale by the queued deltas). + Never raises: apply failures are logged by the writer. + """ + # Lock-free fast path: reads queue-then-busy (see ordering notes below). + if not self._token_queue and not self._token_writer_busy: + return True + batch = None + with self._token_queue_cond: + deadline = time.monotonic() + timeout + while self._token_queue or self._token_writer_busy: + # A live writer is authoritative even when stop-flagged: draining + # here would race its in-flight batch, and newer deltas committing + # before older ones breaks last-non-None-wins / first-accounted- + # route / COALESCE-backfill fields. Only a dead writer lets the + # caller take leftovers; re-checked each wakeup because the writer + # can exit mid-wait with deltas enqueued after its final check. + # busy is claimed while draining so a concurrent flush cannot + # report drained or pop a newer delta while this batch is + # unapplied: a claimed busy means "wait", never "drain alongside". + thread = self._token_writer_thread + if ( + (thread is None or not thread.is_alive()) + and not self._token_writer_busy + ): + self._token_writer_busy = True + batch = list(self._token_queue) + self._token_queue.clear() + break + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + self._token_queue_cond.wait(remaining) + if batch: + try: + self._apply_token_batch(batch) + finally: + with self._token_queue_cond: + self._token_writer_busy = False + self._token_queue_cond.notify_all() + return True + + def _token_writer_loop(self) -> None: + while True: + with self._token_queue_cond: + idle_deadline = time.monotonic() + self._TOKEN_WRITER_IDLE_SECONDS + while not self._token_queue and not self._token_writer_stop: + remaining = idle_deadline - time.monotonic() + if remaining <= 0: + # Retire under the same lock queue_token_counts() uses to + # decide to spawn, so no delta strands behind an exiting worker. + self._token_writer_thread = None + return + self._token_queue_cond.wait(remaining) + if not self._token_queue: + self._token_writer_thread = None + return # stop requested and fully drained + # busy BEFORE clearing the queue: flush's lock-free fast path + # reads queue-then-busy and must never see "empty and idle" + # while a popped batch is unapplied. + self._token_writer_busy = True + batch = list(self._token_queue) + self._token_queue.clear() + try: + self._apply_token_batch(batch) + finally: + with self._token_queue_cond: + self._token_writer_busy = False + self._token_queue_cond.notify_all() + + def _apply_token_batch(self, batch: List[Tuple[str, Dict[str, Any]]]) -> None: + """Apply queued deltas in order, coalescing where safe. Never raises.""" + try: + coalesced = self._coalesce_token_deltas(batch) + except Exception as exc: + # Coalescing must never kill the writer (callers cannot observe a + # dead one); the merge is only an optimization. + logger.warning( + "async token accounting: coalesce failed, applying raw " + "batch: %s", exc, + ) + coalesced = batch + for session_id, kwargs in coalesced: + try: + self.update_token_counts(session_id, **kwargs) + except Exception as exc: + # Accounting loss is logged, never raised into a turn. + logger.warning( + "async token accounting: apply failed (session=%s): %s", + session_id, exc, + ) + + def _coalesce_token_deltas( + self, batch: List[Tuple[str, Dict[str, Any]]] + ) -> List[Tuple[str, Dict[str, Any]]]: + """Merge adjacent incremental deltas with an identical route, so + ordering across sessions and /model switches is preserved exactly. + absolute=True deltas never merge.""" + groups: List[Tuple[Optional[tuple], str, Dict[str, Any]]] = [] + for session_id, kwargs in batch: + key = None + if not kwargs.get("absolute"): + key = (session_id,) + tuple( + kwargs.get(f) for f in self._TOKEN_DELTA_ROUTE_FIELDS + ) + if groups and key is not None and groups[-1][0] == key: + merged = groups[-1][2] + for f in self._TOKEN_DELTA_SUM_FIELDS: + merged[f] = merged.get(f, 0) + kwargs.get(f, 0) + for f in self._TOKEN_DELTA_COST_FIELDS: + value = kwargs.get(f) + if value is not None: + # All-None runs stay None so COALESCE keeps the stored value. + merged[f] = (merged.get(f) or 0.0) + value + else: + groups.append((key, session_id, dict(kwargs))) + return [(sid, kw) for _, sid, kw in groups] + + def _stop_token_writer(self, join_timeout: float = 10.0) -> None: + """Stop the writer thread and drain remaining deltas. Never raises.""" + with self._token_queue_cond: + self._token_writer_stop = True + self._token_queue_cond.notify_all() + thread = self._token_writer_thread + if thread is not None and thread.is_alive(): + thread.join(timeout=join_timeout) + if thread.is_alive(): + # Writer stuck mid-apply: leave deltas unapplied rather than + # race it and misorder/double-count. + logger.warning( + "async token accounting: writer did not stop within %.0fs; " + "%d queued delta(s) not persisted", + join_timeout, len(self._token_queue), + ) + return + # Writer gone: apply leftovers synchronously under the same busy + # protocol. Wait out a flush caller-drain that already claimed busy — + # close() nulls the connection right after this returns and must not + # yank it mid-batch. + with self._token_queue_cond: + deadline = time.monotonic() + join_timeout + while self._token_writer_busy: + remaining = deadline - time.monotonic() + if remaining <= 0: + logger.warning( + "async token accounting: concurrent drain did not " + "finish within %.0fs; %d queued delta(s) not persisted", + join_timeout, len(self._token_queue), + ) + return + self._token_queue_cond.wait(remaining) + # busy BEFORE clearing the queue (same ordering as the writer loop), + # or flush's lock-free fast path could see "empty and idle". + batch = list(self._token_queue) + if batch: + self._token_writer_busy = True + self._token_queue.clear() + if batch: + try: + self._apply_token_batch(batch) + finally: + with self._token_queue_cond: + self._token_writer_busy = False + self._token_queue_cond.notify_all() + + def _drain_token_queue_at_exit(self) -> None: + try: + self._stop_token_writer() + except Exception: + pass # never fatal at interpreter shutdown + + def update_token_counts( + self, + session_id: str, + input_tokens: int = 0, + output_tokens: int = 0, + model: str = None, + cache_read_tokens: int = 0, + cache_write_tokens: int = 0, + reasoning_tokens: int = 0, + estimated_cost_usd: Optional[float] = None, + actual_cost_usd: Optional[float] = None, + cost_status: Optional[str] = None, + cost_source: Optional[str] = None, + pricing_version: Optional[str] = None, + billing_provider: Optional[str] = None, + billing_base_url: Optional[str] = None, + billing_mode: Optional[str] = None, + api_call_count: int = 0, + absolute: bool = False, + ) -> None: + """Update token counters and backfill model if unset. + + *absolute*=False increments (per-API-call deltas, CLI path); + *absolute*=True sets directly (gateway path, where the cached agent + holds cumulative totals). + """ + # Ensure the row exists: under concurrent load the initial + # create_session() may have failed on SQLite locking, and the UPDATE + # would silently affect 0 rows. + self._insert_session_row(session_id, "unknown", model=model) + if absolute: + sql = """UPDATE sessions SET + input_tokens = ?, + output_tokens = ?, + cache_read_tokens = ?, + cache_write_tokens = ?, + reasoning_tokens = ?, + estimated_cost_usd = COALESCE(?, 0), + actual_cost_usd = CASE + WHEN ? IS NULL THEN actual_cost_usd + ELSE ? + END, + cost_status = COALESCE(?, cost_status), + cost_source = COALESCE(?, cost_source), + pricing_version = COALESCE(?, pricing_version), + billing_provider = COALESCE(billing_provider, ?), + billing_base_url = COALESCE(billing_base_url, ?), + billing_mode = COALESCE(billing_mode, ?), + model = COALESCE(model, ?), + api_call_count = ? + WHERE id = ?""" + else: + sql = """UPDATE sessions SET + input_tokens = input_tokens + ?, + output_tokens = output_tokens + ?, + cache_read_tokens = cache_read_tokens + ?, + cache_write_tokens = cache_write_tokens + ?, + reasoning_tokens = reasoning_tokens + ?, + estimated_cost_usd = COALESCE(estimated_cost_usd, 0) + COALESCE(?, 0), + actual_cost_usd = CASE + WHEN ? IS NULL THEN actual_cost_usd + ELSE COALESCE(actual_cost_usd, 0) + ? + END, + cost_status = COALESCE(?, cost_status), + cost_source = COALESCE(?, cost_source), + pricing_version = COALESCE(?, pricing_version), + billing_provider = COALESCE(billing_provider, ?), + billing_base_url = COALESCE(billing_base_url, ?), + billing_mode = COALESCE(billing_mode, ?), + model = COALESCE(model, ?), + api_call_count = COALESCE(api_call_count, 0) + ? + WHERE id = ?""" + has_accounted_usage = bool( + input_tokens or output_tokens or cache_read_tokens + or cache_write_tokens or reasoning_tokens or api_call_count + or estimated_cost_usd or actual_cost_usd + ) + params = ( + input_tokens, + output_tokens, + cache_read_tokens, + cache_write_tokens, + reasoning_tokens, + estimated_cost_usd, + actual_cost_usd, + actual_cost_usd, + cost_status, + cost_source, + pricing_version, + billing_provider if has_accounted_usage else None, + billing_base_url if has_accounted_usage else None, + billing_mode if has_accounted_usage else None, + model if has_accounted_usage else None, + api_call_count, + session_id, + ) + # Per-model attribution: the sessions row keeps one (model, provider) + # pair, so a mid-session /model switch would attribute every token to + # the initial model. Each delta carries the route active at call time + # and is recorded into session_model_usage keyed by it. Only the + # incremental path records here: absolute cumulative updates cannot be + # split back into routes; Insights reconciles the residual instead. + record_model_usage = (not absolute) and ( + input_tokens or output_tokens or cache_read_tokens + or cache_write_tokens or reasoning_tokens or api_call_count + or estimated_cost_usd + ) + + def _do(conn): + row = conn.execute( + "SELECT model, billing_provider, api_call_count FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + existing_model = row["model"] if row is not None else None + existing_provider = row["billing_provider"] if row is not None else None + existing_api_calls = int((row["api_call_count"] if row is not None else 0) or 0) + + # create_session records the requested route before any API call. + # If that fails and fallback succeeds, the first accounted usage is + # the authoritative route; after that keep the row as is (one row + # cannot represent mixed-provider usage). + first_accounted_route = ( + existing_api_calls == 0 + and has_accounted_usage + and bool(model) + and bool(billing_provider) + and (existing_model != model or existing_provider != billing_provider) + ) + if first_accounted_route: + conn.execute( + """UPDATE sessions + SET model = ?, billing_provider = ?, + billing_base_url = ?, billing_mode = ? + WHERE id = ?""", + (model, billing_provider, billing_base_url, billing_mode, session_id), + ) + conn.execute(sql, params) + if record_model_usage: + self._record_model_usage( + conn, + session_id, + model=model, + billing_provider=billing_provider, + billing_base_url=billing_base_url, + billing_mode=billing_mode, + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_read_tokens=cache_read_tokens, + cache_write_tokens=cache_write_tokens, + reasoning_tokens=reasoning_tokens, + estimated_cost_usd=estimated_cost_usd, + actual_cost_usd=actual_cost_usd, + cost_status=cost_status, + cost_source=cost_source, + api_call_count=api_call_count, + ) + self._execute_write(_do) + + def _record_model_usage( + self, + conn, + session_id: str, + *, + model: Optional[str], + billing_provider: Optional[str], + billing_base_url: Optional[str], + billing_mode: Optional[str], + input_tokens: int, + output_tokens: int, + cache_read_tokens: int, + cache_write_tokens: int, + reasoning_tokens: int, + estimated_cost_usd: Optional[float], + actual_cost_usd: Optional[float], + cost_status: Optional[str], + cost_source: Optional[str], + api_call_count: int, + task: str = "", + ) -> None: + """Accumulate a per-API-call usage delta into session_model_usage. + + Runs inside the caller's write transaction, after the ``sessions`` + UPDATE, so per-model rows stay consistent with the summary row. A + missing model/provider falls back to the session row (same COALESCE + behaviour as the summary update). ``task`` is ``''`` for the main loop; + auxiliary calls record their task name via :meth:`record_auxiliary_usage`. + """ + row = conn.execute( + "SELECT model, billing_provider, billing_base_url, billing_mode " + "FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + sess_model = row["model"] if row is not None else None + sess_provider = row["billing_provider"] if row is not None else None + sess_base_url = row["billing_base_url"] if row is not None else None + sess_billing_mode = row["billing_mode"] if row is not None else None + + # Aux rows must NOT inherit the main-loop route (vision on gemini while + # the main loop runs anthropic); missing info stays 'unknown'/empty. + if task: + eff_model = model or "unknown" + eff_provider = billing_provider or "" + eff_base_url = billing_base_url or "" + eff_billing_mode = billing_mode or "" + else: + eff_model = model or sess_model or "unknown" + eff_provider = billing_provider or sess_provider or "" + eff_base_url = billing_base_url or sess_base_url or "" + eff_billing_mode = billing_mode or sess_billing_mode or "" + now = time.time() + conn.execute( + """INSERT INTO session_model_usage ( + session_id, model, billing_provider, billing_base_url, billing_mode, + task, api_call_count, input_tokens, output_tokens, + cache_read_tokens, cache_write_tokens, reasoning_tokens, + estimated_cost_usd, actual_cost_usd, cost_status, cost_source, + first_seen, last_seen + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id, model, billing_provider, billing_base_url, billing_mode, task) + DO UPDATE SET + api_call_count = api_call_count + excluded.api_call_count, + input_tokens = input_tokens + excluded.input_tokens, + output_tokens = output_tokens + excluded.output_tokens, + cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens, + cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens, + reasoning_tokens = reasoning_tokens + excluded.reasoning_tokens, + estimated_cost_usd = estimated_cost_usd + excluded.estimated_cost_usd, + actual_cost_usd = actual_cost_usd + excluded.actual_cost_usd, + cost_status = COALESCE(excluded.cost_status, cost_status), + cost_source = COALESCE(excluded.cost_source, cost_source), + last_seen = excluded.last_seen""", + ( + session_id, + eff_model, + eff_provider, + eff_base_url, + eff_billing_mode, + task or "", + api_call_count or 0, + input_tokens or 0, + output_tokens or 0, + cache_read_tokens or 0, + cache_write_tokens or 0, + reasoning_tokens or 0, + float(estimated_cost_usd or 0.0), + float(actual_cost_usd or 0.0), + cost_status, + cost_source, + now, + now, + ), + ) + + def record_auxiliary_usage( + self, + session_id: str, + task: str, + *, + model: Optional[str] = None, + billing_provider: Optional[str] = None, + billing_base_url: Optional[str] = None, + input_tokens: int = 0, + output_tokens: int = 0, + cache_read_tokens: int = 0, + cache_write_tokens: int = 0, + reasoning_tokens: int = 0, + estimated_cost_usd: Optional[float] = None, + api_call_count: int = 1, + ) -> None: + """Record an auxiliary LLM call's usage (vision, compression, title + generation, ...) against *session_id*. + + Writes a per-(model, provider, task) delta into ``session_model_usage`` + WITHOUT touching the ``sessions`` summary row: the gateway overwrites + session counters with absolute main-loop totals, so aux tokens there + would be clobbered or double-counted. Insights read the union. + ``api_call_count`` may aggregate N calls (background-review forks). + Best-effort: callers must never fail an aux call over accounting. + """ + if not session_id or not task: + return + # FK to sessions.id: same INSERT OR IGNORE guard as update_token_counts. + self._insert_session_row(session_id, "unknown") + + def _do(conn): + self._record_model_usage( + conn, + session_id, + model=model, + billing_provider=billing_provider, + billing_base_url=billing_base_url, + billing_mode=None, + input_tokens=input_tokens or 0, + output_tokens=output_tokens or 0, + cache_read_tokens=cache_read_tokens or 0, + cache_write_tokens=cache_write_tokens or 0, + reasoning_tokens=reasoning_tokens or 0, + estimated_cost_usd=estimated_cost_usd, + actual_cost_usd=None, + cost_status=None, + cost_source=None, + api_call_count=( + 1 if api_call_count is None else int(api_call_count) + ), + task=task, + ) + self._execute_write(_do) + + def usage_totals(self, *, min_message_count: int = 1, include_archived: bool = False) -> Dict[str, float]: + """Tokens and spend across the whole store (one scan), so the sidebar + total does not shrink with paging. Spend prefers the billed figure over + the estimate, the same precedence a single row renders.""" + where = ["parent_session_id IS NULL", "message_count >= ?"] + params: List[Any] = [min_message_count] + if not include_archived: + where.append("COALESCE(archived, 0) = 0") + + row = self._read_one( + f""" + SELECT COALESCE(SUM(COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0)), 0), + COALESCE(SUM(COALESCE(actual_cost_usd, estimated_cost_usd, 0)), 0) + FROM sessions + WHERE {' AND '.join(where)} + """, + params, + ) + + return {"tokens": int(row[0] or 0), "cost_usd": float(row[1] or 0.0)} diff --git a/hermes_state_wal.py b/hermes_state_wal.py new file mode 100644 index 0000000000..a6a1ce9171 --- /dev/null +++ b/hermes_state_wal.py @@ -0,0 +1,821 @@ +"""SQLite journal-mode and PRAGMA policy for state.db. + +Split out of ``hermes_state.py``. Every name is re-imported there so +``hermes_state.`` keeps resolving, and tests that monkeypatch it keep +intercepting because intra-module calls to patched helpers go through a lazy +``from hermes_state import ...`` at call time. +""" + +from __future__ import annotations + +import logging +import sqlite3 +import sys +import threading +import time +from typing import Any, Dict, Optional + +from hermes_cli.sqlite_runtime import ( + is_sqlite_wal_reset_vulnerable as _is_sqlite_wal_reset_vulnerable, +) + +# Log-record parity with the origin module (caplog tests pin "hermes_state"). +logger = logging.getLogger("hermes_state") + + +# --------------------------------------------------------------------------- +# WAL-compatibility fallback +# --------------------------------------------------------------------------- +# WAL needs mmap shared memory and fcntl byte-range locks, which network +# filesystems (NFS, SMB/CIFS, some FUSE, WSL1) don't provide reliably — there +# ``PRAGMA journal_mode=WAL`` raises ``locking protocol`` (SQLITE_PROTOCOL). +# ZFS instead corrupts the -shm file under concurrent connection bursts (COW + +# mmap), presenting as ``disk I/O error``. Propagating either would silently +# break everything backed by state.db/kanban.db, so we fall back to +# ``journal_mode=DELETE`` (pre-WAL default, works on NFS/ZFS): readers block +# during a write, but it works. The WAL-reset-bug gate and the +# never-live-downgrade invariant are documented on apply_wal_with_fallback. +_WAL_INCOMPAT_MARKERS = ( + "locking protocol", # SQLITE_PROTOCOL on NFS/SMB + "not authorized", # Some FUSE mounts block WAL pragma outright + "disk i/o error", # ZFS SHM corruption under concurrent connections +) + + +# SQLite's default is -1 (unlimited), so state.db-wal would keep the high-water +# mark of the largest-ever transaction forever. See _apply_wal_size_limit(). +_WAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024 # 64 MiB + + +# Dedup sets: kanban_db.connect() runs on every kanban operation, so an +# undeduped fallback log line would repeat per connection and fill errors.log. +_wal_fallback_warned_paths: set[str] = set() + + +_wal_fallback_warned_lock = threading.Lock() + + +_wal_reset_bug_warned_paths: set[str] = set() + + +_wal_reset_bug_warned_lock = threading.Lock() + + +# "configured delete overridden by on-disk WAL" ERROR. +_delete_overridden_warned_paths: set[str] = set() + + +_delete_overridden_warned_lock = threading.Lock() + + +def _on_disk_journal_mode(conn: sqlite3.Connection) -> Optional[str]: + """Read the journal mode from the DB header; ``None`` if undeterminable. + + ``None`` (new DB, or PRAGMA failed) sends callers down their fail-closed + "unknown → refuse to downgrade" branch. ``disk i/o error`` can be transient + on virtualized block devices (XFS on cloud hosts), so it is retried a few + times first: transient EIO clears, deterministic filesystem errors do not. + """ + last_exc: Optional[Exception] = None + for _ in range(4): + try: + row = conn.execute("PRAGMA journal_mode").fetchone() + except sqlite3.OperationalError as exc: + last_exc = exc + if "disk i/o error" not in str(exc).lower(): + return None + time.sleep(0.05) + continue + if row is None: + return None + mode = row[0] + if isinstance(mode, bytes): # defensive: sqlite3 occasionally returns bytes + try: + mode = mode.decode("ascii") + except UnicodeDecodeError: + return None + return str(mode).strip().lower() if mode is not None else None + if last_exc is not None: + logger.debug( + "_on_disk_journal_mode: retries exhausted on disk read (%s)", last_exc + ) + return None + + +def _apply_wal_size_limit(conn: sqlite3.Connection) -> None: + """Bound the WAL so it returns space to the OS after big transactions. + + SQLite's default ``journal_size_limit`` is -1: a checkpointed WAL is reused + in place, never truncated, so ``state.db-wal`` keeps the high-water mark + of the largest transaction ever run. One bulk op strands gigabytes — + ``hermes sessions optimize`` on a 3 GB state.db left a 3 GB WAL and filled + the disk, so the maintenance command was self-defeating on the largest + DBs. With a limit, each checkpoint truncates the WAL back to it; 64 MiB is + above normal transaction sizes (steady-state commits never pay a truncate) + while capping slack predictably. kanban_db uses ``wal_autocheckpoint=100``. + + Best-effort: never raises — failure only costs disk slack and must not + prevent the database from opening. + """ + try: + conn.execute(f"PRAGMA journal_size_limit={_WAL_SIZE_LIMIT_BYTES}") + except sqlite3.OperationalError as exc: # pragma: no cover - defensive + logger.debug("journal_size_limit not applied: %s", exc) + + +def _apply_macos_checkpoint_barrier(conn: sqlite3.Connection) -> None: + """Enable ``PRAGMA checkpoint_fullfsync`` on macOS (no-op elsewhere). + + Apple's ``fsync(2)`` guarantees neither data-on-platter nor write ordering, + so WAL's corruption-safety assumption fails on Darwin without ``F_FULLFSYNC``. + A launchd shutdown drops the page cache (power-loss for in-flight pages), so + a checkpoint that "reported" durable can leave a malformed ``state.db``; + a plain in-session kill survives via the page cache. The barrier applies + only at checkpoint boundaries (~+0.1 ms/commit vs ~+4 ms for + ``fullfsync=1``). Best-effort: never raises. + """ + if sys.platform != "darwin": + return + try: + conn.execute("PRAGMA checkpoint_fullfsync=1") + except sqlite3.OperationalError: + pass + + +def _enforce_macos_synchronous_full(conn: sqlite3.Connection) -> None: + """Enforce ``PRAGMA synchronous=FULL`` on macOS to prevent btree corruption. + + With NORMAL, a WAL checkpoint racing process termination (launchd shutdown) + can leave half-written btree pages (``btreeInitPage error 11``) because + Darwin's ``fsync()`` guarantees neither ordering nor durability. Called + after every successful WAL activation so a prior connection's NORMAL never + sticks. Best-effort: never raises. + """ + if sys.platform != "darwin": + return + try: + conn.execute("PRAGMA synchronous=FULL") + except sqlite3.OperationalError: + pass + + +def is_sqlite_wal_reset_vulnerable( + version_info: Optional[tuple] = None, +) -> bool: + """True when the linked SQLite has the WAL-reset bug (3.7.0–3.51.2; + fixed 3.51.3+, backports 3.50.7 / 3.44.6). Pre-WAL libraries are safe. + https://sqlite.org/wal.html#walresetbug + """ + info = version_info if version_info is not None else sqlite3.sqlite_version_info + return _is_sqlite_wal_reset_vulnerable(info) + + +def sqlite_source_id() -> str: + """Return ``sqlite_source_id()``, or an empty string when unavailable.""" + try: + conn = sqlite3.connect(":memory:") + try: + row = conn.execute("SELECT sqlite_source_id()").fetchone() + finally: + conn.close() + except sqlite3.Error: + return "" + if not row or row[0] is None: + return "" + return str(row[0]) + + +def _database_has_content(conn: sqlite3.Connection) -> bool: + """Whether the file already holds pages (existing vs brand-new DB). + + ``PRAGMA page_count`` is a lock-free header read. Fail-quiet: any error + answers False, because the only caller gates a warning on this and an + unknown-answer warning would fire on every fresh database — exactly where + there is provably no operator choice being overwritten. + """ + try: + row = conn.execute("PRAGMA page_count").fetchone() + except sqlite3.Error: + return False + if not row or row[0] is None: + return False + try: + return int(row[0]) > 0 + except (TypeError, ValueError): + return False + + +def resolve_journal_mode() -> str: + """Return the configured journal mode (``wal`` or ``delete``). + + ``database.journal_mode`` in config.yaml is the canonical operator setting; + ``wal`` is the default, ``delete`` is for filesystems without WAL-safe + durability (macOS virtiofs, NFS, SMB). Invalid values fail safe to ``wal``. + """ + try: + from hermes_cli.config import load_config_readonly + + config = load_config_readonly() or {} + database = config.get("database", {}) + if not isinstance(database, dict): + return "wal" + raw = database.get("journal_mode", "wal") + except Exception: + return "wal" + + if not isinstance(raw, str): + return "wal" + mode = raw.strip().lower() + return mode if mode in ("wal", "delete") else "wal" + + +class WalUnsupportedError(sqlite3.OperationalError): + """Raised by :func:`apply_wal_with_fallback` when ``require_wal=True`` and + the filesystem cannot provide WAL — whether SQLite *raised* + ``SQLITE_PROTOCOL`` or (macOS NFS) silently returned the still-effective + mode. Subclasses ``OperationalError`` so existing DB-init handlers still + catch it while WAL-mandating callers can catch the narrower type. + """ + + +def apply_wal_with_fallback( + conn: sqlite3.Connection, + *, + db_label: str = "state.db", + require_wal: bool = False, +) -> str: + """Set ``journal_mode=WAL`` on ``conn``, falling back to DELETE on failure. + + Returns the mode actually set (``"wal"`` or ``"delete"``). Shared by + :class:`SessionDB` and ``hermes_cli.kanban_db.connect`` for identical + fallback behavior. + + On WAL-incompatible filesystems (NFS, SMB, some FUSE, ZFS) SQLite either + raises ``OperationalError`` ("locking protocol" / "disk I/O error") or — + macOS NFS / SMB / AgentFS NFS overlay — silently refuses and leaves the DB + in DELETE. Either way we log at ERROR (a write now blocks readers — a real + concurrency loss) and fall back to DELETE so the feature keeps working. + ``require_wal=True`` raises :class:`WalUnsupportedError` instead; all + current callers keep the default so NFS-homed installs work. + + On SQLite builds with the WAL-reset bug (https://sqlite.org/wal.html#walresetbug, + fixed 3.51.3+, backports 3.50.7 / 3.44.6), refuse to enable WAL on + fresh / non-WAL databases; an already-WAL DB keeps WAL with a warning. + This gate is deliberately RETAINED: an attempt to revert it (theory: DELETE + was "the mode that corrupts") was confounded — its clean WAL result came + from SQLite 3.53.1, which also carries 3.51.0's close()-broken-POSIX-lock + defenses. Re-measured on the bundled 3.50.4 with the lock fix, WAL and + DELETE are both clean, so there is no evidence WAL is safer; keep new + databases out of WAL until a fixed runtime ships. + + Invariant on every path (NFS and WAL-reset alike): never downgrade to + DELETE if the on-disk header reports WAL or the mode cannot be read (see + _on_disk_journal_mode). Other gateway/cron/worker connections may hold the + DB open, and a live downgrade destroys their committed-but-uncheckpointed + transactions. + + The ERROR is deduplicated per ``db_label``: once per process per DB, so + state.db and kanban.db on one NFS mount each log once. + """ + from hermes_state import is_sqlite_wal_reset_vulnerable, resolve_journal_mode + configured = resolve_journal_mode() + + # Vulnerable SQLite: never enable WAL on new/non-WAL files. Resolve the + # operator setting first so an explicit DELETE request still verifies SQLite + # accepted DELETE rather than silently returning MEMORY or another mode. + if is_sqlite_wal_reset_vulnerable(): + return _apply_delete_for_wal_reset_bug( + conn, + db_label=db_label, + require_delete=configured == "delete", + ) + + # Read-only probe — no flock, no checkpoint, no WAL/SHM unlink — so + # WAL-init cannot unlink files other connections hold open. + current_mode = _on_disk_journal_mode(conn) + if current_mode == "wal": + if configured == "delete": + # Never-live-downgrade keeps WAL; tell the operator their delete did not apply. + _log_configured_delete_overridden_once(db_label) + _apply_wal_size_limit(conn) + _apply_macos_checkpoint_barrier(conn) + _enforce_macos_synchronous_full(conn) + return "wal" + + # Honor the canonical database.journal_mode setting (on-disk WAL DBs were + # returned above and are never live-downgraded). + if configured == "delete": + if current_mode is None: + # Probe failed (locked/busy): another process may hold this DB open + # in WAL, so ownership is not provably exclusive and flipping modes + # could destroy a concurrent writer's committed-but-uncheckpointed + # transactions. Fail loudly — the operator asked for DELETE and we + # cannot verify it. + raise sqlite3.OperationalError( + "could not verify journal mode before applying configured " + "journal_mode=delete (database is locked — possible " + "concurrent openers); refusing to downgrade a database " + "this process does not exclusively own" + ) + actual = _set_journal_mode_no_wait(conn, "DELETE") + if actual != "delete": + raise sqlite3.OperationalError( + f"could not set configured journal_mode=delete (got {actual or 'no result'})" + ) + return actual + + # Decide BEFORE the flip whether it would overwrite a mode somebody chose: + # the probe and page_count are only readable while the file is untouched. + # A 0-page DB has no prior choice, and every caller reaches this before + # creating schema, so brand-new databases stay quiet. + _upgrading_existing_db = ( + current_mode is not None + and current_mode != "wal" + and _database_has_content(conn) + ) + + try: + # ``PRAGMA journal_mode=WAL`` RETURNS the resulting mode. Filesystems + # that refuse by *raising* SQLITE_PROTOCOL hit the except branch, but + # macOS NFS, SMB/CIFS and the AgentFS NFS overlay refuse WITHOUT raising + # and just return the still-effective mode. Trust the row, not the + # absence of an exception, or we report a false "wal", skip the + # fallback ERROR, and leave the DB silently in DELETE. + row = conn.execute("PRAGMA journal_mode=WAL").fetchone() + mode = str(row[0]).strip().lower() if row and row[0] is not None else "" + if mode == "wal": + if _upgrading_existing_db: + _log_journal_mode_upgrade_once(db_label, current_mode) + _apply_wal_size_limit(conn) + _apply_macos_checkpoint_barrier(conn) + _enforce_macos_synchronous_full(conn) + return "wal" + # Silent refusal: WAL was not honored, but nothing raised. + silent_exc = WalUnsupportedError( + f"journal_mode=WAL refused without raising (still {mode!r})" + ) + if require_wal: + raise silent_exc + _log_wal_fallback_once(db_label, silent_exc) + return mode or "delete" + except sqlite3.OperationalError as exc: + # The require_wal silent-refusal raise above lands here (subclass of + # OperationalError) — propagate unchanged, skip the marker logic. + if isinstance(exc, WalUnsupportedError): + raise + msg = str(exc).lower() + if not any(marker in msg for marker in _WAL_INCOMPAT_MARKERS): + # Unrelated OperationalError — don't silently swallow. + raise + # ``disk i/o error`` is ambiguous: deterministic WAL-incompatibility on + # ZFS / APFS-CoW (SHM corruption under connection bursts), or a one-shot + # transient EIO (page-cache pressure, brief lock contention). Treating + # a transient EIO as a permanent downgrade signal produced mixed-mode + # corruption (process A downgrades to DELETE while siblings set WAL), + # so retry the pragma: transient EIO clears and we return "wal"; + # deterministic cases keep failing into the guarded DELETE fallback. + if "disk i/o error" in msg: + for _ in range(2): + time.sleep(0.05) + try: + row = conn.execute("PRAGMA journal_mode=WAL").fetchone() + except sqlite3.OperationalError as retry_exc: + if "disk i/o error" not in str(retry_exc).lower(): + raise + exc = retry_exc + continue + mode = ( + str(row[0]).strip().lower() + if row and row[0] is not None + else "" + ) + if mode == "wal": + # Transient EIO cleared and the switch went through; same + # header rewrite, so same upgrade signal. + if _upgrading_existing_db: + _log_journal_mode_upgrade_once(db_label, current_mode) + _apply_wal_size_limit(conn) + _apply_macos_checkpoint_barrier(conn) + _enforce_macos_synchronous_full(conn) + return "wal" + break + # Don't downgrade if another process already set WAL on disk, or if the + # mode cannot be read (probe blocked by a concurrent opener's locks) — + # ownership is not provably exclusive either way. + existing = _on_disk_journal_mode(conn) + if existing == "wal" or existing is None: + raise + if require_wal: + raise WalUnsupportedError(str(exc)) from exc + _log_wal_fallback_once(db_label, exc) + _set_journal_mode_no_wait(conn, "DELETE") + return "delete" + + +def _set_journal_mode_no_wait(conn: sqlite3.Connection, mode: str) -> str: + """Execute ``PRAGMA journal_mode=`` without waiting on other openers. + + The ONLY place a journal-mode switch may be issued for a non-WAL target. + Forces ``busy_timeout=0`` so SQLite's exclusivity requirement becomes a + concurrent-opener detector: leaving WAL needs exclusive access, so if ANY + other connection (this process or another) holds the DB the pragma fails + immediately with ``database is locked`` instead of waiting out a busy + timeout and sneaking the flip between a concurrent writer's transactions — + exactly how committed-but-uncheckpointed WAL transactions get destroyed. + + Callers must treat a raised ``OperationalError`` as "not exclusively + owned: leave the journal mode alone", never as retryable. Returns SQLite's + reported mode (lowercase), or ``""`` if no row. + """ + previous_timeout = 0 + try: + row = conn.execute("PRAGMA busy_timeout").fetchone() + if row and row[0] is not None: + previous_timeout = int(row[0]) + except (sqlite3.OperationalError, TypeError, ValueError): + previous_timeout = 0 + conn.execute("PRAGMA busy_timeout=0") + try: + row = conn.execute(f"PRAGMA journal_mode={mode}").fetchone() + return str(row[0]).strip().lower() if row and row[0] is not None else "" + finally: + try: + conn.execute(f"PRAGMA busy_timeout={previous_timeout}") + except sqlite3.OperationalError: + pass + + +def _apply_delete_for_wal_reset_bug( + conn: sqlite3.Connection, + *, + db_label: str, + require_delete: bool = False, +) -> str: + """Avoid enabling WAL when the linked SQLite has the WAL-reset bug. + + - Already-WAL on disk: leave WAL alone (no live downgrade) and warn. + - Mode unreadable (probe blocked by a concurrent opener's locks): not + provably exclusive — leave the mode alone and warn. Never treat "could + not read the mode" as "not WAL": that confusion once flipped a live WAL + state.db to DELETE under a concurrent writer, destroying its + committed-but-uncheckpointed transactions. + - Otherwise: set DELETE (refusing to wait out concurrent openers) and warn. + - For an explicit operator request, verify SQLite accepted DELETE. + """ + current = _on_disk_journal_mode(conn) + + if current == "wal": + _log_wal_reset_bug_once(db_label, kept_wal=True) + if require_delete: + # Upgrading SQLite (the warning above) doesn't help on a + # WAL-incompatible filesystem; emit the actionable message last. + _log_configured_delete_overridden_once(db_label) + # No TRUNCATE / journal_mode=DELETE while other processes may still + # hold this WAL DB open; same safety rule as the NFS path. + _apply_wal_size_limit(conn) + _apply_macos_checkpoint_barrier(conn) + _enforce_macos_synchronous_full(conn) + return "wal" + + if current is None: + # Probe failed — likely another opener's locks, and the DB may be in + # WAL under a live writer. Never flip a mode we cannot even read. + if require_delete: + raise sqlite3.OperationalError( + "could not verify journal mode before applying configured " + "journal_mode=delete (database is locked — possible " + "concurrent openers); refusing to downgrade a database " + "this process does not exclusively own" + ) + _log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True) + return "wal" + + actual = "" + try: + actual = _set_journal_mode_no_wait(conn, "DELETE") + except sqlite3.OperationalError as exc: + if require_delete: + raise + lowered = str(exc).lower() + if "locked" in lowered or "busy" in lowered: + # A concurrent opener appeared between probe and flip (or already + # held the DB): SQLite refused the exclusive lock. Leave the mode as is. + _log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True) + return current or "delete" + # Best-effort for the automatic fallback: DELETE is normally already + # the default for new file-backed databases. + if require_delete and actual != "delete": + raise sqlite3.OperationalError( + "could not set configured journal_mode=delete " + f"(got {actual or 'no result'})" + ) + _log_wal_reset_bug_once(db_label, kept_wal=False) + return "delete" + + +def _wal_reset_repair_hint() -> str: + """Repair hint matching what ``hermes update`` can actually do for this + install type (uv-managed venv vs git/pip/docker/nix).""" + try: + from hermes_cli.config import ( + detect_install_method, + recommended_update_command_for_method, + get_project_root, + ) + method = detect_install_method(get_project_root()) + cmd = recommended_update_command_for_method(method) + if method in {"git", "unknown"}: + return f"Hermes-managed installs can repair the embedded runtime with `{cmd}`" + if method == "docker": + return f"update the container image with `{cmd}`" + # nix/nixos + return cmd + except Exception: + pass + return ( + "install a Python build bundled with SQLite 3.51.3+ " + "(or backports 3.50.7 / 3.44.6) and restart Hermes" + ) + + +# Dedup state for _log_journal_mode_upgrade_once. +_journal_upgrade_warned_paths: set = set() + + +_journal_upgrade_warned_lock = threading.Lock() + + +def _log_wal_reset_bug_once( + db_label: str, + *, + kept_wal: bool, + indeterminate: bool = False, +) -> None: + """Log once per (process, db_label) about the WAL-reset vulnerability path.""" + from hermes_state import _wal_reset_bug_warned_paths + with _wal_reset_bug_warned_lock: + if db_label in _wal_reset_bug_warned_paths: + return + _wal_reset_bug_warned_paths.add(db_label) + if indeterminate: + action = ( + "journal mode could not be verified or exclusively switched " + "(database is locked — possible concurrent openers); leaving the " + "journal mode untouched (no live downgrade under concurrent " + "openers)" + ) + elif kept_wal: + action = ( + "is already in WAL mode — leaving WAL in place (no live " + "downgrade under concurrent openers)" + ) + else: + action = "using journal_mode=DELETE instead of enabling WAL" + # Install-type-aware so the warning never promises a repair path that + # doesn't exist for git/pip/system Python installs. + repair_hint = _wal_reset_repair_hint() + logger.warning( + "%s: linked SQLite %s (interpreter %s) is vulnerable to the WAL-reset " + "corruption bug (https://sqlite.org/wal.html#walresetbug) — %s. " + "Upgrade to SQLite 3.51.3+ (or backports 3.50.7 / 3.44.6); " + "%s. See `hermes doctor`. This warning fires once per " + "process per database.", + db_label, + sqlite3.sqlite_version, + sys.executable, + action, + repair_hint, + ) + + +def _log_journal_mode_upgrade_once(db_label: str, previous_mode: str) -> None: + """Log a single WARNING per (process, db_label) about a non-WAL -> WAL flip. + + ``PRAGMA journal_mode`` is a property of the FILE: switching an existing DB + to WAL rewrites its header and outlives the process. Operators do set + DELETE on the file directly (the documented WAL-reset-bug mitigation), and + nothing told them the next open would silently put WAL back. + + WARNING, not ERROR: the reverse move is ERROR in ``_log_wal_fallback_once`` + because dropping to DELETE loses concurrency, whereas this direction is + normally desirable (managed_uv repairs DELETE-stuck DBs on update). The + only problem was invisibility, so this names the durable setting without + claiming a degradation. Deduped per process per ``db_label`` because + kanban opens a fresh connection per operation. + """ + from hermes_state import _journal_upgrade_warned_paths + with _journal_upgrade_warned_lock: + if db_label in _journal_upgrade_warned_paths: + return + _journal_upgrade_warned_paths.add(db_label) + logger.warning( + "%s: on-disk journal_mode was %s and has been switched to WAL. This " + "rewrites the database header and persists after this process exits. " + "If %s was a deliberate choice (for example the mitigation for the " + "SQLite WAL-reset bug, or a WAL-unsafe filesystem), setting it with " + "PRAGMA on the file will not survive -- every open re-applies the " + "configured mode. Set `database.journal_mode: delete` in config.yaml " + "to make it stick. This message fires once per process per database.", + db_label, + previous_mode, + previous_mode, + ) + + +def _log_wal_fallback_once(db_label: str, exc: Exception) -> None: + """Log a single ERROR per (process, db_label) about WAL fallback. + + ERROR, not WARNING: silently dropping to DELETE is a real concurrency loss + (under kanban dispatcher + workers a write blocks readers as SQLITE_BUSY). + Deduped because kanban opens a fresh connection per operation. + """ + from hermes_state import _wal_fallback_warned_paths + with _wal_fallback_warned_lock: + if db_label in _wal_fallback_warned_paths: + return + _wal_fallback_warned_paths.add(db_label) + logger.error( + "%s: WAL journal_mode unsupported on this filesystem (%s) — " + "falling back to journal_mode=DELETE (slower rollback-journal " + "mode; reduces concurrency but works on NFS/SMB/FUSE/ZFS). See " + "https://www.sqlite.org/wal.html for details. This message " + "fires once per process per database.", + db_label, + exc, + ) + + +def _log_configured_delete_overridden_once(db_label: str) -> None: + """Log a single ERROR per (process, db_label) when the operator configured + ``journal_mode=delete`` but the on-disk DB is already WAL. + + Never-live-downgrade keeps WAL (a live downgrade causes mixed-mode + corruption); without this the operator would never learn that + ``database.journal_mode: delete`` had no effect and that a one-time + offline ``PRAGMA journal_mode=DELETE`` (no open connections) is required. + """ + from hermes_state import _delete_overridden_warned_paths + with _delete_overridden_warned_lock: + if db_label in _delete_overridden_warned_paths: + return + _delete_overridden_warned_paths.add(db_label) + logger.error( + "%s: database.journal_mode=delete is configured but the on-disk " + "database is already WAL; keeping WAL (a live downgrade under open " + "connections can corrupt the DB). To apply journal_mode=DELETE, stop " + "all connections to this DB and run a one-time offline " + "'PRAGMA journal_mode=DELETE' on the file. This message fires once " + "per process per database.", + db_label, + ) + + +# --------------------------------------------------------------------------- +# Config-driven database pragmas +# --------------------------------------------------------------------------- +# Operators write synchronous as a name; mapped here rather than passed through +# so a typo becomes a warning instead of a silently different durability level. +_SYNCHRONOUS_LEVELS: Dict[str, int] = { + "OFF": 0, + "NORMAL": 1, + "FULL": 2, + "EXTRA": 3, +} + + +_SYNCHRONOUS_NAMES: Dict[int, str] = {v: k for k, v in _SYNCHRONOUS_LEVELS.items()} + + +_SYNCHRONOUS_FULL = 2 + + +def resolve_synchronous_level(raw_value: Any) -> Optional[int]: + """Map a configured ``database.synchronous`` value to its PRAGMA integer. + + Accepts SQLite's names (``OFF``/``NORMAL``/``FULL``/``EXTRA``, any case) or + ``0``-``3``. Anything else returns None so the caller warns and leaves the + level untouched — guessing at a malformed durability setting is worse. + """ + if isinstance(raw_value, bool): + # bool is an int subclass and YAML turns bare `on`/`off` into one. + # "off" is a real durability choice; True is meaningless. + return 0 if raw_value is False else None + if isinstance(raw_value, int): + return raw_value if raw_value in _SYNCHRONOUS_NAMES else None + text = str(raw_value).strip() + if not text: + return None + upper = text.upper() + if upper in _SYNCHRONOUS_LEVELS: + return _SYNCHRONOUS_LEVELS[upper] + try: + value = int(text) + except (TypeError, ValueError): + return None + return value if value in _SYNCHRONOUS_NAMES else None + + +def _apply_synchronous_pragma( + conn: sqlite3.Connection, + raw_value: Any, + *, + db_label: str, +) -> None: + """Set ``PRAGMA synchronous`` from config, never below FULL on macOS. + + Kept out of the integer loop in :func:`apply_database_pragmas`: this PRAGMA + decides whether a commit is on the platter, so an unrecognised value must + not fall through to "SQLite default" the way a bad ``cache_size`` can. + + Darwin floor: :func:`_enforce_macos_synchronous_full` runs during + ``apply_wal_with_fallback()`` and this runs after it, so a configured + ``NORMAL`` would otherwise silently undo the macOS btree protection. + Raising the level on macOS is allowed; lowering it is refused out loud. + """ + level = resolve_synchronous_level(raw_value) + if level is None: + logger.warning( + "%s: ignoring unrecognized database.synchronous=%r " + "(expected OFF, NORMAL, FULL, EXTRA, or 0-3)", + db_label, + raw_value, + ) + return + if sys.platform == "darwin" and level < _SYNCHRONOUS_FULL: + logger.warning( + "%s: refusing database.synchronous=%s on macOS; keeping FULL. " + "Darwin's fsync() does not guarantee write ordering, so a lower " + "level readmits the half-written btree pages FULL exists to " + "prevent.", + db_label, + _SYNCHRONOUS_NAMES[level], + ) + return + try: + conn.execute(f"PRAGMA synchronous={level}") + except sqlite3.OperationalError: + pass + + +def apply_database_pragmas( + conn: sqlite3.Connection, + *, + db_label: str = "state.db", +) -> None: + """Apply optional performance and WAL-sizing PRAGMAs from ``config.yaml``. + + Journal mode is NOT handled here — ``database.journal_mode`` is owned by + :func:`resolve_journal_mode` inside :func:`apply_wal_with_fallback`, under + all the safety guards. + + Keys under ``database:``: ``cache_size`` (negative = KiB, positive = + pages), ``mmap_size`` (bytes, 0 = disabled), ``temp_store`` (0-3), + ``wal_autocheckpoint`` (pages), ``journal_size_limit`` (bytes), and + ``synchronous`` (``OFF``/``NORMAL``/``FULL``/``EXTRA`` or ``0``-``3``). + Unset ``synchronous`` leaves SQLite's default, a *compile-time* constant + (``SQLITE_DEFAULT_WAL_SYNCHRONOUS``) that differs between bundled, distro + and Homebrew builds; setting it explicitly is the only way to know. + + Best-effort: config load or pragma failures are ignored so DB init never + breaks on a malformed ``database:`` section. + """ + try: + # Local import avoids a circular import with hermes_cli.config. + from hermes_cli.config import cfg_get, load_config_readonly + + cfg = load_config_readonly() + except Exception: + return + + # Applied to ALL connection types: writer, read_only, WAL per-thread readers. + for pragma_name in ( + "cache_size", + "mmap_size", + "temp_store", + "wal_autocheckpoint", + "journal_size_limit", + ): + raw_value = cfg_get(cfg, "database", pragma_name, default=None) + if raw_value is None: + continue + try: + value = int(str(raw_value).strip()) + except (TypeError, ValueError): + logger.warning( + "%s: ignoring non-integer database.%s=%r", + db_label, + pragma_name, + raw_value, + ) + continue + try: + conn.execute(f"PRAGMA {pragma_name}={value}") + except sqlite3.OperationalError: + pass + + # Last: the sizing pragmas above cannot change durability, and the macOS + # enforcement ran earlier during WAL activation (see _apply_synchronous_pragma + # for why that ordering needs an explicit floor rather than an override). + raw_synchronous = cfg_get(cfg, "database", "synchronous", default=None) + if raw_synchronous is not None: + _apply_synchronous_pragma(conn, raw_synchronous, db_label=db_label) diff --git a/tests/test_state_db_write_durability.py b/tests/test_state_db_write_durability.py index 4ba56e49f5..04ab8c88f1 100644 --- a/tests/test_state_db_write_durability.py +++ b/tests/test_state_db_write_durability.py @@ -102,8 +102,12 @@ def test_repair_path_has_no_bare_connects() -> None: Source-level guard: the bare form is exactly what regressed, and a unit test on the helper alone would not notice a sixth site being added. """ - source = Path(hermes_state.__file__).read_text(encoding="utf-8") - tree = ast.parse(source, filename=str(hermes_state.__file__)) + # The repair/probe helpers live in hermes_state_repair; hermes_state only + # re-imports them. + import hermes_state_repair + + source = Path(hermes_state_repair.__file__).read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(hermes_state_repair.__file__)) def is_db_path_connect(node: ast.AST) -> bool: if not isinstance(node, ast.Call):