From c89819bec27bd9c61d1e75bb7527d30ddf894eed Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:46:47 -0700 Subject: [PATCH 1/3] refactor(gateway/session): split the two >200-LOC methods into collaborators, unify entry mutators, table-drive platform prompt notes --- gateway/session.py | 889 ++++++++++++++++++++++----------------------- 1 file changed, 438 insertions(+), 451 deletions(-) diff --git a/gateway/session.py b/gateway/session.py index 2c0857ab8c..744da51322 100644 --- a/gateway/session.py +++ b/gateway/session.py @@ -137,9 +137,7 @@ def _is_path_unsafe(value: object, *, strict: bool = True) -> bool: return len(s) >= 2 and s[0].isalpha() and s[1] == ":" -def _is_session_key_unsafe(value: object) -> bool: - """Relaxed traversal check for a logical session_key.""" - return _is_path_unsafe(value, strict=False) +_CHAT_TYPE_PREFIX = {"group": "group: ", "channel": "channel: "} @dataclass @@ -203,11 +201,8 @@ class SessionSource: def _describe(chat_type: str, user_label: str, chat_label: str) -> str: if chat_type == "dm": return f"DM with {user_label}" - if chat_type == "group": - return f"group: {chat_label}" - if chat_type == "channel": - return f"channel: {chat_label}" - return chat_label + prefix = _CHAT_TYPE_PREFIX.get(chat_type, "") + return f"{prefix}{chat_label}" @property def description(self) -> str: @@ -401,6 +396,101 @@ def neutralize_untrusted_inline_text(value: Any, *, max_chars: int = _MAX_PROMPT return text +def _slack_platform_notes(context: SessionContext) -> List[str]: + # Capability note only when Slack tools are actually loaded; otherwise + # keep the disclaimer honest so we never promise tools the agent lacks. + if _slack_tools_loaded(): + lines = ["", ( + "**Platform notes:** You are running inside Slack and have access " + "to Slack-specific tools this session. Consult the available Slack " + "tool schemas for the exact operations supported (e.g. channel " + "history and thread lookups, posting, reactions) — use those tools " + "for Slack-specific requests, and do not promise Slack actions " + "beyond what the loaded tools actually expose." + )] + else: + lines = ["", ( + "**Platform notes:** You are running inside Slack. " + "You do NOT have access to Slack-specific APIs — you cannot search " + "channel history, pin/unpin messages, manage channels, or list users. " + "Do not promise to perform these actions. The gateway may inline the " + "current message's Slack block/attachment payload when available, but " + "you still cannot call Slack APIs yourself." + )] + if context.shared_multi_user_session: + lines.append( + "In shared Slack threads, use the current turn's sender prefix " + "as the only verified current-author mention target. Do not " + "guess or reuse `<@U...>` mentions from names, memory, or prior " + "conversation history." + ) + return lines + + +def _discord_platform_notes(context: SessionContext) -> List[str]: + if _discord_tools_loaded(): + src = context.source + lines = ["", "**Discord IDs (for the `discord` / `discord_admin` tools):**"] + if src.guild_id: + lines.append(f" - Guild: `{src.guild_id}`") + if src.thread_id and src.parent_chat_id: + lines.append(f" - Parent channel: `{src.parent_chat_id}`") + lines.append(f" - Thread: `{src.thread_id}` (use as `channel_id` for fetch_messages etc.)") + else: + lines.append(f" - Channel: `{src.chat_id}`") + if src.message_id: + # The volatile per-turn message id must stay OUT of this cached + # block (it would bust the agent-cache signature every message); + # run.py injects it into the user message instead. + lines.append( + " - Triggering message: provided per-turn in the incoming " + "user message (use it as `message_id` for reply/react/pin)" + ) + else: + lines = ["", ( + "**Platform notes:** You are running inside Discord. " + "You do NOT have access to Discord-specific APIs — you cannot search " + "channel history, pin messages, manage roles, or list server members. " + "Do not promise to perform these actions. If the user asks, explain " + "that you can only read messages sent directly to you and respond." + )] + # Static: live voice-channel state arrives on the user message (it + # changed bytes every turn here and busted the prompt cache). + lines += ["", ( + "Voice-channel state, when relevant, appears in the current " + "message as a `[Voice channel now: ...]` note." + )] + return lines + + +_STATIC_PLATFORM_NOTES = { + Platform.BLUEBUBBLES: ( + "**Platform notes:** You are responding via iMessage. " + "Keep responses short and conversational — think texts, not essays. " + "Structure longer replies as separate short thoughts, each separated " + "by a blank line (double newline). Each block between blank lines " + "will be delivered as its own iMessage bubble, so write accordingly: " + "one idea per bubble, 1–3 sentences each. " + "If the user needs a detailed answer, give the short version first " + "and offer to elaborate." + ), + Platform.YUANBAO: ( + "**Platform notes:** You are running inside Yuanbao. " + "To send a private (DM) message to a user in the current group, " + "use the yb_send_dm tool (look up the recipient by name or pass " + "their user_id). Your normal reply is delivered to the group you " + "are responding in." + ), +} + +# Platform -> extra "Platform notes" lines for the session-context prompt. +_PLATFORM_NOTES = { + Platform.SLACK: _slack_platform_notes, + Platform.DISCORD: _discord_platform_notes, + **{p: (lambda ctx, note=note: ["", note]) for p, note in _STATIC_PLATFORM_NOTES.items()}, +} + + def build_session_context_prompt( context: SessionContext, *, @@ -422,6 +512,10 @@ def build_session_context_prompt( except Exception: pass redact_pii = redact_pii and _is_pii_safe + + def _chat_label(chat_id: str) -> str: + return _hash_chat_id(chat_id) if redact_pii else chat_id + lines = [ "## Current Session Context", "", @@ -444,7 +538,7 @@ def build_session_context_prompt( desc = SessionSource._describe( src.chat_type, src.user_name or (_hash_sender_id(src.user_id) if src.user_id else "user"), - src.chat_name or _hash_chat_id(src.chat_id), + src.chat_name or _chat_label(src.chat_id), ) else: desc = src.description @@ -459,14 +553,13 @@ def build_session_context_prompt( if context.source.platform == Platform.MATRIX: src = context.source - room_name = src.chat_name or src.chat_id - room_id = _hash_chat_id(src.chat_id) if redact_pii else src.chat_id - lines.append("") - lines.append(f"**Matrix Room:** {_format_untrusted_prompt_value(room_name)}") - lines.append(f"**Matrix Room ID:** {room_id}") + lines += [ + "", + f"**Matrix Room:** {_format_untrusted_prompt_value(src.chat_name or src.chat_id)}", + f"**Matrix Room ID:** {_chat_label(src.chat_id)}", + ] if src.thread_id: - thread_id = _hash_chat_id(src.thread_id) if redact_pii else src.thread_id - lines.append(f"**Matrix Thread:** {thread_id}") + lines.append(f"**Matrix Thread:** {_chat_label(src.thread_id)}") lines.append( "**Matrix room boundary:** Treat this turn as scoped to the current " "Matrix room/thread only. Do not assume unresolved references are " @@ -492,94 +585,7 @@ def build_session_context_prompt( uid = _hash_sender_id(uid) lines.append(f"**User ID:** {_format_untrusted_prompt_value(uid)}") - # Platform-specific behavioral notes - if context.source.platform == Platform.SLACK: - # Capability note only when Slack tools are actually loaded; otherwise - # keep the disclaimer honest so we never promise tools the agent lacks. - if _slack_tools_loaded(): - lines.append("") - lines.append( - "**Platform notes:** You are running inside Slack and have access " - "to Slack-specific tools this session. Consult the available Slack " - "tool schemas for the exact operations supported (e.g. channel " - "history and thread lookups, posting, reactions) — use those tools " - "for Slack-specific requests, and do not promise Slack actions " - "beyond what the loaded tools actually expose." - ) - else: - lines.append("") - lines.append( - "**Platform notes:** You are running inside Slack. " - "You do NOT have access to Slack-specific APIs — you cannot search " - "channel history, pin/unpin messages, manage channels, or list users. " - "Do not promise to perform these actions. The gateway may inline the " - "current message's Slack block/attachment payload when available, but " - "you still cannot call Slack APIs yourself." - ) - if context.shared_multi_user_session: - lines.append( - "In shared Slack threads, use the current turn's sender prefix " - "as the only verified current-author mention target. Do not " - "guess or reuse `<@U...>` mentions from names, memory, or prior " - "conversation history." - ) - elif context.source.platform == Platform.DISCORD: - if _discord_tools_loaded(): - src = context.source - id_lines = ["", "**Discord IDs (for the `discord` / `discord_admin` tools):**"] - if src.guild_id: - id_lines.append(f" - Guild: `{src.guild_id}`") - if src.thread_id and src.parent_chat_id: - id_lines.append(f" - Parent channel: `{src.parent_chat_id}`") - id_lines.append(f" - Thread: `{src.thread_id}` (use as `channel_id` for fetch_messages etc.)") - else: - id_lines.append(f" - Channel: `{src.chat_id}`") - if src.message_id: - # The volatile per-turn message id must stay OUT of this cached - # block (it would bust the agent-cache signature every message); - # run.py injects it into the user message instead. - id_lines.append( - " - Triggering message: provided per-turn in the incoming " - "user message (use it as `message_id` for reply/react/pin)" - ) - lines.extend(id_lines) - else: - lines.append("") - lines.append( - "**Platform notes:** You are running inside Discord. " - "You do NOT have access to Discord-specific APIs — you cannot search " - "channel history, pin messages, manage roles, or list server members. " - "Do not promise to perform these actions. If the user asks, explain " - "that you can only read messages sent directly to you and respond." - ) - # Static: live voice-channel state arrives on the user message (it - # changed bytes every turn here and busted the prompt cache). - lines.append("") - lines.append( - "Voice-channel state, when relevant, appears in the current " - "message as a `[Voice channel now: ...]` note." - ) - elif context.source.platform == Platform.BLUEBUBBLES: - lines.append("") - lines.append( - "**Platform notes:** You are responding via iMessage. " - "Keep responses short and conversational — think texts, not essays. " - "Structure longer replies as separate short thoughts, each separated " - "by a blank line (double newline). Each block between blank lines " - "will be delivered as its own iMessage bubble, so write accordingly: " - "one idea per bubble, 1–3 sentences each. " - "If the user needs a detailed answer, give the short version first " - "and offer to elaborate." - ) - elif context.source.platform == Platform.YUANBAO: - lines.append("") - lines.append( - "**Platform notes:** You are running inside Yuanbao. " - "To send a private (DM) message to a user in the current group, " - "use the yb_send_dm tool (look up the recipient by name or pass " - "their user_id). Your normal reply is delivered to the group you " - "are responding in." - ) + lines.extend(_PLATFORM_NOTES.get(context.source.platform, lambda ctx: [])(context)) platforms_list = ["local (files on this machine)"] for p in context.connected_platforms: @@ -589,39 +595,32 @@ def build_session_context_prompt( lines.append(f"**Connected Platforms:** {', '.join(platforms_list)}") if context.home_channels: - lines.append("") - lines.append("**Home Channels (default destinations):**") + lines += ["", "**Home Channels (default destinations):**"] for platform, home in context.home_channels.items(): - hc_id = _hash_chat_id(home.chat_id) if redact_pii else home.chat_id safe_name = _format_untrusted_prompt_value(home.name) - safe_id = _format_untrusted_prompt_value(hc_id) + safe_id = _format_untrusted_prompt_value(_chat_label(home.chat_id)) lines.append(f" - {platform.value}: {safe_name} (ID: {safe_id})") - lines.append("") - lines.append("**Delivery options for scheduled tasks:**") + lines += ["", "**Delivery options for scheduled tasks:**"] from hermes_constants import display_hermes_home if context.source.platform == Platform.LOCAL: lines.append("- `\"origin\"` → Local output (saved to files)") else: - _origin_label = context.source.chat_name or ( - _hash_chat_id(context.source.chat_id) if redact_pii else context.source.chat_id + _origin_label = _format_untrusted_prompt_value( + context.source.chat_name or _chat_label(context.source.chat_id) ) - _origin_label = _format_untrusted_prompt_value(_origin_label) lines.append(f"- `\"origin\"` → Back to this chat ({_origin_label})") lines.append( f"- `\"local\"` → Save to local files only ({display_hermes_home()}/cron/output/)" ) - for platform, home in context.home_channels.items(): home_name = _format_untrusted_prompt_value(home.name) lines.append(f"- `\"{platform.value}\"` → Home channel ({home_name})") - lines.append("") - lines.append("*For explicit targeting, use `\"platform:chat_id\"` format if the user provides a specific chat ID.*") - + lines += ["", "*For explicit targeting, use `\"platform:chat_id\"` format if the user provides a specific chat ID.*"] return "\n".join(lines) @@ -781,7 +780,7 @@ class SessionEntry: raise ValueError( "Invalid session_id: potential directory traversal detected" ) - if _is_session_key_unsafe(session_key): + if _is_path_unsafe(session_key, strict=False): raise ValueError( "Invalid session_key: potential directory traversal detected" ) @@ -1240,7 +1239,7 @@ class SessionStore: if entry.session_id == session_id: return entry.session_key except Exception: - pass + pass # bare stores / foreign entry objects in suites return (getattr(self, "_session_owner_hints", None) or {}).get(session_id) def _db_for_session_id(self, session_id: Optional[str]): @@ -2019,11 +2018,15 @@ class SessionStore: except Exception as exc: log(exc) - def _reopen_session_row(self, session_key: str, session_id: str) -> None: + def _reopen_session_row(self, session_key: str, session_id: str, *, log_prefix: str = "") -> None: + """Best-effort ``reopen_session``; failures are debug-logged only.""" try: self._db_for_key(session_key).reopen_session(session_id) except Exception as exc: - logger.debug("Gateway session DB reopen failed for %s: %s", session_key, exc) + if log_prefix: + logger.debug("%s: %s", log_prefix, exc) + else: + logger.debug("Gateway session DB reopen failed for %s: %s", session_key, exc) def _record_gateway_session_peer( self, @@ -2289,6 +2292,107 @@ class SessionStore: with inflight_lock: self._inflight_sessions.pop(session_key, None) + def _adopt_legacy_slack_entry(self, source: SessionSource, session_key: str) -> None: + """One-time migration of pre-workspace-scope Slack keys. + + MOVE (not copy) the legacy entry so a second workspace with identical + Slack ids cannot attach to the same transcript. Adopt when the legacy + origin names the same workspace; a scope-less DM is claimed once by + the first workspace; a scope-less channel/group is refused (channel + ids collide across workspaces). + """ + legacy_key = self._legacy_slack_session_key(source) + if not legacy_key: + return + migrated: Optional[SessionEntry] = None + with self._lock: + self._ensure_loaded_locked() + legacy_entry = self._entries.get(legacy_key) + if session_key not in self._entries and legacy_entry is not None: + origin_scope = getattr(legacy_entry.origin, "scope_id", None) + if origin_scope is not None: + adopt = origin_scope == source.scope_id + else: + adopt = source.chat_type == "dm" + if adopt and self._claim_legacy_slack_key(legacy_key): + migrated = self._entries.pop(legacy_key) + migrated.session_key = session_key + migrated.origin = source + migrated.platform = source.platform + migrated.chat_type = source.chat_type + self._entries[session_key] = migrated + if migrated is not None: + self._save_entries() + self._record_gateway_session_peer( + migrated.session_id, session_key, source, display_name=migrated.display_name, + ) + + def _route_reset_reason( + self, entry: SessionEntry, source: SessionSource, now: datetime + ) -> Optional[str]: + """Reset decision for an existing route (no lock; DB/config I/O). + + ``suspended`` always resets. Otherwise the reset policy decides; a + still-pending resume marker is additionally freshness-gated — but + ``session_reset.mode: none`` (user opted out of ALL automatic resets) + makes an expired marker fall through to a normal resume, never a + silent fresh session. + """ + if entry.suspended: + return "suspended" + reason = self._should_reset(entry, source) + if reason or not entry.resume_pending: + return reason + policy = self.config.get_reset_policy( + platform=source.platform, session_type=source.chat_type, + ) + if policy.mode == "none": + return None + window = auto_continue_freshness_window() + ref_time = entry.last_resume_marked_at or entry.updated_at + if window > 0 and (now - ref_time).total_seconds() > window: + return "resume_pending_expired" + return None + + def _finish_route_transition( + self, + session_key: str, + *, + end_session_id: Optional[str], + end_reason: str, + create_kwargs: Optional[Dict[str, Any]], + origin: Optional[SessionSource], + display_name: Optional[str], + during: str = "", + ) -> None: + """SQLite side of a routing transition, outside ``_lock``. + + Promotes the predecessor row to an explicit reset boundary (with the + specific reason so state.db is auditable, e.g. ``resume_pending_expired`` + vs a plain ``session_reset``), then INSERTs the new row + routing peer. + Both are best-effort: failures are warned and self-healed by the next + per-turn peer refresh. + """ + if self._db_for_key(session_key) and end_session_id: + self._promote_session_reset( + session_key, end_session_id, end_reason, + log=lambda e: logger.warning( + "Failed to end predecessor session row %s for %s%s: %s — " + "the old row remains open and may win restart recovery " + "until the next successful peer refresh", + end_session_id, session_key, during, e, + ), + ) + if self._db_for_key(session_key) and create_kwargs: + self._create_session_row( + session_key, create_kwargs, origin, display_name, + log=lambda e: logger.warning( + "Failed to create session row %s for %s%s: %s — deferring " + "to the self-healing peer refresh on the next turn", + create_kwargs.get("session_id"), session_key, during, e, + ), + ) + def _get_or_create_session_impl( self, source: SessionSource, @@ -2303,100 +2407,32 @@ class SessionStore: """ session_key = self._generate_session_key(source) now = _now() - - # One-time migration of pre-workspace-scope Slack keys: MOVE (not - # copy) the legacy entry so a second workspace with identical Slack - # ids cannot attach to the same transcript. Adopt when the legacy - # origin names the same workspace; a scope-less DM is claimed once by - # the first workspace; a scope-less channel/group is refused (channel - # ids collide across workspaces). - migrated_legacy_entry: Optional[SessionEntry] = None - legacy_key = self._legacy_slack_session_key(source) - if legacy_key and not force_new: - with self._lock: - self._ensure_loaded_locked() - legacy_entry = self._entries.get(legacy_key) - if session_key not in self._entries and legacy_entry is not None: - origin_scope = getattr(legacy_entry.origin, "scope_id", None) - if origin_scope is not None: - adopt = origin_scope == source.scope_id - else: - adopt = source.chat_type == "dm" - if adopt and self._claim_legacy_slack_key(legacy_key): - migrated_legacy_entry = self._entries.pop(legacy_key) - migrated_legacy_entry.session_key = session_key - migrated_legacy_entry.origin = source - migrated_legacy_entry.platform = source.platform - migrated_legacy_entry.chat_type = source.chat_type - self._entries[session_key] = migrated_legacy_entry - if migrated_legacy_entry is not None: - self._save_entries() - self._record_gateway_session_peer( - migrated_legacy_entry.session_id, - session_key, - source, - display_name=migrated_legacy_entry.display_name, - ) + if not force_new: + self._adopt_legacy_slack_entry(source, session_key) db_end_session_id = None db_create_kwargs = None - existing_session_id = None force_new_observed_entry = None - # ---- Phase 0: lock read -- existing session_id for compression tip ---- - if not force_new: - with self._lock: - self._ensure_loaded_locked() - entry = self._entries.get(session_key) - if entry is not None: - existing_session_id = entry.session_id - - # Compression tip lookup outside the lock (DB I/O). - canonical_existing_session_id = ( - self._compression_tip_for_session_id(existing_session_id) - if existing_session_id - else None - ) - - # ---- Phase 1: lock read -- get entry snapshot for stale/reset checks ---- + # ---- Phase 1: lock read -- entry snapshot for stale/reset checks ---- _stale_session_id = None _entry_for_checks = None with self._lock: self._ensure_loaded_locked() if force_new: force_new_observed_entry = self._entries.get(session_key) - if session_key in self._entries and not force_new: + elif session_key in self._entries: _entry_for_checks = self._entries[session_key] _stale_session_id = _entry_for_checks.session_id - # ---- Phase 1b: no-lock I/O -- stale check + reset policy ---- + # ---- Phase 1b: no-lock I/O -- compression tip + stale check + reset policy ---- + canonical_existing_session_id = None _is_stale = False _reset_reason = None - if _entry_for_checks is not None and _stale_session_id is not None: + if _entry_for_checks is not None: + canonical_existing_session_id = self._compression_tip_for_session_id(_stale_session_id) _is_stale = self._is_session_ended_in_db(_stale_session_id) - if _entry_for_checks.suspended: - _reset_reason = "suspended" - elif _entry_for_checks.resume_pending: - _reset_reason = self._should_reset(_entry_for_checks, source) - if not _reset_reason: - # Freshness-gate stale resume_pending zombies — but honor - # ``session_reset.mode: none`` (user opted out of ALL - # automatic resets): an expired marker then falls through - # to a normal resume, never a silent fresh session. - _policy = self.config.get_reset_policy( - platform=source.platform, - session_type=source.chat_type, - ) - if _policy.mode != "none": - _fw = auto_continue_freshness_window() - _ref_time = ( - _entry_for_checks.last_resume_marked_at - or _entry_for_checks.updated_at - ) - if _fw > 0 and (now - _ref_time).total_seconds() > _fw: - _reset_reason = "resume_pending_expired" - else: - _reset_reason = self._should_reset(_entry_for_checks, source) + _reset_reason = self._route_reset_reason(_entry_for_checks, source, now) # ---- Phase 2: lock write -- apply decisions to _entries ---- _needs_save = False @@ -2418,7 +2454,7 @@ class SessionStore: # A heal rewrites entry.session_id, so it must reach the # sessions.json mirror too (forces the full-rewrite save). _healed = self._heal_compression_tip_locked( - entry, existing_session_id, canonical_existing_session_id + entry, _stale_session_id, canonical_existing_session_id ) # If another thread replaced the entry during our lock-free # window, the stale/reset decisions no longer apply: healthy. @@ -2473,11 +2509,7 @@ class SessionStore: else: self._reopen_session_row(session_key, recovered.session_id) with self._lock: - published = self._entries.get(session_key) - if published is None: - self._entries[session_key] = recovered - published = recovered - entry = published + entry = self._entries.setdefault(session_key, recovered) _needs_save = True if entry is None: @@ -2500,16 +2532,10 @@ class SessionStore: ) with self._lock: current = self._entries.get(session_key) - may_publish = current is None or ( - force_new and current is force_new_observed_entry - ) - if may_publish: + if current is None or (force_new and current is force_new_observed_entry): self._entries[session_key] = candidate - published = candidate - else: - published = current - assert published is not None - entry = published + current = candidate + entry = current _needs_save = True if entry is candidate: db_create_kwargs = self._session_create_kwargs( @@ -2527,31 +2553,14 @@ class SessionStore: else: self._save_entries() - # SQLite operations outside the lock. - if self._db_for_key(session_key) and db_end_session_id: - # Use the specific reset reason so state.db is auditable (e.g. - # "resume_pending_expired" vs a plain idle/daily "session_reset"). - self._promote_session_reset( - session_key, db_end_session_id, - auto_reset_reason if auto_reset_reason else "session_reset", - log=lambda e: logger.warning( - "Failed to end predecessor session row %s for %s: %s — " - "the old row remains open and may win restart recovery " - "until the next successful peer refresh", - db_end_session_id, session_key, e, - ), - ) - - if self._db_for_key(session_key) and db_create_kwargs: - self._create_session_row( - session_key, db_create_kwargs, source, entry.display_name, - log=lambda e: logger.warning( - "Failed to create session row %s for %s: %s — deferring " - "to the self-healing peer refresh on the next turn", - db_create_kwargs.get("session_id"), session_key, e, - ), - ) - + self._finish_route_transition( + session_key, + end_session_id=db_end_session_id, + end_reason=auto_reset_reason if auto_reset_reason else "session_reset", + create_kwargs=db_create_kwargs, + origin=source, + display_name=entry.display_name, + ) return entry @staticmethod @@ -2636,72 +2645,53 @@ class SessionStore: display_name=peer_display_name, ) - def get_session_metadata( - self, - session_key: str, - key: str, - default: Any = None, - ) -> Any: + def _update_entry(self, session_key: str, mutate) -> bool: + """Apply ``mutate(entry)`` under ``_lock`` and full-save; False when the + entry is missing or *mutate* returned False (nothing to persist).""" + with self._lock: + entry = self._entry_locked(session_key) + if entry is None or mutate(entry) is False: + return False + self._save() + return True + + def get_session_metadata(self, session_key: str, key: str, default: Any = None) -> Any: """Return a metadata value stored on a live session entry.""" with self._lock: entry = self._entry_locked(session_key) - if entry is None: - return default - return entry.metadata.get(key, default) + return default if entry is None else entry.metadata.get(key, default) - def set_session_metadata( - self, - session_key: str, - key: str, - value: Any, - ) -> bool: + def set_session_metadata(self, session_key: str, key: str, value: Any) -> bool: """Persist a small, JSON-serializable metadata value on a live entry. Deliberately does NOT advance ``updated_at`` (the user-activity clock behind reset policy and the resume freshness gate): a background write must not make an idle session look fresh. """ - with self._lock: - entry = self._entry_locked(session_key) - if entry is None: - return False - entry.metadata[key] = value - self._save() - return True + return self._update_entry(session_key, lambda e: e.metadata.__setitem__(key, value)) - def set_model_override( - self, session_key: str, override: Optional[Dict[str, Any]] - ) -> None: + def set_model_override(self, session_key: str, override: Optional[Dict[str, Any]]) -> None: """Persist (or clear, with ``None``) the session-scoped /model override; only non-secret keys are written (see ``sanitize_model_override``).""" - with self._lock: - entry = self._entry_locked(session_key) - if entry is None: - return - cleaned = sanitize_model_override(override) + cleaned = sanitize_model_override(override) + + def _apply(entry: SessionEntry): if entry.model_override == cleaned: - return + return False entry.model_override = cleaned - self._save() + + self._update_entry(session_key, _apply) def get_model_override(self, session_key: str) -> Optional[Dict[str, str]]: """Return the persisted /model override for *session_key*, if any.""" with self._lock: entry = self._entry_locked(session_key) - if entry is None: - return None - return dict(entry.model_override) if entry.model_override else None + return dict(entry.model_override) if entry and entry.model_override else None def suspend_session(self, session_key: str) -> bool: """Mark a session suspended so it auto-resets on next access (/stop). Returns True if the session existed.""" - with self._lock: - entry = self._entry_locked(session_key) - if entry is not None: - entry.suspended = True - self._save() - return True - return False + return self._update_entry(session_key, lambda e: setattr(e, "suspended", True)) def mark_turn_active(self, session_key: str) -> Optional[str]: """Persist exact ownership of the agent turn running for *session_key*. @@ -2817,38 +2807,30 @@ class SessionStore: self._save() return cleared - def mark_resume_pending( - self, - session_key: str, - reason: str = "restart_timeout", - ) -> bool: + def mark_resume_pending(self, session_key: str, reason: str = "restart_timeout") -> bool: """Mark a session resumable after a restart interruption (keeps the session_id/transcript, unlike ``suspend_session``). True if marked.""" - with self._lock: - entry = self._entry_locked(session_key) - if entry is not None: - # Never override an explicit ``suspended`` (hard forced-wipe). - if entry.suspended: - return False - entry.resume_pending = True - entry.resume_reason = reason - entry.last_resume_marked_at = _now() - self._save() - return True - return False + def _apply(entry: SessionEntry): + # Never override an explicit ``suspended`` (hard forced-wipe). + if entry.suspended: + return False + entry.resume_pending = True + entry.resume_reason = reason + entry.last_resume_marked_at = _now() + + return self._update_entry(session_key, _apply) def clear_resume_pending(self, session_key: str) -> bool: """Clear the resume-pending flag after a successful resumed turn. Returns True if a flag was cleared.""" - with self._lock: - entry = self._entry_locked(session_key) - if entry is None or not entry.resume_pending: + def _apply(entry: SessionEntry): + if not entry.resume_pending: return False entry.resume_pending = False entry.resume_reason = None entry.last_resume_marked_at = None - self._save() - return True + + return self._update_entry(session_key, _apply) def prune_old_entries(self, max_age_days: int) -> int: """Drop routing entries idle (by ``updated_at``) for more than max_age_days. @@ -2906,15 +2888,10 @@ class SessionStore: def reset_session(self, session_key: str, display_name: Optional[str] = None) -> Optional[SessionEntry]: """Force reset a session, creating a new session ID.""" - db_end_session_id = None - db_create_kwargs = None - new_entry = None - with self._lock: old_entry = self._entry_locked(session_key) if old_entry is None: return None - db_end_session_id = old_entry.session_id now = _now() session_id = _new_session_id(now) new_entry = self._replace_route_locked( @@ -2928,33 +2905,17 @@ class SessionStore: origin=old_entry.origin, source_value=old_entry.platform.value if old_entry.platform else "unknown", display_name=old_entry.display_name, - parent_session_id=db_end_session_id, + parent_session_id=old_entry.session_id, ) - db_create_kwargs["model_config"] = {"_reset_from": db_end_session_id} - - if self._db_for_key(session_key) and db_end_session_id: - self._promote_session_reset( - session_key, db_end_session_id, "session_reset", - log=lambda e: logger.warning( - "Failed to end predecessor session row %s for %s during " - "reset: %s — the old row remains open and may win restart " - "recovery until the next successful peer refresh", - db_end_session_id, session_key, e, - ), - ) - - if self._db_for_key(session_key) and db_create_kwargs: - self._create_session_row( - session_key, db_create_kwargs, old_entry.origin, - new_entry.display_name if new_entry else None, - log=lambda e: logger.warning( - "Failed to create session row %s for %s during reset: %s " - "— deferring to the self-healing peer refresh on the next " - "turn", - session_id, session_key, e, - ), - ) - + self._finish_route_transition( + session_key, + end_session_id=old_entry.session_id, + end_reason="session_reset", + create_kwargs=db_create_kwargs, + origin=old_entry.origin, + display_name=new_entry.display_name, + during=" during reset", + ) return new_entry def _replace_route_locked(self, session_key, old_entry, session_id, now, **fields) -> SessionEntry: @@ -3031,10 +2992,7 @@ class SessionStore: ) if self._db_for_key(session_key): - try: - self._db_for_key(session_key).reopen_session(target_session_id) - except Exception as e: - logger.debug("Session DB reopen_session failed: %s", e) + self._reopen_session_row(session_key, target_session_id, log_prefix="Session DB reopen_session failed") self._record_gateway_session_peer( target_session_id, session_key, @@ -3080,7 +3038,7 @@ class SessionStore: return None with self._lock: entry = self._entry_locked(session_key) - return getattr(entry, "session_id", None) if entry else None + return entry.session_id if entry else None def _get_transcript_drain_lock(self): """Return the lock that serializes pending-queue drain boundaries.""" @@ -3104,43 +3062,147 @@ class SessionStore: session_id = reroutes[session_id] return session_id + def _spool_dropped(self, session_id: str, message: Dict[str, Any]): + """Spool one evicted/undeliverable message to disk; path or None.""" + try: + from gateway.shutdown_flush import spool_dropped_transcript_message + + return spool_dropped_transcript_message(session_id, message) + except Exception: + return None + + def _enqueue_transcript_message(self, session_id: str, message: Dict[str, Any]) -> list: + """Queue *message* (retry lock held); evicts + spools the oldest past the cap. + + Spooling uses the same machinery as shutdown flush so the message is + replayed after DB recovery instead of being lost. + """ + pending = self._dirty_transcripts.setdefault(session_id, []) + pending.append(dict(message)) + if len(pending) > self._MAX_PENDING_PER_SESSION: + spool_path = self._spool_dropped(session_id, pending.pop(0)) + if spool_path is not None: + self._lazy("_spooled_drop_sessions", set).add(session_id) + logger.warning( + "Session DB transcript pending queue full for %s " + "(cap=%d); spooled oldest message to %s for replay " + "after DB recovery", + session_id, self._MAX_PENDING_PER_SESSION, spool_path, + ) + else: + logger.warning( + "Session DB transcript pending queue full for %s " + "(cap=%d); dropping oldest message to make room " + "(on-disk spool unavailable)", + session_id, self._MAX_PENDING_PER_SESSION, + ) + return pending + + def _divert_transcript_after_db_replaced( + self, session_id: str, queue_session_id: str, exc: Exception + ) -> None: + """Stop SQLite writes on a replaced/quarantined handle and divert the backlog. + + Retrying cannot succeed and the FTS rebuild must never run on this + handle; the pending queue goes to the on-disk spool + JSONL fallback. + """ + logger.error( + "Session DB refused further writes on this handle for " + "%s (%s); stopping SQLite writes and diverting pending " + "transcripts to the on-disk fallback: %s", + session_id, type(exc).__name__, exc, + ) + with self._transcript_retry_lock: + remaining = list(self._dirty_transcripts.get(queue_session_id, [])) + self._dirty_transcripts.pop(queue_session_id, None) + self._transcript_append_failures.pop(session_id, None) + for dropped in remaining: + try: + from gateway.shutdown_flush import spool_dropped_transcript_message + + spool_dropped_transcript_message(session_id, dropped) + except Exception: + logger.warning( + "pending fallback failed for replaced " + "state.db transcript on %s", + session_id, + exc_info=True, + ) + try: + from hermes_state import divert_session_transcript_jsonl + divert_session_transcript_jsonl(session_id, remaining) + except Exception: + logger.warning( + "JSONL divert failed for replaced state.db " + "transcript on %s", + session_id, + exc_info=True, + ) + + def _live_compression_child(self, session_id: str) -> str: + """Transitive compression tip of *session_id* if it is a different, still-live + row, else "" (a depth-1 lookup misses multi-hop lineages). + + Uses the PARENT's proven owner handle: the child's id is not published + until after its write succeeds, so a by-id lookup would fall back to + the ambient store. + """ + owner_db = self._db_for_session_id(session_id) + if owner_db is None: + return "" + tip = owner_db.get_compression_tip(session_id) + if tip and tip != session_id: + tip_row = owner_db.get_session(tip) + if tip_row is not None and tip_row.get("ended_at") is None: + return str(tip) + return "" + + def _migrate_transcript_queue_to_child( + self, session_id: str, queue_session_id: str, child_id: str, pending: list, msg + ) -> list: + """Move the retry queue + failure counter from parent to child and publish + the reroute (retry lock held). Returns the child's pending list. + + Older parent backlog must precede messages already queued directly on + the child. Routing is published only AFTER the queue moved (caller), so + new child writes cannot bypass older parent backlog. + """ + if pending and pending[0] is msg: + pending.pop(0) + existing_child_pending = self._dirty_transcripts.get(child_id, []) + if pending: + pending.extend(existing_child_pending) + self._dirty_transcripts[child_id] = pending + elif existing_child_pending: + pending = existing_child_pending + self._dirty_transcripts.pop(queue_session_id, None) + previous_failures = self._transcript_append_failures.pop(queue_session_id, 0) + if previous_failures: + self._transcript_append_failures[child_id] = max( + previous_failures, + self._transcript_append_failures.get(child_id, 0), + ) + self._transcript_reroutes[session_id] = child_id + return pending + + def _publish_transcript_reroute(self, session_id: str, child_id: str) -> None: + """Repoint every route at the compression child and save (index authoritative again).""" + with self._lock: + for entry in self._entries.values(): + if entry.session_id == session_id: + entry.session_id = child_id + self._save() + _hints = getattr(self, "_session_owner_hints", None) + if _hints: + _hints.pop(child_id, None) + def _append_to_transcript_serialized( self, session_id: str, message: Dict[str, Any] ) -> None: """Append a message to a session's transcript (SQLite), draining the per-session retry queue.""" with self._transcript_retry_lock: - pending = self._dirty_transcripts.setdefault(session_id, []) - pending.append(dict(message)) - # Cap the queue; spool the evicted oldest message to disk (same - # machinery as shutdown flush) for replay after DB recovery. - if len(pending) > self._MAX_PENDING_PER_SESSION: - dropped = pending.pop(0) - spool_path = None - try: - from gateway.shutdown_flush import ( - spool_dropped_transcript_message, - ) - spool_path = spool_dropped_transcript_message( - session_id, dropped - ) - except Exception: - spool_path = None - if spool_path is not None: - self._lazy("_spooled_drop_sessions", set).add(session_id) - logger.warning( - "Session DB transcript pending queue full for %s " - "(cap=%d); spooled oldest message to %s for replay " - "after DB recovery", - session_id, self._MAX_PENDING_PER_SESSION, spool_path, - ) - else: - logger.warning( - "Session DB transcript pending queue full for %s " - "(cap=%d); dropping oldest message to make room " - "(on-disk spool unavailable)", - session_id, self._MAX_PENDING_PER_SESSION, - ) + pending = self._enqueue_transcript_message(session_id, message) msg = pending[0] queue_session_id = session_id @@ -3166,60 +3228,14 @@ class SessionStore: ) if isinstance(exc, (StateDbReplacedError, StateDbCorruptError)): - # This handle must not touch the file again (replaced - # generation / quarantined corruption): retrying cannot - # succeed and the FTS rebuild must never run on it. Divert. - logger.error( - "Session DB refused further writes on this handle for " - "%s (%s); stopping SQLite writes and diverting pending " - "transcripts to the on-disk fallback: %s", - session_id, type(exc).__name__, exc, - ) - with self._transcript_retry_lock: - remaining = list(self._dirty_transcripts.get(queue_session_id, [])) - self._dirty_transcripts.pop(queue_session_id, None) - self._transcript_append_failures.pop(session_id, None) - for dropped in remaining: - try: - from gateway.shutdown_flush import ( - spool_dropped_transcript_message, - ) - spool_dropped_transcript_message(session_id, dropped) - except Exception: - logger.warning( - "pending fallback failed for replaced " - "state.db transcript on %s", - session_id, - exc_info=True, - ) - try: - from hermes_state import divert_session_transcript_jsonl - divert_session_transcript_jsonl(session_id, remaining) - except Exception: - logger.warning( - "JSONL divert failed for replaced state.db " - "transcript on %s", - session_id, - exc_info=True, - ) + self._divert_transcript_after_db_replaced(session_id, queue_session_id, exc) return if isinstance(exc, CompressionSessionClosedError): - # Resolve the transitive compression tip (a depth-1 lookup - # misses multi-hop lineages); adopt only a different, - # still-live tip, else fail closed. Use the PARENT's proven - # owner handle: the child's id is not published until - # after its write succeeds, so a by-id lookup would fall - # back to the ambient store. + # Adopt only a different, still-live compression tip, else + # fail closed. _owner_key = self._owner_key_for_session_id(session_id) - _owner_db = self._db_for_session_id(session_id) - child_id = "" - if _owner_db is not None: - tip = _owner_db.get_compression_tip(session_id) - if tip and tip != session_id: - tip_row = _owner_db.get_session(tip) - if tip_row is not None and tip_row.get("ended_at") is None: - child_id = str(tip) + child_id = self._live_compression_child(session_id) if child_id: # Record the child's owner BEFORE writing to it (the # reroute is published only after the write succeeds @@ -3232,40 +3248,11 @@ class SessionStore: exc = reroute_exc else: with self._transcript_retry_lock: - if pending and pending[0] is msg: - pending.pop(0) - existing_child_pending = self._dirty_transcripts.get( - child_id, [] + pending = self._migrate_transcript_queue_to_child( + session_id, queue_session_id, child_id, pending, msg ) - if pending: - # Older parent backlog must precede messages - # already queued directly on the child. - pending.extend(existing_child_pending) - self._dirty_transcripts[child_id] = pending - elif existing_child_pending: - pending = existing_child_pending - self._dirty_transcripts.pop(queue_session_id, None) - previous_failures = self._transcript_append_failures.pop( - queue_session_id, 0 - ) - if previous_failures: - self._transcript_append_failures[child_id] = max( - previous_failures, - self._transcript_append_failures.get(child_id, 0), - ) - self._transcript_reroutes[session_id] = child_id queue_session_id = child_id - # Publish routing only after the retry queue has moved, - # so new child writes cannot bypass older parent backlog. - with self._lock: - for entry in self._entries.values(): - if entry.session_id == session_id: - entry.session_id = child_id - self._save() - # The index is authoritative again; drop the hint. - _hints = getattr(self, "_session_owner_hints", None) - if _hints: - _hints.pop(child_id, None) + self._publish_transcript_reroute(session_id, child_id) if not pending: return msg = pending[0] From d7bdf2788d3a8e3272b419e61e5f5fbbf77133e9 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:15:42 -0700 Subject: [PATCH 2/3] refactor(gateway/session): split SessionStore into persistence/recovery/lifecycle/transcript mixins by call-graph cohesion; compact wire helpers --- gateway/session.py | 2207 +------------------------------- gateway/session_lifecycle.py | 385 ++++++ gateway/session_persistence.py | 656 ++++++++++ gateway/session_recovery.py | 519 ++++++++ gateway/session_transcript.py | 641 ++++++++++ 5 files changed, 2264 insertions(+), 2144 deletions(-) create mode 100644 gateway/session_lifecycle.py create mode 100644 gateway/session_persistence.py create mode 100644 gateway/session_recovery.py create mode 100644 gateway/session_transcript.py diff --git a/gateway/session.py b/gateway/session.py index 744da51322..1c536e6226 100644 --- a/gateway/session.py +++ b/gateway/session.py @@ -17,7 +17,7 @@ import threading import uuid from pathlib import Path from datetime import datetime, timedelta -from dataclasses import dataclass, field, fields, replace +from dataclasses import dataclass, field, fields from typing import Dict, List, Optional, Any logger = logging.getLogger(__name__) @@ -112,9 +112,10 @@ from .whatsapp_identity import ( canonical_whatsapp_identifier, normalize_whatsapp_identifier, # noqa: F401 - re-exported for gateway.session callers ) -from utils import atomic_replace -from agent.turn_context import extract_api_content_sidecar -import contextlib +from gateway.session_persistence import SessionPersistenceMixin +from gateway.session_recovery import SessionRecoveryMixin +from gateway.session_lifecycle import SessionLifecycleMixin +from gateway.session_transcript import SessionTranscriptMixin def _is_path_unsafe(value: object, *, strict: bool = True) -> bool: """True if ``value`` could traverse outside the sessions dir. @@ -218,57 +219,51 @@ class SessionSource: parts.append(f"thread: {self.thread_id}") return ", ".join(parts) + # Wire layout (order matters for byte-stable JSON): always-present fields, + # then truthy-only optionals around the dual-written scope pair. + _ALWAYS_FIELDS = ("chat_id", "chat_name", "chat_type", "user_id", "user_name", "thread_id", "chat_topic") + _OPTIONAL_PRE_SCOPE = ("user_id_alt", "chat_id_alt") + _OPTIONAL_POST_SCOPE = ("parent_chat_id", "message_id", "profile") + _OPTIONAL_TAIL = ("auto_thread_initial_name", "prospective_thread_id") + def to_dict(self) -> Dict[str, Any]: - d = { - "platform": self.platform.value, - "chat_id": self.chat_id, - "chat_name": self.chat_name, - "chat_type": self.chat_type, - "user_id": self.user_id, - "user_name": self.user_name, - "thread_id": self.thread_id, - "chat_topic": self.chat_topic, - } - def _optional(*names: str) -> None: + d = {"platform": self.platform.value} + d.update((name, getattr(self, name)) for name in self._ALWAYS_FIELDS) + + def _optional(names) -> None: for name in names: value = getattr(self, name) if value: d[name] = value - _optional("user_id_alt", "chat_id_alt") + _optional(self._OPTIONAL_PRE_SCOPE) # Dual-write scope_id + deprecated guild_id alias during the migration. scope = self.scope_id if self.scope_id is not None else self.guild_id if scope: d["scope_id"] = scope d["guild_id"] = scope - _optional("parent_chat_id", "message_id", "profile") + _optional(self._OPTIONAL_POST_SCOPE) if self.auto_thread_created: d["auto_thread_created"] = True - _optional("auto_thread_initial_name", "prospective_thread_id") + _optional(self._OPTIONAL_TAIL) return d @classmethod def from_dict(cls, data: Dict[str, Any]) -> "SessionSource": + plain = { + name: data.get(name) + for name in cls._ALWAYS_FIELDS[1:] + cls._OPTIONAL_PRE_SCOPE + cls._OPTIONAL_POST_SCOPE + cls._OPTIONAL_TAIL + if name != "chat_type" + } return cls( platform=Platform(data["platform"]), chat_id=str(data["chat_id"]), - chat_name=data.get("chat_name"), chat_type=data.get("chat_type", "dm"), - user_id=data.get("user_id"), - user_name=data.get("user_name"), - thread_id=data.get("thread_id"), - chat_topic=data.get("chat_topic"), - user_id_alt=data.get("user_id_alt"), - chat_id_alt=data.get("chat_id_alt"), scope_id=data.get("scope_id", data.get("guild_id")), - parent_chat_id=data.get("parent_chat_id"), - message_id=data.get("message_id"), - profile=data.get("profile"), auto_thread_created=bool(data.get("auto_thread_created", False)), - auto_thread_initial_name=data.get("auto_thread_initial_name"), - prospective_thread_id=data.get("prospective_thread_id"), + **plain, ) - + @dataclass @@ -278,13 +273,13 @@ class SessionContext: connected_platforms: List[Platform] home_channels: Dict[Platform, HomeChannel] shared_multi_user_session: bool = False - + # Session metadata session_key: str = "" session_id: str = "" created_at: Optional[datetime] = None updated_at: Optional[datetime] = None - + def to_dict(self) -> Dict[str, Any]: return { "source": self.source.to_dict(), @@ -340,11 +335,9 @@ def _slack_tools_loaded() -> bool: try: from hermes_cli.config import load_config from hermes_cli.tools_config import _get_platform_tools - cfg = load_config() # include_default_mcp_servers defaults True so a default-enabled Slack # MCP server counts too. - enabled = _get_platform_tools(cfg, "slack") - return "slack" in enabled + return "slack" in _get_platform_tools(load_config(), "slack") except Exception: return False @@ -360,8 +353,7 @@ def _discord_tools_loaded() -> bool: if not (get_secret("DISCORD_BOT_TOKEN", "") or "").strip(): return False - cfg = load_config() - enabled = _get_platform_tools(cfg, "discord", include_default_mcp_servers=False) + enabled = _get_platform_tools(load_config(), "discord", include_default_mcp_servers=False) return "discord" in enabled or "discord_admin" in enabled except Exception: return False @@ -649,10 +641,10 @@ class SessionEntry: session_id: str created_at: datetime updated_at: datetime - + # Origin metadata for delivery routing origin: Optional[SessionSource] = None - + # Display metadata display_name: Optional[str] = None platform: Optional[Platform] = None @@ -661,7 +653,7 @@ class SessionEntry: # Small, JSON-serializable per-entry state (e.g. Slack thread watermarks); # persisted in the routing index. metadata: Dict[str, Any] = field(default_factory=dict) - + # Token tracking input_tokens: int = 0 output_tokens: int = 0 @@ -670,10 +662,10 @@ class SessionEntry: total_tokens: int = 0 estimated_cost_usd: float = 0.0 cost_status: str = "unknown" - + # Last API-reported prompt tokens (for accurate compression pre-check) last_prompt_tokens: int = 0 - + # Set when a session was created because the previous one expired; # consumed once by the message handler to inject a notice into context was_auto_reset: bool = False @@ -748,13 +740,10 @@ class SessionEntry: if self.origin: result["origin"] = self.origin.to_dict() return result - + @classmethod def from_dict(cls, data: Dict[str, Any]) -> "SessionEntry": - origin = None - if "origin" in data and isinstance(data["origin"], dict): - origin = SessionSource.from_dict(data["origin"]) - + origin = SessionSource.from_dict(data["origin"]) if isinstance(data.get("origin"), dict) else None platform = None if data.get("platform"): try: @@ -777,13 +766,9 @@ class SessionEntry: # CWE-22: session_id becomes a filename (strict guard); session_key is # a logical routing key where interior ``/`` is legitimate (relaxed). if _is_path_unsafe(session_id): - raise ValueError( - "Invalid session_id: potential directory traversal detected" - ) + raise ValueError("Invalid session_id: potential directory traversal detected") if _is_path_unsafe(session_key, strict=False): - raise ValueError( - "Invalid session_key: potential directory traversal detected" - ) + raise ValueError("Invalid session_key: potential directory traversal detected") defaults = {f.name: f.default for f in fields(cls)} plain = {name: data.get(name, defaults[name]) for name in cls._PLAIN_FIELDS} @@ -825,10 +810,8 @@ def build_channel_continuity_note( """ if source.platform not in (Platform.SLACK, Platform.DISCORD): return None - if not getattr(entry, "reset_had_activity", False): - return None - prev = getattr(entry, "prev_session_id", None) - if not prev: + prev = entry.prev_session_id + if not entry.reset_had_activity or not prev: return None where = "thread" if source.thread_id else "channel" @@ -978,7 +961,12 @@ class AsyncSessionStore: _DB_UNPINNED = object() -class SessionStore: +class SessionStore( + SessionPersistenceMixin, + SessionRecoveryMixin, + SessionLifecycleMixin, + SessionTranscriptMixin, +): """Session storage/retrieval: SQLite (SessionDB) for metadata and transcripts, legacy JSONL fallback when SQLite is unavailable.""" @@ -1018,37 +1006,32 @@ class SessionStore: self._fts_rebuild_attempted = False self._has_active_processes_fn = has_active_processes_fn # Keep the legacy sessions.json mirror (disable via gateway.write_sessions_json). - self._write_sessions_json = bool( - getattr(config, "write_sessions_json", True) - ) + self._write_sessions_json = bool(getattr(config, "write_sessions_json", True)) - # SQLite handles are cached per resolved path and looked up through - # the ``_db`` property rather than bound once here: a multiplexed - # gateway serves every profile from ONE process, and a handle bound in - # __init__ would be frozen to the root home so every profile's rows - # land in the root state.db. Priming the current scope's handle below - # keeps startup diagnostics (live-DB guard, JSONL warning) at - # construction time. + # SQLite handles are cached per resolved path and resolved through the + # ``_db`` property, never bound once here: a multiplexed gateway serves + # every profile from ONE process and a handle frozen to the root home + # would land every profile's rows in the root state.db. Priming the + # current scope below keeps startup diagnostics at construction time. self._db_pinned = _DB_UNPINNED self._db_handles: Dict[Path, Any] = {} self._db_handles_lock = threading.Lock() - # profile name -> its HERMES_HOME; memoized so the per-key store - # lookup is a dict hit, not a profile-directory stat per append. + # profile name -> HERMES_HOME; memoized so per-key store lookup is a + # dict hit, not a profile-directory stat per append. self._profile_home_cache: Dict[str, Optional[Path]] = {} - # session_id -> owning routing key, for ids whose ownership is proven - # but not yet published in ``_entries`` (compression continuation: - # the child row is written before its reroute is published). + # session_id -> owning routing key for ids whose ownership is proven + # but not yet published in ``_entries`` (compression child row is + # written before its reroute is published). self._session_owner_hints: Dict[str, str] = {} from gateway.session_db_recovery import RecoverableHandleCache self._db_handle_cache = RecoverableHandleCache( - handles=self._db_handles, - lock=self._db_handles_lock, + handles=self._db_handles, lock=self._db_handles_lock, ) # The routing index is one process-wide structure keyed by - # ``agent::…``, so it needs exactly one home for its lifetime; - # capture the gateway's own home at startup (before any profile scope - # exists) — see ``_routing_db``. + # ``agent::…`` and needs exactly one home for its lifetime: + # the gateway's own, captured before any profile scope exists + # (see ``_routing_db``). try: from hermes_constants import get_hermes_home @@ -1070,203 +1053,6 @@ class SessionStore: setattr(self, name, value) return value - def _open_session_db_for_active_scope(self, db_path: Optional[Path] = None): - """Return the SessionDB for the profile scope active on this task. - - ``db_path`` pins the store explicitly; otherwise ``_default_db_path()`` - follows the context-local HERMES_HOME installed by - ``_profile_runtime_scope`` (resolving per call, not once in - ``__init__``, is what lets multiplexed profiles reach their own store). - Handles are cached per resolved path; failed opens enter a bounded - backoff during which callers keep using the JSONL fallback. - """ - from hermes_state import _default_db_path, get_shared_session_db - - path = Path(db_path) if db_path is not None else Path(_default_db_path()) - def _open(): - try: - # Process-wide shared registry: one writer connection per path. - return get_shared_session_db(path) - except Exception as e: - if isinstance(e, RuntimeError) and "live-system guard" in str(e): - # Test-isolation guard: must stay a loud failure and is - # deliberately not cached so it fires again next attempt. - raise - print(f"[gateway] Warning: SQLite session store unavailable, falling back to JSONL: {e}") - raise - - return self._db_handle_cache.get( - path, - _open, - non_cacheable=lambda exc: ( - isinstance(exc, RuntimeError) and "live-system guard" in str(exc) - ), - ) - - def _pinned_db(self): - """Return the explicitly pinned DB (``store._db = x``), else ``_DB_UNPINNED``.""" - return getattr(self, "_db_pinned", _DB_UNPINNED) - - @property - def _db(self): - """The SessionDB for the active profile scope, or a pinned override. - - Assigning ``store._db`` pins that value for every subsequent read - (tests install a fake or disable the DB with ``store._db = None``). - Unpinned, each read resolves the scope so a multiplexed profile's - writes reach its own store. - """ - pinned = self._pinned_db() - if pinned is not _DB_UNPINNED: - return pinned - return self._open_session_db_for_active_scope() - - @_db.setter - def _db(self, value) -> None: - self._db_pinned = value - - @property - def _routing_db(self): - """The one store that owns the routing index, whatever scope is active. - - ``_entries`` is a single flat dict holding every profile's keys, so it - must persist to a single file (``_routing_home``), not whichever - profile happens to be scoped — otherwise a rewrite during one - profile's turn and the unscoped startup load see different copies, - and crash markers written under a secondary profile go unrecovered. - A pinned handle still wins. Bare test instances lacking the handle - cache report no DB. - """ - pinned = self._pinned_db() - if pinned is not _DB_UNPINNED: - return pinned - home = getattr(self, "_routing_home", None) - try: - if home is None: - return self._db - return self._open_session_db_for_active_scope(db_path=home / "state.db") - except Exception: - return None - - def _named_profile_for_key(self, session_key: Optional[str]) -> Optional[str]: - """The non-default profile that owns *session_key*, or None. - - None means the ambient store is authoritative (multiplexing off, or - legacy ``agent:main`` namespace). It deliberately does NOT cover "that - profile has no directory" — ownership and resolvability are separate - questions that ``_db_for_key`` answers separately. - """ - if not getattr(self.config, "multiplex_profiles", False): - return None - profile = self._profile_from_session_key(session_key) - if not profile or profile == "default": - return None - return profile - - def _profile_home_for_key(self, session_key: Optional[str]) -> Optional[Path]: - """HERMES_HOME of the profile that owns *session_key*, or None. - - None means only "no live home to point at" — no named owner, or the - owner's directory could not be resolved. - """ - profile = self._named_profile_for_key(session_key) - if profile is None: - return None - cache = self._profile_home_cache - if profile in cache: - return cache[profile] - home: Optional[Path] = None - try: - from hermes_cli.profiles import get_profile_dir, profile_exists - - if profile_exists(profile): - home = Path(get_profile_dir(profile)) - except Exception as exc: - logger.debug("Could not resolve profile home for %r: %s", session_key, exc) - home = None - # Only hits are memoized: a profile directory can be provisioned - # *after* startup (enrollment bridge), and a cached miss would pin - # that profile's rows to the ambient store for the process lifetime. - if home is not None: - cache[profile] = home - return home - - def _db_for_key(self, session_key: Optional[str]): - """The SessionDB holding *session_key*'s rows, whatever scope is active. - - ``_db`` follows the ambient HERMES_HOME, which only the inbound - message path installs; background work (e.g. the expiry watcher) - runs unscoped over every profile's keys and would otherwise write - profile rows into the ROOT store, drifting from the real row until - the stale-route self-heal drops a live conversation. The owning - profile is encoded in the key, so derive the store from it. - """ - pinned = self._pinned_db() - if pinned is not _DB_UNPINNED: - return pinned - profile = self._named_profile_for_key(session_key) - if profile is None: - return self._db - home = self._profile_home_for_key(session_key) - if home is None: - # Named owner we cannot resolve (not provisioned yet, or lookup - # failed). Falling back to the ambient store would split ONE - # session identity across two physical stores — fail closed; - # callers already handle a missing DB. - logger.warning( - "gateway.session: profile %r has no resolvable home (key %r); " - "refusing to fall back to the ambient store", - profile, session_key, - ) - return None - try: - return self._open_session_db_for_active_scope(db_path=home / "state.db") - except Exception: - # Same contract as ``_db``: a failed open degrades to JSONL fallback. - return None - - def _owner_key_for_session_id(self, session_id: Optional[str]) -> Optional[str]: - """The routing key that owns *session_id*, or None. - - The published index is authoritative; ``_session_owner_hints`` covers - the window where ownership is proven but routing not yet published. - Deliberately lock-free: several callers already hold ``_lock``. - """ - if not session_id: - return None - try: - for entry in list(self._entries.values()): - if entry.session_id == session_id: - return entry.session_key - except Exception: - pass # bare stores / foreign entry objects in suites - return (getattr(self, "_session_owner_hints", None) or {}).get(session_id) - - def _db_for_session_id(self, session_id: Optional[str]): - """The SessionDB holding *session_id*'s row (owner recovered from the - index or a pre-published hint; unknown ids fall back to the ambient store).""" - if not session_id: - return self._db - return self._db_for_key(self._owner_key_for_session_id(session_id)) - - def close_all_db_handles(self) -> None: - """Close every SessionDB handle this store opened, one per resolved path. - - Closing just ``store._db`` at shutdown would strand every secondary - profile's handle with its WAL lock held (restart flows then hit - 'database is locked'). Handles are drained under the lock but closed - outside it so concurrent resolvers never wait on N ``close()`` calls. - A pinned handle is deliberately not closed — the pinner owns it. - """ - def _close(db) -> None: - # Shared instances no-op on close(); release the refcount instead. - from hermes_state import release_or_close - try: - release_or_close(db) - except Exception as exc: - logger.debug("SessionDB close error during handle sweep: %s", exc) - - self._db_handle_cache.close_all(_close) def _has_active_processes_safe(self, session_key: str, *, context: str) -> bool: """Return whether a session has active work, failing closed on registry errors.""" @@ -1282,947 +1068,17 @@ class SessionStore: exc, ) return True - - def _ensure_loaded(self) -> None: - """Load sessions index from disk if not already loaded.""" - with self._lock: - self._ensure_loaded_locked() - def _entry_locked(self, session_key: str) -> Optional[SessionEntry]: - """Load the index and return the entry for *session_key*. Lock held.""" - self._ensure_loaded_locked() - return self._entries.get(session_key) - def _routing_scope(self) -> str: - """Namespace for this store's gateway_routing rows: the resolved - sessions_dir, so stores with different dirs never share entries.""" - try: - return str(Path(self.sessions_dir).resolve()) - except Exception: - return str(self.sessions_dir) - def _routing_db_method(self, name: str): - """Bound ``_routing_db.`` if the handle exists and has it, else None.""" - db = self._routing_db - method = getattr(db, name, None) if db else None - return method if callable(method) else None - @staticmethod - def _routing_entry_from_json(key: str, entry_json: str) -> Optional[SessionEntry]: - """Parse one gateway_routing row; None (with a warning) when invalid.""" - try: - entry_data = json.loads(entry_json) - if isinstance(entry_data, dict): - return SessionEntry.from_dict(entry_data) - except (ValueError, KeyError, TypeError) as e: - logger.warning("Skipping invalid routing entry %r: %s", key, e) - return None - def _ensure_loaded_locked(self) -> None: - """Load the routing index. Must be called with self._lock held. - state.db ``gateway_routing`` is primary; sessions.json is the legacy - import for keys the DB lacks (persisted to the DB on the next _save). - """ - if self._loaded: - self._reconcile_recovered_routing_locked() - return - self.sessions_dir.mkdir(parents=True, exist_ok=True) - db_had_entries = False - db_load_succeeded = False - loader = self._routing_db_method("load_gateway_routing_entries") - if loader is not None: - try: - for key, entry_json in loader(scope=self._routing_scope()).items(): - entry = self._routing_entry_from_json(key, entry_json) - if entry is not None: - self._entries[key] = entry - db_had_entries = bool(self._entries) - db_load_succeeded = True - except Exception as e: - logger.warning("gateway.session: state.db routing load failed: %s", e) - # Legacy import: sessions.json only fills keys the DB lacks. - sessions_file = self.sessions_dir / "sessions.json" - if sessions_file.exists(): - try: - with open(sessions_file, "r", encoding="utf-8") as f: - data = json.load(f) - imported = 0 - for key, entry_data in data.items(): - # "_"-prefixed keys are sentinels (e.g. "_README"), not entries. - if key.startswith("_"): - continue - if key in self._entries: - continue - # A non-dict entry (corrupt file) must not abort the - # whole load. - if not isinstance(entry_data, dict): - logger.warning( - "Skipping invalid session entry %r: " - "expected dict, got %s", - key, type(entry_data).__name__, - ) - continue - try: - self._entries[key] = SessionEntry.from_dict(entry_data) - imported += 1 - except (ValueError, KeyError, TypeError) as e: - logger.warning("Skipping invalid session entry %r: %s", key, e) - if imported and db_had_entries: - logger.info( - "gateway.session: imported %d legacy sessions.json " - "entr%s missing from state.db routing table", - imported, "y" if imported == 1 else "ies", - ) - except Exception as e: - print(f"[gateway] Warning: Failed to load sessions: {e}") - self._loaded = True - self._routing_db_loaded = db_load_succeeded - self._routing_fallback_baseline = ( - None - if db_load_succeeded - else {key: entry.to_dict() for key, entry in self._entries.items()} - ) - # A hard crash skips graceful shutdown and leaves sessions.json - # pointing at ended sessions; self-heal before the first message. - self._prune_stale_sessions_locked() - - def _prune_stale_sessions_locked(self) -> None: - """Remove routing entries whose session has ended in state.db (startup, lock held). - - Stale == ``end_reason IS NOT NULL``. Rows absent from the DB are kept; - a ``None`` DB handle is a no-op; DB errors are non-fatal. - """ - if not self._entries: - return - - stale_keys: list = [] - recovered_keys = 0 - try: - for key, entry in self._entries.items(): - # Ask the store that owns the key, not the ambient handle, or a - # live secondary-profile session gets pruned on the root copy. - db = self._db_for_key(key) - if db is None: - continue - row = db.get_session(entry.session_id) - if row is not None and row.get("end_reason") is not None: - recovered_entry = None - if entry.origin is not None: - try: - recovered_entry = self._recover_session_from_db( - session_key=key, - source=entry.origin, - now=_now(), - raise_on_lookup_error=True, - ) - except Exception as exc: - # Indeterminate: keep the only routing handle. - logger.debug( - "gateway.session: recovery lookup failed for stale " - "sessions.json entry %r -> %s: %s", - key, - entry.session_id, - exc, - ) - continue - - # Compression-ended parent with a newer live child for the - # same peer: repoint instead of dropping, or queued/ - # resume-pending work vanishes until the next message. - if recovered_entry is not None and recovered_entry.session_id != entry.session_id: - logger.warning( - "gateway.session: repointing stale sessions.json entry " - "%r from ended %s (end_reason=%r) to recovered %s", - key, - entry.session_id, - row["end_reason"], - recovered_entry.session_id, - ) - self._entries[key] = recovered_entry - recovered_keys += 1 - continue - - # Same-id recovery == successful resume: keep the ORIGINAL - # entry object (the recovered one is rebuilt minimal and - # would drop counters, model_override, resume markers, - # metadata). Nothing changes, so no save. - if recovered_entry is not None: - logger.info( - "gateway.session: reopened ended session %s for " - "sessions.json entry %r (end_reason=%r); keeping route", - entry.session_id, key, row["end_reason"], - ) - continue - - logger.warning( - "gateway.session: pruning stale sessions.json entry " - "%r -> %s (end_reason=%r); left by a crashed gateway", - key, entry.session_id, row["end_reason"], - ) - stale_keys.append(key) - except Exception as exc: - logger.warning( - "gateway.session: stale-entry pruning skipped due to DB error: %s", - exc, - ) - return - - for key in stale_keys: - del self._entries[key] - - if stale_keys or recovered_keys: - self._save() - - def _save(self) -> None: - """Persist the routing index while the caller holds ``_lock``.""" - data, generation = self._snapshot_routing_locked() - self._persist_routing_data(data, generation) - - def _next_routing_generation_locked(self) -> int: - """Bump and return the shared routing counter. Caller holds ``_lock``. - - Full snapshots AND single-entry fast saves MUST allocate from this one - counter: the stale-write protection is a total order over - serialization times and silently breaks otherwise. - """ - self._routing_generation = getattr(self, "_routing_generation", 0) + 1 - return self._routing_generation - - def _reconcile_recovered_routing_locked(self) -> None: - """Merge authoritative rows after a fallback-only startup load.""" - baseline = getattr(self, "_routing_fallback_baseline", None) - if getattr(self, "_routing_db_loaded", False) or baseline is None: - return - - loader = self._routing_db_method("load_gateway_routing_entries") - if loader is None: - return - try: - durable = loader(scope=self._routing_scope()) - except Exception as exc: - logger.warning("gateway.session: recovered state.db routing load failed: %s", exc) - return - - current = {key: entry.to_dict() for key, entry in self._entries.items()} - for key, entry_json in durable.items(): - durable_entry = self._routing_entry_from_json(key, entry_json) - if durable_entry is None: - continue - - if key not in baseline: - # A key created while on fallback wins over a DB-only key; - # otherwise restore the authoritative row that fallback never saw. - self._entries.setdefault(key, durable_entry) - elif key not in current: - # The key was loaded from fallback and deliberately removed. - continue - elif current[key] == baseline[key]: - # Unchanged fallback data yields to the authoritative DB copy. - self._entries[key] = durable_entry - - self._routing_db_loaded = True - self._routing_fallback_baseline = None - - def _snapshot_routing_locked(self) -> tuple[Dict[str, Any], int]: - """Capture immutable routing data and a monotonic generation.""" - self._reconcile_recovered_routing_locked() - return ( - {key: entry.to_dict() for key, entry in self._entries.items()}, - self._next_routing_generation_locked(), - ) - - def _persist_routing_data(self, data: Dict[str, Any], generation: int) -> None: - """Serialize all whole-index writers through one durable write lock.""" - with self._lazy("_save_lock", threading.Lock): - if generation <= getattr(self, "_persisted_routing_generation", 0): - return - # Fold in fast upserts numbered above this snapshot: they were - # serialized after us and a delayed full rewrite must not regress them. - fast_persisted = getattr(self, "_fast_persisted_entries", None) - if fast_persisted: - for key, (revision, entry_json) in fast_persisted.items(): - if revision > generation: - data[key] = json.loads(entry_json) - db_saved = False - replacer = self._routing_db_method("replace_gateway_routing_entries") - if replacer is not None: - try: - replacer( - {k: json.dumps(v) for k, v in data.items()}, - scope=self._routing_scope(), - ) - db_saved = True - except Exception as exc: - logger.warning("gateway.session: state.db routing save failed: %s", exc) - if getattr(self, "_write_sessions_json", True) or not db_saved: - try: - self._save_sessions_json(data) - except Exception as exc: - if not db_saved: - raise - # state.db is authoritative. A failed legacy mirror must not - # report the already-committed primary write as failed. - logger.warning( - "gateway.session: sessions.json mirror save failed " - "after state.db commit: %s", - exc, - ) - self._persisted_routing_generation = generation - # This rewrite supersedes fast records at or below its - # generation; newer ones stay for the next delayed full writer. - if fast_persisted: - for key in [ - k for k, (rev, _) in fast_persisted.items() - if rev <= generation - ]: - del fast_persisted[key] - - def _save_sessions_json(self, data: Dict[str, Any]) -> None: - """Write the legacy sessions.json mirror of the routing index.""" - import tempfile - self.sessions_dir.mkdir(parents=True, exist_ok=True) - sessions_file = self.sessions_dir / "sessions.json" - - # Self-documenting sentinel; "_" keys are skipped on load. Ordered - # first so it renders at the top of the file. - data = { - "_README": ( - "LEGACY MIRROR of the gateway routing index (the primary copy " - "lives in the gateway_routing table in ~/.hermes/state.db). " - "Maps messaging session keys (agent:main::...) to " - "active session IDs. This is NOT the session list. ALL " - "sessions (CLI, TUI, and gateway) live in ~/.hermes/state.db " - "and are shown by `hermes sessions list` and `/sessions`. " - "Disable this file with `gateway.write_sessions_json: false` " - "in config.yaml." - ), - **data, - } - fd, tmp_path = tempfile.mkstemp( - dir=str(self.sessions_dir), suffix=".tmp", prefix=".sessions_" - ) - try: - with os.fdopen(fd, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2) - f.flush() - os.fsync(f.fileno()) - atomic_replace(tmp_path, sessions_file) - except BaseException: - try: - os.unlink(tmp_path) - except OSError as e: - logger.debug("Could not remove temp file %s: %s", tmp_path, e) - raise - - def _save_entries(self) -> None: - """Snapshot latest state under ``_lock`` and persist after releasing it.""" - with self._lock: - data, generation = self._snapshot_routing_locked() - self._persist_routing_data(data, generation) - - def _save_entry( - self, - session_key: str, - *, - entry_data: Optional[Dict[str, Any]] = None, - lock_held: bool = False, - ) -> None: - """Persist ONE routing entry via UPSERT — the per-turn fast path. - - A full index rewrite re-serializes every entry and fsyncs a multi-MB - sessions.json (~50ms at ~1100 keys, twice per turn); a single-row - UPSERT takes well under a millisecond. Invariants: - - - The key -> session_id mapping never changes here; structural - transitions (create/recover/reset/switch/prune/heal) use the full - rewrite, which also refreshes the legacy sessions.json mirror. The - mirror may lag in metadata only; state.db stays primary. - - Ordering: the entry is serialized under ``_lock`` with a revision - from the shared routing generation counter, so a higher number - always means same-or-newer data. Under ``_save_lock`` the upsert is - skipped if a full snapshot or a fast save of this key with a higher - number already persisted. The reverse (delayed full rewrite after a - later fast save) is handled in ``_persist_routing_data``. - - No DB, or a failed upsert, falls back to the full rewrite so - DB-less installs keep sessions.json durable every turn. - - ``entry_data`` persists a candidate before it is published to the live - entry (failure-atomic metadata transitions); the full-save fallback - carries the same candidate. - """ - def _capture() -> Optional[tuple[str, int, Optional[Dict[str, Any]]]]: - entry = self._entries.get(session_key) - if entry is None: - return None - serialized_entry = ( - dict(entry_data) if entry_data is not None else entry.to_dict() - ) - entry_json = json.dumps(serialized_entry) - revision = self._next_routing_generation_locked() - # The O(n) full snapshot is deferred to the fallback branch. - return entry_json, revision, serialized_entry if entry_data is not None else None - - if lock_held: - captured = _capture() - else: - with self._lock: - captured = _capture() - if captured is None: - return - entry_json, revision, candidate_entry = captured - saver = self._routing_db_method("save_gateway_routing_entry") - if saver is not None: - try: - with self._lazy("_save_lock", threading.Lock): - if getattr(self, "_persisted_routing_generation", 0) >= revision: - return - fast_persisted = self._lazy("_fast_persisted_entries", dict) - persisted = fast_persisted.get(session_key) - if persisted is not None and persisted[0] >= revision: - return - saver(session_key, entry_json, scope=self._routing_scope()) - fast_persisted[session_key] = (revision, entry_json) - return - except Exception as exc: - logger.warning( - "gateway.session: single-entry routing save failed for %r " - "(%s); falling back to full index rewrite", - session_key, exc, - ) - if candidate_entry is not None: - # Full-snapshot fallback carrying the candidate transition. - def _snapshot() -> Dict[str, Any]: - return {key: current.to_dict() for key, current in self._entries.items()} - if lock_held: - fallback_data = _snapshot() - else: - with self._lock: - fallback_data = _snapshot() - fallback_data[session_key] = candidate_entry - self._persist_routing_data(fallback_data, revision) - else: - self._save_entries() - - def _resolve_profile_for_key(self, source: Optional[SessionSource] = None) -> Optional[str]: - """Profile namespace for session keys: None when multiplexing is off - (legacy ``agent:main``), else ``source.profile`` or the active profile.""" - if not getattr(self.config, "multiplex_profiles", False): - return None - if source is not None and source.profile: - return source.profile - try: - from hermes_cli.profiles import get_active_profile_name - return get_active_profile_name() or "default" - except Exception: - return None - - @staticmethod - def _profile_from_session_key(session_key: Optional[str]) -> Optional[str]: - """Extract the profile namespace encoded in a gateway session key.""" - if not session_key: - return None - parts = str(session_key).split(":") - if len(parts) < 2 or parts[0] != "agent": - return None - namespace = parts[1] or "main" - return "default" if namespace == "main" else namespace - - @staticmethod - def _active_profile_name() -> str: - try: - from hermes_cli.profiles import get_active_profile_name - return get_active_profile_name() or "default" - except Exception: - return "default" - - def _recovered_row_allowed_for_active_profile( - self, - *, - requested_session_key: str, - recovered: Dict[str, Any], - ) -> bool: - """Prevent a gateway from reviving another profile's row. - - Single-profile: the row's namespace must match the ACTIVE profile. - Multiplexed: it must match the namespace of the requested key (the - active profile is meaningless there). Keyless rows stay adoptable. - """ - recovered_key = str(recovered.get("session_key") or "") - if not recovered_key or recovered_key == requested_session_key: - return True - - recovered_profile = self._profile_from_session_key(recovered_key) - if recovered_profile is None: - return True - - if getattr(self.config, "multiplex_profiles", False): - requested_profile = self._profile_from_session_key(requested_session_key) - return requested_profile is None or recovered_profile == requested_profile - - return recovered_profile == self._active_profile_name() - - def _generate_session_key(self, source: SessionSource, key_source: Optional[SessionSource] = None) -> str: - """Session key for *source* (profile resolved from *source*, key built - from *key_source* when given).""" - return build_session_key( - key_source if key_source is not None else source, - group_sessions_per_user=getattr(self.config, "group_sessions_per_user", True), - thread_sessions_per_user=getattr(self.config, "thread_sessions_per_user", False), - profile=self._resolve_profile_for_key(source), - ) - - def _legacy_slack_session_key(self, source: SessionSource) -> Optional[str]: - """Pre-workspace Slack key for an explicitly scoped source. - - Deliberately Slack-only; an unscoped Slack session may be claimed by - only one workspace because its old key cannot distinguish teams. - """ - if source.platform != Platform.SLACK or not source.scope_id: - return None - return self._generate_session_key( - source, replace(source, scope_id=None, guild_id=None) - ) - - def _claim_legacy_slack_key(self, legacy_key: Optional[str]) -> bool: - """Atomically reserve one ambiguous legacy Slack key for migration.""" - if not legacy_key: - return False - with self._lazy("_legacy_slack_claim_lock", threading.Lock): - claimed = self._lazy("_claimed_legacy_slack_keys", set) - if legacy_key in claimed: - return False - claimed.add(legacy_key) - return True - - @staticmethod - def _recovered_row_matches_source_scope( - recovered: Dict[str, Any], source: SessionSource - ) -> bool: - """Reject recovered rows whose recorded origin belongs to another workspace. - - A workspace-scoped Slack lookup adopts a row only if its origin_json - names the same scope_id; rows without a parseable origin are rejected - (an unattributable transcript is exactly the ambiguity to avoid). - """ - if ( - source.platform != Platform.SLACK - or source.chat_type == "dm" - or not source.scope_id - ): - return True - try: - origin = json.loads(recovered.get("origin_json") or "") - except (TypeError, ValueError): - return False - if not isinstance(origin, dict): - return False - return origin.get("scope_id", origin.get("guild_id")) == source.scope_id - - def _create_entry_from_recovered_row( - self, - *, - row: Dict[str, Any], - session_key: str, - source: SessionSource, - now: datetime, - ) -> SessionEntry: - def _ts(value, default: datetime) -> datetime: - try: - return datetime.fromtimestamp(float(value)) - except (TypeError, ValueError, OSError): - return default - - # An invalid durable timestamp must look old, never freshly active. - created_at = _ts(row.get("started_at"), datetime.fromtimestamp(0)) - # The finder already returns durable recency; no extra round-trip. - last_activity = row.get("last_activity_at") - updated_at = _ts(last_activity, created_at) if last_activity is not None else created_at - had_activity = row.get("_has_messages") - if had_activity is None: - had_activity = bool(row.get("message_count") or 0) or ( - last_activity is not None - ) - return SessionEntry( - session_key=session_key, - session_id=str(row["id"]), - created_at=created_at, - updated_at=updated_at, - origin=source, - display_name=source.chat_name, - platform=source.platform, - chat_type=source.chat_type, - reset_had_activity=bool(had_activity), - ) - - def _find_gateway_session_row( - self, - *, - session_key: str, - source: SessionSource, - allow_peer_fallback: bool, - raise_on_lookup_error: bool = False, - ) -> Optional[Dict[str, Any]]: - """Query one durable gateway session row. - - Scoped Slack lookups disable SessionDB's platform/chat/user fallback: - that tuple does not contain a workspace id and could therefore revive - another team's session. The caller performs one explicit exact lookup - of the old unscoped key instead. - """ - db = self._db_for_key(session_key) - finder = getattr(db, "find_latest_gateway_session_for_peer", None) if db else None - if not callable(finder): - return None - try: - return finder( - source=source.platform.value, - user_id=source.user_id, - session_key=session_key, - chat_id=source.chat_id if allow_peer_fallback else None, - chat_type=source.chat_type if allow_peer_fallback else None, - thread_id=source.thread_id, - ) - except Exception as exc: - logger.debug("Gateway session DB recovery failed for %s: %s", session_key, exc) - if raise_on_lookup_error: - raise - return None - - def _recover_session_from_db( - self, - *, - session_key: str, - source: SessionSource, - now: datetime, - raise_on_lookup_error: bool = False, - ) -> Optional[SessionEntry]: - """Rebuild a missing session-key mapping from durable state.db data. - - Returns ``None`` when no row is recoverable, or when the recovered - session is already overdue under the reset policy — the row is then - durably promoted to a reset boundary instead of resurrected. - """ - entry, migrated_legacy = self._query_recoverable_row( - session_key=session_key, - source=source, - now=now, - raise_on_lookup_error=raise_on_lookup_error, - ) - if entry is None: - return None - reset_reason = self._should_reset(entry, source) - if reset_reason: - self._promote_session_reset( - session_key, entry.session_id, reset_reason, - log=lambda exc: logger.debug( - "Gateway recovered-session reset promotion failed for %s: %s", - session_key, exc, - ), - ) - return None - self._reopen_session_row(session_key, entry.session_id) - if migrated_legacy: - self._record_gateway_session_peer( - entry.session_id, session_key, source, display_name=entry.display_name, - ) - return entry - - def _query_recoverable_session(self, *, session_key, source, now): - """DB-only half of _recover_session_from_db (no lock needed). - - Returns a SessionEntry or None. Caller assigns _entries[key] under - lock. The row is NOT reopened here: the caller evaluates reset policy - first (an agent_close/ws_orphan row may need promotion to a real reset - boundary instead). - """ - entry, migrated_legacy = self._query_recoverable_row( - session_key=session_key, source=source, now=now, - ) - if entry is not None and migrated_legacy: - self._record_gateway_session_peer( - entry.session_id, session_key, source, display_name=entry.display_name, - ) - return entry - - def _query_recoverable_row( - self, *, session_key, source, now, raise_on_lookup_error=False, - ) -> tuple[Optional[SessionEntry], bool]: - """Find and gate a recoverable row -> (entry or None, migrated_legacy). - - The legacy (pre-workspace) Slack key fallback lives here: exact-key - lookup, claimed once per process; ``migrated_legacy`` tells the caller - to rewrite the peer row to the scoped key. - """ - legacy_key = self._legacy_slack_session_key(source) - recovered = self._find_gateway_session_row( - session_key=session_key, - source=source, - allow_peer_fallback=legacy_key is None, - raise_on_lookup_error=raise_on_lookup_error, - ) - migrated_legacy = False - if ( - not recovered - and legacy_key - and self._claim_legacy_slack_key(legacy_key) - ): - recovered = self._find_gateway_session_row( - session_key=legacy_key, - source=source, - allow_peer_fallback=False, - raise_on_lookup_error=raise_on_lookup_error, - ) - migrated_legacy = bool(recovered) - if not isinstance(recovered, dict): - return None, False - if not self._recovered_row_matches_source_scope(recovered, source): - return None, False - if not self._recovered_row_allowed_for_active_profile( - requested_session_key=session_key, - recovered=recovered, - ): - logger.warning( - "Gateway session DB recovery ignored %s for %s because " - "the row belongs to a different profile", - recovered.get("session_key"), - session_key, - ) - return None, False - entry = self._create_entry_from_recovered_row( - row=recovered, session_key=session_key, source=source, now=now, - ) - return entry, migrated_legacy - - def _promote_session_reset(self, session_key: str, session_id: str, reason: str, *, log) -> None: - """End *session_id* with *reason* via ``promote_to_session_reset``. - - Promote (not plain ``end_session``): a row already ended with a - recoverable accidental reason (agent_close / ws_orphan_reap) must be - upgraded to the explicit boundary, or stale-route recovery resurrects - it over the reset. Falls back to ``end_session`` on old SessionDBs. - ``log(exc)`` reports failures (each caller has its own message). - """ - try: - db = self._db_for_key(session_key) - promote = getattr(db, "promote_to_session_reset", None) - if callable(promote): - promote(session_id, reason) - else: - db.end_session(session_id, reason) - except Exception as exc: - log(exc) - - def _reopen_session_row(self, session_key: str, session_id: str, *, log_prefix: str = "") -> None: - """Best-effort ``reopen_session``; failures are debug-logged only.""" - try: - self._db_for_key(session_key).reopen_session(session_id) - except Exception as exc: - if log_prefix: - logger.debug("%s: %s", log_prefix, exc) - else: - logger.debug("Gateway session DB reopen failed for %s: %s", session_key, exc) - - def _record_gateway_session_peer( - self, - session_id: str, - session_key: str, - source: Optional[SessionSource], - display_name: Optional[str] = None, - include_compression_ancestors: bool = False, - ) -> None: - """Persist the routing peer for an existing gateway session row.""" - db = self._db_for_key(session_key) - if not db or not source: - return - recorder = getattr(db, "record_gateway_session_peer", None) - if not callable(recorder): - return - peer = dict( - source=source.platform.value, - user_id=source.user_id, - session_key=session_key, - chat_id=source.chat_id, - chat_type=source.chat_type, - thread_id=source.thread_id, - ) - try: - origin_json = None - with contextlib.suppress(Exception): - origin_json = json.dumps(source.to_dict()) - recorder( - session_id, - **peer, - display_name=display_name or source.chat_name, - origin_json=origin_json, - include_compression_ancestors=include_compression_ancestors, - ) - except TypeError: - # Older SessionDB without display_name/origin_json kwargs. - try: - recorder(session_id, **peer) - except Exception as exc: - logger.debug("Gateway session peer record failed for %s: %s", session_key, exc) - except Exception as exc: - logger.debug("Gateway session peer record failed for %s: %s", session_key, exc) - - def set_expiry_finalized( - self, entry: SessionEntry, *, clear_model_override: bool = True - ) -> None: - """Mark a session entry expiry-finalized in memory, sessions.json, AND state.db. - - Single write-path for the expiry watcher so the durable flag survives - sessions.json loss. ``clear_model_override=False`` = flag only. - """ - with self._lock: - entry.expiry_finalized = True - if clear_model_override: - # Finalization is a conversation boundary: drop the persisted - # /model override so a later message cannot rehydrate it. - entry.model_override = None - self._save() - # Background caller never entered ``_profile_runtime_scope``: resolve - # the store from the key, not the ambient scope. - _db = self._db_for_key(entry.session_key) - if _db: - setter = getattr(_db, "set_expiry_finalized", None) - if callable(setter): - try: - setter(entry.session_id, True) - except Exception as exc: - logger.debug("Session DB expiry_finalized write failed for %s: %s", entry.session_id, exc) - try: - # Without a durable ``session_reset`` end_reason, later agent - # cleanup ends the row as ``agent_close``, which stale-route - # recovery treats as resumable. Promotion only upgrades live/ - # agent_close rows; explicit boundaries are preserved. - _db.promote_to_session_reset(entry.session_id) - except Exception as exc: - logger.debug("Session DB promote_to_session_reset failed for %s: %s", entry.session_id, exc) - - @staticmethod - def _policy_reset_reason(policy, updated_at: datetime) -> Optional[str]: - """Return "idle"/"daily" when *updated_at* is overdue under *policy*, else None.""" - if policy.mode == "none": - return None - now = _now() - if policy.mode in {"idle", "both"} and now > updated_at + timedelta(minutes=policy.idle_minutes): - return "idle" - if policy.mode in {"daily", "both"}: - today_reset = now.replace( - hour=policy.at_hour, minute=0, second=0, microsecond=0, - ) - if now.hour < policy.at_hour: - today_reset -= timedelta(days=1) - if updated_at < today_reset: - return "daily" - return None - - def _is_session_expired(self, entry: SessionEntry) -> bool: - """Whether the entry's reset policy has expired it (entry alone, no source). - - Used by the background expiry watcher. Sessions with active - background processes are never considered expired. - """ - if self._has_active_processes_safe(entry.session_key, context="expiry"): - logger.debug("Session %s not expired — active background processes", entry.session_key) - return False - policy = self.config.get_reset_policy( - platform=entry.platform, - session_type=entry.chat_type, - ) - return self._policy_reset_reason(policy, entry.updated_at) is not None - - def is_session_finalizable(self, entry: SessionEntry) -> bool: - """True if the expiry watcher will *ever* finalize this session. - - A ``mode == "none"`` session never expires, so the agent-cache idle - sweep must reap its agent itself instead of deferring to the watcher - (deferring would pin the agent for the gateway's lifetime). Policy - resolution errors count as "not finalizable" (sweep reaps — safe). - """ - try: - policy = self.config.get_reset_policy( - platform=entry.platform, - session_type=entry.chat_type, - ) - return policy.mode != "none" - except Exception: - return False - - def _is_session_ended_in_db(self, session_id: str) -> bool: - """True iff state.db has this session with a non-null end_reason. - - Same staleness test as ``_prune_stale_sessions_locked`` (no DB, no - row, or DB error -> False, keep). Used by ``get_or_create_session`` - to self-heal at routing time, since the startup prune cannot see a - session ended while the gateway stays alive. Store resolved from the - row's owning profile, not the ambient scope. - """ - db = self._db_for_session_id(session_id) - if not db or not session_id: - return False - try: - row = db.get_session(session_id) - except Exception: - return False - return bool(row is not None and row.get("end_reason") is not None) - - def _should_reset(self, entry: SessionEntry, source: SessionSource) -> Optional[str]: - """Return the reset reason ("idle"/"daily") if policy says reset, else None. - - Sessions with active background processes are never reset. - """ - session_key = self._generate_session_key(source) - if self._has_active_processes_safe(session_key, context="reset"): - logger.debug("Session reset skipped for %s — active background processes", session_key) - return None - policy = self.config.get_reset_policy( - platform=source.platform, - session_type=source.chat_type - ) - return self._policy_reset_reason(policy, entry.updated_at) - - def _compression_tip_for_session_id(self, session_id: Optional[str]) -> Optional[str]: - """Latest compression continuation for *session_id* (heals a mapping - left pointing at a compressed parent by a restart or failed send).""" - if not session_id: - return session_id - db = self._db_for_session_id(session_id) - if db is None: - return session_id - try: - return db.get_compression_tip(session_id) or session_id - except Exception: - logger.debug("Compression-tip lookup failed for session %s", session_id, exc_info=True) - return session_id - - def _heal_compression_tip_locked( - self, - entry: "SessionEntry", - original_session_id: Optional[str], - canonical_session_id: Optional[str], - ) -> bool: - """Rewrite *entry* to the compression continuation if stale. Lock held.""" - if ( - not original_session_id - or not canonical_session_id - or entry.session_id != original_session_id - or canonical_session_id == original_session_id - ): - return False - logger.info( - "SessionStore healed compressed session mapping: %s -> %s", - entry.session_id, - canonical_session_id, - ) - entry.session_id = canonical_session_id - return True def has_any_sessions(self) -> bool: """Whether any session has ever been created (across all platforms). @@ -2292,106 +1148,6 @@ class SessionStore: with inflight_lock: self._inflight_sessions.pop(session_key, None) - def _adopt_legacy_slack_entry(self, source: SessionSource, session_key: str) -> None: - """One-time migration of pre-workspace-scope Slack keys. - - MOVE (not copy) the legacy entry so a second workspace with identical - Slack ids cannot attach to the same transcript. Adopt when the legacy - origin names the same workspace; a scope-less DM is claimed once by - the first workspace; a scope-less channel/group is refused (channel - ids collide across workspaces). - """ - legacy_key = self._legacy_slack_session_key(source) - if not legacy_key: - return - migrated: Optional[SessionEntry] = None - with self._lock: - self._ensure_loaded_locked() - legacy_entry = self._entries.get(legacy_key) - if session_key not in self._entries and legacy_entry is not None: - origin_scope = getattr(legacy_entry.origin, "scope_id", None) - if origin_scope is not None: - adopt = origin_scope == source.scope_id - else: - adopt = source.chat_type == "dm" - if adopt and self._claim_legacy_slack_key(legacy_key): - migrated = self._entries.pop(legacy_key) - migrated.session_key = session_key - migrated.origin = source - migrated.platform = source.platform - migrated.chat_type = source.chat_type - self._entries[session_key] = migrated - if migrated is not None: - self._save_entries() - self._record_gateway_session_peer( - migrated.session_id, session_key, source, display_name=migrated.display_name, - ) - - def _route_reset_reason( - self, entry: SessionEntry, source: SessionSource, now: datetime - ) -> Optional[str]: - """Reset decision for an existing route (no lock; DB/config I/O). - - ``suspended`` always resets. Otherwise the reset policy decides; a - still-pending resume marker is additionally freshness-gated — but - ``session_reset.mode: none`` (user opted out of ALL automatic resets) - makes an expired marker fall through to a normal resume, never a - silent fresh session. - """ - if entry.suspended: - return "suspended" - reason = self._should_reset(entry, source) - if reason or not entry.resume_pending: - return reason - policy = self.config.get_reset_policy( - platform=source.platform, session_type=source.chat_type, - ) - if policy.mode == "none": - return None - window = auto_continue_freshness_window() - ref_time = entry.last_resume_marked_at or entry.updated_at - if window > 0 and (now - ref_time).total_seconds() > window: - return "resume_pending_expired" - return None - - def _finish_route_transition( - self, - session_key: str, - *, - end_session_id: Optional[str], - end_reason: str, - create_kwargs: Optional[Dict[str, Any]], - origin: Optional[SessionSource], - display_name: Optional[str], - during: str = "", - ) -> None: - """SQLite side of a routing transition, outside ``_lock``. - - Promotes the predecessor row to an explicit reset boundary (with the - specific reason so state.db is auditable, e.g. ``resume_pending_expired`` - vs a plain ``session_reset``), then INSERTs the new row + routing peer. - Both are best-effort: failures are warned and self-healed by the next - per-turn peer refresh. - """ - if self._db_for_key(session_key) and end_session_id: - self._promote_session_reset( - session_key, end_session_id, end_reason, - log=lambda e: logger.warning( - "Failed to end predecessor session row %s for %s%s: %s — " - "the old row remains open and may win restart recovery " - "until the next successful peer refresh", - end_session_id, session_key, during, e, - ), - ) - if self._db_for_key(session_key) and create_kwargs: - self._create_session_row( - session_key, create_kwargs, origin, display_name, - log=lambda e: logger.warning( - "Failed to create session row %s for %s%s: %s — deferring " - "to the self-healing peer refresh on the next turn", - create_kwargs.get("session_id"), session_key, during, e, - ), - ) def _get_or_create_session_impl( self, @@ -2563,54 +1319,6 @@ class SessionStore: ) return entry - @staticmethod - def _session_create_kwargs( - *, session_id, session_key, origin, source_value, display_name, parent_session_id, - ) -> Dict[str, Any]: - """kwargs for ``SessionDB.create_session``. - - Identity (origin_json) and lineage (parent/_reset_from) land atomically - in the INSERT so a crash right after cannot strand the row unroutable. - """ - origin_json = None - if origin is not None: - try: - origin_json = json.dumps(origin.to_dict()) - except Exception: - origin_json = None - return { - "session_id": session_id, - "source": source_value, - "user_id": origin.user_id if origin else None, - "session_key": session_key, - "chat_id": origin.chat_id if origin else None, - "chat_type": origin.chat_type if origin else None, - "thread_id": origin.thread_id if origin else None, - "profile_name": origin.profile if origin else None, - "origin_json": origin_json, - "display_name": display_name, - "parent_session_id": parent_session_id, - "model_config": ( - {"_reset_from": parent_session_id} if parent_session_id else None - ), - } - - def _create_session_row(self, session_key, db_create_kwargs, origin, display_name, *, log) -> None: - """INSERT a session row and record its routing peer; ``log(exc)`` on failure. - - A failed create is a routing hazard (visible warning), but the row is - self-healed with full identity by the next per-turn peer refresh. - """ - try: - self._db_for_key(session_key).create_session(**db_create_kwargs) - self._record_gateway_session_peer( - db_create_kwargs["session_id"], - session_key, - origin, - display_name=display_name, - ) - except Exception as e: - log(e) def update_session( self, @@ -2645,15 +1353,6 @@ class SessionStore: display_name=peer_display_name, ) - def _update_entry(self, session_key: str, mutate) -> bool: - """Apply ``mutate(entry)`` under ``_lock`` and full-save; False when the - entry is missing or *mutate* returned False (nothing to persist).""" - with self._lock: - entry = self._entry_locked(session_key) - if entry is None or mutate(entry) is False: - return False - self._save() - return True def get_session_metadata(self, session_key: str, key: str, default: Any = None) -> Any: """Return a metadata value stored on a live session entry.""" @@ -2688,203 +1387,6 @@ class SessionStore: entry = self._entry_locked(session_key) return dict(entry.model_override) if entry and entry.model_override else None - def suspend_session(self, session_key: str) -> bool: - """Mark a session suspended so it auto-resets on next access (/stop). - Returns True if the session existed.""" - return self._update_entry(session_key, lambda e: setattr(e, "suspended", True)) - - def mark_turn_active(self, session_key: str) -> Optional[str]: - """Persist exact ownership of the agent turn running for *session_key*. - - The opaque token is returned to the caller and must be supplied to - :meth:`clear_turn_active`. Re-marking replaces the previous token so - a stale asynchronous unwind cannot clear a newer turn. - """ - token = uuid.uuid4().hex - with self._lock: - entry = self._entry_locked(session_key) - if entry is None: - return None - now = _now() - candidate = entry.to_dict() - candidate["active_turn_token"] = token - candidate["active_turn_started_at"] = now.isoformat() - # Keeps the legacy 120s startup heuristic working for an older - # binary during a rolling downgrade/upgrade window. - candidate["updated_at"] = now.isoformat() - - # Persist before publishing in memory so a failed write cannot - # leak an unowned token through a later unrelated save. - self._save_entry(session_key, entry_data=candidate, lock_held=True) - entry.active_turn_token = token - entry.active_turn_started_at = now - entry.updated_at = now - return token - - def clear_turn_active(self, session_key: str, token: str) -> bool: - """Compare-and-swap clear an active-turn marker. - - Returns ``False`` when the entry disappeared or a newer turn owns it. - """ - with self._lock: - entry = self._entry_locked(session_key) - if entry is None or entry.active_turn_token != token: - return False - candidate = entry.to_dict() - candidate["active_turn_token"] = None - candidate["active_turn_started_at"] = None - - # Keep the live token until the clear is durable (retryable). - self._save_entry(session_key, entry_data=candidate, lock_held=True) - entry.active_turn_token = None - entry.active_turn_started_at = None - return True - - def recover_interrupted_turns( - self, - max_age_seconds: int = 60 * 60, - ) -> int: - """Promote crash-left turn markers into ``resume_pending`` (unclean startup only). - - Old/invalid markers are cleared without resuming; suspended sessions - are never re-armed. Returns the number of newly promoted sessions. - """ - now = _now() - max_age = timedelta(seconds=max(0, max_age_seconds)) - promoted = 0 - changed = False - - with self._lock: - self._ensure_loaded_locked() - for entry in self._entries.values(): - if not entry.active_turn_token: - continue - - started_at = entry.active_turn_started_at - try: - marker_is_stale = ( - started_at is None - or (max_age_seconds > 0 and now - started_at > max_age) - ) - except TypeError: - # Mixed aware/naive timestamps: clear rather than risk an - # unsafe old resume. - marker_is_stale = True - - if not marker_is_stale and not entry.suspended: - if entry.resume_pending: - # A drain-timeout marker is more specific; keep it. - if entry.last_resume_marked_at is None: - entry.last_resume_marked_at = now - else: - entry.resume_pending = True - entry.resume_reason = "restart_interrupted" - # Freshness starts at discovery, not turn start. - entry.last_resume_marked_at = now - promoted += 1 - - entry.active_turn_token = None - entry.active_turn_started_at = None - changed = True - - if changed: - self._save() - - return promoted - - def discard_active_turn_markers(self) -> int: - """Clear orphan turn markers after a verified clean shutdown.""" - cleared = 0 - with self._lock: - self._ensure_loaded_locked() - for entry in self._entries.values(): - if not entry.active_turn_token and entry.active_turn_started_at is None: - continue - entry.active_turn_token = None - entry.active_turn_started_at = None - cleared += 1 - if cleared: - self._save() - return cleared - - def mark_resume_pending(self, session_key: str, reason: str = "restart_timeout") -> bool: - """Mark a session resumable after a restart interruption (keeps the - session_id/transcript, unlike ``suspend_session``). True if marked.""" - def _apply(entry: SessionEntry): - # Never override an explicit ``suspended`` (hard forced-wipe). - if entry.suspended: - return False - entry.resume_pending = True - entry.resume_reason = reason - entry.last_resume_marked_at = _now() - - return self._update_entry(session_key, _apply) - - def clear_resume_pending(self, session_key: str) -> bool: - """Clear the resume-pending flag after a successful resumed turn. - Returns True if a flag was cleared.""" - def _apply(entry: SessionEntry): - if not entry.resume_pending: - return False - entry.resume_pending = False - entry.resume_reason = None - entry.last_resume_marked_at = None - - return self._update_entry(session_key, _apply) - - def prune_old_entries(self, max_age_days: int) -> int: - """Drop routing entries idle (by ``updated_at``) for more than max_age_days. - - Suspended entries and entries with active background processes are - kept. The SQLite transcript stays; only the key -> session_id mapping - is dropped. ``max_age_days <= 0`` disables. Returns the count removed. - """ - if max_age_days is None or max_age_days <= 0: - return 0 - cutoff = _now() - timedelta(days=max_age_days) - removed_keys: list[str] = [] - - with self._lock: - self._ensure_loaded_locked() - for key, entry in list(self._entries.items()): - if entry.suspended: - continue - # The callback is keyed by session_key, NOT session_id. - if self._has_active_processes_safe(entry.session_key, context="prune"): - continue - if entry.updated_at < cutoff: - removed_keys.append(key) - for key in removed_keys: - self._entries.pop(key, None) - if removed_keys: - self._save() - - if removed_keys: - logger.info( - "SessionStore pruned %d entries older than %d days", - len(removed_keys), max_age_days, - ) - return len(removed_keys) - - def suspend_recently_active(self, max_age_seconds: int = 120) -> int: - """Mark sessions active within *max_age_seconds* as ``resume_pending`` - after a crash/fast restart (already-pending and suspended entries are - skipped). Returns the number marked.""" - cutoff = _now() - timedelta(seconds=max_age_seconds) - count = 0 - with self._lock: - self._ensure_loaded_locked() - for entry in self._entries.values(): - if entry.resume_pending: - continue - if not entry.suspended and entry.updated_at >= cutoff: - entry.resume_pending = True - entry.resume_reason = "restart_interrupted" - entry.last_resume_marked_at = _now() - count += 1 - if count: - self._save() - return count def reset_session(self, session_key: str, display_name: Optional[str] = None) -> Optional[SessionEntry]: """Force reset a session, creating a new session ID.""" @@ -2934,38 +1436,6 @@ class SessionStore: self._save() return new_entry - def advance_compression_session( - self, - session_key: str, - expected_session_id: str, - target_session_id: str, - ) -> Optional[SessionEntry]: - """CAS-advance one route along an already-verified compression lineage. - - Unlike ``switch_session`` this never ends/reopens SQLite rows (the - compression transaction owns that). ``None`` means the route moved - after the caller's snapshot (e.g. /new) — caller must fail closed. - """ - if not session_key or not expected_session_id or not target_session_id: - return None - - with self._lock: - entry = self._entry_locked(session_key) - if entry is None: - return None - if entry.session_id == target_session_id: - return entry - if entry.session_id != expected_session_id: - return None - if not self._heal_compression_tip_locked( - entry, - expected_session_id, - target_session_id, - ): - return None - # Bookkeeping, not user activity: leave ``updated_at`` alone. - self._save() - return entry def switch_session(self, session_key: str, target_session_id: str) -> Optional[SessionEntry]: """Point a session key at an existing session ID (``/resume``): ends @@ -3020,10 +1490,7 @@ class SessionStore: return None with self._lock: self._ensure_loaded_locked() - for entry in self._entries.values(): - if entry.session_id == session_id: - return entry - return None + return next((e for e in self._entries.values() if e.session_id == session_id), None) def lookup_by_session_key(self, session_key: str) -> Optional[SessionEntry]: """Return the persisted routing entry for an exact session key.""" @@ -3039,557 +1506,13 @@ class SessionStore: with self._lock: entry = self._entry_locked(session_key) return entry.session_id if entry else None - - def _get_transcript_drain_lock(self): - """Return the lock that serializes pending-queue drain boundaries.""" - return self._lazy("_transcript_drain_lock", threading.RLock) - def append_to_transcript(self, session_id: str, message: Dict[str, Any], skip_db: bool = False) -> None: - """Serialize transcript draining across queue migration boundaries.""" - if not self._db_for_session_id(session_id) or skip_db: - return - with self._get_transcript_drain_lock(): - self._append_to_transcript_serialized( - self._follow_reroutes(session_id), message - ) - def _follow_reroutes(self, session_id: str) -> str: - """Follow the compression reroute chain (cycle-guarded).""" - reroutes = self._lazy("_transcript_reroutes", dict) - seen = set() - while session_id in reroutes and session_id not in seen: - seen.add(session_id) - session_id = reroutes[session_id] - return session_id - - def _spool_dropped(self, session_id: str, message: Dict[str, Any]): - """Spool one evicted/undeliverable message to disk; path or None.""" - try: - from gateway.shutdown_flush import spool_dropped_transcript_message - - return spool_dropped_transcript_message(session_id, message) - except Exception: - return None - - def _enqueue_transcript_message(self, session_id: str, message: Dict[str, Any]) -> list: - """Queue *message* (retry lock held); evicts + spools the oldest past the cap. - - Spooling uses the same machinery as shutdown flush so the message is - replayed after DB recovery instead of being lost. - """ - pending = self._dirty_transcripts.setdefault(session_id, []) - pending.append(dict(message)) - if len(pending) > self._MAX_PENDING_PER_SESSION: - spool_path = self._spool_dropped(session_id, pending.pop(0)) - if spool_path is not None: - self._lazy("_spooled_drop_sessions", set).add(session_id) - logger.warning( - "Session DB transcript pending queue full for %s " - "(cap=%d); spooled oldest message to %s for replay " - "after DB recovery", - session_id, self._MAX_PENDING_PER_SESSION, spool_path, - ) - else: - logger.warning( - "Session DB transcript pending queue full for %s " - "(cap=%d); dropping oldest message to make room " - "(on-disk spool unavailable)", - session_id, self._MAX_PENDING_PER_SESSION, - ) - return pending - - def _divert_transcript_after_db_replaced( - self, session_id: str, queue_session_id: str, exc: Exception - ) -> None: - """Stop SQLite writes on a replaced/quarantined handle and divert the backlog. - - Retrying cannot succeed and the FTS rebuild must never run on this - handle; the pending queue goes to the on-disk spool + JSONL fallback. - """ - logger.error( - "Session DB refused further writes on this handle for " - "%s (%s); stopping SQLite writes and diverting pending " - "transcripts to the on-disk fallback: %s", - session_id, type(exc).__name__, exc, - ) - with self._transcript_retry_lock: - remaining = list(self._dirty_transcripts.get(queue_session_id, [])) - self._dirty_transcripts.pop(queue_session_id, None) - self._transcript_append_failures.pop(session_id, None) - for dropped in remaining: - try: - from gateway.shutdown_flush import spool_dropped_transcript_message - - spool_dropped_transcript_message(session_id, dropped) - except Exception: - logger.warning( - "pending fallback failed for replaced " - "state.db transcript on %s", - session_id, - exc_info=True, - ) - try: - from hermes_state import divert_session_transcript_jsonl - divert_session_transcript_jsonl(session_id, remaining) - except Exception: - logger.warning( - "JSONL divert failed for replaced state.db " - "transcript on %s", - session_id, - exc_info=True, - ) - - def _live_compression_child(self, session_id: str) -> str: - """Transitive compression tip of *session_id* if it is a different, still-live - row, else "" (a depth-1 lookup misses multi-hop lineages). - - Uses the PARENT's proven owner handle: the child's id is not published - until after its write succeeds, so a by-id lookup would fall back to - the ambient store. - """ - owner_db = self._db_for_session_id(session_id) - if owner_db is None: - return "" - tip = owner_db.get_compression_tip(session_id) - if tip and tip != session_id: - tip_row = owner_db.get_session(tip) - if tip_row is not None and tip_row.get("ended_at") is None: - return str(tip) - return "" - - def _migrate_transcript_queue_to_child( - self, session_id: str, queue_session_id: str, child_id: str, pending: list, msg - ) -> list: - """Move the retry queue + failure counter from parent to child and publish - the reroute (retry lock held). Returns the child's pending list. - - Older parent backlog must precede messages already queued directly on - the child. Routing is published only AFTER the queue moved (caller), so - new child writes cannot bypass older parent backlog. - """ - if pending and pending[0] is msg: - pending.pop(0) - existing_child_pending = self._dirty_transcripts.get(child_id, []) - if pending: - pending.extend(existing_child_pending) - self._dirty_transcripts[child_id] = pending - elif existing_child_pending: - pending = existing_child_pending - self._dirty_transcripts.pop(queue_session_id, None) - previous_failures = self._transcript_append_failures.pop(queue_session_id, 0) - if previous_failures: - self._transcript_append_failures[child_id] = max( - previous_failures, - self._transcript_append_failures.get(child_id, 0), - ) - self._transcript_reroutes[session_id] = child_id - return pending - - def _publish_transcript_reroute(self, session_id: str, child_id: str) -> None: - """Repoint every route at the compression child and save (index authoritative again).""" - with self._lock: - for entry in self._entries.values(): - if entry.session_id == session_id: - entry.session_id = child_id - self._save() - _hints = getattr(self, "_session_owner_hints", None) - if _hints: - _hints.pop(child_id, None) - - def _append_to_transcript_serialized( - self, session_id: str, message: Dict[str, Any] - ) -> None: - """Append a message to a session's transcript (SQLite), draining the - per-session retry queue.""" - with self._transcript_retry_lock: - pending = self._enqueue_transcript_message(session_id, message) - msg = pending[0] - queue_session_id = session_id - - def _ack_head() -> bool: - """Pop the acknowledged head (retry lock held). True if queue drained.""" - if pending and pending[0] is msg: - pending.pop(0) - if not pending: - self._dirty_transcripts.pop(queue_session_id, None) - self._transcript_append_failures.pop(session_id, None) - return True - return False - - # DB write outside the retry lock so other sessions can append. - while True: - try: - self._append_transcript_message(session_id, msg) - except Exception as exc: - from hermes_state import ( - CompressionSessionClosedError, - StateDbCorruptError, - StateDbReplacedError, - ) - - if isinstance(exc, (StateDbReplacedError, StateDbCorruptError)): - self._divert_transcript_after_db_replaced(session_id, queue_session_id, exc) - return - - if isinstance(exc, CompressionSessionClosedError): - # Adopt only a different, still-live compression tip, else - # fail closed. - _owner_key = self._owner_key_for_session_id(session_id) - child_id = self._live_compression_child(session_id) - if child_id: - # Record the child's owner BEFORE writing to it (the - # reroute is published only after the write succeeds - # — load-bearing for backlog order). - if _owner_key: - self._lazy("_session_owner_hints", dict)[child_id] = _owner_key - try: - self._append_transcript_message(child_id, msg) - except Exception as reroute_exc: - exc = reroute_exc - else: - with self._transcript_retry_lock: - pending = self._migrate_transcript_queue_to_child( - session_id, queue_session_id, child_id, pending, msg - ) - queue_session_id = child_id - self._publish_transcript_reroute(session_id, child_id) - if not pending: - return - msg = pending[0] - session_id = child_id - continue - else: - # Permanent routing invariant failure, not a transient - # outage: drop it so it cannot poison later writes. - with self._transcript_retry_lock: - _ack_head() - logger.error( - "Session DB transcript append rejected for compression-ended " - "%s with no unique live child; not retrying", - session_id, - ) - return - if self._is_fts_corruption_error(exc) and self._rebuild_fts_once(): - try: - self._append_transcript_message(session_id, msg) - except Exception as retry_exc: - exc = retry_exc - else: - with self._transcript_retry_lock: - _ack_head() - continue - with self._transcript_retry_lock: - failures = self._transcript_append_failures.get(session_id, 0) + 1 - self._transcript_append_failures[session_id] = failures - logger.warning( - "Session DB transcript append failed for %s " - "(failure_count=%d, pending=%d); will retry: %s", - session_id, failures, len(pending), exc, - ) - return - else: - with self._transcript_retry_lock: - queue_empty = _ack_head() - if not queue_empty: - msg = pending[0] - if queue_empty: - # Backlog clear: replay cap-dropped messages spooled to disk. - self._drain_spooled_drops(session_id) - return - continue - - def _drain_spooled_drops(self, session_id: str) -> None: - """Replay cap-dropped spooled transcript messages after DB recovery. - - Best-effort: replay failures keep the spool files for the next - successful flush; nothing here may raise into the caller. - """ - spooled_sessions = getattr(self, "_spooled_drop_sessions", None) - if not spooled_sessions or session_id not in spooled_sessions: - return - try: - from gateway.shutdown_flush import drain_transcript_spool - - _replayed, remaining = drain_transcript_spool( - session_id, - lambda message: self._append_transcript_message( - session_id, message - ), - ) - if not remaining: - spooled_sessions.discard(session_id) - except Exception as exc: - logger.warning("Failed to drain transcript spool for %s: %s", session_id, exc) - - def _append_transcript_message(self, session_id: str, message: Dict[str, Any]) -> None: - """Write one transcript row. Caller handles retry queuing.""" - _db = self._db_for_session_id(session_id) - if _db is None: - # Named profile with no resolvable home yet: defer (caller queues) - # instead of writing into the ambient store. - raise RuntimeError( - f"no owning session store for {session_id}; deferring transcript write" - ) - is_assistant = message.get("role") == "assistant" - _db.append_message( - session_id=session_id, - role=message.get("role", "unknown"), - content=message.get("content"), - tool_name=message.get("tool_name"), - tool_calls=message.get("tool_calls"), - tool_call_id=message.get("tool_call_id"), - reasoning=message.get("reasoning") if is_assistant else None, - reasoning_content=message.get("reasoning_content") if is_assistant else None, - reasoning_details=message.get("reasoning_details") if is_assistant else None, - codex_reasoning_items=message.get("codex_reasoning_items") if is_assistant else None, - codex_message_items=message.get("codex_message_items") if is_assistant else None, - platform_message_id=(message.get("platform_message_id") or message.get("message_id")), - observed=bool(message.get("observed")), - timestamp=message.get("timestamp"), - # Exact bytes sent to the API (prompt-cache-stable replay); must - # survive every persistence path or the next replay diverges. - api_content=extract_api_content_sidecar(message), - # Presentation typing (e.g. "internal_notification"); DB-only. - display_kind=message.get("display_kind"), - display_metadata=message.get("display_metadata"), - ) # Max in-memory pending messages per session (DB persistently broken). - _MAX_PENDING_PER_SESSION = 200 - @staticmethod - def _is_fts_corruption_error(exc: Exception) -> bool: - """True only when the failure is provably scoped to the FTS index. - A bare SQLITE_CORRUPT can mean structural B-tree damage; only errors - naming ``messages_fts`` or carrying FTS provenance (per - ``SessionDB._is_fts_write_corruption_error``) may authorize the - one-shot rebuild-and-retry. Everything else takes the retry path. - """ - text = str(exc).lower() - if "messages_fts" in text: - return True - import sqlite3 - from hermes_state import SessionDB - - if isinstance(exc, sqlite3.DatabaseError): - return SessionDB._is_fts_write_corruption_error(exc) - return False - - def _rebuild_fts_once(self) -> bool: - """Attempt FTS5 ``rebuild`` once per store lifetime; True if any index was rebuilt.""" - if self._fts_rebuild_attempted: - return False - self._fts_rebuild_attempted = True - db = self._db - if db is None or not hasattr(db, "rebuild_fts"): - return False - # WAL split-brain guard: skip when a foreign process holds state.db. - if hasattr(db, "_foreign_state_db_holders"): - foreign_holders = db._foreign_state_db_holders() - if foreign_holders: - logger.warning( - "Skipping Session DB FTS rebuild while foreign processes " - "hold the database or WAL sidecars (%s); canonical " - "transcript writes remain available.", - foreign_holders, - ) - return False - try: - rebuilt = db.rebuild_fts() - except Exception as exc: - logger.warning("Session DB FTS rebuild failed: %s", exc) - return False - if rebuilt: - logger.warning( - "Rebuilt %d Session DB FTS index(es) after append corruption", - rebuilt, - ) - return rebuilt > 0 - - def _clear_dirty_transcript(self, session_id: str) -> None: - """Drop queued pending messages so a rewrite/rewind doesn't re-insert them.""" - with self._transcript_retry_lock: - self._dirty_transcripts.pop(session_id, None) - self._transcript_append_failures.pop(session_id, None) - - def has_platform_message_id( - self, session_id: str, platform_message_id: str - ) -> bool: - """Whether a message with this platform_message_id is persisted (False without a DB).""" - db = self._db_for_session_id(session_id) - if not db: - return False - try: - return db.has_platform_message_id( - session_id, platform_message_id - ) - except Exception: - logger.debug("has_platform_message_id lookup failed", exc_info=True) - return False - - def rewrite_transcript( - self, - session_id: str, - messages: List[Dict[str, Any]], - active_only: bool = False, - reject_active_turn_lease: bool = False, - ) -> bool: - """Replace a session's transcript (/retry, /compress). - - DESTRUCTIVE by default: ``active_only=False`` DELETEs every row - including soft-archived compaction history; pass ``active_only=True`` - for sessions that may carry archived rows. Returns ``True`` when the - write lands (or there is no DB), ``False`` on failure — callers about - to commit a destructive change on top (e.g. /compress repointing) - must check it. ``reject_active_turn_lease`` is for user-initiated - rewrites that do not own the cross-process turn lease. - """ - db = self._db_for_session_id(session_id) - if not db: - return True - with self._get_transcript_drain_lock(): - try: - db.replace_messages( - session_id, - messages, - active_only=active_only, - reject_active_turn_lease=reject_active_turn_lease, - ) - except Exception as e: - logger.debug("Failed to rewrite transcript in DB: %s", e) - return False - self._clear_dirty_transcript(session_id) - return True - - def load_transcript(self, session_id: str) -> List[Dict[str, Any]]: - """Load all messages from a session's transcript (state.db is canonical). - - Reads follow the same routing writes use: the in-memory reroute map - (compression rotation), then the durable compression tip — otherwise - the transcript "vanishes" while every message sits under the child. - """ - if not self._db_for_session_id(session_id): - return [] - session_id = self._follow_reroutes(session_id) - try: - # Durable successor survives restart; the reroute map doesn't. - tip = self._db_for_session_id(session_id).get_compression_tip(session_id) - if tip: - session_id = tip - except Exception: - pass - try: - # repair_alternation: this feeds LIVE REPLAY; heal a durable - # user;user wedge once here instead of on every request. - return self._db_for_session_id(session_id).get_messages_as_conversation( - session_id, repair_alternation=True - ) - except Exception as e: - # Empty history is valid data; a failed canonical read is not — - # live-replay callers must fail closed, not start from []. - logger.error( - "Transcript read failed for session %s; refusing to treat the " - "conversation as empty: %s", - session_id, - e, - exc_info=True, - ) - raise TranscriptReadError(session_id) from e - - def rewind_session( - self, - session_id: str, - n: int = 1, - *, - require_retryable_composite: bool = False, - ) -> Optional[Dict[str, Any]]: - """Back up ``n`` user turns via soft-delete (``active=0``), mirroring CLI ``/undo [N]``. - - Returns ``{"rewound_count", "turns_undone", "target_text"}`` or ``None`` - (no DB / no user turn). ``n`` clamps to the oldest user turn. - ``require_retryable_composite`` is the gateway ``/retry`` guard: the - selected turn must be a composite carrier whose live payload is - losslessly replayable as text before anything changes. - """ - db = self._db_for_session_id(session_id) - if not db: - return None - with self._get_transcript_drain_lock(): - if n < 1: - n = 1 - from agent.context_compressor import ( - retryable_user_text, - split_user_originated_turn, - user_originated_turn_view, - ) - - try: - expected_active_ids = db.get_active_message_ids(session_id) - durable = db.get_messages_as_conversation( - session_id, - include_row_ids=True, - ) - user_indices = [ - index - for index, message in enumerate(durable) - if user_originated_turn_view(message) is not None - ] - if not user_indices: - return None - turns_undone = min(n, len(user_indices)) - target = durable[user_indices[-turns_undone]] - target_id = target.get("_row_id") - if not isinstance(target_id, int): - return None - handoff, target_view = split_user_originated_turn(target) - if target_view is None: - return None - if require_retryable_composite and handoff is None: - return None - except Exception as e: - logger.debug("rewind_session: failed to resolve canonical target: %s", e) - return None - if require_retryable_composite: - # Keep replay-policy failures distinct from persistence errors - # so /retry can explain why the selected carrier is unsafe. - target_text = retryable_user_text(target_view.get("content")) - try: - result = db.rewind_to_message( - session_id, - target_id, - preserve_compaction_handoff=handoff is not None, - expected_active_ids=expected_active_ids, - expected_target_content=target_view.get("content"), - ) - except ValueError as e: - logger.debug("rewind_session: %s", e) - return None - except Exception as e: - logger.debug("rewind_session: rewind_to_message failed: %s", e) - return None - self._clear_dirty_transcript(session_id) - # ``target_view`` is the live projection; a composite carrier's raw - # row holds the summary wrapper and must not be echoed as prompt. - if not require_retryable_composite: - content = target_view.get("content") or "" - if isinstance(content, list): - parts = [ - p.get("text", "") - for p in content - if isinstance(p, dict) and p.get("type") == "text" - ] - target_text = "\n".join(t for t in parts if t) - elif isinstance(content, str): - target_text = content - else: - target_text = "" - return { - "rewound_count": result.get("rewound_count", 0), - "turns_undone": turns_undone, - "target_text": target_text, - } def build_session_context( @@ -3599,13 +1522,11 @@ def build_session_context( ) -> SessionContext: """Build a full session context (for system prompt injection).""" connected = config.get_connected_platforms() - home_channels = {} for platform in connected: home = config.get_home_channel(platform) if home: home_channels[platform] = home - context = SessionContext( source=source, connected_platforms=connected, @@ -3616,11 +1537,9 @@ def build_session_context( thread_sessions_per_user=getattr(config, "thread_sessions_per_user", False), ), ) - if session_entry: context.session_key = session_entry.session_key context.session_id = session_entry.session_id context.created_at = session_entry.created_at context.updated_at = session_entry.updated_at - return context diff --git a/gateway/session_lifecycle.py b/gateway/session_lifecycle.py new file mode 100644 index 0000000000..4497cce47d --- /dev/null +++ b/gateway/session_lifecycle.py @@ -0,0 +1,385 @@ +"""SessionStore reset/expiry policy and crash-recovery markers: idle/daily reset +evaluation, expiry finalization, active-turn tokens, resume_pending, +suspension and pruning. + +Mixin split out of ``gateway/session.py``; bound onto ``SessionStore`` via the MRO. +""" + +from __future__ import annotations + +import logging +import uuid +from datetime import datetime, timedelta +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from gateway.session import SessionEntry, SessionSource + +# Log-record parity with the origin module. +logger = logging.getLogger("gateway.session") + + +class SessionLifecycleMixin: + """SessionStore reset/expiry policy and crash-recovery markers: idle/daily + reset evaluation, expiry finalization, active-turn tokens, resume_pending, + suspension and pruning. + """ + + def set_expiry_finalized( + self, entry: SessionEntry, *, clear_model_override: bool = True + ) -> None: + """Mark a session entry expiry-finalized in memory, sessions.json, AND state.db. + + Single write-path for the expiry watcher so the durable flag survives + sessions.json loss. ``clear_model_override=False`` = flag only. + """ + with self._lock: + entry.expiry_finalized = True + if clear_model_override: + # Finalization is a conversation boundary: drop the persisted + # /model override so a later message cannot rehydrate it. + entry.model_override = None + self._save() + # Background caller never entered ``_profile_runtime_scope``: resolve + # the store from the key, not the ambient scope. + _db = self._db_for_key(entry.session_key) + if _db: + setter = getattr(_db, "set_expiry_finalized", None) + if callable(setter): + try: + setter(entry.session_id, True) + except Exception as exc: + logger.debug("Session DB expiry_finalized write failed for %s: %s", entry.session_id, exc) + try: + # Without a durable ``session_reset`` end_reason, later agent + # cleanup ends the row as ``agent_close``, which stale-route + # recovery treats as resumable. Promotion only upgrades live/ + # agent_close rows; explicit boundaries are preserved. + _db.promote_to_session_reset(entry.session_id) + except Exception as exc: + logger.debug("Session DB promote_to_session_reset failed for %s: %s", entry.session_id, exc) + + @staticmethod + def _policy_reset_reason(policy, updated_at: datetime) -> Optional[str]: + """Return "idle"/"daily" when *updated_at* is overdue under *policy*, else None.""" + from gateway.session import _now + if policy.mode == "none": + return None + now = _now() + if policy.mode in {"idle", "both"} and now > updated_at + timedelta(minutes=policy.idle_minutes): + return "idle" + if policy.mode in {"daily", "both"}: + today_reset = now.replace( + hour=policy.at_hour, minute=0, second=0, microsecond=0, + ) + if now.hour < policy.at_hour: + today_reset -= timedelta(days=1) + if updated_at < today_reset: + return "daily" + return None + + def _is_session_expired(self, entry: SessionEntry) -> bool: + """Whether the entry's reset policy has expired it (entry alone, no source). + + Used by the background expiry watcher. Sessions with active + background processes are never considered expired. + """ + if self._has_active_processes_safe(entry.session_key, context="expiry"): + logger.debug("Session %s not expired — active background processes", entry.session_key) + return False + policy = self.config.get_reset_policy( + platform=entry.platform, + session_type=entry.chat_type, + ) + return self._policy_reset_reason(policy, entry.updated_at) is not None + + def is_session_finalizable(self, entry: SessionEntry) -> bool: + """True if the expiry watcher will *ever* finalize this session. + + A ``mode == "none"`` session never expires, so the agent-cache idle + sweep must reap its agent itself instead of deferring to the watcher + (deferring would pin the agent for the gateway's lifetime). Policy + resolution errors count as "not finalizable" (sweep reaps — safe). + """ + try: + policy = self.config.get_reset_policy( + platform=entry.platform, + session_type=entry.chat_type, + ) + return policy.mode != "none" + except Exception: + return False + + def _is_session_ended_in_db(self, session_id: str) -> bool: + """True iff state.db has this session with a non-null end_reason. + + Same staleness test as ``_prune_stale_sessions_locked`` (no DB, no + row, or DB error -> False, keep). Used by ``get_or_create_session`` + to self-heal at routing time, since the startup prune cannot see a + session ended while the gateway stays alive. Store resolved from the + row's owning profile, not the ambient scope. + """ + db = self._db_for_session_id(session_id) + if not db or not session_id: + return False + try: + row = db.get_session(session_id) + except Exception: + return False + return bool(row is not None and row.get("end_reason") is not None) + + def _should_reset(self, entry: SessionEntry, source: SessionSource) -> Optional[str]: + """Return the reset reason ("idle"/"daily") if policy says reset, else None. + + Sessions with active background processes are never reset. + """ + session_key = self._generate_session_key(source) + if self._has_active_processes_safe(session_key, context="reset"): + logger.debug("Session reset skipped for %s — active background processes", session_key) + return None + policy = self.config.get_reset_policy( + platform=source.platform, + session_type=source.chat_type + ) + return self._policy_reset_reason(policy, entry.updated_at) + + def _route_reset_reason( + self, entry: SessionEntry, source: SessionSource, now: datetime + ) -> Optional[str]: + """Reset decision for an existing route (no lock; DB/config I/O). + + ``suspended`` always resets. Otherwise the reset policy decides; a + still-pending resume marker is additionally freshness-gated — but + ``session_reset.mode: none`` (user opted out of ALL automatic resets) + makes an expired marker fall through to a normal resume, never a + silent fresh session. + """ + from gateway.session import auto_continue_freshness_window + if entry.suspended: + return "suspended" + reason = self._should_reset(entry, source) + if reason or not entry.resume_pending: + return reason + policy = self.config.get_reset_policy( + platform=source.platform, session_type=source.chat_type, + ) + if policy.mode == "none": + return None + window = auto_continue_freshness_window() + ref_time = entry.last_resume_marked_at or entry.updated_at + if window > 0 and (now - ref_time).total_seconds() > window: + return "resume_pending_expired" + return None + + def _update_entry(self, session_key: str, mutate) -> bool: + """Apply ``mutate(entry)`` under ``_lock`` and full-save; False when the + entry is missing or *mutate* returned False (nothing to persist).""" + with self._lock: + entry = self._entry_locked(session_key) + if entry is None or mutate(entry) is False: + return False + self._save() + return True + + def suspend_session(self, session_key: str) -> bool: + """Mark a session suspended so it auto-resets on next access (/stop). + Returns True if the session existed.""" + return self._update_entry(session_key, lambda e: setattr(e, "suspended", True)) + + def mark_turn_active(self, session_key: str) -> Optional[str]: + """Persist exact ownership of the agent turn running for *session_key*. + + The opaque token is returned to the caller and must be supplied to + :meth:`clear_turn_active`. Re-marking replaces the previous token so + a stale asynchronous unwind cannot clear a newer turn. + """ + from gateway.session import _now + token = uuid.uuid4().hex + with self._lock: + entry = self._entry_locked(session_key) + if entry is None: + return None + now = _now() + candidate = entry.to_dict() + candidate["active_turn_token"] = token + candidate["active_turn_started_at"] = now.isoformat() + # Keeps the legacy 120s startup heuristic working for an older + # binary during a rolling downgrade/upgrade window. + candidate["updated_at"] = now.isoformat() + + # Persist before publishing in memory so a failed write cannot + # leak an unowned token through a later unrelated save. + self._save_entry(session_key, entry_data=candidate, lock_held=True) + entry.active_turn_token = token + entry.active_turn_started_at = now + entry.updated_at = now + return token + + def clear_turn_active(self, session_key: str, token: str) -> bool: + """Compare-and-swap clear an active-turn marker. + + Returns ``False`` when the entry disappeared or a newer turn owns it. + """ + with self._lock: + entry = self._entry_locked(session_key) + if entry is None or entry.active_turn_token != token: + return False + candidate = entry.to_dict() + candidate["active_turn_token"] = None + candidate["active_turn_started_at"] = None + + # Keep the live token until the clear is durable (retryable). + self._save_entry(session_key, entry_data=candidate, lock_held=True) + entry.active_turn_token = None + entry.active_turn_started_at = None + return True + + def recover_interrupted_turns( + self, + max_age_seconds: int = 60 * 60, + ) -> int: + """Promote crash-left turn markers into ``resume_pending`` (unclean startup only). + + Old/invalid markers are cleared without resuming; suspended sessions + are never re-armed. Returns the number of newly promoted sessions. + """ + from gateway.session import _now + now = _now() + max_age = timedelta(seconds=max(0, max_age_seconds)) + promoted = 0 + changed = False + + with self._lock: + self._ensure_loaded_locked() + for entry in self._entries.values(): + if not entry.active_turn_token: + continue + + started_at = entry.active_turn_started_at + try: + marker_is_stale = ( + started_at is None + or (max_age_seconds > 0 and now - started_at > max_age) + ) + except TypeError: + # Mixed aware/naive timestamps: clear rather than risk an + # unsafe old resume. + marker_is_stale = True + + if not marker_is_stale and not entry.suspended: + if entry.resume_pending: + # A drain-timeout marker is more specific; keep it. + if entry.last_resume_marked_at is None: + entry.last_resume_marked_at = now + else: + entry.resume_pending = True + entry.resume_reason = "restart_interrupted" + # Freshness starts at discovery, not turn start. + entry.last_resume_marked_at = now + promoted += 1 + + entry.active_turn_token = None + entry.active_turn_started_at = None + changed = True + + if changed: + self._save() + + return promoted + + def discard_active_turn_markers(self) -> int: + """Clear orphan turn markers after a verified clean shutdown.""" + cleared = 0 + with self._lock: + self._ensure_loaded_locked() + for entry in self._entries.values(): + if not entry.active_turn_token and entry.active_turn_started_at is None: + continue + entry.active_turn_token = None + entry.active_turn_started_at = None + cleared += 1 + if cleared: + self._save() + return cleared + + def mark_resume_pending(self, session_key: str, reason: str = "restart_timeout") -> bool: + """Mark a session resumable after a restart interruption (keeps the + session_id/transcript, unlike ``suspend_session``). True if marked.""" + from gateway.session import _now + def _apply(entry: SessionEntry): + # Never override an explicit ``suspended`` (hard forced-wipe). + if entry.suspended: + return False + entry.resume_pending = True + entry.resume_reason = reason + entry.last_resume_marked_at = _now() + + return self._update_entry(session_key, _apply) + + def clear_resume_pending(self, session_key: str) -> bool: + """Clear the resume-pending flag after a successful resumed turn. + Returns True if a flag was cleared.""" + def _apply(entry: SessionEntry): + if not entry.resume_pending: + return False + entry.resume_pending = False + entry.resume_reason = None + entry.last_resume_marked_at = None + + return self._update_entry(session_key, _apply) + + def prune_old_entries(self, max_age_days: int) -> int: + """Drop routing entries idle (by ``updated_at``) for more than max_age_days. + + Suspended entries and entries with active background processes are + kept. The SQLite transcript stays; only the key -> session_id mapping + is dropped. ``max_age_days <= 0`` disables. Returns the count removed. + """ + from gateway.session import _now + if max_age_days is None or max_age_days <= 0: + return 0 + cutoff = _now() - timedelta(days=max_age_days) + removed_keys: list[str] = [] + + with self._lock: + self._ensure_loaded_locked() + for key, entry in list(self._entries.items()): + if entry.suspended: + continue + # The callback is keyed by session_key, NOT session_id. + if self._has_active_processes_safe(entry.session_key, context="prune"): + continue + if entry.updated_at < cutoff: + removed_keys.append(key) + for key in removed_keys: + self._entries.pop(key, None) + if removed_keys: + self._save() + + if removed_keys: + logger.info( + "SessionStore pruned %d entries older than %d days", + len(removed_keys), max_age_days, + ) + return len(removed_keys) + + def suspend_recently_active(self, max_age_seconds: int = 120) -> int: + """Mark sessions active within *max_age_seconds* as ``resume_pending`` + after a crash/fast restart (already-pending and suspended entries are + skipped). Returns the number marked.""" + from gateway.session import _now + cutoff = _now() - timedelta(seconds=max_age_seconds) + count = 0 + with self._lock: + self._ensure_loaded_locked() + for entry in self._entries.values(): + if entry.resume_pending: + continue + if not entry.suspended and entry.updated_at >= cutoff: + entry.resume_pending = True + entry.resume_reason = "restart_interrupted" + entry.last_resume_marked_at = _now() + count += 1 + if count: + self._save() + return count diff --git a/gateway/session_persistence.py b/gateway/session_persistence.py new file mode 100644 index 0000000000..9d1f044da1 --- /dev/null +++ b/gateway/session_persistence.py @@ -0,0 +1,656 @@ +"""SessionStore storage plumbing: per-profile SessionDB handle resolution and the +routing-index load/save paths (state.db gateway_routing primary, sessions.json +legacy mirror). + +Mixin split out of ``gateway/session.py``; bound onto ``SessionStore`` via the MRO. +""" + +from __future__ import annotations + +import logging +import json +import os +import threading +from pathlib import Path +from typing import Any, Dict, Optional +from utils import atomic_replace +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from gateway.session import SessionEntry + +# Log-record parity with the origin module. +logger = logging.getLogger("gateway.session") + + +class SessionPersistenceMixin: + """SessionStore storage plumbing: per-profile SessionDB handle resolution and + the routing-index load/save paths (state.db gateway_routing primary, + sessions.json legacy mirror). + """ + + def _open_session_db_for_active_scope(self, db_path: Optional[Path] = None): + """SessionDB for the profile scope active on this task. + + ``db_path`` pins the store; otherwise ``_default_db_path()`` follows the + context-local HERMES_HOME from ``_profile_runtime_scope`` (resolved per + call so multiplexed profiles reach their own store). Handles are cached + per path; failed opens enter a bounded backoff during which callers keep + using the JSONL fallback. + """ + from hermes_state import _default_db_path, get_shared_session_db + + path = Path(db_path) if db_path is not None else Path(_default_db_path()) + def _open(): + try: + # Process-wide shared registry: one writer connection per path. + return get_shared_session_db(path) + except Exception as e: + if isinstance(e, RuntimeError) and "live-system guard" in str(e): + # Test-isolation guard: must stay a loud failure and is + # deliberately not cached so it fires again next attempt. + raise + print(f"[gateway] Warning: SQLite session store unavailable, falling back to JSONL: {e}") + raise + + return self._db_handle_cache.get( + path, + _open, + non_cacheable=lambda exc: ( + isinstance(exc, RuntimeError) and "live-system guard" in str(exc) + ), + ) + + def _pinned_db(self): + """Return the explicitly pinned DB (``store._db = x``), else ``_DB_UNPINNED``.""" + from gateway.session import _DB_UNPINNED + return getattr(self, "_db_pinned", _DB_UNPINNED) + + @property + def _db(self): + """The SessionDB for the active profile scope, or a pinned override. + + Assigning ``store._db`` pins that value for every subsequent read + (tests install a fake or disable the DB with ``store._db = None``). + Unpinned, each read resolves the scope so a multiplexed profile's + writes reach its own store. + """ + from gateway.session import _DB_UNPINNED + pinned = self._pinned_db() + if pinned is not _DB_UNPINNED: + return pinned + return self._open_session_db_for_active_scope() + + @_db.setter + def _db(self, value) -> None: + self._db_pinned = value + + @property + def _routing_db(self): + """The one store that owns the routing index, whatever scope is active. + + ``_entries`` is one flat dict holding every profile's keys, so it must + persist to ONE file (``_routing_home``), not whichever profile is + scoped — otherwise a mid-turn rewrite and the unscoped startup load see + different copies and crash markers under a secondary profile go + unrecovered. A pinned handle still wins; bare test instances lacking + the handle cache report no DB. + """ + from gateway.session import _DB_UNPINNED + pinned = self._pinned_db() + if pinned is not _DB_UNPINNED: + return pinned + home = getattr(self, "_routing_home", None) + try: + if home is None: + return self._db + return self._open_session_db_for_active_scope(db_path=home / "state.db") + except Exception: + return None + + def _named_profile_for_key(self, session_key: Optional[str]) -> Optional[str]: + """The non-default profile that owns *session_key*, or None. + + None means the ambient store is authoritative (multiplexing off, or + legacy ``agent:main`` namespace). It deliberately does NOT cover "that + profile has no directory" — ownership and resolvability are separate + questions that ``_db_for_key`` answers separately. + """ + if not getattr(self.config, "multiplex_profiles", False): + return None + profile = self._profile_from_session_key(session_key) + if not profile or profile == "default": + return None + return profile + + def _profile_home_for_key(self, session_key: Optional[str]) -> Optional[Path]: + """HERMES_HOME of the profile that owns *session_key*, or None. + + None means only "no live home to point at" — no named owner, or the + owner's directory could not be resolved. + """ + profile = self._named_profile_for_key(session_key) + if profile is None: + return None + cache = self._profile_home_cache + if profile in cache: + return cache[profile] + home: Optional[Path] = None + try: + from hermes_cli.profiles import get_profile_dir, profile_exists + + if profile_exists(profile): + home = Path(get_profile_dir(profile)) + except Exception as exc: + logger.debug("Could not resolve profile home for %r: %s", session_key, exc) + home = None + # Only hits are memoized: a profile directory can be provisioned + # *after* startup (enrollment bridge), and a cached miss would pin + # that profile's rows to the ambient store for the process lifetime. + if home is not None: + cache[profile] = home + return home + + def _db_for_key(self, session_key: Optional[str]): + """The SessionDB holding *session_key*'s rows, whatever scope is active. + + ``_db`` follows the ambient HERMES_HOME that only the inbound message + path installs; background work (expiry watcher) runs unscoped and would + write profile rows into the ROOT store until the stale-route self-heal + drops a live conversation. The owning profile is encoded in the key. + """ + from gateway.session import _DB_UNPINNED + pinned = self._pinned_db() + if pinned is not _DB_UNPINNED: + return pinned + profile = self._named_profile_for_key(session_key) + if profile is None: + return self._db + home = self._profile_home_for_key(session_key) + if home is None: + # Named owner we cannot resolve (not provisioned yet, or lookup + # failed). Falling back to the ambient store would split ONE + # session identity across two physical stores — fail closed; + # callers already handle a missing DB. + logger.warning( + "gateway.session: profile %r has no resolvable home (key %r); " + "refusing to fall back to the ambient store", + profile, session_key, + ) + return None + try: + return self._open_session_db_for_active_scope(db_path=home / "state.db") + except Exception: + # Same contract as ``_db``: a failed open degrades to JSONL fallback. + return None + + def _owner_key_for_session_id(self, session_id: Optional[str]) -> Optional[str]: + """The routing key that owns *session_id*, or None. + + The published index is authoritative; ``_session_owner_hints`` covers + the window where ownership is proven but routing not yet published. + Deliberately lock-free: several callers already hold ``_lock``. + """ + if not session_id: + return None + try: + for entry in list(self._entries.values()): + if entry.session_id == session_id: + return entry.session_key + except Exception: + pass # bare stores / foreign entry objects in suites + return (getattr(self, "_session_owner_hints", None) or {}).get(session_id) + + def _db_for_session_id(self, session_id: Optional[str]): + """The SessionDB holding *session_id*'s row (owner recovered from the + index or a pre-published hint; unknown ids fall back to the ambient store).""" + if not session_id: + return self._db + return self._db_for_key(self._owner_key_for_session_id(session_id)) + + def close_all_db_handles(self) -> None: + """Close every SessionDB handle this store opened (one per path). + + Closing only ``store._db`` would strand secondary profiles' handles with + their WAL lock held ('database is locked' on restart). Drained under the + lock, closed outside it; a pinned handle is the pinner's to close. + """ + def _close(db) -> None: + # Shared instances no-op on close(); release the refcount instead. + from hermes_state import release_or_close + try: + release_or_close(db) + except Exception as exc: + logger.debug("SessionDB close error during handle sweep: %s", exc) + + self._db_handle_cache.close_all(_close) + + def _ensure_loaded(self) -> None: + """Load sessions index from disk if not already loaded.""" + with self._lock: + self._ensure_loaded_locked() + + def _entry_locked(self, session_key: str) -> Optional[SessionEntry]: + """Load the index and return the entry for *session_key*. Lock held.""" + self._ensure_loaded_locked() + return self._entries.get(session_key) + + def _routing_scope(self) -> str: + """Namespace for this store's gateway_routing rows: the resolved + sessions_dir, so stores with different dirs never share entries.""" + try: + return str(Path(self.sessions_dir).resolve()) + except Exception: + return str(self.sessions_dir) + + def _routing_db_method(self, name: str): + """Bound ``_routing_db.`` if the handle exists and has it, else None.""" + db = self._routing_db + method = getattr(db, name, None) if db else None + return method if callable(method) else None + + @staticmethod + def _routing_entry_from_json(key: str, entry_json: str) -> Optional[SessionEntry]: + """Parse one gateway_routing row; None (with a warning) when invalid.""" + from gateway.session import SessionEntry + try: + entry_data = json.loads(entry_json) + if isinstance(entry_data, dict): + return SessionEntry.from_dict(entry_data) + except (ValueError, KeyError, TypeError) as e: + logger.warning("Skipping invalid routing entry %r: %s", key, e) + return None + + def _ensure_loaded_locked(self) -> None: + """Load the routing index. Must be called with self._lock held. + + state.db ``gateway_routing`` is primary; sessions.json is the legacy + import for keys the DB lacks (persisted to the DB on the next _save). + """ + if self._loaded: + self._reconcile_recovered_routing_locked() + return + + self.sessions_dir.mkdir(parents=True, exist_ok=True) + + db_had_entries = False + db_load_succeeded = False + loader = self._routing_db_method("load_gateway_routing_entries") + if loader is not None: + try: + for key, entry_json in loader(scope=self._routing_scope()).items(): + entry = self._routing_entry_from_json(key, entry_json) + if entry is not None: + self._entries[key] = entry + db_had_entries = bool(self._entries) + db_load_succeeded = True + except Exception as e: + logger.warning("gateway.session: state.db routing load failed: %s", e) + + self._import_legacy_sessions_json(db_had_entries) + + self._loaded = True + self._routing_db_loaded = db_load_succeeded + self._routing_fallback_baseline = ( + None + if db_load_succeeded + else {key: entry.to_dict() for key, entry in self._entries.items()} + ) + + # A hard crash skips graceful shutdown and leaves sessions.json + # pointing at ended sessions; self-heal before the first message. + self._prune_stale_sessions_locked() + + def _import_legacy_sessions_json(self, db_had_entries: bool) -> None: + """Legacy import: sessions.json fills only keys the DB lacks. Lock held.""" + from gateway.session import SessionEntry + sessions_file = self.sessions_dir / "sessions.json" + if not sessions_file.exists(): + return + try: + with open(sessions_file, "r", encoding="utf-8") as f: + data = json.load(f) + imported = 0 + for key, entry_data in data.items(): + # "_"-prefixed keys are sentinels (e.g. "_README"), not entries. + if key.startswith("_") or key in self._entries: + continue + # A non-dict entry (corrupt file) must not abort the whole load. + if not isinstance(entry_data, dict): + logger.warning( + "Skipping invalid session entry %r: " + "expected dict, got %s", + key, type(entry_data).__name__, + ) + continue + try: + self._entries[key] = SessionEntry.from_dict(entry_data) + imported += 1 + except (ValueError, KeyError, TypeError) as e: + logger.warning("Skipping invalid session entry %r: %s", key, e) + if imported and db_had_entries: + logger.info( + "gateway.session: imported %d legacy sessions.json " + "entr%s missing from state.db routing table", + imported, "y" if imported == 1 else "ies", + ) + except Exception as e: + print(f"[gateway] Warning: Failed to load sessions: {e}") + + def _prune_stale_sessions_locked(self) -> None: + """Remove routing entries whose session has ended in state.db (startup, lock held). + + Stale == ``end_reason IS NOT NULL``. Rows absent from the DB are kept; + a ``None`` DB handle is a no-op; DB errors are non-fatal. + """ + if not self._entries: + return + + stale_keys: list = [] + recovered_keys = 0 + try: + for key, entry in self._entries.items(): + # Ask the store that owns the key, not the ambient handle, or a + # live secondary-profile session gets pruned on the root copy. + db = self._db_for_key(key) + if db is None: + continue + row = db.get_session(entry.session_id) + if row is None or row.get("end_reason") is None: + continue + verdict = self._stale_entry_verdict(key, entry, row) + if verdict == "prune": + stale_keys.append(key) + elif verdict is not None: + self._entries[key] = verdict + recovered_keys += 1 + except Exception as exc: + logger.warning( + "gateway.session: stale-entry pruning skipped due to DB error: %s", + exc, + ) + return + + for key in stale_keys: + del self._entries[key] + + if stale_keys or recovered_keys: + self._save() + + def _stale_entry_verdict(self, key: str, entry, row): + """For a routing entry whose row has ended: ``"prune"``, a replacement + entry (repoint), or None (keep as-is).""" + from gateway.session import _now + recovered_entry = None + if entry.origin is not None: + try: + recovered_entry = self._recover_session_from_db( + session_key=key, + source=entry.origin, + now=_now(), + raise_on_lookup_error=True, + ) + except Exception as exc: + # Indeterminate: keep the only routing handle. + logger.debug( + "gateway.session: recovery lookup failed for stale " + "sessions.json entry %r -> %s: %s", + key, + entry.session_id, + exc, + ) + return None + + # Compression-ended parent with a newer live child for the same peer: + # repoint instead of dropping, or queued/resume-pending work vanishes + # until the next message. + if recovered_entry is not None and recovered_entry.session_id != entry.session_id: + logger.warning( + "gateway.session: repointing stale sessions.json entry " + "%r from ended %s (end_reason=%r) to recovered %s", + key, + entry.session_id, + row["end_reason"], + recovered_entry.session_id, + ) + return recovered_entry + + # Same-id recovery == successful resume: keep the ORIGINAL entry object + # (the recovered one is rebuilt minimal and would drop counters, + # model_override, resume markers, metadata). Nothing changes, no save. + if recovered_entry is not None: + logger.info( + "gateway.session: reopened ended session %s for " + "sessions.json entry %r (end_reason=%r); keeping route", + entry.session_id, key, row["end_reason"], + ) + return None + + logger.warning( + "gateway.session: pruning stale sessions.json entry " + "%r -> %s (end_reason=%r); left by a crashed gateway", + key, entry.session_id, row["end_reason"], + ) + return "prune" + + def _save(self) -> None: + """Persist the routing index while the caller holds ``_lock``.""" + data, generation = self._snapshot_routing_locked() + self._persist_routing_data(data, generation) + + def _next_routing_generation_locked(self) -> int: + """Bump and return the shared routing counter. Caller holds ``_lock``. + + Full snapshots AND single-entry fast saves MUST allocate from this one + counter: the stale-write protection is a total order over + serialization times and silently breaks otherwise. + """ + self._routing_generation = getattr(self, "_routing_generation", 0) + 1 + return self._routing_generation + + def _reconcile_recovered_routing_locked(self) -> None: + """Merge authoritative rows after a fallback-only startup load.""" + baseline = getattr(self, "_routing_fallback_baseline", None) + if getattr(self, "_routing_db_loaded", False) or baseline is None: + return + + loader = self._routing_db_method("load_gateway_routing_entries") + if loader is None: + return + try: + durable = loader(scope=self._routing_scope()) + except Exception as exc: + logger.warning("gateway.session: recovered state.db routing load failed: %s", exc) + return + + current = {key: entry.to_dict() for key, entry in self._entries.items()} + for key, entry_json in durable.items(): + durable_entry = self._routing_entry_from_json(key, entry_json) + if durable_entry is None: + continue + + if key not in baseline: + # A key created while on fallback wins over a DB-only key; + # otherwise restore the authoritative row that fallback never saw. + self._entries.setdefault(key, durable_entry) + elif key not in current: + # The key was loaded from fallback and deliberately removed. + continue + elif current[key] == baseline[key]: + # Unchanged fallback data yields to the authoritative DB copy. + self._entries[key] = durable_entry + + self._routing_db_loaded = True + self._routing_fallback_baseline = None + + def _snapshot_routing_locked(self) -> tuple[Dict[str, Any], int]: + """Capture immutable routing data and a monotonic generation.""" + self._reconcile_recovered_routing_locked() + return ( + {key: entry.to_dict() for key, entry in self._entries.items()}, + self._next_routing_generation_locked(), + ) + + def _persist_routing_data(self, data: Dict[str, Any], generation: int) -> None: + """Serialize all whole-index writers through one durable write lock.""" + with self._lazy("_save_lock", threading.Lock): + if generation <= getattr(self, "_persisted_routing_generation", 0): + return + # Fold in fast upserts numbered above this snapshot: they were + # serialized after us and a delayed full rewrite must not regress them. + fast_persisted = getattr(self, "_fast_persisted_entries", None) + if fast_persisted: + for key, (revision, entry_json) in fast_persisted.items(): + if revision > generation: + data[key] = json.loads(entry_json) + db_saved = False + replacer = self._routing_db_method("replace_gateway_routing_entries") + if replacer is not None: + try: + replacer( + {k: json.dumps(v) for k, v in data.items()}, + scope=self._routing_scope(), + ) + db_saved = True + except Exception as exc: + logger.warning("gateway.session: state.db routing save failed: %s", exc) + if getattr(self, "_write_sessions_json", True) or not db_saved: + try: + self._save_sessions_json(data) + except Exception as exc: + if not db_saved: + raise + # state.db is authoritative. A failed legacy mirror must not + # report the already-committed primary write as failed. + logger.warning( + "gateway.session: sessions.json mirror save failed " + "after state.db commit: %s", + exc, + ) + self._persisted_routing_generation = generation + # This rewrite supersedes fast records at or below its + # generation; newer ones stay for the next delayed full writer. + if fast_persisted: + for key in [ + k for k, (rev, _) in fast_persisted.items() + if rev <= generation + ]: + del fast_persisted[key] + + def _save_sessions_json(self, data: Dict[str, Any]) -> None: + """Write the legacy sessions.json mirror of the routing index.""" + import tempfile + self.sessions_dir.mkdir(parents=True, exist_ok=True) + sessions_file = self.sessions_dir / "sessions.json" + + # Self-documenting sentinel; "_" keys are skipped on load. Ordered + # first so it renders at the top of the file. + data = { + "_README": ( + "LEGACY MIRROR of the gateway routing index (the primary copy " + "lives in the gateway_routing table in ~/.hermes/state.db). " + "Maps messaging session keys (agent:main::...) to " + "active session IDs. This is NOT the session list. ALL " + "sessions (CLI, TUI, and gateway) live in ~/.hermes/state.db " + "and are shown by `hermes sessions list` and `/sessions`. " + "Disable this file with `gateway.write_sessions_json: false` " + "in config.yaml." + ), + **data, + } + fd, tmp_path = tempfile.mkstemp( + dir=str(self.sessions_dir), suffix=".tmp", prefix=".sessions_" + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + f.flush() + os.fsync(f.fileno()) + atomic_replace(tmp_path, sessions_file) + except BaseException: + try: + os.unlink(tmp_path) + except OSError as e: + logger.debug("Could not remove temp file %s: %s", tmp_path, e) + raise + + def _save_entries(self) -> None: + """Snapshot latest state under ``_lock`` and persist after releasing it.""" + with self._lock: + data, generation = self._snapshot_routing_locked() + self._persist_routing_data(data, generation) + + def _save_entry( + self, + session_key: str, + *, + entry_data: Optional[Dict[str, Any]] = None, + lock_held: bool = False, + ) -> None: + """Persist ONE routing entry via UPSERT — the per-turn fast path + (a full rewrite fsyncs a multi-MB sessions.json, ~50ms at ~1100 keys). + + Invariants: the key -> session_id mapping never changes here — + structural transitions (create/recover/reset/switch/prune/heal) use the + full rewrite, which also refreshes the sessions.json mirror (it may lag + in metadata only; state.db stays primary). The entry is serialized under + ``_lock`` with a revision from the shared routing generation counter + (higher == same-or-newer); under ``_save_lock`` the upsert is skipped if + a full snapshot or a newer fast save of this key already persisted (the + reverse case lives in ``_persist_routing_data``). No DB or a failed + upsert falls back to the full rewrite so DB-less installs stay durable. + ``entry_data`` persists a candidate BEFORE it is published to the live + entry (failure-atomic transitions); the fallback carries the same candidate. + """ + def _capture() -> Optional[tuple[str, int, Optional[Dict[str, Any]]]]: + entry = self._entries.get(session_key) + if entry is None: + return None + serialized_entry = ( + dict(entry_data) if entry_data is not None else entry.to_dict() + ) + entry_json = json.dumps(serialized_entry) + revision = self._next_routing_generation_locked() + # The O(n) full snapshot is deferred to the fallback branch. + return entry_json, revision, serialized_entry if entry_data is not None else None + + if lock_held: + captured = _capture() + else: + with self._lock: + captured = _capture() + if captured is None: + return + entry_json, revision, candidate_entry = captured + saver = self._routing_db_method("save_gateway_routing_entry") + if saver is not None: + try: + with self._lazy("_save_lock", threading.Lock): + if getattr(self, "_persisted_routing_generation", 0) >= revision: + return + fast_persisted = self._lazy("_fast_persisted_entries", dict) + persisted = fast_persisted.get(session_key) + if persisted is not None and persisted[0] >= revision: + return + saver(session_key, entry_json, scope=self._routing_scope()) + fast_persisted[session_key] = (revision, entry_json) + return + except Exception as exc: + logger.warning( + "gateway.session: single-entry routing save failed for %r " + "(%s); falling back to full index rewrite", + session_key, exc, + ) + if candidate_entry is not None: + # Full-snapshot fallback carrying the candidate transition. + def _snapshot() -> Dict[str, Any]: + return {key: current.to_dict() for key, current in self._entries.items()} + if lock_held: + fallback_data = _snapshot() + else: + with self._lock: + fallback_data = _snapshot() + fallback_data[session_key] = candidate_entry + self._persist_routing_data(fallback_data, revision) + else: + self._save_entries() diff --git a/gateway/session_recovery.py b/gateway/session_recovery.py new file mode 100644 index 0000000000..892c57b483 --- /dev/null +++ b/gateway/session_recovery.py @@ -0,0 +1,519 @@ +"""SessionStore durable-row recovery: session-key generation, legacy Slack key +migration, rebuilding a routing entry from state.db, and the SQLite side of +routing transitions (promote/reopen/create/peer). + +Mixin split out of ``gateway/session.py``; bound onto ``SessionStore`` via the MRO. +""" + +from __future__ import annotations + +import logging +import json +import threading +from dataclasses import replace +from datetime import datetime +from gateway.config import Platform +from typing import TYPE_CHECKING, Any, Dict, Optional + +if TYPE_CHECKING: + from gateway.session import SessionEntry, SessionSource + +# Log-record parity with the origin module. +logger = logging.getLogger("gateway.session") + + +def _origin_json(source) -> Optional[str]: + """``source.to_dict()`` as JSON, or None when absent/unserializable.""" + if source is None: + return None + try: + return json.dumps(source.to_dict()) + except Exception: + return None + + +class SessionRecoveryMixin: + """SessionStore durable-row recovery: session-key generation, legacy Slack + key migration, rebuilding a routing entry from state.db, and the SQLite + side of routing transitions (promote/reopen/create/peer). + """ + + def _resolve_profile_for_key(self, source: Optional[SessionSource] = None) -> Optional[str]: + """Profile namespace for session keys: None when multiplexing is off + (legacy ``agent:main``), else ``source.profile`` or the active profile.""" + if not getattr(self.config, "multiplex_profiles", False): + return None + if source is not None and source.profile: + return source.profile + try: + from hermes_cli.profiles import get_active_profile_name + return get_active_profile_name() or "default" + except Exception: + return None + + @staticmethod + def _profile_from_session_key(session_key: Optional[str]) -> Optional[str]: + """Extract the profile namespace encoded in a gateway session key.""" + if not session_key: + return None + parts = str(session_key).split(":") + if len(parts) < 2 or parts[0] != "agent": + return None + namespace = parts[1] or "main" + return "default" if namespace == "main" else namespace + + @staticmethod + def _active_profile_name() -> str: + try: + from hermes_cli.profiles import get_active_profile_name + return get_active_profile_name() or "default" + except Exception: + return "default" + + def _recovered_row_allowed_for_active_profile( + self, + *, + requested_session_key: str, + recovered: Dict[str, Any], + ) -> bool: + """Prevent a gateway from reviving another profile's row. + + Single-profile: the row's namespace must match the ACTIVE profile. + Multiplexed: it must match the namespace of the requested key (the + active profile is meaningless there). Keyless rows stay adoptable. + """ + recovered_key = str(recovered.get("session_key") or "") + if not recovered_key or recovered_key == requested_session_key: + return True + + recovered_profile = self._profile_from_session_key(recovered_key) + if recovered_profile is None: + return True + + if getattr(self.config, "multiplex_profiles", False): + requested_profile = self._profile_from_session_key(requested_session_key) + return requested_profile is None or recovered_profile == requested_profile + + return recovered_profile == self._active_profile_name() + + def _generate_session_key(self, source: SessionSource, key_source: Optional[SessionSource] = None) -> str: + """Session key for *source* (profile resolved from *source*, key built + from *key_source* when given).""" + from gateway.session import build_session_key + return build_session_key( + key_source if key_source is not None else source, + group_sessions_per_user=getattr(self.config, "group_sessions_per_user", True), + thread_sessions_per_user=getattr(self.config, "thread_sessions_per_user", False), + profile=self._resolve_profile_for_key(source), + ) + + def _legacy_slack_session_key(self, source: SessionSource) -> Optional[str]: + """Pre-workspace Slack key for an explicitly scoped source. + + Deliberately Slack-only; an unscoped Slack session may be claimed by + only one workspace because its old key cannot distinguish teams. + """ + if source.platform != Platform.SLACK or not source.scope_id: + return None + return self._generate_session_key( + source, replace(source, scope_id=None, guild_id=None) + ) + + def _claim_legacy_slack_key(self, legacy_key: Optional[str]) -> bool: + """Atomically reserve one ambiguous legacy Slack key for migration.""" + if not legacy_key: + return False + with self._lazy("_legacy_slack_claim_lock", threading.Lock): + claimed = self._lazy("_claimed_legacy_slack_keys", set) + if legacy_key in claimed: + return False + claimed.add(legacy_key) + return True + + @staticmethod + def _recovered_row_matches_source_scope( + recovered: Dict[str, Any], source: SessionSource + ) -> bool: + """Reject recovered rows whose recorded origin belongs to another workspace. + + A workspace-scoped Slack lookup adopts a row only if its origin_json + names the same scope_id; rows without a parseable origin are rejected + (an unattributable transcript is exactly the ambiguity to avoid). + """ + if ( + source.platform != Platform.SLACK + or source.chat_type == "dm" + or not source.scope_id + ): + return True + try: + origin = json.loads(recovered.get("origin_json") or "") + except (TypeError, ValueError): + return False + if not isinstance(origin, dict): + return False + return origin.get("scope_id", origin.get("guild_id")) == source.scope_id + + def _create_entry_from_recovered_row( + self, + *, + row: Dict[str, Any], + session_key: str, + source: SessionSource, + now: datetime, + ) -> SessionEntry: + from gateway.session import SessionEntry + def _ts(value, default: datetime) -> datetime: + try: + return datetime.fromtimestamp(float(value)) + except (TypeError, ValueError, OSError): + return default + + # An invalid durable timestamp must look old, never freshly active. + created_at = _ts(row.get("started_at"), datetime.fromtimestamp(0)) + # The finder already returns durable recency; no extra round-trip. + last_activity = row.get("last_activity_at") + updated_at = _ts(last_activity, created_at) if last_activity is not None else created_at + had_activity = row.get("_has_messages") + if had_activity is None: + had_activity = bool(row.get("message_count") or 0) or ( + last_activity is not None + ) + return SessionEntry( + session_key=session_key, + session_id=str(row["id"]), + created_at=created_at, + updated_at=updated_at, + origin=source, + display_name=source.chat_name, + platform=source.platform, + chat_type=source.chat_type, + reset_had_activity=bool(had_activity), + ) + + def _find_gateway_session_row( + self, + *, + session_key: str, + source: SessionSource, + allow_peer_fallback: bool, + raise_on_lookup_error: bool = False, + ) -> Optional[Dict[str, Any]]: + """Query one durable gateway session row. + + Scoped Slack lookups disable SessionDB's platform/chat/user fallback: + that tuple does not contain a workspace id and could therefore revive + another team's session. The caller performs one explicit exact lookup + of the old unscoped key instead. + """ + db = self._db_for_key(session_key) + finder = getattr(db, "find_latest_gateway_session_for_peer", None) if db else None + if not callable(finder): + return None + try: + return finder( + source=source.platform.value, + user_id=source.user_id, + session_key=session_key, + chat_id=source.chat_id if allow_peer_fallback else None, + chat_type=source.chat_type if allow_peer_fallback else None, + thread_id=source.thread_id, + ) + except Exception as exc: + logger.debug("Gateway session DB recovery failed for %s: %s", session_key, exc) + if raise_on_lookup_error: + raise + return None + + def _recover_session_from_db( + self, + *, + session_key: str, + source: SessionSource, + now: datetime, + raise_on_lookup_error: bool = False, + ) -> Optional[SessionEntry]: + """Rebuild a missing session-key mapping from durable state.db data. + + Returns ``None`` when no row is recoverable, or when the recovered + session is already overdue under the reset policy — the row is then + durably promoted to a reset boundary instead of resurrected. + """ + entry, migrated_legacy = self._query_recoverable_row( + session_key=session_key, + source=source, + now=now, + raise_on_lookup_error=raise_on_lookup_error, + ) + if entry is None: + return None + reset_reason = self._should_reset(entry, source) + if reset_reason: + self._promote_session_reset( + session_key, entry.session_id, reset_reason, + log=lambda exc: logger.debug( + "Gateway recovered-session reset promotion failed for %s: %s", + session_key, exc, + ), + ) + return None + self._reopen_session_row(session_key, entry.session_id) + if migrated_legacy: + self._record_gateway_session_peer( + entry.session_id, session_key, source, display_name=entry.display_name, + ) + return entry + + def _query_recoverable_session(self, *, session_key, source, now): + """DB-only half of _recover_session_from_db (no lock needed). + + Returns a SessionEntry or None. Caller assigns _entries[key] under + lock. The row is NOT reopened here: the caller evaluates reset policy + first (an agent_close/ws_orphan row may need promotion to a real reset + boundary instead). + """ + entry, migrated_legacy = self._query_recoverable_row( + session_key=session_key, source=source, now=now, + ) + if entry is not None and migrated_legacy: + self._record_gateway_session_peer( + entry.session_id, session_key, source, display_name=entry.display_name, + ) + return entry + + def _query_recoverable_row( + self, *, session_key, source, now, raise_on_lookup_error=False, + ) -> tuple[Optional[SessionEntry], bool]: + """Find and gate a recoverable row -> (entry or None, migrated_legacy). + + The legacy (pre-workspace) Slack key fallback lives here: exact-key + lookup, claimed once per process; ``migrated_legacy`` tells the caller + to rewrite the peer row to the scoped key. + """ + legacy_key = self._legacy_slack_session_key(source) + recovered = self._find_gateway_session_row( + session_key=session_key, + source=source, + allow_peer_fallback=legacy_key is None, + raise_on_lookup_error=raise_on_lookup_error, + ) + migrated_legacy = False + if ( + not recovered + and legacy_key + and self._claim_legacy_slack_key(legacy_key) + ): + recovered = self._find_gateway_session_row( + session_key=legacy_key, + source=source, + allow_peer_fallback=False, + raise_on_lookup_error=raise_on_lookup_error, + ) + migrated_legacy = bool(recovered) + if not isinstance(recovered, dict): + return None, False + if not self._recovered_row_matches_source_scope(recovered, source): + return None, False + if not self._recovered_row_allowed_for_active_profile( + requested_session_key=session_key, + recovered=recovered, + ): + logger.warning( + "Gateway session DB recovery ignored %s for %s because " + "the row belongs to a different profile", + recovered.get("session_key"), + session_key, + ) + return None, False + entry = self._create_entry_from_recovered_row( + row=recovered, session_key=session_key, source=source, now=now, + ) + return entry, migrated_legacy + + def _promote_session_reset(self, session_key: str, session_id: str, reason: str, *, log) -> None: + """End *session_id* with *reason* via ``promote_to_session_reset``. + + Promote (not plain ``end_session``): a row already ended with a + recoverable accidental reason (agent_close / ws_orphan_reap) must be + upgraded to the explicit boundary, or stale-route recovery resurrects + it over the reset. Falls back to ``end_session`` on old SessionDBs. + ``log(exc)`` reports failures (each caller has its own message). + """ + try: + db = self._db_for_key(session_key) + promote = getattr(db, "promote_to_session_reset", None) + if callable(promote): + promote(session_id, reason) + else: + db.end_session(session_id, reason) + except Exception as exc: + log(exc) + + def _reopen_session_row(self, session_key: str, session_id: str, *, log_prefix: str = "") -> None: + """Best-effort ``reopen_session``; failures are debug-logged only.""" + try: + self._db_for_key(session_key).reopen_session(session_id) + except Exception as exc: + if log_prefix: + logger.debug("%s: %s", log_prefix, exc) + else: + logger.debug("Gateway session DB reopen failed for %s: %s", session_key, exc) + + def _record_gateway_session_peer( + self, + session_id: str, + session_key: str, + source: Optional[SessionSource], + display_name: Optional[str] = None, + include_compression_ancestors: bool = False, + ) -> None: + """Persist the routing peer for an existing gateway session row.""" + db = self._db_for_key(session_key) + if not db or not source: + return + recorder = getattr(db, "record_gateway_session_peer", None) + if not callable(recorder): + return + peer = dict( + source=source.platform.value, + user_id=source.user_id, + session_key=session_key, + chat_id=source.chat_id, + chat_type=source.chat_type, + thread_id=source.thread_id, + ) + try: + origin_json = _origin_json(source) + recorder( + session_id, + **peer, + display_name=display_name or source.chat_name, + origin_json=origin_json, + include_compression_ancestors=include_compression_ancestors, + ) + except TypeError: + # Older SessionDB without display_name/origin_json kwargs. + try: + recorder(session_id, **peer) + except Exception as exc: + logger.debug("Gateway session peer record failed for %s: %s", session_key, exc) + except Exception as exc: + logger.debug("Gateway session peer record failed for %s: %s", session_key, exc) + + def _adopt_legacy_slack_entry(self, source: SessionSource, session_key: str) -> None: + """One-time migration of pre-workspace-scope Slack keys. + + MOVE (not copy) the legacy entry so a second workspace with identical + Slack ids cannot attach to the same transcript. Adopt when the legacy + origin names the same workspace; a scope-less DM is claimed once by + the first workspace; a scope-less channel/group is refused (channel + ids collide across workspaces). + """ + legacy_key = self._legacy_slack_session_key(source) + if not legacy_key: + return + migrated: Optional[SessionEntry] = None + with self._lock: + self._ensure_loaded_locked() + legacy_entry = self._entries.get(legacy_key) + if session_key not in self._entries and legacy_entry is not None: + origin_scope = getattr(legacy_entry.origin, "scope_id", None) + if origin_scope is not None: + adopt = origin_scope == source.scope_id + else: + adopt = source.chat_type == "dm" + if adopt and self._claim_legacy_slack_key(legacy_key): + migrated = self._entries.pop(legacy_key) + migrated.session_key = session_key + migrated.origin = source + migrated.platform = source.platform + migrated.chat_type = source.chat_type + self._entries[session_key] = migrated + if migrated is not None: + self._save_entries() + self._record_gateway_session_peer( + migrated.session_id, session_key, source, display_name=migrated.display_name, + ) + + def _finish_route_transition( + self, + session_key: str, + *, + end_session_id: Optional[str], + end_reason: str, + create_kwargs: Optional[Dict[str, Any]], + origin: Optional[SessionSource], + display_name: Optional[str], + during: str = "", + ) -> None: + """SQLite side of a routing transition, outside ``_lock``. + + Promotes the predecessor row to an explicit reset boundary (with the + specific reason so state.db is auditable, e.g. ``resume_pending_expired`` + vs a plain ``session_reset``), then INSERTs the new row + routing peer. + Both are best-effort: failures are warned and self-healed by the next + per-turn peer refresh. + """ + if self._db_for_key(session_key) and end_session_id: + self._promote_session_reset( + session_key, end_session_id, end_reason, + log=lambda e: logger.warning( + "Failed to end predecessor session row %s for %s%s: %s — " + "the old row remains open and may win restart recovery " + "until the next successful peer refresh", + end_session_id, session_key, during, e, + ), + ) + if self._db_for_key(session_key) and create_kwargs: + self._create_session_row( + session_key, create_kwargs, origin, display_name, + log=lambda e: logger.warning( + "Failed to create session row %s for %s%s: %s — deferring " + "to the self-healing peer refresh on the next turn", + create_kwargs.get("session_id"), session_key, during, e, + ), + ) + + @staticmethod + def _session_create_kwargs( + *, session_id, session_key, origin, source_value, display_name, parent_session_id, + ) -> Dict[str, Any]: + """kwargs for ``SessionDB.create_session``. + + Identity (origin_json) and lineage (parent/_reset_from) land atomically + in the INSERT so a crash right after cannot strand the row unroutable. + """ + origin_json = _origin_json(origin) + return { + "session_id": session_id, + "source": source_value, + "user_id": origin.user_id if origin else None, + "session_key": session_key, + "chat_id": origin.chat_id if origin else None, + "chat_type": origin.chat_type if origin else None, + "thread_id": origin.thread_id if origin else None, + "profile_name": origin.profile if origin else None, + "origin_json": origin_json, + "display_name": display_name, + "parent_session_id": parent_session_id, + "model_config": ( + {"_reset_from": parent_session_id} if parent_session_id else None + ), + } + + def _create_session_row(self, session_key, db_create_kwargs, origin, display_name, *, log) -> None: + """INSERT a session row and record its routing peer; ``log(exc)`` on failure. + + A failed create is a routing hazard (visible warning), but the row is + self-healed with full identity by the next per-turn peer refresh. + """ + try: + self._db_for_key(session_key).create_session(**db_create_kwargs) + self._record_gateway_session_peer( + db_create_kwargs["session_id"], + session_key, + origin, + display_name=display_name, + ) + except Exception as e: + log(e) diff --git a/gateway/session_transcript.py b/gateway/session_transcript.py new file mode 100644 index 0000000000..e2f4430451 --- /dev/null +++ b/gateway/session_transcript.py @@ -0,0 +1,641 @@ +"""SessionStore transcript I/O: SQLite append with a per-session retry queue, +compression-reroute following, FTS corruption recovery, rewrite/rewind/load. + +Mixin split out of ``gateway/session.py``; bound onto ``SessionStore`` via the MRO. +""" + +from __future__ import annotations + +import logging +import threading +from agent.turn_context import extract_api_content_sidecar +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +if TYPE_CHECKING: + from gateway.session import SessionEntry + +# Log-record parity with the origin module. +logger = logging.getLogger("gateway.session") + + +def _plain_text(content) -> str: + """Text of a message content (str or text-part list); "" for anything else.""" + if isinstance(content, list): + parts = [p.get("text", "") for p in content if isinstance(p, dict) and p.get("type") == "text"] + return "\n".join(t for t in parts if t) + return content if isinstance(content, str) else "" + + +class SessionTranscriptMixin: + """SessionStore transcript I/O: SQLite append with a per-session retry queue, + compression-reroute following, FTS corruption recovery, + rewrite/rewind/load. + """ + + def _compression_tip_for_session_id(self, session_id: Optional[str]) -> Optional[str]: + """Latest compression continuation for *session_id* (heals a mapping + left pointing at a compressed parent by a restart or failed send).""" + if not session_id: + return session_id + db = self._db_for_session_id(session_id) + if db is None: + return session_id + try: + return db.get_compression_tip(session_id) or session_id + except Exception: + logger.debug("Compression-tip lookup failed for session %s", session_id, exc_info=True) + return session_id + + def _heal_compression_tip_locked( + self, + entry: "SessionEntry", + original_session_id: Optional[str], + canonical_session_id: Optional[str], + ) -> bool: + """Rewrite *entry* to the compression continuation if stale. Lock held.""" + if ( + not original_session_id + or not canonical_session_id + or entry.session_id != original_session_id + or canonical_session_id == original_session_id + ): + return False + logger.info( + "SessionStore healed compressed session mapping: %s -> %s", + entry.session_id, + canonical_session_id, + ) + entry.session_id = canonical_session_id + return True + + def advance_compression_session( + self, + session_key: str, + expected_session_id: str, + target_session_id: str, + ) -> Optional[SessionEntry]: + """CAS-advance one route along an already-verified compression lineage. + + Unlike ``switch_session`` this never ends/reopens SQLite rows (the + compression transaction owns that). ``None`` means the route moved + after the caller's snapshot (e.g. /new) — caller must fail closed. + """ + if not session_key or not expected_session_id or not target_session_id: + return None + + with self._lock: + entry = self._entry_locked(session_key) + if entry is None: + return None + if entry.session_id == target_session_id: + return entry + if entry.session_id != expected_session_id: + return None + if not self._heal_compression_tip_locked( + entry, + expected_session_id, + target_session_id, + ): + return None + # Bookkeeping, not user activity: leave ``updated_at`` alone. + self._save() + return entry + + def _get_transcript_drain_lock(self): + """Return the lock that serializes pending-queue drain boundaries.""" + return self._lazy("_transcript_drain_lock", threading.RLock) + + def append_to_transcript(self, session_id: str, message: Dict[str, Any], skip_db: bool = False) -> None: + """Serialize transcript draining across queue migration boundaries.""" + if not self._db_for_session_id(session_id) or skip_db: + return + with self._get_transcript_drain_lock(): + self._append_to_transcript_serialized( + self._follow_reroutes(session_id), message + ) + + def _follow_reroutes(self, session_id: str) -> str: + """Follow the compression reroute chain (cycle-guarded).""" + reroutes = self._lazy("_transcript_reroutes", dict) + seen = set() + while session_id in reroutes and session_id not in seen: + seen.add(session_id) + session_id = reroutes[session_id] + return session_id + + def _spool_dropped(self, session_id: str, message: Dict[str, Any]): + """Spool one evicted/undeliverable message to disk; path or None.""" + try: + from gateway.shutdown_flush import spool_dropped_transcript_message + + return spool_dropped_transcript_message(session_id, message) + except Exception: + return None + + def _enqueue_transcript_message(self, session_id: str, message: Dict[str, Any]) -> list: + """Queue *message* (retry lock held); evicts + spools the oldest past the cap. + + Spooling uses the same machinery as shutdown flush so the message is + replayed after DB recovery instead of being lost. + """ + pending = self._dirty_transcripts.setdefault(session_id, []) + pending.append(dict(message)) + if len(pending) > self._MAX_PENDING_PER_SESSION: + spool_path = self._spool_dropped(session_id, pending.pop(0)) + if spool_path is not None: + self._lazy("_spooled_drop_sessions", set).add(session_id) + logger.warning( + "Session DB transcript pending queue full for %s " + "(cap=%d); spooled oldest message to %s for replay " + "after DB recovery", + session_id, self._MAX_PENDING_PER_SESSION, spool_path, + ) + else: + logger.warning( + "Session DB transcript pending queue full for %s " + "(cap=%d); dropping oldest message to make room " + "(on-disk spool unavailable)", + session_id, self._MAX_PENDING_PER_SESSION, + ) + return pending + + def _divert_transcript_after_db_replaced( + self, session_id: str, queue_session_id: str, exc: Exception + ) -> None: + """Stop SQLite writes on a replaced/quarantined handle and divert the backlog. + + Retrying cannot succeed and the FTS rebuild must never run on this + handle; the pending queue goes to the on-disk spool + JSONL fallback. + """ + logger.error( + "Session DB refused further writes on this handle for " + "%s (%s); stopping SQLite writes and diverting pending " + "transcripts to the on-disk fallback: %s", + session_id, type(exc).__name__, exc, + ) + with self._transcript_retry_lock: + remaining = list(self._dirty_transcripts.get(queue_session_id, [])) + self._dirty_transcripts.pop(queue_session_id, None) + self._transcript_append_failures.pop(session_id, None) + for dropped in remaining: + try: + from gateway.shutdown_flush import spool_dropped_transcript_message + + spool_dropped_transcript_message(session_id, dropped) + except Exception: + logger.warning( + "pending fallback failed for replaced " + "state.db transcript on %s", + session_id, + exc_info=True, + ) + try: + from hermes_state import divert_session_transcript_jsonl + divert_session_transcript_jsonl(session_id, remaining) + except Exception: + logger.warning( + "JSONL divert failed for replaced state.db " + "transcript on %s", + session_id, + exc_info=True, + ) + + def _live_compression_child(self, session_id: str) -> str: + """Transitive compression tip of *session_id* if it is a different, still-live + row, else "" (a depth-1 lookup misses multi-hop lineages). + + Uses the PARENT's proven owner handle: the child's id is not published + until after its write succeeds, so a by-id lookup would fall back to + the ambient store. + """ + owner_db = self._db_for_session_id(session_id) + if owner_db is None: + return "" + tip = owner_db.get_compression_tip(session_id) + if tip and tip != session_id: + tip_row = owner_db.get_session(tip) + if tip_row is not None and tip_row.get("ended_at") is None: + return str(tip) + return "" + + def _migrate_transcript_queue_to_child( + self, session_id: str, queue_session_id: str, child_id: str, pending: list, msg + ) -> list: + """Move the retry queue + failure counter from parent to child and publish + the reroute (retry lock held). Returns the child's pending list. + + Older parent backlog must precede messages already queued directly on + the child. Routing is published only AFTER the queue moved (caller), so + new child writes cannot bypass older parent backlog. + """ + if pending and pending[0] is msg: + pending.pop(0) + existing_child_pending = self._dirty_transcripts.get(child_id, []) + if pending: + pending.extend(existing_child_pending) + self._dirty_transcripts[child_id] = pending + elif existing_child_pending: + pending = existing_child_pending + self._dirty_transcripts.pop(queue_session_id, None) + previous_failures = self._transcript_append_failures.pop(queue_session_id, 0) + if previous_failures: + self._transcript_append_failures[child_id] = max( + previous_failures, + self._transcript_append_failures.get(child_id, 0), + ) + self._transcript_reroutes[session_id] = child_id + return pending + + def _publish_transcript_reroute(self, session_id: str, child_id: str) -> None: + """Repoint every route at the compression child and save (index authoritative again).""" + with self._lock: + for entry in self._entries.values(): + if entry.session_id == session_id: + entry.session_id = child_id + self._save() + _hints = getattr(self, "_session_owner_hints", None) + if _hints: + _hints.pop(child_id, None) + + def _append_to_transcript_serialized( + self, session_id: str, message: Dict[str, Any] + ) -> None: + """Append a message to a session's transcript (SQLite), draining the + per-session retry queue.""" + with self._transcript_retry_lock: + pending = self._enqueue_transcript_message(session_id, message) + msg = pending[0] + queue_session_id = session_id + + def _ack_head() -> bool: + """Pop the acknowledged head (retry lock held). True if queue drained.""" + if pending and pending[0] is msg: + pending.pop(0) + if not pending: + self._dirty_transcripts.pop(queue_session_id, None) + self._transcript_append_failures.pop(session_id, None) + return True + return False + + # DB write outside the retry lock so other sessions can append. + while True: + try: + self._append_transcript_message(session_id, msg) + except Exception as exc: + from hermes_state import ( + CompressionSessionClosedError, + StateDbCorruptError, + StateDbReplacedError, + ) + + if isinstance(exc, (StateDbReplacedError, StateDbCorruptError)): + self._divert_transcript_after_db_replaced(session_id, queue_session_id, exc) + return + + if isinstance(exc, CompressionSessionClosedError): + # Adopt only a different, still-live compression tip, else + # fail closed. + _owner_key = self._owner_key_for_session_id(session_id) + child_id = self._live_compression_child(session_id) + if child_id: + # Record the child's owner BEFORE writing to it (the + # reroute is published only after the write succeeds + # — load-bearing for backlog order). + if _owner_key: + self._lazy("_session_owner_hints", dict)[child_id] = _owner_key + try: + self._append_transcript_message(child_id, msg) + except Exception as reroute_exc: + exc = reroute_exc + else: + with self._transcript_retry_lock: + pending = self._migrate_transcript_queue_to_child( + session_id, queue_session_id, child_id, pending, msg + ) + queue_session_id = child_id + self._publish_transcript_reroute(session_id, child_id) + if not pending: + return + msg = pending[0] + session_id = child_id + continue + else: + # Permanent routing invariant failure, not a transient + # outage: drop it so it cannot poison later writes. + with self._transcript_retry_lock: + _ack_head() + logger.error( + "Session DB transcript append rejected for compression-ended " + "%s with no unique live child; not retrying", + session_id, + ) + return + if self._is_fts_corruption_error(exc) and self._rebuild_fts_once(): + try: + self._append_transcript_message(session_id, msg) + except Exception as retry_exc: + exc = retry_exc + else: + with self._transcript_retry_lock: + _ack_head() + continue + with self._transcript_retry_lock: + failures = self._transcript_append_failures.get(session_id, 0) + 1 + self._transcript_append_failures[session_id] = failures + logger.warning( + "Session DB transcript append failed for %s " + "(failure_count=%d, pending=%d); will retry: %s", + session_id, failures, len(pending), exc, + ) + return + else: + with self._transcript_retry_lock: + queue_empty = _ack_head() + if not queue_empty: + msg = pending[0] + if queue_empty: + # Backlog clear: replay cap-dropped messages spooled to disk. + self._drain_spooled_drops(session_id) + return + continue + + def _drain_spooled_drops(self, session_id: str) -> None: + """Replay cap-dropped spooled transcript messages after DB recovery. + + Best-effort: replay failures keep the spool files for the next + successful flush; nothing here may raise into the caller. + """ + spooled_sessions = getattr(self, "_spooled_drop_sessions", None) + if not spooled_sessions or session_id not in spooled_sessions: + return + try: + from gateway.shutdown_flush import drain_transcript_spool + + _replayed, remaining = drain_transcript_spool( + session_id, + lambda message: self._append_transcript_message( + session_id, message + ), + ) + if not remaining: + spooled_sessions.discard(session_id) + except Exception as exc: + logger.warning("Failed to drain transcript spool for %s: %s", session_id, exc) + + def _append_transcript_message(self, session_id: str, message: Dict[str, Any]) -> None: + """Write one transcript row. Caller handles retry queuing.""" + _db = self._db_for_session_id(session_id) + if _db is None: + # Named profile with no resolvable home yet: defer (caller queues) + # instead of writing into the ambient store. + raise RuntimeError( + f"no owning session store for {session_id}; deferring transcript write" + ) + is_assistant = message.get("role") == "assistant" + _db.append_message( + session_id=session_id, + role=message.get("role", "unknown"), + content=message.get("content"), + tool_name=message.get("tool_name"), + tool_calls=message.get("tool_calls"), + tool_call_id=message.get("tool_call_id"), + reasoning=message.get("reasoning") if is_assistant else None, + reasoning_content=message.get("reasoning_content") if is_assistant else None, + reasoning_details=message.get("reasoning_details") if is_assistant else None, + codex_reasoning_items=message.get("codex_reasoning_items") if is_assistant else None, + codex_message_items=message.get("codex_message_items") if is_assistant else None, + platform_message_id=(message.get("platform_message_id") or message.get("message_id")), + observed=bool(message.get("observed")), + timestamp=message.get("timestamp"), + # Exact bytes sent to the API (prompt-cache-stable replay); must + # survive every persistence path or the next replay diverges. + api_content=extract_api_content_sidecar(message), + # Presentation typing (e.g. "internal_notification"); DB-only. + display_kind=message.get("display_kind"), + display_metadata=message.get("display_metadata"), + ) + + _MAX_PENDING_PER_SESSION = 200 + + @staticmethod + def _is_fts_corruption_error(exc: Exception) -> bool: + """True only when the failure is provably scoped to the FTS index. + + A bare SQLITE_CORRUPT can mean structural B-tree damage; only errors + naming ``messages_fts`` or carrying FTS provenance (per + ``SessionDB._is_fts_write_corruption_error``) may authorize the + one-shot rebuild-and-retry. Everything else takes the retry path. + """ + text = str(exc).lower() + if "messages_fts" in text: + return True + import sqlite3 + + from hermes_state import SessionDB + + if isinstance(exc, sqlite3.DatabaseError): + return SessionDB._is_fts_write_corruption_error(exc) + return False + + def _rebuild_fts_once(self) -> bool: + """Attempt FTS5 ``rebuild`` once per store lifetime; True if any index was rebuilt.""" + if self._fts_rebuild_attempted: + return False + self._fts_rebuild_attempted = True + db = self._db + if db is None or not hasattr(db, "rebuild_fts"): + return False + # WAL split-brain guard: skip when a foreign process holds state.db. + if hasattr(db, "_foreign_state_db_holders"): + foreign_holders = db._foreign_state_db_holders() + if foreign_holders: + logger.warning( + "Skipping Session DB FTS rebuild while foreign processes " + "hold the database or WAL sidecars (%s); canonical " + "transcript writes remain available.", + foreign_holders, + ) + return False + try: + rebuilt = db.rebuild_fts() + except Exception as exc: + logger.warning("Session DB FTS rebuild failed: %s", exc) + return False + if rebuilt: + logger.warning( + "Rebuilt %d Session DB FTS index(es) after append corruption", + rebuilt, + ) + return rebuilt > 0 + + def _clear_dirty_transcript(self, session_id: str) -> None: + """Drop queued pending messages so a rewrite/rewind doesn't re-insert them.""" + with self._transcript_retry_lock: + self._dirty_transcripts.pop(session_id, None) + self._transcript_append_failures.pop(session_id, None) + + def has_platform_message_id( + self, session_id: str, platform_message_id: str + ) -> bool: + """Whether a message with this platform_message_id is persisted (False without a DB).""" + db = self._db_for_session_id(session_id) + if not db: + return False + try: + return db.has_platform_message_id( + session_id, platform_message_id + ) + except Exception: + logger.debug("has_platform_message_id lookup failed", exc_info=True) + return False + + def rewrite_transcript( + self, + session_id: str, + messages: List[Dict[str, Any]], + active_only: bool = False, + reject_active_turn_lease: bool = False, + ) -> bool: + """Replace a session's transcript (/retry, /compress). + + DESTRUCTIVE by default: ``active_only=False`` DELETEs every row incl. + soft-archived compaction history (pass ``active_only=True`` for sessions + that may carry archived rows). True when the write lands or there is no + DB, False on failure — callers committing a destructive change on top + (/compress repointing) must check it. ``reject_active_turn_lease`` is + for user-initiated rewrites that do not own the cross-process turn lease. + """ + db = self._db_for_session_id(session_id) + if not db: + return True + with self._get_transcript_drain_lock(): + try: + db.replace_messages( + session_id, + messages, + active_only=active_only, + reject_active_turn_lease=reject_active_turn_lease, + ) + except Exception as e: + logger.debug("Failed to rewrite transcript in DB: %s", e) + return False + self._clear_dirty_transcript(session_id) + return True + + def load_transcript(self, session_id: str) -> List[Dict[str, Any]]: + """Load all messages from a session's transcript (state.db is canonical). + + Reads follow the same routing writes use: the in-memory reroute map + (compression rotation), then the durable compression tip — otherwise + the transcript "vanishes" while every message sits under the child. + """ + from gateway.session import TranscriptReadError + if not self._db_for_session_id(session_id): + return [] + session_id = self._follow_reroutes(session_id) + try: + # Durable successor survives restart; the reroute map doesn't. + tip = self._db_for_session_id(session_id).get_compression_tip(session_id) + if tip: + session_id = tip + except Exception: + pass + try: + # repair_alternation: this feeds LIVE REPLAY; heal a durable + # user;user wedge once here instead of on every request. + return self._db_for_session_id(session_id).get_messages_as_conversation( + session_id, repair_alternation=True + ) + except Exception as e: + # Empty history is valid data; a failed canonical read is not — + # live-replay callers must fail closed, not start from []. + logger.error( + "Transcript read failed for session %s; refusing to treat the " + "conversation as empty: %s", + session_id, + e, + exc_info=True, + ) + raise TranscriptReadError(session_id) from e + + def rewind_session( + self, + session_id: str, + n: int = 1, + *, + require_retryable_composite: bool = False, + ) -> Optional[Dict[str, Any]]: + """Back up ``n`` user turns via soft-delete (``active=0``), mirroring CLI ``/undo [N]``. + + Returns ``{"rewound_count", "turns_undone", "target_text"}`` or ``None`` + (no DB / no user turn). ``n`` clamps to the oldest user turn. + ``require_retryable_composite`` is the gateway ``/retry`` guard: the + selected turn must be a composite carrier whose live payload is + losslessly replayable as text before anything changes. + """ + db = self._db_for_session_id(session_id) + if not db: + return None + with self._get_transcript_drain_lock(): + if n < 1: + n = 1 + from agent.context_compressor import ( + retryable_user_text, + split_user_originated_turn, + user_originated_turn_view, + ) + + try: + expected_active_ids = db.get_active_message_ids(session_id) + durable = db.get_messages_as_conversation( + session_id, + include_row_ids=True, + ) + user_indices = [ + index + for index, message in enumerate(durable) + if user_originated_turn_view(message) is not None + ] + if not user_indices: + return None + turns_undone = min(n, len(user_indices)) + target = durable[user_indices[-turns_undone]] + target_id = target.get("_row_id") + if not isinstance(target_id, int): + return None + handoff, target_view = split_user_originated_turn(target) + if target_view is None: + return None + if require_retryable_composite and handoff is None: + return None + except Exception as e: + logger.debug("rewind_session: failed to resolve canonical target: %s", e) + return None + if require_retryable_composite: + # Keep replay-policy failures distinct from persistence errors + # so /retry can explain why the selected carrier is unsafe. + target_text = retryable_user_text(target_view.get("content")) + try: + result = db.rewind_to_message( + session_id, + target_id, + preserve_compaction_handoff=handoff is not None, + expected_active_ids=expected_active_ids, + expected_target_content=target_view.get("content"), + ) + except ValueError as e: + logger.debug("rewind_session: %s", e) + return None + except Exception as e: + logger.debug("rewind_session: rewind_to_message failed: %s", e) + return None + self._clear_dirty_transcript(session_id) + # ``target_view`` is the live projection; a composite carrier's raw + # row holds the summary wrapper and must not be echoed as prompt. + if not require_retryable_composite: + target_text = _plain_text(target_view.get("content") or "") + return { + "rewound_count": result.get("rewound_count", 0), + "turns_undone": turns_undone, + "target_text": target_text, + } From 865d77a166be9fd728c1b50ed4e2fd5d943eb57e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:16:59 -0700 Subject: [PATCH 3/3] refactor(gateway/status): drop dead is_gateway_running/clear_planned_stop_marker, unify PID-record liveness, marker-write, JSON-parse and stale-lock helpers, table-drive runtime-status field writes --- gateway/status.py | 914 +++++++++++++++++++--------------------------- 1 file changed, 366 insertions(+), 548 deletions(-) diff --git a/gateway/status.py b/gateway/status.py index 94b1cd0335..1c669b6b38 100644 --- a/gateway/status.py +++ b/gateway/status.py @@ -5,10 +5,12 @@ send_message's check_fn to gate CLI availability). The PID file lives at ``{HERMES_HOME}/gateway.pid``, so separate homes/profiles get separate files. """ +import contextlib import copy import hashlib import json import logging +import math import os import re import shlex @@ -17,13 +19,13 @@ import subprocess import sys import threading import time -from datetime import datetime, timezone from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path -from hermes_constants import get_hermes_home, _get_platform_default_hermes_home from typing import Any, Callable, NamedTuple, Optional + +from hermes_constants import _get_platform_default_hermes_home, get_hermes_home from utils import atomic_json_write -import contextlib if sys.platform == "win32": import msvcrt @@ -35,11 +37,11 @@ _RUNTIME_STATUS_FILE = "gateway_state.json" _LOCKS_DIRNAME = "gateway-locks" _IS_WINDOWS = sys.platform == "win32" _UNSET = object() +_INVALID_JSON = object() _GATEWAY_LOCK_FILENAME = "gateway.lock" _gateway_lock_handle = None -# Windows byte-range locks are mandatory for other readers. Lock a byte well -# past the JSON payload so runtime status / PID readers can still read the file -# while another process holds the mutual-exclusion lock. +# Windows byte-range locks are mandatory for other readers: lock a byte well past +# the JSON payload so status/PID readers can read while another process holds it. _WINDOWS_LOCK_OFFSET = 1024 * 1024 _GATEWAY_RUNNING_PID_CACHE_TTL_SECONDS = 1.0 _gateway_running_pid_cache_lock = threading.Lock() @@ -72,43 +74,27 @@ def record_start_and_check_storm( try: path = _get_starts_log_path() path.parent.mkdir(parents=True, exist_ok=True) - now = datetime.now(timezone.utc).timestamp() - existing: list[float] = [] if path.exists(): for line in path.read_text(encoding="utf-8").splitlines(): - line = line.strip() - if not line: - continue try: existing.append(float(line)) except ValueError: continue - existing.append(now) - recent = [ts for ts in existing if now - ts <= window_s] # Ring-buffer the persisted file so it stays bounded. - keep = max(max_starts * 4, 40) - to_write = existing[-keep:] - + to_write = existing[-max(max_starts * 4, 40):] tmp = path.with_suffix(".tmp") - tmp.write_text( - "\n".join(repr(ts) for ts in to_write) + "\n", encoding="utf-8" - ) + tmp.write_text("\n".join(repr(ts) for ts in to_write) + "\n", encoding="utf-8") os.replace(tmp, path) - if len(recent) > max_starts: - backoff = min( - backoff_cap_s, 5.0 * (2 ** min(len(recent) - max_starts, 6)) - ) + backoff = min(backoff_cap_s, 5.0 * (2 ** min(len(recent) - max_starts, 6))) return StormInfo(count=len(recent), window_s=window_s, backoff_s=backoff) return None except Exception as _e: - logger.debug( - "respawn-storm breaker bookkeeping failed (non-fatal): %s", _e - ) + logger.debug("respawn-storm breaker bookkeeping failed (non-fatal): %s", _e) return None @@ -120,13 +106,11 @@ def _get_process_hermes_home() -> Path: per-session ``_HERMES_HOME_OVERRIDE`` contextvar and would misroute them. """ val = os.environ.get("HERMES_HOME", "").strip() - if val: - return Path(val) - return _get_platform_default_hermes_home() + return Path(val) if val else _get_platform_default_hermes_home() def _canonical_hermes_home(path: Path | str) -> Path: - """Return a stable absolute HERMES_HOME path for persisted identity data.""" + """Stable absolute HERMES_HOME path for persisted identity data.""" return Path(path).expanduser().resolve(strict=False) @@ -145,14 +129,13 @@ def recorded_gateway_home_conflicts( """True when a persisted gateway record names a DIFFERENT HERMES_HOME. Cross-profile kill refusal: a contaminated PID record in one profile's home - can truthfully name another profile's live gateway. Destructive callers must - refuse when the record proves foreign ownership, or profile B's stop/restart - SIGTERMs profile A's gateway and the supervisors enter a mutual restart loop. - - ``expected_home`` overrides the comparison base (e.g. ``profile delete`` on a - target profile). Legacy records without ``hermes_home`` return False (they - prove nothing; callers pair this with PID + start-time guards). A comparison - failure returns True: destructive action + unprovable ownership => fail closed. + can truthfully name another profile's live gateway; destructive callers must + refuse or profile B's stop/restart SIGTERMs profile A's gateway and the + supervisors enter a mutual restart loop. ``expected_home`` overrides the + comparison base (e.g. ``profile delete``). Legacy records without + ``hermes_home`` return False (they prove nothing; callers pair this with + PID + start-time guards). A comparison failure returns True: destructive + action + unprovable ownership => fail closed. """ if not isinstance(record, dict): return False @@ -166,7 +149,7 @@ def recorded_gateway_home_conflicts( return True -# Mirrors hermes_cli.profiles._PROFILE_ID_RE — duplicated here because gateway +# Mirrors hermes_cli.profiles._PROFILE_ID_RE -- duplicated because gateway # identity code must stay import-light (hermes_constants + stdlib only). _PROFILE_LABEL_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") @@ -200,16 +183,15 @@ def _profile_label_for_home(home: Path | str) -> Optional[str]: def scoped_lock_owner_label(record: Optional[dict[str, Any]]) -> Optional[str]: """Profile label for the gateway owning a (machine-global) scoped lock. - Prefers the ``profile`` field stamped by :func:`acquire_scoped_lock`, then - infers from ``hermes_home`` for older locks. None for legacy/malformed - records so callers keep PID-only wording. + Prefers the ``profile`` field stamped by :func:`acquire_scoped_lock` (validated: + it flows into log lines and a suggested CLI command), then infers from + ``hermes_home`` for older locks. None for legacy/malformed records so callers + keep PID-only wording. """ if not isinstance(record, dict): return None profile = record.get("profile") if isinstance(profile, str) and _PROFILE_LABEL_RE.match(profile.strip()): - # Validated: lock files are plain JSON and this flows into log lines - # and a suggested CLI command. return profile.strip() home = record.get("hermes_home") if isinstance(home, str) and home.strip(): @@ -218,24 +200,21 @@ def scoped_lock_owner_label(record: Optional[dict[str, Any]]) -> Optional[str]: def _get_pid_path() -> Path: - """Return the path to the gateway PID file, respecting HERMES_HOME.""" return _get_process_hermes_home() / "gateway.pid" def _get_gateway_lock_path(pid_path: Optional[Path] = None) -> Path: - """Return the path to the runtime gateway lock file.""" if pid_path is not None: return pid_path.with_name(_GATEWAY_LOCK_FILENAME) return _get_process_hermes_home() / _GATEWAY_LOCK_FILENAME def _get_runtime_status_path() -> Path: - """Return the persisted runtime health/status file path.""" return _get_pid_path().with_name(_RUNTIME_STATUS_FILE) def _get_lock_dir() -> Path: - """Return the machine-local directory for token-scoped gateway locks.""" + """Machine-local directory for token-scoped gateway locks.""" override = os.getenv("HERMES_GATEWAY_LOCK_DIR") if override: return Path(override) @@ -254,12 +233,11 @@ _EPOCH_MIN_PLAUSIBLE = 946684800.0 # 2000-01-01T00:00:00Z def normalize_updated_at(value: Any) -> Optional[str]: """Coerce a persisted ``updated_at`` value to an RFC3339 string or ``None``. - ``/api/status`` and ``/health/detailed`` promise ``string | null`` (see - ``web/src/lib/api.ts``), but the file may come from legacy gateways (epoch - floats), hand edits, or corruption. ``str``: accepted iff fromisoformat - parses it (trailing ``Z`` tolerated; naive -> UTC). ``int``/``float``: - epoch seconds; before 2000-01-01, > 1 day in the future, or non-finite -> - None. ``bool`` (an int subclass) and anything else -> None. + ``/api/status`` and ``/health/detailed`` promise ``string | null``, but the + file may come from legacy gateways (epoch floats), hand edits, or corruption. + ``str``: accepted iff fromisoformat parses it (trailing ``Z`` tolerated; naive + -> UTC). ``int``/``float``: epoch seconds; before 2000-01-01, > 1 day in the + future, or non-finite -> None. ``bool`` (an int subclass) and anything else -> None. """ if isinstance(value, bool): return None @@ -277,7 +255,7 @@ def normalize_updated_at(value: Any) -> Optional[str]: return parsed.isoformat() if isinstance(value, (int, float)): seconds = float(value) - if seconds != seconds or seconds in (float("inf"), float("-inf")): + if not math.isfinite(seconds): return None now = datetime.now(timezone.utc).timestamp() if seconds < _EPOCH_MIN_PLAUSIBLE or seconds > now + 86400: @@ -289,19 +267,13 @@ def normalize_updated_at(value: Any) -> Optional[str]: return None -def _assert_process_start_time_matches( - pid: int, expected_start_time: Optional[float] -) -> None: +def _assert_process_start_time_matches(pid: int, expected_start_time: Optional[float]) -> None: """Fail closed unless ``pid`` still names the recorded process object.""" if expected_start_time is None: - raise OSError( - f"refusing to force-kill PID {pid} without a process start-time guard" - ) + raise OSError(f"refusing to force-kill PID {pid} without a process start-time guard") current_start_time = _get_process_start_time(pid) if current_start_time is None: - raise OSError( - f"refusing to force-kill PID {pid}; process start time is unavailable" - ) + raise OSError(f"refusing to force-kill PID {pid}; process start time is unavailable") try: expected = float(expected_start_time) current = float(current_start_time) @@ -342,14 +314,11 @@ def terminate_pid( except FileNotFoundError: os.kill(pid, signal.SIGTERM) return - if result.returncode != 0: details = (result.stderr or result.stdout or "").strip() raise OSError(details or f"taskkill failed for PID {pid}") return - - sig = signal.SIGTERM if not force else getattr(signal, "SIGKILL", signal.SIGTERM) - os.kill(pid, sig) + os.kill(pid, signal.SIGTERM if not force else getattr(signal, "SIGKILL", signal.SIGTERM)) def _scope_hash(identity: str) -> str: @@ -364,13 +333,12 @@ def _get_process_start_time(pid: int) -> Optional[int]: """Stable per-process start-time fingerprint (PID-reuse guard), or None. Linux: field 22 of ``/proc//stat`` (clock ticks since boot). Without - ``/proc`` (macOS/Windows): psutil ``create_time()`` quantized to - centiseconds for stable equality. Units differ per platform but the guard - only ever compares same-host, same-source values. + ``/proc`` (macOS/Windows): psutil ``create_time()`` quantized to centiseconds + for stable equality. Units differ per platform but the guard only ever + compares same-host, same-source values. """ - stat_path = Path(f"/proc/{pid}/stat") try: - return int(stat_path.read_text(encoding="utf-8").split()[21]) + return int(Path(f"/proc/{pid}/stat").read_text(encoding="utf-8").split()[21]) except (FileNotFoundError, IndexError, PermissionError, ValueError, OSError): pass try: @@ -387,15 +355,13 @@ def get_process_start_time(pid: int) -> Optional[int]: def _read_process_cmdline(pid: int) -> Optional[str]: """Process command line as one string: /proc, then ``ps``, then psutil (Windows).""" - cmdline_path = Path(f"/proc/{pid}/cmdline") try: - raw = cmdline_path.read_bytes() + raw = Path(f"/proc/{pid}/cmdline").read_bytes() except (FileNotFoundError, PermissionError, OSError): pass else: if raw: return raw.replace(b"\x00", b" ").decode("utf-8", errors="ignore").strip() - if not _IS_WINDOWS: try: result = subprocess.run( @@ -408,34 +374,30 @@ def _read_process_cmdline(pid: int) -> Optional[str]: return result.stdout.strip() except (OSError, subprocess.TimeoutExpired): pass - try: import psutil # type: ignore - proc = psutil.Process(pid) - cmdline_parts = proc.cmdline() + cmdline_parts = psutil.Process(pid).cmdline() if cmdline_parts: return " ".join(cmdline_parts) except Exception: pass - return None def _gateway_command_subcommand(command: str | None) -> str | None: - """Return the Hermes gateway lifecycle subcommand from a command line. + """Hermes gateway lifecycle subcommand from a command line, or None. - Lifecycle decisions must not fire on loose substring matches: ``"gateway" - in cmdline`` also matched ``gateway status`` and ``python -m tui_gateway``, - making ``restart()`` race a draining process and ``status`` report false - positives. Requires a Hermes entrypoint plus the ``gateway`` subcommand (or - a gateway-dedicated entrypoint). Tokenizes quote-aware so Windows paths with + Lifecycle decisions must not fire on loose substring matches: ``"gateway" in + cmdline`` also matched ``gateway status`` and ``python -m tui_gateway``, making + ``restart()`` race a draining process and ``status`` report false positives. + Requires a Hermes entrypoint plus the ``gateway`` subcommand (or a + gateway-dedicated entrypoint). Tokenizes quote-aware so Windows paths with spaces survive, and strips ``--profile``/``-p`` selectors from anywhere in - argv -- ``_apply_profile_override`` removes them before argparse, so they - (and a profile literally named ``gateway``) can appear on either side. + argv -- ``_apply_profile_override`` removes them before argparse, so they (and + a profile literally named ``gateway``) can appear on either side. """ if not command: return None - try: raw_tokens = shlex.split(command, posix=False) except ValueError: @@ -444,14 +406,12 @@ def _gateway_command_subcommand(command: str | None) -> str | None: tokens = [t.strip("\"'").replace("\\", "/").lower() for t in raw_tokens] if not tokens: return None - # Gateway-dedicated entrypoints carry no subcommand to inspect. for token in tokens: if token == "gateway/run.py" or token.endswith("/gateway/run.py"): return "run" if token.rsplit("/", 1)[-1] in ("hermes-gateway", "hermes-gateway.exe"): return "run" - joined = " ".join(tokens) has_gateway_entry = ( "hermes_cli.main" in joined @@ -460,7 +420,6 @@ def _gateway_command_subcommand(command: str | None) -> str | None: ) if not has_gateway_entry: return None - # Drop --profile X / -p X / --profile=X / -p=X (consumes a VALUE of "gateway" too). filtered: list[str] = [] skip_next = False @@ -474,7 +433,6 @@ def _gateway_command_subcommand(command: str | None) -> str | None: if token.startswith("--profile=") or token.startswith("-p="): continue filtered.append(token) - for i, token in enumerate(filtered): if token != "gateway": continue @@ -485,7 +443,7 @@ def _gateway_command_subcommand(command: str | None) -> str | None: def looks_like_gateway_command_line(command: str | None) -> bool: - """Return True only for a real ``gateway run`` process command line.""" + """True only for a real ``gateway run`` process command line.""" return _gateway_command_subcommand(command) == "run" @@ -493,15 +451,15 @@ def looks_like_gateway_runtime_command_line(command: str | None) -> bool: """True for command lines that can host the gateway runtime (``run`` or ``restart``). Without a service manager the manual restart fallback runs ``run_gateway()`` - in-process, so argv stays ``gateway restart`` while it owns the runtime. - Use only for validating Hermes-owned records / no-supervisor cleanup scans; + in-process, so argv stays ``gateway restart`` while it owns the runtime. Use + only for validating Hermes-owned records / no-supervisor cleanup scans; ``looks_like_gateway_command_line()`` stays strict. """ return _gateway_command_subcommand(command) in {"run", "restart"} def _looks_like_gateway_process(pid: int) -> bool: - """Return True when the live PID still looks like the Hermes gateway.""" + """True when the live PID still looks like the Hermes gateway.""" cmdline = _read_process_cmdline(pid) return bool(cmdline) and looks_like_gateway_command_line(cmdline) @@ -518,9 +476,7 @@ def _record_looks_like_gateway(record: dict[str, Any]) -> bool: def _profile_name_for_home(profile_home: Path) -> Optional[str]: """Profile id for ``/profiles/``; None for the root/default home (bare gateway).""" - if profile_home.parent.name == "profiles": - return profile_home.name - return None + return profile_home.name if profile_home.parent.name == "profiles" else None def _command_line_belongs_to_profile(command: str, profile_home: Path) -> bool: @@ -529,15 +485,13 @@ def _command_line_belongs_to_profile(command: str, profile_home: Path) -> bool: Mirrors ``hermes_cli.gateway._matches_current_profile``: a stale state file can record a PID the OS recycled onto a DIFFERENT profile's live gateway, which still looks like a gateway -- so the dead profile would read running. - Named profiles carry ``-p``/``--profile `` (or explicit - ``HERMES_HOME=``) on argv; the default gateway runs bare. + Named profiles carry ``-p``/``--profile `` (or explicit ``HERMES_HOME=``) + on argv; the default gateway runs bare. Separators are normalized: Windows + str(Path) uses backslashes while an argv HERMES_HOME= may use forward slashes. """ - # Normalize separators: Windows str(Path) uses backslashes while an argv - # HERMES_HOME= value may use forward slashes (Git Bash, JSON) -- or vice versa. command_lc = command.lower().replace("\\", "/") profile_name = _profile_name_for_home(profile_home) home_lc = str(profile_home).lower().replace("\\", "/") - if profile_name is not None and profile_name != "default": profile_lc = profile_name.lower() return ( @@ -545,10 +499,9 @@ def _command_line_belongs_to_profile(command: str, profile_home: Path) -> bool: or f"-p {profile_lc}" in command_lc or f"hermes_home={home_lc}" in command_lc ) - - # Default profile: accept unless argv names some other profile or a - # conflicting explicit HERMES_HOME= (its absence is not disqualifying -- - # HERMES_HOME usually arrives via the environment). + # Default profile: accept unless argv names some other profile or a conflicting + # explicit HERMES_HOME= (its absence is not disqualifying -- HERMES_HOME + # usually arrives via the environment). if "--profile " in command_lc or " -p " in command_lc: return False return not ("hermes_home=" in command_lc and f"hermes_home={home_lc}" not in command_lc) @@ -563,9 +516,9 @@ def _record_matches_live_gateway_pid( """True when a live PID still identifies as this gateway record. Prefer the live command line: a stale record's argv must not make an - unrelated process (PID reuse) count as a gateway. With ``expected_home`` - the live command line must also belong to that profile. When the command - line is unreadable (Windows/permission), fall back to the persisted record. + unrelated process (PID reuse) count as a gateway. With ``expected_home`` the + live command line must also belong to that profile. When the command line is + unreadable (Windows/permission), fall back to the persisted record. """ live_cmdline = _read_process_cmdline(pid) if live_cmdline: @@ -598,21 +551,16 @@ def _get_code_identity_fields() -> dict[str, Any]: from hermes_cli.build_info import get_code_identity identity = get_code_identity() - return { - "code_sha": identity.get("sha"), - "code_version": identity.get("version"), - } + return {"code_sha": identity.get("sha"), "code_version": identity.get("version")} except Exception: return {} -def _pid_record_belongs_to_current_profile( - record: Optional[dict[str, Any]], -) -> bool: +def _pid_record_belongs_to_current_profile(record: Optional[dict[str, Any]]) -> bool: """True when the record's ``hermes_home`` matches the current process. - A record written under a different HERMES_HOME belongs to another profile - and must be ignored, or the default gateway assumes that profile's identity. + A record written under a different HERMES_HOME belongs to another profile and + must be ignored, or the default gateway assumes that profile's identity. Legacy records without the field are accepted conservatively. """ if not isinstance(record, dict): @@ -653,14 +601,18 @@ def _read_text_file(path: Path) -> Optional[str]: return raw or None -def _read_json_file(path: Path) -> Optional[dict[str, Any]]: - raw = _read_text_file(path) +def _parse_json(raw: Optional[str]) -> Any: + """``json.loads`` with ``_INVALID_JSON`` for missing/undecodable text.""" if raw is None: - return None + return _INVALID_JSON try: - payload = json.loads(raw) + return json.loads(raw) except json.JSONDecodeError: - return None + return _INVALID_JSON + + +def _read_json_file(path: Path) -> Optional[dict[str, Any]]: + payload = _parse_json(_read_text_file(path)) return payload if isinstance(payload, dict) else None @@ -676,20 +628,17 @@ def _unlink_quietly(path: Path) -> None: def _read_pid_record(pid_path: Optional[Path] = None) -> Optional[dict]: """PID record as a dict; legacy bare-integer files become ``{"pid": N}``.""" raw = _read_text_file(pid_path or _get_pid_path()) - if raw is None: - return None - try: - payload = json.loads(raw) - except json.JSONDecodeError: + payload = _parse_json(raw) + if payload is _INVALID_JSON: + if raw is None: + return None try: return {"pid": int(raw)} except ValueError: return None if isinstance(payload, int): return {"pid": payload} - if isinstance(payload, dict): - return payload - return None + return payload if isinstance(payload, dict) else None def _read_gateway_lock_record(lock_path: Optional[Path] = None) -> Optional[dict[str, Any]]: @@ -705,13 +654,23 @@ def _pid_from_record(record: Optional[dict[str, Any]]) -> Optional[int]: return None -def _start_time_conflicts(record: dict[str, Any], pid: int) -> bool: +def _start_times_conflict(recorded_start: Any, current_start: Any) -> bool: """PID-reuse guard: True only when BOTH start times are known and differ.""" - recorded_start = record.get("start_time") - current_start = _get_process_start_time(pid) return recorded_start is not None and current_start is not None and current_start != recorded_start +def _start_time_conflicts(record: dict[str, Any], pid: int) -> bool: + return _start_times_conflict(record.get("start_time"), _get_process_start_time(pid)) + + +def _live_pid_from_record(record: Optional[dict[str, Any]]) -> Optional[int]: + """Record's PID when it is alive and passes the start-time PID-reuse guard, else None.""" + pid = _pid_from_record(record) + if pid is None or not _pid_exists(pid) or _start_time_conflicts(record or {}, pid): + return None + return pid + + def _clear_running_pid_cache() -> None: with _gateway_running_pid_cache_lock: _gateway_running_pid_cache.clear() @@ -725,11 +684,7 @@ def _file_cache_signature(path: Path) -> tuple[bool, Optional[int], Optional[int return (True, st.st_mtime_ns, st.st_size) -def _running_pid_cache_signature( - pid_path: Path, - *, - include_runtime_status: bool, -) -> tuple[Any, ...]: +def _running_pid_cache_signature(pid_path: Path, *, include_runtime_status: bool) -> tuple[Any, ...]: parts: list[Any] = [ _file_cache_signature(pid_path), _file_cache_signature(_get_gateway_lock_path(pid_path)), @@ -783,19 +738,18 @@ def _pid_exists(pid: int) -> bool: """Cross-platform "is this PID alive" check that does NOT kill the target. CRITICAL on Windows: ``os.kill(pid, 0)`` is NOT a no-op -- CPython maps - ``sig=0`` to ``CTRL_C_EVENT`` and ``GenerateConsoleCtrlEvent`` sends Ctrl+C - to the target's whole console group (bpo-14484). Prefer psutil; fall back - to ctypes ``OpenProcess``/``WaitForSingleObject`` on Windows and - ``os.kill(pid, 0)`` on POSIX when psutil is unavailable (stripped install / - scaffold phase). + ``sig=0`` to ``CTRL_C_EVENT`` and sends Ctrl+C to the target's whole console + group (bpo-14484). Prefer psutil; fall back to ctypes + ``OpenProcess``/``WaitForSingleObject`` on Windows and ``os.kill(pid, 0)`` on + POSIX when psutil is unavailable (stripped install / scaffold phase). + Zombies are reported dead: they sit in the process table but treating one as + alive makes --replace wait forever under systemd Restart=always, which + respawns before reaping. """ try: import psutil # type: ignore - # Zombies are still in the process table (pid_exists() -> True) but are - # dead: treating one as alive makes --replace wait forever under systemd - # Restart=always, which respawns before reaping. Report them as dead. - # Best-effort: status-read failures fall through to pid_exists(). + # Best-effort zombie check: status-read failures fall through to pid_exists(). try: if psutil.Process(int(pid)).status() == psutil.STATUS_ZOMBIE: return False @@ -804,7 +758,6 @@ def _pid_exists(pid: int) -> bool: except Exception: pass return bool(psutil.pid_exists(int(pid))) - except ImportError: pass # Fall through to stdlib fallback. if _IS_WINDOWS: @@ -827,9 +780,9 @@ def _pid_exists(pid: int) -> bool: err = kernel32.GetLastError() if err == ERROR_INVALID_PARAMETER: return False # PID definitely gone - if err == ERROR_ACCESS_DENIED: - return True # Exists but owned by another user/session - return False # Conservative default for unknown errors + # ACCESS_DENIED: exists but owned by another user/session; any + # other error: conservative default False. + return err == ERROR_ACCESS_DENIED try: # WAIT_TIMEOUT = still running; anything else = gone. return kernel32.WaitForSingleObject(handle, 0) == WAIT_TIMEOUT @@ -837,37 +790,34 @@ def _pid_exists(pid: int) -> bool: kernel32.CloseHandle(handle) except (OSError, AttributeError): return False - else: - # Same zombie case as the psutil path: a zombie answers os.kill(pid, 0). + # POSIX: a zombie answers os.kill(pid, 0), so check its state first. + try: + stat_fields = Path(f"/proc/{int(pid)}/stat").read_text(encoding="utf-8").split() + if len(stat_fields) > 2 and stat_fields[2] == "Z": + return False + except FileNotFoundError: # No /proc (macOS/BSD): use ps state. try: - stat_fields = ( - Path(f"/proc/{int(pid)}/stat").read_text(encoding="utf-8").split() + r = subprocess.run( + ["ps", "-o", "state=", "-p", str(int(pid))], + capture_output=True, + text=True, encoding='utf-8', errors='replace', + timeout=5, ) - if len(stat_fields) > 2 and stat_fields[2] == "Z": + if r.returncode == 0 and r.stdout.strip().startswith("Z"): return False - except FileNotFoundError: # No /proc (macOS/BSD): use ps state. - try: - r = subprocess.run( - ["ps", "-o", "state=", "-p", str(int(pid))], - capture_output=True, - text=True, encoding='utf-8', errors='replace', - timeout=5, - ) - if r.returncode == 0 and r.stdout.strip().startswith("Z"): - return False - except Exception: - pass - except (IndexError, PermissionError, OSError): + except Exception: pass - try: - os.kill(int(pid), 0) # windows-footgun: ok — POSIX-only branch (the whole point of _pid_exists) - return True - except ProcessLookupError: - return False - except PermissionError: - return True # Exists but we can't signal it. - except OSError: - return False + except (IndexError, PermissionError, OSError): + pass + try: + os.kill(int(pid), 0) # windows-footgun: ok — POSIX-only branch (the whole point of _pid_exists) + return True + except ProcessLookupError: + return False + except PermissionError: + return True # Exists but we can't signal it. + except OSError: + return False def _lock_is_held(handle) -> bool: @@ -894,7 +844,6 @@ def acquire_gateway_runtime_lock() -> bool: global _gateway_lock_handle if _gateway_lock_handle is not None: return True - path = _get_gateway_lock_path() path.parent.mkdir(parents=True, exist_ok=True) try: @@ -904,9 +853,6 @@ def acquire_gateway_runtime_lock() -> bool: # root. The directory owner can unlink it; retry once with a fresh file. try: path.unlink() - except OSError: - return False - try: handle = open(path, "a+", encoding="utf-8") except OSError: return False @@ -935,28 +881,25 @@ def release_gateway_runtime_lock() -> None: def owns_gateway_runtime_lock() -> bool: """True when THIS process holds the runtime lock. - ``is_gateway_runtime_lock_active`` answers "does anyone hold it?" and a - file re-probe cannot tell self-ownership apart (a probe on our own flock - succeeds on POSIX); the in-process handle is the only discriminator. + ``is_gateway_runtime_lock_active`` answers "does anyone hold it?" and a file + re-probe cannot tell self-ownership apart (a probe on our own flock succeeds + on POSIX); the in-process handle is the only discriminator. """ return _gateway_lock_handle is not None def is_gateway_runtime_lock_active(lock_path: Optional[Path] = None) -> bool: - """Return True when some process currently owns the gateway runtime lock.""" - global _gateway_lock_handle + """True when some process currently owns the gateway runtime lock.""" resolved_lock_path = lock_path or _get_gateway_lock_path() if _gateway_lock_handle is not None and resolved_lock_path == _get_gateway_lock_path(): return True - if not resolved_lock_path.exists(): return False - try: handle = open(resolved_lock_path, "a+", encoding="utf-8") except PermissionError: - # Stale root-owned lock (launchd session that ran as root): the - # directory owner can unlink it; report inactive so a fresh one is made. + # Stale root-owned lock (launchd session that ran as root): the directory + # owner can unlink it; report inactive so a fresh one is made. with contextlib.suppress(OSError): resolved_lock_path.unlink() return False @@ -1010,6 +953,22 @@ def write_pid_file() -> None: raise +def _apply_set_fields(target: dict[str, Any], fields) -> None: + """Assign each ``(key, value, coerce)`` whose value was explicitly passed (not ``_UNSET``).""" + for key, value, coerce in fields: + if value is not _UNSET: + target[key] = coerce(value) if coerce is not None else value + + +def _coerce_session_store(session_store: Any) -> dict[str, str]: + state = "unknown" + if isinstance(session_store, dict): + candidate = str(session_store.get("status") or "unknown") + if candidate in {"ok", "unavailable", "retrying", "unknown"}: + state = candidate + return {"status": state} + + def write_runtime_status( *, gateway_state: Any = _UNSET, @@ -1034,15 +993,13 @@ def write_runtime_status( payload.setdefault("platforms", {}) if clear_profile_platforms: # Secondary-profile entries are keyed ``:``. A fresh - # process must not inherit them or /api/status stays degraded until - # every old adapter re-emits (removed profiles: forever). + # process must not inherit them or /api/status stays degraded until every + # old adapter re-emits (removed profiles: forever). platforms = payload["platforms"] if not isinstance(platforms, dict): platforms = {} payload["platforms"] = { - key: value - for key, value in platforms.items() - if not isinstance(key, str) or ":" not in key + key: value for key, value in platforms.items() if not isinstance(key, str) or ":" not in key } for key in ("kind", "pid", "argv", "start_time"): payload[key] = current_record[key] @@ -1051,46 +1008,34 @@ def write_runtime_status( # record must describe the CURRENT writer's code. payload.update(_get_code_identity_fields()) - for key, value in (("gateway_state", gateway_state), ("exit_reason", exit_reason)): - if value is not _UNSET: - payload[key] = value - if restart_requested is not _UNSET: - payload["restart_requested"] = bool(restart_requested) - if active_agents is not _UNSET: - payload["active_agents"] = parse_active_agents(active_agents) - if served_profiles is not _UNSET: + _apply_set_fields(payload, ( + ("gateway_state", gateway_state, None), + ("exit_reason", exit_reason, None), + ("restart_requested", restart_requested, bool), + ("active_agents", active_agents, parse_active_agents), # Multiplexed profiles; absent/empty for a single-profile gateway. - payload["served_profiles"] = list(served_profiles or []) - if session_store is not _UNSET: - state = "unknown" - if isinstance(session_store, dict): - candidate = str(session_store.get("status") or "unknown") - if candidate in {"ok", "unavailable", "retrying", "unknown"}: - state = candidate - payload["session_store"] = {"status": state} + ("served_profiles", served_profiles, lambda v: list(v or [])), + ("session_store", session_store, _coerce_session_store), + )) if platform is not _UNSET: platform_payload = payload["platforms"].get(platform, {}) - for key, value in ( - ("state", platform_state), - ("error_code", error_code), - ("error_message", error_message), - ): - if value is not _UNSET: - platform_payload[key] = value - if needs_attention is not _UNSET: - # Reconnect-loop escalation past the attention threshold: a signal - # for owners/fleet monitoring, not a circuit breaker (retry never - # stops). Cleared on successful reconnect. - platform_payload["needs_attention"] = bool(needs_attention) - if retrying_since is not _UNSET: + _apply_set_fields(platform_payload, ( + ("state", platform_state, None), + ("error_code", error_code, None), + ("error_message", error_message, None), + # Reconnect-loop escalation past the attention threshold: a signal for + # owners/fleet monitoring, not a circuit breaker (retry never stops). + # Cleared on successful reconnect. + ("needs_attention", needs_attention, bool), # ISO start of the current retry episode; None clears it. - platform_payload["retrying_since"] = retrying_since + ("retrying_since", retrying_since, None), + )) platform_payload["updated_at"] = _utc_now_iso() - # Per-entry writer provenance: top-level pid/start_time only identify - # the most recent writer, so /api/status distinguishes "written by the - # live process" from "preserved from a prior one" by exact - # (pid, start_time) equality rather than clock heuristics. + # Per-entry writer provenance: top-level pid/start_time only identify the + # most recent writer, so /api/status distinguishes "written by the live + # process" from "preserved from a prior one" by exact (pid, start_time) + # equality rather than clock heuristics. platform_payload["writer_pid"] = current_record["pid"] platform_payload["writer_start_time"] = current_record["start_time"] payload["platforms"][platform] = platform_payload @@ -1126,10 +1071,7 @@ def runtime_status_is_stale( def runtime_status_pid_is_live(record: Optional[dict[str, Any]]) -> bool: """True when the snapshot's PID is alive and passes the start-time PID-reuse guard.""" - pid = _pid_from_record(record) - if pid is None or not _pid_exists(pid): - return False - return not _start_time_conflicts(record, pid) + return _live_pid_from_record(record) is not None def parse_active_agents(raw: Any) -> int: @@ -1144,14 +1086,12 @@ def parse_active_agents(raw: Any) -> int: _DRAINABLE_GATEWAY_STATES = frozenset({"running"}) -def derive_gateway_busy( - *, gateway_running: bool, gateway_state: Any, active_agents: Any -) -> bool: +def derive_gateway_busy(*, gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool: """Busy iff live, in ``running`` state, and ``active_agents > 0`` (the contract NAS gates on). Degrades to False on unknown liveness / other state / unparseable count. - Liveness keys off ``gateway_running``, NEVER ``updated_at`` -- a healthy - idle gateway never advances that timestamp. + Liveness keys off ``gateway_running``, NEVER ``updated_at`` -- a healthy idle + gateway never advances that timestamp. """ if not gateway_running or gateway_state not in _DRAINABLE_GATEWAY_STATES: return False @@ -1171,9 +1111,8 @@ class GatewayLiveness: """Resolved gateway liveness for one dashboard surface. ``source``: which ladder rung answered (logging/tests only -- never branch - product behavior on it). ``probe_error``: a rung raised instead of - answering; lets fail-open callers (kanban dispatcher warning) tell "down" - from "could not tell". + product behavior on it). ``probe_error``: a rung raised instead of answering; + lets fail-open callers (kanban dispatcher warning) tell "down" from "could not tell". """ running: bool @@ -1204,78 +1143,52 @@ def resolve_gateway_liveness( Rung 3 only runs against the LOCAL state record -- the probe body's PID belongs to another host. Pass ``runtime`` if the state file is already read. ``pid_probe``/``runtime_reader``/``runtime_pid_probe`` let the dashboard - inject its ``hermes_cli.web_server`` bindings (test monkeypatch seam). + inject its ``hermes_cli.web_server`` bindings (test monkeypatch seam). A rung + that raises degrades to the next one (never 500 a status endpoint) and sets + ``probe_error``. """ - _pid_probe = pid_probe or ( - get_running_pid_cached if use_cache else get_running_pid - ) + _pid_probe = pid_probe or (get_running_pid_cached if use_cache else get_running_pid) _runtime_reader = runtime_reader or read_runtime_status _runtime_pid_probe = runtime_pid_probe or get_runtime_status_running_pid - - pid_path = (profile_dir / "gateway.pid") if profile_dir is not None else None probe_error = False - try: - # Zero-arg call when unscoped: callers monkeypatch with zero-arg lambdas - # and /api/status's cache signature is keyed on the call shape. - pid = _pid_probe(pid_path) if pid_path is not None else _pid_probe() - except Exception: - # Degrade to the next rung; never 500 a status endpoint. - pid = None - probe_error = True + + def guarded(fn, *args, fallback=None, **kwargs): + nonlocal probe_error + try: + return fn(*args, **kwargs) + except Exception: + probe_error = True + return fallback + + # Zero-arg call when unscoped: callers monkeypatch with zero-arg lambdas and + # /api/status's cache signature is keyed on the call shape. + if profile_dir is not None: + pid = guarded(_pid_probe, profile_dir / "gateway.pid") + else: + pid = guarded(_pid_probe) if pid is not None: return GatewayLiveness(running=True, pid=pid, source="pid") health_body: Optional[dict[str, Any]] = None if health_probe is not None: - try: - alive, health_body = health_probe() - except Exception: - alive, health_body = False, None - probe_error = True + alive, health_body = guarded(health_probe, fallback=(False, None)) if alive: # Display-only PID: it belongs to the remote container. remote_pid = health_body.get("pid") if health_body else None - return GatewayLiveness( - running=True, - pid=remote_pid, - source="health", - health_body=health_body, - ) + return GatewayLiveness(running=True, pid=remote_pid, source="health", health_body=health_body) if runtime is _UNSET: - try: - runtime = ( - _runtime_reader(path=profile_dir / "gateway_state.json") - if profile_dir is not None - else _runtime_reader() - ) - except Exception: - runtime = None - probe_error = True - try: - runtime_pid = ( - _runtime_pid_probe(runtime, expected_home=profile_dir) - if profile_dir is not None - else _runtime_pid_probe(runtime) - ) - except Exception: - runtime_pid = None - probe_error = True + if profile_dir is not None: + runtime = guarded(_runtime_reader, path=profile_dir / "gateway_state.json") + else: + runtime = guarded(_runtime_reader) + if profile_dir is not None: + runtime_pid = guarded(_runtime_pid_probe, runtime, expected_home=profile_dir) + else: + runtime_pid = guarded(_runtime_pid_probe, runtime) if runtime_pid is not None: - return GatewayLiveness( - running=True, - pid=runtime_pid, - source="runtime_status", - health_body=health_body, - ) - - return GatewayLiveness( - running=False, - pid=None, - source="none", - health_body=health_body, - probe_error=probe_error, - ) + return GatewayLiveness(running=True, pid=runtime_pid, source="runtime_status", health_body=health_body) + return GatewayLiveness(running=False, pid=None, source="none", health_body=health_body, probe_error=probe_error) def get_runtime_status_running_pid( @@ -1288,24 +1201,21 @@ def get_runtime_status_running_pid( Conservative fallback to ``get_running_pid()`` for launch-service-managed gateways that have a fresh ``gateway_state.json`` but no ``gateway.pid``. ``expected_home`` scopes the OS-identity check to another profile's home - (dashboard enumeration) so a PID recycled onto a different profile's - gateway is not reported running for the dead one. + (dashboard enumeration) so a PID recycled onto a different profile's gateway + is not reported running for the dead one. """ payload = runtime if runtime is not None else read_runtime_status() if not isinstance(payload, dict): return None if payload.get("gateway_state") in {None, "stopped", "startup_failed"}: return None - - pid = _pid_from_record(payload) - if pid is None or not _pid_exists(pid) or _start_time_conflicts(payload, pid): + pid = _live_pid_from_record(payload) + if pid is None: return None - # Active-profile context: the record's hermes_home must match this process # so a stale record cannot lend another profile's identity. if expected_home is None and not _pid_record_belongs_to_current_profile(payload): return None - if _record_matches_live_gateway_pid(payload, pid, expected_home=expected_home): return pid return None @@ -1330,6 +1240,47 @@ def remove_pid_file() -> None: pass +def _process_is_stopped(pid: int) -> bool: + """True when ``/proc//status`` reports a stopped / tracing-stop state (T/t).""" + try: + proc_status = Path(f"/proc/{pid}/status") + if proc_status.exists(): + for line in proc_status.read_text(encoding="utf-8").splitlines(): + if line.startswith("State:"): + return line.split()[1] in {"T", "t"} + except (OSError, PermissionError): + pass + return False + + +def _scoped_lock_record_is_stale(existing: dict[str, Any], existing_pid: Optional[int]) -> bool: + """True when a foreign scoped-lock record no longer names a live gateway. + + Stale when the PID is missing/dead, its start time changed (PID reuse), or the + live process is not a gateway: its cmdline is readable and says so (also + catches boot-time PID+start_time collisions -- systemd spawns deterministically, + so an unrelated service can land on the same PID and jiffy count). When the + cmdline is unreadable (Windows has no ps) AND start_time is unavailable on + either side (no /proc, psutil failure), consult the lock record's own argv -- + the only identity signal there. Stopped processes (Ctrl+Z / SIGTSTP) look + alive to _pid_exists but are not running; treat as stale so --replace works. + """ + if existing_pid is None or not _pid_exists(existing_pid): + return True + current_start = _get_process_start_time(existing_pid) + if _start_times_conflict(existing.get("start_time"), current_start): + return True + if not _looks_like_gateway_process(existing_pid) and ( + _read_process_cmdline(existing_pid) is not None + or ( + (existing.get("start_time") is None or current_start is None) + and not _record_looks_like_gateway(existing) + ) + ): + return True + return _process_is_stopped(existing_pid) + + def acquire_scoped_lock(scope: str, identity: str, metadata: Optional[dict[str, Any]] = None) -> tuple[bool, Optional[dict[str, Any]]]: """Acquire a machine-local lock keyed by scope + identity (e.g. one Telegram token across homes).""" lock_path = _get_scope_lock_path(scope, identity) @@ -1341,9 +1292,9 @@ def acquire_scoped_lock(scope: str, identity: str, metadata: Optional[dict[str, "metadata": metadata or {}, "updated_at": _utc_now_iso(), } - # Profile label for cross-profile conflict diagnostics ("token already in - # use (PID 559)" alone does not say WHICH profile). Omitted when not - # inferable; readers fall back to hermes_home. + # Profile label for cross-profile conflict diagnostics ("token already in use + # (PID 559)" alone does not say WHICH profile). Omitted when not inferable; + # readers fall back to hermes_home. profile = _profile_label_for_home(_get_process_hermes_home()) if profile: record["profile"] = profile @@ -1355,71 +1306,24 @@ def acquire_scoped_lock(scope: str, identity: str, metadata: Optional[dict[str, _unlink_quietly(lock_path) if existing: existing_pid = _pid_from_record(existing) - # Our own PID: always self-reacquire. start_time guards reuse of OTHER # PIDs; requiring equality here rejects reconnects when the on-disk # record has start_time null (older writers / psutil failure). if existing_pid == os.getpid(): _write_json_file(lock_path, record) return True, existing - - stale = existing_pid is None - if not stale: - if not _pid_exists(existing_pid): - stale = True - else: - current_start = _get_process_start_time(existing_pid) - if ( - existing.get("start_time") is not None - and current_start is not None - and current_start != existing.get("start_time") - ): - stale = True - # Live process is not a gateway: stale when its cmdline is - # readable and says so (also catches boot-time PID+start_time - # collisions -- systemd spawns deterministically, so an unrelated - # service can land on the same PID and jiffy count). When the - # cmdline is unreadable (Windows has no ps) AND start_time is - # unavailable on either side (no /proc, psutil failure), consult - # the lock record's own argv -- the only identity signal there. - if ( - not stale - and not _looks_like_gateway_process(existing_pid) - and ( - _read_process_cmdline(existing_pid) is not None - or ( - (existing.get("start_time") is None or current_start is None) - and not _record_looks_like_gateway(existing) - ) - ) - ): - stale = True - # Stopped processes (Ctrl+Z / SIGTSTP) look alive to _pid_exists - # but are not running; treat as stale so --replace works. - if not stale: - try: - _proc_status = Path(f"/proc/{existing_pid}/status") - if _proc_status.exists(): - for _line in _proc_status.read_text(encoding="utf-8").splitlines(): - if _line.startswith("State:"): - if _line.split()[1] in {"T", "t"}: # stopped / tracing stop - stale = True - break - except (OSError, PermissionError): - pass - if stale: - # Rename to a tombstone instead of unlink(): with unlink()+O_EXCL two - # racing starters could both win (the second unlink deleting the - # first racer's fresh lock). os.replace() lets exactly one claim it. - tombstone = lock_path.with_name(lock_path.name + ".stale") - try: - os.replace(lock_path, tombstone) - except OSError: - pass # Another racer claimed it -- O_EXCL below decides the winner. - else: - _unlink_quietly(tombstone) - else: + if not _scoped_lock_record_is_stale(existing, existing_pid): return False, existing + # Rename to a tombstone instead of unlink(): with unlink()+O_EXCL two + # racing starters could both win (the second unlink deleting the first + # racer's fresh lock). os.replace() lets exactly one claim it. + tombstone = lock_path.with_name(lock_path.name + ".stale") + try: + os.replace(lock_path, tombstone) + except OSError: + pass # Another racer claimed it -- O_EXCL below decides the winner. + else: + _unlink_quietly(tombstone) try: fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) @@ -1435,16 +1339,15 @@ def acquire_scoped_lock(scope: str, identity: str, metadata: Optional[dict[str, def release_scoped_lock(scope: str, identity: str) -> None: - """Release a previously-acquired scope lock when owned by this process.""" + """Release a previously-acquired scope lock when owned by this process. + + Own PID => we own the lock. No start_time equality: on-disk null vs a live + fingerprint would leave the lock stuck across reconnects. + """ lock_path = _get_scope_lock_path(scope, identity) existing = _read_json_file(lock_path) - if not existing: - return - if existing.get("pid") != os.getpid(): - return - # Own PID => we own the lock. No start_time equality: on-disk null vs a live - # fingerprint would leave the lock stuck across reconnects. - _unlink_quietly(lock_path) + if existing and existing.get("pid") == os.getpid(): + _unlink_quietly(lock_path) def release_all_scoped_locks( @@ -1459,24 +1362,20 @@ def release_all_scoped_locks( """ lock_dir = _get_lock_dir() removed = 0 - if lock_dir.exists(): - for lock_file in lock_dir.glob("*.lock"): - if owner_pid is not None: - record = _read_json_file(lock_file) - if not isinstance(record, dict): - continue - if _pid_from_record(record) != owner_pid: - continue - if ( - owner_start_time is not None - and record.get("start_time") != owner_start_time - ): - continue - try: - lock_file.unlink(missing_ok=True) - removed += 1 - except OSError: - pass + if not lock_dir.exists(): + return 0 + for lock_file in lock_dir.glob("*.lock"): + if owner_pid is not None: + record = _read_json_file(lock_file) + if not isinstance(record, dict) or _pid_from_record(record) != owner_pid: + continue + if owner_start_time is not None and record.get("start_time") != owner_start_time: + continue + try: + lock_file.unlink(missing_ok=True) + removed += 1 + except OSError: + pass return removed @@ -1503,7 +1402,6 @@ def _get_takeover_marker_path(hermes_home: Optional[Path] = None) -> Path: def _get_planned_stop_marker_path() -> Path: - """Return the path to the intentional gateway stop marker file.""" return _get_process_hermes_home() / _PLANNED_STOP_MARKER_FILENAME @@ -1541,8 +1439,8 @@ def _pid_marker_names_self(target_pid: int, target_start_time: Any) -> bool: ``_get_process_start_time`` returns None on platforms without /proc (macOS, native Windows -- where the planned-stop watcher matters most). Requiring a - non-None match there would make every consume return False and misclassify - a legitimate ``hermes gateway stop`` as an unexpected exit (revived by the + non-None match there would make every consume return False and misclassify a + legitimate ``hermes gateway stop`` as an unexpected exit (revived by the service manager). So: when both start times are known they must match; when either is unknown, PID equality alone decides (bounded by the marker TTL). Shared by the watcher's non-destructive probe and the authoritative consume @@ -1561,30 +1459,34 @@ def _consume_pid_marker_for_self(path: Path, *, ttl_s: int) -> bool: if parsed is None: return False record, target_pid, target_start_time = parsed - # Cross-profile guard: new markers name the verified TARGET home, which # permits a deliberate cross-HERMES_HOME --replace while ignoring a marker - # accidentally written into another profile's directory. Legacy markers - # have no target field, so keep the original same-replacer-home rule. + # accidentally written into another profile's directory. Legacy markers have + # no target field, so keep the original same-replacer-home rule. our_home = _get_process_hermes_home() target_home = record.get("target_hermes_home") if target_home is not None: - if not isinstance(target_home, str) or not _same_hermes_home( - target_home, our_home - ): + if not isinstance(target_home, str) or not _same_hermes_home(target_home, our_home): return False else: replacer_home = record.get("replacer_hermes_home") - if replacer_home is not None and not _same_hermes_home( - replacer_home, our_home - ): + if replacer_home is not None and not _same_hermes_home(replacer_home, our_home): return False - matches = _pid_marker_names_self(target_pid, target_start_time) _unlink_quietly(path) return matches +def _write_marker(build: Callable[[], tuple[Path, dict[str, Any]]]) -> bool: + """Build and atomically write a marker record; False on any OS failure.""" + try: + path, record = build() + _write_json_file(path, record) + return True + except (OSError, PermissionError): + return False + + def write_takeover_marker( target_pid: int, *, @@ -1599,26 +1501,19 @@ def write_takeover_marker( such callers must fail closed on False (the target's supervisor could otherwise revive it). """ - try: - marker_home = _canonical_hermes_home( - target_home or _get_process_hermes_home() - ) - if target_start_time is _UNSET: - target_start_time = _get_process_start_time(target_pid) - record = { + def build(): + marker_home = _canonical_hermes_home(target_home or _get_process_hermes_home()) + start_time = _get_process_start_time(target_pid) if target_start_time is _UNSET else target_start_time + return _get_takeover_marker_path(marker_home), { "target_pid": target_pid, - "target_start_time": target_start_time, + "target_start_time": start_time, "target_hermes_home": str(marker_home), "replacer_pid": os.getpid(), - "replacer_hermes_home": str( - _canonical_hermes_home(_get_process_hermes_home()) - ), + "replacer_hermes_home": str(_canonical_hermes_home(_get_process_hermes_home())), "written_at": _utc_now_iso(), } - _write_json_file(_get_takeover_marker_path(marker_home), record) - return True - except (OSError, PermissionError): - return False + + return _write_marker(build) def consume_takeover_marker_for_self() -> bool: @@ -1634,9 +1529,7 @@ def clear_takeover_marker(target_home: Optional[Path] = None) -> None: _unlink_quietly(_get_takeover_marker_path(target_home)) -def _validated_scoped_lock_gateway_owner( - record: dict[str, Any], -) -> Optional[tuple[int, int, Path]]: +def _validated_scoped_lock_gateway_owner(record: dict[str, Any]) -> Optional[tuple[int, int, Path]]: """Resolve a live scoped-lock owner to a verified ``(pid, start_time, home)``. A lock file is only a claim: the record, the target home's PID record, and @@ -1645,46 +1538,31 @@ def _validated_scoped_lock_gateway_owner( """ if not isinstance(record, dict) or not _record_looks_like_gateway(record): return None - owner_pid = _pid_from_record(record) if owner_pid is None or owner_pid <= 0 or owner_pid == os.getpid(): return None - owner_start_time = record.get("start_time") if not isinstance(owner_start_time, int) or isinstance(owner_start_time, bool): return None - raw_home = record.get("hermes_home") if not isinstance(raw_home, str) or not raw_home.strip(): return None if not Path(raw_home).expanduser().is_absolute(): return None target_home = _canonical_hermes_home(raw_home) - - if not _pid_exists(owner_pid): + if _scoped_lock_owner_state(owner_pid, owner_start_time) != "same": return None - live_start_time = _get_process_start_time(owner_pid) - if live_start_time is None or live_start_time != owner_start_time: - return None - live_cmdline = _read_process_cmdline(owner_pid) - if live_cmdline is not None and not looks_like_gateway_runtime_command_line( - live_cmdline - ): + if live_cmdline is not None and not looks_like_gateway_runtime_command_line(live_cmdline): return None - pid_record = _read_json_file(target_home / "gateway.pid") if not isinstance(pid_record, dict) or not _record_looks_like_gateway(pid_record): return None if _pid_from_record(pid_record) != owner_pid or pid_record.get("start_time") != owner_start_time: return None - pid_record_home = pid_record.get("hermes_home") - if not isinstance(pid_record_home, str) or not _same_hermes_home( - pid_record_home, target_home - ): + if not isinstance(pid_record_home, str) or not _same_hermes_home(pid_record_home, target_home): return None - return owner_pid, owner_start_time, target_home @@ -1731,9 +1609,7 @@ def _snapshot_gateway_children(pid: int) -> list: return psutil.Process(int(pid)).children(recursive=True) except Exception: - logger.debug( - "Could not snapshot children of gateway PID %d", pid, exc_info=True - ) + logger.debug("Could not snapshot children of gateway PID %d", pid, exc_info=True) return [] @@ -1742,9 +1618,9 @@ def reap_gateway_children(children: list, *, parent_pid: int, timeout: float = 5 Surviving adapter subprocesses keep holding token locks. Call only AFTER the parent is confirmed dead with a snapshot from :func:`_snapshot_gateway_children`. - ``is_running()`` is identity-aware so a recycled child PID is never - signalled; a child whose ppid still equals ``parent_pid`` is skipped (parent - alive => not an orphan). SIGTERM, bounded wait, SIGKILL survivors. Never raises. + ``is_running()`` is identity-aware so a recycled child PID is never signalled; + a child whose ppid still equals ``parent_pid`` is skipped (parent alive => not + an orphan). SIGTERM, bounded wait, SIGKILL survivors. Never raises. """ if _IS_WINDOWS or not children: return 0 @@ -1755,14 +1631,11 @@ def reap_gateway_children(children: list, *, parent_pid: int, timeout: float = 5 live = [] for child in children: try: - if not child.is_running(): - continue - if child.status() == psutil.STATUS_ZOMBIE: + if not child.is_running() or child.status() == psutil.STATUS_ZOMBIE: continue if child.ppid() == parent_pid: logger.debug( - "Skipping child PID %d of old gateway %d: parent " - "still appears alive", + "Skipping child PID %d of old gateway %d: parent still appears alive", child.pid, parent_pid, ) @@ -1800,9 +1673,7 @@ def reap_gateway_children(children: list, *, parent_pid: int, timeout: float = 5 parent_pid, ) except Exception: - logger.debug( - "Child reap for replaced gateway PID %d failed", parent_pid, exc_info=True - ) + logger.debug("Child reap for replaced gateway PID %d failed", parent_pid, exc_info=True) return reaped @@ -1826,7 +1697,6 @@ def take_over_scoped_lock_holder( owner_pid, owner_start_time, target_home = owner # Snapshot while the owner is alive; afterwards children are reparented. owner_children = _snapshot_gateway_children(owner_pid) - replaced = _terminate_scoped_lock_owner_once( owner_pid, owner_start_time, @@ -1847,57 +1717,37 @@ def _terminate_scoped_lock_owner_once( graceful_attempts: int = 20, force_attempts: int = 20, ) -> Optional[int]: - """Marker-write + bounded identity-aware termination of a verified owner.""" - if not write_takeover_marker( - owner_pid, - target_home=target_home, - target_start_time=owner_start_time, - ): - return None + """Marker-write + bounded identity-aware termination of a verified owner. + Each signal step: ``ProcessLookupError`` => already gone (return the PID); + ``PermissionError``/``OSError`` => refuse (None) without escalating. + """ + if not write_takeover_marker(owner_pid, target_home=target_home, target_start_time=owner_start_time): + return None try: state = _scoped_lock_owner_state(owner_pid, owner_start_time) if state == "exited": return owner_pid if state != "same": return None - - try: - terminate_pid(owner_pid, force=False) - except ProcessLookupError: - return owner_pid - except (PermissionError, OSError): - return None - - exited, safe_to_force = _wait_for_scoped_lock_owner_exit( - owner_pid, - owner_start_time, - attempts=graceful_attempts, - delay=0.5, - ) - if exited: - return owner_pid - if not safe_to_force: - return None - - try: - terminate_pid( - owner_pid, - force=True, - expected_start_time=owner_start_time, + for attempts, delay, kwargs in ( + (graceful_attempts, 0.5, {"force": False}), + (force_attempts, 0.25, {"force": True, "expected_start_time": owner_start_time}), + ): + try: + terminate_pid(owner_pid, **kwargs) + except ProcessLookupError: + return owner_pid + except (PermissionError, OSError): + return None + exited, safe_to_force = _wait_for_scoped_lock_owner_exit( + owner_pid, owner_start_time, attempts=attempts, delay=delay ) - except ProcessLookupError: - return owner_pid - except (PermissionError, OSError): - return None - - exited, _ = _wait_for_scoped_lock_owner_exit( - owner_pid, - owner_start_time, - attempts=force_attempts, - delay=0.25, - ) - return owner_pid if exited else None + if exited: + return owner_pid + if not safe_to_force: + return None + return None finally: # The target normally consumes the marker; clean up any remainder. clear_takeover_marker(target_home) @@ -1909,18 +1759,15 @@ def write_planned_stop_marker(target_pid: int) -> bool: Unexpected SIGTERM exits non-zero so service managers revive the gateway; the CLI writes this marker first so a deliberate stop exits cleanly. """ - try: - target_start_time = _get_process_start_time(target_pid) - record = { + def build(): + return _get_planned_stop_marker_path(), { "target_pid": target_pid, - "target_start_time": target_start_time, + "target_start_time": _get_process_start_time(target_pid), "stopper_pid": os.getpid(), "written_at": _utc_now_iso(), } - _write_json_file(_get_planned_stop_marker_path(), record) - return True - except (OSError, PermissionError): - return False + + return _write_marker(build) def consume_planned_stop_marker_for_self() -> bool: @@ -1934,8 +1781,8 @@ def planned_stop_marker_targets_self() -> bool: Used by the watcher thread (``gateway/run.py:_run_planned_stop_watcher``). Unlike :func:`consume_planned_stop_marker_for_self` it never unlinks a matching marker -- the shutdown handler does the authoritative consume. - Malformed/expired markers are still cleaned up; markers naming another - PID are left for that process and report False here. + Malformed/expired markers are still cleaned up; markers naming another PID + are left for that process and report False here. """ parsed = _read_live_pid_marker(_get_planned_stop_marker_path(), _PLANNED_STOP_MARKER_TTL_S) if parsed is None: @@ -1944,11 +1791,6 @@ def planned_stop_marker_targets_self() -> bool: return _pid_marker_names_self(target_pid, target_start_time) -def clear_planned_stop_marker() -> None: - """Remove the planned-stop marker unconditionally.""" - _unlink_quietly(_get_planned_stop_marker_path()) - - def get_running_pid( pid_path: Optional[Path] = None, *, @@ -1959,10 +1801,8 @@ def get_running_pid( resolved_lock_path = _get_gateway_lock_path(resolved_pid_path) if is_gateway_runtime_lock_active(resolved_lock_path): for record in (_read_pid_record(resolved_pid_path), _read_gateway_lock_record(resolved_lock_path)): - pid = _pid_from_record(record) - if pid is None or not _pid_exists(pid) or _start_time_conflicts(record, pid): - continue - if not _pid_record_belongs_to_current_profile(record): + pid = _live_pid_from_record(record) + if pid is None or not _pid_record_belongs_to_current_profile(record): continue if _record_matches_live_gateway_pid(record, pid): return pid @@ -2039,41 +1879,19 @@ def get_running_pid_cached( """ if ttl_seconds <= 0: return get_running_pid(pid_path, cleanup_stale=cleanup_stale) - resolved_pid_path = pid_path or _get_pid_path() include_runtime_status = pid_path is None - signature = _running_pid_cache_signature( - resolved_pid_path, - include_runtime_status=include_runtime_status, - ) + signature = _running_pid_cache_signature(resolved_pid_path, include_runtime_status=include_runtime_status) key = (str(resolved_pid_path), bool(cleanup_stale), include_runtime_status) now = time.monotonic() - with _gateway_running_pid_cache_lock: cached = _gateway_running_pid_cache.get(key) if cached is not None: cached_at, cached_signature, cached_pid = cached if now - cached_at <= ttl_seconds and cached_signature == signature: return cached_pid - pid = get_running_pid(pid_path, cleanup_stale=cleanup_stale) - refreshed_signature = _running_pid_cache_signature( - resolved_pid_path, - include_runtime_status=include_runtime_status, - ) + refreshed_signature = _running_pid_cache_signature(resolved_pid_path, include_runtime_status=include_runtime_status) with _gateway_running_pid_cache_lock: - _gateway_running_pid_cache[key] = ( - time.monotonic(), - refreshed_signature, - pid, - ) + _gateway_running_pid_cache[key] = (time.monotonic(), refreshed_signature, pid) return pid - - -def is_gateway_running( - pid_path: Optional[Path] = None, - *, - cleanup_stale: bool = True, -) -> bool: - """Check if the gateway daemon is currently running.""" - return get_running_pid(pid_path, cleanup_stale=cleanup_stale) is not None