From b4c38f2aa9925ddd869043241adf23427d331bb1 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:23:44 -0700 Subject: [PATCH] refactor(turn): lift run_conversation lease/liveness closures into agent.turn_facade_lease --- agent/turn_facade.py | 517 +++----------------------- agent/turn_facade_lease.py | 390 +++++++++++++++++++ tests/agent/test_turn_facade_lease.py | 124 ++++++ 3 files changed, 576 insertions(+), 455 deletions(-) create mode 100644 agent/turn_facade_lease.py create mode 100644 tests/agent/test_turn_facade_lease.py diff --git a/agent/turn_facade.py b/agent/turn_facade.py index 44350d866b..b4250fa59d 100644 --- a/agent/turn_facade.py +++ b/agent/turn_facade.py @@ -1,17 +1,16 @@ """``AIAgent.run_conversation`` / ``chat`` façade. Turn admission around ``conversation_loop.run_conversation``: durable cross-process session turn lease + -refresher thread, turn-liveness watchdog, relay/accounting/portal scopes, and balanced start/finish marks. -Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s MRO unchanged. +refresher thread and liveness watchdog (``agent.turn_facade_lease``), relay/accounting/portal scopes, and +balanced start/finish marks. Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s +MRO unchanged. """ import logging -import os -import threading import uuid from typing import Any, Dict, List, Optional from agent.lazy_forward import forward as _forward -from tools.interrupt import set_interrupt as _set_interrupt +from tools.interrupt import set_interrupt as _set_interrupt # noqa: F401 (tests patch it here) # Same logger name as the origin module so log records / caplog filters are unchanged. logger = logging.getLogger("run_agent") @@ -36,20 +35,14 @@ class TurnFacadeMixin: ) -> Dict[str, Any]: """Forwarder — see ``agent.conversation_loop.run_conversation``.""" # A review shares this session_id for cache parity: fence review startup or interrupt an admitted - # request and await its exit before opening live-turn instrumentation (#84423). + # request and await its exit before opening live-turn instrumentation. from agent.background_review import cancel_background_review_for_live_turn cancel_background_review_for_live_turn(self) - # Turn liveness for the deferred-review idle queue; start-mark is the first statement of the try - # so the finally's note_turn_finished balances every exit. - from agent.review_idle_queue import QUEUE as _review_queue - - from agent.aux_accounting import ( - reset_accounting_context, - set_accounting_context, - ) from agent import relay_runtime + from agent.aux_accounting import reset_accounting_context, set_accounting_context + from agent.auxiliary_client import scoped_runtime_main from agent.conversation_loop import run_conversation from agent.portal_tags import ( reset_affinity_scope, @@ -58,11 +51,11 @@ class TurnFacadeMixin: set_conversation_context, ) from agent.prompt_cache_scope import declared_conversation_scope_safe - from hermes_cli.observability.relay_shared_metrics import ( - finish_task_run, - start_task_run, - ) + from agent.review_idle_queue import QUEUE as _review_queue from agent.subagent_lifecycle import bind_subagent_parent + from agent.turn_facade_lease import admit_durable_turn_lease + from hermes_cli.observability.relay_shared_metrics import finish_task_run, start_task_run + effective_task_id = task_id or str(uuid.uuid4()) session_id = str(getattr(self, "session_id", None) or "") task_context = { @@ -70,372 +63,38 @@ class TurnFacadeMixin: "task_id": effective_task_id, "platform": getattr(self, "platform", None) or "", } - relay_turn_id = ( - f"{session_id or 'session'}:{effective_task_id}:{uuid.uuid4().hex[:8]}" - ) + relay_turn_id = f"{session_id or 'session'}:{effective_task_id}:{uuid.uuid4().hex[:8]}" self._relay_pending_turn_id = relay_turn_id relay_parent_session_id = ( str(getattr(self, "_parent_session_id", None) or "") if task_context["platform"] == "subagent" else "" ) - relay_lease = None - relay_turn = None - durable_turn_lease = None - durable_turn_lease_stop = None - durable_turn_lease_thread = None - durable_turn_liveness_thread = None - durable_turn_lease_activity_lock = threading.Lock() - durable_turn_lease_turn_active = False - durable_turn_lease_interrupt_message = None - token = None - # Initialized alongside `token`: early returns leave the try before set_affinity_scope() and the - # finally reads this unconditionally (PR #97158). - affinity_token = None - acct_token = None - task_started = False - task_finished = False + relay_lease = relay_turn = lease = None + # Scope tokens start None: early returns leave the try before the set_*() calls and the finally + # resets each one unconditionally. + token = affinity_token = acct_token = None + task_started = task_finished = False relay_outcome = "failed" - def _stop_durable_turn_lease_refresher() -> None: - nonlocal durable_turn_lease_turn_active - with durable_turn_lease_activity_lock: - durable_turn_lease_turn_active = False - if durable_turn_lease_stop is not None: - durable_turn_lease_stop.set() - - def _clear_durable_turn_lease_interrupt() -> None: - """Clear only the interrupt admitted by this turn's refresher.""" - message = durable_turn_lease_interrupt_message - if not message: - return - - def _clear_if_owned() -> None: - if getattr(self, "_interrupt_message", None) != message: - return - self._interrupt_requested = False - self._interrupt_message = None - getattr(self, "_hard_interrupt_requested", threading.Event()).clear() - self._interrupt_thread_signal_pending = False - if self._execution_thread_id is not None: - _set_interrupt(False, self._execution_thread_id) - - redirect_lock = getattr(self, "_pending_redirect_lock", None) - if redirect_lock is None: - _clear_if_owned() - else: - with redirect_lock: - _clear_if_owned() - try: + # Turn liveness for the deferred-review idle queue: first statement of the try so the finally's + # note_turn_finished balances every exit. _review_queue.note_turn_started() - # Durable cross-process lease over load -> run -> flush (Desktop, CLI resume, gateway, background - # delivery sharing state.db, #84234). - _turn_db = getattr(self, "_session_db", None) - _durable_session_exists = False - if _turn_db is not None and session_id: - try: - _durable_session_exists = _turn_db.get_session(session_id) is not None - except Exception: - # A locked / non-WAL read is not proof the row is absent; treating probe failure as - # "fresh" ran fail-open at the exact contention point (#84234). Acquire, or fail closed. - logger.warning( - "Could not check durable session before turn lease; " - "will acquire rather than run without serialization", - exc_info=True, - ) - _durable_session_exists = True - if ( - _turn_db is not None - and session_id - and not getattr(self, "_persist_disabled", False) - # A fresh session id has no durable transcript to race over, and callers may supply an in- - # memory seed before the row exists — reloading would erase it. - and _durable_session_exists - # Check the concrete type: MagicMock-style shims accept any attribute without the protocol. - and callable( - getattr(type(_turn_db), "acquire_session_turn_lease", None) + admission = admit_durable_turn_lease( + self, + session_id=session_id, + relay_turn_id=relay_turn_id, + task_context=task_context, + conversation_history=conversation_history, + ) + if admission.early_result is not None: + relay_outcome = ( + "cancelled" if admission.early_result.get("interrupted") else "timed_out" ) - ): - # Row proven to exist — suppress the redundant create attempt. - self._session_db_created = True - _durable_holder = ( - f"pid={os.getpid()}:turn={relay_turn_id}:platform=" - f"{task_context['platform'] or 'unknown'}" - ) - _lease_ttl = 300.0 - _lease_waited = False - - def _on_session_turn_lease_wait(elapsed: float) -> None: - nonlocal _lease_waited - _lease_waited = True - if elapsed < 1.0: - self._emit_status( - "⏳ Another Hermes process is using this session; " - "waiting for it to finish before starting your turn..." - ) - else: - self._emit_status( - "⏳ Still waiting for the other Hermes process on " - f"this session ({int(elapsed)}s)..." - ) - - if not _turn_db.acquire_session_turn_lease( - session_id, - _durable_holder, - ttl_seconds=_lease_ttl, - wait_seconds=1800.0, - on_wait=_on_session_turn_lease_wait, - should_abort=lambda: getattr(self, "_interrupt_requested", False), - ): - if getattr(self, "_interrupt_requested", False): - logger.info( - "session turn lease wait aborted by interrupt: %s", - session_id, - ) - relay_outcome = "cancelled" - interrupt_msg = ( - "Stopped waiting for another Hermes process on " - "this session. Your message was not processed." - ) - interrupt_result = { - "final_response": interrupt_msg, - "messages": list(conversation_history or []), - "api_calls": 0, - "completed": False, - "interrupted": True, - } - interrupt_message = getattr( - self, "_interrupt_message", None - ) - if interrupt_message: - interrupt_result["interrupt_message"] = ( - interrupt_message - ) - # The finalizer never runs on this early return; clear so a cached agent doesn't fail- - # close the next turn. - try: - self.clear_interrupt() - except Exception: - self._interrupt_requested = False - self._interrupt_message = None - return interrupt_result - # Fail closed like gateway TurnLeaseTimeoutError: surface a resend notice, not a bare - # TimeoutError. - timeout_msg = ( - "⏳ Another Hermes process kept this session busy too " - "long. Your message was not processed - wait for the " - "other process to finish, then send it again." - ) - logger.error( - "session turn lease wait timed out for %s", - session_id, - ) - try: - self._emit_warning(timeout_msg) - except Exception: - logger.debug( - "Failed to emit session turn lease timeout warning", - exc_info=True, - ) - relay_outcome = "timed_out" - return { - "final_response": timeout_msg, - "messages": list(conversation_history or []), - "api_calls": 0, - "completed": False, - "failed": True, - "error": f"session_turn_lease_timeout:{session_id}", - } - - # Assign only after admission so the finally cannot release a holder that never owned the row; - # persist paths read the agent attr so a late flush is fenced in the same SQLite transaction. - durable_turn_lease = _durable_holder - self._active_session_turn_lease_holder = _durable_holder - self._active_session_turn_lease_ttl_seconds = _lease_ttl - if _lease_waited: - self._emit_status( - "Session is free; loading the latest transcript..." - ) - - # The holder may have compressed/rotated the session while we waited: reload only AFTER - # admission, and skip when acquisition was immediate (avoids a needless prompt-cache miss). - if _lease_waited: - latest_session_id = _turn_db.resolve_resume_session_id(session_id) - if latest_session_id: - self.session_id = latest_session_id - task_context["session_id"] = latest_session_id - conversation_history = _turn_db.get_messages_as_conversation( - self.session_id, - repair_alternation=True, - include_row_ids=True, - ) - - # Long turns outlive a fixed TTL: refresh in a daemon thread; holder-qualified UPDATE/DELETE - # fence a late refresher from a successor lease. - durable_turn_lease_stop = threading.Event() - _lease_refresh_interval = float( - getattr(self, "_session_turn_lease_refresh_interval", 60.0) - ) - - # ── Turn liveness watchdog (#95548) ── - # Lease renewal is NOT evidence of progress; a silently stalled turn would renew forever. - # Policy - # lives in agent/turn_liveness.py; this block only wires config + commit/deactivate callbacks. - try: - from hermes_cli.config import ( - load_config_readonly as _liveness_load_config, - ) - _liveness_config = _liveness_load_config() or {} - except Exception: - _liveness_config = {} - from agent import turn_liveness - - _liveness_timeout, _liveness_poll = ( - turn_liveness.resolve_turn_liveness_settings(_liveness_config) - ) - - def _interrupt_turn(message: str) -> None: - # Lease-loss interrupts fire UNCONDITIONALLY (no generation claim): a lost lease means - # this process no longer owns the session. Only the watchdog's stalls can be spuriously - # stale. - nonlocal durable_turn_lease_interrupt_message - with durable_turn_lease_activity_lock: - if ( - durable_turn_lease_stop.is_set() - or not durable_turn_lease_turn_active - ): - return - durable_turn_lease_interrupt_message = message - try: - self.interrupt(message, hard_cancel=True) - except Exception: - self._interrupt_requested = True - self._interrupt_message = message - - def _commit_turn_liveness_abort( - snapshot: "turn_liveness.ActivitySnapshot", - message: str, - ) -> bool: - """Commit point for the watchdog's stall observation. - - Revalidates the observed ``(generation, timestamp)`` under the SAME lock - ``_touch_activity`` uses, so a turn - that resumed while the stall was logged is never hard-cancelled. The revalidated - generation is carried into - ``interrupt`` as ``require_generation``, which consumes it with the first publication in - ONE critical section. - If ``interrupt`` raises, the abort declines FAIL-CLOSED. Returns False when stale or - already winding down. - """ - nonlocal durable_turn_lease_interrupt_message - with self._liveness_activity_lock(): - current_generation = getattr( - self, "_turn_liveness_activity_generation", 0 - ) - if ( - current_generation, - getattr(self, "_last_activity_ts", None), - ) != (snapshot.generation, snapshot.activity_ts): - return False - with durable_turn_lease_activity_lock: - if ( - durable_turn_lease_stop.is_set() - or not durable_turn_lease_turn_active - ): - return False - try: - published = self.interrupt( - message, - hard_cancel=True, - require_generation=current_generation, - ) - except Exception: - # Round-4 (#95663): fail closed — an exceptional path must not turn an unvalidated - # claim into unconditional abort authority. - logger.debug( - "Turn liveness abort interrupt raised; " - "declining the abort", - exc_info=True, - ) - published = False - if published is False: - # Claim went stale between revalidation and the hammer: real progress landed, abandon - # the abort. - return False - with durable_turn_lease_activity_lock: - durable_turn_lease_interrupt_message = message - return True - - def _deactivate_turn_after_liveness_abort() -> None: - """Stop lease renewal after a committed liveness abort. - - A wedge the hard interrupt cannot unwind must not keep the lease alive forever; TTL expiry - lets stale-turn cleanup reclaim the row. - """ - nonlocal durable_turn_lease_turn_active - with durable_turn_lease_activity_lock: - durable_turn_lease_stop.set() - durable_turn_lease_turn_active = False - - def _turn_is_active() -> bool: - with durable_turn_lease_activity_lock: - return durable_turn_lease_turn_active - - def _refresh_durable_turn_lease() -> None: - while not durable_turn_lease_stop.wait(_lease_refresh_interval): - try: - if not _turn_db.refresh_session_turn_lease( - getattr(self, "session_id", None) or session_id, - durable_turn_lease, - ttl_seconds=_lease_ttl, - ): - # finally sets stop then releases; a late holder-fenced miss must not hard- - # interrupt the next turn. - if durable_turn_lease_stop.is_set(): - return - logger.error( - "Lost session turn lease while turn is active: %s", - getattr(self, "session_id", None) or session_id, - ) - _interrupt_turn( - "Session turn lease lost; stopping to protect " - "the transcript." - ) - return - except Exception: - if durable_turn_lease_stop.is_set(): - return - logger.warning( - "Failed to refresh session turn lease: %s", - getattr(self, "session_id", None) or session_id, - exc_info=True, - ) - _interrupt_turn( - "Session turn lease could not be refreshed; " - "stopping to protect the transcript." - ) - return - - durable_turn_lease_thread = threading.Thread( - target=_refresh_durable_turn_lease, - name="session-turn-lease-refresh", - daemon=True, - ) - if _liveness_timeout is not None: - durable_turn_liveness_thread = ( - turn_liveness.TurnLivenessWatchdog( - self, - session_id=getattr(self, "session_id", None) or session_id, - timeout_s=_liveness_timeout, - poll_s=_liveness_poll, - stop_event=durable_turn_lease_stop, - activity_lock=self._liveness_activity_lock(), - is_turn_active=_turn_is_active, - commit_abort=_commit_turn_liveness_abort, - deactivate_turn=_deactivate_turn_after_liveness_abort, - ).make_thread() - ) + return admission.early_result + lease = admission.lease + conversation_history = admission.conversation_history relay_lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation( profile_key=relay_runtime.current_profile_key(), @@ -445,47 +104,30 @@ class TurnFacadeMixin: model=str(getattr(self, "model", None) or ""), ) relay_turn = relay_runtime.SESSION_COORDINATOR.begin_turn( - relay_lease, - turn_id=relay_turn_id, - task_id=effective_task_id, + relay_lease, turn_id=relay_turn_id, task_id=effective_task_id ) - # Keep existing tests and external relay-runtime shims that return - # a minimal turn object compatible with the new opt-out flag. + # Minimal relay-runtime shims (tests, external) may lack the opt-out flag: default enabled. if getattr(relay_turn, "relay_enabled", True): start_task_run( **task_context, parent_session_id=getattr(self, "_parent_session_id", None) or "", ) task_started = True - # Publish the conversation id for ambient Nous Portal tagging: every LLM call in this turn (loop, - # compression, vision, MoA, review forks) inherits `conversation=`. + # Publish the conversation id for ambient Nous Portal tagging: every LLM call in this turn + # (loop, compression, vision, MoA, review forks) inherits `conversation=`. token = set_conversation_context(self._conversation_root_id()) - # Routing/affinity scope the HOST declared; providers fall back to the id above when unset - # (#96811). - affinity_token = set_affinity_scope( - declared_conversation_scope_safe(self) - ) - # Publish session accounting handles so auxiliary calls record usage into session_model_usage - # (#23270). + # Routing/affinity scope the HOST declared; providers fall back to the id above when unset. + affinity_token = set_affinity_scope(declared_conversation_scope_safe(self)) + # Session accounting handles so auxiliary calls record usage into session_model_usage. acct_token = set_accounting_context( - getattr(self, "_session_db", None), - getattr(self, "session_id", None), + getattr(self, "_session_db", None), getattr(self, "session_id", None) ) - from agent.auxiliary_client import scoped_runtime_main # Keep the ContextVar scope local (tokens on the agent may be observed from another thread). with bind_subagent_parent(self), scoped_runtime_main({}): try: - if durable_turn_lease_thread is not None: - with durable_turn_lease_activity_lock: - durable_turn_lease_turn_active = True - # Stamp the activity clock at turn entry (#95663): `_last_activity_ts` persists across - # turns, so without this the watchdog would measure idle from the PREVIOUS turn and - # abort a fresh one. - self._touch_activity("starting new turn") - durable_turn_lease_thread.start() - if durable_turn_liveness_thread is not None: - durable_turn_liveness_thread.start() + if lease is not None: + lease.start() result = run_conversation( self, user_message, @@ -501,10 +143,10 @@ class TurnFacadeMixin: moa_config=moa_config, ) finally: - # Post-loop relay/task finalization must not receive a late refresh interrupt. - _stop_durable_turn_lease_refresher() - # Interrupt clear is deferred until after thread join (outer finally) so a refresher - # firing between stop and join cannot leave an interrupt behind. + # Post-loop relay/task finalization must not receive a late refresh interrupt. The + # interrupt clear itself waits for the thread join in the outer finally. + if lease is not None: + lease.stop_refresher() terminal = result if isinstance(result, dict) else {} if terminal.get("interrupted") is True: relay_outcome = "cancelled" @@ -512,10 +154,7 @@ class TurnFacadeMixin: relay_outcome = "failed" else: relay_outcome = "success" - relay_runtime.SESSION_COORDINATOR.finish_logical_calls( - relay_turn, - outcome=relay_outcome, - ) + relay_runtime.SESSION_COORDINATOR.finish_logical_calls(relay_turn, outcome=relay_outcome) if task_started: task_finished = True finish_task_run(**task_context, result=result) @@ -529,8 +168,7 @@ class TurnFacadeMixin: relay_outcome = "timed_out" if relay_turn is not None: relay_runtime.SESSION_COORDINATOR.finish_logical_calls( - relay_turn, - outcome=relay_outcome, + relay_turn, outcome=relay_outcome ) if task_started and not task_finished: task_finished = True @@ -539,48 +177,20 @@ class TurnFacadeMixin: finally: try: if relay_turn is not None: - relay_runtime.SESSION_COORDINATOR.end_turn( - relay_turn, - outcome=relay_outcome, - ) + relay_runtime.SESSION_COORDINATOR.end_turn(relay_turn, outcome=relay_outcome) finally: try: if relay_lease is not None: - relay_runtime.SESSION_COORDINATOR.release_conversation( - relay_lease - ) + relay_runtime.SESSION_COORDINATOR.release_conversation(relay_lease) finally: - _stop_durable_turn_lease_refresher() - for _durable_thread in ( - durable_turn_lease_thread, - durable_turn_liveness_thread, - ): - if ( - _durable_thread is not None - and _durable_thread.is_alive() - ): - _durable_thread.join(timeout=1.0) - # Clear any refresher interrupt fired between stop and join; must run AFTER join. - _clear_durable_turn_lease_interrupt() - if durable_turn_lease is not None: - try: - _turn_db.release_session_turn_lease( - session_id, durable_turn_lease - ) - except Exception: - logger.error( - "Failed to release session turn lease: %s", - session_id, - exc_info=True, - ) - if ( - getattr(self, "_active_session_turn_lease_holder", None) - == durable_turn_lease - ): - self._active_session_turn_lease_holder = None - self._active_session_turn_lease_ttl_seconds = None - # Always clear mid-turn labels when the turn exits — including - # interrupted early returns that skip finalize_turn. Keep ts. + if lease is not None: + lease.stop_refresher() + lease.join_threads() + # Clear any refresher interrupt fired between stop and join; must run AFTER join. + lease.clear_interrupt() + lease.release() + # Always clear mid-turn labels when the turn exits — including interrupted early + # returns that skip finalize_turn. Keep ts. try: self._reset_activity_labels_after_turn() except Exception: @@ -593,18 +203,15 @@ class TurnFacadeMixin: reset_conversation_context(token) if affinity_token is not None: reset_affinity_scope(affinity_token) - # Balance note_turn_started on every exit so the idle queue's live-turn count cannot leak. + # Balance note_turn_started on every exit so the idle queue's live-turn count + # cannot leak. try: _review_queue.note_turn_finished() except Exception: pass def chat(self, message: str, stream_callback: Optional[callable] = None) -> str: - """Simple chat interface that returns just the final response string. - - ``stream_callback`` is invoked with each text delta during streaming. - """ - result = self.run_conversation(message, stream_callback=stream_callback) - return result["final_response"] + """Final response string of one turn; ``stream_callback`` receives each text delta.""" + return self.run_conversation(message, stream_callback=stream_callback)["final_response"] _run_codex_app_server_turn = _forward("agent.codex_runtime", "run_codex_app_server_turn") diff --git a/agent/turn_facade_lease.py b/agent/turn_facade_lease.py new file mode 100644 index 0000000000..9309ad6f93 --- /dev/null +++ b/agent/turn_facade_lease.py @@ -0,0 +1,390 @@ +"""Durable cross-process session turn lease for ``TurnFacadeMixin.run_conversation``. + +One process at a time may load -> run -> flush a session shared through state.db (Desktop, CLI +resume, gateway, background delivery). ``admit_durable_turn_lease`` acquires the row lease (or +returns the early result the façade must hand back); ``DurableTurnLease`` owns the refresher +daemon thread, the turn-liveness watchdog wiring, and the lease-loss / stall interrupt plumbing. +""" +import logging +import os +import threading +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +# Same logger name as the origin module so log records / caplog filters are unchanged. +logger = logging.getLogger("run_agent") + +LEASE_TTL_SECONDS = 300.0 +LEASE_WAIT_SECONDS = 1800.0 + + +class DurableTurnLease: + """An admitted session turn lease plus the threads that keep it alive and watch the turn. + + ``stop`` is shared by the refresher and the liveness watchdog; ``turn_active`` gates every + interrupt so a late refresher miss can never hard-interrupt the NEXT turn. Both are read and + written only under ``_lock``. + """ + + def __init__(self, agent, db, session_id: str, holder: str) -> None: + self.agent = agent + self.db = db + self.session_id = session_id # id at admission; release always targets this row + self.holder = holder + self.stop = threading.Event() + self.refresh_interval = float(getattr(agent, "_session_turn_lease_refresh_interval", 60.0)) + self._lock = threading.Lock() + self.turn_active = False + self.interrupt_message: Optional[str] = None + self.refresh_thread: Optional[threading.Thread] = None + self.liveness_thread: Optional[threading.Thread] = None + + def _current_session_id(self) -> str: + return getattr(self.agent, "session_id", None) or self.session_id + + # ── thread construction ────────────────────────────────────────────────────── + + def build_threads(self) -> None: + """Create (not start) the refresher thread and, when configured, the liveness watchdog. + + Lease renewal is NOT evidence of progress; a silently stalled turn would renew forever, so the + watchdog (policy in ``agent/turn_liveness.py``) is wired with the commit/deactivate callbacks + below. + """ + self.refresh_thread = threading.Thread( + target=self.refresh_loop, name="session-turn-lease-refresh", daemon=True + ) + try: + from hermes_cli.config import load_config_readonly + + liveness_config = load_config_readonly() or {} + except Exception: + liveness_config = {} + from agent import turn_liveness + + timeout_s, poll_s = turn_liveness.resolve_turn_liveness_settings(liveness_config) + if timeout_s is not None: + self.liveness_thread = turn_liveness.TurnLivenessWatchdog( + self.agent, + session_id=self._current_session_id(), + timeout_s=timeout_s, + poll_s=poll_s, + stop_event=self.stop, + activity_lock=self.agent._liveness_activity_lock(), + is_turn_active=self.is_turn_active, + commit_abort=self.commit_liveness_abort, + deactivate_turn=self.deactivate_after_liveness_abort, + ).make_thread() + + # ── lifecycle ──────────────────────────────────────────────────────────────── + + def start(self) -> None: + with self._lock: + self.turn_active = True + # Stamp the activity clock at turn entry: `_last_activity_ts` persists across turns, so without + # this the watchdog would measure idle from the PREVIOUS turn and abort a fresh one. + self.agent._touch_activity("starting new turn") + self.refresh_thread.start() + if self.liveness_thread is not None: + self.liveness_thread.start() + + def stop_refresher(self) -> None: + with self._lock: + self.turn_active = False + self.stop.set() + + def join_threads(self, timeout: float = 1.0) -> None: + for thread in (self.refresh_thread, self.liveness_thread): + if thread is not None and thread.is_alive(): + thread.join(timeout=timeout) + + def release(self) -> None: + """Release the row and drop the agent's holder attrs (only if they still name this lease).""" + agent = self.agent + try: + self.db.release_session_turn_lease(self.session_id, self.holder) + except Exception: + logger.error("Failed to release session turn lease: %s", self.session_id, exc_info=True) + if getattr(agent, "_active_session_turn_lease_holder", None) == self.holder: + agent._active_session_turn_lease_holder = None + agent._active_session_turn_lease_ttl_seconds = None + + def is_turn_active(self) -> bool: + with self._lock: + return self.turn_active + + # ── interrupts ─────────────────────────────────────────────────────────────── + + def _interrupt_turn(self, message: str) -> None: + # Lease-loss interrupts fire UNCONDITIONALLY (no generation claim): a lost lease means this + # process no longer owns the session. Only the watchdog's stalls can be spuriously stale. + agent = self.agent + with self._lock: + if self.stop.is_set() or not self.turn_active: + return + self.interrupt_message = message + try: + agent.interrupt(message, hard_cancel=True) + except Exception: + agent._interrupt_requested = True + agent._interrupt_message = message + + def commit_liveness_abort(self, snapshot, message: str) -> bool: + """Commit point for the watchdog's stall observation. + + Revalidates the observed ``(generation, timestamp)`` under the SAME lock ``_touch_activity`` + uses, so a turn that resumed while the stall was logged is never hard-cancelled. The + revalidated generation is carried into ``interrupt`` as ``require_generation``, which consumes + it with the first publication in ONE critical section. If ``interrupt`` raises, the abort + declines FAIL-CLOSED. Returns False when stale or already winding down. + """ + agent = self.agent + with agent._liveness_activity_lock(): + current_generation = getattr(agent, "_turn_liveness_activity_generation", 0) + if (current_generation, getattr(agent, "_last_activity_ts", None)) != ( + snapshot.generation, + snapshot.activity_ts, + ): + return False + with self._lock: + if self.stop.is_set() or not self.turn_active: + return False + try: + published = agent.interrupt( + message, hard_cancel=True, require_generation=current_generation + ) + except Exception: + # Fail closed: an exceptional path must not turn an unvalidated claim into abort authority. + logger.debug( + "Turn liveness abort interrupt raised; declining the abort", exc_info=True + ) + published = False + if published is False: + # Claim went stale between revalidation and the hammer: real progress landed. + return False + with self._lock: + self.interrupt_message = message + return True + + def deactivate_after_liveness_abort(self) -> None: + """Stop lease renewal after a committed liveness abort. + + A wedge the hard interrupt cannot unwind must not keep the lease alive forever; TTL expiry + lets stale-turn cleanup reclaim the row. + """ + with self._lock: + self.stop.set() + self.turn_active = False + + def clear_interrupt(self) -> None: + """Clear only the interrupt admitted by this lease's refresher/watchdog. Run AFTER join.""" + message = self.interrupt_message + if not message: + return + agent = self.agent + + def _clear_if_owned() -> None: + # Lazy via the façade so ``patch("agent.turn_facade._set_interrupt")`` keeps intercepting. + from agent.turn_facade import _set_interrupt + + if getattr(agent, "_interrupt_message", None) != message: + return + agent._interrupt_requested = False + agent._interrupt_message = None + getattr(agent, "_hard_interrupt_requested", threading.Event()).clear() + agent._interrupt_thread_signal_pending = False + if agent._execution_thread_id is not None: + _set_interrupt(False, agent._execution_thread_id) + + redirect_lock = getattr(agent, "_pending_redirect_lock", None) + if redirect_lock is None: + _clear_if_owned() + else: + with redirect_lock: + _clear_if_owned() + + # ── refresher thread body ──────────────────────────────────────────────────── + + def refresh_loop(self) -> None: + """Renew the lease every ``refresh_interval``; a miss or error interrupts the turn. + + Long turns outlive the fixed TTL; the holder-qualified UPDATE fences a late refresher from a + successor lease. The façade's finally sets ``stop`` before releasing, so a holder-fenced miss + observed after stop is not a loss. + """ + while not self.stop.wait(self.refresh_interval): + try: + if self.db.refresh_session_turn_lease( + self._current_session_id(), self.holder, ttl_seconds=LEASE_TTL_SECONDS + ): + continue + if self.stop.is_set(): + return + logger.error( + "Lost session turn lease while turn is active: %s", self._current_session_id() + ) + self._interrupt_turn( + "Session turn lease lost; stopping to protect the transcript." + ) + except Exception: + if self.stop.is_set(): + return + logger.warning( + "Failed to refresh session turn lease: %s", + self._current_session_id(), + exc_info=True, + ) + self._interrupt_turn( + "Session turn lease could not be refreshed; stopping to protect the transcript." + ) + return + + +@dataclass +class TurnLeaseAdmission: + """Outcome of ``admit_durable_turn_lease``: exactly one of ``lease`` / ``early_result`` may be set.""" + + lease: Optional[DurableTurnLease] = None + early_result: Optional[Dict[str, Any]] = None + conversation_history: Optional[List[Dict[str, Any]]] = None + + +def _durable_session_exists(db, session_id: str) -> bool: + try: + return db.get_session(session_id) is not None + except Exception: + # A locked / non-WAL read is not proof the row is absent; treating probe failure as "fresh" + # ran fail-open at the exact contention point. Acquire, or fail closed. + logger.warning( + "Could not check durable session before turn lease; " + "will acquire rather than run without serialization", + exc_info=True, + ) + return True + + +def admit_durable_turn_lease( + agent, + *, + session_id: str, + relay_turn_id: str, + task_context: Dict[str, Any], + conversation_history: Optional[List[Dict[str, Any]]], +) -> TurnLeaseAdmission: + """Acquire the session turn lease when the session is durable; build (not start) its threads. + + Mutates ``task_context["session_id"]`` and ``agent.session_id`` when the wait forced a resume-id + reload. Returns an ``early_result`` (interrupted / timed out) instead of a lease when admission + fails; the caller returns it verbatim. + """ + db = getattr(agent, "_session_db", None) + admission = TurnLeaseAdmission(conversation_history=conversation_history) + if db is None or not session_id: + return admission + # A fresh session id has no durable transcript to race over, and callers may supply an in-memory + # seed before the row exists — reloading would erase it. Check the concrete type: MagicMock-style + # shims accept any attribute without the protocol. + exists = _durable_session_exists(db, session_id) + if ( + getattr(agent, "_persist_disabled", False) + or not exists + or not callable(getattr(type(db), "acquire_session_turn_lease", None)) + ): + return admission + # Row proven to exist — suppress the redundant create attempt. + agent._session_db_created = True + holder = ( + f"pid={os.getpid()}:turn={relay_turn_id}:platform={task_context['platform'] or 'unknown'}" + ) + waited = False + + def _on_wait(elapsed: float) -> None: + nonlocal waited + waited = True + if elapsed < 1.0: + agent._emit_status( + "⏳ Another Hermes process is using this session; " + "waiting for it to finish before starting your turn..." + ) + else: + agent._emit_status( + "⏳ Still waiting for the other Hermes process on " + f"this session ({int(elapsed)}s)..." + ) + + if not db.acquire_session_turn_lease( + session_id, + holder, + ttl_seconds=LEASE_TTL_SECONDS, + wait_seconds=LEASE_WAIT_SECONDS, + on_wait=_on_wait, + should_abort=lambda: getattr(agent, "_interrupt_requested", False), + ): + admission.early_result = _lease_not_acquired_result(agent, session_id, conversation_history) + return admission + + # Assign only after admission so the finally cannot release a holder that never owned the row; + # persist paths read the agent attr so a late flush is fenced in the same SQLite transaction. + lease = DurableTurnLease(agent, db, session_id, holder) + agent._active_session_turn_lease_holder = holder + agent._active_session_turn_lease_ttl_seconds = LEASE_TTL_SECONDS + try: + if waited: + agent._emit_status("Session is free; loading the latest transcript...") + # The holder may have compressed/rotated the session while we waited: reload only AFTER + # admission, and skip when acquisition was immediate (avoids a needless prompt-cache miss). + latest_session_id = db.resolve_resume_session_id(session_id) + if latest_session_id: + agent.session_id = latest_session_id + task_context["session_id"] = latest_session_id + admission.conversation_history = db.get_messages_as_conversation( + agent.session_id, repair_alternation=True, include_row_ids=True + ) + lease.build_threads() + except BaseException: + # The façade never saw this lease; release here so an admitted row is not leaked. + lease.release() + raise + admission.lease = lease + return admission + + +def _lease_not_acquired_result(agent, session_id: str, conversation_history) -> Dict[str, Any]: + base = {"messages": list(conversation_history or []), "api_calls": 0, "completed": False} + if getattr(agent, "_interrupt_requested", False): + logger.info("session turn lease wait aborted by interrupt: %s", session_id) + result = { + "final_response": ( + "Stopped waiting for another Hermes process on this session. " + "Your message was not processed." + ), + **base, + "interrupted": True, + } + interrupt_message = getattr(agent, "_interrupt_message", None) + if interrupt_message: + result["interrupt_message"] = interrupt_message + # The finalizer never runs on this early return; clear so a cached agent doesn't fail-close + # the next turn. + try: + agent.clear_interrupt() + except Exception: + agent._interrupt_requested = False + agent._interrupt_message = None + return result + # Fail closed like gateway TurnLeaseTimeoutError: surface a resend notice, not a bare TimeoutError. + timeout_msg = ( + "⏳ Another Hermes process kept this session busy too long. Your message was not " + "processed - wait for the other process to finish, then send it again." + ) + logger.error("session turn lease wait timed out for %s", session_id) + try: + agent._emit_warning(timeout_msg) + except Exception: + logger.debug("Failed to emit session turn lease timeout warning", exc_info=True) + return { + "final_response": timeout_msg, + **base, + "failed": True, + "error": f"session_turn_lease_timeout:{session_id}", + } diff --git a/tests/agent/test_turn_facade_lease.py b/tests/agent/test_turn_facade_lease.py new file mode 100644 index 0000000000..cb73f5818e --- /dev/null +++ b/tests/agent/test_turn_facade_lease.py @@ -0,0 +1,124 @@ +"""Unit tests for agent.turn_facade_lease (admission + lease bracket).""" +import threading +from types import SimpleNamespace + +from agent.turn_facade_lease import ( + LEASE_TTL_SECONDS, + DurableTurnLease, + admit_durable_turn_lease, +) + + +class _Db: + def __init__(self, exists=True, acquired=True): + self.exists = exists + self.acquired = acquired + self.events = [] + + def get_session(self, session_id): + return {"id": session_id} if self.exists else None + + def acquire_session_turn_lease(self, session_id, holder, **kwargs): + self.events.append(("acquire", session_id, holder)) + return self.acquired + + def refresh_session_turn_lease(self, session_id, holder, **kwargs): + return True + + def release_session_turn_lease(self, session_id, holder): + self.events.append(("release", session_id, holder)) + + +def _agent(db, **overrides): + agent = SimpleNamespace( + _session_db=db, + session_id="s1", + _persist_disabled=False, + _interrupt_requested=False, + _interrupt_message=None, + _execution_thread_id=None, + _session_turn_lease_refresh_interval=60.0, + statuses=[], + ) + agent._emit_status = agent.statuses.append + agent._emit_warning = agent.statuses.append + agent._touch_activity = lambda *a, **k: None + agent._liveness_activity_lock = lambda: threading.Lock() + for k, v in overrides.items(): + setattr(agent, k, v) + return agent + + +def _admit(agent, history=None): + return admit_durable_turn_lease( + agent, + session_id="s1", + relay_turn_id="s1:t:abcd", + task_context={"session_id": "s1", "task_id": "t", "platform": "cli"}, + conversation_history=history, + ) + + +def test_no_lease_without_durable_row_or_when_persist_disabled(): + seed = [{"role": "user", "content": "hi"}] + admission = _admit(_agent(_Db(exists=False)), seed) + assert admission.lease is None and admission.early_result is None + assert admission.conversation_history is seed + + db = _Db() + admission = _admit(_agent(db, _persist_disabled=True), seed) + assert admission.lease is None and db.events == [] + + +def test_admission_sets_holder_attrs_and_release_clears_them(monkeypatch): + monkeypatch.setattr( + "agent.turn_liveness.resolve_turn_liveness_settings", lambda cfg: (None, 1.0) + ) + db = _Db() + agent = _agent(db) + admission = _admit(agent) + lease = admission.lease + assert isinstance(lease, DurableTurnLease) + assert agent._session_db_created is True + assert agent._active_session_turn_lease_holder == lease.holder + assert agent._active_session_turn_lease_ttl_seconds == LEASE_TTL_SECONDS + assert lease.holder.startswith("pid=") and ":platform=cli" in lease.holder + assert lease.refresh_thread is not None and lease.liveness_thread is None + assert lease.is_turn_active() is False + + lease.stop_refresher() + lease.join_threads() + lease.clear_interrupt() + lease.release() + assert db.events == [("acquire", "s1", lease.holder), ("release", "s1", lease.holder)] + assert agent._active_session_turn_lease_holder is None + assert agent._active_session_turn_lease_ttl_seconds is None + + +def test_timeout_and_interrupt_early_results(): + agent = _agent(_Db(acquired=False)) + admission = _admit(agent, [{"role": "user", "content": "x"}]) + assert admission.lease is None + assert admission.early_result["failed"] is True + assert admission.early_result["error"] == "session_turn_lease_timeout:s1" + assert admission.early_result["messages"] == [{"role": "user", "content": "x"}] + + agent = _agent(_Db(acquired=False), _interrupt_requested=True, _interrupt_message="stop") + agent.clear_interrupt = lambda: None + admission = _admit(agent) + assert admission.early_result["interrupted"] is True + assert admission.early_result["interrupt_message"] == "stop" + + +def test_interrupt_turn_only_while_active(): + agent = _agent(_Db()) + calls = [] + agent.interrupt = lambda msg, **kw: calls.append(msg) + lease = DurableTurnLease(agent, agent._session_db, "s1", "h") + lease._interrupt_turn("lost") # inactive: ignored + assert calls == [] and lease.interrupt_message is None + lease.turn_active = True + lease._interrupt_turn("lost") + assert calls == ["lost"] and lease.interrupt_message == "lost" + lease.deactivate_after_liveness_abort() + assert lease.stop.is_set() and lease.is_turn_active() is False