refactor(turn): lift run_conversation lease/liveness closures into agent.turn_facade_lease

This commit is contained in:
Teknium
2026-09-02 15:23:44 -07:00
parent 74b928b6a3
commit b4c38f2aa9
3 changed files with 576 additions and 455 deletions
+62 -455
View File
@@ -1,17 +1,16 @@
"""``AIAgent.run_conversation`` / ``chat`` façade.
Turn admission around ``conversation_loop.run_conversation``: durable cross-process session turn lease +
refresher thread, turn-liveness watchdog, relay/accounting/portal scopes, and balanced start/finish marks.
Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s MRO unchanged.
refresher thread and liveness watchdog (``agent.turn_facade_lease``), relay/accounting/portal scopes, and
balanced start/finish marks. Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s
MRO unchanged.
"""
import logging
import os
import threading
import uuid
from typing import Any, Dict, List, Optional
from agent.lazy_forward import forward as _forward
from tools.interrupt import set_interrupt as _set_interrupt
from tools.interrupt import set_interrupt as _set_interrupt # noqa: F401 (tests patch it here)
# Same logger name as the origin module so log records / caplog filters are unchanged.
logger = logging.getLogger("run_agent")
@@ -36,20 +35,14 @@ class TurnFacadeMixin:
) -> Dict[str, Any]:
"""Forwarder — see ``agent.conversation_loop.run_conversation``."""
# A review shares this session_id for cache parity: fence review startup or interrupt an admitted
# request and await its exit before opening live-turn instrumentation (#84423).
# request and await its exit before opening live-turn instrumentation.
from agent.background_review import cancel_background_review_for_live_turn
cancel_background_review_for_live_turn(self)
# Turn liveness for the deferred-review idle queue; start-mark is the first statement of the try
# so the finally's note_turn_finished balances every exit.
from agent.review_idle_queue import QUEUE as _review_queue
from agent.aux_accounting import (
reset_accounting_context,
set_accounting_context,
)
from agent import relay_runtime
from agent.aux_accounting import reset_accounting_context, set_accounting_context
from agent.auxiliary_client import scoped_runtime_main
from agent.conversation_loop import run_conversation
from agent.portal_tags import (
reset_affinity_scope,
@@ -58,11 +51,11 @@ class TurnFacadeMixin:
set_conversation_context,
)
from agent.prompt_cache_scope import declared_conversation_scope_safe
from hermes_cli.observability.relay_shared_metrics import (
finish_task_run,
start_task_run,
)
from agent.review_idle_queue import QUEUE as _review_queue
from agent.subagent_lifecycle import bind_subagent_parent
from agent.turn_facade_lease import admit_durable_turn_lease
from hermes_cli.observability.relay_shared_metrics import finish_task_run, start_task_run
effective_task_id = task_id or str(uuid.uuid4())
session_id = str(getattr(self, "session_id", None) or "")
task_context = {
@@ -70,372 +63,38 @@ class TurnFacadeMixin:
"task_id": effective_task_id,
"platform": getattr(self, "platform", None) or "",
}
relay_turn_id = (
f"{session_id or 'session'}:{effective_task_id}:{uuid.uuid4().hex[:8]}"
)
relay_turn_id = f"{session_id or 'session'}:{effective_task_id}:{uuid.uuid4().hex[:8]}"
self._relay_pending_turn_id = relay_turn_id
relay_parent_session_id = (
str(getattr(self, "_parent_session_id", None) or "")
if task_context["platform"] == "subagent"
else ""
)
relay_lease = None
relay_turn = None
durable_turn_lease = None
durable_turn_lease_stop = None
durable_turn_lease_thread = None
durable_turn_liveness_thread = None
durable_turn_lease_activity_lock = threading.Lock()
durable_turn_lease_turn_active = False
durable_turn_lease_interrupt_message = None
token = None
# Initialized alongside `token`: early returns leave the try before set_affinity_scope() and the
# finally reads this unconditionally (PR #97158).
affinity_token = None
acct_token = None
task_started = False
task_finished = False
relay_lease = relay_turn = lease = None
# Scope tokens start None: early returns leave the try before the set_*() calls and the finally
# resets each one unconditionally.
token = affinity_token = acct_token = None
task_started = task_finished = False
relay_outcome = "failed"
def _stop_durable_turn_lease_refresher() -> None:
nonlocal durable_turn_lease_turn_active
with durable_turn_lease_activity_lock:
durable_turn_lease_turn_active = False
if durable_turn_lease_stop is not None:
durable_turn_lease_stop.set()
def _clear_durable_turn_lease_interrupt() -> None:
"""Clear only the interrupt admitted by this turn's refresher."""
message = durable_turn_lease_interrupt_message
if not message:
return
def _clear_if_owned() -> None:
if getattr(self, "_interrupt_message", None) != message:
return
self._interrupt_requested = False
self._interrupt_message = None
getattr(self, "_hard_interrupt_requested", threading.Event()).clear()
self._interrupt_thread_signal_pending = False
if self._execution_thread_id is not None:
_set_interrupt(False, self._execution_thread_id)
redirect_lock = getattr(self, "_pending_redirect_lock", None)
if redirect_lock is None:
_clear_if_owned()
else:
with redirect_lock:
_clear_if_owned()
try:
# Turn liveness for the deferred-review idle queue: first statement of the try so the finally's
# note_turn_finished balances every exit.
_review_queue.note_turn_started()
# Durable cross-process lease over load -> run -> flush (Desktop, CLI resume, gateway, background
# delivery sharing state.db, #84234).
_turn_db = getattr(self, "_session_db", None)
_durable_session_exists = False
if _turn_db is not None and session_id:
try:
_durable_session_exists = _turn_db.get_session(session_id) is not None
except Exception:
# A locked / non-WAL read is not proof the row is absent; treating probe failure as
# "fresh" ran fail-open at the exact contention point (#84234). Acquire, or fail closed.
logger.warning(
"Could not check durable session before turn lease; "
"will acquire rather than run without serialization",
exc_info=True,
)
_durable_session_exists = True
if (
_turn_db is not None
and session_id
and not getattr(self, "_persist_disabled", False)
# A fresh session id has no durable transcript to race over, and callers may supply an in-
# memory seed before the row exists — reloading would erase it.
and _durable_session_exists
# Check the concrete type: MagicMock-style shims accept any attribute without the protocol.
and callable(
getattr(type(_turn_db), "acquire_session_turn_lease", None)
admission = admit_durable_turn_lease(
self,
session_id=session_id,
relay_turn_id=relay_turn_id,
task_context=task_context,
conversation_history=conversation_history,
)
if admission.early_result is not None:
relay_outcome = (
"cancelled" if admission.early_result.get("interrupted") else "timed_out"
)
):
# Row proven to exist — suppress the redundant create attempt.
self._session_db_created = True
_durable_holder = (
f"pid={os.getpid()}:turn={relay_turn_id}:platform="
f"{task_context['platform'] or 'unknown'}"
)
_lease_ttl = 300.0
_lease_waited = False
def _on_session_turn_lease_wait(elapsed: float) -> None:
nonlocal _lease_waited
_lease_waited = True
if elapsed < 1.0:
self._emit_status(
"⏳ Another Hermes process is using this session; "
"waiting for it to finish before starting your turn..."
)
else:
self._emit_status(
"⏳ Still waiting for the other Hermes process on "
f"this session ({int(elapsed)}s)..."
)
if not _turn_db.acquire_session_turn_lease(
session_id,
_durable_holder,
ttl_seconds=_lease_ttl,
wait_seconds=1800.0,
on_wait=_on_session_turn_lease_wait,
should_abort=lambda: getattr(self, "_interrupt_requested", False),
):
if getattr(self, "_interrupt_requested", False):
logger.info(
"session turn lease wait aborted by interrupt: %s",
session_id,
)
relay_outcome = "cancelled"
interrupt_msg = (
"Stopped waiting for another Hermes process on "
"this session. Your message was not processed."
)
interrupt_result = {
"final_response": interrupt_msg,
"messages": list(conversation_history or []),
"api_calls": 0,
"completed": False,
"interrupted": True,
}
interrupt_message = getattr(
self, "_interrupt_message", None
)
if interrupt_message:
interrupt_result["interrupt_message"] = (
interrupt_message
)
# The finalizer never runs on this early return; clear so a cached agent doesn't fail-
# close the next turn.
try:
self.clear_interrupt()
except Exception:
self._interrupt_requested = False
self._interrupt_message = None
return interrupt_result
# Fail closed like gateway TurnLeaseTimeoutError: surface a resend notice, not a bare
# TimeoutError.
timeout_msg = (
"⏳ Another Hermes process kept this session busy too "
"long. Your message was not processed - wait for the "
"other process to finish, then send it again."
)
logger.error(
"session turn lease wait timed out for %s",
session_id,
)
try:
self._emit_warning(timeout_msg)
except Exception:
logger.debug(
"Failed to emit session turn lease timeout warning",
exc_info=True,
)
relay_outcome = "timed_out"
return {
"final_response": timeout_msg,
"messages": list(conversation_history or []),
"api_calls": 0,
"completed": False,
"failed": True,
"error": f"session_turn_lease_timeout:{session_id}",
}
# Assign only after admission so the finally cannot release a holder that never owned the row;
# persist paths read the agent attr so a late flush is fenced in the same SQLite transaction.
durable_turn_lease = _durable_holder
self._active_session_turn_lease_holder = _durable_holder
self._active_session_turn_lease_ttl_seconds = _lease_ttl
if _lease_waited:
self._emit_status(
"Session is free; loading the latest transcript..."
)
# The holder may have compressed/rotated the session while we waited: reload only AFTER
# admission, and skip when acquisition was immediate (avoids a needless prompt-cache miss).
if _lease_waited:
latest_session_id = _turn_db.resolve_resume_session_id(session_id)
if latest_session_id:
self.session_id = latest_session_id
task_context["session_id"] = latest_session_id
conversation_history = _turn_db.get_messages_as_conversation(
self.session_id,
repair_alternation=True,
include_row_ids=True,
)
# Long turns outlive a fixed TTL: refresh in a daemon thread; holder-qualified UPDATE/DELETE
# fence a late refresher from a successor lease.
durable_turn_lease_stop = threading.Event()
_lease_refresh_interval = float(
getattr(self, "_session_turn_lease_refresh_interval", 60.0)
)
# ── Turn liveness watchdog (#95548) ──
# Lease renewal is NOT evidence of progress; a silently stalled turn would renew forever.
# Policy
# lives in agent/turn_liveness.py; this block only wires config + commit/deactivate callbacks.
try:
from hermes_cli.config import (
load_config_readonly as _liveness_load_config,
)
_liveness_config = _liveness_load_config() or {}
except Exception:
_liveness_config = {}
from agent import turn_liveness
_liveness_timeout, _liveness_poll = (
turn_liveness.resolve_turn_liveness_settings(_liveness_config)
)
def _interrupt_turn(message: str) -> None:
# Lease-loss interrupts fire UNCONDITIONALLY (no generation claim): a lost lease means
# this process no longer owns the session. Only the watchdog's stalls can be spuriously
# stale.
nonlocal durable_turn_lease_interrupt_message
with durable_turn_lease_activity_lock:
if (
durable_turn_lease_stop.is_set()
or not durable_turn_lease_turn_active
):
return
durable_turn_lease_interrupt_message = message
try:
self.interrupt(message, hard_cancel=True)
except Exception:
self._interrupt_requested = True
self._interrupt_message = message
def _commit_turn_liveness_abort(
snapshot: "turn_liveness.ActivitySnapshot",
message: str,
) -> bool:
"""Commit point for the watchdog's stall observation.
Revalidates the observed ``(generation, timestamp)`` under the SAME lock
``_touch_activity`` uses, so a turn
that resumed while the stall was logged is never hard-cancelled. The revalidated
generation is carried into
``interrupt`` as ``require_generation``, which consumes it with the first publication in
ONE critical section.
If ``interrupt`` raises, the abort declines FAIL-CLOSED. Returns False when stale or
already winding down.
"""
nonlocal durable_turn_lease_interrupt_message
with self._liveness_activity_lock():
current_generation = getattr(
self, "_turn_liveness_activity_generation", 0
)
if (
current_generation,
getattr(self, "_last_activity_ts", None),
) != (snapshot.generation, snapshot.activity_ts):
return False
with durable_turn_lease_activity_lock:
if (
durable_turn_lease_stop.is_set()
or not durable_turn_lease_turn_active
):
return False
try:
published = self.interrupt(
message,
hard_cancel=True,
require_generation=current_generation,
)
except Exception:
# Round-4 (#95663): fail closed — an exceptional path must not turn an unvalidated
# claim into unconditional abort authority.
logger.debug(
"Turn liveness abort interrupt raised; "
"declining the abort",
exc_info=True,
)
published = False
if published is False:
# Claim went stale between revalidation and the hammer: real progress landed, abandon
# the abort.
return False
with durable_turn_lease_activity_lock:
durable_turn_lease_interrupt_message = message
return True
def _deactivate_turn_after_liveness_abort() -> None:
"""Stop lease renewal after a committed liveness abort.
A wedge the hard interrupt cannot unwind must not keep the lease alive forever; TTL expiry
lets stale-turn cleanup reclaim the row.
"""
nonlocal durable_turn_lease_turn_active
with durable_turn_lease_activity_lock:
durable_turn_lease_stop.set()
durable_turn_lease_turn_active = False
def _turn_is_active() -> bool:
with durable_turn_lease_activity_lock:
return durable_turn_lease_turn_active
def _refresh_durable_turn_lease() -> None:
while not durable_turn_lease_stop.wait(_lease_refresh_interval):
try:
if not _turn_db.refresh_session_turn_lease(
getattr(self, "session_id", None) or session_id,
durable_turn_lease,
ttl_seconds=_lease_ttl,
):
# finally sets stop then releases; a late holder-fenced miss must not hard-
# interrupt the next turn.
if durable_turn_lease_stop.is_set():
return
logger.error(
"Lost session turn lease while turn is active: %s",
getattr(self, "session_id", None) or session_id,
)
_interrupt_turn(
"Session turn lease lost; stopping to protect "
"the transcript."
)
return
except Exception:
if durable_turn_lease_stop.is_set():
return
logger.warning(
"Failed to refresh session turn lease: %s",
getattr(self, "session_id", None) or session_id,
exc_info=True,
)
_interrupt_turn(
"Session turn lease could not be refreshed; "
"stopping to protect the transcript."
)
return
durable_turn_lease_thread = threading.Thread(
target=_refresh_durable_turn_lease,
name="session-turn-lease-refresh",
daemon=True,
)
if _liveness_timeout is not None:
durable_turn_liveness_thread = (
turn_liveness.TurnLivenessWatchdog(
self,
session_id=getattr(self, "session_id", None) or session_id,
timeout_s=_liveness_timeout,
poll_s=_liveness_poll,
stop_event=durable_turn_lease_stop,
activity_lock=self._liveness_activity_lock(),
is_turn_active=_turn_is_active,
commit_abort=_commit_turn_liveness_abort,
deactivate_turn=_deactivate_turn_after_liveness_abort,
).make_thread()
)
return admission.early_result
lease = admission.lease
conversation_history = admission.conversation_history
relay_lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation(
profile_key=relay_runtime.current_profile_key(),
@@ -445,47 +104,30 @@ class TurnFacadeMixin:
model=str(getattr(self, "model", None) or ""),
)
relay_turn = relay_runtime.SESSION_COORDINATOR.begin_turn(
relay_lease,
turn_id=relay_turn_id,
task_id=effective_task_id,
relay_lease, turn_id=relay_turn_id, task_id=effective_task_id
)
# Keep existing tests and external relay-runtime shims that return
# a minimal turn object compatible with the new opt-out flag.
# Minimal relay-runtime shims (tests, external) may lack the opt-out flag: default enabled.
if getattr(relay_turn, "relay_enabled", True):
start_task_run(
**task_context,
parent_session_id=getattr(self, "_parent_session_id", None) or "",
)
task_started = True
# Publish the conversation id for ambient Nous Portal tagging: every LLM call in this turn (loop,
# compression, vision, MoA, review forks) inherits `conversation=<root>`.
# Publish the conversation id for ambient Nous Portal tagging: every LLM call in this turn
# (loop, compression, vision, MoA, review forks) inherits `conversation=<root>`.
token = set_conversation_context(self._conversation_root_id())
# Routing/affinity scope the HOST declared; providers fall back to the id above when unset
# (#96811).
affinity_token = set_affinity_scope(
declared_conversation_scope_safe(self)
)
# Publish session accounting handles so auxiliary calls record usage into session_model_usage
# (#23270).
# Routing/affinity scope the HOST declared; providers fall back to the id above when unset.
affinity_token = set_affinity_scope(declared_conversation_scope_safe(self))
# Session accounting handles so auxiliary calls record usage into session_model_usage.
acct_token = set_accounting_context(
getattr(self, "_session_db", None),
getattr(self, "session_id", None),
getattr(self, "_session_db", None), getattr(self, "session_id", None)
)
from agent.auxiliary_client import scoped_runtime_main
# Keep the ContextVar scope local (tokens on the agent may be observed from another thread).
with bind_subagent_parent(self), scoped_runtime_main({}):
try:
if durable_turn_lease_thread is not None:
with durable_turn_lease_activity_lock:
durable_turn_lease_turn_active = True
# Stamp the activity clock at turn entry (#95663): `_last_activity_ts` persists across
# turns, so without this the watchdog would measure idle from the PREVIOUS turn and
# abort a fresh one.
self._touch_activity("starting new turn")
durable_turn_lease_thread.start()
if durable_turn_liveness_thread is not None:
durable_turn_liveness_thread.start()
if lease is not None:
lease.start()
result = run_conversation(
self,
user_message,
@@ -501,10 +143,10 @@ class TurnFacadeMixin:
moa_config=moa_config,
)
finally:
# Post-loop relay/task finalization must not receive a late refresh interrupt.
_stop_durable_turn_lease_refresher()
# Interrupt clear is deferred until after thread join (outer finally) so a refresher
# firing between stop and join cannot leave an interrupt behind.
# Post-loop relay/task finalization must not receive a late refresh interrupt. The
# interrupt clear itself waits for the thread join in the outer finally.
if lease is not None:
lease.stop_refresher()
terminal = result if isinstance(result, dict) else {}
if terminal.get("interrupted") is True:
relay_outcome = "cancelled"
@@ -512,10 +154,7 @@ class TurnFacadeMixin:
relay_outcome = "failed"
else:
relay_outcome = "success"
relay_runtime.SESSION_COORDINATOR.finish_logical_calls(
relay_turn,
outcome=relay_outcome,
)
relay_runtime.SESSION_COORDINATOR.finish_logical_calls(relay_turn, outcome=relay_outcome)
if task_started:
task_finished = True
finish_task_run(**task_context, result=result)
@@ -529,8 +168,7 @@ class TurnFacadeMixin:
relay_outcome = "timed_out"
if relay_turn is not None:
relay_runtime.SESSION_COORDINATOR.finish_logical_calls(
relay_turn,
outcome=relay_outcome,
relay_turn, outcome=relay_outcome
)
if task_started and not task_finished:
task_finished = True
@@ -539,48 +177,20 @@ class TurnFacadeMixin:
finally:
try:
if relay_turn is not None:
relay_runtime.SESSION_COORDINATOR.end_turn(
relay_turn,
outcome=relay_outcome,
)
relay_runtime.SESSION_COORDINATOR.end_turn(relay_turn, outcome=relay_outcome)
finally:
try:
if relay_lease is not None:
relay_runtime.SESSION_COORDINATOR.release_conversation(
relay_lease
)
relay_runtime.SESSION_COORDINATOR.release_conversation(relay_lease)
finally:
_stop_durable_turn_lease_refresher()
for _durable_thread in (
durable_turn_lease_thread,
durable_turn_liveness_thread,
):
if (
_durable_thread is not None
and _durable_thread.is_alive()
):
_durable_thread.join(timeout=1.0)
# Clear any refresher interrupt fired between stop and join; must run AFTER join.
_clear_durable_turn_lease_interrupt()
if durable_turn_lease is not None:
try:
_turn_db.release_session_turn_lease(
session_id, durable_turn_lease
)
except Exception:
logger.error(
"Failed to release session turn lease: %s",
session_id,
exc_info=True,
)
if (
getattr(self, "_active_session_turn_lease_holder", None)
== durable_turn_lease
):
self._active_session_turn_lease_holder = None
self._active_session_turn_lease_ttl_seconds = None
# Always clear mid-turn labels when the turn exits — including
# interrupted early returns that skip finalize_turn. Keep ts.
if lease is not None:
lease.stop_refresher()
lease.join_threads()
# Clear any refresher interrupt fired between stop and join; must run AFTER join.
lease.clear_interrupt()
lease.release()
# Always clear mid-turn labels when the turn exits — including interrupted early
# returns that skip finalize_turn. Keep ts.
try:
self._reset_activity_labels_after_turn()
except Exception:
@@ -593,18 +203,15 @@ class TurnFacadeMixin:
reset_conversation_context(token)
if affinity_token is not None:
reset_affinity_scope(affinity_token)
# Balance note_turn_started on every exit so the idle queue's live-turn count cannot leak.
# Balance note_turn_started on every exit so the idle queue's live-turn count
# cannot leak.
try:
_review_queue.note_turn_finished()
except Exception:
pass
def chat(self, message: str, stream_callback: Optional[callable] = None) -> str:
"""Simple chat interface that returns just the final response string.
``stream_callback`` is invoked with each text delta during streaming.
"""
result = self.run_conversation(message, stream_callback=stream_callback)
return result["final_response"]
"""Final response string of one turn; ``stream_callback`` receives each text delta."""
return self.run_conversation(message, stream_callback=stream_callback)["final_response"]
_run_codex_app_server_turn = _forward("agent.codex_runtime", "run_codex_app_server_turn")
+390
View File
@@ -0,0 +1,390 @@
"""Durable cross-process session turn lease for ``TurnFacadeMixin.run_conversation``.
One process at a time may load -> run -> flush a session shared through state.db (Desktop, CLI
resume, gateway, background delivery). ``admit_durable_turn_lease`` acquires the row lease (or
returns the early result the façade must hand back); ``DurableTurnLease`` owns the refresher
daemon thread, the turn-liveness watchdog wiring, and the lease-loss / stall interrupt plumbing.
"""
import logging
import os
import threading
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
# Same logger name as the origin module so log records / caplog filters are unchanged.
logger = logging.getLogger("run_agent")
LEASE_TTL_SECONDS = 300.0
LEASE_WAIT_SECONDS = 1800.0
class DurableTurnLease:
"""An admitted session turn lease plus the threads that keep it alive and watch the turn.
``stop`` is shared by the refresher and the liveness watchdog; ``turn_active`` gates every
interrupt so a late refresher miss can never hard-interrupt the NEXT turn. Both are read and
written only under ``_lock``.
"""
def __init__(self, agent, db, session_id: str, holder: str) -> None:
self.agent = agent
self.db = db
self.session_id = session_id # id at admission; release always targets this row
self.holder = holder
self.stop = threading.Event()
self.refresh_interval = float(getattr(agent, "_session_turn_lease_refresh_interval", 60.0))
self._lock = threading.Lock()
self.turn_active = False
self.interrupt_message: Optional[str] = None
self.refresh_thread: Optional[threading.Thread] = None
self.liveness_thread: Optional[threading.Thread] = None
def _current_session_id(self) -> str:
return getattr(self.agent, "session_id", None) or self.session_id
# ── thread construction ──────────────────────────────────────────────────────
def build_threads(self) -> None:
"""Create (not start) the refresher thread and, when configured, the liveness watchdog.
Lease renewal is NOT evidence of progress; a silently stalled turn would renew forever, so the
watchdog (policy in ``agent/turn_liveness.py``) is wired with the commit/deactivate callbacks
below.
"""
self.refresh_thread = threading.Thread(
target=self.refresh_loop, name="session-turn-lease-refresh", daemon=True
)
try:
from hermes_cli.config import load_config_readonly
liveness_config = load_config_readonly() or {}
except Exception:
liveness_config = {}
from agent import turn_liveness
timeout_s, poll_s = turn_liveness.resolve_turn_liveness_settings(liveness_config)
if timeout_s is not None:
self.liveness_thread = turn_liveness.TurnLivenessWatchdog(
self.agent,
session_id=self._current_session_id(),
timeout_s=timeout_s,
poll_s=poll_s,
stop_event=self.stop,
activity_lock=self.agent._liveness_activity_lock(),
is_turn_active=self.is_turn_active,
commit_abort=self.commit_liveness_abort,
deactivate_turn=self.deactivate_after_liveness_abort,
).make_thread()
# ── lifecycle ────────────────────────────────────────────────────────────────
def start(self) -> None:
with self._lock:
self.turn_active = True
# Stamp the activity clock at turn entry: `_last_activity_ts` persists across turns, so without
# this the watchdog would measure idle from the PREVIOUS turn and abort a fresh one.
self.agent._touch_activity("starting new turn")
self.refresh_thread.start()
if self.liveness_thread is not None:
self.liveness_thread.start()
def stop_refresher(self) -> None:
with self._lock:
self.turn_active = False
self.stop.set()
def join_threads(self, timeout: float = 1.0) -> None:
for thread in (self.refresh_thread, self.liveness_thread):
if thread is not None and thread.is_alive():
thread.join(timeout=timeout)
def release(self) -> None:
"""Release the row and drop the agent's holder attrs (only if they still name this lease)."""
agent = self.agent
try:
self.db.release_session_turn_lease(self.session_id, self.holder)
except Exception:
logger.error("Failed to release session turn lease: %s", self.session_id, exc_info=True)
if getattr(agent, "_active_session_turn_lease_holder", None) == self.holder:
agent._active_session_turn_lease_holder = None
agent._active_session_turn_lease_ttl_seconds = None
def is_turn_active(self) -> bool:
with self._lock:
return self.turn_active
# ── interrupts ───────────────────────────────────────────────────────────────
def _interrupt_turn(self, message: str) -> None:
# Lease-loss interrupts fire UNCONDITIONALLY (no generation claim): a lost lease means this
# process no longer owns the session. Only the watchdog's stalls can be spuriously stale.
agent = self.agent
with self._lock:
if self.stop.is_set() or not self.turn_active:
return
self.interrupt_message = message
try:
agent.interrupt(message, hard_cancel=True)
except Exception:
agent._interrupt_requested = True
agent._interrupt_message = message
def commit_liveness_abort(self, snapshot, message: str) -> bool:
"""Commit point for the watchdog's stall observation.
Revalidates the observed ``(generation, timestamp)`` under the SAME lock ``_touch_activity``
uses, so a turn that resumed while the stall was logged is never hard-cancelled. The
revalidated generation is carried into ``interrupt`` as ``require_generation``, which consumes
it with the first publication in ONE critical section. If ``interrupt`` raises, the abort
declines FAIL-CLOSED. Returns False when stale or already winding down.
"""
agent = self.agent
with agent._liveness_activity_lock():
current_generation = getattr(agent, "_turn_liveness_activity_generation", 0)
if (current_generation, getattr(agent, "_last_activity_ts", None)) != (
snapshot.generation,
snapshot.activity_ts,
):
return False
with self._lock:
if self.stop.is_set() or not self.turn_active:
return False
try:
published = agent.interrupt(
message, hard_cancel=True, require_generation=current_generation
)
except Exception:
# Fail closed: an exceptional path must not turn an unvalidated claim into abort authority.
logger.debug(
"Turn liveness abort interrupt raised; declining the abort", exc_info=True
)
published = False
if published is False:
# Claim went stale between revalidation and the hammer: real progress landed.
return False
with self._lock:
self.interrupt_message = message
return True
def deactivate_after_liveness_abort(self) -> None:
"""Stop lease renewal after a committed liveness abort.
A wedge the hard interrupt cannot unwind must not keep the lease alive forever; TTL expiry
lets stale-turn cleanup reclaim the row.
"""
with self._lock:
self.stop.set()
self.turn_active = False
def clear_interrupt(self) -> None:
"""Clear only the interrupt admitted by this lease's refresher/watchdog. Run AFTER join."""
message = self.interrupt_message
if not message:
return
agent = self.agent
def _clear_if_owned() -> None:
# Lazy via the façade so ``patch("agent.turn_facade._set_interrupt")`` keeps intercepting.
from agent.turn_facade import _set_interrupt
if getattr(agent, "_interrupt_message", None) != message:
return
agent._interrupt_requested = False
agent._interrupt_message = None
getattr(agent, "_hard_interrupt_requested", threading.Event()).clear()
agent._interrupt_thread_signal_pending = False
if agent._execution_thread_id is not None:
_set_interrupt(False, agent._execution_thread_id)
redirect_lock = getattr(agent, "_pending_redirect_lock", None)
if redirect_lock is None:
_clear_if_owned()
else:
with redirect_lock:
_clear_if_owned()
# ── refresher thread body ────────────────────────────────────────────────────
def refresh_loop(self) -> None:
"""Renew the lease every ``refresh_interval``; a miss or error interrupts the turn.
Long turns outlive the fixed TTL; the holder-qualified UPDATE fences a late refresher from a
successor lease. The façade's finally sets ``stop`` before releasing, so a holder-fenced miss
observed after stop is not a loss.
"""
while not self.stop.wait(self.refresh_interval):
try:
if self.db.refresh_session_turn_lease(
self._current_session_id(), self.holder, ttl_seconds=LEASE_TTL_SECONDS
):
continue
if self.stop.is_set():
return
logger.error(
"Lost session turn lease while turn is active: %s", self._current_session_id()
)
self._interrupt_turn(
"Session turn lease lost; stopping to protect the transcript."
)
except Exception:
if self.stop.is_set():
return
logger.warning(
"Failed to refresh session turn lease: %s",
self._current_session_id(),
exc_info=True,
)
self._interrupt_turn(
"Session turn lease could not be refreshed; stopping to protect the transcript."
)
return
@dataclass
class TurnLeaseAdmission:
"""Outcome of ``admit_durable_turn_lease``: exactly one of ``lease`` / ``early_result`` may be set."""
lease: Optional[DurableTurnLease] = None
early_result: Optional[Dict[str, Any]] = None
conversation_history: Optional[List[Dict[str, Any]]] = None
def _durable_session_exists(db, session_id: str) -> bool:
try:
return db.get_session(session_id) is not None
except Exception:
# A locked / non-WAL read is not proof the row is absent; treating probe failure as "fresh"
# ran fail-open at the exact contention point. Acquire, or fail closed.
logger.warning(
"Could not check durable session before turn lease; "
"will acquire rather than run without serialization",
exc_info=True,
)
return True
def admit_durable_turn_lease(
agent,
*,
session_id: str,
relay_turn_id: str,
task_context: Dict[str, Any],
conversation_history: Optional[List[Dict[str, Any]]],
) -> TurnLeaseAdmission:
"""Acquire the session turn lease when the session is durable; build (not start) its threads.
Mutates ``task_context["session_id"]`` and ``agent.session_id`` when the wait forced a resume-id
reload. Returns an ``early_result`` (interrupted / timed out) instead of a lease when admission
fails; the caller returns it verbatim.
"""
db = getattr(agent, "_session_db", None)
admission = TurnLeaseAdmission(conversation_history=conversation_history)
if db is None or not session_id:
return admission
# A fresh session id has no durable transcript to race over, and callers may supply an in-memory
# seed before the row exists — reloading would erase it. Check the concrete type: MagicMock-style
# shims accept any attribute without the protocol.
exists = _durable_session_exists(db, session_id)
if (
getattr(agent, "_persist_disabled", False)
or not exists
or not callable(getattr(type(db), "acquire_session_turn_lease", None))
):
return admission
# Row proven to exist — suppress the redundant create attempt.
agent._session_db_created = True
holder = (
f"pid={os.getpid()}:turn={relay_turn_id}:platform={task_context['platform'] or 'unknown'}"
)
waited = False
def _on_wait(elapsed: float) -> None:
nonlocal waited
waited = True
if elapsed < 1.0:
agent._emit_status(
"⏳ Another Hermes process is using this session; "
"waiting for it to finish before starting your turn..."
)
else:
agent._emit_status(
"⏳ Still waiting for the other Hermes process on "
f"this session ({int(elapsed)}s)..."
)
if not db.acquire_session_turn_lease(
session_id,
holder,
ttl_seconds=LEASE_TTL_SECONDS,
wait_seconds=LEASE_WAIT_SECONDS,
on_wait=_on_wait,
should_abort=lambda: getattr(agent, "_interrupt_requested", False),
):
admission.early_result = _lease_not_acquired_result(agent, session_id, conversation_history)
return admission
# Assign only after admission so the finally cannot release a holder that never owned the row;
# persist paths read the agent attr so a late flush is fenced in the same SQLite transaction.
lease = DurableTurnLease(agent, db, session_id, holder)
agent._active_session_turn_lease_holder = holder
agent._active_session_turn_lease_ttl_seconds = LEASE_TTL_SECONDS
try:
if waited:
agent._emit_status("Session is free; loading the latest transcript...")
# The holder may have compressed/rotated the session while we waited: reload only AFTER
# admission, and skip when acquisition was immediate (avoids a needless prompt-cache miss).
latest_session_id = db.resolve_resume_session_id(session_id)
if latest_session_id:
agent.session_id = latest_session_id
task_context["session_id"] = latest_session_id
admission.conversation_history = db.get_messages_as_conversation(
agent.session_id, repair_alternation=True, include_row_ids=True
)
lease.build_threads()
except BaseException:
# The façade never saw this lease; release here so an admitted row is not leaked.
lease.release()
raise
admission.lease = lease
return admission
def _lease_not_acquired_result(agent, session_id: str, conversation_history) -> Dict[str, Any]:
base = {"messages": list(conversation_history or []), "api_calls": 0, "completed": False}
if getattr(agent, "_interrupt_requested", False):
logger.info("session turn lease wait aborted by interrupt: %s", session_id)
result = {
"final_response": (
"Stopped waiting for another Hermes process on this session. "
"Your message was not processed."
),
**base,
"interrupted": True,
}
interrupt_message = getattr(agent, "_interrupt_message", None)
if interrupt_message:
result["interrupt_message"] = interrupt_message
# The finalizer never runs on this early return; clear so a cached agent doesn't fail-close
# the next turn.
try:
agent.clear_interrupt()
except Exception:
agent._interrupt_requested = False
agent._interrupt_message = None
return result
# Fail closed like gateway TurnLeaseTimeoutError: surface a resend notice, not a bare TimeoutError.
timeout_msg = (
"⏳ Another Hermes process kept this session busy too long. Your message was not "
"processed - wait for the other process to finish, then send it again."
)
logger.error("session turn lease wait timed out for %s", session_id)
try:
agent._emit_warning(timeout_msg)
except Exception:
logger.debug("Failed to emit session turn lease timeout warning", exc_info=True)
return {
"final_response": timeout_msg,
**base,
"failed": True,
"error": f"session_turn_lease_timeout:{session_id}",
}
+124
View File
@@ -0,0 +1,124 @@
"""Unit tests for agent.turn_facade_lease (admission + lease bracket)."""
import threading
from types import SimpleNamespace
from agent.turn_facade_lease import (
LEASE_TTL_SECONDS,
DurableTurnLease,
admit_durable_turn_lease,
)
class _Db:
def __init__(self, exists=True, acquired=True):
self.exists = exists
self.acquired = acquired
self.events = []
def get_session(self, session_id):
return {"id": session_id} if self.exists else None
def acquire_session_turn_lease(self, session_id, holder, **kwargs):
self.events.append(("acquire", session_id, holder))
return self.acquired
def refresh_session_turn_lease(self, session_id, holder, **kwargs):
return True
def release_session_turn_lease(self, session_id, holder):
self.events.append(("release", session_id, holder))
def _agent(db, **overrides):
agent = SimpleNamespace(
_session_db=db,
session_id="s1",
_persist_disabled=False,
_interrupt_requested=False,
_interrupt_message=None,
_execution_thread_id=None,
_session_turn_lease_refresh_interval=60.0,
statuses=[],
)
agent._emit_status = agent.statuses.append
agent._emit_warning = agent.statuses.append
agent._touch_activity = lambda *a, **k: None
agent._liveness_activity_lock = lambda: threading.Lock()
for k, v in overrides.items():
setattr(agent, k, v)
return agent
def _admit(agent, history=None):
return admit_durable_turn_lease(
agent,
session_id="s1",
relay_turn_id="s1:t:abcd",
task_context={"session_id": "s1", "task_id": "t", "platform": "cli"},
conversation_history=history,
)
def test_no_lease_without_durable_row_or_when_persist_disabled():
seed = [{"role": "user", "content": "hi"}]
admission = _admit(_agent(_Db(exists=False)), seed)
assert admission.lease is None and admission.early_result is None
assert admission.conversation_history is seed
db = _Db()
admission = _admit(_agent(db, _persist_disabled=True), seed)
assert admission.lease is None and db.events == []
def test_admission_sets_holder_attrs_and_release_clears_them(monkeypatch):
monkeypatch.setattr(
"agent.turn_liveness.resolve_turn_liveness_settings", lambda cfg: (None, 1.0)
)
db = _Db()
agent = _agent(db)
admission = _admit(agent)
lease = admission.lease
assert isinstance(lease, DurableTurnLease)
assert agent._session_db_created is True
assert agent._active_session_turn_lease_holder == lease.holder
assert agent._active_session_turn_lease_ttl_seconds == LEASE_TTL_SECONDS
assert lease.holder.startswith("pid=") and ":platform=cli" in lease.holder
assert lease.refresh_thread is not None and lease.liveness_thread is None
assert lease.is_turn_active() is False
lease.stop_refresher()
lease.join_threads()
lease.clear_interrupt()
lease.release()
assert db.events == [("acquire", "s1", lease.holder), ("release", "s1", lease.holder)]
assert agent._active_session_turn_lease_holder is None
assert agent._active_session_turn_lease_ttl_seconds is None
def test_timeout_and_interrupt_early_results():
agent = _agent(_Db(acquired=False))
admission = _admit(agent, [{"role": "user", "content": "x"}])
assert admission.lease is None
assert admission.early_result["failed"] is True
assert admission.early_result["error"] == "session_turn_lease_timeout:s1"
assert admission.early_result["messages"] == [{"role": "user", "content": "x"}]
agent = _agent(_Db(acquired=False), _interrupt_requested=True, _interrupt_message="stop")
agent.clear_interrupt = lambda: None
admission = _admit(agent)
assert admission.early_result["interrupted"] is True
assert admission.early_result["interrupt_message"] == "stop"
def test_interrupt_turn_only_while_active():
agent = _agent(_Db())
calls = []
agent.interrupt = lambda msg, **kw: calls.append(msg)
lease = DurableTurnLease(agent, agent._session_db, "s1", "h")
lease._interrupt_turn("lost") # inactive: ignored
assert calls == [] and lease.interrupt_message is None
lease.turn_active = True
lease._interrupt_turn("lost")
assert calls == ["lost"] and lease.interrupt_message == "lost"
lease.deactivate_after_liveness_abort()
assert lease.stop.is_set() and lease.is_turn_active() is False