refactor(turn): lift run_conversation lease/liveness closures into agent.turn_facade_lease
This commit is contained in:
+62
-455
@@ -1,17 +1,16 @@
|
||||
"""``AIAgent.run_conversation`` / ``chat`` façade.
|
||||
|
||||
Turn admission around ``conversation_loop.run_conversation``: durable cross-process session turn lease +
|
||||
refresher thread, turn-liveness watchdog, relay/accounting/portal scopes, and balanced start/finish marks.
|
||||
Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s MRO unchanged.
|
||||
refresher thread and liveness watchdog (``agent.turn_facade_lease``), relay/accounting/portal scopes, and
|
||||
balanced start/finish marks. Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s
|
||||
MRO unchanged.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from agent.lazy_forward import forward as _forward
|
||||
from tools.interrupt import set_interrupt as _set_interrupt
|
||||
from tools.interrupt import set_interrupt as _set_interrupt # noqa: F401 (tests patch it here)
|
||||
|
||||
# Same logger name as the origin module so log records / caplog filters are unchanged.
|
||||
logger = logging.getLogger("run_agent")
|
||||
@@ -36,20 +35,14 @@ class TurnFacadeMixin:
|
||||
) -> Dict[str, Any]:
|
||||
"""Forwarder — see ``agent.conversation_loop.run_conversation``."""
|
||||
# A review shares this session_id for cache parity: fence review startup or interrupt an admitted
|
||||
# request and await its exit before opening live-turn instrumentation (#84423).
|
||||
# request and await its exit before opening live-turn instrumentation.
|
||||
from agent.background_review import cancel_background_review_for_live_turn
|
||||
|
||||
cancel_background_review_for_live_turn(self)
|
||||
|
||||
# Turn liveness for the deferred-review idle queue; start-mark is the first statement of the try
|
||||
# so the finally's note_turn_finished balances every exit.
|
||||
from agent.review_idle_queue import QUEUE as _review_queue
|
||||
|
||||
from agent.aux_accounting import (
|
||||
reset_accounting_context,
|
||||
set_accounting_context,
|
||||
)
|
||||
from agent import relay_runtime
|
||||
from agent.aux_accounting import reset_accounting_context, set_accounting_context
|
||||
from agent.auxiliary_client import scoped_runtime_main
|
||||
from agent.conversation_loop import run_conversation
|
||||
from agent.portal_tags import (
|
||||
reset_affinity_scope,
|
||||
@@ -58,11 +51,11 @@ class TurnFacadeMixin:
|
||||
set_conversation_context,
|
||||
)
|
||||
from agent.prompt_cache_scope import declared_conversation_scope_safe
|
||||
from hermes_cli.observability.relay_shared_metrics import (
|
||||
finish_task_run,
|
||||
start_task_run,
|
||||
)
|
||||
from agent.review_idle_queue import QUEUE as _review_queue
|
||||
from agent.subagent_lifecycle import bind_subagent_parent
|
||||
from agent.turn_facade_lease import admit_durable_turn_lease
|
||||
from hermes_cli.observability.relay_shared_metrics import finish_task_run, start_task_run
|
||||
|
||||
effective_task_id = task_id or str(uuid.uuid4())
|
||||
session_id = str(getattr(self, "session_id", None) or "")
|
||||
task_context = {
|
||||
@@ -70,372 +63,38 @@ class TurnFacadeMixin:
|
||||
"task_id": effective_task_id,
|
||||
"platform": getattr(self, "platform", None) or "",
|
||||
}
|
||||
relay_turn_id = (
|
||||
f"{session_id or 'session'}:{effective_task_id}:{uuid.uuid4().hex[:8]}"
|
||||
)
|
||||
relay_turn_id = f"{session_id or 'session'}:{effective_task_id}:{uuid.uuid4().hex[:8]}"
|
||||
self._relay_pending_turn_id = relay_turn_id
|
||||
relay_parent_session_id = (
|
||||
str(getattr(self, "_parent_session_id", None) or "")
|
||||
if task_context["platform"] == "subagent"
|
||||
else ""
|
||||
)
|
||||
relay_lease = None
|
||||
relay_turn = None
|
||||
durable_turn_lease = None
|
||||
durable_turn_lease_stop = None
|
||||
durable_turn_lease_thread = None
|
||||
durable_turn_liveness_thread = None
|
||||
durable_turn_lease_activity_lock = threading.Lock()
|
||||
durable_turn_lease_turn_active = False
|
||||
durable_turn_lease_interrupt_message = None
|
||||
token = None
|
||||
# Initialized alongside `token`: early returns leave the try before set_affinity_scope() and the
|
||||
# finally reads this unconditionally (PR #97158).
|
||||
affinity_token = None
|
||||
acct_token = None
|
||||
task_started = False
|
||||
task_finished = False
|
||||
relay_lease = relay_turn = lease = None
|
||||
# Scope tokens start None: early returns leave the try before the set_*() calls and the finally
|
||||
# resets each one unconditionally.
|
||||
token = affinity_token = acct_token = None
|
||||
task_started = task_finished = False
|
||||
relay_outcome = "failed"
|
||||
|
||||
def _stop_durable_turn_lease_refresher() -> None:
|
||||
nonlocal durable_turn_lease_turn_active
|
||||
with durable_turn_lease_activity_lock:
|
||||
durable_turn_lease_turn_active = False
|
||||
if durable_turn_lease_stop is not None:
|
||||
durable_turn_lease_stop.set()
|
||||
|
||||
def _clear_durable_turn_lease_interrupt() -> None:
|
||||
"""Clear only the interrupt admitted by this turn's refresher."""
|
||||
message = durable_turn_lease_interrupt_message
|
||||
if not message:
|
||||
return
|
||||
|
||||
def _clear_if_owned() -> None:
|
||||
if getattr(self, "_interrupt_message", None) != message:
|
||||
return
|
||||
self._interrupt_requested = False
|
||||
self._interrupt_message = None
|
||||
getattr(self, "_hard_interrupt_requested", threading.Event()).clear()
|
||||
self._interrupt_thread_signal_pending = False
|
||||
if self._execution_thread_id is not None:
|
||||
_set_interrupt(False, self._execution_thread_id)
|
||||
|
||||
redirect_lock = getattr(self, "_pending_redirect_lock", None)
|
||||
if redirect_lock is None:
|
||||
_clear_if_owned()
|
||||
else:
|
||||
with redirect_lock:
|
||||
_clear_if_owned()
|
||||
|
||||
try:
|
||||
# Turn liveness for the deferred-review idle queue: first statement of the try so the finally's
|
||||
# note_turn_finished balances every exit.
|
||||
_review_queue.note_turn_started()
|
||||
# Durable cross-process lease over load -> run -> flush (Desktop, CLI resume, gateway, background
|
||||
# delivery sharing state.db, #84234).
|
||||
_turn_db = getattr(self, "_session_db", None)
|
||||
_durable_session_exists = False
|
||||
if _turn_db is not None and session_id:
|
||||
try:
|
||||
_durable_session_exists = _turn_db.get_session(session_id) is not None
|
||||
except Exception:
|
||||
# A locked / non-WAL read is not proof the row is absent; treating probe failure as
|
||||
# "fresh" ran fail-open at the exact contention point (#84234). Acquire, or fail closed.
|
||||
logger.warning(
|
||||
"Could not check durable session before turn lease; "
|
||||
"will acquire rather than run without serialization",
|
||||
exc_info=True,
|
||||
)
|
||||
_durable_session_exists = True
|
||||
if (
|
||||
_turn_db is not None
|
||||
and session_id
|
||||
and not getattr(self, "_persist_disabled", False)
|
||||
# A fresh session id has no durable transcript to race over, and callers may supply an in-
|
||||
# memory seed before the row exists — reloading would erase it.
|
||||
and _durable_session_exists
|
||||
# Check the concrete type: MagicMock-style shims accept any attribute without the protocol.
|
||||
and callable(
|
||||
getattr(type(_turn_db), "acquire_session_turn_lease", None)
|
||||
admission = admit_durable_turn_lease(
|
||||
self,
|
||||
session_id=session_id,
|
||||
relay_turn_id=relay_turn_id,
|
||||
task_context=task_context,
|
||||
conversation_history=conversation_history,
|
||||
)
|
||||
if admission.early_result is not None:
|
||||
relay_outcome = (
|
||||
"cancelled" if admission.early_result.get("interrupted") else "timed_out"
|
||||
)
|
||||
):
|
||||
# Row proven to exist — suppress the redundant create attempt.
|
||||
self._session_db_created = True
|
||||
_durable_holder = (
|
||||
f"pid={os.getpid()}:turn={relay_turn_id}:platform="
|
||||
f"{task_context['platform'] or 'unknown'}"
|
||||
)
|
||||
_lease_ttl = 300.0
|
||||
_lease_waited = False
|
||||
|
||||
def _on_session_turn_lease_wait(elapsed: float) -> None:
|
||||
nonlocal _lease_waited
|
||||
_lease_waited = True
|
||||
if elapsed < 1.0:
|
||||
self._emit_status(
|
||||
"⏳ Another Hermes process is using this session; "
|
||||
"waiting for it to finish before starting your turn..."
|
||||
)
|
||||
else:
|
||||
self._emit_status(
|
||||
"⏳ Still waiting for the other Hermes process on "
|
||||
f"this session ({int(elapsed)}s)..."
|
||||
)
|
||||
|
||||
if not _turn_db.acquire_session_turn_lease(
|
||||
session_id,
|
||||
_durable_holder,
|
||||
ttl_seconds=_lease_ttl,
|
||||
wait_seconds=1800.0,
|
||||
on_wait=_on_session_turn_lease_wait,
|
||||
should_abort=lambda: getattr(self, "_interrupt_requested", False),
|
||||
):
|
||||
if getattr(self, "_interrupt_requested", False):
|
||||
logger.info(
|
||||
"session turn lease wait aborted by interrupt: %s",
|
||||
session_id,
|
||||
)
|
||||
relay_outcome = "cancelled"
|
||||
interrupt_msg = (
|
||||
"Stopped waiting for another Hermes process on "
|
||||
"this session. Your message was not processed."
|
||||
)
|
||||
interrupt_result = {
|
||||
"final_response": interrupt_msg,
|
||||
"messages": list(conversation_history or []),
|
||||
"api_calls": 0,
|
||||
"completed": False,
|
||||
"interrupted": True,
|
||||
}
|
||||
interrupt_message = getattr(
|
||||
self, "_interrupt_message", None
|
||||
)
|
||||
if interrupt_message:
|
||||
interrupt_result["interrupt_message"] = (
|
||||
interrupt_message
|
||||
)
|
||||
# The finalizer never runs on this early return; clear so a cached agent doesn't fail-
|
||||
# close the next turn.
|
||||
try:
|
||||
self.clear_interrupt()
|
||||
except Exception:
|
||||
self._interrupt_requested = False
|
||||
self._interrupt_message = None
|
||||
return interrupt_result
|
||||
# Fail closed like gateway TurnLeaseTimeoutError: surface a resend notice, not a bare
|
||||
# TimeoutError.
|
||||
timeout_msg = (
|
||||
"⏳ Another Hermes process kept this session busy too "
|
||||
"long. Your message was not processed - wait for the "
|
||||
"other process to finish, then send it again."
|
||||
)
|
||||
logger.error(
|
||||
"session turn lease wait timed out for %s",
|
||||
session_id,
|
||||
)
|
||||
try:
|
||||
self._emit_warning(timeout_msg)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Failed to emit session turn lease timeout warning",
|
||||
exc_info=True,
|
||||
)
|
||||
relay_outcome = "timed_out"
|
||||
return {
|
||||
"final_response": timeout_msg,
|
||||
"messages": list(conversation_history or []),
|
||||
"api_calls": 0,
|
||||
"completed": False,
|
||||
"failed": True,
|
||||
"error": f"session_turn_lease_timeout:{session_id}",
|
||||
}
|
||||
|
||||
# Assign only after admission so the finally cannot release a holder that never owned the row;
|
||||
# persist paths read the agent attr so a late flush is fenced in the same SQLite transaction.
|
||||
durable_turn_lease = _durable_holder
|
||||
self._active_session_turn_lease_holder = _durable_holder
|
||||
self._active_session_turn_lease_ttl_seconds = _lease_ttl
|
||||
if _lease_waited:
|
||||
self._emit_status(
|
||||
"Session is free; loading the latest transcript..."
|
||||
)
|
||||
|
||||
# The holder may have compressed/rotated the session while we waited: reload only AFTER
|
||||
# admission, and skip when acquisition was immediate (avoids a needless prompt-cache miss).
|
||||
if _lease_waited:
|
||||
latest_session_id = _turn_db.resolve_resume_session_id(session_id)
|
||||
if latest_session_id:
|
||||
self.session_id = latest_session_id
|
||||
task_context["session_id"] = latest_session_id
|
||||
conversation_history = _turn_db.get_messages_as_conversation(
|
||||
self.session_id,
|
||||
repair_alternation=True,
|
||||
include_row_ids=True,
|
||||
)
|
||||
|
||||
# Long turns outlive a fixed TTL: refresh in a daemon thread; holder-qualified UPDATE/DELETE
|
||||
# fence a late refresher from a successor lease.
|
||||
durable_turn_lease_stop = threading.Event()
|
||||
_lease_refresh_interval = float(
|
||||
getattr(self, "_session_turn_lease_refresh_interval", 60.0)
|
||||
)
|
||||
|
||||
# ── Turn liveness watchdog (#95548) ──
|
||||
# Lease renewal is NOT evidence of progress; a silently stalled turn would renew forever.
|
||||
# Policy
|
||||
# lives in agent/turn_liveness.py; this block only wires config + commit/deactivate callbacks.
|
||||
try:
|
||||
from hermes_cli.config import (
|
||||
load_config_readonly as _liveness_load_config,
|
||||
)
|
||||
_liveness_config = _liveness_load_config() or {}
|
||||
except Exception:
|
||||
_liveness_config = {}
|
||||
from agent import turn_liveness
|
||||
|
||||
_liveness_timeout, _liveness_poll = (
|
||||
turn_liveness.resolve_turn_liveness_settings(_liveness_config)
|
||||
)
|
||||
|
||||
def _interrupt_turn(message: str) -> None:
|
||||
# Lease-loss interrupts fire UNCONDITIONALLY (no generation claim): a lost lease means
|
||||
# this process no longer owns the session. Only the watchdog's stalls can be spuriously
|
||||
# stale.
|
||||
nonlocal durable_turn_lease_interrupt_message
|
||||
with durable_turn_lease_activity_lock:
|
||||
if (
|
||||
durable_turn_lease_stop.is_set()
|
||||
or not durable_turn_lease_turn_active
|
||||
):
|
||||
return
|
||||
durable_turn_lease_interrupt_message = message
|
||||
try:
|
||||
self.interrupt(message, hard_cancel=True)
|
||||
except Exception:
|
||||
self._interrupt_requested = True
|
||||
self._interrupt_message = message
|
||||
|
||||
def _commit_turn_liveness_abort(
|
||||
snapshot: "turn_liveness.ActivitySnapshot",
|
||||
message: str,
|
||||
) -> bool:
|
||||
"""Commit point for the watchdog's stall observation.
|
||||
|
||||
Revalidates the observed ``(generation, timestamp)`` under the SAME lock
|
||||
``_touch_activity`` uses, so a turn
|
||||
that resumed while the stall was logged is never hard-cancelled. The revalidated
|
||||
generation is carried into
|
||||
``interrupt`` as ``require_generation``, which consumes it with the first publication in
|
||||
ONE critical section.
|
||||
If ``interrupt`` raises, the abort declines FAIL-CLOSED. Returns False when stale or
|
||||
already winding down.
|
||||
"""
|
||||
nonlocal durable_turn_lease_interrupt_message
|
||||
with self._liveness_activity_lock():
|
||||
current_generation = getattr(
|
||||
self, "_turn_liveness_activity_generation", 0
|
||||
)
|
||||
if (
|
||||
current_generation,
|
||||
getattr(self, "_last_activity_ts", None),
|
||||
) != (snapshot.generation, snapshot.activity_ts):
|
||||
return False
|
||||
with durable_turn_lease_activity_lock:
|
||||
if (
|
||||
durable_turn_lease_stop.is_set()
|
||||
or not durable_turn_lease_turn_active
|
||||
):
|
||||
return False
|
||||
try:
|
||||
published = self.interrupt(
|
||||
message,
|
||||
hard_cancel=True,
|
||||
require_generation=current_generation,
|
||||
)
|
||||
except Exception:
|
||||
# Round-4 (#95663): fail closed — an exceptional path must not turn an unvalidated
|
||||
# claim into unconditional abort authority.
|
||||
logger.debug(
|
||||
"Turn liveness abort interrupt raised; "
|
||||
"declining the abort",
|
||||
exc_info=True,
|
||||
)
|
||||
published = False
|
||||
if published is False:
|
||||
# Claim went stale between revalidation and the hammer: real progress landed, abandon
|
||||
# the abort.
|
||||
return False
|
||||
with durable_turn_lease_activity_lock:
|
||||
durable_turn_lease_interrupt_message = message
|
||||
return True
|
||||
|
||||
def _deactivate_turn_after_liveness_abort() -> None:
|
||||
"""Stop lease renewal after a committed liveness abort.
|
||||
|
||||
A wedge the hard interrupt cannot unwind must not keep the lease alive forever; TTL expiry
|
||||
lets stale-turn cleanup reclaim the row.
|
||||
"""
|
||||
nonlocal durable_turn_lease_turn_active
|
||||
with durable_turn_lease_activity_lock:
|
||||
durable_turn_lease_stop.set()
|
||||
durable_turn_lease_turn_active = False
|
||||
|
||||
def _turn_is_active() -> bool:
|
||||
with durable_turn_lease_activity_lock:
|
||||
return durable_turn_lease_turn_active
|
||||
|
||||
def _refresh_durable_turn_lease() -> None:
|
||||
while not durable_turn_lease_stop.wait(_lease_refresh_interval):
|
||||
try:
|
||||
if not _turn_db.refresh_session_turn_lease(
|
||||
getattr(self, "session_id", None) or session_id,
|
||||
durable_turn_lease,
|
||||
ttl_seconds=_lease_ttl,
|
||||
):
|
||||
# finally sets stop then releases; a late holder-fenced miss must not hard-
|
||||
# interrupt the next turn.
|
||||
if durable_turn_lease_stop.is_set():
|
||||
return
|
||||
logger.error(
|
||||
"Lost session turn lease while turn is active: %s",
|
||||
getattr(self, "session_id", None) or session_id,
|
||||
)
|
||||
_interrupt_turn(
|
||||
"Session turn lease lost; stopping to protect "
|
||||
"the transcript."
|
||||
)
|
||||
return
|
||||
except Exception:
|
||||
if durable_turn_lease_stop.is_set():
|
||||
return
|
||||
logger.warning(
|
||||
"Failed to refresh session turn lease: %s",
|
||||
getattr(self, "session_id", None) or session_id,
|
||||
exc_info=True,
|
||||
)
|
||||
_interrupt_turn(
|
||||
"Session turn lease could not be refreshed; "
|
||||
"stopping to protect the transcript."
|
||||
)
|
||||
return
|
||||
|
||||
durable_turn_lease_thread = threading.Thread(
|
||||
target=_refresh_durable_turn_lease,
|
||||
name="session-turn-lease-refresh",
|
||||
daemon=True,
|
||||
)
|
||||
if _liveness_timeout is not None:
|
||||
durable_turn_liveness_thread = (
|
||||
turn_liveness.TurnLivenessWatchdog(
|
||||
self,
|
||||
session_id=getattr(self, "session_id", None) or session_id,
|
||||
timeout_s=_liveness_timeout,
|
||||
poll_s=_liveness_poll,
|
||||
stop_event=durable_turn_lease_stop,
|
||||
activity_lock=self._liveness_activity_lock(),
|
||||
is_turn_active=_turn_is_active,
|
||||
commit_abort=_commit_turn_liveness_abort,
|
||||
deactivate_turn=_deactivate_turn_after_liveness_abort,
|
||||
).make_thread()
|
||||
)
|
||||
return admission.early_result
|
||||
lease = admission.lease
|
||||
conversation_history = admission.conversation_history
|
||||
|
||||
relay_lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation(
|
||||
profile_key=relay_runtime.current_profile_key(),
|
||||
@@ -445,47 +104,30 @@ class TurnFacadeMixin:
|
||||
model=str(getattr(self, "model", None) or ""),
|
||||
)
|
||||
relay_turn = relay_runtime.SESSION_COORDINATOR.begin_turn(
|
||||
relay_lease,
|
||||
turn_id=relay_turn_id,
|
||||
task_id=effective_task_id,
|
||||
relay_lease, turn_id=relay_turn_id, task_id=effective_task_id
|
||||
)
|
||||
# Keep existing tests and external relay-runtime shims that return
|
||||
# a minimal turn object compatible with the new opt-out flag.
|
||||
# Minimal relay-runtime shims (tests, external) may lack the opt-out flag: default enabled.
|
||||
if getattr(relay_turn, "relay_enabled", True):
|
||||
start_task_run(
|
||||
**task_context,
|
||||
parent_session_id=getattr(self, "_parent_session_id", None) or "",
|
||||
)
|
||||
task_started = True
|
||||
# Publish the conversation id for ambient Nous Portal tagging: every LLM call in this turn (loop,
|
||||
# compression, vision, MoA, review forks) inherits `conversation=<root>`.
|
||||
# Publish the conversation id for ambient Nous Portal tagging: every LLM call in this turn
|
||||
# (loop, compression, vision, MoA, review forks) inherits `conversation=<root>`.
|
||||
token = set_conversation_context(self._conversation_root_id())
|
||||
# Routing/affinity scope the HOST declared; providers fall back to the id above when unset
|
||||
# (#96811).
|
||||
affinity_token = set_affinity_scope(
|
||||
declared_conversation_scope_safe(self)
|
||||
)
|
||||
# Publish session accounting handles so auxiliary calls record usage into session_model_usage
|
||||
# (#23270).
|
||||
# Routing/affinity scope the HOST declared; providers fall back to the id above when unset.
|
||||
affinity_token = set_affinity_scope(declared_conversation_scope_safe(self))
|
||||
# Session accounting handles so auxiliary calls record usage into session_model_usage.
|
||||
acct_token = set_accounting_context(
|
||||
getattr(self, "_session_db", None),
|
||||
getattr(self, "session_id", None),
|
||||
getattr(self, "_session_db", None), getattr(self, "session_id", None)
|
||||
)
|
||||
from agent.auxiliary_client import scoped_runtime_main
|
||||
|
||||
# Keep the ContextVar scope local (tokens on the agent may be observed from another thread).
|
||||
with bind_subagent_parent(self), scoped_runtime_main({}):
|
||||
try:
|
||||
if durable_turn_lease_thread is not None:
|
||||
with durable_turn_lease_activity_lock:
|
||||
durable_turn_lease_turn_active = True
|
||||
# Stamp the activity clock at turn entry (#95663): `_last_activity_ts` persists across
|
||||
# turns, so without this the watchdog would measure idle from the PREVIOUS turn and
|
||||
# abort a fresh one.
|
||||
self._touch_activity("starting new turn")
|
||||
durable_turn_lease_thread.start()
|
||||
if durable_turn_liveness_thread is not None:
|
||||
durable_turn_liveness_thread.start()
|
||||
if lease is not None:
|
||||
lease.start()
|
||||
result = run_conversation(
|
||||
self,
|
||||
user_message,
|
||||
@@ -501,10 +143,10 @@ class TurnFacadeMixin:
|
||||
moa_config=moa_config,
|
||||
)
|
||||
finally:
|
||||
# Post-loop relay/task finalization must not receive a late refresh interrupt.
|
||||
_stop_durable_turn_lease_refresher()
|
||||
# Interrupt clear is deferred until after thread join (outer finally) so a refresher
|
||||
# firing between stop and join cannot leave an interrupt behind.
|
||||
# Post-loop relay/task finalization must not receive a late refresh interrupt. The
|
||||
# interrupt clear itself waits for the thread join in the outer finally.
|
||||
if lease is not None:
|
||||
lease.stop_refresher()
|
||||
terminal = result if isinstance(result, dict) else {}
|
||||
if terminal.get("interrupted") is True:
|
||||
relay_outcome = "cancelled"
|
||||
@@ -512,10 +154,7 @@ class TurnFacadeMixin:
|
||||
relay_outcome = "failed"
|
||||
else:
|
||||
relay_outcome = "success"
|
||||
relay_runtime.SESSION_COORDINATOR.finish_logical_calls(
|
||||
relay_turn,
|
||||
outcome=relay_outcome,
|
||||
)
|
||||
relay_runtime.SESSION_COORDINATOR.finish_logical_calls(relay_turn, outcome=relay_outcome)
|
||||
if task_started:
|
||||
task_finished = True
|
||||
finish_task_run(**task_context, result=result)
|
||||
@@ -529,8 +168,7 @@ class TurnFacadeMixin:
|
||||
relay_outcome = "timed_out"
|
||||
if relay_turn is not None:
|
||||
relay_runtime.SESSION_COORDINATOR.finish_logical_calls(
|
||||
relay_turn,
|
||||
outcome=relay_outcome,
|
||||
relay_turn, outcome=relay_outcome
|
||||
)
|
||||
if task_started and not task_finished:
|
||||
task_finished = True
|
||||
@@ -539,48 +177,20 @@ class TurnFacadeMixin:
|
||||
finally:
|
||||
try:
|
||||
if relay_turn is not None:
|
||||
relay_runtime.SESSION_COORDINATOR.end_turn(
|
||||
relay_turn,
|
||||
outcome=relay_outcome,
|
||||
)
|
||||
relay_runtime.SESSION_COORDINATOR.end_turn(relay_turn, outcome=relay_outcome)
|
||||
finally:
|
||||
try:
|
||||
if relay_lease is not None:
|
||||
relay_runtime.SESSION_COORDINATOR.release_conversation(
|
||||
relay_lease
|
||||
)
|
||||
relay_runtime.SESSION_COORDINATOR.release_conversation(relay_lease)
|
||||
finally:
|
||||
_stop_durable_turn_lease_refresher()
|
||||
for _durable_thread in (
|
||||
durable_turn_lease_thread,
|
||||
durable_turn_liveness_thread,
|
||||
):
|
||||
if (
|
||||
_durable_thread is not None
|
||||
and _durable_thread.is_alive()
|
||||
):
|
||||
_durable_thread.join(timeout=1.0)
|
||||
# Clear any refresher interrupt fired between stop and join; must run AFTER join.
|
||||
_clear_durable_turn_lease_interrupt()
|
||||
if durable_turn_lease is not None:
|
||||
try:
|
||||
_turn_db.release_session_turn_lease(
|
||||
session_id, durable_turn_lease
|
||||
)
|
||||
except Exception:
|
||||
logger.error(
|
||||
"Failed to release session turn lease: %s",
|
||||
session_id,
|
||||
exc_info=True,
|
||||
)
|
||||
if (
|
||||
getattr(self, "_active_session_turn_lease_holder", None)
|
||||
== durable_turn_lease
|
||||
):
|
||||
self._active_session_turn_lease_holder = None
|
||||
self._active_session_turn_lease_ttl_seconds = None
|
||||
# Always clear mid-turn labels when the turn exits — including
|
||||
# interrupted early returns that skip finalize_turn. Keep ts.
|
||||
if lease is not None:
|
||||
lease.stop_refresher()
|
||||
lease.join_threads()
|
||||
# Clear any refresher interrupt fired between stop and join; must run AFTER join.
|
||||
lease.clear_interrupt()
|
||||
lease.release()
|
||||
# Always clear mid-turn labels when the turn exits — including interrupted early
|
||||
# returns that skip finalize_turn. Keep ts.
|
||||
try:
|
||||
self._reset_activity_labels_after_turn()
|
||||
except Exception:
|
||||
@@ -593,18 +203,15 @@ class TurnFacadeMixin:
|
||||
reset_conversation_context(token)
|
||||
if affinity_token is not None:
|
||||
reset_affinity_scope(affinity_token)
|
||||
# Balance note_turn_started on every exit so the idle queue's live-turn count cannot leak.
|
||||
# Balance note_turn_started on every exit so the idle queue's live-turn count
|
||||
# cannot leak.
|
||||
try:
|
||||
_review_queue.note_turn_finished()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def chat(self, message: str, stream_callback: Optional[callable] = None) -> str:
|
||||
"""Simple chat interface that returns just the final response string.
|
||||
|
||||
``stream_callback`` is invoked with each text delta during streaming.
|
||||
"""
|
||||
result = self.run_conversation(message, stream_callback=stream_callback)
|
||||
return result["final_response"]
|
||||
"""Final response string of one turn; ``stream_callback`` receives each text delta."""
|
||||
return self.run_conversation(message, stream_callback=stream_callback)["final_response"]
|
||||
|
||||
_run_codex_app_server_turn = _forward("agent.codex_runtime", "run_codex_app_server_turn")
|
||||
|
||||
@@ -0,0 +1,390 @@
|
||||
"""Durable cross-process session turn lease for ``TurnFacadeMixin.run_conversation``.
|
||||
|
||||
One process at a time may load -> run -> flush a session shared through state.db (Desktop, CLI
|
||||
resume, gateway, background delivery). ``admit_durable_turn_lease`` acquires the row lease (or
|
||||
returns the early result the façade must hand back); ``DurableTurnLease`` owns the refresher
|
||||
daemon thread, the turn-liveness watchdog wiring, and the lease-loss / stall interrupt plumbing.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
# Same logger name as the origin module so log records / caplog filters are unchanged.
|
||||
logger = logging.getLogger("run_agent")
|
||||
|
||||
LEASE_TTL_SECONDS = 300.0
|
||||
LEASE_WAIT_SECONDS = 1800.0
|
||||
|
||||
|
||||
class DurableTurnLease:
|
||||
"""An admitted session turn lease plus the threads that keep it alive and watch the turn.
|
||||
|
||||
``stop`` is shared by the refresher and the liveness watchdog; ``turn_active`` gates every
|
||||
interrupt so a late refresher miss can never hard-interrupt the NEXT turn. Both are read and
|
||||
written only under ``_lock``.
|
||||
"""
|
||||
|
||||
def __init__(self, agent, db, session_id: str, holder: str) -> None:
|
||||
self.agent = agent
|
||||
self.db = db
|
||||
self.session_id = session_id # id at admission; release always targets this row
|
||||
self.holder = holder
|
||||
self.stop = threading.Event()
|
||||
self.refresh_interval = float(getattr(agent, "_session_turn_lease_refresh_interval", 60.0))
|
||||
self._lock = threading.Lock()
|
||||
self.turn_active = False
|
||||
self.interrupt_message: Optional[str] = None
|
||||
self.refresh_thread: Optional[threading.Thread] = None
|
||||
self.liveness_thread: Optional[threading.Thread] = None
|
||||
|
||||
def _current_session_id(self) -> str:
|
||||
return getattr(self.agent, "session_id", None) or self.session_id
|
||||
|
||||
# ── thread construction ──────────────────────────────────────────────────────
|
||||
|
||||
def build_threads(self) -> None:
|
||||
"""Create (not start) the refresher thread and, when configured, the liveness watchdog.
|
||||
|
||||
Lease renewal is NOT evidence of progress; a silently stalled turn would renew forever, so the
|
||||
watchdog (policy in ``agent/turn_liveness.py``) is wired with the commit/deactivate callbacks
|
||||
below.
|
||||
"""
|
||||
self.refresh_thread = threading.Thread(
|
||||
target=self.refresh_loop, name="session-turn-lease-refresh", daemon=True
|
||||
)
|
||||
try:
|
||||
from hermes_cli.config import load_config_readonly
|
||||
|
||||
liveness_config = load_config_readonly() or {}
|
||||
except Exception:
|
||||
liveness_config = {}
|
||||
from agent import turn_liveness
|
||||
|
||||
timeout_s, poll_s = turn_liveness.resolve_turn_liveness_settings(liveness_config)
|
||||
if timeout_s is not None:
|
||||
self.liveness_thread = turn_liveness.TurnLivenessWatchdog(
|
||||
self.agent,
|
||||
session_id=self._current_session_id(),
|
||||
timeout_s=timeout_s,
|
||||
poll_s=poll_s,
|
||||
stop_event=self.stop,
|
||||
activity_lock=self.agent._liveness_activity_lock(),
|
||||
is_turn_active=self.is_turn_active,
|
||||
commit_abort=self.commit_liveness_abort,
|
||||
deactivate_turn=self.deactivate_after_liveness_abort,
|
||||
).make_thread()
|
||||
|
||||
# ── lifecycle ────────────────────────────────────────────────────────────────
|
||||
|
||||
def start(self) -> None:
|
||||
with self._lock:
|
||||
self.turn_active = True
|
||||
# Stamp the activity clock at turn entry: `_last_activity_ts` persists across turns, so without
|
||||
# this the watchdog would measure idle from the PREVIOUS turn and abort a fresh one.
|
||||
self.agent._touch_activity("starting new turn")
|
||||
self.refresh_thread.start()
|
||||
if self.liveness_thread is not None:
|
||||
self.liveness_thread.start()
|
||||
|
||||
def stop_refresher(self) -> None:
|
||||
with self._lock:
|
||||
self.turn_active = False
|
||||
self.stop.set()
|
||||
|
||||
def join_threads(self, timeout: float = 1.0) -> None:
|
||||
for thread in (self.refresh_thread, self.liveness_thread):
|
||||
if thread is not None and thread.is_alive():
|
||||
thread.join(timeout=timeout)
|
||||
|
||||
def release(self) -> None:
|
||||
"""Release the row and drop the agent's holder attrs (only if they still name this lease)."""
|
||||
agent = self.agent
|
||||
try:
|
||||
self.db.release_session_turn_lease(self.session_id, self.holder)
|
||||
except Exception:
|
||||
logger.error("Failed to release session turn lease: %s", self.session_id, exc_info=True)
|
||||
if getattr(agent, "_active_session_turn_lease_holder", None) == self.holder:
|
||||
agent._active_session_turn_lease_holder = None
|
||||
agent._active_session_turn_lease_ttl_seconds = None
|
||||
|
||||
def is_turn_active(self) -> bool:
|
||||
with self._lock:
|
||||
return self.turn_active
|
||||
|
||||
# ── interrupts ───────────────────────────────────────────────────────────────
|
||||
|
||||
def _interrupt_turn(self, message: str) -> None:
|
||||
# Lease-loss interrupts fire UNCONDITIONALLY (no generation claim): a lost lease means this
|
||||
# process no longer owns the session. Only the watchdog's stalls can be spuriously stale.
|
||||
agent = self.agent
|
||||
with self._lock:
|
||||
if self.stop.is_set() or not self.turn_active:
|
||||
return
|
||||
self.interrupt_message = message
|
||||
try:
|
||||
agent.interrupt(message, hard_cancel=True)
|
||||
except Exception:
|
||||
agent._interrupt_requested = True
|
||||
agent._interrupt_message = message
|
||||
|
||||
def commit_liveness_abort(self, snapshot, message: str) -> bool:
|
||||
"""Commit point for the watchdog's stall observation.
|
||||
|
||||
Revalidates the observed ``(generation, timestamp)`` under the SAME lock ``_touch_activity``
|
||||
uses, so a turn that resumed while the stall was logged is never hard-cancelled. The
|
||||
revalidated generation is carried into ``interrupt`` as ``require_generation``, which consumes
|
||||
it with the first publication in ONE critical section. If ``interrupt`` raises, the abort
|
||||
declines FAIL-CLOSED. Returns False when stale or already winding down.
|
||||
"""
|
||||
agent = self.agent
|
||||
with agent._liveness_activity_lock():
|
||||
current_generation = getattr(agent, "_turn_liveness_activity_generation", 0)
|
||||
if (current_generation, getattr(agent, "_last_activity_ts", None)) != (
|
||||
snapshot.generation,
|
||||
snapshot.activity_ts,
|
||||
):
|
||||
return False
|
||||
with self._lock:
|
||||
if self.stop.is_set() or not self.turn_active:
|
||||
return False
|
||||
try:
|
||||
published = agent.interrupt(
|
||||
message, hard_cancel=True, require_generation=current_generation
|
||||
)
|
||||
except Exception:
|
||||
# Fail closed: an exceptional path must not turn an unvalidated claim into abort authority.
|
||||
logger.debug(
|
||||
"Turn liveness abort interrupt raised; declining the abort", exc_info=True
|
||||
)
|
||||
published = False
|
||||
if published is False:
|
||||
# Claim went stale between revalidation and the hammer: real progress landed.
|
||||
return False
|
||||
with self._lock:
|
||||
self.interrupt_message = message
|
||||
return True
|
||||
|
||||
def deactivate_after_liveness_abort(self) -> None:
|
||||
"""Stop lease renewal after a committed liveness abort.
|
||||
|
||||
A wedge the hard interrupt cannot unwind must not keep the lease alive forever; TTL expiry
|
||||
lets stale-turn cleanup reclaim the row.
|
||||
"""
|
||||
with self._lock:
|
||||
self.stop.set()
|
||||
self.turn_active = False
|
||||
|
||||
def clear_interrupt(self) -> None:
|
||||
"""Clear only the interrupt admitted by this lease's refresher/watchdog. Run AFTER join."""
|
||||
message = self.interrupt_message
|
||||
if not message:
|
||||
return
|
||||
agent = self.agent
|
||||
|
||||
def _clear_if_owned() -> None:
|
||||
# Lazy via the façade so ``patch("agent.turn_facade._set_interrupt")`` keeps intercepting.
|
||||
from agent.turn_facade import _set_interrupt
|
||||
|
||||
if getattr(agent, "_interrupt_message", None) != message:
|
||||
return
|
||||
agent._interrupt_requested = False
|
||||
agent._interrupt_message = None
|
||||
getattr(agent, "_hard_interrupt_requested", threading.Event()).clear()
|
||||
agent._interrupt_thread_signal_pending = False
|
||||
if agent._execution_thread_id is not None:
|
||||
_set_interrupt(False, agent._execution_thread_id)
|
||||
|
||||
redirect_lock = getattr(agent, "_pending_redirect_lock", None)
|
||||
if redirect_lock is None:
|
||||
_clear_if_owned()
|
||||
else:
|
||||
with redirect_lock:
|
||||
_clear_if_owned()
|
||||
|
||||
# ── refresher thread body ────────────────────────────────────────────────────
|
||||
|
||||
def refresh_loop(self) -> None:
|
||||
"""Renew the lease every ``refresh_interval``; a miss or error interrupts the turn.
|
||||
|
||||
Long turns outlive the fixed TTL; the holder-qualified UPDATE fences a late refresher from a
|
||||
successor lease. The façade's finally sets ``stop`` before releasing, so a holder-fenced miss
|
||||
observed after stop is not a loss.
|
||||
"""
|
||||
while not self.stop.wait(self.refresh_interval):
|
||||
try:
|
||||
if self.db.refresh_session_turn_lease(
|
||||
self._current_session_id(), self.holder, ttl_seconds=LEASE_TTL_SECONDS
|
||||
):
|
||||
continue
|
||||
if self.stop.is_set():
|
||||
return
|
||||
logger.error(
|
||||
"Lost session turn lease while turn is active: %s", self._current_session_id()
|
||||
)
|
||||
self._interrupt_turn(
|
||||
"Session turn lease lost; stopping to protect the transcript."
|
||||
)
|
||||
except Exception:
|
||||
if self.stop.is_set():
|
||||
return
|
||||
logger.warning(
|
||||
"Failed to refresh session turn lease: %s",
|
||||
self._current_session_id(),
|
||||
exc_info=True,
|
||||
)
|
||||
self._interrupt_turn(
|
||||
"Session turn lease could not be refreshed; stopping to protect the transcript."
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
@dataclass
|
||||
class TurnLeaseAdmission:
|
||||
"""Outcome of ``admit_durable_turn_lease``: exactly one of ``lease`` / ``early_result`` may be set."""
|
||||
|
||||
lease: Optional[DurableTurnLease] = None
|
||||
early_result: Optional[Dict[str, Any]] = None
|
||||
conversation_history: Optional[List[Dict[str, Any]]] = None
|
||||
|
||||
|
||||
def _durable_session_exists(db, session_id: str) -> bool:
|
||||
try:
|
||||
return db.get_session(session_id) is not None
|
||||
except Exception:
|
||||
# A locked / non-WAL read is not proof the row is absent; treating probe failure as "fresh"
|
||||
# ran fail-open at the exact contention point. Acquire, or fail closed.
|
||||
logger.warning(
|
||||
"Could not check durable session before turn lease; "
|
||||
"will acquire rather than run without serialization",
|
||||
exc_info=True,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def admit_durable_turn_lease(
|
||||
agent,
|
||||
*,
|
||||
session_id: str,
|
||||
relay_turn_id: str,
|
||||
task_context: Dict[str, Any],
|
||||
conversation_history: Optional[List[Dict[str, Any]]],
|
||||
) -> TurnLeaseAdmission:
|
||||
"""Acquire the session turn lease when the session is durable; build (not start) its threads.
|
||||
|
||||
Mutates ``task_context["session_id"]`` and ``agent.session_id`` when the wait forced a resume-id
|
||||
reload. Returns an ``early_result`` (interrupted / timed out) instead of a lease when admission
|
||||
fails; the caller returns it verbatim.
|
||||
"""
|
||||
db = getattr(agent, "_session_db", None)
|
||||
admission = TurnLeaseAdmission(conversation_history=conversation_history)
|
||||
if db is None or not session_id:
|
||||
return admission
|
||||
# A fresh session id has no durable transcript to race over, and callers may supply an in-memory
|
||||
# seed before the row exists — reloading would erase it. Check the concrete type: MagicMock-style
|
||||
# shims accept any attribute without the protocol.
|
||||
exists = _durable_session_exists(db, session_id)
|
||||
if (
|
||||
getattr(agent, "_persist_disabled", False)
|
||||
or not exists
|
||||
or not callable(getattr(type(db), "acquire_session_turn_lease", None))
|
||||
):
|
||||
return admission
|
||||
# Row proven to exist — suppress the redundant create attempt.
|
||||
agent._session_db_created = True
|
||||
holder = (
|
||||
f"pid={os.getpid()}:turn={relay_turn_id}:platform={task_context['platform'] or 'unknown'}"
|
||||
)
|
||||
waited = False
|
||||
|
||||
def _on_wait(elapsed: float) -> None:
|
||||
nonlocal waited
|
||||
waited = True
|
||||
if elapsed < 1.0:
|
||||
agent._emit_status(
|
||||
"⏳ Another Hermes process is using this session; "
|
||||
"waiting for it to finish before starting your turn..."
|
||||
)
|
||||
else:
|
||||
agent._emit_status(
|
||||
"⏳ Still waiting for the other Hermes process on "
|
||||
f"this session ({int(elapsed)}s)..."
|
||||
)
|
||||
|
||||
if not db.acquire_session_turn_lease(
|
||||
session_id,
|
||||
holder,
|
||||
ttl_seconds=LEASE_TTL_SECONDS,
|
||||
wait_seconds=LEASE_WAIT_SECONDS,
|
||||
on_wait=_on_wait,
|
||||
should_abort=lambda: getattr(agent, "_interrupt_requested", False),
|
||||
):
|
||||
admission.early_result = _lease_not_acquired_result(agent, session_id, conversation_history)
|
||||
return admission
|
||||
|
||||
# Assign only after admission so the finally cannot release a holder that never owned the row;
|
||||
# persist paths read the agent attr so a late flush is fenced in the same SQLite transaction.
|
||||
lease = DurableTurnLease(agent, db, session_id, holder)
|
||||
agent._active_session_turn_lease_holder = holder
|
||||
agent._active_session_turn_lease_ttl_seconds = LEASE_TTL_SECONDS
|
||||
try:
|
||||
if waited:
|
||||
agent._emit_status("Session is free; loading the latest transcript...")
|
||||
# The holder may have compressed/rotated the session while we waited: reload only AFTER
|
||||
# admission, and skip when acquisition was immediate (avoids a needless prompt-cache miss).
|
||||
latest_session_id = db.resolve_resume_session_id(session_id)
|
||||
if latest_session_id:
|
||||
agent.session_id = latest_session_id
|
||||
task_context["session_id"] = latest_session_id
|
||||
admission.conversation_history = db.get_messages_as_conversation(
|
||||
agent.session_id, repair_alternation=True, include_row_ids=True
|
||||
)
|
||||
lease.build_threads()
|
||||
except BaseException:
|
||||
# The façade never saw this lease; release here so an admitted row is not leaked.
|
||||
lease.release()
|
||||
raise
|
||||
admission.lease = lease
|
||||
return admission
|
||||
|
||||
|
||||
def _lease_not_acquired_result(agent, session_id: str, conversation_history) -> Dict[str, Any]:
|
||||
base = {"messages": list(conversation_history or []), "api_calls": 0, "completed": False}
|
||||
if getattr(agent, "_interrupt_requested", False):
|
||||
logger.info("session turn lease wait aborted by interrupt: %s", session_id)
|
||||
result = {
|
||||
"final_response": (
|
||||
"Stopped waiting for another Hermes process on this session. "
|
||||
"Your message was not processed."
|
||||
),
|
||||
**base,
|
||||
"interrupted": True,
|
||||
}
|
||||
interrupt_message = getattr(agent, "_interrupt_message", None)
|
||||
if interrupt_message:
|
||||
result["interrupt_message"] = interrupt_message
|
||||
# The finalizer never runs on this early return; clear so a cached agent doesn't fail-close
|
||||
# the next turn.
|
||||
try:
|
||||
agent.clear_interrupt()
|
||||
except Exception:
|
||||
agent._interrupt_requested = False
|
||||
agent._interrupt_message = None
|
||||
return result
|
||||
# Fail closed like gateway TurnLeaseTimeoutError: surface a resend notice, not a bare TimeoutError.
|
||||
timeout_msg = (
|
||||
"⏳ Another Hermes process kept this session busy too long. Your message was not "
|
||||
"processed - wait for the other process to finish, then send it again."
|
||||
)
|
||||
logger.error("session turn lease wait timed out for %s", session_id)
|
||||
try:
|
||||
agent._emit_warning(timeout_msg)
|
||||
except Exception:
|
||||
logger.debug("Failed to emit session turn lease timeout warning", exc_info=True)
|
||||
return {
|
||||
"final_response": timeout_msg,
|
||||
**base,
|
||||
"failed": True,
|
||||
"error": f"session_turn_lease_timeout:{session_id}",
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Unit tests for agent.turn_facade_lease (admission + lease bracket)."""
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
|
||||
from agent.turn_facade_lease import (
|
||||
LEASE_TTL_SECONDS,
|
||||
DurableTurnLease,
|
||||
admit_durable_turn_lease,
|
||||
)
|
||||
|
||||
|
||||
class _Db:
|
||||
def __init__(self, exists=True, acquired=True):
|
||||
self.exists = exists
|
||||
self.acquired = acquired
|
||||
self.events = []
|
||||
|
||||
def get_session(self, session_id):
|
||||
return {"id": session_id} if self.exists else None
|
||||
|
||||
def acquire_session_turn_lease(self, session_id, holder, **kwargs):
|
||||
self.events.append(("acquire", session_id, holder))
|
||||
return self.acquired
|
||||
|
||||
def refresh_session_turn_lease(self, session_id, holder, **kwargs):
|
||||
return True
|
||||
|
||||
def release_session_turn_lease(self, session_id, holder):
|
||||
self.events.append(("release", session_id, holder))
|
||||
|
||||
|
||||
def _agent(db, **overrides):
|
||||
agent = SimpleNamespace(
|
||||
_session_db=db,
|
||||
session_id="s1",
|
||||
_persist_disabled=False,
|
||||
_interrupt_requested=False,
|
||||
_interrupt_message=None,
|
||||
_execution_thread_id=None,
|
||||
_session_turn_lease_refresh_interval=60.0,
|
||||
statuses=[],
|
||||
)
|
||||
agent._emit_status = agent.statuses.append
|
||||
agent._emit_warning = agent.statuses.append
|
||||
agent._touch_activity = lambda *a, **k: None
|
||||
agent._liveness_activity_lock = lambda: threading.Lock()
|
||||
for k, v in overrides.items():
|
||||
setattr(agent, k, v)
|
||||
return agent
|
||||
|
||||
|
||||
def _admit(agent, history=None):
|
||||
return admit_durable_turn_lease(
|
||||
agent,
|
||||
session_id="s1",
|
||||
relay_turn_id="s1:t:abcd",
|
||||
task_context={"session_id": "s1", "task_id": "t", "platform": "cli"},
|
||||
conversation_history=history,
|
||||
)
|
||||
|
||||
|
||||
def test_no_lease_without_durable_row_or_when_persist_disabled():
|
||||
seed = [{"role": "user", "content": "hi"}]
|
||||
admission = _admit(_agent(_Db(exists=False)), seed)
|
||||
assert admission.lease is None and admission.early_result is None
|
||||
assert admission.conversation_history is seed
|
||||
|
||||
db = _Db()
|
||||
admission = _admit(_agent(db, _persist_disabled=True), seed)
|
||||
assert admission.lease is None and db.events == []
|
||||
|
||||
|
||||
def test_admission_sets_holder_attrs_and_release_clears_them(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"agent.turn_liveness.resolve_turn_liveness_settings", lambda cfg: (None, 1.0)
|
||||
)
|
||||
db = _Db()
|
||||
agent = _agent(db)
|
||||
admission = _admit(agent)
|
||||
lease = admission.lease
|
||||
assert isinstance(lease, DurableTurnLease)
|
||||
assert agent._session_db_created is True
|
||||
assert agent._active_session_turn_lease_holder == lease.holder
|
||||
assert agent._active_session_turn_lease_ttl_seconds == LEASE_TTL_SECONDS
|
||||
assert lease.holder.startswith("pid=") and ":platform=cli" in lease.holder
|
||||
assert lease.refresh_thread is not None and lease.liveness_thread is None
|
||||
assert lease.is_turn_active() is False
|
||||
|
||||
lease.stop_refresher()
|
||||
lease.join_threads()
|
||||
lease.clear_interrupt()
|
||||
lease.release()
|
||||
assert db.events == [("acquire", "s1", lease.holder), ("release", "s1", lease.holder)]
|
||||
assert agent._active_session_turn_lease_holder is None
|
||||
assert agent._active_session_turn_lease_ttl_seconds is None
|
||||
|
||||
|
||||
def test_timeout_and_interrupt_early_results():
|
||||
agent = _agent(_Db(acquired=False))
|
||||
admission = _admit(agent, [{"role": "user", "content": "x"}])
|
||||
assert admission.lease is None
|
||||
assert admission.early_result["failed"] is True
|
||||
assert admission.early_result["error"] == "session_turn_lease_timeout:s1"
|
||||
assert admission.early_result["messages"] == [{"role": "user", "content": "x"}]
|
||||
|
||||
agent = _agent(_Db(acquired=False), _interrupt_requested=True, _interrupt_message="stop")
|
||||
agent.clear_interrupt = lambda: None
|
||||
admission = _admit(agent)
|
||||
assert admission.early_result["interrupted"] is True
|
||||
assert admission.early_result["interrupt_message"] == "stop"
|
||||
|
||||
|
||||
def test_interrupt_turn_only_while_active():
|
||||
agent = _agent(_Db())
|
||||
calls = []
|
||||
agent.interrupt = lambda msg, **kw: calls.append(msg)
|
||||
lease = DurableTurnLease(agent, agent._session_db, "s1", "h")
|
||||
lease._interrupt_turn("lost") # inactive: ignored
|
||||
assert calls == [] and lease.interrupt_message is None
|
||||
lease.turn_active = True
|
||||
lease._interrupt_turn("lost")
|
||||
assert calls == ["lost"] and lease.interrupt_message == "lost"
|
||||
lease.deactivate_after_liveness_abort()
|
||||
assert lease.stop.is_set() and lease.is_turn_active() is False
|
||||
Reference in New Issue
Block a user