From 7a3aaf01430cba3c5ce2930a724a2ebdd2a2c0f9 Mon Sep 17 00:00:00 2001 From: Ayush Nangia Date: Tue, 18 Aug 2026 00:19:12 +0530 Subject: [PATCH] =?UTF-8?q?feat(deadline):=20SuspectableBackend=20protocol?= =?UTF-8?q?=20=E2=80=94=20mark=20timed-out=20backends=20suspect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3a of the #85125 unified-deadline plan. run_bounded_async and run_bounded_sync accept backend= and call mark_suspect(label + timeout) exactly once on timeout, never on completion. The layer fails open: backends without the protocol (incremental Phase 3b adoption) and raising mark_suspect implementations can never weaken the deadline bound or corrupt the BoundedResult. --- agent/deadline.py | 124 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 108 insertions(+), 16 deletions(-) diff --git a/agent/deadline.py b/agent/deadline.py index fa6bf2a7da..64e46ee414 100644 --- a/agent/deadline.py +++ b/agent/deadline.py @@ -71,7 +71,7 @@ import sys import threading import time from dataclasses import dataclass -from typing import Any, Awaitable, Callable, Optional +from typing import Any, Protocol, Awaitable, Callable, Optional logger = logging.getLogger(__name__) @@ -116,6 +116,41 @@ class DeadlineExpired(TimeoutError): self.timeout_s = timeout_s +class SuspectableBackend(Protocol): + """Phase 3a (#85125): a stateful backend the deadline layer can flag. + + A timed-out stateful backend (MCP connection, browser session, LSP + client) may be left wedged by the abandoned half-finished operation. + ``run_bounded_*`` calls ``mark_suspect`` on timeout so the OWNER can + health-check or recycle the backend before reuse (``ensure_healthy``) + instead of returning a poisoned handle to the cache. Consumers adopt + incrementally (Phase 3b, one backend per PR), so the layer fails open: + backends without the protocol are simply never marked. + """ + + def mark_suspect(self, reason: str) -> None: ... + + def ensure_healthy(self) -> bool: ... + + +def _mark_backend_suspect(backend: object | None, label: str, timeout_s: float) -> None: + """Best-effort ``mark_suspect`` on a timed-out call's backend. + + Never raises: adoption state must not be able to weaken the deadline + bound or corrupt the ``BoundedResult`` the caller is about to receive. + A non-adopting backend (no ``mark_suspect``) is tolerated silently — + Phase 3b lands per-backend, so absence is the norm during adoption. + """ + if backend is None: + return + try: + mark = getattr(backend, "mark_suspect", None) + if callable(mark): + mark(f"{label} timed out after {timeout_s:.1f}s") + except Exception: + logger.debug("deadline mark_suspect failed", exc_info=True) + + @dataclass(frozen=True, kw_only=True) class BoundedResult: """Outcome of a bounded operation. @@ -155,7 +190,9 @@ def clamp_timeout(timeout: Optional[float]) -> Optional[float]: try: value = float(timeout) except (TypeError, ValueError): - logger.warning("clamp_timeout: non-numeric timeout %r; treating as unbounded", timeout) + logger.warning( + "clamp_timeout: non-numeric timeout %r; treating as unbounded", timeout + ) return None if value != value: # NaN logger.warning("clamp_timeout: NaN timeout; treating as unbounded") @@ -170,6 +207,7 @@ def clamp_timeout(timeout: Optional[float]) -> Optional[float]: # registered default. # --------------------------------------------------------------------------- + def _timeouts_section() -> dict: """Read the ``timeouts:`` root section from config.yaml (read-only). @@ -233,7 +271,9 @@ def resolve_timeout( return clamp_timeout(value) except (TypeError, ValueError): pass - logger.warning("timeouts.%s: invalid value %r in config.yaml; ignoring", key, raw) + logger.warning( + "timeouts.%s: invalid value %r in config.yaml; ignoring", key, raw + ) if env_var: env_raw = os.getenv(env_var, "").strip() @@ -256,6 +296,7 @@ def resolve_timeout( # of information loop-blocked hangs otherwise never surface. # --------------------------------------------------------------------------- + def _consume_abandoned(task: "asyncio.Future[Any]") -> None: """Observe an abandoned task's outcome so it never logs 'never retrieved'.""" try: @@ -297,6 +338,7 @@ async def run_bounded_async( label: str = "operation", on_abandon: Optional[Callable[[], Awaitable[Any]]] = None, dump_on_blocked_loop: bool = True, + backend: object | None = None, ) -> BoundedResult: """Await ``awaitable`` under a wall-clock deadline independent of loop timers. @@ -317,7 +359,13 @@ async def run_bounded_async( start = time.monotonic() if timeout_s is None: value = await awaitable - return BoundedResult(timed_out=False, value=value, elapsed_s=time.monotonic() - start, timeout_s=None, label=label) + return BoundedResult( + timed_out=False, + value=value, + elapsed_s=time.monotonic() - start, + timeout_s=None, + label=label, + ) task = asyncio.ensure_future(awaitable) loop = asyncio.get_running_loop() @@ -363,15 +411,32 @@ async def run_bounded_async( if not deadline.done(): deadline.cancel() value = await task - return BoundedResult(timed_out=False, value=value, elapsed_s=time.monotonic() - start, timeout_s=timeout_s, label=label) + return BoundedResult( + timed_out=False, + value=value, + elapsed_s=time.monotonic() - start, + timeout_s=timeout_s, + label=label, + ) task.cancel() task.add_done_callback(_consume_abandoned) if on_abandon is not None: cleanup = asyncio.ensure_future(_run_abandon_cleanup(on_abandon)) cleanup.add_done_callback(_consume_abandoned) - logger.warning("[deadline] %r timed out after %.1fs; task abandoned", label, timeout_s) - return BoundedResult(timed_out=True, value=None, elapsed_s=time.monotonic() - start, timeout_s=timeout_s, label=label) + # Phase 3a (#85125): the abandoned task may leave the backend + # half-wedged; flag it so the owner recycles before reuse. + _mark_backend_suspect(backend, label, timeout_s) + logger.warning( + "[deadline] %r timed out after %.1fs; task abandoned", label, timeout_s + ) + return BoundedResult( + timed_out=True, + value=None, + elapsed_s=time.monotonic() - start, + timeout_s=timeout_s, + label=label, + ) finally: timer.cancel() if watchdog is not None: @@ -386,12 +451,14 @@ async def run_bounded_async( # Bounded execution — sync flavor. # --------------------------------------------------------------------------- + def run_bounded_sync( fn: Callable[[], Any], timeout: Optional[float], *, label: str = "operation", on_timeout: Optional[Callable[[], None]] = None, + backend: object | None = None, ) -> BoundedResult: """Run ``fn`` in a daemon worker thread under a wall-clock deadline. @@ -411,7 +478,13 @@ def run_bounded_sync( timeout_s = clamp_timeout(timeout) start = time.monotonic() if timeout_s is None: - return BoundedResult(timed_out=False, value=fn(), elapsed_s=time.monotonic() - start, timeout_s=None, label=label) + return BoundedResult( + timed_out=False, + value=fn(), + elapsed_s=time.monotonic() - start, + timeout_s=None, + label=label, + ) box: dict[str, Any] = {} done = threading.Event() @@ -424,28 +497,43 @@ def run_bounded_sync( finally: done.set() - thread = threading.Thread( - target=_worker, name=f"deadline-{label}", daemon=True - ) + thread = threading.Thread(target=_worker, name=f"deadline-{label}", daemon=True) thread.start() if not done.wait(timeout_s): - logger.warning("[deadline] %r timed out after %.1fs; worker abandoned", label, timeout_s) + logger.warning( + "[deadline] %r timed out after %.1fs; worker abandoned", label, timeout_s + ) if on_timeout is not None: try: on_timeout() except Exception: logger.debug("deadline on_timeout callback failed", exc_info=True) - return BoundedResult(timed_out=True, value=None, elapsed_s=time.monotonic() - start, timeout_s=timeout_s, label=label) + # Phase 3a (#85125): flag the abandoned call's backend for recycle. + _mark_backend_suspect(backend, label, timeout_s) + return BoundedResult( + timed_out=True, + value=None, + elapsed_s=time.monotonic() - start, + timeout_s=timeout_s, + label=label, + ) if "exc" in box: raise box["exc"] - return BoundedResult(timed_out=False, value=box.get("value"), elapsed_s=time.monotonic() - start, timeout_s=timeout_s, label=label) + return BoundedResult( + timed_out=False, + value=box.get("value"), + elapsed_s=time.monotonic() - start, + timeout_s=timeout_s, + label=label, + ) # --------------------------------------------------------------------------- # Whole-tree process termination. # --------------------------------------------------------------------------- + def kill_process_tree(pid: int, *, sig: Optional[int] = None) -> bool: """Terminate ``pid`` and all its descendants, portably. @@ -490,7 +578,9 @@ def kill_process_tree(pid: int, *, sig: Optional[int] = None) -> bool: # cross-platform contract (False = nothing was terminated). return proc.returncode == 0 except Exception: - logger.debug("kill_process_tree: taskkill failed for pid %s", pid, exc_info=True) + logger.debug( + "kill_process_tree: taskkill failed for pid %s", pid, exc_info=True + ) return False import signal as _signal @@ -523,7 +613,9 @@ def kill_process_tree(pid: int, *, sig: Optional[int] = None) -> bool: # pid leads its own group: one syscall covers the whole group. # (The == check guards against signalling the caller's own group # when pid is not a leader.) - os.killpg(pgid, sig) # windows-footgun: ok — POSIX-only branch (win32 returns above) + os.killpg( + pgid, sig + ) # windows-footgun: ok — POSIX-only branch (win32 returns above) else: os.kill(pid, sig) signalled = True