feat(deadline): SuspectableBackend protocol — mark timed-out backends suspect
Phase 3a of the #85125 unified-deadline plan. run_bounded_async and run_bounded_sync accept backend= and call mark_suspect(label + timeout) exactly once on timeout, never on completion. The layer fails open: backends without the protocol (incremental Phase 3b adoption) and raising mark_suspect implementations can never weaken the deadline bound or corrupt the BoundedResult.
This commit is contained in:
+108
-16
@@ -71,7 +71,7 @@ import sys
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Awaitable, Callable, Optional
|
||||
from typing import Any, Protocol, Awaitable, Callable, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -116,6 +116,41 @@ class DeadlineExpired(TimeoutError):
|
||||
self.timeout_s = timeout_s
|
||||
|
||||
|
||||
class SuspectableBackend(Protocol):
|
||||
"""Phase 3a (#85125): a stateful backend the deadline layer can flag.
|
||||
|
||||
A timed-out stateful backend (MCP connection, browser session, LSP
|
||||
client) may be left wedged by the abandoned half-finished operation.
|
||||
``run_bounded_*`` calls ``mark_suspect`` on timeout so the OWNER can
|
||||
health-check or recycle the backend before reuse (``ensure_healthy``)
|
||||
instead of returning a poisoned handle to the cache. Consumers adopt
|
||||
incrementally (Phase 3b, one backend per PR), so the layer fails open:
|
||||
backends without the protocol are simply never marked.
|
||||
"""
|
||||
|
||||
def mark_suspect(self, reason: str) -> None: ...
|
||||
|
||||
def ensure_healthy(self) -> bool: ...
|
||||
|
||||
|
||||
def _mark_backend_suspect(backend: object | None, label: str, timeout_s: float) -> None:
|
||||
"""Best-effort ``mark_suspect`` on a timed-out call's backend.
|
||||
|
||||
Never raises: adoption state must not be able to weaken the deadline
|
||||
bound or corrupt the ``BoundedResult`` the caller is about to receive.
|
||||
A non-adopting backend (no ``mark_suspect``) is tolerated silently —
|
||||
Phase 3b lands per-backend, so absence is the norm during adoption.
|
||||
"""
|
||||
if backend is None:
|
||||
return
|
||||
try:
|
||||
mark = getattr(backend, "mark_suspect", None)
|
||||
if callable(mark):
|
||||
mark(f"{label} timed out after {timeout_s:.1f}s")
|
||||
except Exception:
|
||||
logger.debug("deadline mark_suspect failed", exc_info=True)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class BoundedResult:
|
||||
"""Outcome of a bounded operation.
|
||||
@@ -155,7 +190,9 @@ def clamp_timeout(timeout: Optional[float]) -> Optional[float]:
|
||||
try:
|
||||
value = float(timeout)
|
||||
except (TypeError, ValueError):
|
||||
logger.warning("clamp_timeout: non-numeric timeout %r; treating as unbounded", timeout)
|
||||
logger.warning(
|
||||
"clamp_timeout: non-numeric timeout %r; treating as unbounded", timeout
|
||||
)
|
||||
return None
|
||||
if value != value: # NaN
|
||||
logger.warning("clamp_timeout: NaN timeout; treating as unbounded")
|
||||
@@ -170,6 +207,7 @@ def clamp_timeout(timeout: Optional[float]) -> Optional[float]:
|
||||
# registered default.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _timeouts_section() -> dict:
|
||||
"""Read the ``timeouts:`` root section from config.yaml (read-only).
|
||||
|
||||
@@ -233,7 +271,9 @@ def resolve_timeout(
|
||||
return clamp_timeout(value)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
logger.warning("timeouts.%s: invalid value %r in config.yaml; ignoring", key, raw)
|
||||
logger.warning(
|
||||
"timeouts.%s: invalid value %r in config.yaml; ignoring", key, raw
|
||||
)
|
||||
|
||||
if env_var:
|
||||
env_raw = os.getenv(env_var, "").strip()
|
||||
@@ -256,6 +296,7 @@ def resolve_timeout(
|
||||
# of information loop-blocked hangs otherwise never surface.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _consume_abandoned(task: "asyncio.Future[Any]") -> None:
|
||||
"""Observe an abandoned task's outcome so it never logs 'never retrieved'."""
|
||||
try:
|
||||
@@ -297,6 +338,7 @@ async def run_bounded_async(
|
||||
label: str = "operation",
|
||||
on_abandon: Optional[Callable[[], Awaitable[Any]]] = None,
|
||||
dump_on_blocked_loop: bool = True,
|
||||
backend: object | None = None,
|
||||
) -> BoundedResult:
|
||||
"""Await ``awaitable`` under a wall-clock deadline independent of loop timers.
|
||||
|
||||
@@ -317,7 +359,13 @@ async def run_bounded_async(
|
||||
start = time.monotonic()
|
||||
if timeout_s is None:
|
||||
value = await awaitable
|
||||
return BoundedResult(timed_out=False, value=value, elapsed_s=time.monotonic() - start, timeout_s=None, label=label)
|
||||
return BoundedResult(
|
||||
timed_out=False,
|
||||
value=value,
|
||||
elapsed_s=time.monotonic() - start,
|
||||
timeout_s=None,
|
||||
label=label,
|
||||
)
|
||||
|
||||
task = asyncio.ensure_future(awaitable)
|
||||
loop = asyncio.get_running_loop()
|
||||
@@ -363,15 +411,32 @@ async def run_bounded_async(
|
||||
if not deadline.done():
|
||||
deadline.cancel()
|
||||
value = await task
|
||||
return BoundedResult(timed_out=False, value=value, elapsed_s=time.monotonic() - start, timeout_s=timeout_s, label=label)
|
||||
return BoundedResult(
|
||||
timed_out=False,
|
||||
value=value,
|
||||
elapsed_s=time.monotonic() - start,
|
||||
timeout_s=timeout_s,
|
||||
label=label,
|
||||
)
|
||||
|
||||
task.cancel()
|
||||
task.add_done_callback(_consume_abandoned)
|
||||
if on_abandon is not None:
|
||||
cleanup = asyncio.ensure_future(_run_abandon_cleanup(on_abandon))
|
||||
cleanup.add_done_callback(_consume_abandoned)
|
||||
logger.warning("[deadline] %r timed out after %.1fs; task abandoned", label, timeout_s)
|
||||
return BoundedResult(timed_out=True, value=None, elapsed_s=time.monotonic() - start, timeout_s=timeout_s, label=label)
|
||||
# Phase 3a (#85125): the abandoned task may leave the backend
|
||||
# half-wedged; flag it so the owner recycles before reuse.
|
||||
_mark_backend_suspect(backend, label, timeout_s)
|
||||
logger.warning(
|
||||
"[deadline] %r timed out after %.1fs; task abandoned", label, timeout_s
|
||||
)
|
||||
return BoundedResult(
|
||||
timed_out=True,
|
||||
value=None,
|
||||
elapsed_s=time.monotonic() - start,
|
||||
timeout_s=timeout_s,
|
||||
label=label,
|
||||
)
|
||||
finally:
|
||||
timer.cancel()
|
||||
if watchdog is not None:
|
||||
@@ -386,12 +451,14 @@ async def run_bounded_async(
|
||||
# Bounded execution — sync flavor.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_bounded_sync(
|
||||
fn: Callable[[], Any],
|
||||
timeout: Optional[float],
|
||||
*,
|
||||
label: str = "operation",
|
||||
on_timeout: Optional[Callable[[], None]] = None,
|
||||
backend: object | None = None,
|
||||
) -> BoundedResult:
|
||||
"""Run ``fn`` in a daemon worker thread under a wall-clock deadline.
|
||||
|
||||
@@ -411,7 +478,13 @@ def run_bounded_sync(
|
||||
timeout_s = clamp_timeout(timeout)
|
||||
start = time.monotonic()
|
||||
if timeout_s is None:
|
||||
return BoundedResult(timed_out=False, value=fn(), elapsed_s=time.monotonic() - start, timeout_s=None, label=label)
|
||||
return BoundedResult(
|
||||
timed_out=False,
|
||||
value=fn(),
|
||||
elapsed_s=time.monotonic() - start,
|
||||
timeout_s=None,
|
||||
label=label,
|
||||
)
|
||||
|
||||
box: dict[str, Any] = {}
|
||||
done = threading.Event()
|
||||
@@ -424,28 +497,43 @@ def run_bounded_sync(
|
||||
finally:
|
||||
done.set()
|
||||
|
||||
thread = threading.Thread(
|
||||
target=_worker, name=f"deadline-{label}", daemon=True
|
||||
)
|
||||
thread = threading.Thread(target=_worker, name=f"deadline-{label}", daemon=True)
|
||||
thread.start()
|
||||
if not done.wait(timeout_s):
|
||||
logger.warning("[deadline] %r timed out after %.1fs; worker abandoned", label, timeout_s)
|
||||
logger.warning(
|
||||
"[deadline] %r timed out after %.1fs; worker abandoned", label, timeout_s
|
||||
)
|
||||
if on_timeout is not None:
|
||||
try:
|
||||
on_timeout()
|
||||
except Exception:
|
||||
logger.debug("deadline on_timeout callback failed", exc_info=True)
|
||||
return BoundedResult(timed_out=True, value=None, elapsed_s=time.monotonic() - start, timeout_s=timeout_s, label=label)
|
||||
# Phase 3a (#85125): flag the abandoned call's backend for recycle.
|
||||
_mark_backend_suspect(backend, label, timeout_s)
|
||||
return BoundedResult(
|
||||
timed_out=True,
|
||||
value=None,
|
||||
elapsed_s=time.monotonic() - start,
|
||||
timeout_s=timeout_s,
|
||||
label=label,
|
||||
)
|
||||
|
||||
if "exc" in box:
|
||||
raise box["exc"]
|
||||
return BoundedResult(timed_out=False, value=box.get("value"), elapsed_s=time.monotonic() - start, timeout_s=timeout_s, label=label)
|
||||
return BoundedResult(
|
||||
timed_out=False,
|
||||
value=box.get("value"),
|
||||
elapsed_s=time.monotonic() - start,
|
||||
timeout_s=timeout_s,
|
||||
label=label,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Whole-tree process termination.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def kill_process_tree(pid: int, *, sig: Optional[int] = None) -> bool:
|
||||
"""Terminate ``pid`` and all its descendants, portably.
|
||||
|
||||
@@ -490,7 +578,9 @@ def kill_process_tree(pid: int, *, sig: Optional[int] = None) -> bool:
|
||||
# cross-platform contract (False = nothing was terminated).
|
||||
return proc.returncode == 0
|
||||
except Exception:
|
||||
logger.debug("kill_process_tree: taskkill failed for pid %s", pid, exc_info=True)
|
||||
logger.debug(
|
||||
"kill_process_tree: taskkill failed for pid %s", pid, exc_info=True
|
||||
)
|
||||
return False
|
||||
|
||||
import signal as _signal
|
||||
@@ -523,7 +613,9 @@ def kill_process_tree(pid: int, *, sig: Optional[int] = None) -> bool:
|
||||
# pid leads its own group: one syscall covers the whole group.
|
||||
# (The == check guards against signalling the caller's own group
|
||||
# when pid is not a leader.)
|
||||
os.killpg(pgid, sig) # windows-footgun: ok — POSIX-only branch (win32 returns above)
|
||||
os.killpg(
|
||||
pgid, sig
|
||||
) # windows-footgun: ok — POSIX-only branch (win32 returns above)
|
||||
else:
|
||||
os.kill(pid, sig)
|
||||
signalled = True
|
||||
|
||||
Reference in New Issue
Block a user