m4
d4b53bfb08
Docker / build (push) Has been cancelled
Test / pytest (ubuntu-latest, 3.12) (push) Has been cancelled
Test / pytest (windows-latest, 3.11) (push) Has been cancelled
Test / pytest (windows-latest, 3.12) (push) Has been cancelled
Lint / ruff (push) Has been cancelled
Test / pytest (ubuntu-latest, 3.11) (push) Has been cancelled
Build / build (push) Has been cancelled
377 lines
16 KiB
Python
377 lines
16 KiB
Python
"""B02 resource failure nodes.
|
|
|
|
Budget ledger (test executions, not the runtime's three close attempts):
|
|
- Original construction failure [nth_model, agent, get_chat_model]: 4/5.
|
|
- Original close failure [2, 99]: 4/5.
|
|
- Original borrowed and explicit-transfer nodes: 3/5 (not rerun here).
|
|
- Hanging close [sync, async]: 5/5 each (budget exhausted; do not rerun).
|
|
- Construction hanging close retirement: 2/5 (RED, GREEN).
|
|
|
|
Execution log after explicit runtime handoff, all commands used uv run pytest:
|
|
1. Hanging close pair -q: exit 1, 2 failed in 6.37s; sync blocked loop,
|
|
async starved the other resource. Watchdog/teardown release was not evidence.
|
|
2. Hanging close pair -q: exit 0, 2 passed in 1.19s.
|
|
3. Construction hanging retirement -q: exit 1, 1 failed in 0.48s;
|
|
completed cleanup did not retire the preparation.
|
|
4. Construction hanging retirement + original close failure pair + original
|
|
construction triple -q: exit 0, 6 passed in 0.66s.
|
|
5. Hanging close pair -q: exit 0, 2 passed in 1.19s.
|
|
6. Public cancellation regression, pair execution 4/5:
|
|
uv run --no-sync pytest tests/test_execution_resource_failures.py::test_hanging_close_isolated_and_recoverable -q
|
|
exit 1, 2 failed in 0.70s. After deliberate release and empty owned_clients,
|
|
public cancel("retry") awaited the permanently failed stop task and raised
|
|
ExceptionGroup containing TimeoutError from _ensure_cleanup (250ms).
|
|
This reproduces the supplied deleg_3a10c94b review finding.
|
|
7. Same command, pair execution 5/5: exit 0, 2 passed in 4.25s.
|
|
Public cancel returns unknown while close hangs; wait_stopped short timeout
|
|
returns unknown; caller cancellation preserves the same stop owner. After
|
|
deliberate release, cancel and wait_stopped return cancelled; public stream
|
|
and committed events contain exactly one run_terminal, and each close ran
|
|
once. No internal cleanup helper is used as recovery evidence.
|
|
No approval, HTTP, full-suite, DB or external-service tests executed.
|
|
"""
|
|
import asyncio
|
|
import threading
|
|
from types import SimpleNamespace as NS
|
|
|
|
import pytest
|
|
|
|
from EvoScientist.llm.contracts import AgentModelSet
|
|
from EvoScientist.llm.runtime import EvoModelRuntime, _EvoWebRun
|
|
|
|
|
|
class Client:
|
|
def __init__(self, failures=0):
|
|
self.calls = 0
|
|
self.failures = failures
|
|
|
|
async def aclose(self):
|
|
self.calls += 1
|
|
if self.calls <= self.failures:
|
|
raise RuntimeError("close failed")
|
|
|
|
|
|
def setup_runtime(monkeypatch):
|
|
runtime = object.__new__(EvoModelRuntime)
|
|
runtime.admission_verifier = NS(require_admission=lambda admission: None)
|
|
runtime._registry_lock = asyncio.Lock()
|
|
runtime._started_grants = {}
|
|
runtime._validate_admission_echo = lambda *args: None
|
|
runtime._validate_still_fresh = lambda *args: None
|
|
admission = NS(grant_id="g", preparation_id="p", reasoning_effort="", provider_run_reserve_microunits=0, unsigned_payload=lambda: {})
|
|
handle = NS(lock=asyncio.Lock(), state="PREPARED", run=None,
|
|
quote=NS(expires_at=10**20), snapshot=NS(),
|
|
input=NS(metadata={}), host=NS(runtime_event_sink=object()))
|
|
runtime._prepared = {"p": handle}
|
|
runtime._prepared_tombstones = {}
|
|
runtime._prepared_requests = {("s", "r", "t"): "p"}
|
|
handle.grant = NS(subject_id="s", request_id="r", turn_id="t")
|
|
handle.request_digest = "request-digest"
|
|
handle.quote.preparation_id = "p"
|
|
monkeypatch.setattr(_EvoWebRun, "_start", lambda self: None)
|
|
return runtime, admission, handle
|
|
|
|
|
|
@pytest.mark.parametrize("failure", ["nth_model", "agent", "get_chat_model"])
|
|
def test_construction_failure_rolls_back_transports(monkeypatch, failure):
|
|
async def scenario():
|
|
import httpx
|
|
from EvoScientist.llm import models
|
|
runtime, admission, handle = setup_runtime(monkeypatch)
|
|
clients = []
|
|
def client():
|
|
result = Client()
|
|
clients.append(result)
|
|
return result
|
|
monkeypatch.setattr(httpx, "Client", client)
|
|
monkeypatch.setattr(httpx, "AsyncClient", client)
|
|
def chat(**kwargs):
|
|
if failure == "get_chat_model":
|
|
raise ValueError("construction")
|
|
return NS()
|
|
monkeypatch.setattr(models, "get_chat_model", chat)
|
|
if failure == "nth_model":
|
|
route = NS(identity=NS(model_id="test"), runtime_provider="openai", invocation_plan=object())
|
|
handle.snapshot.purpose_routes = {"main_agent": (route, route)}
|
|
handle.snapshot.purpose_route_call_bounds = {"main_agent": (NS(route_identity=route.identity),)}
|
|
runtime._model_factory_kwargs = lambda route: {}
|
|
runtime._attach_route_metadata = lambda model, *args: model
|
|
calls = 0
|
|
def factory(**kwargs):
|
|
nonlocal calls
|
|
calls += 1
|
|
if calls == 2:
|
|
raise ValueError("construction")
|
|
return runtime._default_model_factory(**kwargs)
|
|
runtime.model_factory = factory
|
|
def build(*args):
|
|
if failure == "nth_model":
|
|
return EvoModelRuntime._build_model_set(runtime, *args)
|
|
model = runtime._default_model_factory(provider="openai")
|
|
return AgentModelSet(model, model, model)
|
|
runtime._build_model_set = build
|
|
def agent(*args):
|
|
raise ValueError("construction")
|
|
runtime.agent_factory = agent
|
|
with pytest.raises(ValueError, match="construction"):
|
|
await runtime.start_web_run(admission)
|
|
assert clients and all(item.calls == 1 for item in clients)
|
|
assert handle.state != "STARTED"
|
|
assert not runtime._started_grants
|
|
asyncio.run(scenario())
|
|
|
|
|
|
@pytest.mark.parametrize("close_kind", ["sync", "async"])
|
|
def test_hanging_close_isolated_and_recoverable(monkeypatch, close_kind):
|
|
async def scenario():
|
|
runtime, admission, handle = setup_runtime(monkeypatch)
|
|
run = _EvoWebRun(runtime=runtime, admission=admission, prepared=handle,
|
|
agent=None, model_set=AgentModelSet(None, None, None))
|
|
handle.run = run
|
|
handle.state = "STARTED"
|
|
runtime.runtime_instance_id = "resource-test"
|
|
for name in ("request_id", "turn_id", "admission_snapshot_id",
|
|
"admission_id", "hold_id", "turn_fencing_token",
|
|
"billing_fencing_token"):
|
|
setattr(admission, name, name)
|
|
committed = []
|
|
|
|
class Sink:
|
|
async def commit(self, event):
|
|
committed.append(event)
|
|
return "committed"
|
|
|
|
handle.host.runtime_event_sink = Sink()
|
|
release_sync = threading.Event()
|
|
release_async = asyncio.Event()
|
|
started = threading.Event()
|
|
watchdog_fired = threading.Event()
|
|
loop = asyncio.get_running_loop()
|
|
|
|
class HangingSync:
|
|
calls = 0
|
|
|
|
def close(self):
|
|
self.calls += 1
|
|
started.set()
|
|
release_sync.wait()
|
|
|
|
class HangingAsync:
|
|
calls = 0
|
|
|
|
async def aclose(self):
|
|
self.calls += 1
|
|
started.set()
|
|
await release_async.wait()
|
|
|
|
hanging = HangingSync() if close_kind == "sync" else HangingAsync()
|
|
good = Client()
|
|
run._owned_clients.update({id(c): c for c in (hanging, good)})
|
|
pending = []
|
|
|
|
def emergency_release():
|
|
watchdog_fired.set()
|
|
release_sync.set()
|
|
loop.call_soon_threadsafe(release_async.set)
|
|
|
|
# A blocked event loop cannot execute an asyncio timeout or finally.
|
|
watchdog = threading.Timer(5.0, emergency_release)
|
|
watchdog.start()
|
|
try:
|
|
cleanup = asyncio.create_task(run.cancel("resource-test"))
|
|
pending.append(cleanup)
|
|
for _ in range(100):
|
|
if started.is_set() and good.calls == 1:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert not watchdog_fired.is_set(), "sync close blocked the event loop"
|
|
assert started.is_set(), "hanging resource close was not started"
|
|
assert good.calls == 1, "hanging close starved another owned resource"
|
|
assert id(good) not in run._owned_clients
|
|
assert run._owned_clients.get(id(hanging)) is hanging
|
|
assert handle.run is run
|
|
assert run._terminal_event is None
|
|
|
|
# Public observation must be bounded; the run owns finalization.
|
|
done, _ = await asyncio.wait({cleanup}, timeout=3.0)
|
|
assert cleanup in done, "cleanup observation must be bounded"
|
|
first_error = cleanup.exception()
|
|
owner = run._cancel_task
|
|
assert owner is not None and not owner.done()
|
|
assert await run.wait_stopped(timeout=0.01) == "unknown"
|
|
with pytest.raises(TimeoutError):
|
|
await asyncio.wait_for(run.cancel("short-observer"), timeout=0.01)
|
|
assert not owner.done(), "caller timeout must not end the stop owner"
|
|
assert run._terminal_event is None
|
|
assert run._state != "TERMINAL"
|
|
assert hanging.calls == 1, "pending close must never be restarted"
|
|
assert run._owned_clients.get(id(hanging)) is hanging
|
|
assert not watchdog_fired.is_set()
|
|
|
|
# Only this deliberate release is recovery evidence. Teardown
|
|
# releases below cannot turn an earlier failure into a pass.
|
|
release_sync.set()
|
|
release_async.set()
|
|
for _ in range(100):
|
|
if not run._owned_clients:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert not run._owned_clients, "released close was not reconciled"
|
|
assert await asyncio.wait_for(run.cancel("retry"), timeout=1.0) == "cancelled"
|
|
assert await run.wait_stopped(timeout=1.0) == "cancelled"
|
|
assert first_error is None
|
|
assert cleanup.result() == "unknown"
|
|
assert run._cancel_task is owner
|
|
assert run._state == "TERMINAL"
|
|
events = [event async for event in run.stream()]
|
|
assert len(events) == len(committed) == 1
|
|
assert events[0].payload["kind"] == "run_terminal"
|
|
assert events[0].payload["outcome"] == "cancelled"
|
|
assert await run.cancel("duplicate") == "cancelled"
|
|
assert len(committed) == 1
|
|
assert hanging.calls == good.calls == 1
|
|
assert not watchdog_fired.is_set()
|
|
finally:
|
|
watchdog.cancel()
|
|
watchdog.join()
|
|
release_sync.set()
|
|
release_async.set()
|
|
if run._cleanup_task is not None:
|
|
pending.append(run._cleanup_task)
|
|
if pending:
|
|
await asyncio.wait_for(
|
|
asyncio.gather(*pending, return_exceptions=True), timeout=2.0
|
|
)
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_construction_hanging_close_retains_owner_until_retirement(monkeypatch):
|
|
# New distinct node: registered 0/5 before its first execution.
|
|
async def scenario():
|
|
from EvoScientist.llm.runtime import _construction_owner
|
|
runtime, admission, handle = setup_runtime(monkeypatch)
|
|
runtime._prepared_tombstones = {}
|
|
runtime._prepared_requests = {("s", "r", "t"): "p"}
|
|
handle.grant = NS(subject_id="s", request_id="r", turn_id="t")
|
|
handle.request_digest = "request-digest"
|
|
handle.quote.preparation_id = "p"
|
|
release = asyncio.Event()
|
|
good = Client()
|
|
|
|
class Hanging:
|
|
calls = 0
|
|
|
|
async def aclose(self):
|
|
self.calls += 1
|
|
await release.wait()
|
|
|
|
hanging = Hanging()
|
|
|
|
def build(*args):
|
|
owner = _construction_owner.get()
|
|
owner._owned_clients.update({id(c): c for c in (hanging, good)})
|
|
raise ValueError("construction")
|
|
|
|
runtime._build_model_set = build
|
|
start = asyncio.create_task(runtime.start_web_run(admission))
|
|
try:
|
|
done, _ = await asyncio.wait({start}, timeout=1.0)
|
|
assert start in done, "construction rollback held handle lock indefinitely"
|
|
assert start.exception() is not None
|
|
assert handle.state == "CONSTRUCTION_FAILED"
|
|
assert not handle.lock.locked()
|
|
owner = handle.run
|
|
assert owner is not None
|
|
assert runtime._prepared["p"] is handle
|
|
assert owner._owned_clients.get(id(hanging)) is hanging
|
|
assert good.calls == hanging.calls == 1
|
|
assert owner._terminal_event is None
|
|
with pytest.raises(Exception):
|
|
await runtime.start_web_run(admission)
|
|
release.set()
|
|
await asyncio.wait_for(asyncio.shield(owner._cleanup_task), timeout=1.0)
|
|
assert not owner._owned_clients
|
|
assert "p" not in runtime._prepared
|
|
assert ("s", "r", "t") in runtime._prepared_tombstones
|
|
with pytest.raises(Exception):
|
|
await runtime.start_web_run(admission)
|
|
assert hanging.calls == 1
|
|
finally:
|
|
release.set()
|
|
await asyncio.gather(start, return_exceptions=True)
|
|
if handle.run is not None and handle.run._cleanup_task is not None:
|
|
await asyncio.wait_for(
|
|
asyncio.shield(handle.run._cleanup_task), timeout=1.0
|
|
)
|
|
asyncio.run(scenario())
|
|
|
|
|
|
@pytest.mark.parametrize("failures", [2, 99])
|
|
def test_close_failures_are_isolated_and_bounded(monkeypatch, failures):
|
|
async def scenario():
|
|
runtime, admission, handle = setup_runtime(monkeypatch)
|
|
run = _EvoWebRun(runtime=runtime, admission=admission, prepared=handle,
|
|
agent=None, model_set=AgentModelSet(None, None, None))
|
|
bad, good, other = Client(failures), Client(), Client(failures)
|
|
run._owned_clients.update({id(c): c for c in (bad, good, other)})
|
|
caught = None
|
|
try:
|
|
await run._ensure_cleanup()
|
|
except BaseException as exc:
|
|
caught = exc
|
|
assert good.calls == 1, "one failure must not skip remaining resources"
|
|
assert bad.calls == other.calls == 3
|
|
assert len(run._cleanup_failures) == 2
|
|
assert all(len(errors) == min(failures, 3) for errors in run._cleanup_failures.values())
|
|
if failures == 2:
|
|
assert caught is None
|
|
assert not run._owned_clients
|
|
await run._ensure_cleanup()
|
|
else:
|
|
assert isinstance(caught, ExceptionGroup)
|
|
assert len(caught.exceptions) == 2
|
|
assert len(run._owned_clients) == 2
|
|
with pytest.raises(ExceptionGroup):
|
|
await run._terminal_locked("completed")
|
|
assert run._terminal_event is None
|
|
assert bad.calls == other.calls == 3
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_borrowed_model_clients_are_not_closed(monkeypatch):
|
|
async def scenario():
|
|
runtime, admission, handle = setup_runtime(monkeypatch)
|
|
shared = Client()
|
|
model = NS(root_async_client=NS(_client=shared))
|
|
run = _EvoWebRun(runtime=runtime, admission=admission, prepared=handle,
|
|
agent=None, model_set=AgentModelSet(model, model, model))
|
|
await run._ensure_cleanup()
|
|
assert shared.calls == 0
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_custom_factory_explicit_resource_transfer(monkeypatch):
|
|
async def scenario():
|
|
from EvoScientist.llm import contracts
|
|
result_type = getattr(contracts, "ModelFactoryResult", None)
|
|
assert result_type is not None, "explicit factory ownership contract missing"
|
|
runtime, admission, handle = setup_runtime(monkeypatch)
|
|
owned, borrowed = Client(), Client()
|
|
model = NS(root_async_client=borrowed)
|
|
runtime.model_factory = lambda **kwargs: result_type(model, (owned,))
|
|
runtime._model_factory_kwargs = lambda route: {}
|
|
runtime._attach_route_metadata = lambda model, *args: model
|
|
route = NS(identity=NS(model_id="test"), runtime_provider="test", invocation_plan=object())
|
|
handle.snapshot.purpose_route_call_bounds = {"main_agent": (NS(route_identity=route.identity),)}
|
|
def build(*args):
|
|
built = runtime._build_model(route, "main_agent", handle.snapshot)
|
|
assert built is model
|
|
return AgentModelSet(built, built, built)
|
|
runtime._build_model_set = build
|
|
runtime.agent_factory = lambda *args: object()
|
|
run = await runtime.start_web_run(admission)
|
|
await run._ensure_cleanup()
|
|
assert owned.calls == 1
|
|
assert borrowed.calls == 0
|
|
asyncio.run(scenario())
|