EvoScientist-Multi/tests/test_execution_adapter_approval_tools.py

"""B02 fragmented tool approval and cancellation through real runtime runs."""
import asyncio
import json
import os
from pathlib import Path
import subprocess
import sys
import uuid

import pytest

if __name__ == "__main__":
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from tests.test_execution_adapter_stop_contract import (
    HeartbeatExecutor, LoopbackSSE, build_run, isolated_network,
)

NODE = "test_fragmented_manual_checkpoint_child_run_stops_native_tree"


class FragmentedToolSSE(LoopbackSSE):
    async def handle(self, reader, writer):
        task = asyncio.current_task()
        self.tasks.add(task)
        self.writers.add(writer)
        try:
            header = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), 5)
            headers = dict(line.split(b":", 1) for line in header.split(b"\r\n")[1:] if b":" in line)
            length = int(next(v for k, v in headers.items() if k.lower() == b"content-length"))
            assert header.startswith(b"POST /v1/chat/completions ")
            body = json.loads(await reader.readexactly(length))
            self.requests.append(body)
            assert body["stream"] is True
            assert any(t["function"]["name"] == "execute" for t in body["tools"])
            assert len(self.requests) == 1, "unexpected post-tool model request"
            writer.write(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n")
            fragments = ('{"comm', 'and":"b02-', 'heartbeat"}')
            for index, fragment in enumerate(fragments):
                call = {"index": 0, "function": {"arguments": fragment}}
                if index == 0:
                    call.update(id="b02-fragmented", type="function")
                    call["function"]["name"] = "execute"
                chunk = {"id": "b02-fragmented", "object": "chat.completion.chunk", "created": 1,
                         "model": body["model"], "choices": [{"index": 0, "delta": {"tool_calls": [call]}, "finish_reason": None}]}
                writer.write(b"data: " + json.dumps(chunk).encode() + b"\n\n")
                await writer.drain()
                await asyncio.sleep(.02)
            chunk["choices"] = [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]
            writer.write(b"data: " + json.dumps(chunk).encode() + b"\n\ndata: [DONE]\n\n")
            await writer.drain()
            self.requested.set()
        except Exception as exc:
            self.errors.append(exc)
            self.requested.set()
        finally:
            writer.close()
            await writer.wait_closed()
            self.writers.discard(writer)
            self.tasks.discard(task)


def test_fragmented_manual_checkpoint_child_run_stops_native_tree(tmp_path, monkeypatch, isolated_network):
    from EvoScientist import workspace_files
    from EvoScientist.native_sandbox import NativeWorkspaceBackend
    from EvoScientist.llm.contracts import AgentInputV3
    from langgraph.types import Command
    from tests.test_web_model_runtime import _preparation, _admission

    executor = HeartbeatExecutor(tmp_path)
    calls = []
    original = executor.execute

    def counted(command, **kwargs):
        calls.append(command)
        return original(command, **kwargs)

    executor.execute = counted
    scoped = workspace_files.ScopedFilesystemBackend

    class LocalBackend(NativeWorkspaceBackend):
        def __init__(self, root):
            scoped.__init__(self, root)
            self._executor = executor
            self._sandbox_id = "b02-whitelist"

    monkeypatch.setattr(workspace_files, "ScopedFilesystemBackend", LocalBackend)

    async def scenario():
        child = None
        async with FragmentedToolSSE().serve() as server:
            parent, sink = await build_run(tmp_path, monkeypatch, server)
            try:
                await asyncio.wait_for(server.requested.wait(), 8)
                assert await parent.wait_stopped(timeout=8) == "awaiting_input"
                assert not server.errors
                config = {"configurable": {"thread_id": parent._input.checkpoint_thread_id}}
                state = await parent._agent.aget_state(config)
                assert state.interrupts, (state.next, parent._terminal_event)
                payload = parent._terminal_event.payload
                assert payload["outcome"] == "awaiting_input"
                assert payload["checkpoint_thread_id"] == state.config["configurable"]["thread_id"]
                assert payload["checkpoint_id"] == state.config["configurable"]["checkpoint_id"]
                assert payload["checkpoint_id"]
                assert payload["checkpoint_ns"] == state.config["configurable"]["checkpoint_ns"]
                assert payload["pending_interrupts"] == [
                    {"id": item.id, "value": item.value} for item in state.interrupts
                ]
                assert json.loads(json.dumps(payload))["pending_interrupts"] == payload["pending_interrupts"]
                parent_events = [event async for event in parent.stream()]
                assert len([e for e in parent_events if e.payload.get("kind") == "run_terminal"]) == 1
                assert state.values["_verified_review_mode"]["mode"] == "manual"
                assert state.interrupts[0].value["action_requests"][0]["args"] == {"command": "b02-heartbeat"}
                await asyncio.sleep(.15)
                assert calls == [] and not executor.started.is_set()
                assert not executor.child_pid.exists() and not executor.heartbeat.exists()
                runtime = parent._runtime
                authority = runtime.quote_authority
                approved = AgentInputV3(Command(resume={"decisions": [{"type": "approve"}]}), parent._input.checkpoint_thread_id)
                template = _preparation(authority, approved, title_policy="disabled")
                fields = ("turn_id", "thread_id", "subject_id", "requested_model_ref", "plan", "roles",
                          "requires_vision", "reasoning_effort", "title_policy", "gateway_input_digest",
                          "checkpoint_thread_id", "checkpoint_snapshot_id", "turn_fencing_token")
                grant = authority.sign_preparation(**{k: getattr(template, k) for k in fields},
                                                   request_id=str(uuid.uuid4()), ttl_ms=60_000)
                quote = await runtime.prepare_model_run(grant, approved, parent._host)
                child = await runtime.start_web_run(_admission(authority, quote))
                async with asyncio.timeout(8):
                    while not executor.heartbeat.exists() or not executor.child_pid.exists():
                        if child._agent_task.done():
                            pytest.fail(f"approval child ended before execution: {child._terminal_event}")
                        await asyncio.sleep(.02)
                assert calls == ["b02-heartbeat"]
                assert await child.cancel("manual-tool-stop") == "cancelled"
                assert await child.wait_stopped(timeout=4) == "cancelled"
                await executor.assert_stopped_before_teardown()
                assert child._agent_task.done()
                events = [event async for event in child.stream()]
                await asyncio.sleep(.15)
                assert len(server.requests) == 1 and not server.errors
                assert calls == ["b02-heartbeat"]
                assert len([e for e in events if e.payload.get("kind") == "run_terminal"]) == 1
                print(json.dumps({"evidence": "manual_fragmented_runtime_stop", "argument_fragments": 3,
                                  "manual_interrupt": True, "executions": len(calls), "http_requests": len(server.requests),
                                  "worker_finished": executor.finished.is_set(), "child_run_stopped": child._agent_task.done(),
                                  "parent_outcome": payload["outcome"],
                                  "checkpoint_id": payload["checkpoint_id"],
                                  "checkpoint_ns": payload["checkpoint_ns"],
                                  "pending_interrupts": payload["pending_interrupts"]}), flush=True)
            finally:
                if child is not None:
                    await child.cancel("teardown")
                await parent.cancel("teardown")
                executor.emergency_cleanup()

    asyncio.run(scenario())


if __name__ == "__main__":
    repo = Path(__file__).resolve().parents[1]
    root = repo.parent / ".hermes/test-runtime/unified-execution/b02"
    root.mkdir(parents=True, exist_ok=True)
    ledger = root / "attempts.jsonl"
    node = "tests/test_execution_adapter_approval_tools.py::" + NODE
    records = [json.loads(line) for line in ledger.read_text().splitlines()] if ledger.exists() else []
    attempt = 1 + sum(r.get("phase") == "start" and r.get("node") == node for r in records)
    limit = 5
    assert attempt <= limit, "node budget exhausted"
    home = root / (NODE + f"-{attempt}-home")
    config = home / "config"
    config.mkdir(parents=True, exist_ok=True)
    env = {"HOME": str(home), "PATH": str(repo / ".venv/bin") + ":/usr/bin:/bin",
           "PYTHONPATH": str(repo), "PYTHON_DOTENV_DISABLED": "1", "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1",
           "PYTHONDONTWRITEBYTECODE": "1", "EVOSCIENTIST_HOME": str(home), "EVOSCIENTIST_CONFIG_DIR": str(config),
           "XDG_CONFIG_HOME": str(config), "EVOSCIENTIST_DATA_DIR": str(home / "data"),
           "EVOSCIENTIST_WORKSPACE_DIR": str(home / "workspace"), "EVOSCIENTIST_SKILLS_DIR": str(home / "skills"),
           "EVOSCIENTIST_MEMORIES_DIR": str(home / "memory")}
    command = [str(repo / ".venv/bin/python"), "-m", "pytest", "--noconftest", "-p", "no:cacheprovider",
               "-q", "-s", "--tb=short", "--disable-warnings", node]
    with ledger.open("a") as handle:
        if attempt == 1:
            handle.write(json.dumps({"phase": "registered", "node": node, "used": 0, "limit": limit, "command": command}) + "\n")
        elif not any(r.get("node") == node and r.get("limit") == limit for r in records):
            handle.write(json.dumps({"phase": "authorization_extension", "node": node,
                                     "used": attempt - 1, "limit": limit,
                                     "authority": "explicit user total-five authorization",
                                     "preserve_prior_attempts": True}) + "\n")
        handle.write(json.dumps({"phase": "start", "node": node, "attempt": attempt, "command": command}) + "\n")
    try:
        result = subprocess.run(command, cwd=repo, env=env, capture_output=True, text=True, timeout=60)
    except subprocess.TimeoutExpired as exc:
        output = exc.stdout or b""
        if isinstance(output, bytes):
            output = output.decode(errors="replace")
        result = subprocess.CompletedProcess(command, 124, output, "runner timeout\n")
    log = root / (NODE + f"-{attempt}.log")
    log.write_text(result.stdout + result.stderr)
    with ledger.open("a") as handle:
        handle.write(json.dumps({"phase": "result", "node": node, "attempt": attempt, "exit_code": result.returncode,
                                "log": str(log), "remaining": limit-attempt}) + "\n")
    print(result.stdout + result.stderr)
    sys.exit(result.returncode)