Port from lobehub/lobehub#19329: surface environment recreation in terminal tool results
When a persistent Docker container is removed out-of-band or a Vercel sandbox hits a terminal state, the backend silently recreates it and retries. The model then keeps assuming background processes and non-persisted files from earlier commands still exist. Backends now call _mark_recreated() after a successful recovery; BaseEnvironment.execute() folds the one-shot flag into the result as environment_recreated, and finalize_foreground_result() attaches a model-facing warning field explaining what may have been lost. Ported from lobehub/lobehub#19329 (sandbox recreation surfacing), adapted to hermes environment backends and tool-result JSON.
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
"""The model must be told when the backend replaced its container/sandbox
|
||||
mid-command (ported from lobehub/lobehub#19329): silent recovery leaves the
|
||||
agent assuming background processes and unsynced files survived."""
|
||||
|
||||
import json
|
||||
|
||||
from tools.environments import docker as docker_env
|
||||
from tools.environments.local import LocalEnvironment
|
||||
from tools.terminal_tool_result import finalize_foreground_result
|
||||
|
||||
|
||||
def test_execute_folds_recreation_flag_once():
|
||||
"""A pending recreation mark surfaces as ``environment_recreated: True``
|
||||
on the next execute() result and is consumed — the following command
|
||||
reports a clean result. Exercises the real BaseEnvironment.execute path."""
|
||||
env = LocalEnvironment(cwd=".", timeout=30)
|
||||
try:
|
||||
env._mark_recreated()
|
||||
result = env.execute("echo hi")
|
||||
assert result["returncode"] == 0
|
||||
assert result.get("environment_recreated") is True
|
||||
|
||||
result2 = env.execute("echo again")
|
||||
assert "environment_recreated" not in result2
|
||||
finally:
|
||||
env.cleanup()
|
||||
|
||||
|
||||
def test_docker_recovery_marks_pending_and_finalizer_warns(monkeypatch):
|
||||
"""Docker's out-of-band recovery sets the pending mark, and the tool-layer
|
||||
finalizer turns the flag into a model-facing warning — omitted entirely
|
||||
when the backend never flagged a recreation."""
|
||||
env = docker_env.DockerEnvironment.__new__(docker_env.DockerEnvironment)
|
||||
env._container_id = "old"
|
||||
env._labels = {}
|
||||
env._image = ""
|
||||
monkeypatch.setattr(
|
||||
docker_env.DockerEnvironment, "_find_reusable_container",
|
||||
lambda self, *a: ("newcid", "running"))
|
||||
monkeypatch.setattr(docker_env.DockerEnvironment, "init_session", lambda self: None)
|
||||
assert env._recreate_container() is True
|
||||
assert getattr(env, "_recreated_notice_pending", False) is True
|
||||
|
||||
common: dict = dict(
|
||||
command="echo hi", env=env, env_type="docker", effective_task_id="t",
|
||||
task_id="t", session_id="s", session_key="k", workdir=None,
|
||||
command_cwd=None, approval_note=None)
|
||||
|
||||
flagged = json.loads(finalize_foreground_result(
|
||||
result={"output": "hi", "returncode": 0, "environment_recreated": True}, **common))
|
||||
assert "recreated" in flagged.get("environment_recreated", "")
|
||||
assert "may be lost" in flagged["environment_recreated"]
|
||||
|
||||
clean = json.loads(finalize_foreground_result(
|
||||
result={"output": "hi", "returncode": 0}, **common))
|
||||
assert "environment_recreated" not in clean
|
||||
@@ -476,6 +476,15 @@ class BaseEnvironment(ABC):
|
||||
trigger their FileSyncManager here; bind-mount backends and Local don't."""
|
||||
pass
|
||||
|
||||
def _mark_recreated(self) -> None:
|
||||
"""Flag that the live container/sandbox was replaced while serving the
|
||||
current command. ``execute`` folds the flag into the result as
|
||||
``environment_recreated`` so the tool layer can warn the model that
|
||||
background processes died and non-persisted files may be gone —
|
||||
without this the recovery is silent and the model keeps assuming the
|
||||
old workspace state (lobehub/lobehub#19329 class)."""
|
||||
self._recreated_notice_pending = True
|
||||
|
||||
# --- Unified execute() ---
|
||||
def execute(
|
||||
self,
|
||||
@@ -566,6 +575,9 @@ class BaseEnvironment(ABC):
|
||||
{"output": f"[Command timed out after {effective_timeout}s]", "returncode": 124}
|
||||
if bounded.timed_out else bounded.value)
|
||||
self._update_cwd(result)
|
||||
if getattr(self, "_recreated_notice_pending", False):
|
||||
self._recreated_notice_pending = False
|
||||
result["environment_recreated"] = True
|
||||
return result
|
||||
|
||||
def _kill_spawned_tree(self, spawned) -> None:
|
||||
|
||||
@@ -918,6 +918,7 @@ class DockerEnvironment(BaseEnvironment):
|
||||
return False
|
||||
|
||||
logger.info("Recovery successful — new container %s", (self._container_id or "")[:12])
|
||||
self._mark_recreated()
|
||||
return True
|
||||
|
||||
def execute(self, command: str, cwd: str = "", **kwargs) -> dict:
|
||||
|
||||
@@ -265,6 +265,7 @@ class VercelSandboxEnvironment(BaseEnvironment):
|
||||
return
|
||||
logger.warning("Vercel: sandbox entered state %s for task %s; recreating", status, self._task_id)
|
||||
self._close_sandbox_client(sandbox)
|
||||
self._mark_recreated()
|
||||
self._attach_fresh_sandbox(requested_cwd)
|
||||
|
||||
def _run_checked(self, script: str, label: str) -> None:
|
||||
|
||||
@@ -76,6 +76,18 @@ _EXIT_CODE_SEMANTICS: dict[str, dict[int, str]] = {
|
||||
"git": {1: "Non-zero exit (often normal — e.g. 'git diff' returns 1 when files differ)"},
|
||||
}
|
||||
|
||||
# Model-facing warning attached when the backend replaced its container/sandbox
|
||||
# mid-command (out-of-band removal, terminal sandbox state). Persistent-filesystem
|
||||
# state was restored, but background processes and anything outside the synced
|
||||
# paths are gone (ported from lobehub/lobehub#19329).
|
||||
_ENV_RECREATED_NOTE = (
|
||||
"The execution environment was recreated while running this command "
|
||||
"(the previous container/sandbox was gone). Persistent files were restored "
|
||||
"where the backend supports it, but background processes and any files "
|
||||
"outside persisted paths from earlier commands may be lost — verify state "
|
||||
"before relying on prior work."
|
||||
)
|
||||
|
||||
|
||||
def _interpret_exit_code(command: str, exit_code: int) -> str | None:
|
||||
"""Note for a non-zero exit code that is informational rather than an
|
||||
@@ -251,6 +263,7 @@ def finalize_foreground_result(
|
||||
# metadata is present only when output overflowed the capture window.
|
||||
optional_fields: list[tuple[str, Any]] = [
|
||||
("cwd", changed_cwd),
|
||||
("environment_recreated", _ENV_RECREATED_NOTE if result.get("environment_recreated") else None),
|
||||
*_redact_spill_file(result.get("full_output_path"), result.get("output_total_chars"), command),
|
||||
("verification_evidence", _verification_evidence(
|
||||
command, command_cwd, session_id or task_id or effective_task_id or "default",
|
||||
|
||||
Reference in New Issue
Block a user