Port from lobehub/lobehub#19329: surface environment recreation in terminal tool results

When a persistent Docker container is removed out-of-band or a Vercel
sandbox hits a terminal state, the backend silently recreates it and
retries. The model then keeps assuming background processes and
non-persisted files from earlier commands still exist.

Backends now call _mark_recreated() after a successful recovery;
BaseEnvironment.execute() folds the one-shot flag into the result as
environment_recreated, and finalize_foreground_result() attaches a
model-facing warning field explaining what may have been lost.

Ported from lobehub/lobehub#19329 (sandbox recreation surfacing),
adapted to hermes environment backends and tool-result JSON.
This commit is contained in:
Teknium
2026-09-09 21:17:23 -07:00
parent 6f62aff7f2
commit a49a9d79b3
5 changed files with 83 additions and 0 deletions
@@ -0,0 +1,56 @@
"""The model must be told when the backend replaced its container/sandbox
mid-command (ported from lobehub/lobehub#19329): silent recovery leaves the
agent assuming background processes and unsynced files survived."""
import json
from tools.environments import docker as docker_env
from tools.environments.local import LocalEnvironment
from tools.terminal_tool_result import finalize_foreground_result
def test_execute_folds_recreation_flag_once():
"""A pending recreation mark surfaces as ``environment_recreated: True``
on the next execute() result and is consumed — the following command
reports a clean result. Exercises the real BaseEnvironment.execute path."""
env = LocalEnvironment(cwd=".", timeout=30)
try:
env._mark_recreated()
result = env.execute("echo hi")
assert result["returncode"] == 0
assert result.get("environment_recreated") is True
result2 = env.execute("echo again")
assert "environment_recreated" not in result2
finally:
env.cleanup()
def test_docker_recovery_marks_pending_and_finalizer_warns(monkeypatch):
"""Docker's out-of-band recovery sets the pending mark, and the tool-layer
finalizer turns the flag into a model-facing warning — omitted entirely
when the backend never flagged a recreation."""
env = docker_env.DockerEnvironment.__new__(docker_env.DockerEnvironment)
env._container_id = "old"
env._labels = {}
env._image = ""
monkeypatch.setattr(
docker_env.DockerEnvironment, "_find_reusable_container",
lambda self, *a: ("newcid", "running"))
monkeypatch.setattr(docker_env.DockerEnvironment, "init_session", lambda self: None)
assert env._recreate_container() is True
assert getattr(env, "_recreated_notice_pending", False) is True
common: dict = dict(
command="echo hi", env=env, env_type="docker", effective_task_id="t",
task_id="t", session_id="s", session_key="k", workdir=None,
command_cwd=None, approval_note=None)
flagged = json.loads(finalize_foreground_result(
result={"output": "hi", "returncode": 0, "environment_recreated": True}, **common))
assert "recreated" in flagged.get("environment_recreated", "")
assert "may be lost" in flagged["environment_recreated"]
clean = json.loads(finalize_foreground_result(
result={"output": "hi", "returncode": 0}, **common))
assert "environment_recreated" not in clean
+12
View File
@@ -476,6 +476,15 @@ class BaseEnvironment(ABC):
trigger their FileSyncManager here; bind-mount backends and Local don't."""
pass
def _mark_recreated(self) -> None:
"""Flag that the live container/sandbox was replaced while serving the
current command. ``execute`` folds the flag into the result as
``environment_recreated`` so the tool layer can warn the model that
background processes died and non-persisted files may be gone —
without this the recovery is silent and the model keeps assuming the
old workspace state (lobehub/lobehub#19329 class)."""
self._recreated_notice_pending = True
# --- Unified execute() ---
def execute(
self,
@@ -566,6 +575,9 @@ class BaseEnvironment(ABC):
{"output": f"[Command timed out after {effective_timeout}s]", "returncode": 124}
if bounded.timed_out else bounded.value)
self._update_cwd(result)
if getattr(self, "_recreated_notice_pending", False):
self._recreated_notice_pending = False
result["environment_recreated"] = True
return result
def _kill_spawned_tree(self, spawned) -> None:
+1
View File
@@ -918,6 +918,7 @@ class DockerEnvironment(BaseEnvironment):
return False
logger.info("Recovery successful — new container %s", (self._container_id or "")[:12])
self._mark_recreated()
return True
def execute(self, command: str, cwd: str = "", **kwargs) -> dict:
+1
View File
@@ -265,6 +265,7 @@ class VercelSandboxEnvironment(BaseEnvironment):
return
logger.warning("Vercel: sandbox entered state %s for task %s; recreating", status, self._task_id)
self._close_sandbox_client(sandbox)
self._mark_recreated()
self._attach_fresh_sandbox(requested_cwd)
def _run_checked(self, script: str, label: str) -> None:
+13
View File
@@ -76,6 +76,18 @@ _EXIT_CODE_SEMANTICS: dict[str, dict[int, str]] = {
"git": {1: "Non-zero exit (often normal — e.g. 'git diff' returns 1 when files differ)"},
}
# Model-facing warning attached when the backend replaced its container/sandbox
# mid-command (out-of-band removal, terminal sandbox state). Persistent-filesystem
# state was restored, but background processes and anything outside the synced
# paths are gone (ported from lobehub/lobehub#19329).
_ENV_RECREATED_NOTE = (
"The execution environment was recreated while running this command "
"(the previous container/sandbox was gone). Persistent files were restored "
"where the backend supports it, but background processes and any files "
"outside persisted paths from earlier commands may be lost — verify state "
"before relying on prior work."
)
def _interpret_exit_code(command: str, exit_code: int) -> str | None:
"""Note for a non-zero exit code that is informational rather than an
@@ -251,6 +263,7 @@ def finalize_foreground_result(
# metadata is present only when output overflowed the capture window.
optional_fields: list[tuple[str, Any]] = [
("cwd", changed_cwd),
("environment_recreated", _ENV_RECREATED_NOTE if result.get("environment_recreated") else None),
*_redact_spill_file(result.get("full_output_path"), result.get("output_total_chars"), command),
("verification_evidence", _verification_evidence(
command, command_cwd, session_id or task_id or effective_task_id or "default",