"""Base class for all Hermes execution environment backends. Unified spawn-per-call model: every command spawns a fresh ``bash -c`` process. A session snapshot (env vars, functions, aliases) is captured once at init and re-sourced before each command. CWD persists via in-band stdout markers (remote) or a temp file (local). Cohesive pieces live in sibling modules and are re-exported here so ``from tools.environments.base import X`` / ``patch("tools.environments.base.X")`` keep working: ``base_output`` (collector, ProcessHandle, stdin/drain plumbing), ``base_session_env`` (snapshot/wrapper shell scripting), ``base_wait`` (tracing). """ import json import logging import os import shlex import threading import time import uuid from abc import ABC, abstractmethod from pathlib import Path from typing import Callable, Iterable from hermes_constants import get_hermes_home from tools.interrupt import is_interrupted, is_thread_interrupted from tools.environments.base_output import ( # noqa: F401 ProcessHandle, _BoundedOutputCollector, _ThreadedProcessHandle, _UNBOUNDED_CAPTURE_CHARS, _finalize_wait_result, _new_output_collector, _pipe_stdin, _popen_bash, _start_drain_thread, ) from tools.environments.base_session_env import ( # noqa: F401 _SHELL_ENV_NAME_RE, _SNAP_TMP, _SNAP_TMP_SUFFIX, _SNAPSHOT_EXCLUDED_ENV_REGEX, _cwd_marker, _export_dump_excluding_session_vars, _snapshot_bootstrap_script, _split_cwd_marker, _wrap_command_script, ) from tools.environments.base_wait import _WaitTrace from tools.environments.path_utils import ( # noqa: F401 _SANDBOX_DIR_HASH_LEN, _SANDBOX_DIR_MAX_LEN, _SANDBOX_DIR_UNSAFE_RE, sanitize_task_id_for_path, ) logger = logging.getLogger(__name__) # Opt-in debug tracing for the interrupt/activity/poll machinery # (HERMES_DEBUG_INTERRUPT=1). Off by default to avoid flooding gateway logs. _DEBUG_INTERRUPT = bool(os.getenv("HERMES_DEBUG_INTERRUPT")) # Extra seconds the ``run_bounded_sync`` backstop waits past the inner # ``_wait_for_process`` deadline. The inner poll loop returns partial output + # returncode 124; the outer bound only fires when that loop itself never # returns (a blocked wait that silently disables asyncio timers). Keep it # small so a healthy timeout still comes from the inner path. _EXECUTE_WAIT_BOUND_GRACE_S = 2.0 if _DEBUG_INTERRUPT: # quiet_mode forces the `tools` logger to ERROR on CLI startup, which would # swallow every trace; force this logger back to INFO in the opt-in case. logger.setLevel(logging.INFO) # Thread-local activity callback: the agent sets it before a tool call so # long-running _wait_for_process loops can report liveness to the gateway. _activity_callback_local = threading.local() class EnvironmentConnectionError(RuntimeError): """Infrastructure/connection-class failure of a terminal backend. Raised when the backend itself is unreachable (SSH host down, Docker daemon not running, remote file sync failing on a dead link) — never for a command that merely exited nonzero. Subclassing RuntimeError keeps every existing ``except RuntimeError`` catcher working. ``terminal_tool`` turns this into a structured ``status: "degraded"`` result (config ``terminal.degraded_mode``); the failed backend is never cached, so a later call retries from scratch. """ def __init__(self, reason: str, *, retry_hint: str = ""): super().__init__(reason) self.reason = reason self.retry_hint = retry_hint or ( "This is an infrastructure failure, not a command failure. " "Verify the backend is reachable (network, service running, " "credentials), then retry the same command — recovery is " "automatic once the backend is back." ) def set_activity_callback(cb: Callable[[str], None] | None) -> None: """Register a callback that _wait_for_process fires periodically.""" _activity_callback_local.callback = cb def get_activity_callback() -> Callable[[str], None] | None: """Return the thread-local activity callback (see ``set_activity_callback``). For callers that must capture the calling thread's callback before handing work to another thread — a freshly spawned thread cannot read it back. """ return getattr(_activity_callback_local, "callback", None) def touch_activity_if_due(state: dict, label: str) -> None: """Fire the activity callback at most once every ``state['interval']`` seconds. *state* must contain ``last_touch`` and ``start`` (monotonic timestamps); optional ``interval`` overrides the default 10 s cadence. Swallows all exceptions so callers don't need their own try/except. """ now = time.monotonic() interval = state.get("interval", 10.0) if now - state["last_touch"] < interval: return state["last_touch"] = now try: cb = get_activity_callback() if cb: elapsed = int(now - state["start"]) cb(f"{label} ({elapsed}s elapsed)") except Exception: pass def get_sandbox_dir() -> Path: """Host-side root for all sandbox storage (Docker workspaces, Singularity overlays/SIF cache). ``TERMINAL_SANDBOX_DIR`` overrides ``{HERMES_HOME}/sandboxes``.""" custom = os.getenv("TERMINAL_SANDBOX_DIR") p = Path(custom) if custom else get_hermes_home() / "sandboxes" p.mkdir(parents=True, exist_ok=True) return p def _load_json_store(path: Path) -> dict: """Load a JSON file as a dict, returning ``{}`` on any error.""" if path.exists(): try: return json.loads(path.read_text(encoding="utf-8")) except Exception: pass return {} def _save_json_store(path: Path, data: dict) -> None: """Write *data* as pretty-printed JSON to *path*.""" path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, indent=2), encoding="utf-8") def _file_mtime_key(host_path: str) -> tuple[float, int] | None: """Return ``(mtime, size)`` for cache comparison, or ``None`` if unreadable.""" try: st = Path(host_path).stat() return (st.st_mtime, st.st_size) except OSError: return None # --------------------------------------------------------------------------- # BaseEnvironment # --------------------------------------------------------------------------- class BaseEnvironment(ABC): """Common interface and unified execution flow for all Hermes backends. Subclasses implement ``_run_bash()`` and ``cleanup()``. The base class provides ``execute()`` with session snapshot sourcing, CWD tracking, interrupt handling, and timeout enforcement. """ # Subclasses that embed stdin as a heredoc (Modal, Daytona) set this. _stdin_mode: str = "pipe" # "pipe" or "heredoc" # True only when commands execute on the SAME host as the Hermes process # (LocalEnvironment). Controller-host facts (sys.platform, Path.home()) # describe the execution target only when this is True. is_local: bool = False # Snapshot creation timeout (override for slow cold-starts). _snapshot_timeout: int = 30 # Local and Docker override this because they resolve allowlisted values # through the active profile scope; other backends keep plain snapshots. _profile_scoped_passthrough: bool = False def get_temp_dir(self) -> str: """Backend temp directory for session artifacts (``/tmp`` in sandboxes; LocalEnvironment overrides for Termux where only ``TMPDIR`` is writable).""" return "/tmp" def __init__(self, cwd: str, timeout: int, env: dict = None): self.cwd = cwd self.timeout = timeout self.env = env or {} self._session_id = uuid.uuid4().hex[:12] temp_dir = self.get_temp_dir().rstrip("/") or "/" self._snapshot_path = f"{temp_dir}/hermes-snap-{self._session_id}.sh" self._cwd_file = f"{temp_dir}/hermes-cwd-{self._session_id}.txt" self._cwd_marker = _cwd_marker(self._session_id) self._snapshot_ready = False self._snapshot_passthrough_names: set[str] = set() # True when login bash is unusable (e.g. broken Git-for-Windows startup) # so execute() must fall back to non-login ``bash -c``, not ``bash -l``. self._prefer_nonlogin = False # ------------------------------------------------------------------ # Abstract methods # ------------------------------------------------------------------ def _run_bash( self, cmd_string: str, *, login: bool = False, timeout: int = 120, stdin_data: str | None = None, ) -> ProcessHandle: """Spawn a bash process to run *cmd_string*; every backend overrides this.""" raise NotImplementedError(f"{type(self).__name__} must implement _run_bash()") @abstractmethod def cleanup(self): """Release backend resources (container, instance, connection).""" ... # ------------------------------------------------------------------ # Session snapshot (init_session) # ------------------------------------------------------------------ def _additional_profile_scoped_passthrough_names(self) -> Iterable[str]: """Return backend-specific names that must not persist in snapshots.""" return () def _snapshot_excluded_passthrough_names(self) -> tuple[str, ...]: """Profile-scoped names that must not persist in the snapshot. Monotonic for the environment lifetime: an allowlist can be cleared after a value was captured, and retaining the exclusion keeps that old value from leaking to a later profile through the shared snapshot. """ if not self._profile_scoped_passthrough: return () try: from agent.secret_scope import is_multiplex_active if is_multiplex_active(): from tools.env_passthrough import get_all_passthrough names = ( *get_all_passthrough(), *self._additional_profile_scoped_passthrough_names(), ) self._snapshot_passthrough_names.update( name for name in names if isinstance(name, str) and _SHELL_ENV_NAME_RE.fullmatch(name) ) except Exception: logger.debug( "Could not refresh profile-scoped snapshot exclusions", exc_info=True, ) return tuple(sorted(self._snapshot_passthrough_names)) def init_session(self): """Capture the login shell environment into the snapshot file. Called once after construction. On success ``_snapshot_ready`` is set so commands source the snapshot instead of running under ``bash -l``. On failure, fall back to ``bash -l`` per command — unless a non-login probe shows login bash itself is dead, in which case prefer ``bash -c``. """ bootstrap = _snapshot_bootstrap_script( # ``_quote_cwd_for_cd`` / ``_quote_shell_path`` (not bare shlex.quote) # let the Windows subclass rewrite ``C:\\...`` to ``/c/...`` so the # bootstrap ``cd`` resolves and MSYS doesn't choke on drive paths. quoted_cwd=self._quote_cwd_for_cd(self.cwd), quoted_snap=self._quote_shell_path(self._snapshot_path), snap_tmp_template=self._quote_shell_path(self._snapshot_path + _SNAP_TMP_SUFFIX), excluded_names=self._snapshot_excluded_passthrough_names(), cwd_marker=self._cwd_marker, ) try: proc = self._run_bash(bootstrap, login=True, timeout=self._snapshot_timeout) result = self._wait_for_process(proc, timeout=self._snapshot_timeout) if int(result.get("returncode") or 0) != 0: raise RuntimeError( f"snapshot bootstrap failed with exit code {result.get('returncode')}" ) self._snapshot_ready = True self._update_cwd(result) logger.info( "Session snapshot created (session=%s, cwd=%s)", self._session_id, self.cwd, ) except Exception as exc: self._snapshot_ready = False self._prefer_nonlogin, detail = self._probe_nonlogin_fallback(str(exc)) if self._prefer_nonlogin: logger.warning( "init_session failed (session=%s): %s — " "login bash unusable; falling back to non-login bash -c", self._session_id, exc, ) else: logger.warning( "init_session failed (session=%s): %s — " "falling back to bash -l per command", self._session_id, detail, ) def _probe_nonlogin_fallback(self, detail: str) -> tuple[bool, str]: """Run ``true`` under non-login bash; return ``(prefer_nonlogin, detail)``.""" probe_timeout = min(15, self._snapshot_timeout) try: probe = self._run_bash("true", login=False, timeout=probe_timeout) probe_result = self._wait_for_process(probe, timeout=probe_timeout) prefer_nonlogin = int(probe_result.get("returncode") or 0) == 0 if not prefer_nonlogin: detail = (probe_result.get("stdout") or detail).strip() or detail return prefer_nonlogin, detail except Exception as probe_exc: return False, f"{detail}; non-login probe: {probe_exc}" # ------------------------------------------------------------------ # Command wrapping # ------------------------------------------------------------------ @staticmethod def _quote_cwd_for_cd(cwd: str) -> str: """Quote a ``cd`` target while preserving ``~`` expansion (``~/...`` goes through ``$HOME`` so suffixes with spaces stay one word).""" if cwd == "~": return cwd if cwd == "~/": return "$HOME" if cwd.startswith("~/"): return f"$HOME/{shlex.quote(cwd[2:])}" return shlex.quote(cwd) def _quote_shell_path(self, path: str) -> str: """Quote *path* for a bash script. LocalEnvironment overrides this to rewrite native/mixed Windows paths to ``/c/...``; remote backends are POSIX.""" return shlex.quote(path) def _wrap_command(self, command: str, cwd: str) -> str: """Build the full bash script that sources snapshot, cd's, runs command, re-dumps env vars, and emits CWD markers (see ``_wrap_command_script``).""" return _wrap_command_script( command, quoted_cwd=self._quote_cwd_for_cd(cwd), quoted_snap=self._quote_shell_path(self._snapshot_path), snap_tmp_template=self._quote_shell_path(self._snapshot_path + _SNAP_TMP_SUFFIX), passthrough_names=self._snapshot_excluded_passthrough_names(), snapshot_ready=self._snapshot_ready, cwd_marker=self._cwd_marker, ) @staticmethod def _embed_stdin_heredoc(command: str, stdin_data: str) -> str: """Append stdin_data as a shell heredoc to the command string (SDK backends).""" delimiter = f"HERMES_STDIN_{uuid.uuid4().hex[:12]}" return f"{command} << '{delimiter}'\n{stdin_data}\n{delimiter}" # ------------------------------------------------------------------ # Process lifecycle # ------------------------------------------------------------------ def _wait_for_process( self, proc: ProcessHandle, timeout: int = 120, *, bounded_capture: bool = False, watch_interrupt_tid: int | None = None, ) -> dict: """Poll-based wait with interrupt checking and stdout draining. Shared across all backends — not overridden. ``bounded_capture=True`` (foreground terminal-tool path only) retains at most ``tool_output.max_bytes`` in a head/tail window so a verbose subprocess cannot OOM the process; the default keeps full fidelity for internal consumers (file-op ``cat`` reads, RPC reads) where truncation corrupts data. Fires the activity callback every 10s so the gateway's inactivity timeout doesn't kill long commands. ``watch_interrupt_tid`` is the tool-worker thread that submitted this wait: ``execute()`` may move the wait onto a ``run_bounded_sync`` worker while ``/stop`` still interrupts the original tid, so both bits are honored. A ``KeyboardInterrupt``/``SystemExit`` mid-poll kills the process first — the local backend spawns into its own process group (``os.setsid``), so an unkilled child would be orphaned to PPID=1 and keep running. """ output = _new_output_collector(proc, bounded_capture) drain_thread = _start_drain_thread(proc, output) deadline = time.monotonic() + timeout _now = time.monotonic() _activity_state = {"last_touch": _now, "start": _now} trace = _WaitTrace(proc, timeout, enabled=_DEBUG_INTERRUPT, logger=logger) trace.enter() def _kill_and_join(): self._kill_process(proc) drain_thread.join(timeout=2) try: # Adaptive poll: start at 5ms so fast commands return in ~6ms, back # off exponentially toward 200ms so long builds don't pay poll CPU. _poll_sleep = 0.005 while proc.poll() is None: trace.iterations += 1 if is_interrupted() or is_thread_interrupted(watch_interrupt_tid): trace.interrupted() _kill_and_join() return self._finalize_wait_result( output, output.render(suffix="\n[Command interrupted]"), 130 ) if time.monotonic() > deadline: trace.timed_out() _kill_and_join() timeout_msg = f"\n[Command timed out after {timeout}s]" rendered = output.render(suffix=timeout_msg) if output.total_chars == 0: rendered = rendered.lstrip() return self._finalize_wait_result(output, rendered, 124) touch_activity_if_due(_activity_state, "terminal command running") trace.heartbeat() time.sleep(_poll_sleep) if _poll_sleep < 0.2: _poll_sleep = min(_poll_sleep * 1.5, 0.2) except (KeyboardInterrupt, SystemExit): trace.exception_exit() try: _kill_and_join() except Exception: pass # cleanup is best-effort raise # The drain thread exits promptly after bash does (~300ms idle check); # a long join here would itself indicate a bug in the drain loop. drain_thread.join(timeout=2) try: proc.stdout.close() except Exception: pass trace.natural_exit(proc.returncode) # Join the stdin writer before reading its error list: a child that # exits without reading stdin can otherwise race ahead of a recorded # encode failure. The timeout is a pure safety net (write raises # BrokenPipeError once the pipe closes). stdin_thread = getattr(proc, "_hermes_stdin_thread", None) if stdin_thread is not None: stdin_thread.join(timeout=5) rendered = output.render() result = self._finalize_wait_result(output, rendered, proc.returncode) stdin_errors = getattr(proc, "_hermes_stdin_errors", None) if stdin_errors: err = str(stdin_errors[0]) result["stdin_error"] = err result["output"] = rendered + f"\n[stdin write failed: {err}]" return result _finalize_wait_result = staticmethod(_finalize_wait_result) def _kill_process(self, proc: ProcessHandle): """Terminate a process. Subclasses may override for process-group kill.""" try: proc.kill() except (ProcessLookupError, PermissionError, OSError): pass # ------------------------------------------------------------------ # CWD extraction # ------------------------------------------------------------------ def _update_cwd(self, result: dict): """Extract CWD from command output. Override for local file-based read.""" self._extract_cwd_from_output(result) def _extract_cwd_from_output(self, result: dict): """Parse the ``__HERMES_CWD_{session}__`` marker from ``result["output"]``. Updates ``self.cwd`` and strips the marker line. Sets ``result["cwd_observed"]`` (and ``result["cwd"]``) only when THIS command emitted a marker: the wrapper prints it after the command returns, so a killed/timed-out command emits none and ``self.cwd`` keeps the previous value. The environment is shared across sessions, so callers must not attribute an unobserved cwd to this session, and concurrent callers must read ``result["cwd"]`` rather than ``self.cwd``. """ split = _split_cwd_marker(result.get("output", ""), self._cwd_marker) if split is None: return cwd_path, cleaned = split if cwd_path: self.cwd = cwd_path result["cwd_observed"] = True result["cwd"] = cwd_path result["output"] = cleaned # ------------------------------------------------------------------ # Hooks # ------------------------------------------------------------------ def _before_execute(self) -> None: """Hook before each command. Remote backends (SSH, Modal, Daytona) trigger their FileSyncManager here; bind-mount backends and Local don't.""" pass # ------------------------------------------------------------------ # Unified execute() # ------------------------------------------------------------------ def execute( self, command: str, cwd: str = "", *, timeout: int | None = None, stdin_data: str | None = None, rewrite_compound_background: bool = True, bounded_capture: bool = False, ) -> dict: """Execute a command, return {"output": str, "returncode": int}. ``bounded_capture=True`` caps retention at ``tool_output.max_bytes`` WHILE draining; only the foreground terminal tool may set it — internal full-fidelity consumers (file-op ``cat`` reads feeding the patch engine, RPC reads, log reads) MUST leave it False or data is corrupted. The wait is bounded by ``agent.deadline.run_bounded_sync`` so a wedged poll loop cannot hang past ``timeout`` and silently disable every asyncio timer in the process. """ self._before_execute() exec_command, sudo_stdin = self._prepare_command(command) # Guard against the `A && B &` subshell-wait trap by default; callers # that already produce shell-safe wrappers (spawn_via_env) pass False. if rewrite_compound_background: from tools.terminal_tool import _rewrite_compound_background exec_command = _rewrite_compound_background(exec_command) effective_timeout = timeout or self.timeout effective_cwd = cwd or self.cwd # Merge sudo stdin with caller stdin. if sudo_stdin is not None: effective_stdin = sudo_stdin + (stdin_data or "") else: effective_stdin = stdin_data if effective_stdin and self._stdin_mode == "heredoc": exec_command = self._embed_stdin_heredoc(exec_command, effective_stdin) effective_stdin = None wrapped = self._wrap_command(exec_command, effective_cwd) # Login shell if the snapshot failed (so the user's profile still # loads), unless login itself is broken — then non-login is the only path. login = not self._snapshot_ready and not self._prefer_nonlogin parent_tid = threading.current_thread().ident # The activity callback is thread-local and the wait runs on the # deadline worker, so copy it across or long commands look idle. parent_activity_cb = get_activity_callback() proc_holder: list = [] def _spawn_and_wait() -> dict: if parent_activity_cb is not None: set_activity_callback(parent_activity_cb) spawned = self._run_bash( wrapped, login=login, timeout=effective_timeout, stdin_data=effective_stdin ) proc_holder.append(spawned) return self._wait_for_process( spawned, timeout=effective_timeout, bounded_capture=bounded_capture, watch_interrupt_tid=parent_tid, ) def _on_timeout() -> None: if not proc_holder: return self._kill_spawned_tree(proc_holder[0]) # Hard wall-clock backstop: ``_wait_for_process`` polls to # ``effective_timeout`` on the tool thread; if that is the event-loop # thread, or the wait never returns (Windows pipe/poll hang), every # asyncio timer is silently disabled. ``run_bounded_sync`` drives expiry # from a daemon worker + ``Event.wait`` so a blocked loop cannot disable # it; the grace lets the inner loop return the partial-output 124 path. from agent.deadline import run_bounded_sync try: bound_s = float(effective_timeout) + _EXECUTE_WAIT_BOUND_GRACE_S except (TypeError, ValueError): # A non-numeric timeout must not disable the backstop; use the 120s default. bound_s = 120.0 + _EXECUTE_WAIT_BOUND_GRACE_S try: bounded = run_bounded_sync( _spawn_and_wait, bound_s, label=f"terminal.wait:{type(self).__name__}", on_timeout=_on_timeout, ) except (KeyboardInterrupt, SystemExit): _on_timeout() raise if bounded.timed_out: timeout_msg = f"\n[Command timed out after {effective_timeout}s]" result = {"output": timeout_msg.lstrip(), "returncode": 124} else: result = bounded.value self._update_cwd(result) return result def _kill_spawned_tree(self, spawned) -> None: """Best-effort kill of a wedged spawned process and its tree (backstop path).""" try: self._kill_process(spawned) except Exception: logger.debug("terminal wait-bound kill_process failed", exc_info=True) pid = getattr(spawned, "pid", None) if not pid: return try: from agent.deadline import kill_process_tree kill_process_tree(int(pid)) except Exception: logger.debug("terminal wait-bound kill_process_tree failed", exc_info=True) # ------------------------------------------------------------------ # Shared helpers # ------------------------------------------------------------------ def stop(self): """Alias for cleanup (compat with older callers).""" self.cleanup() def __del__(self): try: self.cleanup() except Exception: pass def _prepare_command(self, command: str) -> tuple[str, str | None]: """Transform sudo commands if SUDO_PASSWORD is available.""" from tools.terminal_tool import _transform_sudo_command return _transform_sudo_command(command)