"""Vercel Sandbox execution environment. Runs commands in Vercel cloud sandboxes through the shared ``BaseEnvironment`` shell contract. With persistence enabled, task-scoped snapshot ids are stored under ``HERMES_HOME`` and new sandboxes are restored from them on task reuse. """ from __future__ import annotations from functools import cache from dataclasses import dataclass from datetime import timedelta import logging import math import os import shlex import threading import time from pathlib import Path from typing import TYPE_CHECKING, Any import httpx from hermes_constants import get_hermes_home from tools.environments.base import BaseEnvironment, _ThreadedProcessHandle, _load_json_store, _save_json_store from tools.environments.file_sync import FileSyncManager, iter_sync_files, quoted_rm_command from tools.environments.remote_common import ensure_lazy_dep logger = logging.getLogger(__name__) if TYPE_CHECKING: from vercel.sandbox import Resources, Sandbox, SandboxStatus, WriteFile DEFAULT_VERCEL_CWD = "/vercel/sandbox" _DEFAULT_CONTAINER_DISK_MB = 51200 def _ensure_vercel_sdk() -> None: """Lazy-install vercel SDK on demand. Idempotent.""" # The vercel SDK (>=0.7) ships default-on usage telemetry posting to # telemetry.vercel.com. Hermes policy is no outbound telemetry without # explicit opt-in, so disable it before the SDK is ever imported. Only the # default is set — an explicit user value (e.g. "0") is never overridden. os.environ.setdefault("VERCEL_TELEMETRY_DISABLED", "1") ensure_lazy_dep("terminal.vercel") _CREATE_RETRY_ATTEMPTS = 3 _WRITE_RETRY_ATTEMPTS = 3 _TRANSIENT_STATUS_CODES = frozenset({408, 425, 429, 500, 502, 503, 504}) _RETRY_BACKOFF_STEP = timedelta(milliseconds=100) _MIN_SANDBOX_TIMEOUT = timedelta(minutes=5) _MIN_RUNNING_WAIT = timedelta(seconds=1) _RUNNING_WAIT_TIMEOUT = timedelta(seconds=30) _RUNNING_WAIT_POLL_INTERVAL = timedelta(milliseconds=250) _STOP_TIMEOUT = timedelta(seconds=15) _STOP_POLL_INTERVAL = timedelta(milliseconds=500) _SNAPSHOT_STORE_NAME = "vercel_sandbox_snapshots.json" _SNAPSHOT_ID_KEYS = ("snapshot_id", "snapshotId", "id") _MISSING = object() def _exception_chain(exc: BaseException) -> list[BaseException]: chain: list[BaseException] = [] current: BaseException | None = exc seen: set[int] = set() while current is not None and id(current) not in seen: chain.append(current) seen.add(id(current)) current = current.__cause__ or current.__context__ return chain def _extract_status_code(exc: BaseException) -> int | None: response = getattr(exc, "response", None) for value in (getattr(exc, "status_code", None), getattr(response, "status_code", None)): if isinstance(value, int): return value return None def _is_transient_vercel_error(exc: BaseException) -> bool: for error in _exception_chain(exc): error_name = type(error).__name__.lower() if (_extract_status_code(error) in _TRANSIENT_STATUS_CODES or isinstance(error, (httpx.NetworkError, httpx.ProtocolError, httpx.ReadError)) or "ratelimit" in error_name or "servererror" in error_name): return True return False def _retry_vercel_call(label: str, callback, *, attempts: int): backoff_seconds = _RETRY_BACKOFF_STEP.total_seconds() for attempt in range(1, attempts + 1): try: return callback() except Exception as exc: if attempt >= attempts or not _is_transient_vercel_error(exc): raise logger.warning("Vercel: %s failed (%s); retrying %d/%d", label, exc, attempt, attempts) time.sleep(backoff_seconds * attempt) def _coerce_text(value: Any) -> str: if isinstance(value, bytes): return value.decode("utf-8", errors="replace") return "" if value is None else str(value) def _extract_result_output(result: Any) -> str: try: return _coerce_text(result.output()) except (AttributeError, TypeError): return _coerce_text(result) def _extract_result_returncode(result: Any) -> int: exit_code = getattr(result, "exit_code", _MISSING) if exit_code is _MISSING: exit_code = getattr(result, "returncode", None) return exit_code if isinstance(exit_code, int) else 1 def _load_snapshots() -> dict: return _load_json_store(get_hermes_home() / _SNAPSHOT_STORE_NAME) def _save_snapshots(data: dict) -> None: _save_json_store(get_hermes_home() / _SNAPSHOT_STORE_NAME, data) def _get_snapshot_id(task_id: str) -> str | None: if not task_id: return None snapshot_id = _load_snapshots().get(task_id) return snapshot_id if isinstance(snapshot_id, str) and snapshot_id else None def _store_snapshot(task_id: str, snapshot_id: str) -> None: if not task_id or not snapshot_id: return snapshots = _load_snapshots() snapshots[task_id] = snapshot_id _save_snapshots(snapshots) def _delete_snapshot(task_id: str, snapshot_id: str | None = None) -> None: if not task_id: return snapshots = _load_snapshots() existing = snapshots.get(task_id) if existing is None or (snapshot_id is not None and existing != snapshot_id): return snapshots.pop(task_id, None) _save_snapshots(snapshots) def _extract_snapshot_id(snapshot: Any) -> str | None: """Accept SDK objects or raw dicts; attribute lookup first, then dict keys.""" getters = [lambda k: getattr(snapshot, k, None)] if isinstance(snapshot, dict): getters.append(snapshot.get) for get in getters: for key in _SNAPSHOT_ID_KEYS: value = get(key) if isinstance(value, str) and value: return value return None @cache def _sandbox_status_type() -> type[SandboxStatus]: _ensure_vercel_sdk() from vercel.sandbox import SandboxStatus return SandboxStatus @cache def _terminal_sandbox_states() -> frozenset[SandboxStatus]: SandboxStatus = _sandbox_status_type() return frozenset({SandboxStatus.ABORTED, SandboxStatus.FAILED, SandboxStatus.STOPPED}) @dataclass(frozen=True, slots=True) class _SandboxCreateParams: timeout: timedelta runtime: str | None = None resources: Resources | None = None class VercelSandboxEnvironment(BaseEnvironment): """Vercel cloud sandbox backend.""" _stdin_mode = "heredoc" def __init__(self, runtime: str | None = None, cwd: str = DEFAULT_VERCEL_CWD, timeout: int = 60, cpu: float = 1, memory: int = 5120, disk: int = _DEFAULT_CONTAINER_DISK_MB, persistent_filesystem: bool = True, task_id: str = "default"): super().__init__(cwd=cwd, timeout=timeout) self._runtime = runtime or None self._persistent = persistent_filesystem self._task_id = task_id self._requested_cwd = cwd self._lock = threading.Lock() self._sandbox: Sandbox | None = None self._workspace_root = DEFAULT_VERCEL_CWD self._remote_home = DEFAULT_VERCEL_CWD self._sync_manager: FileSyncManager | None = None self._create_params = self._build_create_params(cpu=cpu, memory=memory, disk=disk) self._attach_fresh_sandbox(cwd) self._sync_manager.sync(force=True) self.init_session() def _require_sandbox(self) -> Sandbox: sandbox = self._sandbox if sandbox is None: raise RuntimeError("Vercel sandbox is not attached") return sandbox def _attach_fresh_sandbox(self, requested_cwd: str) -> None: self._sandbox = self._create_sandbox() self._configure_attached_sandbox(requested_cwd=requested_cwd) def _remote_hermes_dir(self) -> str: home = self._remote_home return "/.hermes" if home == "/" else f"{home.rstrip('/')}/.hermes" def _build_create_params(self, *, cpu: float, memory: int, disk: int) -> _SandboxCreateParams: if disk not in {0, _DEFAULT_CONTAINER_DISK_MB}: raise ValueError( "Vercel Sandbox does not support configurable container_disk. " "Use the default shared setting." ) _ensure_vercel_sdk() from vercel.sandbox import Resources sandbox_timeout = max(timedelta(seconds=max(self.timeout, 0)), _MIN_SANDBOX_TIMEOUT) vcpus = math.floor(cpu) if cpu > 0 else None memory_mb = memory if memory > 0 else None resources = Resources(vcpus=vcpus, memory=memory_mb) if vcpus is not None or memory_mb is not None else None return _SandboxCreateParams(timeout=sandbox_timeout, runtime=self._runtime, resources=resources) def _create_sandbox(self) -> Sandbox: _ensure_vercel_sdk() from vercel.sandbox import Sandbox params = self._create_params snapshot_id = _get_snapshot_id(self._task_id) if self._persistent else None if snapshot_id: try: return _retry_vercel_call( "sandbox restore", lambda: Sandbox.create(timeout=params.timeout, runtime=params.runtime, resources=params.resources, source={"type": "snapshot", "snapshot_id": snapshot_id}), attempts=_CREATE_RETRY_ATTEMPTS, ) except Exception as exc: logger.warning( "Vercel: failed to restore snapshot %s for task %s; " "falling back to a fresh sandbox: %s", snapshot_id, self._task_id, exc, ) _delete_snapshot(self._task_id, snapshot_id) return _retry_vercel_call( "sandbox create", lambda: Sandbox.create(timeout=params.timeout, runtime=params.runtime, resources=params.resources), attempts=_CREATE_RETRY_ATTEMPTS, ) def _configure_attached_sandbox(self, *, requested_cwd: str) -> None: self._wait_for_running() self._workspace_root = self._detect_workspace_root() self._remote_home = self._detect_remote_home() container_base = self._remote_hermes_dir() self._sync_manager = FileSyncManager( get_files_fn=lambda: iter_sync_files(container_base), upload_fn=self._vercel_upload, delete_fn=self._vercel_delete, bulk_upload_fn=self._vercel_bulk_upload, bulk_download_fn=self._vercel_bulk_download, ) if requested_cwd == "~": self.cwd = self._remote_home elif requested_cwd in {"", DEFAULT_VERCEL_CWD}: self.cwd = self._workspace_root else: self.cwd = requested_cwd def _detect_workspace_root(self) -> str: cwd = self._require_sandbox().sandbox.cwd return cwd if cwd.startswith("/") else DEFAULT_VERCEL_CWD def _detect_remote_home(self) -> str: sandbox = self._require_sandbox() try: result = sandbox.run_command("sh", ["-lc", 'printf %s "$HOME"'], cwd=self._workspace_root) except Exception as exc: logger.debug("Vercel: home detection failed for task %s: %s", self._task_id, exc) return self._workspace_root home = _extract_result_output(result).strip() return home if home.startswith("/") else self._workspace_root def _wait_for_running(self, timeout: timedelta = _RUNNING_WAIT_TIMEOUT) -> None: sandbox = self._require_sandbox() SandboxStatus = _sandbox_status_type() status = sandbox.status if status is None or status == SandboxStatus.RUNNING: return if status in _terminal_sandbox_states(): raise RuntimeError(f"Sandbox entered terminal state: {status}") try: sandbox.wait_for_status(SandboxStatus.RUNNING, timeout=max(timeout, _MIN_RUNNING_WAIT), poll_interval=_RUNNING_WAIT_POLL_INTERVAL) except TimeoutError as exc: status = sandbox.status if status in _terminal_sandbox_states(): raise RuntimeError(f"Sandbox entered terminal state: {status}") from exc raise RuntimeError(f"Sandbox did not reach running state (last status: {status})") from exc def _close_sandbox_client(self, sandbox: Sandbox | None) -> None: if sandbox is None: return try: sandbox.client.close() except Exception: pass def _stop_sandbox(self, sandbox: Sandbox | None) -> None: if sandbox is None: return try: sandbox.stop(blocking=True, timeout=_STOP_TIMEOUT, poll_interval=_STOP_POLL_INTERVAL) except TypeError: try: sandbox.stop() except Exception: pass except Exception: pass def _snapshot_sandbox(self, sandbox: Sandbox) -> str | None: if not self._persistent or not self._task_id: return None try: snapshot = sandbox.snapshot() except Exception as exc: logger.warning("Vercel: filesystem snapshot failed for task %s: %s", self._task_id, exc) return None snapshot_id = _extract_snapshot_id(snapshot) if not snapshot_id: logger.warning("Vercel: filesystem snapshot for task %s did not return a snapshot id", self._task_id) return None _store_snapshot(self._task_id, snapshot_id) logger.info("Vercel: saved filesystem snapshot %s for task %s", snapshot_id, self._task_id) return snapshot_id def _ensure_sandbox_ready(self) -> None: sandbox = self._sandbox requested_cwd = self.cwd or self._requested_cwd or DEFAULT_VERCEL_CWD if sandbox is None: self._attach_fresh_sandbox(requested_cwd) return try: sandbox.refresh() except Exception as exc: logger.warning("Vercel: sandbox refresh failed for task %s: %s; recreating", self._task_id, exc) self._close_sandbox_client(sandbox) self._attach_fresh_sandbox(requested_cwd) return status = sandbox.status if status in _terminal_sandbox_states(): logger.warning("Vercel: sandbox entered state %s for task %s; recreating", status, self._task_id) self._close_sandbox_client(sandbox) self._attach_fresh_sandbox(requested_cwd) return self._wait_for_running() def _vercel_upload(self, host_path: str, remote_path: str) -> None: self._vercel_bulk_upload([(host_path, remote_path)]) def _vercel_bulk_upload(self, files: list[tuple[str, str]]) -> None: if not files: return payload: list[WriteFile] = [ {"path": remote_path, "content": Path(host_path).read_bytes()} for host_path, remote_path in files ] sandbox = self._require_sandbox() _retry_vercel_call("write_files", lambda: sandbox.write_files(payload), attempts=_WRITE_RETRY_ATTEMPTS) def _vercel_delete(self, remote_paths: list[str]) -> None: if not remote_paths: return result = self._require_sandbox().run_command( "bash", ["-lc", quoted_rm_command(remote_paths)], cwd=self._workspace_root, ) if _extract_result_returncode(result) != 0: raise RuntimeError(f"Vercel delete failed: {_extract_result_output(result).strip()}") def _vercel_bulk_download(self, dest_tar_path: Path) -> None: archive_member = self._remote_hermes_dir().lstrip("/") remote_tar = f"/tmp/.hermes_sync.{os.getpid()}.tar" sandbox = self._require_sandbox() try: result = sandbox.run_command( "bash", ["-lc", f"tar cf {shlex.quote(remote_tar)} -C / {shlex.quote(archive_member)}"], cwd=self._workspace_root, ) if _extract_result_returncode(result) != 0: raise RuntimeError(f"Vercel bulk download failed: {_extract_result_output(result).strip()}") sandbox.download_file(remote_tar, dest_tar_path) finally: try: sandbox.run_command("bash", ["-lc", f"rm -f {shlex.quote(remote_tar)}"], cwd=self._workspace_root) except Exception: pass def _before_execute(self) -> None: with self._lock: self._ensure_sandbox_ready() if self._sync_manager is not None: self._sync_manager.sync() def _run_bash(self, cmd_string: str, *, login: bool = False, timeout: int = 120, stdin_data: str | None = None): """``timeout`` is not forwarded (the SDK has no per-exec timeout); the base ``_wait_for_process`` enforces it by killing the sandbox via ``cancel_fn``. ``stdin_data`` is discarded: ``_stdin_mode = "heredoc"`` makes the base ``execute()`` embed stdin into the command string before calling this.""" del timeout, stdin_data sandbox = self._require_sandbox() workspace_root = self._workspace_root lock = self._lock def cancel() -> None: with lock: self._stop_sandbox(sandbox) def exec_fn() -> tuple[str, int]: result = sandbox.run_command("bash", ["-lc" if login else "-c", cmd_string], cwd=workspace_root) return _extract_result_output(result), _extract_result_returncode(result) return _ThreadedProcessHandle(exec_fn, cancel_fn=cancel) def cleanup(self): with self._lock: sandbox = self._sandbox sync_manager = self._sync_manager if sandbox is not None and sync_manager is not None: try: sync_manager.sync_back() except Exception as exc: logger.warning("Vercel: sync_back failed for task %s: %s", self._task_id, exc) self._sandbox = None self._sync_manager = None if sandbox is None: return self._snapshot_sandbox(sandbox) # Always stop the sandbox during cleanup to avoid resource leaks (matches Modal/Daytona). self._stop_sandbox(sandbox) self._close_sandbox_client(sandbox)