diff --git a/tests/tools/test_docker_network_config.py b/tests/tools/test_docker_network_config.py index ee0eef9ead..d2c779f53a 100644 --- a/tests/tools/test_docker_network_config.py +++ b/tests/tools/test_docker_network_config.py @@ -30,7 +30,11 @@ def test_sibling_container_config_sites_carry_docker_network(): import tools.code_execution_tool as code_execution_tool import tools.file_tools as file_tools - for module in (terminal_tool, file_tools, code_execution_tool): + # file_tools no longer builds its own container_config: it goes through + # the shared _create_configured_env, which is what carries docker_network. + assert "_create_configured_env(" in inspect.getsource(file_tools) + + for module in (terminal_tool, code_execution_tool): tree = ast.parse(inspect.getsource(module)) sites = 0 for node in ast.walk(tree): diff --git a/tools/file_state.py b/tools/file_state.py index 77dd0577e0..b063ec5229 100644 --- a/tools/file_state.py +++ b/tools/file_state.py @@ -1,24 +1,11 @@ """Cross-agent file state coordination. Prevents mangled edits when concurrent subagents (same process, same -filesystem) touch the same file: subagent B writes a file that subagent A -already read, so A's next write would clobber B's changes with stale content. -Complements the single-agent path-overlap check in -``run_agent._should_parallelize_tool_batch``. - -A process-wide ``FileStateRegistry`` tracks, per resolved path: - * per-agent read stamps {task_id: {path: (mtime, read_ts, partial)}} - * last writer globally {path: (task_id, write_ts)} - * a per-path ``threading.Lock`` for read->modify->write sections - -Hooks used by the file tools: ``record_read`` (read_file), ``note_write`` -(after write_file/patch), ``check_stale`` (BEFORE write_file/patch), -``lock_path`` (wrap the whole read->modify->write block) and ``writes_since`` -(delegate_tool's subagent-completion reminder). - -All methods are no-ops when ``HERMES_DISABLE_FILE_STATE_GUARD=1``. This is -separate from ``file_tools._read_tracker``, which handles per-task -consecutive-read loop detection. +filesystem) touch the same file: B writes a file A already read, so A's next +write would clobber B's changes. Complements the single-agent path-overlap +check in ``run_agent._should_parallelize_tool_batch``. A process-wide +``FileStateRegistry`` tracks per-agent read stamps, the global last writer and +a per-path lock; every method is a no-op under ``HERMES_DISABLE_FILE_STATE_GUARD=1``. """ from __future__ import annotations @@ -35,12 +22,41 @@ from typing import Dict, Iterable, List, Optional, Tuple # so the model re-reads in full. ReadStamp = Tuple[float, float, bool] -# Bounded so long sessions don't accumulate unbounded state; oldest by -# insertion order are dropped on overflow. +# Bounded so long sessions don't accumulate unbounded state. _MAX_PATHS_PER_AGENT = 4096 _MAX_GLOBAL_WRITERS = 4096 +def _disabled() -> bool: + # Re-read each call so tests can toggle via monkeypatch.setenv. + return os.environ.get("HERMES_DISABLE_FILE_STATE_GUARD", "").strip() == "1" + + +def _mtime_or_none(resolved: str) -> Optional[float]: + try: + return os.path.getmtime(resolved) + except OSError: + return None + + +def _fmt_ts(ts: float) -> str: + # Short wall-clock for warnings; avoids datetime formatting on the hot path. + return time.strftime("%H:%M:%S", time.localtime(ts)) + + +def _evict_oldest(container, cap: int) -> None: + """Pop entries until *container* is within *cap* (sets: arbitrary; dicts: oldest + by insertion order). An eviction only costs one redundant re-send or staleness check.""" + for _ in range(len(container) - cap): + try: + if isinstance(container, set): + container.pop() + else: + container.pop(next(iter(container))) + except (StopIteration, KeyError): + break + + class FileStateRegistry: """Process-wide coordinator for cross-agent file edits.""" @@ -51,50 +67,31 @@ class FileStateRegistry: self._meta_lock = threading.Lock() # guards _path_locks self._state_lock = threading.Lock() # guards _reads + _last_writer - def _lock_for(self, resolved: str) -> threading.Lock: - with self._meta_lock: - lock = self._path_locks.get(resolved) - if lock is None: - lock = threading.Lock() - self._path_locks[resolved] = lock - return lock - @contextmanager def lock_path(self, resolved: str): """Per-path lock: threads on the same path serialize, different paths proceed.""" - lock = self._lock_for(resolved) - lock.acquire() - try: + with self._meta_lock: + lock = self._path_locks.setdefault(resolved, threading.Lock()) + with lock: yield - finally: - lock.release() - def record_read( - self, - task_id: str, - resolved: str, - *, - partial: bool = False, - mtime: Optional[float] = None, - ) -> None: + def _stamp(self, task_id: str, resolved: str, mtime: float, now: float, partial: bool) -> None: + """Caller holds ``_state_lock``.""" + agent_reads = self._reads[task_id] + agent_reads[resolved] = (float(mtime), now, bool(partial)) + _evict_oldest(agent_reads, _MAX_PATHS_PER_AGENT) + + def record_read(self, task_id: str, resolved: str, *, partial: bool = False, + mtime: Optional[float] = None) -> None: if _disabled(): return mtime = _mtime_or_none(resolved) if mtime is None else mtime if mtime is None: return - now = time.time() with self._state_lock: - agent_reads = self._reads[task_id] - agent_reads[resolved] = (float(mtime), now, bool(partial)) - _cap_dict(agent_reads, _MAX_PATHS_PER_AGENT) + self._stamp(task_id, resolved, mtime, time.time(), partial) - def note_write( - self, - task_id: str, - resolved: str, - *, - mtime: Optional[float] = None, - ) -> None: + def note_write(self, task_id: str, resolved: str, *, mtime: Optional[float] = None) -> None: """Record a successful write: global last-writer AND this agent's own read stamp (a write is an implicit read of the current content).""" if _disabled(): @@ -105,27 +102,19 @@ class FileStateRegistry: now = time.time() with self._state_lock: self._last_writer[resolved] = (task_id, now) - _cap_dict(self._last_writer, _MAX_GLOBAL_WRITERS) - self._reads[task_id][resolved] = (float(mtime), now, False) - _cap_dict(self._reads[task_id], _MAX_PATHS_PER_AGENT) + _evict_oldest(self._last_writer, _MAX_GLOBAL_WRITERS) + self._stamp(task_id, resolved, mtime, now, False) def check_stale(self, task_id: str, resolved: str) -> Optional[str]: - """Model-facing warning if this write would be stale, else ``None``. - - Checked in severity order: (1) a sibling subagent wrote after this - agent's last read; (2) mtime drifted since our read (external edit) or - the read was partial; (3) this agent never read the file. Never raises - — callers decide whether to block or warn. - """ + """Model-facing warning if this write would be stale, else ``None``. Severity + order: sibling wrote after our read > mtime drift / partial read > never read.""" if _disabled(): return None with self._state_lock: stamp = self._reads.get(task_id, {}).get(resolved) last_writer = self._last_writer.get(resolved) - # Never read and no write record: net-new file or first touch — - # existing sensitive-path / file-exists logic handles it. - if stamp is None and last_writer is None: + if stamp is None and last_writer is None: # net-new file / first touch return None current_mtime = _mtime_or_none(resolved) if current_mtime is None: @@ -139,16 +128,14 @@ class FileStateRegistry: f"{resolved} was modified by sibling subagent " f"{writer_tid!r} but this agent never read it. " "Read the file before writing to avoid overwriting " - "the sibling's changes." - ) + "the sibling's changes.") read_ts = stamp[1] if writer_ts > read_ts: return ( f"{resolved} was modified by sibling subagent " f"{writer_tid!r} at {_fmt_ts(writer_ts)} — after " f"this agent's last read at {_fmt_ts(read_ts)}. " - "Re-read the file before writing." - ) + "Re-read the file before writing.") if stamp is not None: read_mtime, _read_ts, partial = stamp @@ -156,27 +143,20 @@ class FileStateRegistry: return ( f"{resolved} was modified since you last read it " "on disk (external edit or unrecorded writer). " - "Re-read the file before writing." - ) + "Re-read the file before writing.") if partial: return ( f"{resolved} was last read with offset/limit pagination " "(partial view). Re-read the whole file before " - "overwriting it." - ) + "overwriting it.") return None return ( f"{resolved} was not read by this agent. " - "Read the file first so you can write an informed edit." - ) + "Read the file first so you can write an informed edit.") - def writes_since( - self, - exclude_task_id: str, - since_ts: float, - paths: Iterable[str], - ) -> Dict[str, List[str]]: + def writes_since(self, exclude_task_id: str, since_ts: float, + paths: Iterable[str]) -> Dict[str, List[str]]: """``{writer_task_id: [paths]}`` for writes after ``since_ts`` by agents other than ``exclude_task_id`` (delegate_task's "subagent modified files you previously read" reminder).""" @@ -213,36 +193,6 @@ def get_registry() -> FileStateRegistry: return _registry -def _disabled() -> bool: - # Re-read each call so tests can toggle via monkeypatch.setenv. - return os.environ.get("HERMES_DISABLE_FILE_STATE_GUARD", "").strip() == "1" - - -def _mtime_or_none(resolved: str) -> Optional[float]: - try: - return os.path.getmtime(resolved) - except OSError: - return None - - -def _fmt_ts(ts: float) -> str: - # Short wall-clock for warnings; avoids datetime formatting on the hot path. - return time.strftime("%H:%M:%S", time.localtime(ts)) - - -def _cap_dict(d: dict, limit: int) -> None: - """Trim ``d`` to ``limit`` entries by dropping the insertion-order oldest.""" - over = len(d) - limit - if over <= 0: - return - it = iter(d) - for _ in range(over): - try: - d.pop(next(it)) - except (StopIteration, KeyError): - break - - # Convenience wrappers (short names used at call sites). def record_read(task_id: str, resolved_or_path: str | Path, *, partial: bool = False) -> None: _registry.record_read(task_id, str(resolved_or_path), partial=partial) @@ -260,11 +210,7 @@ def lock_path(resolved_or_path: str | Path): return _registry.lock_path(str(resolved_or_path)) -def writes_since( - exclude_task_id: str, - since_ts: float, - paths: Iterable[str | Path], -) -> Dict[str, List[str]]: +def writes_since(exclude_task_id: str, since_ts: float, paths: Iterable[str | Path]) -> Dict[str, List[str]]: return _registry.writes_since(exclude_task_id, since_ts, [str(p) for p in paths]) @@ -280,5 +226,4 @@ __all__ = [ "check_stale", "lock_path", "writes_since", - "known_reads", -] + "known_reads"] diff --git a/tools/file_tools.py b/tools/file_tools.py index a76ec2e8af..a6e74da744 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -1,12 +1,10 @@ #!/usr/bin/env python3 """File Tools Module - LLM agent file manipulation tools. -Companion modules (every name is re-imported here so ``tools.file_tools.X`` -keeps working for callers and test patches): - * ``file_tools_paths`` — task-aware path resolution / ``~`` expansion. - * ``file_tools_write_guards`` — sensitive-path, protected-instruction, - approval, mirror and binary-document guards. - * ``file_tools_read_tracking`` — per-task dedup / loop-detection / staleness state. +Companions (every name re-imported here so ``tools.file_tools.X`` keeps working +for callers and test patches): ``file_tools_paths`` (task-aware resolution), +``file_tools_write_guards`` (write-side guards), ``file_tools_read_tracking`` +(per-task dedup / loop-detection / staleness state). """ import base64 @@ -17,87 +15,45 @@ import os import re import stat import threading +import time +from contextlib import ExitStack from pathlib import Path from agent.file_safety import get_read_block_error from tools.binary_extensions import has_binary_extension from tools.file_operations import ( - ShellFileOperations, - normalize_read_pagination, - normalize_search_pagination, -) + ShellFileOperations, normalize_read_pagination, normalize_search_pagination) from tools import file_state from agent.redact import redact_sensitive_text from tools.file_tools_paths import ( # noqa: F401 (re-exported) - _CONTAINER_PATH_BACKENDS_FALLBACK, - _TERMINAL_CWD_SENTINELS, - _authoritative_workspace_root, - _configured_terminal_cwd, - _expand_tilde, - _normalize_without_host_deref, - _path_resolution_warning, - _registered_task_cwd_override, - _resolve_base_dir, - _resolve_path, - _resolve_path_for_task, - _sentinel_free_abs_cwd, - _terminal_env_type_for_task, - _uses_container_paths, -) + _CONTAINER_PATH_BACKENDS_FALLBACK, _TERMINAL_CWD_SENTINELS, _authoritative_workspace_root, + _configured_terminal_cwd, _expand_tilde, _normalize_without_host_deref, + _path_resolution_warning, _registered_task_cwd_override, _resolve_base_dir, _resolve_path, + _resolve_path_for_task, _sentinel_free_abs_cwd, _terminal_env_type_for_task, + _uses_container_paths) from tools.file_tools_write_guards import ( # noqa: F401 (re-exported) - _PROTECTED_INSTRUCTION_BASENAMES, - _READ_DEDUP_STATUS_MESSAGE, - _SENSITIVE_EXACT_PATHS, - _SENSITIVE_PATH_PREFIXES, - _check_approval_required_write, - _check_binary_document_write, - _check_cross_profile_path, - _check_protected_instruction_write, - _check_sensitive_path, - _get_container_mirror_prefix_for_task, - _get_hermes_config_resolved, - _get_real_hermes_home, - _is_internal_file_status_text, - _is_internal_file_tool_content, - _looks_like_read_file_line_numbered_content, - _protected_instruction_config, - _protected_instruction_reason, - _request_protected_instruction_approval, -) + _PROTECTED_INSTRUCTION_BASENAMES, _READ_DEDUP_STATUS_MESSAGE, _SENSITIVE_EXACT_PATHS, + _SENSITIVE_PATH_PREFIXES, _check_approval_required_write, _check_binary_document_write, + _check_cross_profile_path, _check_protected_instruction_write, _check_sensitive_path, + _get_container_mirror_prefix_for_task, _get_hermes_config_resolved, _get_real_hermes_home, + _is_internal_file_status_text, _is_internal_file_tool_content, + _looks_like_read_file_line_numbered_content, _protected_instruction_config, + _protected_instruction_reason, _request_protected_instruction_approval) from tools.file_tools_read_tracking import ( # noqa: F401 (re-exported) - _DEDUP_CAP, - _NOT_FOUND_CAP, - _NOT_FOUND_TTL_SECONDS, - _READ_HISTORY_CAP, - _READ_TIMESTAMPS_CAP, - _bump_consecutive, - _cap_read_tracker_data, - _check_file_staleness, - _check_not_found_cache, - _invalidate_dedup_for_path, - _mark_verification_stale, - _patch_failure_lock, - _patch_failure_tracker, - _read_tracker, - _read_tracker_lock, - _record_not_found, - _record_patch_failure, - _reset_patch_failures, - _task_data, - _update_read_timestamp, - notify_other_tool_call, - reset_file_dedup, -) + _DEDUP_CAP, _NOT_FOUND_CAP, _NOT_FOUND_TTL_SECONDS, _READ_HISTORY_CAP, _READ_TIMESTAMPS_CAP, + _bump_consecutive, _cap_read_tracker_data, _check_file_staleness, _check_not_found_cache, + _invalidate_dedup_for_path, _mark_verification_stale, _patch_failure_lock, + _patch_failure_tracker, _read_tracker, _read_tracker_lock, _record_not_found, + _record_patch_failure, _reset_patch_failures, _task_data, _update_read_timestamp, + notify_other_tool_call, reset_file_dedup) logger = logging.getLogger(__name__) _EXPECTED_WRITE_ERRNOS = {errno.EACCES, errno.EPERM, errno.EROFS} -# --------------------------------------------------------------------------- # Read-size guard. Model-agnostic, so characters proxy tokens: 100K chars is # ~25-35K tokens across typical tokenisers. Configurable: file_read_max_chars. -# --------------------------------------------------------------------------- _DEFAULT_MAX_READ_CHARS = 100_000 _max_read_chars_cached: int | None = None @@ -105,18 +61,14 @@ _max_read_chars_cached: int | None = None def _get_max_read_chars() -> int: """Return ``file_read_max_chars`` from config.yaml (cached per process; default on missing/invalid).""" global _max_read_chars_cached - if _max_read_chars_cached is not None: - return _max_read_chars_cached - try: - from hermes_cli.config import load_config - cfg = load_config() - val = cfg.get("file_read_max_chars") - if isinstance(val, (int, float)) and val > 0: - _max_read_chars_cached = int(val) - return _max_read_chars_cached - except Exception: - pass - _max_read_chars_cached = _DEFAULT_MAX_READ_CHARS + if _max_read_chars_cached is None: + try: + from hermes_cli.config import load_config + val = load_config().get("file_read_max_chars") + except Exception: + val = None + valid = isinstance(val, (int, float)) and val > 0 + _max_read_chars_cached = int(val) if valid else _DEFAULT_MAX_READ_CHARS return _max_read_chars_cached @@ -124,11 +76,8 @@ def _truncate_to_char_budget(content: str, max_chars: int) -> tuple[str, int, bo """Trim line-numbered ``read_file`` content to the last COMPLETE line within *max_chars*. Returns ``(kept_text, lines_kept, truncated)`` so the caller can offer a - ``next_offset`` instead of rejecting the read. Lines are already clamped to - ``get_max_line_length()`` upstream; the overflow handled here is the - accumulation of many lines under the line-count limit (logs, wide CSV). - If not even the first line fits it is clamped mid-line (Python slicing - never splits a code point) so the read is never empty and the cursor advances. + ``next_offset`` instead of rejecting the read. If not even the first line + fits it is clamped mid-line so the read is never empty and the cursor advances. """ if len(content) <= max_chars: return content, (content.count("\n") + 1 if content else 0), False @@ -142,10 +91,8 @@ def _truncate_to_char_budget(content: str, max_chars: int) -> tuple[str, int, bo break kept.append(line) running += addition - if not kept: kept.append(lines[0][:max_chars]) - return "\n".join(kept), len(kept), True @@ -154,30 +101,25 @@ def _apply_char_budget(result_dict: dict, content: str, offset: int, total_lines continuation hint, and return the trimmed text.""" trimmed, lines_kept, _ = _truncate_to_char_budget(content, max_chars) next_offset = offset + lines_kept - shown_end = offset + lines_kept - 1 result_dict["content"] = trimmed result_dict["truncated"] = True result_dict["truncated_by"] = "bytes" result_dict["next_offset"] = next_offset result_dict["hint"] = ( f"Output truncated at the {max_chars:,}-char read budget after " - f"{lines_kept} line(s) (showing lines {offset}-{shown_end} of " - f"{total_lines}). Use offset={next_offset} to continue." - ) + f"{lines_kept} line(s) (showing lines {offset}-{next_offset - 1} of " + f"{total_lines}). Use offset={next_offset} to continue.") if len(trimmed.split("\n", 1)[0]) >= max_chars: result_dict["hint"] += ( " Note: the first line alone exceeded the budget and was " - "clamped mid-line; its remainder is not retrievable via " - "offset." - ) + "clamped mid-line; its remainder is not retrievable via offset.") return trimmed # Above this size, a wide read (limit > 200) gets a hint toward targeted reads. _LARGE_FILE_HINT_BYTES = 512_000 -# Device/fd paths whose reads hang the process (infinite output or blocking on -# input). Checked by path only — no I/O. +# Device/fd paths whose reads hang the process. Checked by path only — no I/O. _BLOCKED_DEVICE_PATHS = frozenset({ "/dev/zero", "/dev/random", "/dev/urandom", "/dev/full", # never reach EOF "/dev/stdin", "/dev/tty", "/dev/console", # block on input @@ -189,16 +131,12 @@ _BLOCKED_DEVICE_PATHS = frozenset({ _BLOCKED_PROC_SUFFIXES = ( "/fd/0", "/fd/1", "/fd/2", # stdio aliases "/environ", "/cmdline", "/maps", "/smaps", "/smaps_rollup", "/numa_maps", - "/mem", "/auxv", "/pagemap", -) + "/mem", "/auxv", "/pagemap") def _file_ops_uses_host_paths(file_ops) -> bool: - """True when *file_ops* targets the same host filesystem as Hermes. - - Only then may we stat paths or rewrite V4A headers to host-absolute paths; - a container/remote backend has its own filesystem namespace. - """ + """True when *file_ops* targets the host filesystem (only then may we stat paths + or rewrite V4A headers to host-absolute paths; sandboxes have their own namespace).""" env = getattr(file_ops, "env", None) if env is None: return True @@ -209,39 +147,29 @@ def _file_ops_uses_host_paths(file_ops) -> bool: return isinstance(env, LocalEnvironment) -# V4A file headers. ``\s*`` after ``***`` mirrors patch_parser's leniency -# (``***Update File:`` with no space parses and applies, so it must be checked). -_V4A_SINGLE_HEADER_RE = re.compile(r'^(\*\*\*\s*(?:Update|Add|Delete)\s+File:\s*)(.+)$', re.MULTILINE) -_V4A_SINGLE_HEADER_OP_RE = re.compile(r'^\*\*\*\s*(Update|Add|Delete)\s+File:\s*(.+)$', re.MULTILINE) +# V4A file headers: group 1 = header prefix, 2 = op, 3 = path. ``\s*`` after +# ``***`` mirrors patch_parser's leniency (``***Update File:`` applies, so it +# must be checked). +_V4A_SINGLE_HEADER_RE = re.compile(r'^(\*\*\*\s*(Update|Add|Delete)\s+File:\s*)(.+)$', re.MULTILINE) _V4A_MOVE_HEADER_RE = re.compile(r'^(\*\*\*\s*Move\s+File:\s*)(.+?)\s*->\s*(.+)$', re.MULTILINE) -def _rewrite_v4a_patch_paths_for_host( - patch: str, - path_to_resolved: dict, - file_ops, -) -> str: - """Rewrite V4A ``*** Update/Add/Delete/Move File:`` headers to the resolved host paths. +def _rewrite_v4a_patch_paths_for_host(patch: str, path_to_resolved: dict, file_ops) -> str: + """Rewrite V4A file headers to the resolved host paths (host backends only). - ``patch_tool`` resolves every header against the task's workspace for - locking/staleness/reporting; the shell layer must patch those SAME files - rather than re-resolving a relative header against its own cwd (which can - differ — the git-worktree cwd bug). Only applied for host-filesystem backends. + The shell layer must patch the SAME files ``patch_tool`` resolved for + locking/staleness, not re-resolve a relative header against its own cwd + (which can differ — the git-worktree cwd bug). """ if not _file_ops_uses_host_paths(file_ops): return patch - def _resolved_or_original(raw: str) -> str: + def _res(raw: str) -> str: raw = raw.strip() return path_to_resolved.get(raw) or raw - patch = _V4A_SINGLE_HEADER_RE.sub( - lambda m: f"{m.group(1)}{_resolved_or_original(m.group(2))}", patch, - ) - return _V4A_MOVE_HEADER_RE.sub( - lambda m: f"{m.group(1)}{_resolved_or_original(m.group(2))} -> {_resolved_or_original(m.group(3))}", - patch, - ) + patch = _V4A_SINGLE_HEADER_RE.sub(lambda m: f"{m.group(1)}{_res(m.group(3))}", patch) + return _V4A_MOVE_HEADER_RE.sub(lambda m: f"{m.group(1)}{_res(m.group(2))} -> {_res(m.group(3))}", patch) def _is_blocked_device_path(path: str) -> bool: @@ -255,9 +183,8 @@ def _is_blocked_device_path(path: str) -> bool: def _is_blocked_device(filepath: str, base_dir: str | Path | None = None) -> bool: """True if the path (literal, any symlink hop, or final realpath) is a blocked device. - The literal path is checked first so aliases like /dev/stdin are caught - before they resolve to terminal-specific paths; each symlink hop is checked - so an alias to a device cannot bypass the guard. + Literal first so /dev/stdin is caught before resolving to a terminal path; + every symlink hop is checked so an alias cannot bypass the guard. """ expanded = _expand_tilde(filepath) if base_dir is not None and not os.path.isabs(expanded): @@ -290,145 +217,76 @@ def _is_blocked_device(filepath: str, base_dir: str | Path | None = None) -> boo return _is_blocked_device_path(resolved) -def _search_result_read_block_error(path: str, task_id: str = "default") -> str | None: - """Read-safety error for a search result path, resolved against the task cwd - (search backends may return cwd-relative paths; the process cwd can differ).""" - try: - resolved = _resolve_path_for_task(path, task_id) - except (OSError, ValueError, RuntimeError): - return get_read_block_error(path) - return get_read_block_error(str(resolved)) - - def _filter_read_blocked_search_results(result, task_id: str = "default") -> int: - """Remove credential/cache/env paths from a SearchResult in-place; return the omitted count.""" + """Remove credential/cache/env paths from a SearchResult in-place; return the omitted count. + + Each path is resolved against the task cwd first (search backends may + return cwd-relative paths; the process cwd can differ). + """ omitted = 0 - def _blocked(path: str) -> bool: + def _allowed(path: str) -> bool: nonlocal omitted - if _search_result_read_block_error(path, task_id): + try: + target = str(_resolve_path_for_task(path, task_id)) + except (OSError, ValueError, RuntimeError): + target = path + if get_read_block_error(target): omitted += 1 - return True - return False + return False + return True if getattr(result, "matches", None): - result.matches = [m for m in result.matches if not _blocked(m.path)] + result.matches = [m for m in result.matches if _allowed(m.path)] if getattr(result, "files", None): - result.files = [f for f in result.files if not _blocked(f)] + result.files = [f for f in result.files if _allowed(f)] if getattr(result, "counts", None): - result.counts = {f: c for f, c in result.counts.items() if not _blocked(f)} + result.counts = {f: c for f, c in result.counts.items() if _allowed(f)} return omitted def _is_expected_write_exception(exc: Exception) -> bool: """Return True for expected write denials that should not hit error logs.""" - if isinstance(exc, PermissionError): - return True - return isinstance(exc, OSError) and exc.errno in _EXPECTED_WRITE_ERRNOS + return isinstance(exc, PermissionError) or ( + isinstance(exc, OSError) and exc.errno in _EXPECTED_WRITE_ERRNOS) -# --------------------------------------------------------------------------- -# ShellFileOperations per terminal environment -# --------------------------------------------------------------------------- +# ── ShellFileOperations per terminal environment ───────────────────────── _file_ops_lock = threading.Lock() _file_ops_cache: dict = {} -# Per-backend config key that names the sandbox image (overridable per task). -_ENV_IMAGE_KEYS = { - "docker": "docker_image", - "singularity": "singularity_image", - "modal": "modal_image", - "daytona": "daytona_image", -} -_CONTAINER_CONFIG_DEFAULTS = ( - ("container_cpu", 1), - ("container_memory", 5120), - ("container_disk", 51200), - ("container_persistent", True), - ("vercel_runtime", ""), - ("docker_volumes", []), - ("docker_mount_cwd_to_workspace", False), - ("docker_forward_env", []), - ("docker_run_as_host_user", False), - ("docker_network", True), -) - def _create_terminal_env_for_file_ops(raw_task_id: str, task_id: str): - """Build the terminal environment for *task_id* from config + per-task overrides. - - Mirrors terminal_tool's own environment construction so a file tool that - runs before any terminal command uses the configured backend (docker, - modal, ...) rather than always defaulting to local. - """ + """Build the terminal environment for *task_id* via the shared ``_create_configured_env``, + so a file tool that runs before any terminal command still gets the configured backend.""" from tools.terminal_tool import ( - _CONTAINER_BACKENDS, - _create_environment, - _get_env_config, - _is_container_backend, - _is_unusable_container_cwd, - _resolve_task_host_cwd, - get_session_cwd, - resolve_task_overrides, - ) + _CONTAINER_BACKENDS, _create_configured_env, _get_env_config, _is_unusable_container_cwd, + _resolve_task_host_cwd, _select_image, get_session_cwd, resolve_task_overrides) config = _get_env_config() env_type = config["env_type"] overrides = resolve_task_overrides(raw_task_id) - - image_key = _ENV_IMAGE_KEYS.get(env_type) - image = (overrides.get(image_key) or config[image_key]) if image_key else "" - try: recorded_cwd = get_session_cwd(raw_task_id) except Exception: recorded_cwd = None cwd = overrides.get("cwd") or recorded_cwd or config["cwd"] - # Re-apply the container cwd guard _get_env_config() already ran on - # config["cwd"]: a gateway/TUI/ACP cwd override is a raw HOST path, and - # ``docker run -w `` starts the container in a directory that - # doesn't exist there, so search_files & co silently return nothing. - # Valid in-container overrides (/workspace, /root, ...) pass untouched. + # Re-apply the container cwd guard: a gateway/TUI/ACP override is a raw HOST + # path and ``docker run -w `` makes search_files & co silently + # return nothing. Valid in-container overrides (/workspace, /root) pass. if env_type in _CONTAINER_BACKENDS and _is_unusable_container_cwd(cwd): if cwd != config["cwd"]: logger.info( "Ignoring host/relative cwd override %r for %s backend " "(won't exist in sandbox). Using %r instead.", - cwd, env_type, config["cwd"], - ) + cwd, env_type, config["cwd"]) cwd = config["cwd"] logger.info("Creating new %s environment for task %s...", env_type, task_id[:8]) - - container_config = None - if _is_container_backend(env_type): - container_config = {key: config.get(key, default) for key, default in _CONTAINER_CONFIG_DEFAULTS} - - ssh_config = None - if env_type == "ssh": - ssh_config = { - "host": config.get("ssh_host", ""), - "user": config.get("ssh_user", ""), - "port": config.get("ssh_port", 22), - "key": config.get("ssh_key", ""), - "persistent": config.get("ssh_persistent", False), - } - - local_config = None - if env_type == "local": - local_config = { - "persistent": config.get("local_persistent", False), - } - - terminal_env = _create_environment( - env_type=env_type, - image=image, - cwd=cwd, - timeout=config["timeout"], - ssh_config=ssh_config, - container_config=container_config, - local_config=local_config, - task_id=task_id, + terminal_env = _create_configured_env( + config, env_type, image=_select_image(env_type, overrides, config), cwd=cwd, + timeout=config["timeout"], task_id=task_id, host_cwd=_resolve_task_host_cwd(config, raw_task_id), + local_config={"persistent": config.get("local_persistent", False)} if env_type == "local" else None, ) return env_type, terminal_env @@ -436,26 +294,20 @@ def _create_terminal_env_for_file_ops(raw_task_id: str, task_id: str): def _get_file_ops(task_id: str = "default") -> ShellFileOperations: """Get or create ShellFileOperations for the task's terminal environment. - Thread-safe via terminal_tool's per-task creation locks, so concurrent tool - calls never build duplicate sandboxes. Subagent task_ids collapse to - "default" (``_resolve_container_task_id``) so delegate_task children share - the parent's container and cached file_ops; RL/benchmark task_ids with a - registered env override keep their isolation. + Uses terminal_tool's per-task creation locks (no duplicate sandboxes). + Subagent task_ids collapse to "default" (``_resolve_container_task_id``) so + delegate_task children share the parent's container; RL/benchmark task_ids + with a registered env override keep their isolation. """ from tools.terminal_tool import ( - _active_environments, _env_lock, - _last_activity, _start_cleanup_thread, - _creation_locks, - _creation_locks_lock, - _resolve_container_task_id, - ) - import time + _active_environments, _env_lock, _last_activity, _start_cleanup_thread, + _creation_locks, _creation_locks_lock, _resolve_container_task_id, + get_session_cwd, record_session_cwd) raw_task_id = task_id or "default" task_id = _resolve_container_task_id(raw_task_id) - # Fast path: cached AND the environment is still alive (the cleanup thread - # may have killed it). + # Fast path: cached AND the environment is still alive (cleanup thread may have killed it). with _file_ops_lock: cached = _file_ops_cache.get(task_id) if cached is not None: @@ -463,47 +315,32 @@ def _get_file_ops(task_id: str = "default") -> ShellFileOperations: if task_id in _active_environments: _last_activity[task_id] = time.time() return cached - else: - # Environment was cleaned up: rescue the old cwd into the - # session record before dropping the stale entry, FILL-ONLY — - # ``cached.cwd`` is a snapshot of the SHARED env's cwd, not - # attributable to this session, so never overwrite a record - # the session wrote for itself. - old_cwd = getattr(cached, "cwd", None) - if old_cwd: - try: - from tools.terminal_tool import ( - get_session_cwd, - record_session_cwd, - ) - if get_session_cwd(raw_task_id) is None: - record_session_cwd(raw_task_id, old_cwd) - except Exception: - pass - with _file_ops_lock: - _file_ops_cache.pop(task_id, None) + # Env was cleaned up: rescue its cwd into the session record FILL-ONLY + # (``cached.cwd`` is the SHARED env's cwd, not this session's own). + old_cwd = getattr(cached, "cwd", None) + if old_cwd: + try: + if get_session_cwd(raw_task_id) is None: + record_session_cwd(raw_task_id, old_cwd) + except Exception: + pass + with _file_ops_lock: + _file_ops_cache.pop(task_id, None) with _creation_locks_lock: - if task_id not in _creation_locks: - _creation_locks[task_id] = threading.Lock() - task_lock = _creation_locks[task_id] + task_lock = _creation_locks.setdefault(task_id, threading.Lock()) with task_lock: # Double-check: another thread may have created it while we waited. with _env_lock: - if task_id in _active_environments: + terminal_env = _active_environments.get(task_id) + if terminal_env is not None: _last_activity[task_id] = time.time() - terminal_env = _active_environments[task_id] - else: - terminal_env = None - if terminal_env is None: env_type, terminal_env = _create_terminal_env_for_file_ops(raw_task_id, task_id) - with _env_lock: _active_environments[task_id] = terminal_env _last_activity[task_id] = time.time() - _start_cleanup_thread() logger.info("%s environment ready for task %s", env_type, task_id[:8]) @@ -526,84 +363,59 @@ _SPECIAL_FILE_KINDS = ( (stat.S_ISFIFO, "a FIFO (named pipe)"), (stat.S_ISSOCK, "a socket"), (stat.S_ISCHR, "a character device"), - (stat.S_ISBLK, "a block device"), -) + (stat.S_ISBLK, "a block device")) def _special_file_kind(path) -> str | None: """Human name for a non-regular file type that would hang a read, else None. - Stat-based sibling of the name-based ``_is_blocked_device`` guard: a FIFO - or socket inside a workspace hangs ``read_file`` as hard as ``/dev/zero`` - but carries no recognizable name. Host filesystems only (see - ``_file_ops_uses_host_paths``). Missing/unstattable paths return None and - flow to the normal read path's own error handling. + Stat-based sibling of ``_is_blocked_device``: a FIFO/socket in a workspace + hangs like ``/dev/zero`` but has no recognizable name. Host filesystems + only; unstattable paths return None and flow to the normal read path. """ try: - st = os.stat(os.fspath(path)) # follows symlinks, matching a real read + mode = os.stat(os.fspath(path)).st_mode # follows symlinks, matching a real read except OSError: return None - mode = st.st_mode if stat.S_ISREG(mode) or stat.S_ISDIR(mode): return None - for predicate, label in _SPECIAL_FILE_KINDS: - if predicate(mode): - return label - return "a special (non-regular) file" + return next((label for predicate, label in _SPECIAL_FILE_KINDS if predicate(mode)), + "a special (non-regular) file") def _read_extracted_document(path: str, _resolved, offset: int, limit: int, task_id: str) -> str | None: """Render an extractable document (.docx/.xlsx/.pdf/...) as paginated text. - Returns the JSON result, a tool_error for a binary document whose - extraction failed for an actionable reason (size cap, encrypted, - malformed), or ``None`` to fall through to the normal read path. Runs - BEFORE the binary-extension guard so Office files can render as text. + Returns the JSON result, a tool_error for an actionable extraction failure + (size cap, encrypted, malformed), or ``None`` to fall through to the normal + read path. Runs BEFORE the binary-extension guard. """ from tools.read_extract import ( - ANYDOC_EXTENSIONS, - EXTRACTABLE_EXTENSIONS, - MAX_DOCUMENT_BYTES, - ExtractionError, - extract_document_bytes, - is_extractable_document, - ) + ANYDOC_EXTENSIONS, EXTRACTABLE_EXTENSIONS, MAX_DOCUMENT_BYTES, ExtractionError, + extract_document_bytes, is_extractable_document) if not is_extractable_document(str(_resolved)): return None file_ops = _get_file_ops(task_id) try: - binary = file_ops.read_file_bytes( - str(_resolved), max_bytes=MAX_DOCUMENT_BYTES - ) + binary = file_ops.read_file_bytes(str(_resolved), max_bytes=MAX_DOCUMENT_BYTES) if binary.error or binary.base64_content is None: raise ExtractionError(binary.error or "Document bytes unavailable") - document_bytes = base64.b64decode( - binary.base64_content, validate=True - ) - extracted_text = extract_document_bytes( - document_bytes, str(_resolved) - ) + document_bytes = base64.b64decode(binary.base64_content, validate=True) + extracted_text = extract_document_bytes(document_bytes, str(_resolved)) except (ExtractionError, ValueError, base64.binascii.Error) as exc: logger.debug("document extraction failed for %s", path, exc_info=True) - # Binary document formats surface the specific failure: the fallthrough - # can only yield a generic binary-file error or garbage bytes. .ipynb - # (plain JSON) and byte-transport errors (ValueError/binascii) fall - # through — only a specific ExtractionError carries an actionable reason. + # Binary formats surface the specific failure (fallthrough would only + # give a generic binary error); .ipynb and byte-transport errors fall through. _doc_ext = _resolved.suffix.lower() _binary_doc = _doc_ext in ANYDOC_EXTENSIONS or ( - _doc_ext in EXTRACTABLE_EXTENSIONS and _doc_ext != ".ipynb" - ) - if ( - _binary_doc - and isinstance(exc, ExtractionError) - and not str(exc).startswith("Unsupported document type") - ): + _doc_ext in EXTRACTABLE_EXTENSIONS and _doc_ext != ".ipynb") + if (_binary_doc and isinstance(exc, ExtractionError) + and not str(exc).startswith("Unsupported document type")): return tool_error( f"Cannot read '{path}' ({_doc_ext}): document " f"extraction failed — {exc}. Use terminal utilities " - "to inspect or convert the file." - ) + "to inspect or convert the file.") return None lines = extracted_text.splitlines() @@ -615,13 +427,11 @@ def _read_extracted_document(path: str, _resolved, offset: int, limit: int, task "total_lines": total_lines, "file_size": binary.file_size, "truncated": total_lines > end_line, - "extracted_document": True, - } + "extracted_document": True} if result_dict["truncated"]: result_dict["hint"] = ( f"Use offset={end_line + 1} to continue reading " - f"(showing {offset}-{min(end_line, total_lines)} of {total_lines} lines)" - ) + f"(showing {offset}-{min(end_line, total_lines)} of {total_lines} lines)") max_chars = _get_max_read_chars() if len(result_dict["content"]) > max_chars: _apply_char_budget(result_dict, result_dict["content"], offset, total_lines, max_chars) @@ -648,8 +458,7 @@ def _dedup_stub_or_block(task_data: dict, dedup_key: tuple, path: str) -> str: "still current. Proceed with your task using " "the information you already have.", path=path, - already_read=hits + 1, - ) + already_read=hits + 1) return json.dumps({ "status": "unchanged", @@ -664,14 +473,10 @@ def _record_successful_read(task_data: dict, task_id: str, path: str, resolved_s offset: int, limit: int, dedup_key: tuple, *, partial: bool) -> int: """Bookkeeping after a real (non-stub) read; returns the consecutive-read count. - Per-task tracker (under the lock): clear this key's stub-loop counter, add to - history, bump the consecutive counter, and store the mtime — it feeds both - dedup and the write/patch staleness warning. Then, OUTSIDE our lock so the - registry's own locking isn't nested under it, the cross-agent registry - (lets write/patch detect sibling-subagent writes after our read) and the - background-review read-mark (a FULL read of a skill file counts like - skill_view so a follow-up skill_manage(patch) is accepted; no-op outside - review forks). + Per-task tracker under the lock (stub counter, history, consecutive count, + mtime for dedup + staleness). Then OUTSIDE our lock (no nested locking): the + cross-agent registry, and the background-review read-mark (a FULL read of a + skill file counts like skill_view so a follow-up skill_manage(patch) is accepted). """ with _read_tracker_lock: task_data["dedup_hits"].pop(dedup_key, None) @@ -693,7 +498,6 @@ def _record_successful_read(task_data: dict, task_id: str, path: str, resolved_s if not partial: try: from tools.skill_manager_tool import mark_background_review_skill_read - mark_background_review_skill_read(Path(resolved_str)) except Exception: logger.debug("background-review read-mark failed", exc_info=True) @@ -714,8 +518,7 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = if _is_blocked_device(path, base_dir=device_base): return tool_error( f"Cannot read '{path}': this is a device file that would " - "block or produce infinite output." - ) + "block or produce infinite output.") _resolved = _resolve_path_for_task(path, task_id) @@ -729,9 +532,7 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = f"'{path}' is {kind}, not a regular file — reading " "it would block indefinitely, so no read was " "attempted. Use terminal utilities if you need to " - "interact with it." - ), - }) + "interact with it.")}) extracted = _read_extracted_document(path, _resolved, offset, limit, task_id) if extracted is not None: @@ -740,17 +541,13 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = # The extension is a claim, so this message names only the extension; # the content-sniffing path names the actual magic-byte type. if has_binary_extension(str(_resolved)): - _ext = _resolved.suffix.lower() return tool_error( - f"Cannot read binary file '{path}' ({_ext}). " - "Use vision_analyze for images, or terminal to inspect binary files." - ) + f"Cannot read binary file '{path}' ({_resolved.suffix.lower()}). " + "Use vision_analyze for images, or terminal to inspect binary files.") - # Hermes internal denylist: blocks prompt injection via catalog/hub - # metadata and credential stores under HERMES_HOME. Pass the resolved - # path: get_read_block_error's own resolve() runs against the process - # cwd, so a relative "auth.json" read with TERMINAL_CWD == HERMES_HOME - # would otherwise miss the denylist. + # Hermes internal denylist (prompt injection via catalog metadata, + # credential stores). Pass the RESOLVED path: the denylist's own + # resolve() uses the process cwd and would miss a relative "auth.json". block_error = get_read_block_error(str(_resolved)) if block_error: return tool_error(block_error) @@ -766,7 +563,6 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = with _read_tracker_lock: task_data = _task_data(task_id) cached_mtime = task_data.get("dedup", {}).get(dedup_key) - if cached_mtime is not None: try: if os.path.getmtime(resolved_str) == cached_mtime: @@ -774,67 +570,56 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = except OSError: pass # stat failed — fall through to full read - file_ops = _get_file_ops(task_id) - result = file_ops.read_file(path, offset, limit) + result = _get_file_ops(task_id).read_file(path, offset, limit) result_dict = result.to_dict() # Cache a not-found result for retries. Deliberately NO early return: - # error results still flow through the tracking block below - # (consecutive-loop detection, dedup bookkeeping) exactly as before the - # cache existed — serving from the cache is the optimization, recording - # must stay side-effect-identical. + # error results still flow through the tracking below unchanged. _err = result_dict.get("error") or "" if isinstance(_err, str) and _err.startswith("File not found:"): _record_not_found("read", resolved_str, task_id, json.dumps(result_dict, ensure_ascii=False)) - # Char budget is checked on the FORMATTED content (that is what enters - # context) and BEFORE redaction, to skip the regex pass on huge content. - # Graceful truncation instead of rejection: the model gets what fits plus - # a next_offset, rather than guessing a smaller limit and burning a turn. + # Char budget on the FORMATTED content (what enters context), BEFORE + # redaction (skip the regex pass on huge content); truncate gracefully + # with a next_offset instead of rejecting. file_size = result_dict.get("file_size", 0) max_chars = _get_max_read_chars() if len(result.content or "") > max_chars: result.content = _apply_char_budget( result_dict, result.content or "", offset, - result_dict.get("total_lines", "unknown"), max_chars, - ) - + result_dict.get("total_lines", "unknown"), max_chars) if result.content: result.content = redact_sensitive_text(result.content, file_read=True) result_dict["content"] = result.content if (file_size and file_size > _LARGE_FILE_HINT_BYTES - and limit > 200 - and result_dict.get("truncated")): + and limit > 200 and result_dict.get("truncated")): result_dict.setdefault("_hint", ( f"This file is large ({file_size:,} bytes). " "Consider reading only the section you need with offset and limit " - "to keep context usage efficient." - )) + "to keep context usage efficient.")) count = _record_successful_read(task_data, task_id, path, resolved_str, offset, limit, dedup_key, partial=(offset > 1) or bool(result_dict.get("truncated"))) - if count >= 4: return tool_error( f"BLOCKED: You have read this exact file region {count} times in a row. " "The content has NOT changed. You already have this information. " "STOP re-reading and proceed with your task.", path=path, - already_read=count, - ) - elif count >= 3: + already_read=count) + if count >= 3: result_dict["_warning"] = ( f"You have read this exact file region {count} times consecutively. " "The content has not changed since your last read. Use the information you already have. " - "If you are stuck in a loop, stop reading and proceed with writing or responding." - ) - + "If you are stuck in a loop, stop reading and proceed with writing or responding.") return json.dumps(result_dict, ensure_ascii=False) except Exception as e: return tool_error(str(e)) +# ── Shared write/patch plumbing ────────────────────────────────────────── + def _resolve_or_none(filepath: str, task_id: str) -> str | None: """Task-resolved path string, or None when resolution fails for any reason.""" try: @@ -852,13 +637,10 @@ def _write_precheck_error(paths: list[str], content_paths: list[str], task_id: s prompt covers every path of a multi-file patch. """ for p in paths: - err = _check_sensitive_path(p, task_id) + err = _check_sensitive_path(p, task_id) or ( + None if cross_profile else _check_cross_profile_path(p, task_id)) if err: return err - if not cross_profile: - err = _check_cross_profile_path(p, task_id) - if err: - return err for p in content_paths: err = _check_binary_document_write(p, task_id) if err: @@ -867,6 +649,31 @@ def _write_precheck_error(paths: list[str], content_paths: list[str], task_id: s or _check_approval_required_write(paths, task_id)) +def _edit_warnings(paths: list[str], path_to_resolved: dict, task_id: str) -> list[str]: + """One pre-edit warning per path, in priority order: cross-agent registry + (names the sibling subagent) > per-task staleness > workspace divergence + (relative path resolving outside the terminal's cwd — the worktree-cwd bug).""" + warnings: list[str] = [] + for p in paths: + r = path_to_resolved.get(p) + w = (file_state.check_stale(task_id, r) if r else None) or _check_file_staleness(p, task_id) + if not w and r: + w = _path_resolution_warning(p, Path(r), task_id) + if w: + warnings.append(w) + return warnings + + +def _note_edited(task_id: str, paths: list[str], path_to_resolved: dict, session_id: str | None) -> None: + """Post-success bookkeeping: verification-stale marker, then per path refresh + the read stamp (no false staleness on the next edit) and record the write.""" + _mark_verification_stale(task_id, [path_to_resolved.get(p) or p for p in paths], session_id=session_id) + for p in paths: + _update_read_timestamp(p, task_id) + if path_to_resolved.get(p): + file_state.note_write(task_id, path_to_resolved[p]) + + def write_file_tool(path: str, content: str, task_id: str = "default", cross_profile: bool = False, session_id: str | None = None) -> str: @@ -877,71 +684,41 @@ def write_file_tool(path: str, content: str, task_id: str = "default", cross-PROFILE guard it was named for no longer exists). """ # write_file checks the binary-document guard before the mirror guard. - sensitive_err = _check_sensitive_path(path, task_id) - if sensitive_err: - return tool_error(sensitive_err) - binary_doc_err = _check_binary_document_write(path, task_id) - if binary_doc_err: - return tool_error(binary_doc_err) - protected_err = _check_protected_instruction_write([path], task_id) - if protected_err: - return tool_error(protected_err) - approval_err = _check_approval_required_write([path], task_id) - if approval_err: - return tool_error(approval_err) - if not cross_profile: - cross_warning = _check_cross_profile_path(path, task_id) - if cross_warning: - return tool_error(cross_warning) - if _is_internal_file_tool_content(content): - return tool_error( - "Refusing to write internal read_file display text as file content. " - "Strip read_file line-number prefixes or reconstruct the intended " - "file contents before writing." - ) + err = (_check_sensitive_path(path, task_id) + or _check_binary_document_write(path, task_id) + or _check_protected_instruction_write([path], task_id) + or _check_approval_required_write([path], task_id) + or (None if cross_profile else _check_cross_profile_path(path, task_id))) + if not err and _is_internal_file_tool_content(content): + err = ("Refusing to write internal read_file display text as file content. " + "Strip read_file line-number prefixes or reconstruct the intended " + "file contents before writing.") + if err: + return tool_error(err) try: # Resolution failure falls back to the legacy unlocked path (the write # still proceeds; the per-task staleness check still runs). _resolved = _resolve_or_none(path, task_id) - - if _resolved is None: - stale_warning = _check_file_staleness(path, task_id) - file_ops = _get_file_ops(task_id) - result = file_ops.write_file(path, content) - result_dict = result.to_dict() - if stale_warning: - result_dict["_warning"] = stale_warning - if not result_dict.get("error"): - _mark_verification_stale(task_id, [path], session_id=session_id) - _update_read_timestamp(path, task_id) - return json.dumps(result_dict, ensure_ascii=False) - - # Per-path lock serializes read→modify→write across concurrent - # subagents; different paths stay fully parallel. - with file_state.lock_path(_resolved): - # Warning priority: cross-agent (names the sibling subagent) > - # per-task staleness > workspace divergence (relative path resolving - # outside the terminal's cwd — the worktree-cwd bug). - cross_warning = file_state.check_stale(task_id, _resolved) - stale_warning = _check_file_staleness(path, task_id) - cwd_warning = _path_resolution_warning(path, Path(_resolved), task_id) - file_ops = _get_file_ops(task_id) - result = file_ops.write_file(_resolved, content) - result_dict = result.to_dict() - effective_warning = cross_warning or stale_warning or cwd_warning - if effective_warning: - result_dict["_warning"] = effective_warning - # Always report the ABSOLUTE path written so a wrong-cwd mismatch is - # visible in the response instead of silently landing elsewhere. - result_dict["resolved_path"] = _resolved - if not result_dict.get("error"): - result_dict["files_modified"] = [_resolved] - _mark_verification_stale(task_id, [_resolved], session_id=session_id) - # Refresh stamps after the write so consecutive edits by the same - # task don't trigger false staleness warnings. - _update_read_timestamp(path, task_id) - if not result_dict.get("error"): - file_state.note_write(task_id, _resolved) + path_to_resolved = {path: _resolved} + with ExitStack() as _lock: + if _resolved: + # Per-path lock serializes read→modify→write across concurrent + # subagents; different paths stay fully parallel. + _lock.enter_context(file_state.lock_path(_resolved)) + warnings = _edit_warnings([path], path_to_resolved, task_id) + result_dict = _get_file_ops(task_id).write_file(_resolved or path, content).to_dict() + if warnings: + result_dict["_warning"] = warnings[0] + if _resolved: + # Always report the ABSOLUTE path written so a wrong-cwd mismatch + # is visible in the response instead of silently landing elsewhere. + result_dict["resolved_path"] = _resolved + if result_dict.get("error"): + _update_read_timestamp(path, task_id) + else: + if _resolved: + result_dict["files_modified"] = [_resolved] + _note_edited(task_id, [path], path_to_resolved, session_id) return json.dumps(result_dict, ensure_ascii=False) except Exception as e: if _is_expected_write_exception(e): @@ -954,42 +731,28 @@ def write_file_tool(path: str, content: str, task_id: str = "default", def _collect_v4a_header_paths(patch: str) -> tuple[list[str], list[str]] | str: """Extract every path named in V4A headers, rejecting ``..`` traversal. - Returns ``(all_paths, content_write_paths)`` or a tool_error string. - Header paths come from patch CONTENT (skill text, web extract, prompt - injection) so they are more attacker-influenceable than the explicit - ``path=`` arg, which keeps its legitimate ``..`` use. Move headers check - BOTH endpoints (a Move onto /etc/crontab must hit the sensitive-path check). - Delete/Move don't write text, so only Update/Add feed the binary-document guard. + Returns ``(all_paths, content_write_paths)`` or a tool_error string. Header + paths come from patch CONTENT (more attacker-influenceable than ``path=``, + which keeps its legitimate ``..`` use). Move headers check BOTH endpoints; + only Update/Add write text and feed the binary-document guard. """ from tools.path_security import has_traversal_component - def _reject_v4a_traversal(v4a_path: str) -> str | None: + headers = [(m.group(3), m.group(2) in ("Update", "Add")) for m in _V4A_SINGLE_HEADER_RE.finditer(patch)] + headers += [(g, False) for m in _V4A_MOVE_HEADER_RE.finditer(patch) for g in (m.group(2), m.group(3))] + paths: list[str] = [] + content_paths: list[str] = [] + for raw, writes_text in headers: + v4a_path = raw.strip() if has_traversal_component(v4a_path): return tool_error( f"V4A patch header contains '..' traversal: {v4a_path!r}. " "Use the agent's cwd-relative path (no '..') or an absolute " "path in '*** Update File:' / '*** Add File:' / " - "'*** Delete File:' / '*** Move File:' headers." - ) - return None - - paths: list[str] = [] - content_paths: list[str] = [] - for _m in _V4A_SINGLE_HEADER_OP_RE.finditer(patch): - _op = _m.group(1) - v4a_path = _m.group(2).strip() - _err = _reject_v4a_traversal(v4a_path) - if _err: - return _err + "'*** Delete File:' / '*** Move File:' headers.") paths.append(v4a_path) - if _op in ("Update", "Add"): + if writes_text: content_paths.append(v4a_path) - for _m in _V4A_MOVE_HEADER_RE.finditer(patch): - for v4a_path in (_m.group(2).strip(), _m.group(3).strip()): - _err = _reject_v4a_traversal(v4a_path) - if _err: - return _err - paths.append(v4a_path) return paths, content_paths @@ -1017,30 +780,11 @@ def patch_tool(mode: str = "replace", path: str = None, old_string: str = None, # Lock paths in sorted, deduplicated order so concurrent callers with # overlapping multi-file patches can't deadlock (every caller locks in # the same order). An unresolvable path is simply not locked. - _path_to_resolved: dict[str, str] = { - _p: _resolve_or_none(_p, task_id) for _p in _paths_to_check - } - _resolved_paths = sorted({_r for _r in _path_to_resolved.values() if _r}) - - # ExitStack: one lock per path; degenerates to a single lock (or none - # for an unresolvable path) without special-casing. - from contextlib import ExitStack + _path_to_resolved: dict[str, str] = {_p: _resolve_or_none(_p, task_id) for _p in _paths_to_check} with ExitStack() as _locks: - for _r in _resolved_paths: + for _r in sorted({_r for _r in _path_to_resolved.values() if _r}): _locks.enter_context(file_state.lock_path(_r)) - - # Warning priority per path: cross-agent registry (names the - # sibling) > per-task staleness > workspace divergence. - stale_warnings: list[str] = [] - for _p in _paths_to_check: - _r = _path_to_resolved[_p] - _cross = file_state.check_stale(task_id, _r) if _r else None - _sw = _cross or _check_file_staleness(_p, task_id) - if not _sw and _r: - _sw = _path_resolution_warning(_p, Path(_r), task_id) - if _sw: - stale_warnings.append(_sw) - + stale_warnings = _edit_warnings(_paths_to_check, _path_to_resolved, task_id) file_ops = _get_file_ops(task_id) # Hand the shell layer the RESOLVED targets so both layers agree on @@ -1055,48 +799,30 @@ def patch_tool(mode: str = "replace", path: str = None, old_string: str = None, elif mode == "patch": if not patch: return tool_error("patch content required") - patch_for_ops = _rewrite_v4a_patch_paths_for_host( - patch, _path_to_resolved, file_ops - ) - result = file_ops.patch_v4a(patch_for_ops) + result = file_ops.patch_v4a(_rewrite_v4a_patch_paths_for_host(patch, _path_to_resolved, file_ops)) else: return tool_error(f"Unknown mode: {mode}") result_dict = result.to_dict() if stale_warnings: - result_dict["_warning"] = stale_warnings[0] if len(stale_warnings) == 1 else " | ".join(stale_warnings) - # Report the ABSOLUTE path(s) actually patched so a wrong-cwd - # mismatch is visible instead of silently landing elsewhere. - _resolved_modified = [ - _path_to_resolved.get(_p) or _p for _p in _paths_to_check - ] + result_dict["_warning"] = " | ".join(stale_warnings) if not result_dict.get("error"): + # Report the ABSOLUTE path(s) actually patched so a wrong-cwd + # mismatch is visible instead of silently landing elsewhere. + _resolved_modified = [_path_to_resolved.get(_p) or _p for _p in _paths_to_check] result_dict["files_modified"] = _resolved_modified if len(_resolved_modified) == 1: result_dict["resolved_path"] = _resolved_modified[0] - _mark_verification_stale(task_id, _resolved_modified, session_id=session_id) - # Refresh stamps for every patched path (no false staleness on - # the next edit) and clear failure counters so a future miss - # starts a fresh count. - for _p in _paths_to_check: - _update_read_timestamp(_p, task_id) - _r = _path_to_resolved.get(_p) - if _r: - file_state.note_write(task_id, _r) - _reset_patch_failures(task_id, [ - _r for _r in (_path_to_resolved.get(_p) for _p in _paths_to_check) if _r - ]) - # old_string-not-found hint. Per-file failure escalation is tracked for - # replace mode only (V4A failures are rare; the generic hint suffices). - # The generic hint is suppressed when patch_replace already attached a - # richer "Did you mean?" snippet. The escalating hint after 3 failures - # exists so the model recognises the loop and changes approach. + _note_edited(task_id, _paths_to_check, _path_to_resolved, session_id) + # Clear failure counters so a future miss starts a fresh count. + _reset_patch_failures(task_id, [_r for _r in _path_to_resolved.values() if _r]) + # old_string-not-found hint. Failure escalation is tracked for replace + # mode only (V4A misses are rare); the generic hint is suppressed when + # patch_replace already attached a richer "Did you mean?" snippet. if result_dict.get("error") and "Could not find" in str(result_dict["error"]): failure_count = 0 if mode == "replace" and path: - resolved = _path_to_resolved.get(path) or path - failure_count = _record_patch_failure(task_id, resolved) - + failure_count = _record_patch_failure(task_id, _path_to_resolved.get(path) or path) if failure_count >= 3: result_dict["_hint"] = ( f"This is failure #{failure_count} patching {path!r}. " @@ -1105,13 +831,11 @@ def patch_tool(mode: str = "replace", path: str = None, old_string: str = None, "content, (2) use a longer / more unique old_string with " "surrounding context lines, or (3) use write_file to " "replace the entire file if the targeted region is hard " - "to anchor." - ) + "to anchor.") elif "Did you mean one of these sections?" not in str(result_dict["error"]): result_dict["_hint"] = ( "old_string not found. Use read_file to verify the current " - "content, or search_files to locate the text." - ) + "content, or search_files to locate the text.") return json.dumps(result_dict, ensure_ascii=False) except Exception as e: return tool_error(str(e)) @@ -1127,19 +851,10 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", # Pagination args are part of the key so paging through truncated # results doesn't trip the repeated-search guard. - search_key = ( - "search", - pattern, - target, - str(path), - file_glob or "", - limit, - offset, - ) + search_key = ("search", pattern, target, str(path), file_glob or "", limit, offset) with _read_tracker_lock: task_data = _read_tracker.setdefault(task_id, { - "last_key": None, "consecutive": 0, "read_history": set(), - }) + "last_key": None, "consecutive": 0, "read_history": set()}) count = _bump_consecutive(task_data, search_key) if count >= 4: @@ -1148,44 +863,39 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", "The results have NOT changed. You already have this information. " "STOP re-searching and proceed with your task.", pattern=pattern, - already_searched=count, - ) + already_searched=count) try: - resolved_path = _resolve_path_for_task(path, task_id) - except (OSError, ValueError, RuntimeError): - resolved_path = None - block_error = get_read_block_error(str(resolved_path) if resolved_path else path) + resolved_search_path = str(_resolve_path_for_task(path, task_id)) + except (OSError, ValueError, RuntimeError) as exc: + resolved_search_path = path + # A RuntimeError still surfaces as the tool error unless the raw + # path is itself denylisted (that error wins). + if isinstance(exc, RuntimeError) and not get_read_block_error(path): + raise + block_error = get_read_block_error(resolved_search_path) if block_error: return tool_error(block_error) # A missing search root costs two shells (search + parent listing for # "Similar paths"); cache the miss so a retry skips both. - try: - resolved_search_path = str(_resolve_path_for_task(path, task_id)) - except (OSError, ValueError): - resolved_search_path = path cached_search_nf = _check_not_found_cache("search", resolved_search_path, task_id) if cached_search_nf is not None: return cached_search_nf - file_ops = _get_file_ops(task_id) - result = file_ops.search( + result = _get_file_ops(task_id).search( pattern=pattern, path=path, target=target, file_glob=file_glob, - limit=limit, offset=offset, output_mode=output_mode, context=context - ) + limit=limit, offset=offset, output_mode=output_mode, context=context) omitted = _filter_read_blocked_search_results(result, task_id) - if hasattr(result, 'matches'): - for m in result.matches: - if hasattr(m, 'content') and m.content: - m.content = redact_sensitive_text(m.content, file_read=True) + for m in getattr(result, "matches", None) or (): + if getattr(m, "content", None): + m.content = redact_sensitive_text(m.content, file_read=True) result_dict = result.to_dict(densify=True) if omitted: result_dict["_omitted"] = ( f"{omitted} result(s) omitted because they target credential, " - "token, cache, or secret-bearing environment files." - ) + "token, cache, or secret-bearing environment files.") # No early return on a cached miss — same rationale as the read path. _search_err = result_dict.get("error") or "" @@ -1195,8 +905,7 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", if count >= 3: result_dict["_warning"] = ( f"You have run this exact search {count} times consecutively. " - "The results have not changed. Use the information you already have." - ) + "The results have not changed. Use the information you already have.") result_json = json.dumps(result_dict, ensure_ascii=False) if result_dict.get("truncated"): diff --git a/tools/file_tools_paths.py b/tools/file_tools_paths.py index bb79f5c8bd..a4246049be 100644 --- a/tools/file_tools_paths.py +++ b/tools/file_tools_paths.py @@ -1,11 +1,9 @@ """Path resolution for the file tools: task-aware base dir, ``~`` expansion, workspace-divergence warning. -Companion to ``tools.file_tools`` (which re-imports every name here). The core -invariant: the base directory used to anchor relative paths is ALWAYS absolute +Core invariant: the base directory anchoring relative paths is ALWAYS absolute and derived from the task's terminal cwd, never from the process cwd unless no -other anchor exists. A relative or sentinel ``TERMINAL_CWD`` would otherwise -silently anchor edits to the agent process cwd (e.g. the main repo while a -worktree session is active). +other anchor exists (a relative/sentinel ``TERMINAL_CWD`` would silently anchor +edits to the agent process cwd, e.g. the main repo during a worktree session). """ import os @@ -13,9 +11,8 @@ import posixpath import sys from pathlib import Path, PurePosixPath -# ``TERMINAL_CWD`` values that mean "not configured", not a directory to resolve -# against ("." from a stale config; "auto"/"cwd" are setup-wizard placeholders). -# The gateway sanitizes the same set at import time (gateway/run.py). +# ``TERMINAL_CWD`` values that mean "not configured" ("." from a stale config; +# "auto"/"cwd" are wizard placeholders). gateway/run.py sanitizes the same set. _TERMINAL_CWD_SENTINELS = frozenset({"", ".", "./", "auto", "cwd"}) _CONTAINER_PATH_BACKENDS_FALLBACK = frozenset({"docker", "singularity", "modal", "daytona", "vercel_sandbox"}) # Backend name inferred from the live environment's class name (first match wins). @@ -23,13 +20,8 @@ _ENV_CLASS_NAME_HINTS = ("local", "ssh", "docker", "singularity", "modal", "dayt def _expand_tilde(path: str) -> str: - """Expand ``~`` using the effective profile home when available. - - In-process file tools share the gateway process's HOME, which may differ - from the profile-specific HOME interactive CLI sessions use; mirroring - ``hermes_constants.get_subprocess_home()`` keeps ``~`` consistent across - interactive and gateway-driven (cron) runs. - """ + """Expand ``~`` using the effective profile home (``get_subprocess_home``) so + gateway/cron runs, whose process HOME may differ, agree with interactive CLI sessions.""" if not path or "~" not in path: return path try: @@ -47,11 +39,7 @@ def _terminal_env_type_for_task(task_id: str = "default") -> str: """Best-effort terminal backend type for path-resolution decisions.""" try: from tools.terminal_tool import ( - _active_environments, - _env_lock, - _get_env_config, - _resolve_container_task_id, - ) + _active_environments, _env_lock, _get_env_config, _resolve_container_task_id) try: container_key = _resolve_container_task_id(task_id) @@ -61,14 +49,11 @@ def _terminal_env_type_for_task(task_id: str = "default") -> str: env = _active_environments.get(container_key) or _active_environments.get(task_id) if env is not None: name = env.__class__.__name__.lower() - for hint in _ENV_CLASS_NAME_HINTS: - if hint in name: - return hint + hint = next((h for h in _ENV_CLASS_NAME_HINTS if h in name), None) stamped = getattr(env, "_hermes_backend_name", None) - if isinstance(stamped, str) and stamped: - return stamped - cfg = _get_env_config() - return str(cfg.get("env_type") or os.getenv("TERMINAL_ENV") or "local").lower() + if hint or (isinstance(stamped, str) and stamped): + return hint or stamped + return str(_get_env_config().get("env_type") or os.getenv("TERMINAL_ENV") or "local").lower() except Exception: return str(os.getenv("TERMINAL_ENV") or "local").lower() @@ -84,36 +69,24 @@ def _uses_container_paths(task_id: str = "default") -> bool: def _normalize_without_host_deref(path: str | Path | PurePosixPath) -> PurePosixPath: - """Normalize path syntax without following host symlinks. - - Container paths are meaningful inside the sandbox; ``Path.resolve()`` on the - host could dereference a host-side symlink (e.g. ``/workspace``) and rewrite - the path before Docker sees it. - """ + """Normalize path syntax without following host symlinks: container paths are + meaningful inside the sandbox, and a host-side ``/workspace`` symlink must not rewrite them.""" return PurePosixPath(posixpath.normpath(str(path))) def _sentinel_free_abs_cwd(raw: str | None) -> str | None: - """Return *raw* expanded when it is a non-sentinel ABSOLUTE anchor, else ``None``. - - A relative anchor is meaningless without knowing which cwd it is relative - to — exactly the ambiguity that misroutes worktree edits. - """ + """Return *raw* expanded when it is a non-sentinel ABSOLUTE anchor, else ``None`` + (a relative anchor is exactly the ambiguity that misroutes worktree edits).""" raw = str(raw or "").strip() if raw.lower() in _TERMINAL_CWD_SENTINELS: return None expanded = _expand_tilde(raw) - if not os.path.isabs(expanded): - return None - return expanded + return expanded if os.path.isabs(expanded) else None def _configured_terminal_cwd() -> str | None: """Return ``$TERMINAL_CWD`` only when it names a real (absolute, non-sentinel) anchor. - - Scope-aware: under gateway multiplexing the routed profile's cwd lives in - the per-turn terminal scope, not the process env. - """ + Scope-aware: under gateway multiplexing the routed profile's cwd lives in the per-turn scope.""" from agent.runtime_cwd import scope_terminal_cwd return _sentinel_free_abs_cwd(scope_terminal_cwd() or None) @@ -122,10 +95,8 @@ def _configured_terminal_cwd() -> str | None: def _registered_task_cwd_override(task_id: str = "default") -> str | None: """Return a registered cwd override keyed by the RAW task id, when available. - ``terminal_tool`` collapses CWD-only task overrides to the shared - ``"default"`` environment (TUI/dashboard/ACP sessions share one sandbox), - but the cwd value itself stays keyed by the raw session id — so read the - raw override before falling back to the collapsed container key. + ``terminal_tool`` collapses CWD-only overrides to the shared ``"default"`` + env, but the cwd value stays keyed by the raw session id. """ try: from tools.terminal_tool import resolve_task_overrides @@ -140,10 +111,9 @@ def _registered_task_cwd_override(task_id: str = "default") -> str | None: def _authoritative_workspace_root(task_id: str = "default") -> str | None: """Best-effort absolute workspace root, or ``None`` when no reliable anchor exists. - Order: (1) the session's own cwd record (written on every completed terminal - command; per-session, so one session's ``cd`` never leaks into another); - (2) a registered raw-keyed task/session cwd override (TUI/Desktop/ACP); - (3) a sentinel-free absolute ``$TERMINAL_CWD`` (``-w`` sessions). + Order: (1) the session's own cwd record (per-session, so one session's + ``cd`` never leaks into another); (2) a registered raw-keyed cwd override + (TUI/Desktop/ACP); (3) a sentinel-free absolute ``$TERMINAL_CWD``. """ try: from tools.terminal_tool import get_session_cwd @@ -151,86 +121,55 @@ def _authoritative_workspace_root(task_id: str = "default") -> str | None: recorded = get_session_cwd(task_id) except Exception: recorded = None - if recorded: - return recorded - registered = _registered_task_cwd_override(task_id) - if registered: - return registered - return _configured_terminal_cwd() + return recorded or _registered_task_cwd_override(task_id) or _configured_terminal_cwd() + + +def _host_text(text: str, container_paths: bool) -> str: + """Expand ``~``; on host backends also translate Git Bash ``/c/Users/...`` drive + paths before Path sees them. Container/WSL Linux paths are never rewritten.""" + if not container_paths: + from tools.environments.local import _msys_to_windows_path + + text = _msys_to_windows_path(text) + return _expand_tilde(text) + + +def _anchor(text: str, base, container_paths: bool) -> Path | PurePosixPath: + """Return *text* as an absolute, normalized path, joining it onto ``base()`` when + relative. Container: pure-posix, no host deref. Host: resolve() (win32: ntpath normpath).""" + if container_paths: + if not posixpath.isabs(text): + text = posixpath.join(str(base()), text) + return _normalize_without_host_deref(text) + if sys.platform == "win32": + import ntpath + + if not ntpath.isabs(text): + text = ntpath.join(str(base()), text) + return Path(ntpath.normpath(text)) + p = Path(text) + if not p.is_absolute(): + p = Path(base()) / p + return p.resolve() def _resolve_base_dir( - task_id: str = "default", - *, - container_paths: bool | None = None, -) -> Path | PurePosixPath: - """Return the ABSOLUTE base directory for resolving relative paths. - - Uses ``_authoritative_workspace_root`` (live cwd → registered override → - ``$TERMINAL_CWD``), falling back to the process cwd only as a last resort. - Sentinel/relative ``TERMINAL_CWD`` values are rejected outright rather than - anchored to the process cwd, so the result never depends on where the - agent process happens to run. - """ + task_id: str = "default", *, container_paths: bool | None = None) -> Path | PurePosixPath: + """Return the ABSOLUTE base directory for resolving relative paths: + ``_authoritative_workspace_root``, else the process cwd as a last resort.""" root = _authoritative_workspace_root(task_id) if container_paths is None: container_paths = _uses_container_paths(task_id) - base_text = _expand_tilde(root) if root else os.getcwd() - if container_paths: - if not posixpath.isabs(base_text): - base_text = posixpath.join(os.getcwd(), base_text) - return _normalize_without_host_deref(base_text) - # Git Bash ``pwd -P`` reports ``/c/Users/...``; translate before Path so - # relative file-tool paths don't anchor under a nonexistent ``\\c\\Users``. - from tools.environments.local import _msys_to_windows_path - - base_text = _msys_to_windows_path(base_text) - if sys.platform == "win32": - import ntpath - - if not ntpath.isabs(base_text): - base_text = ntpath.join(os.getcwd(), base_text) - return Path(ntpath.normpath(base_text)) - base = Path(base_text) - if not base.is_absolute(): - # A backend reporting a relative cwd is anchored to the process cwd - # once, here, so the result no longer depends on cwd at resolve(). - base = Path(os.getcwd()) / base - return base.resolve() + # A backend's relative cwd is anchored to the process cwd once, here. + return _anchor(_host_text(root or os.getcwd(), container_paths), os.getcwd, container_paths) def _resolve_path_for_task(filepath: str, task_id: str = "default") -> Path | PurePosixPath: - """Resolve *filepath* against the task's absolute base directory. - - Absolute inputs are returned resolved-but-unanchored. On native Windows, - Git Bash / MSYS drive paths (``/c/Users/...``) are translated first so - they aren't treated as relative ``\\c\\Users\\...`` under the process cwd; - container/WSL Linux paths are never rewritten. - """ + """Resolve *filepath* against the task's absolute base directory + (absolute inputs are returned resolved-but-unanchored).""" container_paths = _uses_container_paths(task_id) - if container_paths: - expanded = _expand_tilde(filepath) - if posixpath.isabs(expanded): - return _normalize_without_host_deref(expanded) - resolved = _resolve_base_dir(task_id, container_paths=True) / expanded - return _normalize_without_host_deref(resolved) - - from tools.environments.local import _msys_to_windows_path - - expanded = _expand_tilde(_msys_to_windows_path(filepath)) - if sys.platform == "win32": - import ntpath - - if ntpath.isabs(expanded): - return Path(ntpath.normpath(expanded)) - joined = ntpath.join(str(_resolve_base_dir(task_id, container_paths=False)), expanded) - return Path(ntpath.normpath(joined)) - - p = Path(expanded) - if p.is_absolute(): - return p.resolve() - resolved = _resolve_base_dir(task_id, container_paths=False) / p - return resolved.resolve() + return _anchor(_host_text(filepath, container_paths), + lambda: _resolve_base_dir(task_id, container_paths=container_paths), container_paths) # Back-compat alias (imported by agent.context_references and tests). @@ -238,13 +177,9 @@ _resolve_path = _resolve_path_for_task def _path_resolution_warning(filepath: str, resolved: Path, task_id: str = "default") -> str | None: - """Warn when a RELATIVE path resolved OUTSIDE the task's workspace root. - - Surfaces the worktree-cwd divergence the moment it matters — the edit is - about to land in a different checkout than the terminal's cwd. ``None`` for - absolute paths, an unknown root, or a path correctly under the root. Fires - on the very first write even before any ``cd`` populated the cwd registry. - """ + """Warn when a RELATIVE path resolved OUTSIDE the task's workspace root (the + edit is about to land in a different checkout than the terminal's cwd). + ``None`` for absolute paths, an unknown root, or a path under the root.""" try: if Path(_expand_tilde(filepath)).is_absolute(): return None @@ -255,16 +190,13 @@ def _path_resolution_warning(filepath: str, resolved: Path, task_id: str = "defa root = _normalize_without_host_deref(Path(_expand_tilde(workspace_root))) else: root = Path(_expand_tilde(workspace_root)).resolve() - try: - resolved.relative_to(root) + if resolved.is_relative_to(root): return None - except ValueError: - return ( - f"Relative path {filepath!r} resolved to {str(resolved)!r}, which is " - f"OUTSIDE the active workspace ({str(root)!r}). The edit will land in " - f"a different directory than the terminal's cwd. If this is not " - f"intended (e.g. a git-worktree session writing into the main " - f"checkout), pass an absolute path under the workspace instead." - ) + return ( + f"Relative path {filepath!r} resolved to {str(resolved)!r}, which is " + f"OUTSIDE the active workspace ({str(root)!r}). The edit will land in " + f"a different directory than the terminal's cwd. If this is not " + f"intended (e.g. a git-worktree session writing into the main " + f"checkout), pass an absolute path under the workspace instead.") except Exception: return None diff --git a/tools/file_tools_read_tracking.py b/tools/file_tools_read_tracking.py index 1789c76bdc..86c639a876 100644 --- a/tools/file_tools_read_tracking.py +++ b/tools/file_tools_read_tracking.py @@ -1,23 +1,12 @@ """Per-task read/search bookkeeping for the file tools. -Process-lifetime state behind ``read_file`` / ``search_files`` / ``write_file`` / -``patch``; ``tools.file_tools`` re-imports every name here. Per task_id -``_read_tracker`` stores: - - last_key / consecutive most recent read-or-search key and its repeat count - (loop detection; reset by any OTHER tool call). - read_history set of (path, offset, limit) — diagnostic summaries only. - dedup (resolved_path, offset, limit) -> mtime; skip identical - re-reads of unchanged files. Cleared on context - compression (the original content was summarised away). - dedup_hits per-key count of stub returns, to break stub loops. - read_timestamps resolved_path -> mtime at last read/write by this task; - write/patch warn when the file changed underneath. - not_found (op, resolved_path) -> (monotonic, cached error JSON); - short-TTL negative cache for retried missing paths. - -Every container is hard-capped (``_cap_read_tracker_data``) so a long CLI -session accretes a few hundred KB at most instead of ~1.5MB per 10k reads. +Process-lifetime state behind read_file/search_files/write_file/patch; +``tools.file_tools`` re-imports every name here. Per task_id ``_read_tracker`` +stores: ``last_key``/``consecutive`` (loop detection; reset by any OTHER tool +call), ``read_history`` (diagnostics), ``dedup`` (key -> mtime; cleared on +context compression), ``dedup_hits`` (stub-loop breaker), ``read_timestamps`` +(staleness warnings) and ``not_found`` (short-TTL negative cache). Every +container is hard-capped (``_cap_read_tracker_data``) so long sessions stay small. """ import logging @@ -25,6 +14,7 @@ import os import threading import time +from tools.file_state import _evict_oldest from tools.file_tools_paths import _authoritative_workspace_root, _resolve_path_for_task logger = logging.getLogger("tools.file_tools") @@ -33,8 +23,7 @@ _read_tracker_lock = threading.Lock() _read_tracker: dict = {} # Consecutive patch failures per (task_id, resolved_path); escalates the hint -# when the model keeps failing the same file (stale view, ambiguous old_string). -# Reset on a successful patch to that path. +# when the model keeps failing the same file. Reset on a successful patch. _patch_failure_lock = threading.Lock() _patch_failure_tracker: dict = {} # {task_id: {resolved_path: count}} _PATCH_FAILURE_PATHS_CAP = 64 @@ -48,24 +37,13 @@ _NOT_FOUND_CAP = 500 _NOT_FOUND_TTL_SECONDS = 60.0 # a path that didn't exist may be created soon -def _new_task_data() -> dict: - return { - "last_key": None, "consecutive": 0, - "read_history": set(), "dedup": {}, - "dedup_hits": {}, "read_timestamps": {}, - } - - def _task_data(task_id: str) -> dict: - """Get-or-create the tracker entry for *task_id*, back-filling any missing keys. - - Must be called with ``_read_tracker_lock`` held. Entries created by older - code paths (or injected by tests) may lack the newer containers. - """ - task_data = _read_tracker.setdefault(task_id, _new_task_data()) - for key, factory in (("dedup", dict), ("dedup_hits", dict), ("read_timestamps", dict)): - if key not in task_data: - task_data[key] = factory() + """Get-or-create the tracker entry for *task_id*, back-filling missing containers + (search_tool / tests create partial entries). Lock must be held.""" + task_data = _read_tracker.setdefault(task_id, { + "last_key": None, "consecutive": 0, "read_history": set()}) + for key in ("dedup", "dedup_hits", "read_timestamps"): + task_data.setdefault(key, {}) return task_data @@ -74,11 +52,8 @@ def _record_patch_failure(task_id: str, resolved_path: str) -> int: with _patch_failure_lock: task_failures = _patch_failure_tracker.setdefault(task_id, {}) # Evict the oldest entry once a task has failed on many distinct files. - if len(task_failures) >= _PATCH_FAILURE_PATHS_CAP and resolved_path not in task_failures: - try: - del task_failures[next(iter(task_failures))] - except StopIteration: - pass + if resolved_path not in task_failures: + _evict_oldest(task_failures, _PATCH_FAILURE_PATHS_CAP - 1) task_failures[resolved_path] = task_failures.get(resolved_path, 0) + 1 return task_failures[resolved_path] @@ -89,29 +64,10 @@ def _reset_patch_failures(task_id: str, resolved_paths: list) -> None: return with _patch_failure_lock: task_failures = _patch_failure_tracker.get(task_id) - if not task_failures: - return - for rp in resolved_paths: + for rp in resolved_paths if task_failures else (): task_failures.pop(rp, None) -def _evict_oldest(container, cap: int) -> None: - """Pop entries until *container* is within *cap*. - - Sets pop arbitrary entries (they only feed diagnostic summaries); dicts pop - oldest by insertion order. An evicted entry costs one redundant re-send - (dedup) or one non-mtime staleness check — graceful degradation, not a bug. - """ - for _ in range(len(container) - cap): - try: - if isinstance(container, set): - container.pop() - else: - container.pop(next(iter(container))) - except (StopIteration, KeyError): - break - - def _cap_read_tracker_data(task_data: dict) -> None: """Enforce size caps on the per-task sub-containers. Call with ``_read_tracker_lock`` held.""" # Caps are read at call time so tests can monkeypatch the module constants. @@ -120,44 +76,48 @@ def _cap_read_tracker_data(task_data: dict) -> None: ("dedup", _DEDUP_CAP), ("dedup_hits", _DEDUP_CAP), ("read_timestamps", _READ_TIMESTAMPS_CAP), - ("not_found", _NOT_FOUND_CAP), - ): + ("not_found", _NOT_FOUND_CAP)): container = task_data.get(key) if container is not None and len(container) > cap: _evict_oldest(container, cap) +def _resolved_or_none(filepath: str, task_id: str) -> str | None: + try: + return str(_resolve_path_for_task(filepath, task_id)) + except (OSError, ValueError): + return None + + +def _pop_not_found(op: str, resolved_str: str, task_id: str) -> None: + """Drop the negative-cache entry for *(op, resolved_str)*. Lock must be held.""" + task_data = _read_tracker.get(task_id) + nf = task_data.get("not_found") if task_data else None + if nf: + nf.pop((op, resolved_str), None) + + def _check_not_found_cache(op: str, resolved_str: str, task_id: str) -> str | None: """Return cached not-found JSON for *(op, resolved_str)* if still fresh. - Skips the subprocess + similar-name walk when the model retries the same - missing path. *op* is "read" or "search" (different error JSON shapes). - Evicted by TTL, by write_file/patch on the path, or by any other tool call. + *op* is "read" or "search" (different error JSON shapes). Evicted by TTL, + by write_file/patch on the path, or by any other tool call. """ with _read_tracker_lock: task_data = _read_tracker.get(task_id) - if not task_data: - return None - nf = task_data.get("not_found") - if not nf: - return None - entry = nf.get((op, resolved_str)) + entry = (task_data.get("not_found") or {}).get((op, resolved_str)) if task_data else None if entry is None: return None ts, cached_json = entry if time.monotonic() - ts > _NOT_FOUND_TTL_SECONDS: - nf.pop((op, resolved_str), None) + _pop_not_found(op, resolved_str, task_id) return None - # The path may have been created since the miss was cached (terminal, - # another agent, ...) — the "check → create → read" pattern is common, so - # serving a stale miss breaks it. The stat runs OUTSIDE the global tracker - # lock: a hung stat on a dead network mount must not stall every task. + # "check → create → read" is common, so never serve a stale miss for a path + # that now exists. The stat runs OUTSIDE the tracker lock: a hung stat on a + # dead network mount must not stall every task. if os.path.exists(resolved_str): with _read_tracker_lock: - task_data = _read_tracker.get(task_id) - nf = task_data.get("not_found") if task_data else None - if nf: - nf.pop((op, resolved_str), None) + _pop_not_found(op, resolved_str, task_id) return None return cached_json @@ -166,8 +126,7 @@ def _record_not_found(op: str, resolved_str: str, task_id: str, error_json: str) """Cache a not-found error so the next *op* call for *resolved_str* skips I/O.""" with _read_tracker_lock: task_data = _task_data(task_id) - nf = task_data.setdefault("not_found", {}) - nf[(op, resolved_str)] = (time.monotonic(), error_json) + task_data.setdefault("not_found", {})[(op, resolved_str)] = (time.monotonic(), error_json) _cap_read_tracker_data(task_data) @@ -182,11 +141,8 @@ def _bump_consecutive(task_data: dict, key: tuple) -> int: def reset_file_dedup(task_id: str = None): - """Clear the read-dedup cache (one task, or all when ``task_id`` is None). - - Called after context compression: the original read content was summarised - away, so a "file unchanged" stub would point at content no longer in context. - """ + """Clear the read-dedup cache (one task, or all when ``task_id`` is None). Called + after context compression: a "file unchanged" stub would point at summarised-away content.""" with _read_tracker_lock: if task_id: targets = [_read_tracker[task_id]] if _read_tracker.get(task_id) else [] @@ -201,33 +157,25 @@ def reset_file_dedup(task_id: str = None): def notify_other_tool_call(task_id: str = "default"): """Reset the consecutive read/search counter for a task. - Called by the dispatcher for every tool OTHER than read_file/search_files, - so loop detection only fires on truly consecutive repeats. Also clears the - stub-hit counters and the not-found cache: any other tool may have created - a previously-missing path (the serve-side stat covers most cases; clearing - covers the rest, e.g. permission flips). + Called by the dispatcher for every tool OTHER than read_file/search_files. + Also clears stub-hit counters and the not-found cache: any other tool may + have created a previously-missing path (or flipped its permissions). """ with _read_tracker_lock: task_data = _read_tracker.get(task_id) if task_data: task_data["last_key"] = None task_data["consecutive"] = 0 - if "dedup_hits" in task_data: - task_data["dedup_hits"].clear() - nf = task_data.get("not_found") - if nf: - nf.clear() + for key in ("dedup_hits", "not_found"): + if task_data.get(key): + task_data[key].clear() def _invalidate_dedup_for_path(filepath: str, task_id: str) -> None: - """Evict every dedup entry (all offset/limit ranges) and not-found entry for *filepath*. - - Called after write_file/patch so the next read returns fresh content - instead of a stale "unchanged" stub. Acquires ``_read_tracker_lock`` itself. - """ - try: - resolved = str(_resolve_path_for_task(filepath, task_id)) - except (OSError, ValueError): + """Evict every dedup entry (all offset/limit ranges) and not-found entry for *filepath* + after a write, so the next read returns fresh content. Acquires the lock itself.""" + resolved = _resolved_or_none(filepath, task_id) + if resolved is None: return with _read_tracker_lock: task_data = _read_tracker.get(task_id) @@ -237,20 +185,20 @@ def _invalidate_dedup_for_path(filepath: str, task_id: str) -> None: if dedup: for k in [k for k in dedup if k[0] == resolved]: del dedup[k] - nf = task_data.get("not_found") - if nf: - nf.pop(("read", resolved), None) - nf.pop(("search", resolved), None) + _pop_not_found("read", resolved, task_id) + _pop_not_found("search", resolved, task_id) def _update_read_timestamp(filepath: str, task_id: str) -> None: """After a successful write: invalidate dedup and refresh the stored mtime so consecutive edits by the same task don't trigger false staleness warnings.""" _invalidate_dedup_for_path(filepath, task_id) + resolved = _resolved_or_none(filepath, task_id) + if resolved is None: + return try: - resolved = str(_resolve_path_for_task(filepath, task_id)) current_mtime = os.path.getmtime(resolved) - except (OSError, ValueError): + except OSError: return with _read_tracker_lock: task_data = _read_tracker.get(task_id) @@ -261,19 +209,13 @@ def _update_read_timestamp(filepath: str, task_id: str) -> None: def _check_file_staleness(filepath: str, task_id: str) -> str | None: """Warn (don't block) when the file's mtime changed since this task last read it. - - ``None`` when never read, fresh, or unstattable (a deleted file is the - write's problem to report). - """ - try: - resolved = str(_resolve_path_for_task(filepath, task_id)) - except (OSError, ValueError): + ``None`` when never read, fresh, or unstattable (a deleted file is the write's problem).""" + resolved = _resolved_or_none(filepath, task_id) + if resolved is None: return None with _read_tracker_lock: task_data = _read_tracker.get(task_id) - if not task_data: - return None - read_mtime = task_data.get("read_timestamps", {}).get(resolved) + read_mtime = task_data.get("read_timestamps", {}).get(resolved) if task_data else None if read_mtime is None: return None try: @@ -284,21 +226,14 @@ def _check_file_staleness(filepath: str, task_id: str) -> str | None: return ( f"Warning: {filepath} was modified since you last read it " "(external edit or concurrent agent). The content you read may be " - "stale. Consider re-reading the file to verify before writing." - ) + "stale. Consider re-reading the file to verify before writing.") return None -def _mark_verification_stale( - task_id: str, - resolved_paths: list[str], - session_id: str | None = None, -) -> None: - """Best-effort note that successful edits made prior verification stale. - - The workspace cwd is the first edited path's project root when one is - recognised, else the task's workspace root, else the first path's parent. - """ +def _mark_verification_stale(task_id: str, resolved_paths: list[str], + session_id: str | None = None) -> None: + """Best-effort note that successful edits made prior verification stale. cwd: the + first edited path's recognised project root, else the workspace root, else the first parent.""" from pathlib import Path paths = [p for p in resolved_paths if p] @@ -308,22 +243,9 @@ def _mark_verification_stale( from agent.coding_context import project_facts_for from agent.verification_evidence import mark_workspace_edited - cwd = None - for path in paths: - try: - candidate = str(Path(path).parent) - except Exception: - continue - if project_facts_for(candidate): - cwd = candidate - break - if cwd is None: - cwd = _authoritative_workspace_root(task_id) - if cwd is None: - try: - cwd = str(Path(paths[0]).parent) - except Exception: - cwd = None + parents = [str(Path(p).parent) for p in paths] + cwd = (next((c for c in parents if project_facts_for(c)), None) + or _authoritative_workspace_root(task_id) or parents[0]) mark_workspace_edited(session_id=session_id or task_id, cwd=cwd, paths=paths) except Exception: logger.debug("verification stale marker failed", exc_info=True) diff --git a/tools/file_tools_write_guards.py b/tools/file_tools_write_guards.py index 294ce52715..147a1a2f35 100644 --- a/tools/file_tools_write_guards.py +++ b/tools/file_tools_write_guards.py @@ -2,14 +2,10 @@ Every guard returns ``None`` when the write may proceed, else an error string the tool returns verbatim. ``tools.file_tools`` re-imports every name here. - -Guards (in the order the tools apply them): - * ``_check_sensitive_path`` — system paths + the Hermes config.yaml (hard deny). - * ``_check_binary_document_write`` — text write would corrupt an Office/PDF container. - * ``_check_protected_instruction_write`` — AGENTS.md-style files: ALWAYS ask, no yolo bypass. - * ``_check_approval_required_write`` — ~/.ssh/config-style files: normal approval gate. - * ``_check_cross_profile_path`` — sandbox-mirror writes the host never reads (lost-work guard). - * ``_is_internal_file_tool_content`` — refuse to persist read_file display text as a file. +Guards, in the order the tools apply them: ``_check_sensitive_path`` (hard +deny), ``_check_binary_document_write``, ``_check_protected_instruction_write`` +(ALWAYS ask), ``_check_approval_required_write`` (normal gate), +``_check_cross_profile_path`` (sandbox-mirror lost-work), ``_is_internal_file_tool_content``. """ import fnmatch @@ -25,29 +21,51 @@ from tools.file_tools_paths import _expand_tilde, _resolve_path_for_task _SENSITIVE_PATH_PREFIXES = ( "/etc/", "/boot/", "/usr/lib/systemd/", "/private/etc/", - "/private/var/db/", "/private/var/root/", -) + "/private/var/db/", "/private/var/root/") _SENSITIVE_EXACT_PATHS = {"/var/run/docker.sock", "/run/docker.sock"} _hermes_config_resolved: str | None = None _hermes_config_resolved_loaded = False +_real_hermes_home_cached: str | None = None +_real_hermes_home_loaded = False + + +def _cached_lookup(slot: str, flag: str, primary, fallback) -> str | None: + """Fill module global *slot* once (guarded by *flag*) from ``primary()``, else + ``fallback()``, else None. Module globals so tests can monkeypatch the slots.""" + g = globals() + if not g[flag]: + g[flag] = True + try: + g[slot] = primary() + except Exception: + try: + g[slot] = fallback() + except Exception: + g[slot] = None + return g[slot] + + +def _config_path_resolved() -> str: + from hermes_cli.config import get_config_path + return str(get_config_path().resolve()) + + +def _hermes_home_real() -> str: + from hermes_constants import get_hermes_home + return os.path.realpath(str(get_hermes_home())) def _get_hermes_config_resolved() -> str | None: """Return the resolved absolute path of the Hermes config file (cached).""" - global _hermes_config_resolved, _hermes_config_resolved_loaded - if _hermes_config_resolved_loaded: - return _hermes_config_resolved - _hermes_config_resolved_loaded = True - try: - from hermes_cli.config import get_config_path - _hermes_config_resolved = str(get_config_path().resolve()) - except Exception: - try: - _hermes_config_resolved = str(Path(_expand_tilde("~/.hermes/config.yaml")).resolve()) - except Exception: - _hermes_config_resolved = None - return _hermes_config_resolved + return _cached_lookup("_hermes_config_resolved", "_hermes_config_resolved_loaded", _config_path_resolved, + lambda: str(Path(_expand_tilde("~/.hermes/config.yaml")).resolve())) + + +def _get_real_hermes_home() -> str | None: + """Return the realpath of the authoritative Hermes home (cached).""" + return _cached_lookup("_real_hermes_home_cached", "_real_hermes_home_loaded", _hermes_home_real, + lambda: os.path.realpath(_expand_tilde("~/.hermes"))) def _resolved_or_raw(filepath: str, task_id: str) -> str: @@ -60,79 +78,40 @@ def _resolved_or_raw(filepath: str, task_id: str) -> str: def _check_sensitive_path(filepath: str, task_id: str = "default") -> str | None: """Return an error message if the path targets a sensitive system location.""" - resolved = _resolved_or_raw(filepath, task_id) - normalized = os.path.normpath(_expand_tilde(filepath)) - _err = ( - f"Refusing to write to sensitive system path: {filepath}\n" - "Use the terminal tool with sudo if you need to modify system files." - ) - for prefix in _SENSITIVE_PATH_PREFIXES: - if resolved.startswith(prefix) or normalized.startswith(prefix): - return _err - if resolved in _SENSITIVE_EXACT_PATHS or normalized in _SENSITIVE_EXACT_PATHS: - return _err + candidates = (_resolved_or_raw(filepath, task_id), os.path.normpath(_expand_tilde(filepath))) + if any(c.startswith(_SENSITIVE_PATH_PREFIXES) or c in _SENSITIVE_EXACT_PATHS for c in candidates): + return ( + f"Refusing to write to sensitive system path: {filepath}\n" + "Use the terminal tool with sudo if you need to modify system files.") # approvals.mode and other security settings live in config.yaml; a # prompt-injected agent could silently disable exec approval by editing it. hermes_config = _get_hermes_config_resolved() - if hermes_config and (resolved == hermes_config or normalized == hermes_config): + if hermes_config and hermes_config in candidates: return ( f"Refusing to write to Hermes config file: {filepath}\n" "Agent cannot modify security-sensitive configuration. " - "Edit ~/.hermes/config.yaml directly or use 'hermes config' instead." - ) + "Edit ~/.hermes/config.yaml directly or use 'hermes config' instead.") return None -# --------------------------------------------------------------------------- -# Protected agent-instruction files (always-ask approval gate) -# --------------------------------------------------------------------------- +# ── Protected agent-instruction files (always-ask approval gate) ───────── # Files that steer FUTURE agent behavior are a prompt-injection persistence -# vector: an injected edit to AGENTS.md / CLAUDE.md / SOUL.md / .cursorrules -# (or a project-local .hermes tree) outlives the turn and poisons every later -# session. Writes ALWAYS require human approval — even under --yolo — and fail -# closed when no human channel exists. (Ported from Roo-Code's -# RooProtectedController; the terminal-tool vector is gated separately.) -# -# Basenames match in ANY directory (instruction files load from cwd trees) and -# case-insensitively (case-insensitive filesystems; loaders probe variants). +# vector (AGENTS.md / CLAUDE.md / SOUL.md / .cursorrules / project .hermes tree). +# Writes ALWAYS require human approval — even under --yolo — and fail closed +# without a human channel. Basenames match in ANY directory, case-insensitively. _PROTECTED_INSTRUCTION_BASENAMES = frozenset({ - "agents.md", "claude.md", "soul.md", ".cursorrules", -}) - -_real_hermes_home_cached: str | None = None -_real_hermes_home_loaded = False - - -def _get_real_hermes_home() -> str | None: - """Return the realpath of the authoritative Hermes home (cached).""" - global _real_hermes_home_cached, _real_hermes_home_loaded - if _real_hermes_home_loaded: - return _real_hermes_home_cached - _real_hermes_home_loaded = True - try: - from hermes_constants import get_hermes_home - _real_hermes_home_cached = os.path.realpath(str(get_hermes_home())) - except Exception: - try: - _real_hermes_home_cached = os.path.realpath(_expand_tilde("~/.hermes")) - except Exception: - _real_hermes_home_cached = None - return _real_hermes_home_cached + "agents.md", "claude.md", "soul.md", ".cursorrules"}) def _protected_instruction_config() -> tuple[bool, list[str]]: """Return ``(enabled, extra_patterns)`` from ``security.protected_instruction_files`` / - ``security.protected_instruction_extra_patterns`` (fnmatch on basename). - - Config read failures keep the gate ON — fail-safe for a security boundary. - """ + ``security.protected_instruction_extra_patterns`` (fnmatch on basename). Config read + failures keep the gate ON — fail-safe for a security boundary.""" try: from hermes_cli.config import load_config, cfg_get cfg = load_config() - enabled = cfg_get(cfg, "security", "protected_instruction_files", - default=True) - extra = cfg_get(cfg, "security", "protected_instruction_extra_patterns", - default=[]) + enabled = cfg_get(cfg, "security", "protected_instruction_files", default=True) + extra = cfg_get(cfg, "security", "protected_instruction_extra_patterns", default=[]) except Exception: return True, [] if not isinstance(enabled, bool): @@ -146,11 +125,7 @@ def _protected_instruction_reason(filepath: str, task_id: str = "default", *, enabled: bool | None = None, extra_patterns: list[str] | None = None) -> str | None: """Return a short label when ``filepath`` targets a protected instruction file, else ``None``. - - Matches BOTH the normalized input and its realpath so neither a symlink - pointing AT a protected file nor a protected name that is itself a symlink - escapes; ``..`` traversal is neutralized by normpath/realpath first. - """ + Matches BOTH the normalized input and its realpath so no symlink direction escapes.""" if enabled is None or extra_patterns is None: enabled, extra_patterns = _protected_instruction_config() if not enabled: @@ -166,18 +141,15 @@ def _protected_instruction_reason(filepath: str, task_id: str = "default", # mirror guard, write_approval); this gate targets PROJECT-LOCAL files only. # Must run before the ``.hermes`` component rule, which would match the home. real_home = _get_real_hermes_home() - if real_home and (resolved == real_home - or resolved.startswith(real_home + os.sep)): + if real_home and (resolved == real_home or resolved.startswith(real_home + os.sep)): return None for candidate in (normalized, resolved): base = os.path.basename(candidate) base_lower = base.lower() - if base_lower in _PROTECTED_INSTRUCTION_BASENAMES: + if base_lower in _PROTECTED_INSTRUCTION_BASENAMES or any( + fnmatch.fnmatch(base_lower, pattern.lower()) for pattern in extra_patterns): return base - for pattern in extra_patterns: - if fnmatch.fnmatch(base_lower, pattern.lower()): - return base # Project-local .hermes config dirs (/.hermes/config.yaml) steer # behavior too. Only the IMMEDIATE parent counts — matching any ancestor # would gate every write inside a checkout living under ~/.hermes. @@ -187,43 +159,39 @@ def _protected_instruction_reason(filepath: str, task_id: str = "default", return None -def _request_protected_instruction_approval( - reasons: list[str], task_id: str = "default") -> str | None: - """Ask the human to approve a write to protected instruction file(s). +_APPROVAL_UNAVAILABLE = "requires approval but the approval subsystem is unavailable." +_NO_HUMAN = "requires approval but no interactive user or gateway is present to approve it." - Returns ``None`` when approved, else a BLOCKED error string. Deliberately - NOT routed through ``_run_approval_gate``: that honors --yolo and - session/permanent allowlists, and this gate is one-operation approval EVERY - time with no persisted scope. Fail-closed when no human channel exists. + +def _request_protected_instruction_approval(reasons: list[str], task_id: str = "default") -> str | None: + """Ask the human to approve a write to protected instruction file(s); ``None`` when approved. + + Deliberately NOT routed through ``_run_approval_gate`` (honors --yolo and + allowlists): this gate is one-operation approval EVERY time, no persisted + scope, fail-closed without a human channel. """ targets = ", ".join(dict.fromkeys(reasons)) description = ( f"Write to protected agent-instruction file(s): {targets}. " "These files steer future agent behavior; approval is always " - "required (not bypassed by auto-approve)." - ) + "required (not bypassed by auto-approve).") display = f"" blocked = ( f"BLOCKED: write to protected agent-instruction file(s) ({targets}) " "{why} The user has NOT consented to this write. Do NOT retry it or " "attempt the same edit via another path (terminal, execute_code, " - "etc.)." - ) - timed_out = blocked.format( - why="approval prompt timed out without a user response. " - "Silence is not consent.") + "etc.).") + timed_out = blocked.format(why="approval prompt timed out without a user response. Silence is not consent.") denied = blocked.format(why="was denied by the user.") try: import tools.approval as _approval except Exception: - return blocked.format(why="requires approval but the approval " - "subsystem is unavailable.") + return blocked.format(why=_APPROVAL_UNAVAILABLE) # Gateway surface: block on the button round-trip when a notify callback # is registered for this session. One-operation only — no scope buttons. session_key = _approval.get_current_session_key() - notify_cb = None try: with _approval._lock: notify_cb = _approval._gateway_notify_cbs.get(session_key) @@ -237,82 +205,48 @@ def _request_protected_instruction_approval( "pattern_keys": ["protected_instruction_file"], "description": description, "allow_permanent": False, - "allow_session": False, - } - decision = _approval._await_gateway_decision( - session_key, notify_cb, approval_data, surface="gateway", - ) + "allow_session": False} + decision = _approval._await_gateway_decision(session_key, notify_cb, approval_data, surface="gateway") if decision.get("notify_failed"): - return blocked.format( - why="requires approval but the approval request could not " - "be delivered.") - choice = decision.get("choice") - # Any tapped scope is a one-operation grant; nothing is persisted. - if decision.get("resolved") and choice in {"once", "session", "always"}: - return None - if not decision.get("resolved"): - return timed_out - return denied - - # CLI surface: per-thread approval callback (prompt_toolkit panel). - callback = None - try: - from tools.terminal_tool import _get_approval_callback - callback = _get_approval_callback() - except Exception: - callback = None - - if callback is not None: + return blocked.format(why="requires approval but the approval request could not be delivered.") + choice, timed = decision.get("choice"), not decision.get("resolved") + else: + # CLI surface: per-thread approval callback (prompt_toolkit panel). + try: + from tools.terminal_tool import _get_approval_callback + callback = _get_approval_callback() + except Exception: + callback = None + if callback is None: + # No human channel (script, cron, background thread): fail closed — + # auto-approving here would recreate the persistence vector. + return blocked.format(why=_NO_HUMAN) choice = _approval.prompt_dangerous_approval( - display, description, - allow_permanent=False, - allow_session=False, - approval_callback=callback, - ) - if choice in {"once", "session", "always"}: - return None - if choice == "timeout": - return timed_out - return denied - - # No human channel (script, cron, background thread): fail closed — - # auto-approving here would recreate the persistence vector. - return blocked.format( - why="requires approval but no interactive user or gateway is " - "present to approve it.") + display, description, allow_permanent=False, allow_session=False, approval_callback=callback) + timed = choice == "timeout" + # Any tapped scope is a one-operation grant; nothing is persisted. + if not timed and choice in {"once", "session", "always"}: + return None + return timed_out if timed else denied -def _check_protected_instruction_write(paths: list[str], - task_id: str = "default") -> str | None: - """Gate a write/patch touching protected instruction files. - - ONE protected file gates the ENTIRE multi-file patch: a single prompt lists - every protected target and a deny applies nothing (atomic all-or-nothing - beats a partially-applied patch). - """ +def _check_protected_instruction_write(paths: list[str], task_id: str = "default") -> str | None: + """Gate a write/patch touching protected instruction files. ONE protected file gates + the ENTIRE multi-file patch (one prompt, all-or-nothing).""" enabled, extra = _protected_instruction_config() if not enabled: return None - reasons = [ - r for r in ( - _protected_instruction_reason(p, task_id, enabled=enabled, extra_patterns=extra) - for p in paths - ) if r - ] + reasons = [r for r in (_protected_instruction_reason(p, task_id, enabled=enabled, extra_patterns=extra) + for p in paths) if r] if not reasons: return None return _request_protected_instruction_approval(reasons, task_id) -def _check_approval_required_write(paths: list[str], - task_id: str = "default") -> str | None: - """Gate a write/patch touching an approval-required path (``~/.ssh/config``). - - Not credentials and not hard-denied, but they can steer process execution - (SSH ``ProxyCommand`` / ``Match exec``). Unlike the protected-instruction - gate this is a routine user edit: the prompt offers once/session/always and - honors --yolo. Fail-closed when no interactive/gateway channel exists. - """ +def _check_approval_required_write(paths: list[str], task_id: str = "default") -> str | None: + """Gate a write/patch touching an approval-required path (``~/.ssh/config`` can steer + execution via ``ProxyCommand``). Routine gate: once/session/always, honors --yolo, + fail-closed without an interactive/gateway channel.""" try: from agent.file_safety import is_write_approval_required except Exception: @@ -326,26 +260,22 @@ def _check_approval_required_write(paths: list[str], description = ( f"Write to SSH client config file(s): {display_targets}. " "The SSH config can carry ProxyCommand / Match exec directives that " - "run commands, so writes require your approval." - ) + "run commands, so writes require your approval.") blocked = ( f"BLOCKED: write to SSH config file(s) ({display_targets}) " "{why} Do NOT retry it via another path (terminal, execute_code) " - "without the user's explicit consent." - ) + "without the user's explicit consent.") try: import tools.approval as _approval except Exception: - return blocked.format(why="requires approval but the approval " - "subsystem is unavailable.") + return blocked.format(why=_APPROVAL_UNAVAILABLE) result = _approval._run_approval_gate( pattern_key="ssh_config_write", description=description, display_target=f"", - cron_deny_message=blocked.format( - why="requires approval but this cron session denies it."), + cron_deny_message=blocked.format(why="requires approval but this cron session denies it."), single_query_deny_message=blocked.format( why="requires approval but single-query (-q) sessions run " "without a user present to approve it. To allow flagged " @@ -353,10 +283,7 @@ def _check_approval_required_write(paths: list[str], "approve in config.yaml."), autoapprove_log_prefix="ssh_config_write", fail_closed_when_no_human=True, - no_human_block_message=blocked.format( - why="requires approval but no interactive user or gateway is " - "present to approve it."), - ) + no_human_block_message=blocked.format(why=_NO_HUMAN)) if result.get("approved"): return None return result.get("message") or blocked.format(why="was denied.") @@ -366,74 +293,41 @@ def _get_container_mirror_prefix_for_task(task_id: str = "default") -> str | Non """Return the container-side Hermes mirror prefix for persistent Docker file tools.""" try: from tools.terminal_tool import ( - _active_environments, - _env_lock, - _get_env_config, - _resolve_container_task_id, - ) - + _active_environments, _env_lock, _get_env_config, _resolve_container_task_id) container_key = _resolve_container_task_id(task_id) - except Exception: - return None - - try: with _env_lock: env = _active_environments.get(container_key) or _active_environments.get(task_id) - if env is not None: - if env.__class__.__name__ == "DockerEnvironment" and bool( - getattr(env, "_persistent", False) - ): - return "/root/.hermes" - return None - + persistent_docker = (env.__class__.__name__ == "DockerEnvironment" + and bool(getattr(env, "_persistent", False))) + return "/root/.hermes" if persistent_docker else None config = _get_env_config() except Exception: return None - if config.get("env_type") == "docker" and config.get("container_persistent", True): return "/root/.hermes" return None def _check_cross_profile_path(filepath: str, task_id: str = "default") -> str | None: - """Soft-guard: warn when ``filepath`` lands on a host-side or Docker sandbox - MIRROR of Hermes state — a write the host process never reads (lost work). - - Not profile isolation: the former cross-PROFILE guard was removed by - maintainer decision (profiles were never isolated). ``cross_profile=True`` - on the tools still bypasses these mirror guards (name kept for replay compat). - Fails open on import error — the sensitive-path guard and denylist still apply. - """ + """Soft-guard: warn when ``filepath`` lands on a host-side or Docker sandbox MIRROR of + Hermes state (a write the host never reads). Not profile isolation — that guard was + removed; ``cross_profile=True`` keeps bypassing this one for replay compat. Fails open.""" try: - from agent.file_safety import ( - get_container_mirror_warning, - get_sandbox_mirror_warning, - ) + from agent.file_safety import get_container_mirror_warning, get_sandbox_mirror_warning except Exception: return None - resolved = _resolved_or_raw(filepath, task_id) - warning = get_sandbox_mirror_warning(resolved) if warning is not None: return warning - - return get_container_mirror_warning( - resolved, - mirror_prefix=_get_container_mirror_prefix_for_task(task_id), - ) + return get_container_mirror_warning(resolved, mirror_prefix=_get_container_mirror_prefix_for_task(task_id)) def _check_binary_document_write(filepath: str, task_id: str = "default") -> str | None: - """Reject text-tool writes that would corrupt a binary document. - - ``read_file`` auto-extracts Office/PDF to text, so the model plausibly - believes it holds the file's bytes and writes edited text back — which can - never form a valid container. Opaque formats (.docx/.xlsx/.pptx/.odt/...) - are always rejected; .pdf only when OVERWRITING an existing regular file - (raw PDF syntax is text-authorable, so new-file creation stays allowed). - """ + """Reject text-tool writes that would corrupt a binary document (read_file showed + EXTRACTED text, so the model may write it back). Opaque formats are always rejected; + .pdf only when OVERWRITING an existing file (raw PDF syntax is text-authorable).""" if has_opaque_document_extension(filepath): ext = filepath[filepath.rfind("."):].lower() return ( @@ -442,8 +336,7 @@ def _check_binary_document_write(filepath: str, task_id: str = "default") -> str "corrupt the file (read_file showed you EXTRACTED text, not the real " "bytes). Use the docx/xlsx/powerpoint skills or a library like " "python-docx/openpyxl/python-pptx via the terminal to create or edit " - "this document." - ) + "this document.") if is_pdf_path(filepath): try: resolved = Path(_resolve_path_for_task(filepath, task_id)) @@ -456,75 +349,48 @@ def _check_binary_document_write(filepath: str, task_id: str = "default") -> str "read_file showed you EXTRACTED text, not the real bytes — writing " "text back would destroy the document. Use the pdf skill or a PDF " "library via the terminal to modify it. (Creating a NEW .pdf file " - "is allowed.)" - ) + "is allowed.)") except OSError: pass return None -# --------------------------------------------------------------------------- -# Internal display text must never be persisted as file content -# --------------------------------------------------------------------------- +# ── Internal display text must never be persisted as file content ──────── _READ_DEDUP_STATUS_MESSAGE = ( "File unchanged since last read. The content from " "the earlier read_file result in this conversation is " - "still current — refer to that instead of re-reading." -) + "still current — refer to that instead of re-reading.") def _is_internal_file_status_text(content: str) -> bool: - """True when content is the read_file dedup status message (verbatim or lightly framed). - - Models echo the message verbatim OR wrap it with short framing ("Note:", - a trailing comment). Any write whose stripped body contains the full - message and is <=2x its length is status-dominated — a real file quoting - this message would be dramatically longer. - """ + """True when content is the read_file dedup status message, verbatim or lightly framed + (contains the full message and is <=2x its length — a real file quoting it would be longer).""" if not isinstance(content, str): return False stripped = content.strip() - if not stripped: - return False - if stripped == _READ_DEDUP_STATUS_MESSAGE: - return True - return (_READ_DEDUP_STATUS_MESSAGE in stripped - and len(stripped) <= 2 * len(_READ_DEDUP_STATUS_MESSAGE)) + return bool(stripped) and _READ_DEDUP_STATUS_MESSAGE in stripped and ( + len(stripped) <= 2 * len(_READ_DEDUP_STATUS_MESSAGE)) def _looks_like_read_file_line_numbered_content(content: str) -> bool: - """True for content dominated by read_file's ``LINE_NUM|CONTENT`` display. - - Rejects writes whose non-empty lines are mostly (>=60%) consecutive - numbered lines, while allowing sparse literal pipe content such as a - single ``1|value`` line. - """ + """True for content dominated by read_file's ``LINE_NUM|CONTENT`` display (>=60% of + non-empty lines are consecutive numbered lines; a lone ``1|value`` is allowed).""" if not isinstance(content, str): return False - lines = [line for line in content.splitlines() if line.strip()] if len(lines) < 2: return False - numbered: list[int] = [] for line in lines: prefix, sep, _rest = line.lstrip().partition("|") if sep and prefix.isdigit(): numbered.append(int(prefix)) - if len(numbered) < 2 or len(numbered) / len(lines) < 0.6: return False - - consecutive_pairs = sum( - 1 for prev, current in zip(numbered, numbered[1:]) - if current == prev + 1 - ) + consecutive_pairs = sum(1 for prev, current in zip(numbered, numbered[1:]) if current == prev + 1) return consecutive_pairs >= len(numbered) - 1 def _is_internal_file_tool_content(content: str) -> bool: """Return True when content is file-tool display text, not intended file bytes.""" - return ( - _is_internal_file_status_text(content) - or _looks_like_read_file_line_numbered_content(content) - ) + return _is_internal_file_status_text(content) or _looks_like_read_file_line_numbered_content(content) diff --git a/tools/fuzzy_match.py b/tools/fuzzy_match.py index a991c00f6b..ab9491e36f 100644 --- a/tools/fuzzy_match.py +++ b/tools/fuzzy_match.py @@ -1,67 +1,321 @@ #!/usr/bin/env python3 """Fuzzy find-and-replace for LLM-generated edits. -Tries an ordered chain of increasingly permissive matching strategies (see -:mod:`tools.fuzzy_match_strategies`) so whitespace, indentation, escaping and -Unicode drift in tool-call arguments still land on the intended region. +An ordered chain of increasingly permissive strategies (``STRATEGIES``) lets +whitespace, indentation, escaping and Unicode drift in tool-call arguments +still land on the intended region:: new_content, match_count, strategy, error = fuzzy_find_and_replace( content, old_string, new_string, replace_all=False) """ +import re from difflib import SequenceMatcher -from typing import List, Optional, Tuple +from typing import Callable, Optional -from tools.fuzzy_match_strategies import ( # noqa: F401 — re-exported names - SIMILARITY_STRATEGIES, - STRATEGIES, - UNICODE_MAP, - _build_orig_to_norm_map, - _calculate_line_positions, - _invert_norm_map, - _map_normalized_positions, - _map_positions_norm_to_orig, - _norm_end_to_orig, - _strategy_block_anchor, - _strategy_context_aware, - _strategy_escape_normalized, - _strategy_exact, - _strategy_indentation_flexible, - _strategy_line_trimmed, - _strategy_trimmed_boundary, - _strategy_unicode_normalized, - _strategy_whitespace_normalized, - _unicode_normalize, -) +Span = tuple[int, int] IDENTICAL_STRINGS_ERROR = ( "No edit was applied because old_string and new_string are identical. " "Provide the existing text to replace in old_string and the changed " - "replacement text in new_string." -) + "replacement text in new_string.") +UNICODE_MAP = { + "\u201c": '"', "\u201d": '"', # smart double quotes + "\u2018": "'", "\u2019": "'", # smart single quotes + "\u2014": "--", "\u2013": "-", # em/en dashes + "\u2026": "...", "\u00a0": " ", # ellipsis and non-breaking space + "\u2212": "-", # typographic minus (math/scientific docs) + # Space-separator family (Zs): otherwise such files fall to the similarity fallback. + "\u2000": " ", "\u2001": " ", "\u2002": " ", "\u2003": " ", + "\u2004": " ", "\u2005": " ", "\u2006": " ", "\u2007": " ", + "\u2008": " ", "\u2009": " ", "\u200a": " ", "\u202f": " ", + "\u205f": " ", "\u3000": " "} + + +def _unicode_normalize(text: str) -> str: + """Map typographic Unicode variants to ASCII equivalents.""" + for char, repl in UNICODE_MAP.items(): + text = text.replace(char, repl) + return text + + +# ── Position helpers ───────────────────────────────────────────────────── + +def _calculate_line_positions(content_lines: list[str], start_line: int, + end_line: int, content_length: int) -> Span: + """Character span covering ``content_lines[start_line:end_line]`` (end exclusive).""" + start_pos = sum(len(line) + 1 for line in content_lines[:start_line]) + end_pos = sum(len(line) + 1 for line in content_lines[:end_line]) - 1 + return start_pos, min(content_length, end_pos) + + +def _window_spans(content: str, content_lines: list[str], n: int, + accept: Callable[[int], bool]) -> list[Span]: + """Spans of every ``n``-line window starting at ``i`` for which ``accept(i)``.""" + return [ + _calculate_line_positions(content_lines, i, i + n, len(content)) + for i in range(len(content_lines) - n + 1) if accept(i)] + + +def _match_transformed_lines(content: str, pattern: str, + transform: Callable[[list[str]], list[str]]) -> list[Span]: + """Match ``pattern`` against ``content`` after applying ``transform`` to each line block.""" + content_lines = content.split('\n') + pattern_norm = transform(pattern.split('\n')) + n = len(pattern_norm) + return _window_spans(content, content_lines, n, + lambda i: transform(content_lines[i:i + n]) == pattern_norm) + + +def _strip_boundary(lines: list[str]) -> list[str]: + """Strip whitespace on the first and last lines only.""" + lines = list(lines) + lines[0] = lines[0].strip() + if len(lines) > 1: + lines[-1] = lines[-1].strip() + return lines + + +def _build_orig_to_norm_map(original: str) -> list[int]: + """Map each original index to its index in ``_unicode_normalize(original)`` + (replacements can expand one char into several). Length ``len(original)+1``; + the last entry is a sentinel one past the final character.""" + result: list[int] = [] + norm_pos = 0 + for char in original: + result.append(norm_pos) + repl = UNICODE_MAP.get(char) + norm_pos += len(repl) if repl is not None else 1 + result.append(norm_pos) + return result + + +def _invert_norm_map(orig_to_norm: list[int]) -> dict[int, int]: + """norm_pos -> first original position mapping to it.""" + inverted: dict[int, int] = {} + for orig_pos, norm_pos in enumerate(orig_to_norm[:-1]): + inverted.setdefault(norm_pos, orig_pos) + return inverted + + +def _norm_end_to_orig(orig_to_norm: list[int], orig_start: int, norm_end: int) -> int: + """Walk from ``orig_start`` until the mapped position reaches ``norm_end``.""" + orig_len = len(orig_to_norm) - 1 + orig_end = orig_start + while orig_end < orig_len and orig_to_norm[orig_end] < norm_end: + orig_end += 1 + return orig_end + + +def _map_positions_norm_to_orig(orig_to_norm: list[int], norm_matches: list[Span]) -> list[Span]: + """Convert spans in the normalised string to original-string spans.""" + norm_to_orig_start = _invert_norm_map(orig_to_norm) + results: list[Span] = [] + for norm_start, norm_end in norm_matches: + if norm_start in norm_to_orig_start: + orig_start = norm_to_orig_start[norm_start] + results.append((orig_start, _norm_end_to_orig(orig_to_norm, orig_start, norm_end))) + return results + + +def _map_normalized_positions(original: str, normalized: str, + normalized_matches: list[Span]) -> list[Span]: + """Best-effort span mapping for ``[ \\t]+`` -> ``' '`` whitespace collapsing.""" + orig_to_norm = [] # orig_to_norm[i] = position in normalized + orig_idx = norm_idx = 0 + while orig_idx < len(original) and norm_idx < len(normalized): + if original[orig_idx] == normalized[norm_idx]: + orig_to_norm.append(norm_idx) + orig_idx += 1 + norm_idx += 1 + elif original[orig_idx] in ' \t' and normalized[norm_idx] == ' ': + # Collapsed run: advance norm_idx only once the run is consumed. + orig_to_norm.append(norm_idx) + orig_idx += 1 + if orig_idx < len(original) and original[orig_idx] not in ' \t': + norm_idx += 1 + else: + # Extra whitespace in original, or a mismatch normalization should + # never produce — either way, pin to the current norm_idx. + orig_to_norm.append(norm_idx) + orig_idx += 1 + orig_to_norm.extend([len(normalized)] * (len(original) - orig_idx)) + + norm_to_orig_start = {} + norm_to_orig_end = {} + for orig_pos, norm_pos in enumerate(orig_to_norm): + norm_to_orig_start.setdefault(norm_pos, orig_pos) + norm_to_orig_end[norm_pos] = orig_pos + + original_matches = [] + for norm_start, norm_end in normalized_matches: + if norm_start in norm_to_orig_start: + orig_start = norm_to_orig_start[norm_start] + else: + orig_start = min(i for i, n in enumerate(orig_to_norm) if n >= norm_start) + if norm_end - 1 in norm_to_orig_end: + orig_end = norm_to_orig_end[norm_end - 1] + 1 + else: + orig_end = orig_start + (norm_end - norm_start) + # Absorb trailing collapsed whitespace only when the normalized match + # itself ended in a space; otherwise the first whitespace after the + # match is a word boundary that must survive. + if norm_end < len(normalized) and normalized[norm_end - 1] == ' ': + while orig_end < len(original) and original[orig_end] in ' \t': + orig_end += 1 + original_matches.append((orig_start, min(orig_end, len(original)))) + return original_matches + + +# ── Strategies ─────────────────────────────────────────────────────────── +# Each takes ``(content, pattern)`` and returns ``(start, end)`` spans in the +# ORIGINAL content. + +def _strategy_exact(content: str, pattern: str) -> list[Span]: + """Strategy 1: exact, non-overlapping occurrences (str.replace semantics — + overlapping spans would corrupt the file under replace_all).""" + return [m.span() for m in re.finditer(re.escape(pattern), content)] + + +def _strategy_line_trimmed(content: str, pattern: str) -> list[Span]: + """Strategy 2: strip each line before comparing.""" + return _match_transformed_lines(content, pattern, lambda ls: [l.strip() for l in ls]) + + +def _strategy_whitespace_normalized(content: str, pattern: str) -> list[Span]: + """Strategy 3: collapse runs of spaces/tabs to a single space.""" + def normalize(s): + return re.sub(r'[ \t]+', ' ', s) + + content_normalized = normalize(content) + matches_in_normalized = _strategy_exact(content_normalized, normalize(pattern)) + if not matches_in_normalized: + return [] + return _map_normalized_positions(content, content_normalized, matches_in_normalized) + + +def _strategy_indentation_flexible(content: str, pattern: str) -> list[Span]: + """Strategy 4: ignore leading indentation entirely.""" + return _match_transformed_lines(content, pattern, lambda ls: [l.lstrip() for l in ls]) + + +def _strategy_escape_normalized(content: str, pattern: str) -> list[Span]: + """Strategy 5: treat literal ``\\n``/``\\t``/``\\r`` in the pattern as control chars.""" + pattern_unescaped = pattern.replace('\\n', '\n').replace('\\t', '\t').replace('\\r', '\r') + if pattern_unescaped == pattern: + return [] + return _strategy_exact(content, pattern_unescaped) + + +def _strategy_trimmed_boundary(content: str, pattern: str) -> list[Span]: + """Strategy 6: strip whitespace on the first and last lines only.""" + return _match_transformed_lines(content, pattern, _strip_boundary) + + +def _strategy_unicode_normalized(content: str, pattern: str) -> list[Span]: + """Strategy 7: exact/line-trimmed match after Unicode->ASCII normalisation of both sides.""" + norm_pattern = _unicode_normalize(pattern) + norm_content = _unicode_normalize(content) + if norm_content == content and norm_pattern == pattern: + return [] + norm_matches = (_strategy_exact(norm_content, norm_pattern) + or _strategy_line_trimmed(norm_content, norm_pattern)) + if not norm_matches: + return [] + return _map_positions_norm_to_orig(_build_orig_to_norm_map(content), norm_matches) + + +def _strategy_block_anchor(content: str, pattern: str) -> list[Span]: + """Strategy 8: anchor on first+last lines, similarity-score the middle.""" + pattern_lines = _unicode_normalize(pattern).split('\n') + if len(pattern_lines) < 2: + return [] + first_line = pattern_lines[0].strip() + last_line = pattern_lines[-1].strip() + n = len(pattern_lines) + + # Match on normalized lines; compute offsets from the ORIGINAL lines so + # multi-char expansions (em-dash -> '--') don't shift positions. + norm_content_lines = _unicode_normalize(content).split('\n') + potential_matches = { + i for i in range(len(norm_content_lines) - n + 1) + if norm_content_lines[i].strip() == first_line + and norm_content_lines[i + n - 1].strip() == last_line} + # Looser thresholds (0.10/0.30) matched unrelated blocks; these are the safe floor. + threshold = 0.50 if len(potential_matches) == 1 else 0.70 + pattern_middle = '\n'.join(pattern_lines[1:-1]) + + def similar(i: int) -> bool: + if i not in potential_matches: + return False + if n <= 2: + return True + content_middle = '\n'.join(norm_content_lines[i + 1:i + n - 1]) + return SequenceMatcher(None, content_middle, pattern_middle).ratio() >= threshold + + return _window_spans(content, content.split('\n'), n, similar) + + +def _strategy_context_aware(content: str, pattern: str) -> list[Span]: + """Strategy 9 (last resort): anchored per-line similarity, every non-blank line >= 0.80. + The anchor pre-filter bounds the scan; the all-lines rule stops coincidental matches.""" + pattern_lines = pattern.split('\n') + content_lines = content.split('\n') + n = len(pattern_lines) + if n > len(content_lines): + return [] + first_pat = pattern_lines[0].strip() + last_pat = pattern_lines[-1].strip() + + def _sim(a: str, b: str) -> float: + return 1.0 if a == b else SequenceMatcher(None, a, b).ratio() + + def accept(i: int) -> bool: + block_lines = content_lines[i:i + n] + if _sim(first_pat, block_lines[0].strip()) < 0.80: + return False + if _sim(last_pat, block_lines[-1].strip()) < 0.80: + return False + return all( + not p_line.strip() or _sim(p_line.strip(), c_line.strip()) >= 0.80 + for p_line, c_line in zip(pattern_lines, block_lines)) + + return _window_spans(content, content_lines, n, accept) + + +# Ordered chain: precise strategies first, similarity-based last. +STRATEGIES: list[tuple[str, Callable[[str, str], list[Span]]]] = [ + ("exact", _strategy_exact), + ("line_trimmed", _strategy_line_trimmed), + ("whitespace_normalized", _strategy_whitespace_normalized), + ("indentation_flexible", _strategy_indentation_flexible), + ("escape_normalized", _strategy_escape_normalized), + ("trimmed_boundary", _strategy_trimmed_boundary), + ("unicode_normalized", _strategy_unicode_normalized), + ("block_anchor", _strategy_block_anchor), + ("context_aware", _strategy_context_aware)] + +# Matches from these only *approximately* resemble old_string — fine for one +# unique replacement, never safe under replace_all. +SIMILARITY_STRATEGIES = frozenset({"block_anchor", "context_aware"}) + + +# ── Orchestrator ───────────────────────────────────────────────────────── def is_already_applied(content: str, old_string: str, new_string: str) -> bool: - """True when the requested edit is already present (re-sent edit -> success-shaped no-op). - - Conservative: new_string must be non-trivial (>= 8 chars stripped) and - appear EXACTLY; when it differs from old_string, old_string must be gone. - """ - if not new_string or len(new_string.strip()) < 8: + """True when the edit is already present (re-sent edit -> success-shaped no-op). + Conservative: new_string non-trivial (>= 8 chars) and present EXACTLY; old_string gone.""" + if not new_string or len(new_string.strip()) < 8 or new_string not in content: return False - if new_string not in content: - return False - if old_string == new_string: - return True - return old_string not in content + return old_string == new_string or old_string not in content -def _matched_regions(content: str, matches: List[Tuple[int, int]]) -> str: +def _matched_regions(content: str, matches: list[Span]) -> str: return "".join(content[start:end] for start, end in matches) -def _format_match_locations(content: str, matches: List[Tuple[int, int]], - cap: int = 5) -> str: +def _format_match_locations(content: str, matches: list[Span], cap: int = 5) -> str: """Render up to ``cap`` match positions as 'L: ' rows.""" rows = [] for start, _end in matches[:cap]: @@ -81,7 +335,7 @@ def _format_match_locations(content: str, matches: List[Tuple[int, int]], def fuzzy_find_and_replace(content: str, old_string: str, new_string: str, - replace_all: bool = False) -> Tuple[str, int, Optional[str], Optional[str]]: + replace_all: bool = False) -> tuple[str, int, Optional[str], Optional[str]]: """Find and replace via the strategy chain. Returns ``(new_content, match_count, strategy_name, error)``; on failure @@ -106,15 +360,13 @@ def fuzzy_find_and_replace(content: str, old_string: str, new_string: str, return content, 0, None, ( f"Found {len(matches)} matches for old_string. " f"Provide more context to make it unique, or use replace_all=True. " - f"Matches:\n{locations}" - ) + f"Matches:\n{locations}") if replace_all and len(matches) > 1 and strategy_name in SIMILARITY_STRATEGIES: return content, 0, None, ( f"Found {len(matches)} approximate matches via the " f"'{strategy_name}' strategy; replace_all only applies to exact " f"matches. Provide the precise text (whitespace included) so an " - f"exact/line-trimmed match can be made." - ) + f"exact/line-trimmed match can be made.") # Non-exact matches came through some normalization, so new_string may # carry serialization drift the file doesn't have. @@ -125,13 +377,10 @@ def fuzzy_find_and_replace(content: str, old_string: str, new_string: str, effective_new = _maybe_unescape_new_string(new_string, content, matches) if strategy_name == "unicode_normalized": - effective_new = _preserve_unicode_in_replacement( - content, matches, old_string, effective_new, - ) + effective_new = _preserve_unicode_in_replacement(content, matches, old_string, effective_new) new_content = _apply_replacements( content, matches, effective_new, - old_string=old_string if strategy_name != "exact" else None, - ) + old_string=old_string if strategy_name != "exact" else None) return new_content, len(matches), strategy_name, None return content, 0, None, "Could not find a match for old_string in the file" @@ -139,14 +388,10 @@ def fuzzy_find_and_replace(content: str, old_string: str, new_string: str, # ── Escape-drift guards ────────────────────────────────────────────────── -def _detect_escape_drift(content: str, matches: List[Tuple[int, int]], +def _detect_escape_drift(content: str, matches: list[Span], old_string: str, new_string: str) -> Optional[str]: - """Error string when new_string carries tool-call escape artifacts, else None. - - Fires on ``\\'``/``\\"`` present in both old_string and new_string but - absent from the matched region (spurious shell-style escaping), and on - JSON double-escaped backslash runs (see ``_detect_backslash_doubling``). - """ + """Error string when new_string carries tool-call escape artifacts, else None: + ``\\'``/``\\"`` in both strings but not the matched region, or doubled backslash runs.""" has_quote_suspects = "\\'" in new_string or '\\"' in new_string if not has_quote_suspects and "\\" not in old_string: return None @@ -163,47 +408,27 @@ def _detect_escape_drift(content: str, matches: List[Tuple[int, int]], f"serialization artifact where an apostrophe or quote got " f"prefixed with a spurious backslash. Re-read the file with " f"read_file and pass old_string/new_string without " - f"backslash-escaping {plain!r} characters." - ) + f"backslash-escaping {plain!r} characters.") return _detect_backslash_doubling(matched_regions, old_string, new_string) -def _backslash_runs(s: str) -> List[int]: +def _backslash_runs(s: str) -> list[int]: """Lengths of maximal backslash runs in ``s``, in order.""" - runs: List[int] = [] - n = 0 - for ch in s: - if ch == "\\": - n += 1 - elif n: - runs.append(n) - n = 0 - if n: - runs.append(n) - return runs + return [len(run) for run in re.findall(r"\\+", s)] def _detect_backslash_doubling(matched_regions: str, old_string: str, new_string: str) -> Optional[str]: - """Detect old_string whose every backslash run is exactly 2x the file's. - - That pattern means the arguments were JSON-escaped one extra time; a - similarity strategy still matches, and writing new_string verbatim would - double every backslash in the file. Requires the same run count, a - non-trivial signal (a run >= 2 or 2+ runs), and new_string not already - matching the file's counts. - """ + """Detect old_string whose every backslash run is exactly 2x the file's (arguments + JSON-escaped one extra time). Requires the same run count, a non-trivial signal + (a run >= 2 or 2+ runs), and new_string not already matching the file's counts.""" old_runs = _backslash_runs(old_string) file_runs = _backslash_runs(matched_regions) - if not old_runs or not file_runs or len(old_runs) != len(file_runs): - return None - if old_runs == file_runs: - return None - if any(o != f * 2 for o, f in zip(old_runs, file_runs)): - return None - if not (any(f >= 2 for f in file_runs) or len(file_runs) >= 2): - return None - if _backslash_runs(new_string) == file_runs: + if (not old_runs or not file_runs or len(old_runs) != len(file_runs) + or old_runs == file_runs + or any(o != f * 2 for o, f in zip(old_runs, file_runs)) + or not (any(f >= 2 for f in file_runs) or len(file_runs) >= 2) + or _backslash_runs(new_string) == file_runs): return None return ( "Escape-drift detected: every backslash run in old_string is exactly " @@ -212,29 +437,20 @@ def _detect_backslash_doubling(matched_regions: str, old_string: str, "were JSON-escaped one extra time; applying new_string verbatim would " "double every backslash in the file. Re-read the file with read_file " "and resend old_string/new_string with the backslash counts exactly " - "as they appear in the file." - ) + "as they appear in the file.") -def _maybe_unescape_new_string(new_string: str, content: str, - matches: List[Tuple[int, int]]) -> str: - """Convert literal ``\\t``/``\\r`` in new_string to control chars, per sequence, - only when the matched file region already contains the real control char. - - Files that legitimately contain the two-char string (e.g. ``sep = "\\t"``) - have a backslash+t in the region, not a tab, so they're left alone. - ``\\n`` is deliberately excluded: newlines serialize correctly through - JSON and rewriting them would mangle escape sequences in source literals. - """ +def _maybe_unescape_new_string(new_string: str, content: str, matches: list[Span]) -> str: + """Convert literal ``\\t``/``\\r`` in new_string to control chars, per sequence, only + when the matched region already contains the real control char (so ``sep = "\\t"`` files + are left alone). ``\\n`` is excluded: rewriting it would mangle source escape literals.""" if "\\t" not in new_string and "\\r" not in new_string: return new_string matched_regions = _matched_regions(content, matches) - out = new_string - if "\\t" in out and "\t" in matched_regions: - out = out.replace("\\t", "\t") - if "\\r" in out and "\r" in matched_regions: - out = out.replace("\\r", "\r") - return out + for literal, control in (("\\t", "\t"), ("\\r", "\r")): + if literal in new_string and control in matched_regions: + new_string = new_string.replace(literal, control) + return new_string # ── Replacement shaping ────────────────────────────────────────────────── @@ -244,20 +460,13 @@ def _leading_whitespace(line: str) -> str: def _first_meaningful_line(text: str) -> Optional[str]: - for line in text.split("\n"): - if line.strip(): - return line - return None + return next((line for line in text.split("\n") if line.strip()), None) def _reindent_replacement(file_region: str, old_string: str, new_string: str) -> str: - """Re-anchor ``new_string``'s indentation onto the file's actual base indent. - - After a non-exact match the LLM's base indent (first non-blank line of - old_string) may differ from the file's. Each non-blank new_string line - swaps the LLM base prefix for the file's, preserving relative nesting; - lines shallower than the LLM base are anchored to the file base. - """ + """Re-anchor ``new_string``'s indentation onto the file's actual base indent after a + non-exact match: swap the LLM base prefix (first non-blank old_string line) for the + file's, preserving relative nesting; shallower lines anchor to the file base.""" if not new_string: return new_string old_first = _first_meaningful_line(old_string) @@ -269,7 +478,7 @@ def _reindent_replacement(file_region: str, old_string: str, new_string: str) -> if old_indent == file_indent: return new_string - out_lines: List[str] = [] + out_lines: list[str] = [] for line in new_string.split("\n"): if not line.strip(): out_lines.append(line) @@ -280,17 +489,10 @@ def _reindent_replacement(file_region: str, old_string: str, new_string: str) -> return "\n".join(out_lines) -def _preserve_unicode_in_replacement( - content: str, matches: List[Tuple[int, int]], - old_string: str, new_string: str, -) -> str: - """Apply only the old->new edits onto the file's original (Unicode) text. - - After a unicode_normalized match, writing the LLM's ASCII new_string - verbatim would flatten the file's em-dashes/smart quotes. Diff the - normalized old_string against new_string and keep the file's original - characters for every ``equal`` span. - """ +def _preserve_unicode_in_replacement(content: str, matches: list[Span], + old_string: str, new_string: str) -> str: + """Apply only the old->new edits onto the file's original (Unicode) text, so a + unicode_normalized match doesn't flatten the file's em-dashes/smart quotes.""" file_region = _matched_regions(content, matches) norm_old = _unicode_normalize(old_string) if norm_old != _unicode_normalize(file_region): @@ -299,7 +501,7 @@ def _preserve_unicode_in_replacement( file_orig_to_norm = _build_orig_to_norm_map(file_region) file_norm_to_orig = _invert_norm_map(file_orig_to_norm) - result_parts: List[str] = [] + result_parts: list[str] = [] for tag, i1, i2, j1, j2 in SequenceMatcher(None, norm_old, new_string).get_opcodes(): if tag == "equal": orig_start = file_norm_to_orig.get(i1, 0) @@ -310,13 +512,10 @@ def _preserve_unicode_in_replacement( return "".join(result_parts) -def _apply_replacements(content: str, matches: List[Tuple[int, int]], +def _apply_replacements(content: str, matches: list[Span], new_string: str, old_string: Optional[str] = None) -> str: - """Splice ``new_string`` over each span (end-to-start so offsets stay valid). - - ``old_string`` non-None signals a non-exact match: new_string is - re-indented per region to the file's actual indentation. - """ + """Splice ``new_string`` over each span (end-to-start so offsets stay valid); + ``old_string`` non-None (non-exact match) re-indents it per region.""" result = content for start, end in sorted(matches, key=lambda x: x[0], reverse=True): adjusted = new_string @@ -330,12 +529,9 @@ def _apply_replacements(content: str, matches: List[Tuple[int, int]], def _visualize_whitespace(line: str) -> str: """Render the leading whitespace run visibly (→ = tab, · = space).""" - i = 0 - prefix = [] - while i < len(line) and line[i] in (" ", "\t"): - prefix.append("→" if line[i] == "\t" else "·") - i += 1 - return "".join(prefix) + line[i:] + stripped = line.lstrip(" \t") + prefix = line[:len(line) - len(stripped)] + return prefix.replace("\t", "→").replace(" ", "·") + stripped def find_closest_lines(old_string: str, content: str, context_lines: int = 2, max_results: int = 3) -> str: @@ -347,25 +543,15 @@ def find_closest_lines(old_string: str, content: str, context_lines: int = 2, ma if not old_lines or not content_lines: return "" - anchor = old_lines[0].strip() + anchor = old_lines[0].strip() or next((l.strip() for l in old_lines if l.strip()), "") if not anchor: - candidates = [l.strip() for l in old_lines if l.strip()] - if not candidates: - return "" - anchor = candidates[0] - - scored = [] - for i, line in enumerate(content_lines): - stripped = line.strip() - if not stripped: - continue - ratio = SequenceMatcher(None, anchor, stripped).ratio() - if ratio > 0.3: - scored.append((ratio, i)) - if not scored: return "" - scored.sort(key=lambda x: -x[0]) - top = scored[:max_results] + + scored = sorted(((SequenceMatcher(None, anchor, line.strip()).ratio(), i) + for i, line in enumerate(content_lines) if line.strip()), key=lambda x: -x[0]) + top = [s for s in scored if s[0] > 0.3][:max_results] + if not top: + return "" parts = [] seen_ranges = set() @@ -376,11 +562,7 @@ def find_closest_lines(old_string: str, content: str, context_lines: int = 2, ma continue seen_ranges.add((start, end)) parts.append("\n".join( - f"{start + j + 1:4d}| {content_lines[start + j]}" - for j in range(end - start) - )) - if not parts: - return "" + f"{start + j + 1:4d}| {content_lines[start + j]}" for j in range(end - start))) result = "\n---\n".join(parts) # Whitespace-shaped miss: best line equals the anchor once stripped. Show @@ -391,23 +573,15 @@ def find_closest_lines(old_string: str, content: str, context_lines: int = 2, ma "\n\nWhitespace difference detected (→ = tab, · = space):\n" f" file has: {_visualize_whitespace(best_line)}\n" f" you sent: {_visualize_whitespace(old_lines[0])}\n" - "Use the exact whitespace shown in 'file has'." - ) + "Use the exact whitespace shown in 'file has'.") return result def format_no_match_hint(error: Optional[str], match_count: int, old_string: str, content: str) -> str: - """'\\n\\nDid you mean...' snippet for plain no-match errors only, else ''. - - Ambiguous-match, escape-drift and identical-strings errors also have - ``match_count == 0`` but a hint would mislead there. - """ - if match_count != 0: - return "" - if not error or not error.startswith("Could not find"): + """'\\n\\nDid you mean...' snippet for plain no-match errors only, else '' (ambiguous / + escape-drift / identical errors also have ``match_count == 0`` but a hint would mislead).""" + if match_count != 0 or not error or not error.startswith("Could not find"): return "" hint = find_closest_lines(old_string, content) - if not hint: - return "" - return "\n\nDid you mean one of these sections?\n" + hint + return "\n\nDid you mean one of these sections?\n" + hint if hint else "" diff --git a/tools/fuzzy_match_strategies.py b/tools/fuzzy_match_strategies.py deleted file mode 100644 index eb3e3464bd..0000000000 --- a/tools/fuzzy_match_strategies.py +++ /dev/null @@ -1,334 +0,0 @@ -"""Matching strategies for :mod:`tools.fuzzy_match`. - -Each strategy takes ``(content, pattern)`` and returns a list of -``(start, end)`` spans in the ORIGINAL content. ``STRATEGIES`` is the ordered -chain the orchestrator tries; ``SIMILARITY_STRATEGIES`` names the ones whose -matches are approximate and therefore unsafe under ``replace_all``. -""" - -import re -from difflib import SequenceMatcher -from typing import Callable, Dict, List, Tuple - -UNICODE_MAP = { - "\u201c": '"', "\u201d": '"', # smart double quotes - "\u2018": "'", "\u2019": "'", # smart single quotes - "\u2014": "--", "\u2013": "-", # em/en dashes - "\u2026": "...", "\u00a0": " ", # ellipsis and non-breaking space - "\u2212": "-", # typographic minus (math/scientific docs) - # Space-separator family (Zs) beyond NBSP: files with typographic spacing - # otherwise miss every precise strategy and fall through to the - # similarity fallback, which can pick the wrong region. - "\u2000": " ", "\u2001": " ", "\u2002": " ", "\u2003": " ", - "\u2004": " ", "\u2005": " ", "\u2006": " ", "\u2007": " ", - "\u2008": " ", "\u2009": " ", "\u200a": " ", "\u202f": " ", - "\u205f": " ", "\u3000": " ", -} - -Span = Tuple[int, int] - - -def _unicode_normalize(text: str) -> str: - """Map typographic Unicode variants to ASCII equivalents.""" - for char, repl in UNICODE_MAP.items(): - text = text.replace(char, repl) - return text - - -# ── Position helpers ───────────────────────────────────────────────────── - -def _calculate_line_positions(content_lines: List[str], start_line: int, - end_line: int, content_length: int) -> Span: - """Character span covering ``content_lines[start_line:end_line]`` (end exclusive).""" - start_pos = sum(len(line) + 1 for line in content_lines[:start_line]) - end_pos = sum(len(line) + 1 for line in content_lines[:end_line]) - 1 - return start_pos, min(content_length, end_pos) - - -def _match_transformed_lines(content: str, pattern: str, - transform: Callable[[str], str]) -> List[Span]: - """Match ``pattern`` against ``content`` after applying ``transform`` per line.""" - content_lines = content.split('\n') - norm_lines = [transform(line) for line in content_lines] - pattern_norm = '\n'.join(transform(line) for line in pattern.split('\n')) - n = pattern_norm.count('\n') + 1 - matches = [] - for i in range(len(norm_lines) - n + 1): - if '\n'.join(norm_lines[i:i + n]) == pattern_norm: - matches.append(_calculate_line_positions(content_lines, i, i + n, len(content))) - return matches - - -def _build_orig_to_norm_map(original: str) -> List[int]: - """Map each original index to its index in ``_unicode_normalize(original)``. - - UNICODE_MAP replacements can expand one char into several, so the map is - needed to translate normalised spans back. Length is ``len(original)+1``; - the last entry is a sentinel one past the final character. - """ - result: List[int] = [] - norm_pos = 0 - for char in original: - result.append(norm_pos) - repl = UNICODE_MAP.get(char) - norm_pos += len(repl) if repl is not None else 1 - result.append(norm_pos) - return result - - -def _invert_norm_map(orig_to_norm: List[int]) -> Dict[int, int]: - """norm_pos -> first original position mapping to it.""" - inverted: Dict[int, int] = {} - for orig_pos, norm_pos in enumerate(orig_to_norm[:-1]): - if norm_pos not in inverted: - inverted[norm_pos] = orig_pos - return inverted - - -def _norm_end_to_orig(orig_to_norm: List[int], orig_start: int, norm_end: int) -> int: - """Walk from ``orig_start`` until the mapped position reaches ``norm_end``.""" - orig_len = len(orig_to_norm) - 1 - orig_end = orig_start - while orig_end < orig_len and orig_to_norm[orig_end] < norm_end: - orig_end += 1 - return orig_end - - -def _map_positions_norm_to_orig(orig_to_norm: List[int], - norm_matches: List[Span]) -> List[Span]: - """Convert spans in the normalised string to original-string spans.""" - norm_to_orig_start = _invert_norm_map(orig_to_norm) - results: List[Span] = [] - for norm_start, norm_end in norm_matches: - if norm_start not in norm_to_orig_start: - continue - orig_start = norm_to_orig_start[norm_start] - results.append((orig_start, _norm_end_to_orig(orig_to_norm, orig_start, norm_end))) - return results - - -def _map_normalized_positions(original: str, normalized: str, - normalized_matches: List[Span]) -> List[Span]: - """Best-effort span mapping for ``[ \\t]+`` -> ``' '`` whitespace collapsing.""" - orig_to_norm = [] # orig_to_norm[i] = position in normalized - orig_idx = norm_idx = 0 - while orig_idx < len(original) and norm_idx < len(normalized): - if original[orig_idx] == normalized[norm_idx]: - orig_to_norm.append(norm_idx) - orig_idx += 1 - norm_idx += 1 - elif original[orig_idx] in ' \t' and normalized[norm_idx] == ' ': - # Collapsed run: advance norm_idx only once the run is consumed. - orig_to_norm.append(norm_idx) - orig_idx += 1 - if orig_idx < len(original) and original[orig_idx] not in ' \t': - norm_idx += 1 - else: - # Extra whitespace in original, or a mismatch that normalization - # should never produce — either way, pin to the current norm_idx. - orig_to_norm.append(norm_idx) - orig_idx += 1 - while orig_idx < len(original): - orig_to_norm.append(len(normalized)) - orig_idx += 1 - - norm_to_orig_start = {} - norm_to_orig_end = {} - for orig_pos, norm_pos in enumerate(orig_to_norm): - if norm_pos not in norm_to_orig_start: - norm_to_orig_start[norm_pos] = orig_pos - norm_to_orig_end[norm_pos] = orig_pos - - original_matches = [] - for norm_start, norm_end in normalized_matches: - if norm_start in norm_to_orig_start: - orig_start = norm_to_orig_start[norm_start] - else: - orig_start = min(i for i, n in enumerate(orig_to_norm) if n >= norm_start) - if norm_end - 1 in norm_to_orig_end: - orig_end = norm_to_orig_end[norm_end - 1] + 1 - else: - orig_end = orig_start + (norm_end - norm_start) - # Absorb trailing collapsed whitespace only when the normalized match - # itself ended in a space; otherwise the first whitespace after the - # match is a word boundary that must survive (#52491). - if norm_end < len(normalized) and normalized[norm_end - 1] == ' ': - while orig_end < len(original) and original[orig_end] in ' \t': - orig_end += 1 - original_matches.append((orig_start, min(orig_end, len(original)))) - return original_matches - - -# ── Strategies ─────────────────────────────────────────────────────────── - -def _strategy_exact(content: str, pattern: str) -> List[Span]: - """Strategy 1: exact, non-overlapping occurrences (str.replace semantics).""" - matches = [] - start = 0 - while True: - pos = content.find(pattern, start) - if pos == -1: - break - matches.append((pos, pos + len(pattern))) - # Advance past the whole match: overlapping spans would corrupt the - # file under replace_all (reverse-order apply on stale offsets). - start = pos + len(pattern) - return matches - - -def _strategy_line_trimmed(content: str, pattern: str) -> List[Span]: - """Strategy 2: strip each line before comparing.""" - return _match_transformed_lines(content, pattern, str.strip) - - -def _strategy_whitespace_normalized(content: str, pattern: str) -> List[Span]: - """Strategy 3: collapse runs of spaces/tabs to a single space.""" - def normalize(s): - return re.sub(r'[ \t]+', ' ', s) - - content_normalized = normalize(content) - matches_in_normalized = _strategy_exact(content_normalized, normalize(pattern)) - if not matches_in_normalized: - return [] - return _map_normalized_positions(content, content_normalized, matches_in_normalized) - - -def _strategy_indentation_flexible(content: str, pattern: str) -> List[Span]: - """Strategy 4: ignore leading indentation entirely.""" - return _match_transformed_lines(content, pattern, str.lstrip) - - -def _strategy_escape_normalized(content: str, pattern: str) -> List[Span]: - """Strategy 5: treat literal ``\\n``/``\\t``/``\\r`` in the pattern as control chars.""" - pattern_unescaped = pattern.replace('\\n', '\n').replace('\\t', '\t').replace('\\r', '\r') - if pattern_unescaped == pattern: - return [] - return _strategy_exact(content, pattern_unescaped) - - -def _strategy_trimmed_boundary(content: str, pattern: str) -> List[Span]: - """Strategy 6: strip whitespace on the first and last lines only.""" - pattern_lines = pattern.split('\n') - pattern_lines[0] = pattern_lines[0].strip() - if len(pattern_lines) > 1: - pattern_lines[-1] = pattern_lines[-1].strip() - modified_pattern = '\n'.join(pattern_lines) - n = len(pattern_lines) - - content_lines = content.split('\n') - matches = [] - for i in range(len(content_lines) - n + 1): - check_lines = content_lines[i:i + n] - check_lines[0] = check_lines[0].strip() - if n > 1: - check_lines[-1] = check_lines[-1].strip() - if '\n'.join(check_lines) == modified_pattern: - matches.append(_calculate_line_positions(content_lines, i, i + n, len(content))) - return matches - - -def _strategy_unicode_normalized(content: str, pattern: str) -> List[Span]: - """Strategy 7: exact/line-trimmed match after Unicode->ASCII normalisation of both sides.""" - norm_pattern = _unicode_normalize(pattern) - norm_content = _unicode_normalize(content) - if norm_content == content and norm_pattern == pattern: - return [] - norm_matches = _strategy_exact(norm_content, norm_pattern) - if not norm_matches: - norm_matches = _strategy_line_trimmed(norm_content, norm_pattern) - if not norm_matches: - return [] - return _map_positions_norm_to_orig(_build_orig_to_norm_map(content), norm_matches) - - -def _strategy_block_anchor(content: str, pattern: str) -> List[Span]: - """Strategy 8: anchor on first+last lines, similarity-score the middle.""" - pattern_lines = _unicode_normalize(pattern).split('\n') - if len(pattern_lines) < 2: - return [] - first_line = pattern_lines[0].strip() - last_line = pattern_lines[-1].strip() - n = len(pattern_lines) - - # Match on normalized lines; compute offsets from the ORIGINAL lines so - # multi-char expansions (em-dash -> '--') don't shift positions. - norm_content_lines = _unicode_normalize(content).split('\n') - orig_content_lines = content.split('\n') - - potential_matches = [ - i for i in range(len(norm_content_lines) - n + 1) - if norm_content_lines[i].strip() == first_line - and norm_content_lines[i + n - 1].strip() == last_line - ] - # Looser thresholds (0.10/0.30) matched unrelated blocks; these are the safe floor. - threshold = 0.50 if len(potential_matches) == 1 else 0.70 - - matches = [] - for i in potential_matches: - if n <= 2: - similarity = 1.0 - else: - content_middle = '\n'.join(norm_content_lines[i + 1:i + n - 1]) - pattern_middle = '\n'.join(pattern_lines[1:-1]) - similarity = SequenceMatcher(None, content_middle, pattern_middle).ratio() - if similarity >= threshold: - matches.append(_calculate_line_positions(orig_content_lines, i, i + n, len(content))) - return matches - - -def _strategy_context_aware(content: str, pattern: str) -> List[Span]: - """Strategy 9 (last resort): anchored per-line similarity, every non-blank line >= 0.80. - - The first/last-line anchor pre-filter keeps a miss from being an - O(file x pattern) scan; the all-lines requirement stops one coincidental - line match from replacing an unrelated block. - """ - pattern_lines = pattern.split('\n') - content_lines = content.split('\n') - n = len(pattern_lines) - if n > len(content_lines): - return [] - - first_pat = pattern_lines[0].strip() - last_pat = pattern_lines[-1].strip() - ANCHOR_THRESHOLD = 0.80 - - def _sim(a: str, b: str) -> float: - if a == b: - return 1.0 - return SequenceMatcher(None, a, b).ratio() - - matches = [] - for i in range(len(content_lines) - n + 1): - block_lines = content_lines[i:i + n] - if _sim(first_pat, block_lines[0].strip()) < ANCHOR_THRESHOLD: - continue - if _sim(last_pat, block_lines[-1].strip()) < ANCHOR_THRESHOLD: - continue - all_match = True - for p_line, c_line in zip(pattern_lines, block_lines): - p_stripped = p_line.strip() - if p_stripped and _sim(p_stripped, c_line.strip()) < 0.80: - all_match = False - break - if all_match: - matches.append(_calculate_line_positions(content_lines, i, i + n, len(content))) - return matches - - -# Ordered chain: precise strategies first, similarity-based last. -STRATEGIES: List[Tuple[str, Callable[[str, str], List[Span]]]] = [ - ("exact", _strategy_exact), - ("line_trimmed", _strategy_line_trimmed), - ("whitespace_normalized", _strategy_whitespace_normalized), - ("indentation_flexible", _strategy_indentation_flexible), - ("escape_normalized", _strategy_escape_normalized), - ("trimmed_boundary", _strategy_trimmed_boundary), - ("unicode_normalized", _strategy_unicode_normalized), - ("block_anchor", _strategy_block_anchor), - ("context_aware", _strategy_context_aware), -] - -# Matches from these only *approximately* resemble old_string — fine for one -# unique replacement, never safe under replace_all. -SIMILARITY_STRATEGIES = frozenset({"block_anchor", "context_aware"})