Merge branch 'simp/r3-32-A' into simp/r3-32
This commit is contained in:
@@ -30,7 +30,11 @@ def test_sibling_container_config_sites_carry_docker_network():
|
||||
import tools.code_execution_tool as code_execution_tool
|
||||
import tools.file_tools as file_tools
|
||||
|
||||
for module in (terminal_tool, file_tools, code_execution_tool):
|
||||
# file_tools no longer builds its own container_config: it goes through
|
||||
# the shared _create_configured_env, which is what carries docker_network.
|
||||
assert "_create_configured_env(" in inspect.getsource(file_tools)
|
||||
|
||||
for module in (terminal_tool, code_execution_tool):
|
||||
tree = ast.parse(inspect.getsource(module))
|
||||
sites = 0
|
||||
for node in ast.walk(tree):
|
||||
|
||||
+63
-118
@@ -1,24 +1,11 @@
|
||||
"""Cross-agent file state coordination.
|
||||
|
||||
Prevents mangled edits when concurrent subagents (same process, same
|
||||
filesystem) touch the same file: subagent B writes a file that subagent A
|
||||
already read, so A's next write would clobber B's changes with stale content.
|
||||
Complements the single-agent path-overlap check in
|
||||
``run_agent._should_parallelize_tool_batch``.
|
||||
|
||||
A process-wide ``FileStateRegistry`` tracks, per resolved path:
|
||||
* per-agent read stamps {task_id: {path: (mtime, read_ts, partial)}}
|
||||
* last writer globally {path: (task_id, write_ts)}
|
||||
* a per-path ``threading.Lock`` for read->modify->write sections
|
||||
|
||||
Hooks used by the file tools: ``record_read`` (read_file), ``note_write``
|
||||
(after write_file/patch), ``check_stale`` (BEFORE write_file/patch),
|
||||
``lock_path`` (wrap the whole read->modify->write block) and ``writes_since``
|
||||
(delegate_tool's subagent-completion reminder).
|
||||
|
||||
All methods are no-ops when ``HERMES_DISABLE_FILE_STATE_GUARD=1``. This is
|
||||
separate from ``file_tools._read_tracker``, which handles per-task
|
||||
consecutive-read loop detection.
|
||||
filesystem) touch the same file: B writes a file A already read, so A's next
|
||||
write would clobber B's changes. Complements the single-agent path-overlap
|
||||
check in ``run_agent._should_parallelize_tool_batch``. A process-wide
|
||||
``FileStateRegistry`` tracks per-agent read stamps, the global last writer and
|
||||
a per-path lock; every method is a no-op under ``HERMES_DISABLE_FILE_STATE_GUARD=1``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -35,12 +22,41 @@ from typing import Dict, Iterable, List, Optional, Tuple
|
||||
# so the model re-reads in full.
|
||||
ReadStamp = Tuple[float, float, bool]
|
||||
|
||||
# Bounded so long sessions don't accumulate unbounded state; oldest by
|
||||
# insertion order are dropped on overflow.
|
||||
# Bounded so long sessions don't accumulate unbounded state.
|
||||
_MAX_PATHS_PER_AGENT = 4096
|
||||
_MAX_GLOBAL_WRITERS = 4096
|
||||
|
||||
|
||||
def _disabled() -> bool:
|
||||
# Re-read each call so tests can toggle via monkeypatch.setenv.
|
||||
return os.environ.get("HERMES_DISABLE_FILE_STATE_GUARD", "").strip() == "1"
|
||||
|
||||
|
||||
def _mtime_or_none(resolved: str) -> Optional[float]:
|
||||
try:
|
||||
return os.path.getmtime(resolved)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _fmt_ts(ts: float) -> str:
|
||||
# Short wall-clock for warnings; avoids datetime formatting on the hot path.
|
||||
return time.strftime("%H:%M:%S", time.localtime(ts))
|
||||
|
||||
|
||||
def _evict_oldest(container, cap: int) -> None:
|
||||
"""Pop entries until *container* is within *cap* (sets: arbitrary; dicts: oldest
|
||||
by insertion order). An eviction only costs one redundant re-send or staleness check."""
|
||||
for _ in range(len(container) - cap):
|
||||
try:
|
||||
if isinstance(container, set):
|
||||
container.pop()
|
||||
else:
|
||||
container.pop(next(iter(container)))
|
||||
except (StopIteration, KeyError):
|
||||
break
|
||||
|
||||
|
||||
class FileStateRegistry:
|
||||
"""Process-wide coordinator for cross-agent file edits."""
|
||||
|
||||
@@ -51,50 +67,31 @@ class FileStateRegistry:
|
||||
self._meta_lock = threading.Lock() # guards _path_locks
|
||||
self._state_lock = threading.Lock() # guards _reads + _last_writer
|
||||
|
||||
def _lock_for(self, resolved: str) -> threading.Lock:
|
||||
with self._meta_lock:
|
||||
lock = self._path_locks.get(resolved)
|
||||
if lock is None:
|
||||
lock = threading.Lock()
|
||||
self._path_locks[resolved] = lock
|
||||
return lock
|
||||
|
||||
@contextmanager
|
||||
def lock_path(self, resolved: str):
|
||||
"""Per-path lock: threads on the same path serialize, different paths proceed."""
|
||||
lock = self._lock_for(resolved)
|
||||
lock.acquire()
|
||||
try:
|
||||
with self._meta_lock:
|
||||
lock = self._path_locks.setdefault(resolved, threading.Lock())
|
||||
with lock:
|
||||
yield
|
||||
finally:
|
||||
lock.release()
|
||||
|
||||
def record_read(
|
||||
self,
|
||||
task_id: str,
|
||||
resolved: str,
|
||||
*,
|
||||
partial: bool = False,
|
||||
mtime: Optional[float] = None,
|
||||
) -> None:
|
||||
def _stamp(self, task_id: str, resolved: str, mtime: float, now: float, partial: bool) -> None:
|
||||
"""Caller holds ``_state_lock``."""
|
||||
agent_reads = self._reads[task_id]
|
||||
agent_reads[resolved] = (float(mtime), now, bool(partial))
|
||||
_evict_oldest(agent_reads, _MAX_PATHS_PER_AGENT)
|
||||
|
||||
def record_read(self, task_id: str, resolved: str, *, partial: bool = False,
|
||||
mtime: Optional[float] = None) -> None:
|
||||
if _disabled():
|
||||
return
|
||||
mtime = _mtime_or_none(resolved) if mtime is None else mtime
|
||||
if mtime is None:
|
||||
return
|
||||
now = time.time()
|
||||
with self._state_lock:
|
||||
agent_reads = self._reads[task_id]
|
||||
agent_reads[resolved] = (float(mtime), now, bool(partial))
|
||||
_cap_dict(agent_reads, _MAX_PATHS_PER_AGENT)
|
||||
self._stamp(task_id, resolved, mtime, time.time(), partial)
|
||||
|
||||
def note_write(
|
||||
self,
|
||||
task_id: str,
|
||||
resolved: str,
|
||||
*,
|
||||
mtime: Optional[float] = None,
|
||||
) -> None:
|
||||
def note_write(self, task_id: str, resolved: str, *, mtime: Optional[float] = None) -> None:
|
||||
"""Record a successful write: global last-writer AND this agent's own
|
||||
read stamp (a write is an implicit read of the current content)."""
|
||||
if _disabled():
|
||||
@@ -105,27 +102,19 @@ class FileStateRegistry:
|
||||
now = time.time()
|
||||
with self._state_lock:
|
||||
self._last_writer[resolved] = (task_id, now)
|
||||
_cap_dict(self._last_writer, _MAX_GLOBAL_WRITERS)
|
||||
self._reads[task_id][resolved] = (float(mtime), now, False)
|
||||
_cap_dict(self._reads[task_id], _MAX_PATHS_PER_AGENT)
|
||||
_evict_oldest(self._last_writer, _MAX_GLOBAL_WRITERS)
|
||||
self._stamp(task_id, resolved, mtime, now, False)
|
||||
|
||||
def check_stale(self, task_id: str, resolved: str) -> Optional[str]:
|
||||
"""Model-facing warning if this write would be stale, else ``None``.
|
||||
|
||||
Checked in severity order: (1) a sibling subagent wrote after this
|
||||
agent's last read; (2) mtime drifted since our read (external edit) or
|
||||
the read was partial; (3) this agent never read the file. Never raises
|
||||
— callers decide whether to block or warn.
|
||||
"""
|
||||
"""Model-facing warning if this write would be stale, else ``None``. Severity
|
||||
order: sibling wrote after our read > mtime drift / partial read > never read."""
|
||||
if _disabled():
|
||||
return None
|
||||
with self._state_lock:
|
||||
stamp = self._reads.get(task_id, {}).get(resolved)
|
||||
last_writer = self._last_writer.get(resolved)
|
||||
|
||||
# Never read and no write record: net-new file or first touch —
|
||||
# existing sensitive-path / file-exists logic handles it.
|
||||
if stamp is None and last_writer is None:
|
||||
if stamp is None and last_writer is None: # net-new file / first touch
|
||||
return None
|
||||
current_mtime = _mtime_or_none(resolved)
|
||||
if current_mtime is None:
|
||||
@@ -139,16 +128,14 @@ class FileStateRegistry:
|
||||
f"{resolved} was modified by sibling subagent "
|
||||
f"{writer_tid!r} but this agent never read it. "
|
||||
"Read the file before writing to avoid overwriting "
|
||||
"the sibling's changes."
|
||||
)
|
||||
"the sibling's changes.")
|
||||
read_ts = stamp[1]
|
||||
if writer_ts > read_ts:
|
||||
return (
|
||||
f"{resolved} was modified by sibling subagent "
|
||||
f"{writer_tid!r} at {_fmt_ts(writer_ts)} — after "
|
||||
f"this agent's last read at {_fmt_ts(read_ts)}. "
|
||||
"Re-read the file before writing."
|
||||
)
|
||||
"Re-read the file before writing.")
|
||||
|
||||
if stamp is not None:
|
||||
read_mtime, _read_ts, partial = stamp
|
||||
@@ -156,27 +143,20 @@ class FileStateRegistry:
|
||||
return (
|
||||
f"{resolved} was modified since you last read it "
|
||||
"on disk (external edit or unrecorded writer). "
|
||||
"Re-read the file before writing."
|
||||
)
|
||||
"Re-read the file before writing.")
|
||||
if partial:
|
||||
return (
|
||||
f"{resolved} was last read with offset/limit pagination "
|
||||
"(partial view). Re-read the whole file before "
|
||||
"overwriting it."
|
||||
)
|
||||
"overwriting it.")
|
||||
return None
|
||||
|
||||
return (
|
||||
f"{resolved} was not read by this agent. "
|
||||
"Read the file first so you can write an informed edit."
|
||||
)
|
||||
"Read the file first so you can write an informed edit.")
|
||||
|
||||
def writes_since(
|
||||
self,
|
||||
exclude_task_id: str,
|
||||
since_ts: float,
|
||||
paths: Iterable[str],
|
||||
) -> Dict[str, List[str]]:
|
||||
def writes_since(self, exclude_task_id: str, since_ts: float,
|
||||
paths: Iterable[str]) -> Dict[str, List[str]]:
|
||||
"""``{writer_task_id: [paths]}`` for writes after ``since_ts`` by agents
|
||||
other than ``exclude_task_id`` (delegate_task's "subagent modified files
|
||||
you previously read" reminder)."""
|
||||
@@ -213,36 +193,6 @@ def get_registry() -> FileStateRegistry:
|
||||
return _registry
|
||||
|
||||
|
||||
def _disabled() -> bool:
|
||||
# Re-read each call so tests can toggle via monkeypatch.setenv.
|
||||
return os.environ.get("HERMES_DISABLE_FILE_STATE_GUARD", "").strip() == "1"
|
||||
|
||||
|
||||
def _mtime_or_none(resolved: str) -> Optional[float]:
|
||||
try:
|
||||
return os.path.getmtime(resolved)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _fmt_ts(ts: float) -> str:
|
||||
# Short wall-clock for warnings; avoids datetime formatting on the hot path.
|
||||
return time.strftime("%H:%M:%S", time.localtime(ts))
|
||||
|
||||
|
||||
def _cap_dict(d: dict, limit: int) -> None:
|
||||
"""Trim ``d`` to ``limit`` entries by dropping the insertion-order oldest."""
|
||||
over = len(d) - limit
|
||||
if over <= 0:
|
||||
return
|
||||
it = iter(d)
|
||||
for _ in range(over):
|
||||
try:
|
||||
d.pop(next(it))
|
||||
except (StopIteration, KeyError):
|
||||
break
|
||||
|
||||
|
||||
# Convenience wrappers (short names used at call sites).
|
||||
def record_read(task_id: str, resolved_or_path: str | Path, *, partial: bool = False) -> None:
|
||||
_registry.record_read(task_id, str(resolved_or_path), partial=partial)
|
||||
@@ -260,11 +210,7 @@ def lock_path(resolved_or_path: str | Path):
|
||||
return _registry.lock_path(str(resolved_or_path))
|
||||
|
||||
|
||||
def writes_since(
|
||||
exclude_task_id: str,
|
||||
since_ts: float,
|
||||
paths: Iterable[str | Path],
|
||||
) -> Dict[str, List[str]]:
|
||||
def writes_since(exclude_task_id: str, since_ts: float, paths: Iterable[str | Path]) -> Dict[str, List[str]]:
|
||||
return _registry.writes_since(exclude_task_id, since_ts, [str(p) for p in paths])
|
||||
|
||||
|
||||
@@ -280,5 +226,4 @@ __all__ = [
|
||||
"check_stale",
|
||||
"lock_path",
|
||||
"writes_since",
|
||||
"known_reads",
|
||||
]
|
||||
"known_reads"]
|
||||
|
||||
+262
-553
File diff suppressed because it is too large
Load Diff
+72
-140
@@ -1,11 +1,9 @@
|
||||
"""Path resolution for the file tools: task-aware base dir, ``~`` expansion, workspace-divergence warning.
|
||||
|
||||
Companion to ``tools.file_tools`` (which re-imports every name here). The core
|
||||
invariant: the base directory used to anchor relative paths is ALWAYS absolute
|
||||
Core invariant: the base directory anchoring relative paths is ALWAYS absolute
|
||||
and derived from the task's terminal cwd, never from the process cwd unless no
|
||||
other anchor exists. A relative or sentinel ``TERMINAL_CWD`` would otherwise
|
||||
silently anchor edits to the agent process cwd (e.g. the main repo while a
|
||||
worktree session is active).
|
||||
other anchor exists (a relative/sentinel ``TERMINAL_CWD`` would silently anchor
|
||||
edits to the agent process cwd, e.g. the main repo during a worktree session).
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -13,9 +11,8 @@ import posixpath
|
||||
import sys
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
# ``TERMINAL_CWD`` values that mean "not configured", not a directory to resolve
|
||||
# against ("." from a stale config; "auto"/"cwd" are setup-wizard placeholders).
|
||||
# The gateway sanitizes the same set at import time (gateway/run.py).
|
||||
# ``TERMINAL_CWD`` values that mean "not configured" ("." from a stale config;
|
||||
# "auto"/"cwd" are wizard placeholders). gateway/run.py sanitizes the same set.
|
||||
_TERMINAL_CWD_SENTINELS = frozenset({"", ".", "./", "auto", "cwd"})
|
||||
_CONTAINER_PATH_BACKENDS_FALLBACK = frozenset({"docker", "singularity", "modal", "daytona", "vercel_sandbox"})
|
||||
# Backend name inferred from the live environment's class name (first match wins).
|
||||
@@ -23,13 +20,8 @@ _ENV_CLASS_NAME_HINTS = ("local", "ssh", "docker", "singularity", "modal", "dayt
|
||||
|
||||
|
||||
def _expand_tilde(path: str) -> str:
|
||||
"""Expand ``~`` using the effective profile home when available.
|
||||
|
||||
In-process file tools share the gateway process's HOME, which may differ
|
||||
from the profile-specific HOME interactive CLI sessions use; mirroring
|
||||
``hermes_constants.get_subprocess_home()`` keeps ``~`` consistent across
|
||||
interactive and gateway-driven (cron) runs.
|
||||
"""
|
||||
"""Expand ``~`` using the effective profile home (``get_subprocess_home``) so
|
||||
gateway/cron runs, whose process HOME may differ, agree with interactive CLI sessions."""
|
||||
if not path or "~" not in path:
|
||||
return path
|
||||
try:
|
||||
@@ -47,11 +39,7 @@ def _terminal_env_type_for_task(task_id: str = "default") -> str:
|
||||
"""Best-effort terminal backend type for path-resolution decisions."""
|
||||
try:
|
||||
from tools.terminal_tool import (
|
||||
_active_environments,
|
||||
_env_lock,
|
||||
_get_env_config,
|
||||
_resolve_container_task_id,
|
||||
)
|
||||
_active_environments, _env_lock, _get_env_config, _resolve_container_task_id)
|
||||
|
||||
try:
|
||||
container_key = _resolve_container_task_id(task_id)
|
||||
@@ -61,14 +49,11 @@ def _terminal_env_type_for_task(task_id: str = "default") -> str:
|
||||
env = _active_environments.get(container_key) or _active_environments.get(task_id)
|
||||
if env is not None:
|
||||
name = env.__class__.__name__.lower()
|
||||
for hint in _ENV_CLASS_NAME_HINTS:
|
||||
if hint in name:
|
||||
return hint
|
||||
hint = next((h for h in _ENV_CLASS_NAME_HINTS if h in name), None)
|
||||
stamped = getattr(env, "_hermes_backend_name", None)
|
||||
if isinstance(stamped, str) and stamped:
|
||||
return stamped
|
||||
cfg = _get_env_config()
|
||||
return str(cfg.get("env_type") or os.getenv("TERMINAL_ENV") or "local").lower()
|
||||
if hint or (isinstance(stamped, str) and stamped):
|
||||
return hint or stamped
|
||||
return str(_get_env_config().get("env_type") or os.getenv("TERMINAL_ENV") or "local").lower()
|
||||
except Exception:
|
||||
return str(os.getenv("TERMINAL_ENV") or "local").lower()
|
||||
|
||||
@@ -84,36 +69,24 @@ def _uses_container_paths(task_id: str = "default") -> bool:
|
||||
|
||||
|
||||
def _normalize_without_host_deref(path: str | Path | PurePosixPath) -> PurePosixPath:
|
||||
"""Normalize path syntax without following host symlinks.
|
||||
|
||||
Container paths are meaningful inside the sandbox; ``Path.resolve()`` on the
|
||||
host could dereference a host-side symlink (e.g. ``/workspace``) and rewrite
|
||||
the path before Docker sees it.
|
||||
"""
|
||||
"""Normalize path syntax without following host symlinks: container paths are
|
||||
meaningful inside the sandbox, and a host-side ``/workspace`` symlink must not rewrite them."""
|
||||
return PurePosixPath(posixpath.normpath(str(path)))
|
||||
|
||||
|
||||
def _sentinel_free_abs_cwd(raw: str | None) -> str | None:
|
||||
"""Return *raw* expanded when it is a non-sentinel ABSOLUTE anchor, else ``None``.
|
||||
|
||||
A relative anchor is meaningless without knowing which cwd it is relative
|
||||
to — exactly the ambiguity that misroutes worktree edits.
|
||||
"""
|
||||
"""Return *raw* expanded when it is a non-sentinel ABSOLUTE anchor, else ``None``
|
||||
(a relative anchor is exactly the ambiguity that misroutes worktree edits)."""
|
||||
raw = str(raw or "").strip()
|
||||
if raw.lower() in _TERMINAL_CWD_SENTINELS:
|
||||
return None
|
||||
expanded = _expand_tilde(raw)
|
||||
if not os.path.isabs(expanded):
|
||||
return None
|
||||
return expanded
|
||||
return expanded if os.path.isabs(expanded) else None
|
||||
|
||||
|
||||
def _configured_terminal_cwd() -> str | None:
|
||||
"""Return ``$TERMINAL_CWD`` only when it names a real (absolute, non-sentinel) anchor.
|
||||
|
||||
Scope-aware: under gateway multiplexing the routed profile's cwd lives in
|
||||
the per-turn terminal scope, not the process env.
|
||||
"""
|
||||
Scope-aware: under gateway multiplexing the routed profile's cwd lives in the per-turn scope."""
|
||||
from agent.runtime_cwd import scope_terminal_cwd
|
||||
|
||||
return _sentinel_free_abs_cwd(scope_terminal_cwd() or None)
|
||||
@@ -122,10 +95,8 @@ def _configured_terminal_cwd() -> str | None:
|
||||
def _registered_task_cwd_override(task_id: str = "default") -> str | None:
|
||||
"""Return a registered cwd override keyed by the RAW task id, when available.
|
||||
|
||||
``terminal_tool`` collapses CWD-only task overrides to the shared
|
||||
``"default"`` environment (TUI/dashboard/ACP sessions share one sandbox),
|
||||
but the cwd value itself stays keyed by the raw session id — so read the
|
||||
raw override before falling back to the collapsed container key.
|
||||
``terminal_tool`` collapses CWD-only overrides to the shared ``"default"``
|
||||
env, but the cwd value stays keyed by the raw session id.
|
||||
"""
|
||||
try:
|
||||
from tools.terminal_tool import resolve_task_overrides
|
||||
@@ -140,10 +111,9 @@ def _registered_task_cwd_override(task_id: str = "default") -> str | None:
|
||||
def _authoritative_workspace_root(task_id: str = "default") -> str | None:
|
||||
"""Best-effort absolute workspace root, or ``None`` when no reliable anchor exists.
|
||||
|
||||
Order: (1) the session's own cwd record (written on every completed terminal
|
||||
command; per-session, so one session's ``cd`` never leaks into another);
|
||||
(2) a registered raw-keyed task/session cwd override (TUI/Desktop/ACP);
|
||||
(3) a sentinel-free absolute ``$TERMINAL_CWD`` (``-w`` sessions).
|
||||
Order: (1) the session's own cwd record (per-session, so one session's
|
||||
``cd`` never leaks into another); (2) a registered raw-keyed cwd override
|
||||
(TUI/Desktop/ACP); (3) a sentinel-free absolute ``$TERMINAL_CWD``.
|
||||
"""
|
||||
try:
|
||||
from tools.terminal_tool import get_session_cwd
|
||||
@@ -151,86 +121,55 @@ def _authoritative_workspace_root(task_id: str = "default") -> str | None:
|
||||
recorded = get_session_cwd(task_id)
|
||||
except Exception:
|
||||
recorded = None
|
||||
if recorded:
|
||||
return recorded
|
||||
registered = _registered_task_cwd_override(task_id)
|
||||
if registered:
|
||||
return registered
|
||||
return _configured_terminal_cwd()
|
||||
return recorded or _registered_task_cwd_override(task_id) or _configured_terminal_cwd()
|
||||
|
||||
|
||||
def _host_text(text: str, container_paths: bool) -> str:
|
||||
"""Expand ``~``; on host backends also translate Git Bash ``/c/Users/...`` drive
|
||||
paths before Path sees them. Container/WSL Linux paths are never rewritten."""
|
||||
if not container_paths:
|
||||
from tools.environments.local import _msys_to_windows_path
|
||||
|
||||
text = _msys_to_windows_path(text)
|
||||
return _expand_tilde(text)
|
||||
|
||||
|
||||
def _anchor(text: str, base, container_paths: bool) -> Path | PurePosixPath:
|
||||
"""Return *text* as an absolute, normalized path, joining it onto ``base()`` when
|
||||
relative. Container: pure-posix, no host deref. Host: resolve() (win32: ntpath normpath)."""
|
||||
if container_paths:
|
||||
if not posixpath.isabs(text):
|
||||
text = posixpath.join(str(base()), text)
|
||||
return _normalize_without_host_deref(text)
|
||||
if sys.platform == "win32":
|
||||
import ntpath
|
||||
|
||||
if not ntpath.isabs(text):
|
||||
text = ntpath.join(str(base()), text)
|
||||
return Path(ntpath.normpath(text))
|
||||
p = Path(text)
|
||||
if not p.is_absolute():
|
||||
p = Path(base()) / p
|
||||
return p.resolve()
|
||||
|
||||
|
||||
def _resolve_base_dir(
|
||||
task_id: str = "default",
|
||||
*,
|
||||
container_paths: bool | None = None,
|
||||
) -> Path | PurePosixPath:
|
||||
"""Return the ABSOLUTE base directory for resolving relative paths.
|
||||
|
||||
Uses ``_authoritative_workspace_root`` (live cwd → registered override →
|
||||
``$TERMINAL_CWD``), falling back to the process cwd only as a last resort.
|
||||
Sentinel/relative ``TERMINAL_CWD`` values are rejected outright rather than
|
||||
anchored to the process cwd, so the result never depends on where the
|
||||
agent process happens to run.
|
||||
"""
|
||||
task_id: str = "default", *, container_paths: bool | None = None) -> Path | PurePosixPath:
|
||||
"""Return the ABSOLUTE base directory for resolving relative paths:
|
||||
``_authoritative_workspace_root``, else the process cwd as a last resort."""
|
||||
root = _authoritative_workspace_root(task_id)
|
||||
if container_paths is None:
|
||||
container_paths = _uses_container_paths(task_id)
|
||||
base_text = _expand_tilde(root) if root else os.getcwd()
|
||||
if container_paths:
|
||||
if not posixpath.isabs(base_text):
|
||||
base_text = posixpath.join(os.getcwd(), base_text)
|
||||
return _normalize_without_host_deref(base_text)
|
||||
# Git Bash ``pwd -P`` reports ``/c/Users/...``; translate before Path so
|
||||
# relative file-tool paths don't anchor under a nonexistent ``\\c\\Users``.
|
||||
from tools.environments.local import _msys_to_windows_path
|
||||
|
||||
base_text = _msys_to_windows_path(base_text)
|
||||
if sys.platform == "win32":
|
||||
import ntpath
|
||||
|
||||
if not ntpath.isabs(base_text):
|
||||
base_text = ntpath.join(os.getcwd(), base_text)
|
||||
return Path(ntpath.normpath(base_text))
|
||||
base = Path(base_text)
|
||||
if not base.is_absolute():
|
||||
# A backend reporting a relative cwd is anchored to the process cwd
|
||||
# once, here, so the result no longer depends on cwd at resolve().
|
||||
base = Path(os.getcwd()) / base
|
||||
return base.resolve()
|
||||
# A backend's relative cwd is anchored to the process cwd once, here.
|
||||
return _anchor(_host_text(root or os.getcwd(), container_paths), os.getcwd, container_paths)
|
||||
|
||||
|
||||
def _resolve_path_for_task(filepath: str, task_id: str = "default") -> Path | PurePosixPath:
|
||||
"""Resolve *filepath* against the task's absolute base directory.
|
||||
|
||||
Absolute inputs are returned resolved-but-unanchored. On native Windows,
|
||||
Git Bash / MSYS drive paths (``/c/Users/...``) are translated first so
|
||||
they aren't treated as relative ``\\c\\Users\\...`` under the process cwd;
|
||||
container/WSL Linux paths are never rewritten.
|
||||
"""
|
||||
"""Resolve *filepath* against the task's absolute base directory
|
||||
(absolute inputs are returned resolved-but-unanchored)."""
|
||||
container_paths = _uses_container_paths(task_id)
|
||||
if container_paths:
|
||||
expanded = _expand_tilde(filepath)
|
||||
if posixpath.isabs(expanded):
|
||||
return _normalize_without_host_deref(expanded)
|
||||
resolved = _resolve_base_dir(task_id, container_paths=True) / expanded
|
||||
return _normalize_without_host_deref(resolved)
|
||||
|
||||
from tools.environments.local import _msys_to_windows_path
|
||||
|
||||
expanded = _expand_tilde(_msys_to_windows_path(filepath))
|
||||
if sys.platform == "win32":
|
||||
import ntpath
|
||||
|
||||
if ntpath.isabs(expanded):
|
||||
return Path(ntpath.normpath(expanded))
|
||||
joined = ntpath.join(str(_resolve_base_dir(task_id, container_paths=False)), expanded)
|
||||
return Path(ntpath.normpath(joined))
|
||||
|
||||
p = Path(expanded)
|
||||
if p.is_absolute():
|
||||
return p.resolve()
|
||||
resolved = _resolve_base_dir(task_id, container_paths=False) / p
|
||||
return resolved.resolve()
|
||||
return _anchor(_host_text(filepath, container_paths),
|
||||
lambda: _resolve_base_dir(task_id, container_paths=container_paths), container_paths)
|
||||
|
||||
|
||||
# Back-compat alias (imported by agent.context_references and tests).
|
||||
@@ -238,13 +177,9 @@ _resolve_path = _resolve_path_for_task
|
||||
|
||||
|
||||
def _path_resolution_warning(filepath: str, resolved: Path, task_id: str = "default") -> str | None:
|
||||
"""Warn when a RELATIVE path resolved OUTSIDE the task's workspace root.
|
||||
|
||||
Surfaces the worktree-cwd divergence the moment it matters — the edit is
|
||||
about to land in a different checkout than the terminal's cwd. ``None`` for
|
||||
absolute paths, an unknown root, or a path correctly under the root. Fires
|
||||
on the very first write even before any ``cd`` populated the cwd registry.
|
||||
"""
|
||||
"""Warn when a RELATIVE path resolved OUTSIDE the task's workspace root (the
|
||||
edit is about to land in a different checkout than the terminal's cwd).
|
||||
``None`` for absolute paths, an unknown root, or a path under the root."""
|
||||
try:
|
||||
if Path(_expand_tilde(filepath)).is_absolute():
|
||||
return None
|
||||
@@ -255,16 +190,13 @@ def _path_resolution_warning(filepath: str, resolved: Path, task_id: str = "defa
|
||||
root = _normalize_without_host_deref(Path(_expand_tilde(workspace_root)))
|
||||
else:
|
||||
root = Path(_expand_tilde(workspace_root)).resolve()
|
||||
try:
|
||||
resolved.relative_to(root)
|
||||
if resolved.is_relative_to(root):
|
||||
return None
|
||||
except ValueError:
|
||||
return (
|
||||
f"Relative path {filepath!r} resolved to {str(resolved)!r}, which is "
|
||||
f"OUTSIDE the active workspace ({str(root)!r}). The edit will land in "
|
||||
f"a different directory than the terminal's cwd. If this is not "
|
||||
f"intended (e.g. a git-worktree session writing into the main "
|
||||
f"checkout), pass an absolute path under the workspace instead."
|
||||
)
|
||||
return (
|
||||
f"Relative path {filepath!r} resolved to {str(resolved)!r}, which is "
|
||||
f"OUTSIDE the active workspace ({str(root)!r}). The edit will land in "
|
||||
f"a different directory than the terminal's cwd. If this is not "
|
||||
f"intended (e.g. a git-worktree session writing into the main "
|
||||
f"checkout), pass an absolute path under the workspace instead.")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@@ -1,23 +1,12 @@
|
||||
"""Per-task read/search bookkeeping for the file tools.
|
||||
|
||||
Process-lifetime state behind ``read_file`` / ``search_files`` / ``write_file`` /
|
||||
``patch``; ``tools.file_tools`` re-imports every name here. Per task_id
|
||||
``_read_tracker`` stores:
|
||||
|
||||
last_key / consecutive most recent read-or-search key and its repeat count
|
||||
(loop detection; reset by any OTHER tool call).
|
||||
read_history set of (path, offset, limit) — diagnostic summaries only.
|
||||
dedup (resolved_path, offset, limit) -> mtime; skip identical
|
||||
re-reads of unchanged files. Cleared on context
|
||||
compression (the original content was summarised away).
|
||||
dedup_hits per-key count of stub returns, to break stub loops.
|
||||
read_timestamps resolved_path -> mtime at last read/write by this task;
|
||||
write/patch warn when the file changed underneath.
|
||||
not_found (op, resolved_path) -> (monotonic, cached error JSON);
|
||||
short-TTL negative cache for retried missing paths.
|
||||
|
||||
Every container is hard-capped (``_cap_read_tracker_data``) so a long CLI
|
||||
session accretes a few hundred KB at most instead of ~1.5MB per 10k reads.
|
||||
Process-lifetime state behind read_file/search_files/write_file/patch;
|
||||
``tools.file_tools`` re-imports every name here. Per task_id ``_read_tracker``
|
||||
stores: ``last_key``/``consecutive`` (loop detection; reset by any OTHER tool
|
||||
call), ``read_history`` (diagnostics), ``dedup`` (key -> mtime; cleared on
|
||||
context compression), ``dedup_hits`` (stub-loop breaker), ``read_timestamps``
|
||||
(staleness warnings) and ``not_found`` (short-TTL negative cache). Every
|
||||
container is hard-capped (``_cap_read_tracker_data``) so long sessions stay small.
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -25,6 +14,7 @@ import os
|
||||
import threading
|
||||
import time
|
||||
|
||||
from tools.file_state import _evict_oldest
|
||||
from tools.file_tools_paths import _authoritative_workspace_root, _resolve_path_for_task
|
||||
|
||||
logger = logging.getLogger("tools.file_tools")
|
||||
@@ -33,8 +23,7 @@ _read_tracker_lock = threading.Lock()
|
||||
_read_tracker: dict = {}
|
||||
|
||||
# Consecutive patch failures per (task_id, resolved_path); escalates the hint
|
||||
# when the model keeps failing the same file (stale view, ambiguous old_string).
|
||||
# Reset on a successful patch to that path.
|
||||
# when the model keeps failing the same file. Reset on a successful patch.
|
||||
_patch_failure_lock = threading.Lock()
|
||||
_patch_failure_tracker: dict = {} # {task_id: {resolved_path: count}}
|
||||
_PATCH_FAILURE_PATHS_CAP = 64
|
||||
@@ -48,24 +37,13 @@ _NOT_FOUND_CAP = 500
|
||||
_NOT_FOUND_TTL_SECONDS = 60.0 # a path that didn't exist may be created soon
|
||||
|
||||
|
||||
def _new_task_data() -> dict:
|
||||
return {
|
||||
"last_key": None, "consecutive": 0,
|
||||
"read_history": set(), "dedup": {},
|
||||
"dedup_hits": {}, "read_timestamps": {},
|
||||
}
|
||||
|
||||
|
||||
def _task_data(task_id: str) -> dict:
|
||||
"""Get-or-create the tracker entry for *task_id*, back-filling any missing keys.
|
||||
|
||||
Must be called with ``_read_tracker_lock`` held. Entries created by older
|
||||
code paths (or injected by tests) may lack the newer containers.
|
||||
"""
|
||||
task_data = _read_tracker.setdefault(task_id, _new_task_data())
|
||||
for key, factory in (("dedup", dict), ("dedup_hits", dict), ("read_timestamps", dict)):
|
||||
if key not in task_data:
|
||||
task_data[key] = factory()
|
||||
"""Get-or-create the tracker entry for *task_id*, back-filling missing containers
|
||||
(search_tool / tests create partial entries). Lock must be held."""
|
||||
task_data = _read_tracker.setdefault(task_id, {
|
||||
"last_key": None, "consecutive": 0, "read_history": set()})
|
||||
for key in ("dedup", "dedup_hits", "read_timestamps"):
|
||||
task_data.setdefault(key, {})
|
||||
return task_data
|
||||
|
||||
|
||||
@@ -74,11 +52,8 @@ def _record_patch_failure(task_id: str, resolved_path: str) -> int:
|
||||
with _patch_failure_lock:
|
||||
task_failures = _patch_failure_tracker.setdefault(task_id, {})
|
||||
# Evict the oldest entry once a task has failed on many distinct files.
|
||||
if len(task_failures) >= _PATCH_FAILURE_PATHS_CAP and resolved_path not in task_failures:
|
||||
try:
|
||||
del task_failures[next(iter(task_failures))]
|
||||
except StopIteration:
|
||||
pass
|
||||
if resolved_path not in task_failures:
|
||||
_evict_oldest(task_failures, _PATCH_FAILURE_PATHS_CAP - 1)
|
||||
task_failures[resolved_path] = task_failures.get(resolved_path, 0) + 1
|
||||
return task_failures[resolved_path]
|
||||
|
||||
@@ -89,29 +64,10 @@ def _reset_patch_failures(task_id: str, resolved_paths: list) -> None:
|
||||
return
|
||||
with _patch_failure_lock:
|
||||
task_failures = _patch_failure_tracker.get(task_id)
|
||||
if not task_failures:
|
||||
return
|
||||
for rp in resolved_paths:
|
||||
for rp in resolved_paths if task_failures else ():
|
||||
task_failures.pop(rp, None)
|
||||
|
||||
|
||||
def _evict_oldest(container, cap: int) -> None:
|
||||
"""Pop entries until *container* is within *cap*.
|
||||
|
||||
Sets pop arbitrary entries (they only feed diagnostic summaries); dicts pop
|
||||
oldest by insertion order. An evicted entry costs one redundant re-send
|
||||
(dedup) or one non-mtime staleness check — graceful degradation, not a bug.
|
||||
"""
|
||||
for _ in range(len(container) - cap):
|
||||
try:
|
||||
if isinstance(container, set):
|
||||
container.pop()
|
||||
else:
|
||||
container.pop(next(iter(container)))
|
||||
except (StopIteration, KeyError):
|
||||
break
|
||||
|
||||
|
||||
def _cap_read_tracker_data(task_data: dict) -> None:
|
||||
"""Enforce size caps on the per-task sub-containers. Call with ``_read_tracker_lock`` held."""
|
||||
# Caps are read at call time so tests can monkeypatch the module constants.
|
||||
@@ -120,44 +76,48 @@ def _cap_read_tracker_data(task_data: dict) -> None:
|
||||
("dedup", _DEDUP_CAP),
|
||||
("dedup_hits", _DEDUP_CAP),
|
||||
("read_timestamps", _READ_TIMESTAMPS_CAP),
|
||||
("not_found", _NOT_FOUND_CAP),
|
||||
):
|
||||
("not_found", _NOT_FOUND_CAP)):
|
||||
container = task_data.get(key)
|
||||
if container is not None and len(container) > cap:
|
||||
_evict_oldest(container, cap)
|
||||
|
||||
|
||||
def _resolved_or_none(filepath: str, task_id: str) -> str | None:
|
||||
try:
|
||||
return str(_resolve_path_for_task(filepath, task_id))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _pop_not_found(op: str, resolved_str: str, task_id: str) -> None:
|
||||
"""Drop the negative-cache entry for *(op, resolved_str)*. Lock must be held."""
|
||||
task_data = _read_tracker.get(task_id)
|
||||
nf = task_data.get("not_found") if task_data else None
|
||||
if nf:
|
||||
nf.pop((op, resolved_str), None)
|
||||
|
||||
|
||||
def _check_not_found_cache(op: str, resolved_str: str, task_id: str) -> str | None:
|
||||
"""Return cached not-found JSON for *(op, resolved_str)* if still fresh.
|
||||
|
||||
Skips the subprocess + similar-name walk when the model retries the same
|
||||
missing path. *op* is "read" or "search" (different error JSON shapes).
|
||||
Evicted by TTL, by write_file/patch on the path, or by any other tool call.
|
||||
*op* is "read" or "search" (different error JSON shapes). Evicted by TTL,
|
||||
by write_file/patch on the path, or by any other tool call.
|
||||
"""
|
||||
with _read_tracker_lock:
|
||||
task_data = _read_tracker.get(task_id)
|
||||
if not task_data:
|
||||
return None
|
||||
nf = task_data.get("not_found")
|
||||
if not nf:
|
||||
return None
|
||||
entry = nf.get((op, resolved_str))
|
||||
entry = (task_data.get("not_found") or {}).get((op, resolved_str)) if task_data else None
|
||||
if entry is None:
|
||||
return None
|
||||
ts, cached_json = entry
|
||||
if time.monotonic() - ts > _NOT_FOUND_TTL_SECONDS:
|
||||
nf.pop((op, resolved_str), None)
|
||||
_pop_not_found(op, resolved_str, task_id)
|
||||
return None
|
||||
# The path may have been created since the miss was cached (terminal,
|
||||
# another agent, ...) — the "check → create → read" pattern is common, so
|
||||
# serving a stale miss breaks it. The stat runs OUTSIDE the global tracker
|
||||
# lock: a hung stat on a dead network mount must not stall every task.
|
||||
# "check → create → read" is common, so never serve a stale miss for a path
|
||||
# that now exists. The stat runs OUTSIDE the tracker lock: a hung stat on a
|
||||
# dead network mount must not stall every task.
|
||||
if os.path.exists(resolved_str):
|
||||
with _read_tracker_lock:
|
||||
task_data = _read_tracker.get(task_id)
|
||||
nf = task_data.get("not_found") if task_data else None
|
||||
if nf:
|
||||
nf.pop((op, resolved_str), None)
|
||||
_pop_not_found(op, resolved_str, task_id)
|
||||
return None
|
||||
return cached_json
|
||||
|
||||
@@ -166,8 +126,7 @@ def _record_not_found(op: str, resolved_str: str, task_id: str, error_json: str)
|
||||
"""Cache a not-found error so the next *op* call for *resolved_str* skips I/O."""
|
||||
with _read_tracker_lock:
|
||||
task_data = _task_data(task_id)
|
||||
nf = task_data.setdefault("not_found", {})
|
||||
nf[(op, resolved_str)] = (time.monotonic(), error_json)
|
||||
task_data.setdefault("not_found", {})[(op, resolved_str)] = (time.monotonic(), error_json)
|
||||
_cap_read_tracker_data(task_data)
|
||||
|
||||
|
||||
@@ -182,11 +141,8 @@ def _bump_consecutive(task_data: dict, key: tuple) -> int:
|
||||
|
||||
|
||||
def reset_file_dedup(task_id: str = None):
|
||||
"""Clear the read-dedup cache (one task, or all when ``task_id`` is None).
|
||||
|
||||
Called after context compression: the original read content was summarised
|
||||
away, so a "file unchanged" stub would point at content no longer in context.
|
||||
"""
|
||||
"""Clear the read-dedup cache (one task, or all when ``task_id`` is None). Called
|
||||
after context compression: a "file unchanged" stub would point at summarised-away content."""
|
||||
with _read_tracker_lock:
|
||||
if task_id:
|
||||
targets = [_read_tracker[task_id]] if _read_tracker.get(task_id) else []
|
||||
@@ -201,33 +157,25 @@ def reset_file_dedup(task_id: str = None):
|
||||
def notify_other_tool_call(task_id: str = "default"):
|
||||
"""Reset the consecutive read/search counter for a task.
|
||||
|
||||
Called by the dispatcher for every tool OTHER than read_file/search_files,
|
||||
so loop detection only fires on truly consecutive repeats. Also clears the
|
||||
stub-hit counters and the not-found cache: any other tool may have created
|
||||
a previously-missing path (the serve-side stat covers most cases; clearing
|
||||
covers the rest, e.g. permission flips).
|
||||
Called by the dispatcher for every tool OTHER than read_file/search_files.
|
||||
Also clears stub-hit counters and the not-found cache: any other tool may
|
||||
have created a previously-missing path (or flipped its permissions).
|
||||
"""
|
||||
with _read_tracker_lock:
|
||||
task_data = _read_tracker.get(task_id)
|
||||
if task_data:
|
||||
task_data["last_key"] = None
|
||||
task_data["consecutive"] = 0
|
||||
if "dedup_hits" in task_data:
|
||||
task_data["dedup_hits"].clear()
|
||||
nf = task_data.get("not_found")
|
||||
if nf:
|
||||
nf.clear()
|
||||
for key in ("dedup_hits", "not_found"):
|
||||
if task_data.get(key):
|
||||
task_data[key].clear()
|
||||
|
||||
|
||||
def _invalidate_dedup_for_path(filepath: str, task_id: str) -> None:
|
||||
"""Evict every dedup entry (all offset/limit ranges) and not-found entry for *filepath*.
|
||||
|
||||
Called after write_file/patch so the next read returns fresh content
|
||||
instead of a stale "unchanged" stub. Acquires ``_read_tracker_lock`` itself.
|
||||
"""
|
||||
try:
|
||||
resolved = str(_resolve_path_for_task(filepath, task_id))
|
||||
except (OSError, ValueError):
|
||||
"""Evict every dedup entry (all offset/limit ranges) and not-found entry for *filepath*
|
||||
after a write, so the next read returns fresh content. Acquires the lock itself."""
|
||||
resolved = _resolved_or_none(filepath, task_id)
|
||||
if resolved is None:
|
||||
return
|
||||
with _read_tracker_lock:
|
||||
task_data = _read_tracker.get(task_id)
|
||||
@@ -237,20 +185,20 @@ def _invalidate_dedup_for_path(filepath: str, task_id: str) -> None:
|
||||
if dedup:
|
||||
for k in [k for k in dedup if k[0] == resolved]:
|
||||
del dedup[k]
|
||||
nf = task_data.get("not_found")
|
||||
if nf:
|
||||
nf.pop(("read", resolved), None)
|
||||
nf.pop(("search", resolved), None)
|
||||
_pop_not_found("read", resolved, task_id)
|
||||
_pop_not_found("search", resolved, task_id)
|
||||
|
||||
|
||||
def _update_read_timestamp(filepath: str, task_id: str) -> None:
|
||||
"""After a successful write: invalidate dedup and refresh the stored mtime so
|
||||
consecutive edits by the same task don't trigger false staleness warnings."""
|
||||
_invalidate_dedup_for_path(filepath, task_id)
|
||||
resolved = _resolved_or_none(filepath, task_id)
|
||||
if resolved is None:
|
||||
return
|
||||
try:
|
||||
resolved = str(_resolve_path_for_task(filepath, task_id))
|
||||
current_mtime = os.path.getmtime(resolved)
|
||||
except (OSError, ValueError):
|
||||
except OSError:
|
||||
return
|
||||
with _read_tracker_lock:
|
||||
task_data = _read_tracker.get(task_id)
|
||||
@@ -261,19 +209,13 @@ def _update_read_timestamp(filepath: str, task_id: str) -> None:
|
||||
|
||||
def _check_file_staleness(filepath: str, task_id: str) -> str | None:
|
||||
"""Warn (don't block) when the file's mtime changed since this task last read it.
|
||||
|
||||
``None`` when never read, fresh, or unstattable (a deleted file is the
|
||||
write's problem to report).
|
||||
"""
|
||||
try:
|
||||
resolved = str(_resolve_path_for_task(filepath, task_id))
|
||||
except (OSError, ValueError):
|
||||
``None`` when never read, fresh, or unstattable (a deleted file is the write's problem)."""
|
||||
resolved = _resolved_or_none(filepath, task_id)
|
||||
if resolved is None:
|
||||
return None
|
||||
with _read_tracker_lock:
|
||||
task_data = _read_tracker.get(task_id)
|
||||
if not task_data:
|
||||
return None
|
||||
read_mtime = task_data.get("read_timestamps", {}).get(resolved)
|
||||
read_mtime = task_data.get("read_timestamps", {}).get(resolved) if task_data else None
|
||||
if read_mtime is None:
|
||||
return None
|
||||
try:
|
||||
@@ -284,21 +226,14 @@ def _check_file_staleness(filepath: str, task_id: str) -> str | None:
|
||||
return (
|
||||
f"Warning: {filepath} was modified since you last read it "
|
||||
"(external edit or concurrent agent). The content you read may be "
|
||||
"stale. Consider re-reading the file to verify before writing."
|
||||
)
|
||||
"stale. Consider re-reading the file to verify before writing.")
|
||||
return None
|
||||
|
||||
|
||||
def _mark_verification_stale(
|
||||
task_id: str,
|
||||
resolved_paths: list[str],
|
||||
session_id: str | None = None,
|
||||
) -> None:
|
||||
"""Best-effort note that successful edits made prior verification stale.
|
||||
|
||||
The workspace cwd is the first edited path's project root when one is
|
||||
recognised, else the task's workspace root, else the first path's parent.
|
||||
"""
|
||||
def _mark_verification_stale(task_id: str, resolved_paths: list[str],
|
||||
session_id: str | None = None) -> None:
|
||||
"""Best-effort note that successful edits made prior verification stale. cwd: the
|
||||
first edited path's recognised project root, else the workspace root, else the first parent."""
|
||||
from pathlib import Path
|
||||
|
||||
paths = [p for p in resolved_paths if p]
|
||||
@@ -308,22 +243,9 @@ def _mark_verification_stale(
|
||||
from agent.coding_context import project_facts_for
|
||||
from agent.verification_evidence import mark_workspace_edited
|
||||
|
||||
cwd = None
|
||||
for path in paths:
|
||||
try:
|
||||
candidate = str(Path(path).parent)
|
||||
except Exception:
|
||||
continue
|
||||
if project_facts_for(candidate):
|
||||
cwd = candidate
|
||||
break
|
||||
if cwd is None:
|
||||
cwd = _authoritative_workspace_root(task_id)
|
||||
if cwd is None:
|
||||
try:
|
||||
cwd = str(Path(paths[0]).parent)
|
||||
except Exception:
|
||||
cwd = None
|
||||
parents = [str(Path(p).parent) for p in paths]
|
||||
cwd = (next((c for c in parents if project_facts_for(c)), None)
|
||||
or _authoritative_workspace_root(task_id) or parents[0])
|
||||
mark_workspace_edited(session_id=session_id or task_id, cwd=cwd, paths=paths)
|
||||
except Exception:
|
||||
logger.debug("verification stale marker failed", exc_info=True)
|
||||
|
||||
+133
-267
@@ -2,14 +2,10 @@
|
||||
|
||||
Every guard returns ``None`` when the write may proceed, else an error string
|
||||
the tool returns verbatim. ``tools.file_tools`` re-imports every name here.
|
||||
|
||||
Guards (in the order the tools apply them):
|
||||
* ``_check_sensitive_path`` — system paths + the Hermes config.yaml (hard deny).
|
||||
* ``_check_binary_document_write`` — text write would corrupt an Office/PDF container.
|
||||
* ``_check_protected_instruction_write`` — AGENTS.md-style files: ALWAYS ask, no yolo bypass.
|
||||
* ``_check_approval_required_write`` — ~/.ssh/config-style files: normal approval gate.
|
||||
* ``_check_cross_profile_path`` — sandbox-mirror writes the host never reads (lost-work guard).
|
||||
* ``_is_internal_file_tool_content`` — refuse to persist read_file display text as a file.
|
||||
Guards, in the order the tools apply them: ``_check_sensitive_path`` (hard
|
||||
deny), ``_check_binary_document_write``, ``_check_protected_instruction_write``
|
||||
(ALWAYS ask), ``_check_approval_required_write`` (normal gate),
|
||||
``_check_cross_profile_path`` (sandbox-mirror lost-work), ``_is_internal_file_tool_content``.
|
||||
"""
|
||||
|
||||
import fnmatch
|
||||
@@ -25,29 +21,51 @@ from tools.file_tools_paths import _expand_tilde, _resolve_path_for_task
|
||||
_SENSITIVE_PATH_PREFIXES = (
|
||||
"/etc/", "/boot/", "/usr/lib/systemd/",
|
||||
"/private/etc/",
|
||||
"/private/var/db/", "/private/var/root/",
|
||||
)
|
||||
"/private/var/db/", "/private/var/root/")
|
||||
_SENSITIVE_EXACT_PATHS = {"/var/run/docker.sock", "/run/docker.sock"}
|
||||
|
||||
_hermes_config_resolved: str | None = None
|
||||
_hermes_config_resolved_loaded = False
|
||||
_real_hermes_home_cached: str | None = None
|
||||
_real_hermes_home_loaded = False
|
||||
|
||||
|
||||
def _cached_lookup(slot: str, flag: str, primary, fallback) -> str | None:
|
||||
"""Fill module global *slot* once (guarded by *flag*) from ``primary()``, else
|
||||
``fallback()``, else None. Module globals so tests can monkeypatch the slots."""
|
||||
g = globals()
|
||||
if not g[flag]:
|
||||
g[flag] = True
|
||||
try:
|
||||
g[slot] = primary()
|
||||
except Exception:
|
||||
try:
|
||||
g[slot] = fallback()
|
||||
except Exception:
|
||||
g[slot] = None
|
||||
return g[slot]
|
||||
|
||||
|
||||
def _config_path_resolved() -> str:
|
||||
from hermes_cli.config import get_config_path
|
||||
return str(get_config_path().resolve())
|
||||
|
||||
|
||||
def _hermes_home_real() -> str:
|
||||
from hermes_constants import get_hermes_home
|
||||
return os.path.realpath(str(get_hermes_home()))
|
||||
|
||||
|
||||
def _get_hermes_config_resolved() -> str | None:
|
||||
"""Return the resolved absolute path of the Hermes config file (cached)."""
|
||||
global _hermes_config_resolved, _hermes_config_resolved_loaded
|
||||
if _hermes_config_resolved_loaded:
|
||||
return _hermes_config_resolved
|
||||
_hermes_config_resolved_loaded = True
|
||||
try:
|
||||
from hermes_cli.config import get_config_path
|
||||
_hermes_config_resolved = str(get_config_path().resolve())
|
||||
except Exception:
|
||||
try:
|
||||
_hermes_config_resolved = str(Path(_expand_tilde("~/.hermes/config.yaml")).resolve())
|
||||
except Exception:
|
||||
_hermes_config_resolved = None
|
||||
return _hermes_config_resolved
|
||||
return _cached_lookup("_hermes_config_resolved", "_hermes_config_resolved_loaded", _config_path_resolved,
|
||||
lambda: str(Path(_expand_tilde("~/.hermes/config.yaml")).resolve()))
|
||||
|
||||
|
||||
def _get_real_hermes_home() -> str | None:
|
||||
"""Return the realpath of the authoritative Hermes home (cached)."""
|
||||
return _cached_lookup("_real_hermes_home_cached", "_real_hermes_home_loaded", _hermes_home_real,
|
||||
lambda: os.path.realpath(_expand_tilde("~/.hermes")))
|
||||
|
||||
|
||||
def _resolved_or_raw(filepath: str, task_id: str) -> str:
|
||||
@@ -60,79 +78,40 @@ def _resolved_or_raw(filepath: str, task_id: str) -> str:
|
||||
|
||||
def _check_sensitive_path(filepath: str, task_id: str = "default") -> str | None:
|
||||
"""Return an error message if the path targets a sensitive system location."""
|
||||
resolved = _resolved_or_raw(filepath, task_id)
|
||||
normalized = os.path.normpath(_expand_tilde(filepath))
|
||||
_err = (
|
||||
f"Refusing to write to sensitive system path: {filepath}\n"
|
||||
"Use the terminal tool with sudo if you need to modify system files."
|
||||
)
|
||||
for prefix in _SENSITIVE_PATH_PREFIXES:
|
||||
if resolved.startswith(prefix) or normalized.startswith(prefix):
|
||||
return _err
|
||||
if resolved in _SENSITIVE_EXACT_PATHS or normalized in _SENSITIVE_EXACT_PATHS:
|
||||
return _err
|
||||
candidates = (_resolved_or_raw(filepath, task_id), os.path.normpath(_expand_tilde(filepath)))
|
||||
if any(c.startswith(_SENSITIVE_PATH_PREFIXES) or c in _SENSITIVE_EXACT_PATHS for c in candidates):
|
||||
return (
|
||||
f"Refusing to write to sensitive system path: {filepath}\n"
|
||||
"Use the terminal tool with sudo if you need to modify system files.")
|
||||
# approvals.mode and other security settings live in config.yaml; a
|
||||
# prompt-injected agent could silently disable exec approval by editing it.
|
||||
hermes_config = _get_hermes_config_resolved()
|
||||
if hermes_config and (resolved == hermes_config or normalized == hermes_config):
|
||||
if hermes_config and hermes_config in candidates:
|
||||
return (
|
||||
f"Refusing to write to Hermes config file: {filepath}\n"
|
||||
"Agent cannot modify security-sensitive configuration. "
|
||||
"Edit ~/.hermes/config.yaml directly or use 'hermes config' instead."
|
||||
)
|
||||
"Edit ~/.hermes/config.yaml directly or use 'hermes config' instead.")
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Protected agent-instruction files (always-ask approval gate)
|
||||
# ---------------------------------------------------------------------------
|
||||
# ── Protected agent-instruction files (always-ask approval gate) ─────────
|
||||
# Files that steer FUTURE agent behavior are a prompt-injection persistence
|
||||
# vector: an injected edit to AGENTS.md / CLAUDE.md / SOUL.md / .cursorrules
|
||||
# (or a project-local .hermes tree) outlives the turn and poisons every later
|
||||
# session. Writes ALWAYS require human approval — even under --yolo — and fail
|
||||
# closed when no human channel exists. (Ported from Roo-Code's
|
||||
# RooProtectedController; the terminal-tool vector is gated separately.)
|
||||
#
|
||||
# Basenames match in ANY directory (instruction files load from cwd trees) and
|
||||
# case-insensitively (case-insensitive filesystems; loaders probe variants).
|
||||
# vector (AGENTS.md / CLAUDE.md / SOUL.md / .cursorrules / project .hermes tree).
|
||||
# Writes ALWAYS require human approval — even under --yolo — and fail closed
|
||||
# without a human channel. Basenames match in ANY directory, case-insensitively.
|
||||
_PROTECTED_INSTRUCTION_BASENAMES = frozenset({
|
||||
"agents.md", "claude.md", "soul.md", ".cursorrules",
|
||||
})
|
||||
|
||||
_real_hermes_home_cached: str | None = None
|
||||
_real_hermes_home_loaded = False
|
||||
|
||||
|
||||
def _get_real_hermes_home() -> str | None:
|
||||
"""Return the realpath of the authoritative Hermes home (cached)."""
|
||||
global _real_hermes_home_cached, _real_hermes_home_loaded
|
||||
if _real_hermes_home_loaded:
|
||||
return _real_hermes_home_cached
|
||||
_real_hermes_home_loaded = True
|
||||
try:
|
||||
from hermes_constants import get_hermes_home
|
||||
_real_hermes_home_cached = os.path.realpath(str(get_hermes_home()))
|
||||
except Exception:
|
||||
try:
|
||||
_real_hermes_home_cached = os.path.realpath(_expand_tilde("~/.hermes"))
|
||||
except Exception:
|
||||
_real_hermes_home_cached = None
|
||||
return _real_hermes_home_cached
|
||||
"agents.md", "claude.md", "soul.md", ".cursorrules"})
|
||||
|
||||
|
||||
def _protected_instruction_config() -> tuple[bool, list[str]]:
|
||||
"""Return ``(enabled, extra_patterns)`` from ``security.protected_instruction_files`` /
|
||||
``security.protected_instruction_extra_patterns`` (fnmatch on basename).
|
||||
|
||||
Config read failures keep the gate ON — fail-safe for a security boundary.
|
||||
"""
|
||||
``security.protected_instruction_extra_patterns`` (fnmatch on basename). Config read
|
||||
failures keep the gate ON — fail-safe for a security boundary."""
|
||||
try:
|
||||
from hermes_cli.config import load_config, cfg_get
|
||||
cfg = load_config()
|
||||
enabled = cfg_get(cfg, "security", "protected_instruction_files",
|
||||
default=True)
|
||||
extra = cfg_get(cfg, "security", "protected_instruction_extra_patterns",
|
||||
default=[])
|
||||
enabled = cfg_get(cfg, "security", "protected_instruction_files", default=True)
|
||||
extra = cfg_get(cfg, "security", "protected_instruction_extra_patterns", default=[])
|
||||
except Exception:
|
||||
return True, []
|
||||
if not isinstance(enabled, bool):
|
||||
@@ -146,11 +125,7 @@ def _protected_instruction_reason(filepath: str, task_id: str = "default",
|
||||
*, enabled: bool | None = None,
|
||||
extra_patterns: list[str] | None = None) -> str | None:
|
||||
"""Return a short label when ``filepath`` targets a protected instruction file, else ``None``.
|
||||
|
||||
Matches BOTH the normalized input and its realpath so neither a symlink
|
||||
pointing AT a protected file nor a protected name that is itself a symlink
|
||||
escapes; ``..`` traversal is neutralized by normpath/realpath first.
|
||||
"""
|
||||
Matches BOTH the normalized input and its realpath so no symlink direction escapes."""
|
||||
if enabled is None or extra_patterns is None:
|
||||
enabled, extra_patterns = _protected_instruction_config()
|
||||
if not enabled:
|
||||
@@ -166,18 +141,15 @@ def _protected_instruction_reason(filepath: str, task_id: str = "default",
|
||||
# mirror guard, write_approval); this gate targets PROJECT-LOCAL files only.
|
||||
# Must run before the ``.hermes`` component rule, which would match the home.
|
||||
real_home = _get_real_hermes_home()
|
||||
if real_home and (resolved == real_home
|
||||
or resolved.startswith(real_home + os.sep)):
|
||||
if real_home and (resolved == real_home or resolved.startswith(real_home + os.sep)):
|
||||
return None
|
||||
|
||||
for candidate in (normalized, resolved):
|
||||
base = os.path.basename(candidate)
|
||||
base_lower = base.lower()
|
||||
if base_lower in _PROTECTED_INSTRUCTION_BASENAMES:
|
||||
if base_lower in _PROTECTED_INSTRUCTION_BASENAMES or any(
|
||||
fnmatch.fnmatch(base_lower, pattern.lower()) for pattern in extra_patterns):
|
||||
return base
|
||||
for pattern in extra_patterns:
|
||||
if fnmatch.fnmatch(base_lower, pattern.lower()):
|
||||
return base
|
||||
# Project-local .hermes config dirs (<repo>/.hermes/config.yaml) steer
|
||||
# behavior too. Only the IMMEDIATE parent counts — matching any ancestor
|
||||
# would gate every write inside a checkout living under ~/.hermes.
|
||||
@@ -187,43 +159,39 @@ def _protected_instruction_reason(filepath: str, task_id: str = "default",
|
||||
return None
|
||||
|
||||
|
||||
def _request_protected_instruction_approval(
|
||||
reasons: list[str], task_id: str = "default") -> str | None:
|
||||
"""Ask the human to approve a write to protected instruction file(s).
|
||||
_APPROVAL_UNAVAILABLE = "requires approval but the approval subsystem is unavailable."
|
||||
_NO_HUMAN = "requires approval but no interactive user or gateway is present to approve it."
|
||||
|
||||
Returns ``None`` when approved, else a BLOCKED error string. Deliberately
|
||||
NOT routed through ``_run_approval_gate``: that honors --yolo and
|
||||
session/permanent allowlists, and this gate is one-operation approval EVERY
|
||||
time with no persisted scope. Fail-closed when no human channel exists.
|
||||
|
||||
def _request_protected_instruction_approval(reasons: list[str], task_id: str = "default") -> str | None:
|
||||
"""Ask the human to approve a write to protected instruction file(s); ``None`` when approved.
|
||||
|
||||
Deliberately NOT routed through ``_run_approval_gate`` (honors --yolo and
|
||||
allowlists): this gate is one-operation approval EVERY time, no persisted
|
||||
scope, fail-closed without a human channel.
|
||||
"""
|
||||
targets = ", ".join(dict.fromkeys(reasons))
|
||||
description = (
|
||||
f"Write to protected agent-instruction file(s): {targets}. "
|
||||
"These files steer future agent behavior; approval is always "
|
||||
"required (not bypassed by auto-approve)."
|
||||
)
|
||||
"required (not bypassed by auto-approve).")
|
||||
display = f"<write to {targets}>"
|
||||
blocked = (
|
||||
f"BLOCKED: write to protected agent-instruction file(s) ({targets}) "
|
||||
"{why} The user has NOT consented to this write. Do NOT retry it or "
|
||||
"attempt the same edit via another path (terminal, execute_code, "
|
||||
"etc.)."
|
||||
)
|
||||
timed_out = blocked.format(
|
||||
why="approval prompt timed out without a user response. "
|
||||
"Silence is not consent.")
|
||||
"etc.).")
|
||||
timed_out = blocked.format(why="approval prompt timed out without a user response. Silence is not consent.")
|
||||
denied = blocked.format(why="was denied by the user.")
|
||||
|
||||
try:
|
||||
import tools.approval as _approval
|
||||
except Exception:
|
||||
return blocked.format(why="requires approval but the approval "
|
||||
"subsystem is unavailable.")
|
||||
return blocked.format(why=_APPROVAL_UNAVAILABLE)
|
||||
|
||||
# Gateway surface: block on the button round-trip when a notify callback
|
||||
# is registered for this session. One-operation only — no scope buttons.
|
||||
session_key = _approval.get_current_session_key()
|
||||
notify_cb = None
|
||||
try:
|
||||
with _approval._lock:
|
||||
notify_cb = _approval._gateway_notify_cbs.get(session_key)
|
||||
@@ -237,82 +205,48 @@ def _request_protected_instruction_approval(
|
||||
"pattern_keys": ["protected_instruction_file"],
|
||||
"description": description,
|
||||
"allow_permanent": False,
|
||||
"allow_session": False,
|
||||
}
|
||||
decision = _approval._await_gateway_decision(
|
||||
session_key, notify_cb, approval_data, surface="gateway",
|
||||
)
|
||||
"allow_session": False}
|
||||
decision = _approval._await_gateway_decision(session_key, notify_cb, approval_data, surface="gateway")
|
||||
if decision.get("notify_failed"):
|
||||
return blocked.format(
|
||||
why="requires approval but the approval request could not "
|
||||
"be delivered.")
|
||||
choice = decision.get("choice")
|
||||
# Any tapped scope is a one-operation grant; nothing is persisted.
|
||||
if decision.get("resolved") and choice in {"once", "session", "always"}:
|
||||
return None
|
||||
if not decision.get("resolved"):
|
||||
return timed_out
|
||||
return denied
|
||||
|
||||
# CLI surface: per-thread approval callback (prompt_toolkit panel).
|
||||
callback = None
|
||||
try:
|
||||
from tools.terminal_tool import _get_approval_callback
|
||||
callback = _get_approval_callback()
|
||||
except Exception:
|
||||
callback = None
|
||||
|
||||
if callback is not None:
|
||||
return blocked.format(why="requires approval but the approval request could not be delivered.")
|
||||
choice, timed = decision.get("choice"), not decision.get("resolved")
|
||||
else:
|
||||
# CLI surface: per-thread approval callback (prompt_toolkit panel).
|
||||
try:
|
||||
from tools.terminal_tool import _get_approval_callback
|
||||
callback = _get_approval_callback()
|
||||
except Exception:
|
||||
callback = None
|
||||
if callback is None:
|
||||
# No human channel (script, cron, background thread): fail closed —
|
||||
# auto-approving here would recreate the persistence vector.
|
||||
return blocked.format(why=_NO_HUMAN)
|
||||
choice = _approval.prompt_dangerous_approval(
|
||||
display, description,
|
||||
allow_permanent=False,
|
||||
allow_session=False,
|
||||
approval_callback=callback,
|
||||
)
|
||||
if choice in {"once", "session", "always"}:
|
||||
return None
|
||||
if choice == "timeout":
|
||||
return timed_out
|
||||
return denied
|
||||
|
||||
# No human channel (script, cron, background thread): fail closed —
|
||||
# auto-approving here would recreate the persistence vector.
|
||||
return blocked.format(
|
||||
why="requires approval but no interactive user or gateway is "
|
||||
"present to approve it.")
|
||||
display, description, allow_permanent=False, allow_session=False, approval_callback=callback)
|
||||
timed = choice == "timeout"
|
||||
# Any tapped scope is a one-operation grant; nothing is persisted.
|
||||
if not timed and choice in {"once", "session", "always"}:
|
||||
return None
|
||||
return timed_out if timed else denied
|
||||
|
||||
|
||||
def _check_protected_instruction_write(paths: list[str],
|
||||
task_id: str = "default") -> str | None:
|
||||
"""Gate a write/patch touching protected instruction files.
|
||||
|
||||
ONE protected file gates the ENTIRE multi-file patch: a single prompt lists
|
||||
every protected target and a deny applies nothing (atomic all-or-nothing
|
||||
beats a partially-applied patch).
|
||||
"""
|
||||
def _check_protected_instruction_write(paths: list[str], task_id: str = "default") -> str | None:
|
||||
"""Gate a write/patch touching protected instruction files. ONE protected file gates
|
||||
the ENTIRE multi-file patch (one prompt, all-or-nothing)."""
|
||||
enabled, extra = _protected_instruction_config()
|
||||
if not enabled:
|
||||
return None
|
||||
reasons = [
|
||||
r for r in (
|
||||
_protected_instruction_reason(p, task_id, enabled=enabled, extra_patterns=extra)
|
||||
for p in paths
|
||||
) if r
|
||||
]
|
||||
reasons = [r for r in (_protected_instruction_reason(p, task_id, enabled=enabled, extra_patterns=extra)
|
||||
for p in paths) if r]
|
||||
if not reasons:
|
||||
return None
|
||||
return _request_protected_instruction_approval(reasons, task_id)
|
||||
|
||||
|
||||
def _check_approval_required_write(paths: list[str],
|
||||
task_id: str = "default") -> str | None:
|
||||
"""Gate a write/patch touching an approval-required path (``~/.ssh/config``).
|
||||
|
||||
Not credentials and not hard-denied, but they can steer process execution
|
||||
(SSH ``ProxyCommand`` / ``Match exec``). Unlike the protected-instruction
|
||||
gate this is a routine user edit: the prompt offers once/session/always and
|
||||
honors --yolo. Fail-closed when no interactive/gateway channel exists.
|
||||
"""
|
||||
def _check_approval_required_write(paths: list[str], task_id: str = "default") -> str | None:
|
||||
"""Gate a write/patch touching an approval-required path (``~/.ssh/config`` can steer
|
||||
execution via ``ProxyCommand``). Routine gate: once/session/always, honors --yolo,
|
||||
fail-closed without an interactive/gateway channel."""
|
||||
try:
|
||||
from agent.file_safety import is_write_approval_required
|
||||
except Exception:
|
||||
@@ -326,26 +260,22 @@ def _check_approval_required_write(paths: list[str],
|
||||
description = (
|
||||
f"Write to SSH client config file(s): {display_targets}. "
|
||||
"The SSH config can carry ProxyCommand / Match exec directives that "
|
||||
"run commands, so writes require your approval."
|
||||
)
|
||||
"run commands, so writes require your approval.")
|
||||
blocked = (
|
||||
f"BLOCKED: write to SSH config file(s) ({display_targets}) "
|
||||
"{why} Do NOT retry it via another path (terminal, execute_code) "
|
||||
"without the user's explicit consent."
|
||||
)
|
||||
"without the user's explicit consent.")
|
||||
|
||||
try:
|
||||
import tools.approval as _approval
|
||||
except Exception:
|
||||
return blocked.format(why="requires approval but the approval "
|
||||
"subsystem is unavailable.")
|
||||
return blocked.format(why=_APPROVAL_UNAVAILABLE)
|
||||
|
||||
result = _approval._run_approval_gate(
|
||||
pattern_key="ssh_config_write",
|
||||
description=description,
|
||||
display_target=f"<write to {display_targets}>",
|
||||
cron_deny_message=blocked.format(
|
||||
why="requires approval but this cron session denies it."),
|
||||
cron_deny_message=blocked.format(why="requires approval but this cron session denies it."),
|
||||
single_query_deny_message=blocked.format(
|
||||
why="requires approval but single-query (-q) sessions run "
|
||||
"without a user present to approve it. To allow flagged "
|
||||
@@ -353,10 +283,7 @@ def _check_approval_required_write(paths: list[str],
|
||||
"approve in config.yaml."),
|
||||
autoapprove_log_prefix="ssh_config_write",
|
||||
fail_closed_when_no_human=True,
|
||||
no_human_block_message=blocked.format(
|
||||
why="requires approval but no interactive user or gateway is "
|
||||
"present to approve it."),
|
||||
)
|
||||
no_human_block_message=blocked.format(why=_NO_HUMAN))
|
||||
if result.get("approved"):
|
||||
return None
|
||||
return result.get("message") or blocked.format(why="was denied.")
|
||||
@@ -366,74 +293,41 @@ def _get_container_mirror_prefix_for_task(task_id: str = "default") -> str | Non
|
||||
"""Return the container-side Hermes mirror prefix for persistent Docker file tools."""
|
||||
try:
|
||||
from tools.terminal_tool import (
|
||||
_active_environments,
|
||||
_env_lock,
|
||||
_get_env_config,
|
||||
_resolve_container_task_id,
|
||||
)
|
||||
|
||||
_active_environments, _env_lock, _get_env_config, _resolve_container_task_id)
|
||||
container_key = _resolve_container_task_id(task_id)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
try:
|
||||
with _env_lock:
|
||||
env = _active_environments.get(container_key) or _active_environments.get(task_id)
|
||||
|
||||
if env is not None:
|
||||
if env.__class__.__name__ == "DockerEnvironment" and bool(
|
||||
getattr(env, "_persistent", False)
|
||||
):
|
||||
return "/root/.hermes"
|
||||
return None
|
||||
|
||||
persistent_docker = (env.__class__.__name__ == "DockerEnvironment"
|
||||
and bool(getattr(env, "_persistent", False)))
|
||||
return "/root/.hermes" if persistent_docker else None
|
||||
config = _get_env_config()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
if config.get("env_type") == "docker" and config.get("container_persistent", True):
|
||||
return "/root/.hermes"
|
||||
return None
|
||||
|
||||
|
||||
def _check_cross_profile_path(filepath: str, task_id: str = "default") -> str | None:
|
||||
"""Soft-guard: warn when ``filepath`` lands on a host-side or Docker sandbox
|
||||
MIRROR of Hermes state — a write the host process never reads (lost work).
|
||||
|
||||
Not profile isolation: the former cross-PROFILE guard was removed by
|
||||
maintainer decision (profiles were never isolated). ``cross_profile=True``
|
||||
on the tools still bypasses these mirror guards (name kept for replay compat).
|
||||
Fails open on import error — the sensitive-path guard and denylist still apply.
|
||||
"""
|
||||
"""Soft-guard: warn when ``filepath`` lands on a host-side or Docker sandbox MIRROR of
|
||||
Hermes state (a write the host never reads). Not profile isolation — that guard was
|
||||
removed; ``cross_profile=True`` keeps bypassing this one for replay compat. Fails open."""
|
||||
try:
|
||||
from agent.file_safety import (
|
||||
get_container_mirror_warning,
|
||||
get_sandbox_mirror_warning,
|
||||
)
|
||||
from agent.file_safety import get_container_mirror_warning, get_sandbox_mirror_warning
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
resolved = _resolved_or_raw(filepath, task_id)
|
||||
|
||||
warning = get_sandbox_mirror_warning(resolved)
|
||||
if warning is not None:
|
||||
return warning
|
||||
|
||||
return get_container_mirror_warning(
|
||||
resolved,
|
||||
mirror_prefix=_get_container_mirror_prefix_for_task(task_id),
|
||||
)
|
||||
return get_container_mirror_warning(resolved, mirror_prefix=_get_container_mirror_prefix_for_task(task_id))
|
||||
|
||||
|
||||
def _check_binary_document_write(filepath: str, task_id: str = "default") -> str | None:
|
||||
"""Reject text-tool writes that would corrupt a binary document.
|
||||
|
||||
``read_file`` auto-extracts Office/PDF to text, so the model plausibly
|
||||
believes it holds the file's bytes and writes edited text back — which can
|
||||
never form a valid container. Opaque formats (.docx/.xlsx/.pptx/.odt/...)
|
||||
are always rejected; .pdf only when OVERWRITING an existing regular file
|
||||
(raw PDF syntax is text-authorable, so new-file creation stays allowed).
|
||||
"""
|
||||
"""Reject text-tool writes that would corrupt a binary document (read_file showed
|
||||
EXTRACTED text, so the model may write it back). Opaque formats are always rejected;
|
||||
.pdf only when OVERWRITING an existing file (raw PDF syntax is text-authorable)."""
|
||||
if has_opaque_document_extension(filepath):
|
||||
ext = filepath[filepath.rfind("."):].lower()
|
||||
return (
|
||||
@@ -442,8 +336,7 @@ def _check_binary_document_write(filepath: str, task_id: str = "default") -> str
|
||||
"corrupt the file (read_file showed you EXTRACTED text, not the real "
|
||||
"bytes). Use the docx/xlsx/powerpoint skills or a library like "
|
||||
"python-docx/openpyxl/python-pptx via the terminal to create or edit "
|
||||
"this document."
|
||||
)
|
||||
"this document.")
|
||||
if is_pdf_path(filepath):
|
||||
try:
|
||||
resolved = Path(_resolve_path_for_task(filepath, task_id))
|
||||
@@ -456,75 +349,48 @@ def _check_binary_document_write(filepath: str, task_id: str = "default") -> str
|
||||
"read_file showed you EXTRACTED text, not the real bytes — writing "
|
||||
"text back would destroy the document. Use the pdf skill or a PDF "
|
||||
"library via the terminal to modify it. (Creating a NEW .pdf file "
|
||||
"is allowed.)"
|
||||
)
|
||||
"is allowed.)")
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal display text must never be persisted as file content
|
||||
# ---------------------------------------------------------------------------
|
||||
# ── Internal display text must never be persisted as file content ────────
|
||||
_READ_DEDUP_STATUS_MESSAGE = (
|
||||
"File unchanged since last read. The content from "
|
||||
"the earlier read_file result in this conversation is "
|
||||
"still current — refer to that instead of re-reading."
|
||||
)
|
||||
"still current — refer to that instead of re-reading.")
|
||||
|
||||
|
||||
def _is_internal_file_status_text(content: str) -> bool:
|
||||
"""True when content is the read_file dedup status message (verbatim or lightly framed).
|
||||
|
||||
Models echo the message verbatim OR wrap it with short framing ("Note:",
|
||||
a trailing comment). Any write whose stripped body contains the full
|
||||
message and is <=2x its length is status-dominated — a real file quoting
|
||||
this message would be dramatically longer.
|
||||
"""
|
||||
"""True when content is the read_file dedup status message, verbatim or lightly framed
|
||||
(contains the full message and is <=2x its length — a real file quoting it would be longer)."""
|
||||
if not isinstance(content, str):
|
||||
return False
|
||||
stripped = content.strip()
|
||||
if not stripped:
|
||||
return False
|
||||
if stripped == _READ_DEDUP_STATUS_MESSAGE:
|
||||
return True
|
||||
return (_READ_DEDUP_STATUS_MESSAGE in stripped
|
||||
and len(stripped) <= 2 * len(_READ_DEDUP_STATUS_MESSAGE))
|
||||
return bool(stripped) and _READ_DEDUP_STATUS_MESSAGE in stripped and (
|
||||
len(stripped) <= 2 * len(_READ_DEDUP_STATUS_MESSAGE))
|
||||
|
||||
|
||||
def _looks_like_read_file_line_numbered_content(content: str) -> bool:
|
||||
"""True for content dominated by read_file's ``LINE_NUM|CONTENT`` display.
|
||||
|
||||
Rejects writes whose non-empty lines are mostly (>=60%) consecutive
|
||||
numbered lines, while allowing sparse literal pipe content such as a
|
||||
single ``1|value`` line.
|
||||
"""
|
||||
"""True for content dominated by read_file's ``LINE_NUM|CONTENT`` display (>=60% of
|
||||
non-empty lines are consecutive numbered lines; a lone ``1|value`` is allowed)."""
|
||||
if not isinstance(content, str):
|
||||
return False
|
||||
|
||||
lines = [line for line in content.splitlines() if line.strip()]
|
||||
if len(lines) < 2:
|
||||
return False
|
||||
|
||||
numbered: list[int] = []
|
||||
for line in lines:
|
||||
prefix, sep, _rest = line.lstrip().partition("|")
|
||||
if sep and prefix.isdigit():
|
||||
numbered.append(int(prefix))
|
||||
|
||||
if len(numbered) < 2 or len(numbered) / len(lines) < 0.6:
|
||||
return False
|
||||
|
||||
consecutive_pairs = sum(
|
||||
1 for prev, current in zip(numbered, numbered[1:])
|
||||
if current == prev + 1
|
||||
)
|
||||
consecutive_pairs = sum(1 for prev, current in zip(numbered, numbered[1:]) if current == prev + 1)
|
||||
return consecutive_pairs >= len(numbered) - 1
|
||||
|
||||
|
||||
def _is_internal_file_tool_content(content: str) -> bool:
|
||||
"""Return True when content is file-tool display text, not intended file bytes."""
|
||||
return (
|
||||
_is_internal_file_status_text(content)
|
||||
or _looks_like_read_file_line_numbered_content(content)
|
||||
)
|
||||
return _is_internal_file_status_text(content) or _looks_like_read_file_line_numbered_content(content)
|
||||
|
||||
+352
-178
@@ -1,67 +1,321 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fuzzy find-and-replace for LLM-generated edits.
|
||||
|
||||
Tries an ordered chain of increasingly permissive matching strategies (see
|
||||
:mod:`tools.fuzzy_match_strategies`) so whitespace, indentation, escaping and
|
||||
Unicode drift in tool-call arguments still land on the intended region.
|
||||
An ordered chain of increasingly permissive strategies (``STRATEGIES``) lets
|
||||
whitespace, indentation, escaping and Unicode drift in tool-call arguments
|
||||
still land on the intended region::
|
||||
|
||||
new_content, match_count, strategy, error = fuzzy_find_and_replace(
|
||||
content, old_string, new_string, replace_all=False)
|
||||
"""
|
||||
|
||||
import re
|
||||
from difflib import SequenceMatcher
|
||||
from typing import List, Optional, Tuple
|
||||
from typing import Callable, Optional
|
||||
|
||||
from tools.fuzzy_match_strategies import ( # noqa: F401 — re-exported names
|
||||
SIMILARITY_STRATEGIES,
|
||||
STRATEGIES,
|
||||
UNICODE_MAP,
|
||||
_build_orig_to_norm_map,
|
||||
_calculate_line_positions,
|
||||
_invert_norm_map,
|
||||
_map_normalized_positions,
|
||||
_map_positions_norm_to_orig,
|
||||
_norm_end_to_orig,
|
||||
_strategy_block_anchor,
|
||||
_strategy_context_aware,
|
||||
_strategy_escape_normalized,
|
||||
_strategy_exact,
|
||||
_strategy_indentation_flexible,
|
||||
_strategy_line_trimmed,
|
||||
_strategy_trimmed_boundary,
|
||||
_strategy_unicode_normalized,
|
||||
_strategy_whitespace_normalized,
|
||||
_unicode_normalize,
|
||||
)
|
||||
Span = tuple[int, int]
|
||||
|
||||
IDENTICAL_STRINGS_ERROR = (
|
||||
"No edit was applied because old_string and new_string are identical. "
|
||||
"Provide the existing text to replace in old_string and the changed "
|
||||
"replacement text in new_string."
|
||||
)
|
||||
"replacement text in new_string.")
|
||||
|
||||
UNICODE_MAP = {
|
||||
"\u201c": '"', "\u201d": '"', # smart double quotes
|
||||
"\u2018": "'", "\u2019": "'", # smart single quotes
|
||||
"\u2014": "--", "\u2013": "-", # em/en dashes
|
||||
"\u2026": "...", "\u00a0": " ", # ellipsis and non-breaking space
|
||||
"\u2212": "-", # typographic minus (math/scientific docs)
|
||||
# Space-separator family (Zs): otherwise such files fall to the similarity fallback.
|
||||
"\u2000": " ", "\u2001": " ", "\u2002": " ", "\u2003": " ",
|
||||
"\u2004": " ", "\u2005": " ", "\u2006": " ", "\u2007": " ",
|
||||
"\u2008": " ", "\u2009": " ", "\u200a": " ", "\u202f": " ",
|
||||
"\u205f": " ", "\u3000": " "}
|
||||
|
||||
|
||||
def _unicode_normalize(text: str) -> str:
|
||||
"""Map typographic Unicode variants to ASCII equivalents."""
|
||||
for char, repl in UNICODE_MAP.items():
|
||||
text = text.replace(char, repl)
|
||||
return text
|
||||
|
||||
|
||||
# ── Position helpers ─────────────────────────────────────────────────────
|
||||
|
||||
def _calculate_line_positions(content_lines: list[str], start_line: int,
|
||||
end_line: int, content_length: int) -> Span:
|
||||
"""Character span covering ``content_lines[start_line:end_line]`` (end exclusive)."""
|
||||
start_pos = sum(len(line) + 1 for line in content_lines[:start_line])
|
||||
end_pos = sum(len(line) + 1 for line in content_lines[:end_line]) - 1
|
||||
return start_pos, min(content_length, end_pos)
|
||||
|
||||
|
||||
def _window_spans(content: str, content_lines: list[str], n: int,
|
||||
accept: Callable[[int], bool]) -> list[Span]:
|
||||
"""Spans of every ``n``-line window starting at ``i`` for which ``accept(i)``."""
|
||||
return [
|
||||
_calculate_line_positions(content_lines, i, i + n, len(content))
|
||||
for i in range(len(content_lines) - n + 1) if accept(i)]
|
||||
|
||||
|
||||
def _match_transformed_lines(content: str, pattern: str,
|
||||
transform: Callable[[list[str]], list[str]]) -> list[Span]:
|
||||
"""Match ``pattern`` against ``content`` after applying ``transform`` to each line block."""
|
||||
content_lines = content.split('\n')
|
||||
pattern_norm = transform(pattern.split('\n'))
|
||||
n = len(pattern_norm)
|
||||
return _window_spans(content, content_lines, n,
|
||||
lambda i: transform(content_lines[i:i + n]) == pattern_norm)
|
||||
|
||||
|
||||
def _strip_boundary(lines: list[str]) -> list[str]:
|
||||
"""Strip whitespace on the first and last lines only."""
|
||||
lines = list(lines)
|
||||
lines[0] = lines[0].strip()
|
||||
if len(lines) > 1:
|
||||
lines[-1] = lines[-1].strip()
|
||||
return lines
|
||||
|
||||
|
||||
def _build_orig_to_norm_map(original: str) -> list[int]:
|
||||
"""Map each original index to its index in ``_unicode_normalize(original)``
|
||||
(replacements can expand one char into several). Length ``len(original)+1``;
|
||||
the last entry is a sentinel one past the final character."""
|
||||
result: list[int] = []
|
||||
norm_pos = 0
|
||||
for char in original:
|
||||
result.append(norm_pos)
|
||||
repl = UNICODE_MAP.get(char)
|
||||
norm_pos += len(repl) if repl is not None else 1
|
||||
result.append(norm_pos)
|
||||
return result
|
||||
|
||||
|
||||
def _invert_norm_map(orig_to_norm: list[int]) -> dict[int, int]:
|
||||
"""norm_pos -> first original position mapping to it."""
|
||||
inverted: dict[int, int] = {}
|
||||
for orig_pos, norm_pos in enumerate(orig_to_norm[:-1]):
|
||||
inverted.setdefault(norm_pos, orig_pos)
|
||||
return inverted
|
||||
|
||||
|
||||
def _norm_end_to_orig(orig_to_norm: list[int], orig_start: int, norm_end: int) -> int:
|
||||
"""Walk from ``orig_start`` until the mapped position reaches ``norm_end``."""
|
||||
orig_len = len(orig_to_norm) - 1
|
||||
orig_end = orig_start
|
||||
while orig_end < orig_len and orig_to_norm[orig_end] < norm_end:
|
||||
orig_end += 1
|
||||
return orig_end
|
||||
|
||||
|
||||
def _map_positions_norm_to_orig(orig_to_norm: list[int], norm_matches: list[Span]) -> list[Span]:
|
||||
"""Convert spans in the normalised string to original-string spans."""
|
||||
norm_to_orig_start = _invert_norm_map(orig_to_norm)
|
||||
results: list[Span] = []
|
||||
for norm_start, norm_end in norm_matches:
|
||||
if norm_start in norm_to_orig_start:
|
||||
orig_start = norm_to_orig_start[norm_start]
|
||||
results.append((orig_start, _norm_end_to_orig(orig_to_norm, orig_start, norm_end)))
|
||||
return results
|
||||
|
||||
|
||||
def _map_normalized_positions(original: str, normalized: str,
|
||||
normalized_matches: list[Span]) -> list[Span]:
|
||||
"""Best-effort span mapping for ``[ \\t]+`` -> ``' '`` whitespace collapsing."""
|
||||
orig_to_norm = [] # orig_to_norm[i] = position in normalized
|
||||
orig_idx = norm_idx = 0
|
||||
while orig_idx < len(original) and norm_idx < len(normalized):
|
||||
if original[orig_idx] == normalized[norm_idx]:
|
||||
orig_to_norm.append(norm_idx)
|
||||
orig_idx += 1
|
||||
norm_idx += 1
|
||||
elif original[orig_idx] in ' \t' and normalized[norm_idx] == ' ':
|
||||
# Collapsed run: advance norm_idx only once the run is consumed.
|
||||
orig_to_norm.append(norm_idx)
|
||||
orig_idx += 1
|
||||
if orig_idx < len(original) and original[orig_idx] not in ' \t':
|
||||
norm_idx += 1
|
||||
else:
|
||||
# Extra whitespace in original, or a mismatch normalization should
|
||||
# never produce — either way, pin to the current norm_idx.
|
||||
orig_to_norm.append(norm_idx)
|
||||
orig_idx += 1
|
||||
orig_to_norm.extend([len(normalized)] * (len(original) - orig_idx))
|
||||
|
||||
norm_to_orig_start = {}
|
||||
norm_to_orig_end = {}
|
||||
for orig_pos, norm_pos in enumerate(orig_to_norm):
|
||||
norm_to_orig_start.setdefault(norm_pos, orig_pos)
|
||||
norm_to_orig_end[norm_pos] = orig_pos
|
||||
|
||||
original_matches = []
|
||||
for norm_start, norm_end in normalized_matches:
|
||||
if norm_start in norm_to_orig_start:
|
||||
orig_start = norm_to_orig_start[norm_start]
|
||||
else:
|
||||
orig_start = min(i for i, n in enumerate(orig_to_norm) if n >= norm_start)
|
||||
if norm_end - 1 in norm_to_orig_end:
|
||||
orig_end = norm_to_orig_end[norm_end - 1] + 1
|
||||
else:
|
||||
orig_end = orig_start + (norm_end - norm_start)
|
||||
# Absorb trailing collapsed whitespace only when the normalized match
|
||||
# itself ended in a space; otherwise the first whitespace after the
|
||||
# match is a word boundary that must survive.
|
||||
if norm_end < len(normalized) and normalized[norm_end - 1] == ' ':
|
||||
while orig_end < len(original) and original[orig_end] in ' \t':
|
||||
orig_end += 1
|
||||
original_matches.append((orig_start, min(orig_end, len(original))))
|
||||
return original_matches
|
||||
|
||||
|
||||
# ── Strategies ───────────────────────────────────────────────────────────
|
||||
# Each takes ``(content, pattern)`` and returns ``(start, end)`` spans in the
|
||||
# ORIGINAL content.
|
||||
|
||||
def _strategy_exact(content: str, pattern: str) -> list[Span]:
|
||||
"""Strategy 1: exact, non-overlapping occurrences (str.replace semantics —
|
||||
overlapping spans would corrupt the file under replace_all)."""
|
||||
return [m.span() for m in re.finditer(re.escape(pattern), content)]
|
||||
|
||||
|
||||
def _strategy_line_trimmed(content: str, pattern: str) -> list[Span]:
|
||||
"""Strategy 2: strip each line before comparing."""
|
||||
return _match_transformed_lines(content, pattern, lambda ls: [l.strip() for l in ls])
|
||||
|
||||
|
||||
def _strategy_whitespace_normalized(content: str, pattern: str) -> list[Span]:
|
||||
"""Strategy 3: collapse runs of spaces/tabs to a single space."""
|
||||
def normalize(s):
|
||||
return re.sub(r'[ \t]+', ' ', s)
|
||||
|
||||
content_normalized = normalize(content)
|
||||
matches_in_normalized = _strategy_exact(content_normalized, normalize(pattern))
|
||||
if not matches_in_normalized:
|
||||
return []
|
||||
return _map_normalized_positions(content, content_normalized, matches_in_normalized)
|
||||
|
||||
|
||||
def _strategy_indentation_flexible(content: str, pattern: str) -> list[Span]:
|
||||
"""Strategy 4: ignore leading indentation entirely."""
|
||||
return _match_transformed_lines(content, pattern, lambda ls: [l.lstrip() for l in ls])
|
||||
|
||||
|
||||
def _strategy_escape_normalized(content: str, pattern: str) -> list[Span]:
|
||||
"""Strategy 5: treat literal ``\\n``/``\\t``/``\\r`` in the pattern as control chars."""
|
||||
pattern_unescaped = pattern.replace('\\n', '\n').replace('\\t', '\t').replace('\\r', '\r')
|
||||
if pattern_unescaped == pattern:
|
||||
return []
|
||||
return _strategy_exact(content, pattern_unescaped)
|
||||
|
||||
|
||||
def _strategy_trimmed_boundary(content: str, pattern: str) -> list[Span]:
|
||||
"""Strategy 6: strip whitespace on the first and last lines only."""
|
||||
return _match_transformed_lines(content, pattern, _strip_boundary)
|
||||
|
||||
|
||||
def _strategy_unicode_normalized(content: str, pattern: str) -> list[Span]:
|
||||
"""Strategy 7: exact/line-trimmed match after Unicode->ASCII normalisation of both sides."""
|
||||
norm_pattern = _unicode_normalize(pattern)
|
||||
norm_content = _unicode_normalize(content)
|
||||
if norm_content == content and norm_pattern == pattern:
|
||||
return []
|
||||
norm_matches = (_strategy_exact(norm_content, norm_pattern)
|
||||
or _strategy_line_trimmed(norm_content, norm_pattern))
|
||||
if not norm_matches:
|
||||
return []
|
||||
return _map_positions_norm_to_orig(_build_orig_to_norm_map(content), norm_matches)
|
||||
|
||||
|
||||
def _strategy_block_anchor(content: str, pattern: str) -> list[Span]:
|
||||
"""Strategy 8: anchor on first+last lines, similarity-score the middle."""
|
||||
pattern_lines = _unicode_normalize(pattern).split('\n')
|
||||
if len(pattern_lines) < 2:
|
||||
return []
|
||||
first_line = pattern_lines[0].strip()
|
||||
last_line = pattern_lines[-1].strip()
|
||||
n = len(pattern_lines)
|
||||
|
||||
# Match on normalized lines; compute offsets from the ORIGINAL lines so
|
||||
# multi-char expansions (em-dash -> '--') don't shift positions.
|
||||
norm_content_lines = _unicode_normalize(content).split('\n')
|
||||
potential_matches = {
|
||||
i for i in range(len(norm_content_lines) - n + 1)
|
||||
if norm_content_lines[i].strip() == first_line
|
||||
and norm_content_lines[i + n - 1].strip() == last_line}
|
||||
# Looser thresholds (0.10/0.30) matched unrelated blocks; these are the safe floor.
|
||||
threshold = 0.50 if len(potential_matches) == 1 else 0.70
|
||||
pattern_middle = '\n'.join(pattern_lines[1:-1])
|
||||
|
||||
def similar(i: int) -> bool:
|
||||
if i not in potential_matches:
|
||||
return False
|
||||
if n <= 2:
|
||||
return True
|
||||
content_middle = '\n'.join(norm_content_lines[i + 1:i + n - 1])
|
||||
return SequenceMatcher(None, content_middle, pattern_middle).ratio() >= threshold
|
||||
|
||||
return _window_spans(content, content.split('\n'), n, similar)
|
||||
|
||||
|
||||
def _strategy_context_aware(content: str, pattern: str) -> list[Span]:
|
||||
"""Strategy 9 (last resort): anchored per-line similarity, every non-blank line >= 0.80.
|
||||
The anchor pre-filter bounds the scan; the all-lines rule stops coincidental matches."""
|
||||
pattern_lines = pattern.split('\n')
|
||||
content_lines = content.split('\n')
|
||||
n = len(pattern_lines)
|
||||
if n > len(content_lines):
|
||||
return []
|
||||
first_pat = pattern_lines[0].strip()
|
||||
last_pat = pattern_lines[-1].strip()
|
||||
|
||||
def _sim(a: str, b: str) -> float:
|
||||
return 1.0 if a == b else SequenceMatcher(None, a, b).ratio()
|
||||
|
||||
def accept(i: int) -> bool:
|
||||
block_lines = content_lines[i:i + n]
|
||||
if _sim(first_pat, block_lines[0].strip()) < 0.80:
|
||||
return False
|
||||
if _sim(last_pat, block_lines[-1].strip()) < 0.80:
|
||||
return False
|
||||
return all(
|
||||
not p_line.strip() or _sim(p_line.strip(), c_line.strip()) >= 0.80
|
||||
for p_line, c_line in zip(pattern_lines, block_lines))
|
||||
|
||||
return _window_spans(content, content_lines, n, accept)
|
||||
|
||||
|
||||
# Ordered chain: precise strategies first, similarity-based last.
|
||||
STRATEGIES: list[tuple[str, Callable[[str, str], list[Span]]]] = [
|
||||
("exact", _strategy_exact),
|
||||
("line_trimmed", _strategy_line_trimmed),
|
||||
("whitespace_normalized", _strategy_whitespace_normalized),
|
||||
("indentation_flexible", _strategy_indentation_flexible),
|
||||
("escape_normalized", _strategy_escape_normalized),
|
||||
("trimmed_boundary", _strategy_trimmed_boundary),
|
||||
("unicode_normalized", _strategy_unicode_normalized),
|
||||
("block_anchor", _strategy_block_anchor),
|
||||
("context_aware", _strategy_context_aware)]
|
||||
|
||||
# Matches from these only *approximately* resemble old_string — fine for one
|
||||
# unique replacement, never safe under replace_all.
|
||||
SIMILARITY_STRATEGIES = frozenset({"block_anchor", "context_aware"})
|
||||
|
||||
|
||||
# ── Orchestrator ─────────────────────────────────────────────────────────
|
||||
|
||||
def is_already_applied(content: str, old_string: str, new_string: str) -> bool:
|
||||
"""True when the requested edit is already present (re-sent edit -> success-shaped no-op).
|
||||
|
||||
Conservative: new_string must be non-trivial (>= 8 chars stripped) and
|
||||
appear EXACTLY; when it differs from old_string, old_string must be gone.
|
||||
"""
|
||||
if not new_string or len(new_string.strip()) < 8:
|
||||
"""True when the edit is already present (re-sent edit -> success-shaped no-op).
|
||||
Conservative: new_string non-trivial (>= 8 chars) and present EXACTLY; old_string gone."""
|
||||
if not new_string or len(new_string.strip()) < 8 or new_string not in content:
|
||||
return False
|
||||
if new_string not in content:
|
||||
return False
|
||||
if old_string == new_string:
|
||||
return True
|
||||
return old_string not in content
|
||||
return old_string == new_string or old_string not in content
|
||||
|
||||
|
||||
def _matched_regions(content: str, matches: List[Tuple[int, int]]) -> str:
|
||||
def _matched_regions(content: str, matches: list[Span]) -> str:
|
||||
return "".join(content[start:end] for start, end in matches)
|
||||
|
||||
|
||||
def _format_match_locations(content: str, matches: List[Tuple[int, int]],
|
||||
cap: int = 5) -> str:
|
||||
def _format_match_locations(content: str, matches: list[Span], cap: int = 5) -> str:
|
||||
"""Render up to ``cap`` match positions as 'L<line>: <snippet>' rows."""
|
||||
rows = []
|
||||
for start, _end in matches[:cap]:
|
||||
@@ -81,7 +335,7 @@ def _format_match_locations(content: str, matches: List[Tuple[int, int]],
|
||||
|
||||
|
||||
def fuzzy_find_and_replace(content: str, old_string: str, new_string: str,
|
||||
replace_all: bool = False) -> Tuple[str, int, Optional[str], Optional[str]]:
|
||||
replace_all: bool = False) -> tuple[str, int, Optional[str], Optional[str]]:
|
||||
"""Find and replace via the strategy chain.
|
||||
|
||||
Returns ``(new_content, match_count, strategy_name, error)``; on failure
|
||||
@@ -106,15 +360,13 @@ def fuzzy_find_and_replace(content: str, old_string: str, new_string: str,
|
||||
return content, 0, None, (
|
||||
f"Found {len(matches)} matches for old_string. "
|
||||
f"Provide more context to make it unique, or use replace_all=True. "
|
||||
f"Matches:\n{locations}"
|
||||
)
|
||||
f"Matches:\n{locations}")
|
||||
if replace_all and len(matches) > 1 and strategy_name in SIMILARITY_STRATEGIES:
|
||||
return content, 0, None, (
|
||||
f"Found {len(matches)} approximate matches via the "
|
||||
f"'{strategy_name}' strategy; replace_all only applies to exact "
|
||||
f"matches. Provide the precise text (whitespace included) so an "
|
||||
f"exact/line-trimmed match can be made."
|
||||
)
|
||||
f"exact/line-trimmed match can be made.")
|
||||
|
||||
# Non-exact matches came through some normalization, so new_string may
|
||||
# carry serialization drift the file doesn't have.
|
||||
@@ -125,13 +377,10 @@ def fuzzy_find_and_replace(content: str, old_string: str, new_string: str,
|
||||
|
||||
effective_new = _maybe_unescape_new_string(new_string, content, matches)
|
||||
if strategy_name == "unicode_normalized":
|
||||
effective_new = _preserve_unicode_in_replacement(
|
||||
content, matches, old_string, effective_new,
|
||||
)
|
||||
effective_new = _preserve_unicode_in_replacement(content, matches, old_string, effective_new)
|
||||
new_content = _apply_replacements(
|
||||
content, matches, effective_new,
|
||||
old_string=old_string if strategy_name != "exact" else None,
|
||||
)
|
||||
old_string=old_string if strategy_name != "exact" else None)
|
||||
return new_content, len(matches), strategy_name, None
|
||||
|
||||
return content, 0, None, "Could not find a match for old_string in the file"
|
||||
@@ -139,14 +388,10 @@ def fuzzy_find_and_replace(content: str, old_string: str, new_string: str,
|
||||
|
||||
# ── Escape-drift guards ──────────────────────────────────────────────────
|
||||
|
||||
def _detect_escape_drift(content: str, matches: List[Tuple[int, int]],
|
||||
def _detect_escape_drift(content: str, matches: list[Span],
|
||||
old_string: str, new_string: str) -> Optional[str]:
|
||||
"""Error string when new_string carries tool-call escape artifacts, else None.
|
||||
|
||||
Fires on ``\\'``/``\\"`` present in both old_string and new_string but
|
||||
absent from the matched region (spurious shell-style escaping), and on
|
||||
JSON double-escaped backslash runs (see ``_detect_backslash_doubling``).
|
||||
"""
|
||||
"""Error string when new_string carries tool-call escape artifacts, else None:
|
||||
``\\'``/``\\"`` in both strings but not the matched region, or doubled backslash runs."""
|
||||
has_quote_suspects = "\\'" in new_string or '\\"' in new_string
|
||||
if not has_quote_suspects and "\\" not in old_string:
|
||||
return None
|
||||
@@ -163,47 +408,27 @@ def _detect_escape_drift(content: str, matches: List[Tuple[int, int]],
|
||||
f"serialization artifact where an apostrophe or quote got "
|
||||
f"prefixed with a spurious backslash. Re-read the file with "
|
||||
f"read_file and pass old_string/new_string without "
|
||||
f"backslash-escaping {plain!r} characters."
|
||||
)
|
||||
f"backslash-escaping {plain!r} characters.")
|
||||
return _detect_backslash_doubling(matched_regions, old_string, new_string)
|
||||
|
||||
|
||||
def _backslash_runs(s: str) -> List[int]:
|
||||
def _backslash_runs(s: str) -> list[int]:
|
||||
"""Lengths of maximal backslash runs in ``s``, in order."""
|
||||
runs: List[int] = []
|
||||
n = 0
|
||||
for ch in s:
|
||||
if ch == "\\":
|
||||
n += 1
|
||||
elif n:
|
||||
runs.append(n)
|
||||
n = 0
|
||||
if n:
|
||||
runs.append(n)
|
||||
return runs
|
||||
return [len(run) for run in re.findall(r"\\+", s)]
|
||||
|
||||
|
||||
def _detect_backslash_doubling(matched_regions: str, old_string: str,
|
||||
new_string: str) -> Optional[str]:
|
||||
"""Detect old_string whose every backslash run is exactly 2x the file's.
|
||||
|
||||
That pattern means the arguments were JSON-escaped one extra time; a
|
||||
similarity strategy still matches, and writing new_string verbatim would
|
||||
double every backslash in the file. Requires the same run count, a
|
||||
non-trivial signal (a run >= 2 or 2+ runs), and new_string not already
|
||||
matching the file's counts.
|
||||
"""
|
||||
"""Detect old_string whose every backslash run is exactly 2x the file's (arguments
|
||||
JSON-escaped one extra time). Requires the same run count, a non-trivial signal
|
||||
(a run >= 2 or 2+ runs), and new_string not already matching the file's counts."""
|
||||
old_runs = _backslash_runs(old_string)
|
||||
file_runs = _backslash_runs(matched_regions)
|
||||
if not old_runs or not file_runs or len(old_runs) != len(file_runs):
|
||||
return None
|
||||
if old_runs == file_runs:
|
||||
return None
|
||||
if any(o != f * 2 for o, f in zip(old_runs, file_runs)):
|
||||
return None
|
||||
if not (any(f >= 2 for f in file_runs) or len(file_runs) >= 2):
|
||||
return None
|
||||
if _backslash_runs(new_string) == file_runs:
|
||||
if (not old_runs or not file_runs or len(old_runs) != len(file_runs)
|
||||
or old_runs == file_runs
|
||||
or any(o != f * 2 for o, f in zip(old_runs, file_runs))
|
||||
or not (any(f >= 2 for f in file_runs) or len(file_runs) >= 2)
|
||||
or _backslash_runs(new_string) == file_runs):
|
||||
return None
|
||||
return (
|
||||
"Escape-drift detected: every backslash run in old_string is exactly "
|
||||
@@ -212,29 +437,20 @@ def _detect_backslash_doubling(matched_regions: str, old_string: str,
|
||||
"were JSON-escaped one extra time; applying new_string verbatim would "
|
||||
"double every backslash in the file. Re-read the file with read_file "
|
||||
"and resend old_string/new_string with the backslash counts exactly "
|
||||
"as they appear in the file."
|
||||
)
|
||||
"as they appear in the file.")
|
||||
|
||||
|
||||
def _maybe_unescape_new_string(new_string: str, content: str,
|
||||
matches: List[Tuple[int, int]]) -> str:
|
||||
"""Convert literal ``\\t``/``\\r`` in new_string to control chars, per sequence,
|
||||
only when the matched file region already contains the real control char.
|
||||
|
||||
Files that legitimately contain the two-char string (e.g. ``sep = "\\t"``)
|
||||
have a backslash+t in the region, not a tab, so they're left alone.
|
||||
``\\n`` is deliberately excluded: newlines serialize correctly through
|
||||
JSON and rewriting them would mangle escape sequences in source literals.
|
||||
"""
|
||||
def _maybe_unescape_new_string(new_string: str, content: str, matches: list[Span]) -> str:
|
||||
"""Convert literal ``\\t``/``\\r`` in new_string to control chars, per sequence, only
|
||||
when the matched region already contains the real control char (so ``sep = "\\t"`` files
|
||||
are left alone). ``\\n`` is excluded: rewriting it would mangle source escape literals."""
|
||||
if "\\t" not in new_string and "\\r" not in new_string:
|
||||
return new_string
|
||||
matched_regions = _matched_regions(content, matches)
|
||||
out = new_string
|
||||
if "\\t" in out and "\t" in matched_regions:
|
||||
out = out.replace("\\t", "\t")
|
||||
if "\\r" in out and "\r" in matched_regions:
|
||||
out = out.replace("\\r", "\r")
|
||||
return out
|
||||
for literal, control in (("\\t", "\t"), ("\\r", "\r")):
|
||||
if literal in new_string and control in matched_regions:
|
||||
new_string = new_string.replace(literal, control)
|
||||
return new_string
|
||||
|
||||
|
||||
# ── Replacement shaping ──────────────────────────────────────────────────
|
||||
@@ -244,20 +460,13 @@ def _leading_whitespace(line: str) -> str:
|
||||
|
||||
|
||||
def _first_meaningful_line(text: str) -> Optional[str]:
|
||||
for line in text.split("\n"):
|
||||
if line.strip():
|
||||
return line
|
||||
return None
|
||||
return next((line for line in text.split("\n") if line.strip()), None)
|
||||
|
||||
|
||||
def _reindent_replacement(file_region: str, old_string: str, new_string: str) -> str:
|
||||
"""Re-anchor ``new_string``'s indentation onto the file's actual base indent.
|
||||
|
||||
After a non-exact match the LLM's base indent (first non-blank line of
|
||||
old_string) may differ from the file's. Each non-blank new_string line
|
||||
swaps the LLM base prefix for the file's, preserving relative nesting;
|
||||
lines shallower than the LLM base are anchored to the file base.
|
||||
"""
|
||||
"""Re-anchor ``new_string``'s indentation onto the file's actual base indent after a
|
||||
non-exact match: swap the LLM base prefix (first non-blank old_string line) for the
|
||||
file's, preserving relative nesting; shallower lines anchor to the file base."""
|
||||
if not new_string:
|
||||
return new_string
|
||||
old_first = _first_meaningful_line(old_string)
|
||||
@@ -269,7 +478,7 @@ def _reindent_replacement(file_region: str, old_string: str, new_string: str) ->
|
||||
if old_indent == file_indent:
|
||||
return new_string
|
||||
|
||||
out_lines: List[str] = []
|
||||
out_lines: list[str] = []
|
||||
for line in new_string.split("\n"):
|
||||
if not line.strip():
|
||||
out_lines.append(line)
|
||||
@@ -280,17 +489,10 @@ def _reindent_replacement(file_region: str, old_string: str, new_string: str) ->
|
||||
return "\n".join(out_lines)
|
||||
|
||||
|
||||
def _preserve_unicode_in_replacement(
|
||||
content: str, matches: List[Tuple[int, int]],
|
||||
old_string: str, new_string: str,
|
||||
) -> str:
|
||||
"""Apply only the old->new edits onto the file's original (Unicode) text.
|
||||
|
||||
After a unicode_normalized match, writing the LLM's ASCII new_string
|
||||
verbatim would flatten the file's em-dashes/smart quotes. Diff the
|
||||
normalized old_string against new_string and keep the file's original
|
||||
characters for every ``equal`` span.
|
||||
"""
|
||||
def _preserve_unicode_in_replacement(content: str, matches: list[Span],
|
||||
old_string: str, new_string: str) -> str:
|
||||
"""Apply only the old->new edits onto the file's original (Unicode) text, so a
|
||||
unicode_normalized match doesn't flatten the file's em-dashes/smart quotes."""
|
||||
file_region = _matched_regions(content, matches)
|
||||
norm_old = _unicode_normalize(old_string)
|
||||
if norm_old != _unicode_normalize(file_region):
|
||||
@@ -299,7 +501,7 @@ def _preserve_unicode_in_replacement(
|
||||
file_orig_to_norm = _build_orig_to_norm_map(file_region)
|
||||
file_norm_to_orig = _invert_norm_map(file_orig_to_norm)
|
||||
|
||||
result_parts: List[str] = []
|
||||
result_parts: list[str] = []
|
||||
for tag, i1, i2, j1, j2 in SequenceMatcher(None, norm_old, new_string).get_opcodes():
|
||||
if tag == "equal":
|
||||
orig_start = file_norm_to_orig.get(i1, 0)
|
||||
@@ -310,13 +512,10 @@ def _preserve_unicode_in_replacement(
|
||||
return "".join(result_parts)
|
||||
|
||||
|
||||
def _apply_replacements(content: str, matches: List[Tuple[int, int]],
|
||||
def _apply_replacements(content: str, matches: list[Span],
|
||||
new_string: str, old_string: Optional[str] = None) -> str:
|
||||
"""Splice ``new_string`` over each span (end-to-start so offsets stay valid).
|
||||
|
||||
``old_string`` non-None signals a non-exact match: new_string is
|
||||
re-indented per region to the file's actual indentation.
|
||||
"""
|
||||
"""Splice ``new_string`` over each span (end-to-start so offsets stay valid);
|
||||
``old_string`` non-None (non-exact match) re-indents it per region."""
|
||||
result = content
|
||||
for start, end in sorted(matches, key=lambda x: x[0], reverse=True):
|
||||
adjusted = new_string
|
||||
@@ -330,12 +529,9 @@ def _apply_replacements(content: str, matches: List[Tuple[int, int]],
|
||||
|
||||
def _visualize_whitespace(line: str) -> str:
|
||||
"""Render the leading whitespace run visibly (→ = tab, · = space)."""
|
||||
i = 0
|
||||
prefix = []
|
||||
while i < len(line) and line[i] in (" ", "\t"):
|
||||
prefix.append("→" if line[i] == "\t" else "·")
|
||||
i += 1
|
||||
return "".join(prefix) + line[i:]
|
||||
stripped = line.lstrip(" \t")
|
||||
prefix = line[:len(line) - len(stripped)]
|
||||
return prefix.replace("\t", "→").replace(" ", "·") + stripped
|
||||
|
||||
|
||||
def find_closest_lines(old_string: str, content: str, context_lines: int = 2, max_results: int = 3) -> str:
|
||||
@@ -347,25 +543,15 @@ def find_closest_lines(old_string: str, content: str, context_lines: int = 2, ma
|
||||
if not old_lines or not content_lines:
|
||||
return ""
|
||||
|
||||
anchor = old_lines[0].strip()
|
||||
anchor = old_lines[0].strip() or next((l.strip() for l in old_lines if l.strip()), "")
|
||||
if not anchor:
|
||||
candidates = [l.strip() for l in old_lines if l.strip()]
|
||||
if not candidates:
|
||||
return ""
|
||||
anchor = candidates[0]
|
||||
|
||||
scored = []
|
||||
for i, line in enumerate(content_lines):
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
ratio = SequenceMatcher(None, anchor, stripped).ratio()
|
||||
if ratio > 0.3:
|
||||
scored.append((ratio, i))
|
||||
if not scored:
|
||||
return ""
|
||||
scored.sort(key=lambda x: -x[0])
|
||||
top = scored[:max_results]
|
||||
|
||||
scored = sorted(((SequenceMatcher(None, anchor, line.strip()).ratio(), i)
|
||||
for i, line in enumerate(content_lines) if line.strip()), key=lambda x: -x[0])
|
||||
top = [s for s in scored if s[0] > 0.3][:max_results]
|
||||
if not top:
|
||||
return ""
|
||||
|
||||
parts = []
|
||||
seen_ranges = set()
|
||||
@@ -376,11 +562,7 @@ def find_closest_lines(old_string: str, content: str, context_lines: int = 2, ma
|
||||
continue
|
||||
seen_ranges.add((start, end))
|
||||
parts.append("\n".join(
|
||||
f"{start + j + 1:4d}| {content_lines[start + j]}"
|
||||
for j in range(end - start)
|
||||
))
|
||||
if not parts:
|
||||
return ""
|
||||
f"{start + j + 1:4d}| {content_lines[start + j]}" for j in range(end - start)))
|
||||
result = "\n---\n".join(parts)
|
||||
|
||||
# Whitespace-shaped miss: best line equals the anchor once stripped. Show
|
||||
@@ -391,23 +573,15 @@ def find_closest_lines(old_string: str, content: str, context_lines: int = 2, ma
|
||||
"\n\nWhitespace difference detected (→ = tab, · = space):\n"
|
||||
f" file has: {_visualize_whitespace(best_line)}\n"
|
||||
f" you sent: {_visualize_whitespace(old_lines[0])}\n"
|
||||
"Use the exact whitespace shown in 'file has'."
|
||||
)
|
||||
"Use the exact whitespace shown in 'file has'.")
|
||||
return result
|
||||
|
||||
|
||||
def format_no_match_hint(error: Optional[str], match_count: int,
|
||||
old_string: str, content: str) -> str:
|
||||
"""'\\n\\nDid you mean...' snippet for plain no-match errors only, else ''.
|
||||
|
||||
Ambiguous-match, escape-drift and identical-strings errors also have
|
||||
``match_count == 0`` but a hint would mislead there.
|
||||
"""
|
||||
if match_count != 0:
|
||||
return ""
|
||||
if not error or not error.startswith("Could not find"):
|
||||
"""'\\n\\nDid you mean...' snippet for plain no-match errors only, else '' (ambiguous /
|
||||
escape-drift / identical errors also have ``match_count == 0`` but a hint would mislead)."""
|
||||
if match_count != 0 or not error or not error.startswith("Could not find"):
|
||||
return ""
|
||||
hint = find_closest_lines(old_string, content)
|
||||
if not hint:
|
||||
return ""
|
||||
return "\n\nDid you mean one of these sections?\n" + hint
|
||||
return "\n\nDid you mean one of these sections?\n" + hint if hint else ""
|
||||
|
||||
@@ -1,334 +0,0 @@
|
||||
"""Matching strategies for :mod:`tools.fuzzy_match`.
|
||||
|
||||
Each strategy takes ``(content, pattern)`` and returns a list of
|
||||
``(start, end)`` spans in the ORIGINAL content. ``STRATEGIES`` is the ordered
|
||||
chain the orchestrator tries; ``SIMILARITY_STRATEGIES`` names the ones whose
|
||||
matches are approximate and therefore unsafe under ``replace_all``.
|
||||
"""
|
||||
|
||||
import re
|
||||
from difflib import SequenceMatcher
|
||||
from typing import Callable, Dict, List, Tuple
|
||||
|
||||
UNICODE_MAP = {
|
||||
"\u201c": '"', "\u201d": '"', # smart double quotes
|
||||
"\u2018": "'", "\u2019": "'", # smart single quotes
|
||||
"\u2014": "--", "\u2013": "-", # em/en dashes
|
||||
"\u2026": "...", "\u00a0": " ", # ellipsis and non-breaking space
|
||||
"\u2212": "-", # typographic minus (math/scientific docs)
|
||||
# Space-separator family (Zs) beyond NBSP: files with typographic spacing
|
||||
# otherwise miss every precise strategy and fall through to the
|
||||
# similarity fallback, which can pick the wrong region.
|
||||
"\u2000": " ", "\u2001": " ", "\u2002": " ", "\u2003": " ",
|
||||
"\u2004": " ", "\u2005": " ", "\u2006": " ", "\u2007": " ",
|
||||
"\u2008": " ", "\u2009": " ", "\u200a": " ", "\u202f": " ",
|
||||
"\u205f": " ", "\u3000": " ",
|
||||
}
|
||||
|
||||
Span = Tuple[int, int]
|
||||
|
||||
|
||||
def _unicode_normalize(text: str) -> str:
|
||||
"""Map typographic Unicode variants to ASCII equivalents."""
|
||||
for char, repl in UNICODE_MAP.items():
|
||||
text = text.replace(char, repl)
|
||||
return text
|
||||
|
||||
|
||||
# ── Position helpers ─────────────────────────────────────────────────────
|
||||
|
||||
def _calculate_line_positions(content_lines: List[str], start_line: int,
|
||||
end_line: int, content_length: int) -> Span:
|
||||
"""Character span covering ``content_lines[start_line:end_line]`` (end exclusive)."""
|
||||
start_pos = sum(len(line) + 1 for line in content_lines[:start_line])
|
||||
end_pos = sum(len(line) + 1 for line in content_lines[:end_line]) - 1
|
||||
return start_pos, min(content_length, end_pos)
|
||||
|
||||
|
||||
def _match_transformed_lines(content: str, pattern: str,
|
||||
transform: Callable[[str], str]) -> List[Span]:
|
||||
"""Match ``pattern`` against ``content`` after applying ``transform`` per line."""
|
||||
content_lines = content.split('\n')
|
||||
norm_lines = [transform(line) for line in content_lines]
|
||||
pattern_norm = '\n'.join(transform(line) for line in pattern.split('\n'))
|
||||
n = pattern_norm.count('\n') + 1
|
||||
matches = []
|
||||
for i in range(len(norm_lines) - n + 1):
|
||||
if '\n'.join(norm_lines[i:i + n]) == pattern_norm:
|
||||
matches.append(_calculate_line_positions(content_lines, i, i + n, len(content)))
|
||||
return matches
|
||||
|
||||
|
||||
def _build_orig_to_norm_map(original: str) -> List[int]:
|
||||
"""Map each original index to its index in ``_unicode_normalize(original)``.
|
||||
|
||||
UNICODE_MAP replacements can expand one char into several, so the map is
|
||||
needed to translate normalised spans back. Length is ``len(original)+1``;
|
||||
the last entry is a sentinel one past the final character.
|
||||
"""
|
||||
result: List[int] = []
|
||||
norm_pos = 0
|
||||
for char in original:
|
||||
result.append(norm_pos)
|
||||
repl = UNICODE_MAP.get(char)
|
||||
norm_pos += len(repl) if repl is not None else 1
|
||||
result.append(norm_pos)
|
||||
return result
|
||||
|
||||
|
||||
def _invert_norm_map(orig_to_norm: List[int]) -> Dict[int, int]:
|
||||
"""norm_pos -> first original position mapping to it."""
|
||||
inverted: Dict[int, int] = {}
|
||||
for orig_pos, norm_pos in enumerate(orig_to_norm[:-1]):
|
||||
if norm_pos not in inverted:
|
||||
inverted[norm_pos] = orig_pos
|
||||
return inverted
|
||||
|
||||
|
||||
def _norm_end_to_orig(orig_to_norm: List[int], orig_start: int, norm_end: int) -> int:
|
||||
"""Walk from ``orig_start`` until the mapped position reaches ``norm_end``."""
|
||||
orig_len = len(orig_to_norm) - 1
|
||||
orig_end = orig_start
|
||||
while orig_end < orig_len and orig_to_norm[orig_end] < norm_end:
|
||||
orig_end += 1
|
||||
return orig_end
|
||||
|
||||
|
||||
def _map_positions_norm_to_orig(orig_to_norm: List[int],
|
||||
norm_matches: List[Span]) -> List[Span]:
|
||||
"""Convert spans in the normalised string to original-string spans."""
|
||||
norm_to_orig_start = _invert_norm_map(orig_to_norm)
|
||||
results: List[Span] = []
|
||||
for norm_start, norm_end in norm_matches:
|
||||
if norm_start not in norm_to_orig_start:
|
||||
continue
|
||||
orig_start = norm_to_orig_start[norm_start]
|
||||
results.append((orig_start, _norm_end_to_orig(orig_to_norm, orig_start, norm_end)))
|
||||
return results
|
||||
|
||||
|
||||
def _map_normalized_positions(original: str, normalized: str,
|
||||
normalized_matches: List[Span]) -> List[Span]:
|
||||
"""Best-effort span mapping for ``[ \\t]+`` -> ``' '`` whitespace collapsing."""
|
||||
orig_to_norm = [] # orig_to_norm[i] = position in normalized
|
||||
orig_idx = norm_idx = 0
|
||||
while orig_idx < len(original) and norm_idx < len(normalized):
|
||||
if original[orig_idx] == normalized[norm_idx]:
|
||||
orig_to_norm.append(norm_idx)
|
||||
orig_idx += 1
|
||||
norm_idx += 1
|
||||
elif original[orig_idx] in ' \t' and normalized[norm_idx] == ' ':
|
||||
# Collapsed run: advance norm_idx only once the run is consumed.
|
||||
orig_to_norm.append(norm_idx)
|
||||
orig_idx += 1
|
||||
if orig_idx < len(original) and original[orig_idx] not in ' \t':
|
||||
norm_idx += 1
|
||||
else:
|
||||
# Extra whitespace in original, or a mismatch that normalization
|
||||
# should never produce — either way, pin to the current norm_idx.
|
||||
orig_to_norm.append(norm_idx)
|
||||
orig_idx += 1
|
||||
while orig_idx < len(original):
|
||||
orig_to_norm.append(len(normalized))
|
||||
orig_idx += 1
|
||||
|
||||
norm_to_orig_start = {}
|
||||
norm_to_orig_end = {}
|
||||
for orig_pos, norm_pos in enumerate(orig_to_norm):
|
||||
if norm_pos not in norm_to_orig_start:
|
||||
norm_to_orig_start[norm_pos] = orig_pos
|
||||
norm_to_orig_end[norm_pos] = orig_pos
|
||||
|
||||
original_matches = []
|
||||
for norm_start, norm_end in normalized_matches:
|
||||
if norm_start in norm_to_orig_start:
|
||||
orig_start = norm_to_orig_start[norm_start]
|
||||
else:
|
||||
orig_start = min(i for i, n in enumerate(orig_to_norm) if n >= norm_start)
|
||||
if norm_end - 1 in norm_to_orig_end:
|
||||
orig_end = norm_to_orig_end[norm_end - 1] + 1
|
||||
else:
|
||||
orig_end = orig_start + (norm_end - norm_start)
|
||||
# Absorb trailing collapsed whitespace only when the normalized match
|
||||
# itself ended in a space; otherwise the first whitespace after the
|
||||
# match is a word boundary that must survive (#52491).
|
||||
if norm_end < len(normalized) and normalized[norm_end - 1] == ' ':
|
||||
while orig_end < len(original) and original[orig_end] in ' \t':
|
||||
orig_end += 1
|
||||
original_matches.append((orig_start, min(orig_end, len(original))))
|
||||
return original_matches
|
||||
|
||||
|
||||
# ── Strategies ───────────────────────────────────────────────────────────
|
||||
|
||||
def _strategy_exact(content: str, pattern: str) -> List[Span]:
|
||||
"""Strategy 1: exact, non-overlapping occurrences (str.replace semantics)."""
|
||||
matches = []
|
||||
start = 0
|
||||
while True:
|
||||
pos = content.find(pattern, start)
|
||||
if pos == -1:
|
||||
break
|
||||
matches.append((pos, pos + len(pattern)))
|
||||
# Advance past the whole match: overlapping spans would corrupt the
|
||||
# file under replace_all (reverse-order apply on stale offsets).
|
||||
start = pos + len(pattern)
|
||||
return matches
|
||||
|
||||
|
||||
def _strategy_line_trimmed(content: str, pattern: str) -> List[Span]:
|
||||
"""Strategy 2: strip each line before comparing."""
|
||||
return _match_transformed_lines(content, pattern, str.strip)
|
||||
|
||||
|
||||
def _strategy_whitespace_normalized(content: str, pattern: str) -> List[Span]:
|
||||
"""Strategy 3: collapse runs of spaces/tabs to a single space."""
|
||||
def normalize(s):
|
||||
return re.sub(r'[ \t]+', ' ', s)
|
||||
|
||||
content_normalized = normalize(content)
|
||||
matches_in_normalized = _strategy_exact(content_normalized, normalize(pattern))
|
||||
if not matches_in_normalized:
|
||||
return []
|
||||
return _map_normalized_positions(content, content_normalized, matches_in_normalized)
|
||||
|
||||
|
||||
def _strategy_indentation_flexible(content: str, pattern: str) -> List[Span]:
|
||||
"""Strategy 4: ignore leading indentation entirely."""
|
||||
return _match_transformed_lines(content, pattern, str.lstrip)
|
||||
|
||||
|
||||
def _strategy_escape_normalized(content: str, pattern: str) -> List[Span]:
|
||||
"""Strategy 5: treat literal ``\\n``/``\\t``/``\\r`` in the pattern as control chars."""
|
||||
pattern_unescaped = pattern.replace('\\n', '\n').replace('\\t', '\t').replace('\\r', '\r')
|
||||
if pattern_unescaped == pattern:
|
||||
return []
|
||||
return _strategy_exact(content, pattern_unescaped)
|
||||
|
||||
|
||||
def _strategy_trimmed_boundary(content: str, pattern: str) -> List[Span]:
|
||||
"""Strategy 6: strip whitespace on the first and last lines only."""
|
||||
pattern_lines = pattern.split('\n')
|
||||
pattern_lines[0] = pattern_lines[0].strip()
|
||||
if len(pattern_lines) > 1:
|
||||
pattern_lines[-1] = pattern_lines[-1].strip()
|
||||
modified_pattern = '\n'.join(pattern_lines)
|
||||
n = len(pattern_lines)
|
||||
|
||||
content_lines = content.split('\n')
|
||||
matches = []
|
||||
for i in range(len(content_lines) - n + 1):
|
||||
check_lines = content_lines[i:i + n]
|
||||
check_lines[0] = check_lines[0].strip()
|
||||
if n > 1:
|
||||
check_lines[-1] = check_lines[-1].strip()
|
||||
if '\n'.join(check_lines) == modified_pattern:
|
||||
matches.append(_calculate_line_positions(content_lines, i, i + n, len(content)))
|
||||
return matches
|
||||
|
||||
|
||||
def _strategy_unicode_normalized(content: str, pattern: str) -> List[Span]:
|
||||
"""Strategy 7: exact/line-trimmed match after Unicode->ASCII normalisation of both sides."""
|
||||
norm_pattern = _unicode_normalize(pattern)
|
||||
norm_content = _unicode_normalize(content)
|
||||
if norm_content == content and norm_pattern == pattern:
|
||||
return []
|
||||
norm_matches = _strategy_exact(norm_content, norm_pattern)
|
||||
if not norm_matches:
|
||||
norm_matches = _strategy_line_trimmed(norm_content, norm_pattern)
|
||||
if not norm_matches:
|
||||
return []
|
||||
return _map_positions_norm_to_orig(_build_orig_to_norm_map(content), norm_matches)
|
||||
|
||||
|
||||
def _strategy_block_anchor(content: str, pattern: str) -> List[Span]:
|
||||
"""Strategy 8: anchor on first+last lines, similarity-score the middle."""
|
||||
pattern_lines = _unicode_normalize(pattern).split('\n')
|
||||
if len(pattern_lines) < 2:
|
||||
return []
|
||||
first_line = pattern_lines[0].strip()
|
||||
last_line = pattern_lines[-1].strip()
|
||||
n = len(pattern_lines)
|
||||
|
||||
# Match on normalized lines; compute offsets from the ORIGINAL lines so
|
||||
# multi-char expansions (em-dash -> '--') don't shift positions.
|
||||
norm_content_lines = _unicode_normalize(content).split('\n')
|
||||
orig_content_lines = content.split('\n')
|
||||
|
||||
potential_matches = [
|
||||
i for i in range(len(norm_content_lines) - n + 1)
|
||||
if norm_content_lines[i].strip() == first_line
|
||||
and norm_content_lines[i + n - 1].strip() == last_line
|
||||
]
|
||||
# Looser thresholds (0.10/0.30) matched unrelated blocks; these are the safe floor.
|
||||
threshold = 0.50 if len(potential_matches) == 1 else 0.70
|
||||
|
||||
matches = []
|
||||
for i in potential_matches:
|
||||
if n <= 2:
|
||||
similarity = 1.0
|
||||
else:
|
||||
content_middle = '\n'.join(norm_content_lines[i + 1:i + n - 1])
|
||||
pattern_middle = '\n'.join(pattern_lines[1:-1])
|
||||
similarity = SequenceMatcher(None, content_middle, pattern_middle).ratio()
|
||||
if similarity >= threshold:
|
||||
matches.append(_calculate_line_positions(orig_content_lines, i, i + n, len(content)))
|
||||
return matches
|
||||
|
||||
|
||||
def _strategy_context_aware(content: str, pattern: str) -> List[Span]:
|
||||
"""Strategy 9 (last resort): anchored per-line similarity, every non-blank line >= 0.80.
|
||||
|
||||
The first/last-line anchor pre-filter keeps a miss from being an
|
||||
O(file x pattern) scan; the all-lines requirement stops one coincidental
|
||||
line match from replacing an unrelated block.
|
||||
"""
|
||||
pattern_lines = pattern.split('\n')
|
||||
content_lines = content.split('\n')
|
||||
n = len(pattern_lines)
|
||||
if n > len(content_lines):
|
||||
return []
|
||||
|
||||
first_pat = pattern_lines[0].strip()
|
||||
last_pat = pattern_lines[-1].strip()
|
||||
ANCHOR_THRESHOLD = 0.80
|
||||
|
||||
def _sim(a: str, b: str) -> float:
|
||||
if a == b:
|
||||
return 1.0
|
||||
return SequenceMatcher(None, a, b).ratio()
|
||||
|
||||
matches = []
|
||||
for i in range(len(content_lines) - n + 1):
|
||||
block_lines = content_lines[i:i + n]
|
||||
if _sim(first_pat, block_lines[0].strip()) < ANCHOR_THRESHOLD:
|
||||
continue
|
||||
if _sim(last_pat, block_lines[-1].strip()) < ANCHOR_THRESHOLD:
|
||||
continue
|
||||
all_match = True
|
||||
for p_line, c_line in zip(pattern_lines, block_lines):
|
||||
p_stripped = p_line.strip()
|
||||
if p_stripped and _sim(p_stripped, c_line.strip()) < 0.80:
|
||||
all_match = False
|
||||
break
|
||||
if all_match:
|
||||
matches.append(_calculate_line_positions(content_lines, i, i + n, len(content)))
|
||||
return matches
|
||||
|
||||
|
||||
# Ordered chain: precise strategies first, similarity-based last.
|
||||
STRATEGIES: List[Tuple[str, Callable[[str, str], List[Span]]]] = [
|
||||
("exact", _strategy_exact),
|
||||
("line_trimmed", _strategy_line_trimmed),
|
||||
("whitespace_normalized", _strategy_whitespace_normalized),
|
||||
("indentation_flexible", _strategy_indentation_flexible),
|
||||
("escape_normalized", _strategy_escape_normalized),
|
||||
("trimmed_boundary", _strategy_trimmed_boundary),
|
||||
("unicode_normalized", _strategy_unicode_normalized),
|
||||
("block_anchor", _strategy_block_anchor),
|
||||
("context_aware", _strategy_context_aware),
|
||||
]
|
||||
|
||||
# Matches from these only *approximately* resemble old_string — fine for one
|
||||
# unique replacement, never safe under replace_all.
|
||||
SIMILARITY_STRATEGIES = frozenset({"block_anchor", "context_aware"})
|
||||
Reference in New Issue
Block a user