Merge branch 'simp/r3-33-M' into simp/r3-33

This commit is contained in:
Teknium
2026-09-03 01:45:19 -07:00
5 changed files with 515 additions and 781 deletions
+121 -171
View File
@@ -1,14 +1,9 @@
#!/usr/bin/env python3
"""V4A patch format parser and applier (format used by codex, cline, etc.).
*** Begin Patch / *** End Patch wrap the operations:
*** Update File: p.py then hunks: ``@@ hint @@``, `` ctx``, ``-old``, ``+new``
*** Add File: n.py then ``+`` lines; *** Delete File: o.py; *** Move File: a -> b
operations, error = parse_v4a_patch(patch_content)
result = apply_v4a_operations(operations, file_ops)
"""
"""V4A patch parser/applier (codex, cline). ``*** Begin Patch``/``*** End Patch`` wrap ops:
``*** Update File: p`` + hunks (``@@ hint @@``, `` ctx``, ``-old``, ``+new``); ``*** Add File: n``
+ ``+`` lines; ``*** Delete File: o``; ``*** Move File: a -> b``. Entry points:
``parse_v4a_patch(text) -> (ops, error)`` and ``apply_v4a_operations(ops, file_ops)``."""
import contextlib
import difflib
import inspect
import re
@@ -42,7 +37,6 @@ class PatchOperation:
file_path: str
new_path: Optional[str] = None # MOVE only
hunks: List[Hunk] = field(default_factory=list)
content: Optional[str] = None # ADD only
# Markers must occupy the whole line at column 0 so content lines that merely
@@ -58,12 +52,9 @@ _HINT_RE = re.compile(r'@@\s*(.+?)\s*@@')
def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[str]]:
"""Parse a V4A patch -> ``(operations, None)`` (``[]`` for an empty patch is not an
error) or ``([], "Parse error: ...")`` for malformed operations."""
# Tolerate CRLF bodies: a stray ``\r`` would otherwise end up in every
# HunkLine.content and defeat the anchored Begin/End markers.
"""-> ``(operations, None)`` (empty patch = ``[]``, no error) or ``([], "Parse error: …")``."""
# Tolerate CRLF: a stray ``\r`` would land in every HunkLine.content and defeat the markers.
lines = [ln[:-1] if ln.endswith('\r') else ln for ln in patch_content.split('\n')]
start_idx = -1 # parse from the top when no Begin marker is present
end_idx = len(lines)
for i, line in enumerate(lines):
@@ -72,7 +63,6 @@ def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[
elif _END_MARKER.match(line):
end_idx = i
break
operations: List[PatchOperation] = []
current_op: Optional[PatchOperation] = None
current_hunk: Optional[Hunk] = None
@@ -87,8 +77,7 @@ def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[
operations.append(current_op)
for line in lines[start_idx + 1:end_idx]:
op_match = next(
((kind, m) for kind, rx in _OP_MARKERS if (m := rx.match(line))), None)
op_match = next(((kind, m) for kind, rx in _OP_MARKERS if (m := rx.match(line))), None)
if op_match:
kind, m = op_match
_flush()
@@ -96,8 +85,8 @@ def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[
operation=kind,
file_path=m.group(1).strip(),
new_path=m.group(2).strip() if kind is OperationType.MOVE else None)
# UPDATE hunks start lazily (at '@@' or the first hunk line); ADD
# collects all '+' lines into one hunk; DELETE/MOVE are complete.
# UPDATE hunks start lazily ('@@' or first hunk line); ADD collects all '+' lines
# into one hunk; DELETE/MOVE are complete.
current_hunk = Hunk() if kind is OperationType.ADD else None
if kind in (OperationType.DELETE, OperationType.MOVE):
operations.append(current_op)
@@ -115,18 +104,16 @@ def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[
elif line[0] != '\\': # "\ No newline at end of file" marker is skipped
current_hunk.lines.append(HunkLine(' ', line)) # implicit context line
_flush()
parse_errors: List[str] = []
for op in operations:
if not op.file_path:
parse_errors.append("Operation with empty file path")
if op.operation == OperationType.UPDATE and not op.hunks:
if op.operation is OperationType.UPDATE and not op.hunks:
parse_errors.append(f"UPDATE {op.file_path!r}: no hunks found")
if op.operation == OperationType.MOVE and not op.new_path:
parse_errors.append(f"MOVE {op.file_path!r}: missing destination path (expected 'src -> dst')")
if parse_errors:
return [], "Parse error: " + "; ".join(parse_errors)
return operations, None
if op.operation is OperationType.MOVE and not op.new_path:
parse_errors.append(
f"MOVE {op.file_path!r}: missing destination path (expected 'src -> dst')")
return ([], "Parse error: " + "; ".join(parse_errors)) if parse_errors else (operations, None)
def _count_occurrences(text: str, pattern: str) -> int:
@@ -136,66 +123,57 @@ def _count_occurrences(text: str, pattern: str) -> int:
def _split_hunk(hunk: Hunk) -> Tuple[List[str], List[str]]:
"""``(search_lines, replace_lines)``: context+removed vs context+added."""
search = [l.content for l in hunk.lines if l.prefix in {' ', '-'}]
replace = [l.content for l in hunk.lines if l.prefix in {' ', '+'}]
return search, replace
return ([l.content for l in hunk.lines if l.prefix != '+'],
[l.content for l in hunk.lines if l.prefix != '-'])
def _no_match_hint(error: Optional[str], search_pattern: str, content: str) -> str:
"""Best-effort 'Did you mean...' suffix; never lets a hint failure mask the real error."""
try:
with contextlib.suppress(Exception):
from tools.fuzzy_match import format_no_match_hint
return format_no_match_hint(error, 0, search_pattern, content)
except Exception:
return ""
return ""
def _hint_ambiguity(content: str, hint: str, tail: str = "") -> Tuple[int, str]:
"""(occurrences, error) for an addition-only hunk's context hint; error is '' when unique."""
occurrences = _count_occurrences(content, hint)
if occurrences > 1:
return occurrences, (f"context hint '{hint}' is ambiguous "
f"({occurrences} occurrences){tail}")
return occurrences, ""
n = _count_occurrences(content, hint)
return n, f"context hint '{hint}' is ambiguous ({n} occurrences){tail}" if n > 1 else ""
def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> List[str]:
"""Dry-run every operation; return error strings (empty list = safe to apply). UPDATE
hunks are simulated in order so later hunks see post-earlier-hunk content, as apply will."""
"""Dry-run every operation -> error strings (empty = safe). UPDATE hunks are simulated in
order so later hunks see post-earlier-hunk content, exactly as apply will."""
from tools.fuzzy_match import fuzzy_find_and_replace, is_already_applied
errors: List[str] = []
real_change_count = 0
# Virtual overlay so inter-op state validates (e.g. a MOVE creating the destination
# a later UPDATE targets): path -> content from an earlier op; paths MOVE/DELETE removed.
# Overlay so inter-op state validates (a MOVE creating the path a later UPDATE targets).
pending_content: dict = {}
removed_paths: set = set()
def _read(path: str):
if path in removed_paths and path not in pending_content:
return None, "file not found"
def _read(path: str) -> Tuple[Optional[str], Optional[str]]:
if path in pending_content:
return pending_content[path], None
if path in removed_paths:
return None, "file not found"
r = file_ops.read_file_raw(path)
return (None, r.error) if r.error else (r.content, None)
def _validate_update(op: PatchOperation) -> None:
nonlocal real_change_count
content, read_err = _read(op.file_path)
simulated, read_err = _read(op.file_path)
if read_err:
errors.append(f"{op.file_path}: {read_err}")
return
simulated = content
for hunk_index, hunk in enumerate(op.hunks, start=1):
search_lines, replace_lines = _split_hunk(hunk)
if not any(l.prefix in '-+' for l in hunk.lines):
# Inert anchor hunk (context only) — models emit these
# between real changes; ignore without failing the patch.
if search_lines == replace_lines:
# Context-only anchor hunks (models emit these between changes) are inert; identical
# -/+ lines are skipped by apply as a no-op — neither may fail validation.
real_change_count += any(l.prefix in '-+' for l in hunk.lines)
continue
real_change_count += 1
if not search_lines:
# Addition-only hunk: the context hint must be unique.
if not search_lines: # addition-only: the context hint must be unique
if hunk.context_hint:
occurrences, ambiguous = _hint_ambiguity(simulated, hunk.context_hint)
if occurrences == 0:
@@ -204,19 +182,13 @@ def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> Lis
elif ambiguous:
errors.append(f"{op.file_path}: addition-only hunk {ambiguous}")
continue
search_pattern = '\n'.join(search_lines)
replacement = '\n'.join(replace_lines)
if search_lines == replace_lines:
# Identical -/+ lines: apply skips it as a no-op, so
# validation must not reject it with the identical-strings error.
continue
search_pattern, replacement = '\n'.join(search_lines), '\n'.join(replace_lines)
new_simulated, count, _strategy, match_error = fuzzy_find_and_replace(
simulated, search_pattern, replacement, replace_all=False)
if count:
simulated = new_simulated
elif not is_already_applied(simulated or "", search_pattern, replacement):
# Already-applied hunks (edit landed in a prior call) are no-ops so
# multi-hunk patches don't fail wholesale; apply performs the same skip.
# Already-applied hunks are no-ops (apply performs the same skip).
label = f"'{hunk.context_hint}'" if hunk.context_hint else "(no hint)"
errors.append(
f"{op.file_path}: hunk {hunk_index} {label} not found"
@@ -224,18 +196,20 @@ def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> Lis
+ _no_match_hint(match_error, search_pattern, simulated))
pending_content[op.file_path] = simulated
def _remove(path: str) -> None:
removed_paths.add(path)
pending_content.pop(path, None)
for op in operations:
if op.operation == OperationType.UPDATE:
_validate_update(op)
continue
real_change_count += 1
if op.operation == OperationType.DELETE:
_content, read_err = _read(op.file_path)
if read_err:
if _read(op.file_path)[1]:
errors.append(f"{op.file_path}: file not found for deletion")
else:
removed_paths.add(op.file_path)
pending_content.pop(op.file_path, None)
_remove(op.file_path)
elif op.operation == OperationType.MOVE:
if not op.new_path:
errors.append(f"{op.file_path}: MOVE operation missing destination path")
@@ -243,16 +217,12 @@ def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> Lis
src_content, src_err = _read(op.file_path)
if src_err:
errors.append(f"{op.file_path}: source file not found for move")
_dst, dst_err = _read(op.new_path)
if not dst_err:
if not _read(op.new_path)[1]:
errors.append(f"{op.new_path}: destination already exists — move would overwrite")
# Only a cleanly-validated move updates the overlay.
if not src_err and dst_err:
elif not src_err: # only a cleanly-validated move updates the overlay
pending_content[op.new_path] = src_content if src_content is not None else ""
pending_content.pop(op.file_path, None)
removed_paths.add(op.file_path)
# ADD: parent directory creation handled by write_file; no pre-check needed.
_remove(op.file_path)
# ADD: write_file creates parent directories; no pre-check needed.
if not errors and real_change_count == 0:
errors.append("Patch contains no changes (only context lines were provided)")
return errors
@@ -262,104 +232,101 @@ def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> Lis
ApplyResult = Tuple[bool, str, Optional[str], Optional[dict]]
def _fail(error: str) -> ApplyResult:
return False, error, None, None
def _written(result: Any, diff: str) -> ApplyResult:
"""Outcome of a write: its error, else success with LSP/lint propagated from the WriteResult."""
if result.error:
return _fail(result.error)
return True, diff, getattr(result, "lsp_diagnostics", None), getattr(result, "lint", None)
def _unified_diff(path: str, old: str, new: Optional[str]) -> str:
"""Unified diff ``a/path`` -> ``b/path`` (``new=None`` = deletion, ``/dev/null``)."""
return ''.join(difflib.unified_diff(
old.splitlines(keepends=True), [] if new is None else new.splitlines(keepends=True),
fromfile=f"a/{path}", tofile="/dev/null" if new is None else f"b/{path}"))
def apply_v4a_operations(operations: List[PatchOperation], file_ops: Any) -> 'PatchResult':
"""Validate all operations, then apply them (two-phase, atomic on validation failure).
A phase-2 failure (validate/apply race) is reported with a ``git diff`` note since state
may be inconsistent. ``file_ops`` needs read_file_raw/write_file/delete_file/move_file."""
"""Two-phase: validate everything, then apply (atomic on validation failure). A phase-2
failure (validate/apply race) carries a ``git diff`` note since state may be inconsistent.
``file_ops`` needs read_file_raw/write_file/delete_file/move_file."""
from tools.file_operations import PatchResult # avoid circular import
validation_errors = _validate_operations(operations, file_ops)
if validation_errors:
def _bullets(errs: List[str]) -> str:
return "\n".join(f" • {e}" for e in errs)
if errors := _validate_operations(operations, file_ops):
return PatchResult(
success=False,
error="Patch validation failed (no files were modified):\n"
+ "\n".join(f" • {e}" for e in validation_errors))
error="Patch validation failed (no files were modified):\n" + _bullets(errors))
files: Dict[str, List[str]] = {"created": [], "deleted": [], "modified": []}
all_diffs: List[str] = []
# V4A bypasses the WriteResult/PatchResult plumbing that write_file uses,
# so LSP diagnostics and lint must be propagated explicitly per file.
# V4A bypasses write_file's WriteResult plumbing: LSP diagnostics and lint propagate per file.
lsp_blocks: List[str] = []
errors: List[str] = []
lint_results: Dict[str, dict] = {}
for op in operations:
handler, verb, bucket = _APPLY_DISPATCH[op.operation]
try:
handler, verb, bucket = _APPLY_DISPATCH[op.operation]
ok, payload, lsp, lint = handler(op, file_ops)
if not ok:
errors.append(f"Failed to {verb} {op.file_path}: {payload}")
continue
label = op.file_path
if op.operation is OperationType.MOVE:
label = f"{op.file_path} -> {op.new_path}"
files[bucket].append(label)
all_diffs.append(payload)
if lsp:
lsp_blocks.append(lsp)
if lint:
lint_results[op.file_path] = lint
except Exception as e:
errors.append(f"Error processing {op.file_path}: {str(e)}")
# Each LSP block carries its own <diagnostics file="..."> header, so plain
# concatenation keeps per-file attribution.
result_kwargs = dict(
ok, payload = None, str(e)
if not ok:
prefix = f"Failed to {verb}" if ok is False else "Error processing"
errors.append(f"{prefix} {op.file_path}: {payload}")
continue
is_move = op.operation is OperationType.MOVE
files[bucket].append(f"{op.file_path} -> {op.new_path}" if is_move else op.file_path)
all_diffs.append(payload)
if lsp:
lsp_blocks.append(lsp)
if lint:
lint_results[op.file_path] = lint
# Each LSP block carries its own <diagnostics file="..."> header; joining keeps attribution.
return PatchResult(
success=not errors,
error=("Apply phase failed (state may be inconsistent — run `git diff` to assess):\n"
+ _bullets(errors)) if errors else None,
diff='\n'.join(all_diffs),
files_modified=files["modified"], files_created=files["created"], files_deleted=files["deleted"],
lint=lint_results if lint_results else None,
lsp_diagnostics="\n\n".join(lsp_blocks) if lsp_blocks else None)
if errors:
return PatchResult(
success=False,
error="Apply phase failed (state may be inconsistent — run `git diff` to assess):\n"
+ "\n".join(f" • {e}" for e in errors),
**result_kwargs)
return PatchResult(success=True, **result_kwargs)
lint=lint_results or None, lsp_diagnostics="\n\n".join(lsp_blocks) or None)
def _write_file_accepts_pre_content(file_ops: Any) -> bool:
"""True when ``file_ops.write_file`` accepts ``pre_content``. Decided from the signature,
not by catching TypeError around the call, so a TypeError raised *inside* a capable
write_file propagates instead of triggering a duplicate write."""
try:
"""Whether ``file_ops.write_file`` accepts ``pre_content`` — read from the signature, not by
catching TypeError around the call, so a TypeError raised *inside* it can't double-write."""
with contextlib.suppress(TypeError, ValueError):
params = inspect.signature(file_ops.write_file).parameters
except (TypeError, ValueError):
return False
return "pre_content" in params or any(
p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values())
return "pre_content" in params or any(
p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values())
return False
def _apply_add(op: PatchOperation, file_ops: Any) -> ApplyResult:
"""Create a file from the hunks' '+' lines."""
content_lines = [line.content for hunk in op.hunks for line in hunk.lines if line.prefix == '+']
result = file_ops.write_file(op.file_path, '\n'.join(content_lines))
if result.error:
return False, result.error, None, None
diff = f"--- /dev/null\n+++ b/{op.file_path}\n" + '\n'.join(f"+{line}" for line in content_lines)
return True, diff, getattr(result, "lsp_diagnostics", None), getattr(result, "lint", None)
return _written(result, diff)
def _apply_delete(op: PatchOperation, file_ops: Any) -> ApplyResult:
"""Delete a file, producing a real unified diff of the removed content."""
# Validation already confirmed existence; the re-read guards against races.
read_result = file_ops.read_file_raw(op.file_path)
read_result = file_ops.read_file_raw(op.file_path) # re-read guards validate/apply races
if read_result.error:
return False, f"Cannot delete {op.file_path}: file not found", None, None
return _fail(f"Cannot delete {op.file_path}: file not found")
result = file_ops.delete_file(op.file_path)
if result.error:
return False, result.error, None, None
diff = ''.join(difflib.unified_diff(
read_result.content.splitlines(keepends=True), [],
fromfile=f"a/{op.file_path}", tofile="/dev/null"))
return True, diff or f"# Deleted: {op.file_path}", None, None
diff = _unified_diff(op.file_path, read_result.content, None) or f"# Deleted: {op.file_path}"
return _fail(result.error) if result.error else (True, diff, None, None)
def _apply_move(op: PatchOperation, file_ops: Any) -> ApplyResult:
result = file_ops.move_file(op.file_path, op.new_path)
if result.error:
return False, result.error, None, None
return True, f"# Moved: {op.file_path} -> {op.new_path}", None, None
return _fail(result.error) if result.error else (
True, f"# Moved: {op.file_path} -> {op.new_path}", None, None)
def _insert_addition_only(new_content: str, hunk: Hunk, insert_text: str) -> Tuple[Optional[str], Optional[str]]:
@@ -371,71 +338,54 @@ def _insert_addition_only(new_content: str, hunk: Hunk, insert_text: str) -> Tup
return None, f"Addition-only hunk: {ambiguous}"
if occurrences == 1:
eol = new_content.find('\n', new_content.find(hunk.context_hint))
if eol != -1:
return new_content[:eol + 1] + insert_text + '\n' + new_content[eol + 1:], None
return new_content + '\n' + insert_text, None
# Hint not found — append at end as a safe fallback.
if eol == -1:
return new_content + '\n' + insert_text, None
return new_content[:eol + 1] + insert_text + '\n' + new_content[eol + 1:], None
# No hint / hint not found — append at end as a safe fallback.
return new_content.rstrip('\n') + '\n' + insert_text + '\n', None
def _apply_update(op: PatchOperation, file_ops: Any) -> ApplyResult:
"""Apply each hunk via fuzzy replace, then write once."""
from tools.fuzzy_match import fuzzy_find_and_replace, is_already_applied
# Raw read: no line-number prefixes or per-line truncation.
read_result = file_ops.read_file_raw(op.file_path)
read_result = file_ops.read_file_raw(op.file_path) # raw: no line numbers / truncation
if read_result.error:
return False, f"Cannot read file: {read_result.error}", None, None
current_content = read_result.content
new_content = current_content
return _fail(f"Cannot read file: {read_result.error}")
current_content = new_content = read_result.content
for hunk in op.hunks:
search_lines, replace_lines = _split_hunk(hunk)
if search_lines and search_lines == replace_lines:
continue
search_pattern, replacement = '\n'.join(search_lines), '\n'.join(replace_lines)
if not search_lines:
new_content, err = _insert_addition_only(new_content, hunk, '\n'.join(replace_lines))
new_content, err = _insert_addition_only(new_content, hunk, replacement)
if err:
return False, err, None, None
return _fail(err)
continue
search_pattern = '\n'.join(search_lines)
replacement = '\n'.join(replace_lines)
new_content, count, _strategy, error = fuzzy_find_and_replace(
new_content, search_pattern, replacement, replace_all=False)
if not (error and count == 0):
continue
# Retry inside a window around the context hint, if any.
hint_pos = new_content.find(hunk.context_hint) if hunk.context_hint else -1
if hint_pos != -1:
window_start = max(0, hint_pos - 500)
window_end = min(len(new_content), hint_pos + 2000)
window_new, count, _strategy, error = fuzzy_find_and_replace(
new_content[window_start:window_end], search_pattern, replacement, replace_all=False
)
new_content[window_start:window_end], search_pattern, replacement, replace_all=False)
if count > 0:
new_content = new_content[:window_start] + window_new + new_content[window_end:]
error = None
if error:
# Mirror the validation-phase already-applied skip, or the two
# phases disagree and the whole patch fails here.
# Mirror validation's already-applied skip, else the two phases disagree and fail here.
if is_already_applied(new_content, search_pattern, replacement):
continue
return False, f"Could not apply hunk: {error}" + _no_match_hint(error, search_pattern, new_content), None, None
hint = _no_match_hint(error, search_pattern, new_content)
return _fail(f"Could not apply hunk: {error}" + hint)
# Pass pre_content to skip a redundant re-read inside write_file when supported.
if _write_file_accepts_pre_content(file_ops):
write_result = file_ops.write_file(op.file_path, new_content, pre_content=current_content)
else:
write_result = file_ops.write_file(op.file_path, new_content)
if write_result.error:
return False, write_result.error, None, None
diff = ''.join(difflib.unified_diff(
current_content.splitlines(keepends=True), new_content.splitlines(keepends=True),
fromfile=f"a/{op.file_path}", tofile=f"b/{op.file_path}"))
return True, diff, getattr(write_result, "lsp_diagnostics", None), getattr(write_result, "lint", None)
extra = {"pre_content": current_content} if _write_file_accepts_pre_content(file_ops) else {}
write_result = file_ops.write_file(op.file_path, new_content, **extra)
return _written(write_result, _unified_diff(op.file_path, current_content, new_content))
# operation -> (handler, verb for error text, files_* bucket)
+123 -194
View File
@@ -1,16 +1,14 @@
"""Stdlib document-to-text extraction for ``read_file``.
Jupyter, DOCX and XLSX need no dependencies. The optional ``firecrawl-anydoc`` package
(imports as ``anydoc``) widens coverage to legacy Office, OpenDocument, RTF, EPUB and PDF.
The stdlib extractors stay authoritative for their three formats so behavior is identical
with or without anydoc. Malformed documents raise :class:`ExtractionError`; callers then
fall back to normal text/binary handling.
"""
"""Document-to-text extraction for ``read_file``: stdlib Jupyter/DOCX/XLSX (always
authoritative for those three), plus legacy Office/OpenDocument/RTF/EPUB/PDF when the
optional ``firecrawl-anydoc`` package (imports as ``anydoc``) is installed. Malformed
documents raise :class:`ExtractionError`; callers fall back to text/binary handling."""
from __future__ import annotations
import contextlib
import functools
import importlib
import itertools
import json
import os
import posixpath
@@ -29,12 +27,10 @@ __all__ = ["EXTRACTABLE_EXTENSIONS", "ExtractionError", "extract_document_bytes"
"extract_document_text", "is_extractable_document"]
EXTRACTABLE_EXTENSIONS = frozenset({".ipynb", ".docx", ".xlsx"})
# Formats handled only when the optional anydoc converter is installed.
ANYDOC_EXTENSIONS = frozenset({
".doc", ".docm", ".ppt", ".pps", ".pot", ".pptx", ".pptm", ".ppsx", ".ppsm",
".xls", ".xlsm", ".xlsb", ".odt", ".ods", ".odp", ".rtf", ".epub", ".pdf"})
# anydoc loads whole files with no streaming and the read_file char budget only applies
# after conversion — cap the input size.
# anydoc loads whole files (no streaming); read_file's char budget applies only post-conversion.
MAX_ANYDOC_BYTES = 50 * 1024 * 1024
MAX_DOCUMENT_BYTES = 50 * 1024 * 1024
_MAX_XLSX_ROWS_PER_SHEET = 5000
@@ -59,15 +55,15 @@ def _extension(path: str) -> str:
_ANYDOC_UNSET = object()
_anydoc_module: Any = _ANYDOC_UNSET
_anydoc_lock = threading.Lock()
# After a failed load, wait this long before retrying: the attempt can shell out
# to pip, so retrying every call would hammer the network where install can't succeed.
# Cooldown after a failed load: the attempt can shell out to pip, so retrying every call would
# hammer the network where install can't succeed.
ANYDOC_RETRY_SECONDS = 300.0
_anydoc_failed_at: Optional[float] = None
def _anydoc() -> Optional[Any]:
"""Lazily import the optional anydoc converter; None when unavailable. A failed load is
retried after ANYDOC_RETRY_SECONDS so one transient pip/network blip does not stick."""
"""Lazily import the optional anydoc converter (None when unavailable; failures retried after
ANYDOC_RETRY_SECONDS so one transient pip/network blip does not stick)."""
global _anydoc_module, _anydoc_failed_at
if _anydoc_module is not _ANYDOC_UNSET:
return _anydoc_module
@@ -79,8 +75,7 @@ def _anydoc() -> Optional[Any]:
return None
try:
from tools.lazy_deps import ensure as _lazy_ensure
# prompt=False: read_file must never block on an install prompt.
_lazy_ensure("tool.doc_extract", prompt=False)
_lazy_ensure("tool.doc_extract", prompt=False) # read_file must never block on a prompt
_anydoc_module = importlib.import_module("anydoc")
except Exception: # install failure, ImportError or a broken native binding
_anydoc_failed_at = time.monotonic()
@@ -101,23 +96,19 @@ def _check_size(size: int, limit: int) -> None:
@contextlib.contextmanager
def _temp_copy(data: bytes, suffix: str) -> Iterator[str]:
"""Materialize backend bytes in a private host temp file; removed even when parsing fails."""
temp_path = ""
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh:
fh.write(data)
try:
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh:
fh.write(data)
temp_path = fh.name
yield temp_path
yield fh.name
finally:
if temp_path:
with contextlib.suppress(OSError):
os.unlink(temp_path)
with contextlib.suppress(OSError):
os.unlink(fh.name)
def extract_document_text(path: str) -> str:
ext = _extension(path)
extractor = _STDLIB_EXTRACTORS.get(ext)
if extractor is not None:
return extractor(path)
if ext in _STDLIB_EXTRACTORS:
return _STDLIB_EXTRACTORS[ext](path)
if ext in ANYDOC_EXTENSIONS:
return _extract_anydoc(path)
raise ExtractionError(f"Unsupported document type: {path!r}")
@@ -131,14 +122,12 @@ def extract_document_bytes(data: bytes, path: str) -> str:
return _extract_anydoc_bytes(data, path)
if ext not in EXTRACTABLE_EXTENSIONS:
raise ExtractionError(f"Unsupported document type: {path!r}")
# The stdlib extractors are path-oriented.
with _temp_copy(data, ext) as temp_path:
return extract_document_text(temp_path)
with _temp_copy(data, ext) as temp_path: # the stdlib extractors are path-oriented
return _STDLIB_EXTRACTORS[ext](temp_path)
def _anydoc_missing_error(path: str) -> str:
"""Teaching error for anydoc-gated formats (deliberately absent from the schema so only
sessions that hit one pay for it)."""
"""Teaching text for anydoc-gated formats (not in the schema: only sessions hitting one pay)."""
return (
f"Cannot convert {path!r}: this format needs the optional anydoc "
"converter, which is not installed (install blocked or first "
@@ -149,10 +138,10 @@ def _anydoc_missing_error(path: str) -> str:
def _hosted_ocr_config() -> tuple:
"""Resolve hosted-OCR settings: (enabled, api_key, api_url). Never raises; no network.
Maintainer decision: the ONLY route is a direct ``FIRECRAWL_API_KEY`` (anydoc defaults the
api_url); the Nous gateway is NOT used — its Parse proxy live-probed broken (revisit when
it grows Parse support). ``file_tools.hosted_ocr: false`` disables even with a key."""
"""(enabled, api_key, api_url); never raises, no network. Maintainer decision: the ONLY route
is a direct ``FIRECRAWL_API_KEY`` (anydoc defaults api_url); the Nous gateway's Parse proxy
live-probed broken, so it is NOT used. ``file_tools.hosted_ocr: false`` disables even with a
key."""
api_key = os.environ.get("FIRECRAWL_API_KEY") or None
enabled = api_key is not None
with contextlib.suppress(Exception):
@@ -164,35 +153,30 @@ def _hosted_ocr_config() -> tuple:
def hosted_ocr_available() -> bool:
"""Public probe for read_file's schema line; a key failing at conversion time lands in
the NEEDS-OCR warning instead."""
"""Probe for read_file's schema line; a key failing at conversion time surfaces in NEEDS-OCR."""
return _hosted_ocr_config()[0]
def _needs_ocr_warning(path: str, pages, hosted_error: str = "") -> str:
"""Result text when anydoc raises NeedsOcrError and hosted OCR is off/failed. Hints at
CHECKING for an OCR skill (never names one) and never advertises the hosted_ocr knob."""
"""NeedsOcrError result when hosted OCR is off/failed; hints at CHECKING for an OCR skill
(never names one) and never advertises the hosted_ocr knob."""
page_list = ", ".join(str(p) for p in pages) if pages else "unknown"
msg = (
hosted = f"Hosted OCR was attempted and failed ({hosted_error}). " if hosted_error else ""
return (
f"[NEEDS OCR: pages {page_list} of this PDF are scanned images "
"with no text layer — their content is MISSING below. ")
if hosted_error:
msg += f"Hosted OCR was attempted and failed ({hosted_error}). "
msg += (
f"with no text layer — their content is MISSING below. {hosted}"
"If the missing pages matter: render just those pages with "
f"`pdftoppm -jpeg -r 150 -f <first> -l <last> '{path}' /tmp/page` "
"and inspect via vision_analyze, or check whether an OCR skill is "
"available (skills_list).")
return msg + "]\n"
"available (skills_list).]\n")
def _finalize_anydoc_text(text: Any, path: str, pdf_note: Callable[[], str]) -> str:
"""Normalize converter output and, for PDFs, PREPEND the coverage note (read_file
paginates: a footer may never be fetched). Covers PARTIAL gaps without NeedsOcrError."""
"""Normalize converter output; PDFs get the coverage note PREPENDED (read_file paginates, so a
footer may never be fetched) — this covers PARTIAL scan gaps that raise no NeedsOcrError."""
if not isinstance(text, str) or not text.strip():
raise ExtractionError("Document contains no extractable text")
note = pdf_note() if Path(path).suffix.lower() == ".pdf" else ""
return (note or "") + text.rstrip("\n") + "\n"
return (pdf_note() if Path(path).suffix.lower() == ".pdf" else "") + text.rstrip("\n") + "\n"
def _ocr_scanned_pdf(mod: Any, path: str, exc: BaseException) -> str:
@@ -206,40 +190,33 @@ def _ocr_scanned_pdf(mod: Any, path: str, exc: BaseException) -> str:
return mod.to_markdown(path, ocr="hosted", **extra).rstrip("\n") + "\n"
except Exception as hosted_exc: # noqa: BLE001
hosted_error = f"{type(hosted_exc).__name__}: {hosted_exc}"
# No route / disabled / hosted failed: whole doc is scans — the warning IS the result.
return _needs_ocr_warning(path, pages, hosted_error)
def _require_anydoc(path: str) -> Any:
mod = _anydoc()
if mod is None:
raise ExtractionError(_anydoc_missing_error(path))
return mod
return _needs_ocr_warning(path, pages, hosted_error) # whole doc is scans: the warning IS it
def _extract_anydoc(path: str) -> str:
mod = _require_anydoc(path)
try:
size = os.path.getsize(path)
except OSError as exc:
raise ExtractionError(str(exc)) from exc
_check_size(size, MAX_ANYDOC_BYTES)
mod = _anydoc()
if mod is None:
raise ExtractionError(_anydoc_missing_error(path))
try:
_check_size(os.path.getsize(path), MAX_ANYDOC_BYTES)
text = mod.to_markdown(path)
except ExtractionError:
raise
except OSError as exc:
raise ExtractionError(str(exc)) from exc
except Exception as exc:
needs_ocr = getattr(mod, "NeedsOcrError", None)
if needs_ocr is not None and isinstance(exc, needs_ocr):
return _ocr_scanned_pdf(mod, path, exc)
# anydoc raises one ConvertError subclass per failure mode (Unsupported, Malformed,
# Encrypted, ResourceLimit, MissingPart); all mean "no meaningful text".
# Any ConvertError subclass (Unsupported/Malformed/Encrypted/...) = "no meaningful text".
raise ExtractionError(f"{type(exc).__name__}: {exc}") from exc
return _finalize_anydoc_text(text, path, lambda: _pdf_coverage_note(path))
def _extract_anydoc_bytes(data: bytes, path: str) -> str:
mod = _require_anydoc(path)
mod = _anydoc()
if mod is None:
raise ExtractionError(_anydoc_missing_error(path))
_check_size(len(data), MAX_ANYDOC_BYTES)
try:
text = mod.to_markdown_bytes(data)
@@ -248,17 +225,13 @@ def _extract_anydoc_bytes(data: bytes, path: str) -> str:
return _finalize_anydoc_text(text, path, lambda: _pdf_coverage_note_from_bytes(data, path))
# ── Scanned-PDF coverage detection: text-layer extractors return nothing for scanned
# pages, so a mostly-scanned PDF converts "successfully" into headers with empty bodies —
# silent data loss. Count per-page text via pdftotext (form-feed separated) and warn.
# ── Scanned-PDF coverage: text-layer extractors return nothing for scanned pages, so a mostly
# scanned PDF converts "successfully" into silent data loss. Count per-page text via pdftotext.
PDF_EMPTY_PAGE_CHARS = 20 # fewer extracted chars than this = empty page
# Warn when empty pages reach both MIN_EMPTY and MIN_RATIO, or ABSOLUTE_EMPTY alone.
PDF_COVERAGE_MIN_EMPTY = 2
PDF_COVERAGE_MIN_RATIO = 0.2
PDF_COVERAGE_ABSOLUTE_EMPTY = 10
PDF_COVERAGE_MIN_EMPTY, PDF_COVERAGE_MIN_RATIO, PDF_COVERAGE_ABSOLUTE_EMPTY = 2, 0.2, 10
PDF_PAGE_SCAN_TIMEOUT = 20.0
# Cap the per-gap breakdown so alternating text/scan pages can't balloon the warning.
PDF_GAP_MAP_MAX_ENTRIES = 20
PDF_GAP_MAP_MAX_ENTRIES = 20 # cap so alternating text/scan pages can't balloon the warning
_GAP_CONTEXT_CHARS = 60
@@ -279,14 +252,10 @@ def _pdf_page_texts(path: str) -> Optional[list[str]]:
def _gap_map(counts: list[int], texts: list[str], empty: list[int]) -> str:
"""Per-gap breakdown, each empty range labeled with the last text seen before
it (usually a section header), so the agent can pick WHICH gaps to OCR."""
ranges: list[list[int]] = [] # sorted 1-based page numbers -> [start, end] runs
for p in empty:
if ranges and p == ranges[-1][1] + 1:
ranges[-1][1] = p
else:
ranges.append([p, p])
"""Per-gap breakdown labeled with the text before each gap, so the agent picks which to OCR."""
# Sorted 1-based page numbers -> (start, end) runs; consecutive pages share ``page - index``.
runs = [list(g) for _k, g in itertools.groupby(enumerate(empty), lambda e: e[1] - e[0])]
ranges = [(run[0][1], run[-1][1]) for run in runs]
lines: list[str] = []
for a, b in ranges[:PDF_GAP_MAP_MAX_ENTRIES]:
label = ""
@@ -305,17 +274,17 @@ def _gap_map(counts: list[int], texts: list[str], empty: list[int]) -> str:
def _pdf_coverage_note(path: str, display_path: Optional[str] = None) -> str:
"""Warning header when many PDF pages produced no text, else ''. ``path`` is scanned
(may be a host temp file); ``display_path`` is what the recovery command shows."""
"""Warning header when many pages yielded no text, else ''. ``display_path`` (default ``path``,
which may be a host temp file) is what the recovery command shows."""
texts = _pdf_page_texts(path)
if not texts or len(texts) < 2:
return ""
counts = [len(page.strip()) for page in texts]
empty = [i + 1 for i, n in enumerate(counts) if n < PDF_EMPTY_PAGE_CHARS]
total = len(counts)
if len(empty) < PDF_COVERAGE_MIN_EMPTY or (
len(empty) / total < PDF_COVERAGE_MIN_RATIO and len(empty) < PDF_COVERAGE_ABSOLUTE_EMPTY
):
n_empty = len(empty)
enough = n_empty / total >= PDF_COVERAGE_MIN_RATIO or n_empty >= PDF_COVERAGE_ABSOLUTE_EMPTY
if n_empty < PDF_COVERAGE_MIN_EMPTY or not enough:
return ""
shown = display_path or path
return (
@@ -335,13 +304,10 @@ def _pdf_coverage_note(path: str, display_path: Optional[str] = None) -> str:
def _pdf_coverage_note_from_bytes(data: bytes, display_path: str) -> str:
"""Coverage note for backend-transferred PDF bytes via a host temp copy (pdftotext is
path-oriented); the recovery command still names ``display_path``."""
try:
with _temp_copy(data, ".pdf") as temp_path:
return _pdf_coverage_note(temp_path, display_path=display_path)
except OSError:
return ""
"""Coverage note for backend PDF bytes via a host temp copy (pdftotext needs a path)."""
with contextlib.suppress(OSError), _temp_copy(data, ".pdf") as temp_path:
return _pdf_coverage_note(temp_path, display_path=display_path)
return ""
def _joined(lines: list[str], empty_error: str) -> str:
@@ -352,11 +318,10 @@ def _joined(lines: list[str], empty_error: str) -> str:
def _source_text(source) -> str:
if isinstance(source, str):
return source
"""Notebook source/text fields are a str or a list of str fragments."""
if isinstance(source, list):
return "".join(item for item in source if isinstance(item, str))
return ""
source = "".join(item for item in source if isinstance(item, str))
return source if isinstance(source, str) else ""
def _human_size(n_bytes: int) -> str:
@@ -366,30 +331,24 @@ def _human_size(n_bytes: int) -> str:
def _base64_bytes(payload: str) -> int:
"""Approximate decoded size of a base64 payload (whitespace ignored)."""
clean = re.sub(r"[^0-9+/=A-Za-z]", "", payload)
padding = min(2, len(clean) - len(clean.rstrip("=")))
return max(0, (len(clean) * 3) // 4 - padding)
return max(0, (len(clean) * 3) // 4 - min(2, len(clean) - len(clean.rstrip("="))))
def _clean_stream_text(text: str) -> str:
"""Strip ANSI escapes; keep only the final ``\\r`` frame of each line (tqdm redraws)."""
from tools.ansi_strip import strip_ansi
lines = []
for line in strip_ansi(text).replace("\r\n", "\n").split("\n"):
frames = [frame for frame in line.split("\r") if frame]
lines.append(frames[-1] if frames else "")
return "\n".join(lines)
return "\n".join(([f for f in line.split("\r") if f] or [""])[-1]
for line in strip_ansi(text).replace("\r\n", "\n").split("\n"))
# Per-output-block truncation so one runaway training log cannot flood the extraction.
_MAX_OUTPUT_CHARS = 20_000
_MAX_OUTPUT_CHARS = 20_000 # per code cell, so one runaway training log cannot flood the extraction
# nbformat v3 stores mime data flat on the output dict under these keys.
_V3_MIME_KEYS = (("png", "image/png"), ("jpeg", "image/jpeg"), ("svg", "image/svg+xml"), ("html", "text/html"))
def _notebook_output_text(output: Any) -> str:
"""Render one notebook output as compact text: stream text, tracebacks and textual
results kept; token-heavy payloads (images, HTML, widgets) become sized placeholders.
Handles nbformat v4 and legacy v3 (``pyout``/``pyerr``) shapes."""
"""One notebook output as compact text: stream/traceback/textual results kept; token-heavy
payloads (images, HTML, widgets) become sized placeholders. Handles v4 and legacy v3 shapes."""
if not isinstance(output, dict):
return ""
otype = output.get("output_type")
@@ -397,52 +356,41 @@ def _notebook_output_text(output: Any) -> str:
body = _clean_stream_text(_source_text(output.get("text", "")))
return body if body.strip() else ""
if otype in {"error", "pyerr"}:
traceback = output.get("traceback")
tb_text = ""
if isinstance(traceback, list):
tb_text = _clean_stream_text(
"\n".join(line for line in traceback if isinstance(line, str)))
tb = output.get("traceback")
tb_text = _clean_stream_text("\n".join(filter(lambda l: isinstance(l, str), tb))
if isinstance(tb, list) else "")
header = f"Error: {output.get('ename', '')}: {output.get('evalue', '')}".rstrip(": ")
return f"{header}\n{tb_text}".rstrip()
if otype not in {"execute_result", "display_data", "pyout"}:
return ""
data = output.get("data")
if not isinstance(data, dict): # legacy v3: mime payloads sit flat on the output dict
data = {"text/plain": output["text"]} if isinstance(output.get("text"), (str, list)) else {}
data.update((mime, output[k]) for k, mime in _V3_MIME_KEYS if k in output)
if "application/vnd.jupyter.widget-view+json" in data:
return "[interactive widget — omitted]"
# Prefer readable text: models consume text/plain far better than markup.
for mime in ("text/plain", "text/markdown"):
if mime in data:
body = _clean_stream_text(_source_text(data[mime]))
if body.strip():
return body
for mime in ("text/plain", "text/markdown"): # models consume text far better than markup
body = _clean_stream_text(_source_text(data[mime])) if mime in data else ""
if body.strip():
return body
for mime, value in data.items():
if isinstance(mime, str) and mime.startswith("image/"):
size = _base64_bytes(_source_text(value))
return f"[{mime} output — {_human_size(size)}, omitted]"
return f"[{mime} output — {_human_size(_base64_bytes(_source_text(value)))}, omitted]"
if "text/html" in data:
html = _source_text(data["text/html"])
return f"[text/html output — {len(html):,} chars, omitted]"
mimes = ", ".join(str(m) for m in data) or "unknown"
return f"[{mimes} output — omitted]"
return f"[text/html output — {len(_source_text(data['text/html'])):,} chars, omitted]"
return f"[{', '.join(str(m) for m in data) or 'unknown'} output — omitted]"
def _notebook_outputs(cell: dict, jq_pointer: str = "", filename: str = "") -> str:
outputs = cell.get("outputs")
if not isinstance(outputs, list):
return ""
blocks = [text for text in (_notebook_output_text(o) for o in outputs) if text]
if not blocks:
return ""
joined = "\n".join(blocks)
if len(joined) > _MAX_OUTPUT_CHARS:
omitted = len(joined) - _MAX_OUTPUT_CHARS
hint = f" — full output: jq -r '{jq_pointer}' {filename}" if jq_pointer and filename else ""
joined = joined[:_MAX_OUTPUT_CHARS] + f"\n… [{omitted:,} output chars truncated{hint}]"
return joined
joined = "\n".join(filter(None, map(_notebook_output_text, outputs)))
if len(joined) <= _MAX_OUTPUT_CHARS:
return joined
hint = f" — full output: jq -r '{jq_pointer}' {filename}" if jq_pointer and filename else ""
omitted = len(joined) - _MAX_OUTPUT_CHARS
return joined[:_MAX_OUTPUT_CHARS] + f"\n… [{omitted:,} output chars truncated{hint}]"
_CELL_LABELS = {"markdown": "Markdown", "code": "Code", "raw": "Raw"}
@@ -459,7 +407,7 @@ def _extract_notebook(path: str) -> str:
raw_cells = nb.get("cells")
if isinstance(raw_cells, list):
cells = [(f".cells[{i}].outputs", cell) for i, cell in enumerate(raw_cells)]
else:
else: # nbformat v3: cells live under worksheets
cells = [
(f".worksheets[{wi}].cells[{ci}].outputs", cell)
for wi, ws in enumerate(nb.get("worksheets", [])) if isinstance(ws, dict)
@@ -470,18 +418,16 @@ def _extract_notebook(path: str) -> str:
counts = dict.fromkeys(_CELL_LABELS, 0)
out: list[str] = []
for jq_pointer, cell in cells:
if not isinstance(cell, dict):
continue
typ = cell.get("cell_type")
typ = cell.get("cell_type") if isinstance(cell, dict) else None
if typ not in _CELL_LABELS:
continue
counts[typ] += 1
suffix = f" {counts[typ]}" if typ != "raw" else ""
out.extend((f"# ── {_CELL_LABELS[typ]} cell{suffix} ──", _source_text(cell.get("source", "")).rstrip("\n"), ""))
if typ == "code":
rendered = _notebook_outputs(cell, jq_pointer, nb_name)
if rendered:
out.extend((f"# ── Output (cell {counts[typ]}) ──", rendered.rstrip("\n"), ""))
source = _source_text(cell.get("source", "")).rstrip("\n")
out += [f"# ── {_CELL_LABELS[typ]} cell{suffix} ──", source, ""]
rendered = _notebook_outputs(cell, jq_pointer, nb_name) if typ == "code" else ""
if rendered:
out += [f"# ── Output (cell {counts[typ]}) ──", rendered.rstrip("\n"), ""]
return _joined(out, "Notebook contains no readable cells")
@@ -491,24 +437,21 @@ def _open_zip(path: str, kind: str) -> Iterator[zipfile.ZipFile]:
try:
with zipfile.ZipFile(path) as zf:
yield zf
except zipfile.BadZipFile as exc:
raise ExtractionError(f"Not a valid {kind}: {exc}") from exc
except OSError as exc:
raise ExtractionError(str(exc)) from exc
except (zipfile.BadZipFile, OSError) as exc:
bad_zip = isinstance(exc, zipfile.BadZipFile)
raise ExtractionError(f"Not a valid {kind}: {exc}" if bad_zip else str(exc)) from exc
def _zip_xml(zf: zipfile.ZipFile, name: str, optional: bool = False) -> Any:
"""Parse a package part; ``optional`` parts yield None when absent or malformed."""
"""Parse a package part; ``optional`` parts yield an empty element when absent or malformed."""
try:
return ET.fromstring(zf.read(name))
except KeyError as exc:
except (KeyError, ET.ParseError) as exc:
if optional:
return None
raise ExtractionError(f"Missing {name}") from exc
except ET.ParseError as exc:
if optional:
return None
raise ExtractionError(f"Malformed XML in {name}: {exc}") from exc
return ET.Element("missing")
raise ExtractionError(
f"Missing {name}" if isinstance(exc, KeyError) else f"Malformed XML in {name}: {exc}"
) from exc
def _extract_docx(path: str) -> str:
@@ -518,9 +461,9 @@ def _extract_docx(path: str) -> str:
breaks = {f"{w}tab": "\t", f"{w}br": "\n", f"{w}cr": "\n"}
lines: list[str] = []
for para in root.iter(f"{w}p"):
buf = [(node.text or "") if node.tag == f"{w}t" else breaks.get(node.tag, "")
for node in para.iter()]
lines.extend("".join(buf).split("\n"))
text = "".join(
(n.text or "") if n.tag == f"{w}t" else breaks.get(n.tag, "") for n in para.iter())
lines.extend(text.split("\n"))
return _joined(lines, "DOCX contains no extractable text")
@@ -529,38 +472,27 @@ def _extract_xlsx(path: str) -> str:
with _open_zip(path, "XLSX") as zf:
names = set(zf.namelist())
sst = _zip_xml(zf, "xl/sharedStrings.xml", optional=True)
shared = [] if sst is None else [
"".join(t.text or "" for t in item.iter(f"{s}t")) for item in sst.iter(f"{s}si")]
shared = ["".join(t.text or "" for t in item.iter(f"{s}t")) for item in sst.iter(f"{s}si")]
rels_root = _zip_xml(zf, "xl/_rels/workbook.xml.rels", optional=True)
rels = {} if rels_root is None else {
rel.get("Id", ""): rel.get("Target", "")
for rel in rels_root.iter(f"{pr}Relationship") if rel.get("Id")}
rels = {rel.get("Id", ""): rel.get("Target", "")
for rel in rels_root.iter(f"{pr}Relationship") if rel.get("Id")}
out: list[str] = []
for sheet in _zip_xml(zf, "xl/workbook.xml").iter(f"{s}sheet"):
if sheet.get("state", "visible") in {"hidden", "veryHidden"}:
continue
target = rels.get(sheet.get(f"{r}id", ""), "").lstrip("/")
part = posixpath.normpath(target if target.startswith("xl/") else f"xl/{target}")
if part not in names:
if sheet.get("state", "visible") in {"hidden", "veryHidden"} or part not in names:
continue
try:
with contextlib.suppress(ET.ParseError):
rows = _sheet_rows(zf.read(part), shared)
except ET.ParseError:
continue
out.append(f"# ── Sheet: {sheet.get('name', 'Sheet')} ──")
out.extend("\t".join(row) for row in rows)
if not rows:
out.append("(empty)")
out.append("")
out += [f"# ── Sheet: {sheet.get('name', 'Sheet')} ──",
*(["\t".join(row) for row in rows] or ["(empty)"]), ""]
return _joined(out, "XLSX has no visible sheets with content")
def _col_index(ref: str) -> int:
idx = 0
for ch in ref:
if not ch.isalpha():
break
idx = idx * 26 + ord(ch.upper()) - ord("A") + 1
"""0-based column of a cell ref: ``A1`` -> 0, ``AB7`` -> 27 (bijective base-26 letters)."""
idx = functools.reduce(lambda acc, ch: acc * 26 + ord(ch.upper()) - ord("A") + 1,
itertools.takewhile(str.isalpha, ref), 0)
return max(idx - 1, 0)
@@ -568,18 +500,15 @@ def _sheet_rows(xml_bytes: bytes, shared: list[str]) -> list[list[str]]:
root = ET.fromstring(xml_bytes)
s = f"{{{_NS_S}}}"
rows: list[list[str]] = []
for row in root.iter(f"{s}row"):
if len(rows) >= _MAX_XLSX_ROWS_PER_SHEET:
break
for row in itertools.islice(root.iter(f"{s}row"), _MAX_XLSX_ROWS_PER_SHEET):
cells: dict[int, str] = {}
max_col = -1
for cell in row.iter(f"{s}c"):
col = _col_index(cell.get("r", "")) if cell.get("r") else max_col + 1
if col >= _MAX_XLSX_COLS:
continue
cells[col] = _cell_value(cell, shared, s)
max_col = max(max_col, col)
rows.append([cells.get(i, "") for i in range(max_col + 1)] if max_col >= 0 else [])
if col < _MAX_XLSX_COLS:
cells[col] = _cell_value(cell, shared, s)
max_col = max(max_col, col)
rows.append([cells.get(i, "") for i in range(max_col + 1)])
while rows and not any(value.strip() for value in rows[-1]):
rows.pop()
return rows
+97 -141
View File
@@ -1,11 +1,7 @@
"""Sanitize tool JSON schemas for broad LLM-backend compatibility.
Strict backends reject shapes OpenAI/Anthropic accept: llama.cpp's grammar converter fails
on ``{"type": "object"}`` without ``properties``, bare-string schemas and ``type`` arrays;
Anthropic rejects nullable ``anyOf`` at the top of ``input_schema``; Fireworks rejects
``default`` beside ``$ref``; Codex rejects top-level combinators. This module walks the
final schema tree on a deep copy and fixes only those shapes.
"""
"""Sanitize tool JSON schemas for strict LLM backends. llama.cpp's grammar converter fails on
``{"type": "object"}`` without ``properties``, bare-string schemas and ``type`` arrays; Anthropic
rejects nullable ``anyOf`` at the top of ``input_schema``; Fireworks rejects ``default`` beside
``$ref``; Codex rejects top-level combinators. Walks a deep copy and fixes only those shapes."""
from __future__ import annotations
@@ -16,15 +12,12 @@ from typing import Any, Callable
logger = logging.getLogger(__name__)
# Anthropic (and Bedrock/Vertex/Azure fronting it) reject property keys not matching this;
# one bad key anywhere in the tools array 400s the request (Cloudflare's MCP ships 61).
# Anthropic (and Bedrock/Vertex/Azure fronting it) reject property keys not matching this; one bad
# key anywhere in the tools array 400s the request (Cloudflare's MCP ships 61).
_PROP_KEY_RE = re.compile(r"^[a-zA-Z0-9_.-]{1,64}$")
_PROP_KEY_BAD_CHARS = re.compile(r"[^a-zA-Z0-9_.-]")
_UNION_KEYS = ("anyOf", "oneOf")
# Outer-node metadata carried onto a union's replacement node.
_UNION_META_KEYS = ("title", "description", "default", "examples")
_UNION_META_KEYS = ("title", "description", "default", "examples") # copied onto replacements
def _empty_object() -> dict:
@@ -47,56 +40,45 @@ def sanitize_property_key(key: str) -> str:
def _rename_property_keys(props: dict, path: str) -> dict[str, str]:
"""{original_key: conforming_key} for one properties dict (identity entries omitted).
Deterministic (insertion order, numeric suffixes on collision) so the model-visible
schema and the dispatch-time reverse map from the registry's original schema agree."""
Deterministic (insertion order, numeric suffixes on collision) so the model-visible schema
and the dispatch-time reverse map from the registry's original schema agree."""
renames: dict[str, str] = {}
taken = {k for k in props if _PROP_KEY_RE.match(k)}
for key in props:
if _PROP_KEY_RE.match(key):
continue
for key in (k for k in props if not _PROP_KEY_RE.match(k)):
base = sanitize_property_key(key)
candidate, i = base, 2
while candidate in taken:
suffix = f"_{i}"
candidate = base[: 64 - len(suffix)] + suffix
i += 1
candidate, i = base[: 64 - len(f"_{i}")] + f"_{i}", i + 1
taken.add(candidate)
renames[key] = candidate
logger.debug(
"schema_sanitizer[%s]: renamed property key %r -> %r "
"(provider key-pattern compat)", path, key, candidate)
logger.debug("schema_sanitizer[%s]: renamed property key %r -> %r "
"(provider key-pattern compat)", path, key, candidate)
return renames
def unrename_tool_args(params_schema: Any, args: Any) -> Any:
"""Map sanitized property keys in model-emitted args back to wire names. ``params_schema``
is the ORIGINAL registry schema; recurses into objects/array items; unknown keys pass."""
if not isinstance(params_schema, dict) or not isinstance(args, dict):
return args
props = params_schema.get("properties")
if not isinstance(props, dict):
"""Map sanitized keys in model-emitted args back to wire names. ``params_schema`` is the
ORIGINAL registry schema; recurses into objects/array items; unknown keys pass through."""
props = params_schema.get("properties") if isinstance(params_schema, dict) else None
if not isinstance(props, dict) or not isinstance(args, dict):
return args
reverse = {v: k for k, v in _rename_property_keys(props, "<unrename>").items()}
out = {}
for key, value in args.items():
orig = reverse.get(key, key)
subschema = props.get(orig)
if isinstance(subschema, dict):
if isinstance(value, dict):
value = unrename_tool_args(subschema, value)
elif isinstance(value, list) and isinstance(subschema.get("items"), dict):
value = [
unrename_tool_args(subschema["items"], item) if isinstance(item, dict) else item
for item in value]
sub = props.get(orig) if isinstance(props.get(orig), dict) else {}
if isinstance(value, dict) and sub:
value = unrename_tool_args(sub, value)
elif isinstance(value, list) and isinstance(sub.get("items"), dict):
value = [unrename_tool_args(sub["items"], item) if isinstance(item, dict) else item
for item in value]
out[orig] = value
return out
def sanitize_tool_schemas(tools: list[dict]) -> list[dict]:
"""Deep-copied ``tools`` (OpenAI format) with sanitized parameter schemas; safe to mutate."""
if not tools:
return tools
return [_sanitize_single_tool(tool) for tool in tools]
return [_sanitize_single_tool(tool) for tool in tools] if tools else tools
def _sanitize_single_tool(tool: dict) -> dict:
@@ -110,9 +92,7 @@ def _sanitize_single_tool(tool: dict) -> dict:
return out
name = fn.get("name", "<tool>")
top = _sanitize_node(params, path=name)
# Guarantee the top level is an object with properties.
if not isinstance(top, dict):
top = {}
top = top if isinstance(top, dict) else {} # guarantee an object with properties on top
top["type"] = "object"
if not isinstance(top.get("properties"), dict):
top["properties"] = {}
@@ -124,16 +104,14 @@ def _sanitize_single_tool(tool: dict) -> dict:
return out
# Sibling keywords strict JSON Schema validators reject alongside ``$ref``.
_REF_FORBIDDEN_SIBLINGS = frozenset({"default"})
_REF_FORBIDDEN_SIBLINGS = frozenset({"default"}) # strict validators reject these beside ``$ref``
def _strip_ref_siblings(node: Any) -> Any:
"""Recursively drop forbidden siblings of ``$ref`` (Fireworks rejects ``default`` there)."""
def strip(out: dict) -> dict:
if "$ref" in out:
for key in _REF_FORBIDDEN_SIBLINGS:
out.pop(key, None)
for key in _REF_FORBIDDEN_SIBLINGS if "$ref" in out else ():
out.pop(key, None)
return out
return _rewrite(node, strip)
@@ -143,18 +121,15 @@ _TOP_LEVEL_FORBIDDEN_KEYS = ("allOf", "anyOf", "oneOf", "enum", "not")
def _strip_top_level_combinators(params: dict, *, path: str = "<tool>") -> dict:
"""Drop combinators from the TOP level only (Codex rejects them there). They are usually
conditional-required hints, so validity is unchanged (handlers re-validate); nested
combinators are preserved."""
conditional-required hints, so validity is unchanged (handlers re-validate); nested ones
stay."""
if not isinstance(params, dict):
return params
out = dict(params)
for key in _TOP_LEVEL_FORBIDDEN_KEYS:
if key in out:
logger.debug(
"schema_sanitizer[%s]: stripped top-level %r combinator "
"from tool parameters (strict-backend compat)",
path, key)
out.pop(key, None)
for key in [k for k in _TOP_LEVEL_FORBIDDEN_KEYS if k in out]:
logger.debug("schema_sanitizer[%s]: stripped top-level %r combinator "
"from tool parameters (strict-backend compat)", path, key)
del out[key]
return out
@@ -163,20 +138,18 @@ def _is_null_branch(item: Any) -> bool:
def _carry_union_meta(outer: dict, replacement: dict, *, skip_default_on_ref: bool) -> None:
"""Copy outer-union metadata onto *replacement* where absent."""
"""Copy outer-union metadata onto *replacement* where absent (``default`` is illegal beside
``$ref`` on strict backends, hence ``skip_default_on_ref``)."""
for meta_key in _UNION_META_KEYS:
if meta_key in outer and meta_key not in replacement:
# ``default`` is illegal alongside ``$ref`` on strict backends.
if skip_default_on_ref and meta_key == "default" and "$ref" in replacement:
continue
if meta_key in outer and meta_key not in replacement and not (
skip_default_on_ref and meta_key == "default" and "$ref" in replacement):
replacement[meta_key] = outer[meta_key]
def strip_nullable_unions(schema: Any, *, keep_nullable_hint: bool = True) -> Any:
"""Collapse ``anyOf``/``oneOf`` nullable unions to the single non-null branch. MCP/Pydantic
optional fields arrive as ``{"anyOf": [{"type": "string"}, {"type": "null"}]}``; Anthropic
rejects the null branch and optionality is already in the parent's ``required``. Only
collapses when a null branch was dropped AND exactly one non-null branch survives.
"""Collapse ``anyOf``/``oneOf`` nullable unions (MCP/Pydantic optional fields) to the single
non-null branch: Anthropic rejects the null branch and optionality already lives in the parent's
``required``. Only when a null branch was dropped AND exactly one non-null branch survives.
``keep_nullable_hint`` sets ``nullable: true`` for runtime ``"null"`` → ``None`` coercion."""
def collapse(stripped: dict) -> Any:
for key in _UNION_KEYS:
@@ -189,7 +162,7 @@ def strip_nullable_unions(schema: Any, *, keep_nullable_hint: bool = True) -> An
if keep_nullable_hint:
replacement.setdefault("nullable", True)
_carry_union_meta(stripped, replacement, skip_default_on_ref=True)
return _rewrite(replacement, collapse)
return _rewrite(replacement, collapse) # the survivor may itself be a union
return stripped
return _rewrite(schema, collapse)
@@ -199,24 +172,23 @@ _CONST_PRIMITIVE_TYPES: dict[type, str] = {
def _const_branch_type(branch: Any) -> str | None:
"""JSON-Schema primitive type of a pure ``const`` branch, else None: a primitive ``const``
whose declared ``type`` (if any) matches; only ``title``/``description`` may accompany it."""
"""Primitive JSON-Schema type of a pure ``const`` branch (declared ``type``, if any, must match;
only ``title``/``description`` may accompany it), else None."""
if not isinstance(branch, dict) or "const" not in branch \
or set(branch) - {"const", "type", "title", "description"}:
return None
# ``type(value)`` lookup (not isinstance): bool is a subclass of int.
json_type = _CONST_PRIMITIVE_TYPES.get(type(branch["const"]))
return json_type if json_type is not None and branch.get("type") in (None, json_type) else None
return json_type if branch.get("type") in (None, json_type) else None
def collapse_const_unions(schema: Any) -> Any:
"""Collapse ``anyOf``/``oneOf`` unions of same-typed consts to ``enum`` (ported from
block/goose ``tool_schema_normalize.rs``, Apache-2.0). Rust/TS MCP servers emit
``{"anyOf": [{"const": "red"}, {"const": "green"}]}``, which strict backends mishandle.
Applies only when EVERY non-null branch is a pure ``const`` of one primitive type
(``bool`` never merges with ``integer``); one ``{"type": "null"}`` branch is tolerated
and recorded as ``nullable: true``. Branch order is kept; outer metadata carried over;
input never mutated."""
"""Collapse ``anyOf``/``oneOf`` unions of same-typed consts (Rust/TS MCP servers emit
``{"anyOf": [{"const": "red"}, {"const": "green"}]}``) to ``enum``; ported from block/goose
``tool_schema_normalize.rs`` (Apache-2.0). Only when EVERY non-null branch is a pure ``const``
of one primitive type (``bool`` never merges with ``integer``); one ``{"type": "null"}`` branch
is tolerated as ``nullable: true``. Branch order kept; outer metadata carried; input never
mutated."""
def collapse(out: dict) -> Any:
for key in _UNION_KEYS:
variants = out.get(key)
@@ -240,52 +212,48 @@ def collapse_const_unions(schema: Any) -> Any:
_BARE_TYPE_NAMES = frozenset({"object", "string", "number", "integer", "boolean", "array", "null"})
# Keys whose values are NOT schemas (recursing would treat "path" as a bare-string schema).
# Values that are NOT schemas (recursing would treat a required name like "path" as a bare schema).
_NON_SCHEMA_LIST_KEYS = frozenset({"required", "enum", "examples", "dependentRequired"})
def _normalize_type_array(value: list, out: dict) -> None:
"""Normalize a ``type: [...]`` array into *out* (llama.cpp and Gemini-via-OpenAI reject
arrays). Per AI-SDK: one non-null type → ``type: X`` (+ ``nullable`` if ``null`` present);
several → ``anyOf`` of single-type schemas so EVERY branch survives; none → ``null`` or
the object fallback. Ported from anomalyco/opencode#31877."""
"""Normalize a ``type: [...]`` array into *out* (llama.cpp and Gemini-via-OpenAI reject arrays).
Per AI-SDK: one non-null type → ``type: X`` (+ ``nullable`` if ``null`` present); several →
``anyOf`` of single-type schemas so EVERY branch survives; none → ``null``/object fallback."""
has_null = "null" in value
non_null = [t for t in value if isinstance(t, str) and t != "null"]
if len(non_null) == 1:
out["type"] = non_null[0]
elif len(non_null) >= 2:
out["anyOf"] = [{"type": t} for t in non_null]
else:
if not non_null:
out["type"] = "null" if has_null else "object"
return
if len(non_null) == 1:
out["type"] = non_null[0]
else:
out["anyOf"] = [{"type": t} for t in non_null]
if has_null:
out.setdefault("nullable", True)
def _sanitize_node(node: Any, path: str) -> Any:
"""Recursively sanitize a JSON-Schema fragment: bare-string schemas become ``{"type":
<value>}`` (unknown strings → permissive object); object nodes gain ``properties: {}``;
``type`` arrays are normalized; property keys are renamed to the provider-safe pattern
and ``required`` follows, with entries missing from ``properties`` pruned."""
"""Recursively sanitize a JSON-Schema fragment: bare-string schemas → ``{"type": <value>}``
(unknown strings → permissive object); object nodes gain ``properties: {}``; ``type`` arrays
are normalized; property keys are renamed to the provider-safe pattern and ``required``
follows, with entries missing from ``properties`` pruned."""
if isinstance(node, str):
if node in _BARE_TYPE_NAMES:
logger.debug(
"schema_sanitizer[%s]: replacing bare-string schema %r with {'type': %r}",
path, node, node)
logger.debug("schema_sanitizer[%s]: replacing bare-string schema %r with {'type': %r}",
path, node, node)
return _empty_object() if node == "object" else {"type": node}
logger.debug(
"schema_sanitizer[%s]: replacing non-schema string %r "
"with empty object schema", path, node)
logger.debug("schema_sanitizer[%s]: replacing non-schema string %r "
"with empty object schema", path, node)
return _empty_object()
if isinstance(node, list):
return [_sanitize_node(item, f"{path}[{i}]") for i, item in enumerate(node)]
if not isinstance(node, dict):
return node
# Renames computed up front so ``required`` remaps even when it precedes ``properties``.
prop_renames: dict[str, str] = {}
if isinstance(node.get("properties"), dict):
prop_renames = _rename_property_keys(node["properties"], f"{path}.properties")
props_in = node.get("properties")
prop_renames = (_rename_property_keys(props_in, f"{path}.properties")
if isinstance(props_in, dict) else {})
out: dict = {}
for key, value in node.items():
if key == "type" and isinstance(value, list):
@@ -293,64 +261,55 @@ def _sanitize_node(node: Any, path: str) -> Any:
elif key in {"properties", "$defs", "definitions"} and isinstance(value, dict):
renames = prop_renames if key == "properties" else {}
out[key] = {
renames.get(sub_k, sub_k): _sanitize_node(sub_v, f"{path}.{key}.{renames.get(sub_k, sub_k)}")
for sub_k, sub_v in value.items()}
renames.get(k, k): _sanitize_node(v, f"{path}.{key}.{renames.get(k, k)}")
for k, v in value.items()}
elif key in {"items", "additionalProperties"}:
# Bool ``additionalProperties`` is valid; bool ``items`` is non-standard but preserved.
out[key] = value if isinstance(value, bool) else _sanitize_node(value, f"{path}.{key}")
elif key in {"anyOf", "oneOf", "allOf"} and isinstance(value, list):
out[key] = [_sanitize_node(item, f"{path}.{key}[{i}]") for i, item in enumerate(value)]
elif key in _NON_SCHEMA_LIST_KEYS:
if key == "required" and prop_renames and isinstance(value, list):
out[key] = [prop_renames.get(r, r) if isinstance(r, str) else r for r in value]
else:
out[key] = copy.deepcopy(value) if isinstance(value, (list, dict)) else value
else:
else: # anyOf/oneOf/allOf and any other nested schema recurse (lists index the path)
out[key] = _sanitize_node(value, f"{path}.{key}") if isinstance(value, (dict, list)) else value
if out.get("type") == "object":
if not isinstance(out.get("properties"), dict):
out["properties"] = {}
if isinstance(out.get("required"), list):
props = out.get("properties") or {}
valid = [r for r in out["required"] if isinstance(r, str) and r in props]
if not valid:
out.pop("required", None)
elif len(valid) != len(out["required"]):
valid = [r for r in out["required"] if isinstance(r, str) and r in out["properties"]]
if valid:
out["required"] = valid
else:
del out["required"]
return out
# ---- Reactive strips — only invoked after a backend rejects a schema -----------------------
# ---- Reactive strips — only invoked after a backend rejects a schema ----
_STRIP_ON_RECOVERY_KEYS = frozenset({"pattern", "format"})
_SCHEMA_MARKERS = frozenset({"type", "anyOf", "oneOf", "allOf"}) # a node with one IS a schema
def _dict_nodes(node: Any):
"""Pre-order walk yielding every dict node; each is yielded before its values are visited,
so a consumer may mutate it in place."""
"""Pre-order walk over every dict node (yielded before its values, so it may be mutated)."""
if isinstance(node, dict):
yield node
for v in node.values():
yield from _dict_nodes(v)
elif isinstance(node, list):
for item in node:
yield from _dict_nodes(item)
children = node.values() if isinstance(node, dict) else node if isinstance(node, list) else ()
for child in children:
yield from _dict_nodes(child)
def _reactive_strip(
tools: list[dict], strip_node: Callable[[dict], int], log_msg: str) -> tuple[list[dict], int]:
"""Apply *strip_node* (returns keywords removed) to every dict node of each tool's
parameters, in place. Handles OpenAI (``{"function": {"parameters": ..}}``) and Responses
(``{"name": .., "parameters": ..}``) formats. Returns ``(tools, stripped_count)``."""
if not tools:
return tools, 0
"""Apply *strip_node* (-> keywords removed) to every dict node of each tool's parameters, in
place; OpenAI (``{"function": {"parameters"}}``) and Responses (``{"parameters"}``) formats."""
stripped = 0
for tool in tools:
for tool in tools or ():
if not isinstance(tool, dict):
continue
fn = tool.get("function")
params = fn.get("parameters") if isinstance(fn, dict) else None
if not isinstance(params, dict):
params = tool.get("parameters")
params = params if isinstance(params, dict) else tool.get("parameters")
if isinstance(params, dict):
stripped += sum(strip_node(node) for node in _dict_nodes(params))
if stripped:
@@ -359,16 +318,14 @@ def _reactive_strip(
def strip_pattern_and_format(tools: list[dict]) -> tuple[list[dict], int]:
"""Strip ``pattern``/``format`` keywords from tool schemas, in place. Reactive: only after
llama.cpp's grammar converter rejected a schema (its regex engine is a small ECMAScript
subset), since cloud providers use these as prompting hints. Only strips beside
``type``/combinators, so a property literally *named* ``pattern`` is untouched."""
"""Strip ``pattern``/``format`` in place — reactive, only after llama.cpp's grammar converter
rejected a schema (its regex engine is a small ECMAScript subset); cloud providers use these as
prompting hints. Only beside ``type``/combinators, so a property *named* ``pattern`` stays."""
def _strip(node: dict) -> int:
if not ("type" in node or "anyOf" in node or "oneOf" in node or "allOf" in node):
return 0
hits = [k for k in node if k in _STRIP_ON_RECOVERY_KEYS]
is_schema = bool(node.keys() & _SCHEMA_MARKERS)
hits = [k for k in node if k in _STRIP_ON_RECOVERY_KEYS] if is_schema else []
for k in hits:
node.pop(k, None)
del node[k]
return len(hits)
return _reactive_strip(
tools, _strip,
@@ -377,13 +334,12 @@ def strip_pattern_and_format(tools: list[dict]) -> tuple[list[dict], int]:
def strip_slash_enum(tools: list[dict]) -> tuple[list[dict], int]:
"""Strip ``enum`` keywords whose string values contain ``/``, in place. xAI compiles
schemas to a grammar that rejects ``/`` in enum values (HTTP 400 before any token) —
typically MCP enums of HuggingFace model IDs. The constraint is a prompting hint only."""
"""Strip ``enum`` keywords whose string values contain ``/``, in place: xAI's grammar compiler
rejects them (HTTP 400 before any token) — typically MCP enums of HuggingFace model IDs."""
def _strip(node: dict) -> int:
enum_val = node.get("enum")
if isinstance(enum_val, list) and any(isinstance(v, str) and "/" in v for v in enum_val):
node.pop("enum", None)
del node["enum"]
return 1
return 0
return _reactive_strip(
+112 -164
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import contextlib
import os
import re
import shlex
@@ -11,9 +12,7 @@ from pathlib import Path
from typing import Callable
from tools.approval import (
_bash_exec_payload,
_deobfuscate_shell_word_for_detection,
_iter_shell_command_starts,
_bash_exec_payload, _deobfuscate_shell_word_for_detection, _iter_shell_command_starts,
_read_shell_word)
# bisect drives repeated checkouts of the running root — the exact skew hazard guarded here.
@@ -24,8 +23,7 @@ _WORKTREE_TARGET_ACTIONS = frozenset({"move", "remove"})
_STASH_SAFE_ACTIONS = frozenset({"list", "show", "create", "store", "drop", "clear"})
_RESET_WORKTREE_MODES = frozenset({"--hard", "--merge", "--keep"})
# `reset`/`stash`/`clean`/`restore` reach this set only in their SAFE forms (_mutates_worktree
# classifies the dangerous forms first); listing them just skips a pointless
# `git config --get alias.<sub>` subprocess for `stash list`, `reset --soft`, `clean -n`.
# runs first); listing them skips a pointless `git config --get alias.<sub>` subprocess.
_KNOWN_GIT_BUILTINS = frozenset({
"add", "am", "apply", "blame", "branch", "bundle", "cat-file", "clean", "clone", "commit",
"config", "describe", "diff", "fetch", "format-patch", "grep", "help", "init", "log",
@@ -64,36 +62,31 @@ class _Heredoc:
@dataclass
class _ShellContext:
class _ShellContext: # one `(` / `$(` / backtick nesting level and its live quote state
kind: str
opener: int
quote: str | None = None
def get_running_source_root() -> Path | None:
"""Return the source checkout backing this process, if there is one."""
try:
"""The source checkout backing this process, if there is one."""
with contextlib.suppress(OSError, RuntimeError):
root = Path(__file__).resolve().parent.parent
except (OSError, RuntimeError):
return None
return root if (root / ".git").exists() else None
return root if (root / ".git").exists() else None
return None
def _resolve(path_str: str, base: Path) -> Path:
path = Path(os.path.expanduser(path_str))
if not path.is_absolute():
path = base / path
try:
path = base / Path(os.path.expanduser(path_str)) # ``/`` keeps an absolute right operand
with contextlib.suppress(OSError, RuntimeError, ValueError):
return path.resolve()
except (OSError, RuntimeError, ValueError):
return path
return path
def _is_within(path: Path, root: Path) -> bool:
try:
with contextlib.suppress(OSError, RuntimeError, ValueError):
return path == root or path.is_relative_to(root)
except (OSError, RuntimeError, ValueError):
return False
return False
def _executable_name(value: str) -> str:
@@ -101,6 +94,7 @@ def _executable_name(value: str) -> str:
def _shell_words_at(command: str, start: int) -> list[str]:
"""Deobfuscated words of the simple command at ``start`` (stops at a newline; max 64)."""
words: list[str] = []
cursor = start
for _ in range(64):
@@ -116,13 +110,10 @@ def _consume_options(
words: list[str], start: int, options_with_arg: frozenset[str] = _NO_OPTIONS) -> int:
"""Index of the first positional at/after ``start`` (``--`` ends options)."""
index = start
while index < len(words):
option = words[index]
if option == "--":
while index < len(words) and words[index].startswith("-") and words[index] != "-":
if words[index] == "--":
return index + 1
if not option.startswith("-") or option == "-":
break
index += 2 if "=" not in option and option in options_with_arg else 1
index += 2 if "=" not in words[index] and words[index] in options_with_arg else 1
return index
@@ -140,9 +131,8 @@ def _command_parts(words: list[str]) -> tuple[dict[str, str], str | None, list[s
wrapper_options = _WRAPPER_OPTIONS_WITH_ARG.get(executable)
if wrapper_options is None:
return env, words[index], words[index + 1 :]
# `command -v/-V` only reports; nothing runs.
if executable == "command" and words[index + 1 : index + 2] in (["-v"], ["-V"]):
return env, None, []
break # `command -v/-V` only reports; nothing runs
index = _consume_options(words, index + 1, wrapper_options)
return env, None, []
@@ -157,15 +147,14 @@ def _scope_keys(command: str, starts: list[int]) -> dict[int, tuple[int, ...]]:
context = contexts[-1]
quote = context.quote
char = command[cursor]
nested = len(contexts) > 1
if quote == "'":
if char == "'":
context.quote = None
closes = quote is None and len(contexts) > 1 # an unquoted closer may pop a scope
if quote is not None and char == quote:
context.quote = None
elif quote == "'":
pass # single quotes: no escapes, no substitutions
elif char == "\\" and cursor + 1 < start:
cursor += 1
elif quote == '"' and char == '"':
context.quote = None
elif quote is None and char in {"'", '"'}:
elif quote is None and char in "'\"":
context.quote = char
# Unquoted or inside double quotes: substitutions still open scopes.
elif command.startswith("$(", cursor):
@@ -173,28 +162,27 @@ def _scope_keys(command: str, starts: list[int]) -> dict[int, tuple[int, ...]]:
cursor += 1
elif quote is None and char == "(":
contexts.append(_ShellContext("(", cursor))
elif quote is None and char == ")" and nested and contexts[-1].kind in {"(", "$("}:
elif (char == ")" and closes and context.kind in {"(", "$("}) or (
char == "`" and closes and context.kind == "`"):
contexts.pop()
elif char == "`":
if quote is None and nested and contexts[-1].kind == "`":
contexts.pop()
else:
contexts.append(_ShellContext("`", cursor))
contexts.append(_ShellContext("`", cursor))
cursor += 1
scopes[start] = tuple(item.opener for item in contexts[1:])
return scopes
def _operator_before(command: str, start: int) -> str | None:
"""The list/grouping operator (or newline) immediately preceding a command start."""
head = command[:start].rstrip()
if head[-2:] in {"&&", "||"}:
return head[-2:]
if head[-1:] in {";", "|", "&", "(", "{"}:
return head[-1:]
for tail in (head[-2:], head[-1:]):
if tail in {"&&", "||", ";", "|", "&", "(", "{"}:
return tail
return "\n" if "\n" in command[len(head):start] else None
def _cd_target(executable: str, args: list[str], cwd: Path) -> Path | None:
"""Directory a ``cd``/``pushd`` would land in (existing dirs only), else None."""
if _executable_name(executable) not in {"cd", "pushd"}:
return None
index = _consume_options(args, 0)
@@ -205,17 +193,15 @@ def _cd_target(executable: str, args: list[str], cwd: Path) -> Path | None:
def _shell_script_arg(args: list[str]) -> str | None:
"""Return the script string owned by a shell's ``-c``, if present. approval.py's
``_bash_exec_payload`` parses bash's real option grammar (``-o pipefail -c '<script>'``
hides ``-c`` behind an operand); when it finds no ``-c``, fall back to a permissive
positional scan, since zsh/dash/ksh option letters (``zsh -yc``) fall outside bash's
alphabet and would otherwise make this block-guard fail open."""
"""Script string owned by a shell's ``-c``, if present. ``_bash_exec_payload`` parses bash's
real option grammar (``-o pipefail -c '<script>'``); when it finds no ``-c``, fall back to a
permissive scan: zsh/dash/ksh letters (``zsh -yc``) would otherwise fail this guard open."""
has_c, payload = _bash_exec_payload(args)
if has_c:
return payload
for index, arg in enumerate(args):
if arg == "--" or not arg.startswith("-"):
break
return None
if "c" in arg[1:]:
return args[index + 1] if index + 1 < len(args) else None
return None
@@ -233,7 +219,7 @@ def _heredoc_specs(line: str) -> list[_Heredoc]:
index += 1 # skip the escaped character too
elif char == quote:
quote = None
elif char in {"'", '"'}:
elif char in "'\"":
quote = char
if quote or not line.startswith("<<", index) or line.startswith("<<<", index):
index += 1
@@ -241,52 +227,47 @@ def _heredoc_specs(line: str) -> list[_Heredoc]:
opener = _HEREDOC_OPENER_RE.match(line, index)
if opener is None: # unterminated quoted delimiter: give up on this line
break
operator_at, index = index, opener.end()
strip_tabs = bool(opener.group("dash"))
header, index = line[:index], opener.end()
delimiter = opener.group("quoted") if opener.group("q") else opener.group("bare")
if not delimiter:
continue
header = line[:operator_at]
starts = list(_iter_shell_command_starts(header))
words = _shell_words_at(header, starts[-1]) if starts else []
_, executable, args = _command_parts(words)
_, executable, args = _command_parts(_shell_words_at(header, starts[-1]) if starts else [])
# A bare shell (no -c script, no script operand) executes the body itself.
execute_as_shell = bool(
executable
and _executable_name(executable) in _SHELL_EXECUTABLES
executable and _executable_name(executable) in _SHELL_EXECUTABLES
and _shell_script_arg(args) is None
and not any(arg and not arg.startswith("-") for arg in args))
specs.append(_Heredoc(delimiter, strip_tabs, execute_as_shell))
specs.append(_Heredoc(delimiter, bool(opener.group("dash")), execute_as_shell))
return specs
def _mask_heredocs(command: str) -> tuple[str, list[str]]:
"""Blank heredoc bodies; return (masked command, bodies a bare shell would execute).
Unterminated heredocs run to end of input and are still reported."""
"""Blank heredoc bodies -> (masked command, bodies a bare shell would execute). Unterminated
heredocs run to end of input and are still reported."""
output: list[str] = []
pending: list[_Heredoc] = []
finished: list[_Heredoc] = []
for line in command.splitlines(keepends=True):
if pending:
current = pending[0]
candidate = line.rstrip("\r\n")
if current.strip_tabs:
candidate = candidate.lstrip("\t")
if candidate == current.delimiter:
finished.append(pending.pop(0))
else:
current.body.append(line)
output.append("".join(char if char in {"\r", "\n"} else " " for char in line))
if not pending:
output.append(line)
pending.extend(_heredoc_specs(line))
continue
output.append(line)
pending.extend(_heredoc_specs(line))
current = pending[0]
candidate = line.rstrip("\r\n")
if (candidate.lstrip("\t") if current.strip_tabs else candidate) == current.delimiter:
finished.append(pending.pop(0))
else:
current.body.append(line)
output.append(re.sub(r"[^\r\n]", " ", line))
shell_scripts = ["".join(spec.body) for spec in finished + pending if spec.execute_as_shell]
return "".join(output), shell_scripts
def _record_alias(config: str, aliases: dict[str, str]) -> None:
"""Record an inline ``-c alias.<name>=<value>`` git config override."""
if config.lower().startswith("alias.") and "=" in config:
key, value = config.split("=", 1)
key, sep, value = config.partition("=")
if sep and key.lower().startswith("alias."):
aliases[key[6:].lower()] = value
@@ -305,88 +286,70 @@ def _git_target_and_subcommand(
break
if not arg.startswith("-"):
break
# Separate-argument form (`-C dir`) vs attached form (`-Cdir`, `--work-tree=dir`, `-cK=V`).
if arg in _GIT_GLOBAL_OPTIONS_WITH_ARG:
if index + 1 < len(args):
value = args[index + 1]
if arg == "-C":
target = _resolve(value, target)
elif arg == "--work-tree":
work_tree = value
elif arg == "-c":
_record_alias(value, aliases)
option, value = arg, args[index + 1] if index + 1 < len(args) else None
index += 2
continue
if arg.startswith("-C") and len(arg) > 2:
target = _resolve(arg[2:], target)
elif arg.startswith("--work-tree="):
work_tree = arg.split("=", 1)[1]
elif arg.startswith("-calias."):
_record_alias(arg[2:], aliases)
index += 1
else:
option, value = next(((o, arg[len(o):]) for o in ("-C", "--work-tree=", "-c")
if arg.startswith(o) and len(arg) > len(o)), (None, None))
index += 1
if option == "-C" and value is not None:
target = _resolve(value, target)
elif option and option.startswith("--work-tree") and value is not None:
work_tree = value
elif option == "-c" and value is not None:
_record_alias(value, aliases) # only alias.* overrides matter; others are ignored
explicit_work_tree = work_tree or env.get("GIT_WORK_TREE")
if explicit_work_tree:
target = _resolve(explicit_work_tree, target)
return target, args[index].lower() if index < len(args) else None, args[index + 1 :], aliases
def _has_short_flag(arg: str, letter: str) -> bool:
return arg.startswith("-") and letter in arg[1:]
def _reset_mutates(args: list[str]) -> bool:
return any(arg in _RESET_WORKTREE_MODES or _RESET_HARD_RE.fullmatch(arg) for arg in args)
def _stash_mutates(args: list[str]) -> bool:
action = next((arg for arg in args if not arg.startswith("-")), "push")
return action not in _STASH_SAFE_ACTIONS
def _clean_mutates(args: list[str]) -> bool:
return not any(
arg == "--dry-run" or (not arg.startswith("--") and _has_short_flag(arg, "n"))
def _has_flag(args: list[str], long: str, letter: str, short_only: bool = False) -> bool:
"""``--long`` present, or ``letter`` inside a dash-prefixed arg (``-fdn``; ``short_only``
additionally excludes ``--`` args from the letter scan)."""
return any(
arg == long or (arg.startswith("-") and letter in arg[1:]
and not (short_only and arg.startswith("--")))
for arg in args)
def _restore_mutates(args: list[str]) -> bool:
staged = any(arg == "--staged" or _has_short_flag(arg, "S") for arg in args)
worktree = any(arg == "--worktree" or _has_short_flag(arg, "W") for arg in args)
return worktree or not staged
# Subcommands whose worktree impact depends on their arguments.
# Subcommands whose worktree impact depends on their arguments -> predicate(args).
_CONDITIONAL_MUTATIONS: dict[str, Callable[[list[str]], bool]] = {
"reset": _reset_mutates,
"stash": _stash_mutates,
"clean": _clean_mutates,
"restore": _restore_mutates}
"reset": lambda args: any(
arg in _RESET_WORKTREE_MODES or _RESET_HARD_RE.fullmatch(arg) for arg in args),
"stash": lambda args: next(
(arg for arg in args if not arg.startswith("-")), "push") not in _STASH_SAFE_ACTIONS,
"clean": lambda args: not _has_flag(args, "--dry-run", "n", short_only=True),
# `restore` touches the worktree unless ONLY --staged was requested.
"restore": lambda args: (_has_flag(args, "--worktree", "W")
or not _has_flag(args, "--staged", "S"))}
def _mutates_worktree(subcommand: str, args: list[str]) -> bool:
check = _CONDITIONAL_MUTATIONS.get(subcommand)
return check(args) if check is not None else subcommand in _WORKTREE_MUTATIONS
check = _CONDITIONAL_MUTATIONS.get(subcommand, lambda _args: subcommand in _WORKTREE_MUTATIONS)
return check(args)
def _inspect_git_worktree(args: list[str], cwd: Path, root: Path) -> str | None:
"""Block `worktree remove|move` aimed at the running root, from any directory."""
action_index = _consume_options(args, 0)
action = args[action_index].lower() if action_index < len(args) else None
if action not in _WORKTREE_TARGET_ACTIONS:
return None
target_index = _consume_options(args, action_index + 1)
if target_index < len(args) and _resolve(args[target_index], cwd) == root:
if (action in _WORKTREE_TARGET_ACTIONS and target_index < len(args)
and _resolve(args[target_index], cwd) == root):
return f"git worktree {action}"
return None
def _read_git_alias(executable: str, target: Path, alias: str) -> str | None:
try:
with contextlib.suppress(OSError, subprocess.SubprocessError):
result = subprocess.run(
[executable, "-C", str(target), "config", "--get", f"alias.{alias}"],
capture_output=True, text=True, timeout=1, check=False)
except (OSError, subprocess.SubprocessError):
return None
return (result.stdout.strip() or None) if result.returncode == 0 else None
return (result.stdout.strip() or None) if result.returncode == 0 else None
return None
def _inspect_git(
@@ -396,8 +359,7 @@ def _inspect_git(
args, current_dir, env)
if subcommand is None:
return None
# `worktree` names its victim as an argument, so the cwd check does not apply.
if subcommand == "worktree":
if subcommand == "worktree": # names its victim as an argument: the cwd check does not apply
return _inspect_git_worktree(sub_args, target, root)
if not _is_within(target, root):
return None
@@ -405,18 +367,16 @@ def _inspect_git(
return f"git {subcommand}"
if subcommand in _KNOWN_GIT_BUILTINS or depth >= _MAX_RECURSION:
return None
alias = inline_aliases.get(subcommand)
if alias is None:
alias = _read_git_alias(executable, target, subcommand)
alias = (inline_aliases[subcommand] if subcommand in inline_aliases
else _read_git_alias(executable, target, subcommand))
if not alias:
return None
if alias.startswith("!"):
if alias.startswith("!"): # shell alias: scan it as a command
return _find_mutation(alias[1:], target, root, depth + 1)
try:
with contextlib.suppress(ValueError):
alias_args = shlex.split(alias, posix=True)
except ValueError:
return None
return _inspect_git(executable, [*alias_args, *sub_args], target, {}, root, depth + 1)
return _inspect_git(executable, [*alias_args, *sub_args], target, {}, root, depth + 1)
return None
def _inspect_github_cli(
@@ -425,9 +385,8 @@ def _inspect_github_cli(
if not _is_within(current_dir, root):
return None
index = _consume_options(args, 0, frozenset({"-R", "--repo", "--hostname"}))
if args[index : index + 2] == ["pr", "checkout"]:
return f"{_executable_name(executable)} pr checkout"
return None
is_checkout = args[index : index + 2] == ["pr", "checkout"]
return f"{_executable_name(executable)} pr checkout" if is_checkout else None
def _inspect_shell(
@@ -439,9 +398,7 @@ def _inspect_shell(
# executable name -> inspector(executable, args, current_dir, env, root, depth)
_INSPECTORS: dict[str, Callable[..., str | None]] = {
"git": _inspect_git,
"gh": _inspect_github_cli,
"hub": _inspect_github_cli,
"git": _inspect_git, "gh": _inspect_github_cli, "hub": _inspect_github_cli,
**{shell: _inspect_shell for shell in _SHELL_EXECUTABLES}}
@@ -451,8 +408,7 @@ def _find_mutation(command: str, cwd: Path, root: Path, depth: int = 0) -> str |
return None
masked_command, heredoc_scripts = _mask_heredocs(command)
for script in heredoc_scripts:
operation = _find_mutation(script, cwd, root, depth + 1)
if operation:
if operation := _find_mutation(script, cwd, root, depth + 1):
return operation
starts = sorted(set(_iter_shell_command_starts(masked_command)))
scopes = _scope_keys(masked_command, starts)
@@ -462,50 +418,42 @@ def _find_mutation(command: str, cwd: Path, root: Path, depth: int = 0) -> str |
for start in starts:
scope = scopes[start]
cwd_by_scope.setdefault(scope, cwd_by_scope.get(scope[:-1], cwd))
operator = _operator_before(masked_command, start)
pending = pending_cd.pop(scope, None)
if pending is not None and operator in {"&&", ";", "\n"}:
if pending is not None and _operator_before(masked_command, start) in {"&&", ";", "\n"}:
cwd_by_scope[scope] = pending
env, executable, args = _command_parts(_shell_words_at(masked_command, start))
if executable is None:
continue
current_dir = cwd_by_scope[scope]
cd_target = _cd_target(executable, args, current_dir)
if cd_target is not None:
if (cd_target := _cd_target(executable, args, current_dir)) is not None:
pending_cd[scope] = cd_target
continue
inspect = _INSPECTORS.get(_executable_name(executable))
if inspect is not None:
operation = inspect(executable, args, current_dir, env, root, depth)
if operation:
return operation
elif (inspect := _INSPECTORS.get(_executable_name(executable))) and (
operation := inspect(executable, args, current_dir, env, root, depth)):
return operation
return None
def guard_active() -> bool:
"""Whether the self-repo git guard applies on this platform. Windows-only: NTFS locks
loaded .py/.pyd files, so overwriting the live checkout can corrupt the running
process. On POSIX open handles keep the old inode alive; the mixed-module hazard is
limited to later lazy imports — not worth blocking every git workflow for."""
"""Windows-only: NTFS locks loaded .py/.pyd files, so overwriting the live checkout can
corrupt the running process. On POSIX open handles keep the old inode alive; the mixed-module
hazard is limited to later lazy imports — not worth blocking every git workflow for."""
return os.name == "nt"
def detect_self_repo_git_mutation(
command: str, cwd: str | None, source_root: Path | None = None) -> tuple[bool, str | None]:
"""Return whether a command would rewrite the live source checkout."""
"""-> (blocked, message): whether a command would rewrite the live source checkout."""
root = source_root if source_root is not None else get_running_source_root()
if root is None or not command:
return False, None
root = _resolve(str(root), Path("/"))
operation = _find_mutation(command, _resolve(cwd, Path("/")) if cwd else Path("/"), root)
operation = _find_mutation(command, _resolve(cwd or "/", Path("/")), root)
return (True, _block_message(operation, root)) if operation is not None else (False, None)
def _block_message(operation: str, root: Path) -> str:
# Suggest a disk-backed scratch dir: /tmp is usually tmpfs (see message).
hermes_home = os.environ.get("HERMES_HOME", "").strip()
home = Path(hermes_home).expanduser() if hermes_home else Path.home() / ".hermes"
scratch = home / "scratch"
scratch = (Path(hermes_home).expanduser() if hermes_home else Path.home() / ".hermes") / "scratch"
return (
f"Blocked: `{operation}` would rewrite Hermes's live source checkout "
f"({root}) and can mix module versions in this running process. "
+62 -111
View File
@@ -1,19 +1,18 @@
"""Conservative heredoc masking for shell-command scanners (terminal '&' guard, blocked-command
checks, cron lifecycle_guard) that false-positive on heredoc *bodies*. Stripping every body is
unsafe the other way (a fake ``<<`` in quotes can swallow a real operator; unquoted bodies
expand; ``bash <<'EOF'`` executes), so a body is masked ONLY when every delimiter is quoted,
every heredoc has an exact terminator line, the opener is a single command (no ``;|&``,
``$(...)``, backticks or process substitution) and the consumer is an allowlisted non-shell
interpreter. Otherwise the command is returned untouched: a false positive is acceptable,
hiding shell syntax from a guard is not. Masked bodies keep their newline count (re.MULTILINE)."""
"""Conservative heredoc masking for shell-command scanners ('&' guard, blocked-command checks,
cron lifecycle_guard) that false-positive on heredoc *bodies*. Stripping every body is unsafe the
other way (a fake ``<<`` in quotes can swallow an operator; unquoted bodies expand; ``bash <<'EOF'``
executes), so a body is masked ONLY when every delimiter is quoted, every heredoc has an exact
terminator line, the opener is a single command (no ``;|&``, ``$(...)``, backticks, process
substitution) and the consumer is an allowlisted non-shell interpreter. Otherwise the command is
returned untouched: a false positive is acceptable, hiding shell syntax from a guard is not.
Masked bodies keep their newline count (re.MULTILINE)."""
from __future__ import annotations
import re
# Non-shell interpreters whose quoted heredoc bodies are program text/data for
# THAT interpreter. Optional VAR=... assignments, ``env`` and a path prefix are
# allowed. Deliberately narrow: anything unmatched keeps its body visible.
# Non-shell interpreters whose quoted heredoc bodies are data for THAT interpreter; optional
# VAR=... assignments, ``env`` and a path prefix allowed. Narrow on purpose: unmatched = visible.
_INERT_HEREDOC_CONSUMER_RE = re.compile(
r"^\s*(?:[A-Z_][A-Z0-9_]*=\S+\s+)*(?:env\s+)?(?:[A-Za-z0-9_./-]+/)?"
r"(?:python(?:3(?:\.\d+)*)?|osascript|cat)(?=\s|$)",
@@ -24,12 +23,9 @@ def _span_end(command: str, cursor: int, closer: str) -> int:
"""Index just past the backslash-aware span opened at ``cursor``."""
end = cursor + 1
while end < len(command):
if command[end] == "\\" and end + 1 < len(command):
end += 2
continue
if command[end] == closer:
return end + 1
end += 1
end += 2 if command[end] == "\\" and end + 1 < len(command) else 1
return end
@@ -39,20 +35,15 @@ def _mask_simple_quotes(command: str) -> str:
cursor = 0
while cursor < len(command):
char = command[cursor]
if char == "'":
closing = command.find("'", cursor + 1)
if closing == -1:
result.append(command[cursor:])
break
result.append("''")
cursor = closing + 1
elif char == '"':
end = _span_end(command, cursor, '"')
if not command[cursor:end].endswith('"'):
result.append(command[cursor:])
break
if char in "'\"": # single quotes have no escapes; double quotes are backslash-aware
end = (command.find("'", cursor + 1) + 1 if char == "'"
else _span_end(command, cursor, '"'))
segment = command[cursor:end]
result.append(segment if "$(" in segment or "`" in segment else '""')
if not segment.endswith(char):
result.append(command[cursor:])
break
keep = char == '"' and ("$(" in segment or "`" in segment)
result.append(segment if keep else char * 2)
cursor = end
elif char == "`":
end = _span_end(command, cursor, "`")
@@ -68,118 +59,88 @@ def _parse_heredoc_operator(command: str, index: int):
"""Parse one ``<<`` opener -> ``(end_index, delimiter, strip_tabs, quoted)`` or None."""
if not command.startswith("<<", index) or command.startswith("<<<", index):
return None
cursor = index + 2
strip_tabs = cursor < len(command) and command[cursor] == "-"
if strip_tabs:
cursor += 1
strip_tabs = command.startswith("-", index + 2)
cursor = index + 3 if strip_tabs else index + 2
while cursor < len(command) and command[cursor] in " \t":
cursor += 1
if cursor >= len(command) or command[cursor] in "\r\n":
return None
delimiter: list[str] = []
quoted = False
while cursor < len(command):
while cursor < len(command) and not (command[cursor].isspace() or command[cursor] in ";&|<>()"):
char = command[cursor]
if char.isspace() or char in ";&|<>()":
break
if char == "\\":
if char == "\\": # backslash-escaped char: quoted, literal
if cursor + 1 >= len(command) or command[cursor + 1] in "\r\n":
return None
quoted = True
delimiter.append(command[cursor + 1])
cursor += 2
continue
if char in "'\"":
elif char in "'\"":
quoted = True
quote = char
cursor += 1
while cursor < len(command) and command[cursor] != quote:
while cursor < len(command) and command[cursor] != char:
current = command[cursor]
if current in "\r\n":
return None
if quote == '"' and current == "\\":
if char == '"' and current == "\\":
if cursor + 1 >= len(command):
return None
following = command[cursor + 1]
if following in {"$", "`", '"', "\\", "\n"}:
delimiter.append(following)
cursor += 2
continue
# In double quotes, backslash is literal before other chars.
if command[cursor + 1] in '$`"\\\n': # else backslash is literal in dquotes
cursor += 1
current = command[cursor]
delimiter.append(current)
cursor += 1
if cursor >= len(command):
if cursor >= len(command): # unterminated quote
return None
cursor += 1
continue
delimiter.append(char)
cursor += 1
else:
delimiter.append(char)
cursor += 1
if not delimiter and not quoted:
return None
return cursor, "".join(delimiter), strip_tabs, quoted
def _scan_heredoc_command_unit(command: str, start: int):
"""Scan one logical command -> ``(end, specs, unknown_operator, has_list_operator)``:
an unparseable ``<<`` (caller must fail closed) / unquoted ``;|&`` on the opener."""
"""Scan one logical command -> ``(end, specs, unknown_operator, has_list_operator)``: an
unparseable ``<<`` (caller must fail closed) / an unquoted ``;|&`` on the opener line."""
cursor = start
quote = None
comment = False
specs = []
unknown_operator = False
has_list_operator = False
while cursor < len(command):
char = command[cursor]
if comment:
if char == "\n":
return cursor, specs, unknown_operator, has_list_operator
cursor += 1
continue
if quote is not None:
if quote in {'"', "`"} and char == "\\" and cursor + 1 < len(command):
cursor += 2
continue
if char == "\n" and (comment or quote is None):
break
# Backslash escapes (incl. line continuations) outside single quotes skip the next char.
escaped = char == "\\" and quote != "'" and not comment and cursor + 1 < len(command)
if comment or quote is not None or escaped:
if char == quote:
quote = None
cursor += 1
continue
if char == "\\" and cursor + 1 < len(command):
# Includes line continuations: the logical command keeps going.
cursor += 2
continue
if char in "'\"`":
cursor += 2 if escaped else 1
elif char in "'\"`":
quote = char
cursor += 1
continue
if char == "#":
previous = command[cursor - 1] if cursor > start else ""
if cursor == start or previous.isspace() or previous in ";&|()":
comment = True
cursor += 1
continue
if char == "\n":
return cursor, specs, unknown_operator, has_list_operator
if command.startswith("<<<", cursor):
elif char == "#" and (cursor == start or command[cursor - 1].isspace()
or command[cursor - 1] in ";&|()"):
comment = True
cursor += 1
elif command.startswith("<<<", cursor):
cursor += 3
continue
if command.startswith("<<", cursor):
elif command.startswith("<<", cursor):
parsed = _parse_heredoc_operator(command, cursor)
if parsed is None:
unknown_operator = True
cursor += 2
continue
cursor, delimiter, strip_tabs, quoted = parsed
specs.append((delimiter, strip_tabs, quoted))
continue
if char in ";|&":
has_list_operator = True
cursor += 1
return len(command), specs, unknown_operator, has_list_operator
else:
cursor, delimiter, strip_tabs, quoted = parsed
specs.append((delimiter, strip_tabs, quoted))
else:
has_list_operator = has_list_operator or char in ";|&"
cursor += 1
return cursor, specs, unknown_operator, has_list_operator
def _find_heredoc_close(
@@ -200,14 +161,12 @@ def _find_heredoc_close(
def strip_inert_heredoc_bodies(command: str) -> str:
"""Mask heredoc bodies that are provably inert data (see module docstring)."""
# Runs on every terminal call: skip the state machine when no '<<' exists,
# and stop scanning once past the last '<<'.
# Runs on every terminal call: skip the state machine when no '<<' exists; stop past the last.
if "<<" not in command:
return command
last_opener_index = command.rfind("<<")
ranges: list[tuple[int, int]] = []
command_start = 0
while command_start <= last_opener_index:
command_end, specs, unknown_operator, has_list_operator = (
_scan_heredoc_command_unit(command, command_start))
@@ -219,9 +178,7 @@ def strip_inert_heredoc_bodies(command: str) -> str:
command_start = command_end + 1
continue
if command_end >= len(command):
# Opener with no body line: unterminated — leave visible.
return command
return command # opener with no body line: unterminated — leave visible
body_cursor = command_end + 1
body_ranges: list[tuple[int, int]] = []
for delimiter, strip_tabs, _quoted in specs:
@@ -230,22 +187,16 @@ def strip_inert_heredoc_bodies(command: str) -> str:
return command # unterminated
body_ranges.append((body_cursor, close_end))
body_cursor = close_end
if all(quoted for _delimiter, _strip_tabs, quoted in specs) and not has_list_operator:
masked_opener = _mask_simple_quotes(command[command_start:command_end])
nested_scope = any(m in masked_opener for m in ("$(", "`", "<(", ">("))
if not nested_scope and _INERT_HEREDOC_CONSUMER_RE.search(masked_opener):
if (not any(m in masked_opener for m in ("$(", "`", "<(", ">("))
and _INERT_HEREDOC_CONSUMER_RE.search(masked_opener)):
ranges.extend(body_ranges)
command_start = body_cursor
if not ranges:
return command
# Single-pass rebuild: ranges are sorted and non-overlapping.
# Single-pass rebuild (ranges are sorted and non-overlapping), bodies -> their newlines only.
parts: list[str] = []
previous = 0
for start, end in ranges:
parts.append(command[previous:start])
parts.append("\n" * command.count("\n", start, end))
parts += [command[previous:start], "\n" * command.count("\n", start, end)]
previous = end
parts.append(command[previous:])
return "".join(parts)
return "".join(parts) + command[previous:]