Merge branch 'simp/r3-33-M' into simp/r3-33
This commit is contained in:
+121
-171
@@ -1,14 +1,9 @@
|
||||
#!/usr/bin/env python3
|
||||
"""V4A patch format parser and applier (format used by codex, cline, etc.).
|
||||
|
||||
*** Begin Patch / *** End Patch wrap the operations:
|
||||
*** Update File: p.py then hunks: ``@@ hint @@``, `` ctx``, ``-old``, ``+new``
|
||||
*** Add File: n.py then ``+`` lines; *** Delete File: o.py; *** Move File: a -> b
|
||||
|
||||
operations, error = parse_v4a_patch(patch_content)
|
||||
result = apply_v4a_operations(operations, file_ops)
|
||||
"""
|
||||
"""V4A patch parser/applier (codex, cline). ``*** Begin Patch``/``*** End Patch`` wrap ops:
|
||||
``*** Update File: p`` + hunks (``@@ hint @@``, `` ctx``, ``-old``, ``+new``); ``*** Add File: n``
|
||||
+ ``+`` lines; ``*** Delete File: o``; ``*** Move File: a -> b``. Entry points:
|
||||
``parse_v4a_patch(text) -> (ops, error)`` and ``apply_v4a_operations(ops, file_ops)``."""
|
||||
|
||||
import contextlib
|
||||
import difflib
|
||||
import inspect
|
||||
import re
|
||||
@@ -42,7 +37,6 @@ class PatchOperation:
|
||||
file_path: str
|
||||
new_path: Optional[str] = None # MOVE only
|
||||
hunks: List[Hunk] = field(default_factory=list)
|
||||
content: Optional[str] = None # ADD only
|
||||
|
||||
|
||||
# Markers must occupy the whole line at column 0 so content lines that merely
|
||||
@@ -58,12 +52,9 @@ _HINT_RE = re.compile(r'@@\s*(.+?)\s*@@')
|
||||
|
||||
|
||||
def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[str]]:
|
||||
"""Parse a V4A patch -> ``(operations, None)`` (``[]`` for an empty patch is not an
|
||||
error) or ``([], "Parse error: ...")`` for malformed operations."""
|
||||
# Tolerate CRLF bodies: a stray ``\r`` would otherwise end up in every
|
||||
# HunkLine.content and defeat the anchored Begin/End markers.
|
||||
"""-> ``(operations, None)`` (empty patch = ``[]``, no error) or ``([], "Parse error: …")``."""
|
||||
# Tolerate CRLF: a stray ``\r`` would land in every HunkLine.content and defeat the markers.
|
||||
lines = [ln[:-1] if ln.endswith('\r') else ln for ln in patch_content.split('\n')]
|
||||
|
||||
start_idx = -1 # parse from the top when no Begin marker is present
|
||||
end_idx = len(lines)
|
||||
for i, line in enumerate(lines):
|
||||
@@ -72,7 +63,6 @@ def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[
|
||||
elif _END_MARKER.match(line):
|
||||
end_idx = i
|
||||
break
|
||||
|
||||
operations: List[PatchOperation] = []
|
||||
current_op: Optional[PatchOperation] = None
|
||||
current_hunk: Optional[Hunk] = None
|
||||
@@ -87,8 +77,7 @@ def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[
|
||||
operations.append(current_op)
|
||||
|
||||
for line in lines[start_idx + 1:end_idx]:
|
||||
op_match = next(
|
||||
((kind, m) for kind, rx in _OP_MARKERS if (m := rx.match(line))), None)
|
||||
op_match = next(((kind, m) for kind, rx in _OP_MARKERS if (m := rx.match(line))), None)
|
||||
if op_match:
|
||||
kind, m = op_match
|
||||
_flush()
|
||||
@@ -96,8 +85,8 @@ def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[
|
||||
operation=kind,
|
||||
file_path=m.group(1).strip(),
|
||||
new_path=m.group(2).strip() if kind is OperationType.MOVE else None)
|
||||
# UPDATE hunks start lazily (at '@@' or the first hunk line); ADD
|
||||
# collects all '+' lines into one hunk; DELETE/MOVE are complete.
|
||||
# UPDATE hunks start lazily ('@@' or first hunk line); ADD collects all '+' lines
|
||||
# into one hunk; DELETE/MOVE are complete.
|
||||
current_hunk = Hunk() if kind is OperationType.ADD else None
|
||||
if kind in (OperationType.DELETE, OperationType.MOVE):
|
||||
operations.append(current_op)
|
||||
@@ -115,18 +104,16 @@ def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[
|
||||
elif line[0] != '\\': # "\ No newline at end of file" marker is skipped
|
||||
current_hunk.lines.append(HunkLine(' ', line)) # implicit context line
|
||||
_flush()
|
||||
|
||||
parse_errors: List[str] = []
|
||||
for op in operations:
|
||||
if not op.file_path:
|
||||
parse_errors.append("Operation with empty file path")
|
||||
if op.operation == OperationType.UPDATE and not op.hunks:
|
||||
if op.operation is OperationType.UPDATE and not op.hunks:
|
||||
parse_errors.append(f"UPDATE {op.file_path!r}: no hunks found")
|
||||
if op.operation == OperationType.MOVE and not op.new_path:
|
||||
parse_errors.append(f"MOVE {op.file_path!r}: missing destination path (expected 'src -> dst')")
|
||||
if parse_errors:
|
||||
return [], "Parse error: " + "; ".join(parse_errors)
|
||||
return operations, None
|
||||
if op.operation is OperationType.MOVE and not op.new_path:
|
||||
parse_errors.append(
|
||||
f"MOVE {op.file_path!r}: missing destination path (expected 'src -> dst')")
|
||||
return ([], "Parse error: " + "; ".join(parse_errors)) if parse_errors else (operations, None)
|
||||
|
||||
|
||||
def _count_occurrences(text: str, pattern: str) -> int:
|
||||
@@ -136,66 +123,57 @@ def _count_occurrences(text: str, pattern: str) -> int:
|
||||
|
||||
def _split_hunk(hunk: Hunk) -> Tuple[List[str], List[str]]:
|
||||
"""``(search_lines, replace_lines)``: context+removed vs context+added."""
|
||||
search = [l.content for l in hunk.lines if l.prefix in {' ', '-'}]
|
||||
replace = [l.content for l in hunk.lines if l.prefix in {' ', '+'}]
|
||||
return search, replace
|
||||
return ([l.content for l in hunk.lines if l.prefix != '+'],
|
||||
[l.content for l in hunk.lines if l.prefix != '-'])
|
||||
|
||||
|
||||
def _no_match_hint(error: Optional[str], search_pattern: str, content: str) -> str:
|
||||
"""Best-effort 'Did you mean...' suffix; never lets a hint failure mask the real error."""
|
||||
try:
|
||||
with contextlib.suppress(Exception):
|
||||
from tools.fuzzy_match import format_no_match_hint
|
||||
return format_no_match_hint(error, 0, search_pattern, content)
|
||||
except Exception:
|
||||
return ""
|
||||
return ""
|
||||
|
||||
|
||||
def _hint_ambiguity(content: str, hint: str, tail: str = "") -> Tuple[int, str]:
|
||||
"""(occurrences, error) for an addition-only hunk's context hint; error is '' when unique."""
|
||||
occurrences = _count_occurrences(content, hint)
|
||||
if occurrences > 1:
|
||||
return occurrences, (f"context hint '{hint}' is ambiguous "
|
||||
f"({occurrences} occurrences){tail}")
|
||||
return occurrences, ""
|
||||
n = _count_occurrences(content, hint)
|
||||
return n, f"context hint '{hint}' is ambiguous ({n} occurrences){tail}" if n > 1 else ""
|
||||
|
||||
|
||||
def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> List[str]:
|
||||
"""Dry-run every operation; return error strings (empty list = safe to apply). UPDATE
|
||||
hunks are simulated in order so later hunks see post-earlier-hunk content, as apply will."""
|
||||
"""Dry-run every operation -> error strings (empty = safe). UPDATE hunks are simulated in
|
||||
order so later hunks see post-earlier-hunk content, exactly as apply will."""
|
||||
from tools.fuzzy_match import fuzzy_find_and_replace, is_already_applied
|
||||
|
||||
errors: List[str] = []
|
||||
real_change_count = 0
|
||||
|
||||
# Virtual overlay so inter-op state validates (e.g. a MOVE creating the destination
|
||||
# a later UPDATE targets): path -> content from an earlier op; paths MOVE/DELETE removed.
|
||||
# Overlay so inter-op state validates (a MOVE creating the path a later UPDATE targets).
|
||||
pending_content: dict = {}
|
||||
removed_paths: set = set()
|
||||
|
||||
def _read(path: str):
|
||||
if path in removed_paths and path not in pending_content:
|
||||
return None, "file not found"
|
||||
def _read(path: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
if path in pending_content:
|
||||
return pending_content[path], None
|
||||
if path in removed_paths:
|
||||
return None, "file not found"
|
||||
r = file_ops.read_file_raw(path)
|
||||
return (None, r.error) if r.error else (r.content, None)
|
||||
|
||||
def _validate_update(op: PatchOperation) -> None:
|
||||
nonlocal real_change_count
|
||||
content, read_err = _read(op.file_path)
|
||||
simulated, read_err = _read(op.file_path)
|
||||
if read_err:
|
||||
errors.append(f"{op.file_path}: {read_err}")
|
||||
return
|
||||
simulated = content
|
||||
for hunk_index, hunk in enumerate(op.hunks, start=1):
|
||||
search_lines, replace_lines = _split_hunk(hunk)
|
||||
if not any(l.prefix in '-+' for l in hunk.lines):
|
||||
# Inert anchor hunk (context only) — models emit these
|
||||
# between real changes; ignore without failing the patch.
|
||||
if search_lines == replace_lines:
|
||||
# Context-only anchor hunks (models emit these between changes) are inert; identical
|
||||
# -/+ lines are skipped by apply as a no-op — neither may fail validation.
|
||||
real_change_count += any(l.prefix in '-+' for l in hunk.lines)
|
||||
continue
|
||||
real_change_count += 1
|
||||
if not search_lines:
|
||||
# Addition-only hunk: the context hint must be unique.
|
||||
if not search_lines: # addition-only: the context hint must be unique
|
||||
if hunk.context_hint:
|
||||
occurrences, ambiguous = _hint_ambiguity(simulated, hunk.context_hint)
|
||||
if occurrences == 0:
|
||||
@@ -204,19 +182,13 @@ def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> Lis
|
||||
elif ambiguous:
|
||||
errors.append(f"{op.file_path}: addition-only hunk {ambiguous}")
|
||||
continue
|
||||
search_pattern = '\n'.join(search_lines)
|
||||
replacement = '\n'.join(replace_lines)
|
||||
if search_lines == replace_lines:
|
||||
# Identical -/+ lines: apply skips it as a no-op, so
|
||||
# validation must not reject it with the identical-strings error.
|
||||
continue
|
||||
search_pattern, replacement = '\n'.join(search_lines), '\n'.join(replace_lines)
|
||||
new_simulated, count, _strategy, match_error = fuzzy_find_and_replace(
|
||||
simulated, search_pattern, replacement, replace_all=False)
|
||||
if count:
|
||||
simulated = new_simulated
|
||||
elif not is_already_applied(simulated or "", search_pattern, replacement):
|
||||
# Already-applied hunks (edit landed in a prior call) are no-ops so
|
||||
# multi-hunk patches don't fail wholesale; apply performs the same skip.
|
||||
# Already-applied hunks are no-ops (apply performs the same skip).
|
||||
label = f"'{hunk.context_hint}'" if hunk.context_hint else "(no hint)"
|
||||
errors.append(
|
||||
f"{op.file_path}: hunk {hunk_index} {label} not found"
|
||||
@@ -224,18 +196,20 @@ def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> Lis
|
||||
+ _no_match_hint(match_error, search_pattern, simulated))
|
||||
pending_content[op.file_path] = simulated
|
||||
|
||||
def _remove(path: str) -> None:
|
||||
removed_paths.add(path)
|
||||
pending_content.pop(path, None)
|
||||
|
||||
for op in operations:
|
||||
if op.operation == OperationType.UPDATE:
|
||||
_validate_update(op)
|
||||
continue
|
||||
real_change_count += 1
|
||||
if op.operation == OperationType.DELETE:
|
||||
_content, read_err = _read(op.file_path)
|
||||
if read_err:
|
||||
if _read(op.file_path)[1]:
|
||||
errors.append(f"{op.file_path}: file not found for deletion")
|
||||
else:
|
||||
removed_paths.add(op.file_path)
|
||||
pending_content.pop(op.file_path, None)
|
||||
_remove(op.file_path)
|
||||
elif op.operation == OperationType.MOVE:
|
||||
if not op.new_path:
|
||||
errors.append(f"{op.file_path}: MOVE operation missing destination path")
|
||||
@@ -243,16 +217,12 @@ def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> Lis
|
||||
src_content, src_err = _read(op.file_path)
|
||||
if src_err:
|
||||
errors.append(f"{op.file_path}: source file not found for move")
|
||||
_dst, dst_err = _read(op.new_path)
|
||||
if not dst_err:
|
||||
if not _read(op.new_path)[1]:
|
||||
errors.append(f"{op.new_path}: destination already exists — move would overwrite")
|
||||
# Only a cleanly-validated move updates the overlay.
|
||||
if not src_err and dst_err:
|
||||
elif not src_err: # only a cleanly-validated move updates the overlay
|
||||
pending_content[op.new_path] = src_content if src_content is not None else ""
|
||||
pending_content.pop(op.file_path, None)
|
||||
removed_paths.add(op.file_path)
|
||||
# ADD: parent directory creation handled by write_file; no pre-check needed.
|
||||
|
||||
_remove(op.file_path)
|
||||
# ADD: write_file creates parent directories; no pre-check needed.
|
||||
if not errors and real_change_count == 0:
|
||||
errors.append("Patch contains no changes (only context lines were provided)")
|
||||
return errors
|
||||
@@ -262,104 +232,101 @@ def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> Lis
|
||||
ApplyResult = Tuple[bool, str, Optional[str], Optional[dict]]
|
||||
|
||||
|
||||
def _fail(error: str) -> ApplyResult:
|
||||
return False, error, None, None
|
||||
|
||||
|
||||
def _written(result: Any, diff: str) -> ApplyResult:
|
||||
"""Outcome of a write: its error, else success with LSP/lint propagated from the WriteResult."""
|
||||
if result.error:
|
||||
return _fail(result.error)
|
||||
return True, diff, getattr(result, "lsp_diagnostics", None), getattr(result, "lint", None)
|
||||
|
||||
|
||||
def _unified_diff(path: str, old: str, new: Optional[str]) -> str:
|
||||
"""Unified diff ``a/path`` -> ``b/path`` (``new=None`` = deletion, ``/dev/null``)."""
|
||||
return ''.join(difflib.unified_diff(
|
||||
old.splitlines(keepends=True), [] if new is None else new.splitlines(keepends=True),
|
||||
fromfile=f"a/{path}", tofile="/dev/null" if new is None else f"b/{path}"))
|
||||
|
||||
|
||||
def apply_v4a_operations(operations: List[PatchOperation], file_ops: Any) -> 'PatchResult':
|
||||
"""Validate all operations, then apply them (two-phase, atomic on validation failure).
|
||||
A phase-2 failure (validate/apply race) is reported with a ``git diff`` note since state
|
||||
may be inconsistent. ``file_ops`` needs read_file_raw/write_file/delete_file/move_file."""
|
||||
"""Two-phase: validate everything, then apply (atomic on validation failure). A phase-2
|
||||
failure (validate/apply race) carries a ``git diff`` note since state may be inconsistent.
|
||||
``file_ops`` needs read_file_raw/write_file/delete_file/move_file."""
|
||||
from tools.file_operations import PatchResult # avoid circular import
|
||||
|
||||
validation_errors = _validate_operations(operations, file_ops)
|
||||
if validation_errors:
|
||||
def _bullets(errs: List[str]) -> str:
|
||||
return "\n".join(f" • {e}" for e in errs)
|
||||
|
||||
if errors := _validate_operations(operations, file_ops):
|
||||
return PatchResult(
|
||||
success=False,
|
||||
error="Patch validation failed (no files were modified):\n"
|
||||
+ "\n".join(f" • {e}" for e in validation_errors))
|
||||
|
||||
error="Patch validation failed (no files were modified):\n" + _bullets(errors))
|
||||
files: Dict[str, List[str]] = {"created": [], "deleted": [], "modified": []}
|
||||
all_diffs: List[str] = []
|
||||
# V4A bypasses the WriteResult/PatchResult plumbing that write_file uses,
|
||||
# so LSP diagnostics and lint must be propagated explicitly per file.
|
||||
# V4A bypasses write_file's WriteResult plumbing: LSP diagnostics and lint propagate per file.
|
||||
lsp_blocks: List[str] = []
|
||||
errors: List[str] = []
|
||||
lint_results: Dict[str, dict] = {}
|
||||
|
||||
for op in operations:
|
||||
handler, verb, bucket = _APPLY_DISPATCH[op.operation]
|
||||
try:
|
||||
handler, verb, bucket = _APPLY_DISPATCH[op.operation]
|
||||
ok, payload, lsp, lint = handler(op, file_ops)
|
||||
if not ok:
|
||||
errors.append(f"Failed to {verb} {op.file_path}: {payload}")
|
||||
continue
|
||||
label = op.file_path
|
||||
if op.operation is OperationType.MOVE:
|
||||
label = f"{op.file_path} -> {op.new_path}"
|
||||
files[bucket].append(label)
|
||||
all_diffs.append(payload)
|
||||
if lsp:
|
||||
lsp_blocks.append(lsp)
|
||||
if lint:
|
||||
lint_results[op.file_path] = lint
|
||||
except Exception as e:
|
||||
errors.append(f"Error processing {op.file_path}: {str(e)}")
|
||||
|
||||
# Each LSP block carries its own <diagnostics file="..."> header, so plain
|
||||
# concatenation keeps per-file attribution.
|
||||
result_kwargs = dict(
|
||||
ok, payload = None, str(e)
|
||||
if not ok:
|
||||
prefix = f"Failed to {verb}" if ok is False else "Error processing"
|
||||
errors.append(f"{prefix} {op.file_path}: {payload}")
|
||||
continue
|
||||
is_move = op.operation is OperationType.MOVE
|
||||
files[bucket].append(f"{op.file_path} -> {op.new_path}" if is_move else op.file_path)
|
||||
all_diffs.append(payload)
|
||||
if lsp:
|
||||
lsp_blocks.append(lsp)
|
||||
if lint:
|
||||
lint_results[op.file_path] = lint
|
||||
# Each LSP block carries its own <diagnostics file="..."> header; joining keeps attribution.
|
||||
return PatchResult(
|
||||
success=not errors,
|
||||
error=("Apply phase failed (state may be inconsistent — run `git diff` to assess):\n"
|
||||
+ _bullets(errors)) if errors else None,
|
||||
diff='\n'.join(all_diffs),
|
||||
files_modified=files["modified"], files_created=files["created"], files_deleted=files["deleted"],
|
||||
lint=lint_results if lint_results else None,
|
||||
lsp_diagnostics="\n\n".join(lsp_blocks) if lsp_blocks else None)
|
||||
if errors:
|
||||
return PatchResult(
|
||||
success=False,
|
||||
error="Apply phase failed (state may be inconsistent — run `git diff` to assess):\n"
|
||||
+ "\n".join(f" • {e}" for e in errors),
|
||||
**result_kwargs)
|
||||
return PatchResult(success=True, **result_kwargs)
|
||||
lint=lint_results or None, lsp_diagnostics="\n\n".join(lsp_blocks) or None)
|
||||
|
||||
|
||||
def _write_file_accepts_pre_content(file_ops: Any) -> bool:
|
||||
"""True when ``file_ops.write_file`` accepts ``pre_content``. Decided from the signature,
|
||||
not by catching TypeError around the call, so a TypeError raised *inside* a capable
|
||||
write_file propagates instead of triggering a duplicate write."""
|
||||
try:
|
||||
"""Whether ``file_ops.write_file`` accepts ``pre_content`` — read from the signature, not by
|
||||
catching TypeError around the call, so a TypeError raised *inside* it can't double-write."""
|
||||
with contextlib.suppress(TypeError, ValueError):
|
||||
params = inspect.signature(file_ops.write_file).parameters
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return "pre_content" in params or any(
|
||||
p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values())
|
||||
return "pre_content" in params or any(
|
||||
p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values())
|
||||
return False
|
||||
|
||||
|
||||
def _apply_add(op: PatchOperation, file_ops: Any) -> ApplyResult:
|
||||
"""Create a file from the hunks' '+' lines."""
|
||||
content_lines = [line.content for hunk in op.hunks for line in hunk.lines if line.prefix == '+']
|
||||
result = file_ops.write_file(op.file_path, '\n'.join(content_lines))
|
||||
if result.error:
|
||||
return False, result.error, None, None
|
||||
diff = f"--- /dev/null\n+++ b/{op.file_path}\n" + '\n'.join(f"+{line}" for line in content_lines)
|
||||
return True, diff, getattr(result, "lsp_diagnostics", None), getattr(result, "lint", None)
|
||||
return _written(result, diff)
|
||||
|
||||
|
||||
def _apply_delete(op: PatchOperation, file_ops: Any) -> ApplyResult:
|
||||
"""Delete a file, producing a real unified diff of the removed content."""
|
||||
# Validation already confirmed existence; the re-read guards against races.
|
||||
read_result = file_ops.read_file_raw(op.file_path)
|
||||
read_result = file_ops.read_file_raw(op.file_path) # re-read guards validate/apply races
|
||||
if read_result.error:
|
||||
return False, f"Cannot delete {op.file_path}: file not found", None, None
|
||||
return _fail(f"Cannot delete {op.file_path}: file not found")
|
||||
result = file_ops.delete_file(op.file_path)
|
||||
if result.error:
|
||||
return False, result.error, None, None
|
||||
diff = ''.join(difflib.unified_diff(
|
||||
read_result.content.splitlines(keepends=True), [],
|
||||
fromfile=f"a/{op.file_path}", tofile="/dev/null"))
|
||||
return True, diff or f"# Deleted: {op.file_path}", None, None
|
||||
diff = _unified_diff(op.file_path, read_result.content, None) or f"# Deleted: {op.file_path}"
|
||||
return _fail(result.error) if result.error else (True, diff, None, None)
|
||||
|
||||
|
||||
def _apply_move(op: PatchOperation, file_ops: Any) -> ApplyResult:
|
||||
result = file_ops.move_file(op.file_path, op.new_path)
|
||||
if result.error:
|
||||
return False, result.error, None, None
|
||||
return True, f"# Moved: {op.file_path} -> {op.new_path}", None, None
|
||||
return _fail(result.error) if result.error else (
|
||||
True, f"# Moved: {op.file_path} -> {op.new_path}", None, None)
|
||||
|
||||
|
||||
def _insert_addition_only(new_content: str, hunk: Hunk, insert_text: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
@@ -371,71 +338,54 @@ def _insert_addition_only(new_content: str, hunk: Hunk, insert_text: str) -> Tup
|
||||
return None, f"Addition-only hunk: {ambiguous}"
|
||||
if occurrences == 1:
|
||||
eol = new_content.find('\n', new_content.find(hunk.context_hint))
|
||||
if eol != -1:
|
||||
return new_content[:eol + 1] + insert_text + '\n' + new_content[eol + 1:], None
|
||||
return new_content + '\n' + insert_text, None
|
||||
# Hint not found — append at end as a safe fallback.
|
||||
if eol == -1:
|
||||
return new_content + '\n' + insert_text, None
|
||||
return new_content[:eol + 1] + insert_text + '\n' + new_content[eol + 1:], None
|
||||
# No hint / hint not found — append at end as a safe fallback.
|
||||
return new_content.rstrip('\n') + '\n' + insert_text + '\n', None
|
||||
|
||||
|
||||
def _apply_update(op: PatchOperation, file_ops: Any) -> ApplyResult:
|
||||
"""Apply each hunk via fuzzy replace, then write once."""
|
||||
from tools.fuzzy_match import fuzzy_find_and_replace, is_already_applied
|
||||
|
||||
# Raw read: no line-number prefixes or per-line truncation.
|
||||
read_result = file_ops.read_file_raw(op.file_path)
|
||||
read_result = file_ops.read_file_raw(op.file_path) # raw: no line numbers / truncation
|
||||
if read_result.error:
|
||||
return False, f"Cannot read file: {read_result.error}", None, None
|
||||
current_content = read_result.content
|
||||
new_content = current_content
|
||||
|
||||
return _fail(f"Cannot read file: {read_result.error}")
|
||||
current_content = new_content = read_result.content
|
||||
for hunk in op.hunks:
|
||||
search_lines, replace_lines = _split_hunk(hunk)
|
||||
if search_lines and search_lines == replace_lines:
|
||||
continue
|
||||
search_pattern, replacement = '\n'.join(search_lines), '\n'.join(replace_lines)
|
||||
if not search_lines:
|
||||
new_content, err = _insert_addition_only(new_content, hunk, '\n'.join(replace_lines))
|
||||
new_content, err = _insert_addition_only(new_content, hunk, replacement)
|
||||
if err:
|
||||
return False, err, None, None
|
||||
return _fail(err)
|
||||
continue
|
||||
|
||||
search_pattern = '\n'.join(search_lines)
|
||||
replacement = '\n'.join(replace_lines)
|
||||
new_content, count, _strategy, error = fuzzy_find_and_replace(
|
||||
new_content, search_pattern, replacement, replace_all=False)
|
||||
if not (error and count == 0):
|
||||
continue
|
||||
|
||||
# Retry inside a window around the context hint, if any.
|
||||
hint_pos = new_content.find(hunk.context_hint) if hunk.context_hint else -1
|
||||
if hint_pos != -1:
|
||||
window_start = max(0, hint_pos - 500)
|
||||
window_end = min(len(new_content), hint_pos + 2000)
|
||||
window_new, count, _strategy, error = fuzzy_find_and_replace(
|
||||
new_content[window_start:window_end], search_pattern, replacement, replace_all=False
|
||||
)
|
||||
new_content[window_start:window_end], search_pattern, replacement, replace_all=False)
|
||||
if count > 0:
|
||||
new_content = new_content[:window_start] + window_new + new_content[window_end:]
|
||||
error = None
|
||||
if error:
|
||||
# Mirror the validation-phase already-applied skip, or the two
|
||||
# phases disagree and the whole patch fails here.
|
||||
# Mirror validation's already-applied skip, else the two phases disagree and fail here.
|
||||
if is_already_applied(new_content, search_pattern, replacement):
|
||||
continue
|
||||
return False, f"Could not apply hunk: {error}" + _no_match_hint(error, search_pattern, new_content), None, None
|
||||
|
||||
hint = _no_match_hint(error, search_pattern, new_content)
|
||||
return _fail(f"Could not apply hunk: {error}" + hint)
|
||||
# Pass pre_content to skip a redundant re-read inside write_file when supported.
|
||||
if _write_file_accepts_pre_content(file_ops):
|
||||
write_result = file_ops.write_file(op.file_path, new_content, pre_content=current_content)
|
||||
else:
|
||||
write_result = file_ops.write_file(op.file_path, new_content)
|
||||
if write_result.error:
|
||||
return False, write_result.error, None, None
|
||||
|
||||
diff = ''.join(difflib.unified_diff(
|
||||
current_content.splitlines(keepends=True), new_content.splitlines(keepends=True),
|
||||
fromfile=f"a/{op.file_path}", tofile=f"b/{op.file_path}"))
|
||||
return True, diff, getattr(write_result, "lsp_diagnostics", None), getattr(write_result, "lint", None)
|
||||
extra = {"pre_content": current_content} if _write_file_accepts_pre_content(file_ops) else {}
|
||||
write_result = file_ops.write_file(op.file_path, new_content, **extra)
|
||||
return _written(write_result, _unified_diff(op.file_path, current_content, new_content))
|
||||
|
||||
|
||||
# operation -> (handler, verb for error text, files_* bucket)
|
||||
|
||||
+123
-194
@@ -1,16 +1,14 @@
|
||||
"""Stdlib document-to-text extraction for ``read_file``.
|
||||
|
||||
Jupyter, DOCX and XLSX need no dependencies. The optional ``firecrawl-anydoc`` package
|
||||
(imports as ``anydoc``) widens coverage to legacy Office, OpenDocument, RTF, EPUB and PDF.
|
||||
The stdlib extractors stay authoritative for their three formats so behavior is identical
|
||||
with or without anydoc. Malformed documents raise :class:`ExtractionError`; callers then
|
||||
fall back to normal text/binary handling.
|
||||
"""
|
||||
"""Document-to-text extraction for ``read_file``: stdlib Jupyter/DOCX/XLSX (always
|
||||
authoritative for those three), plus legacy Office/OpenDocument/RTF/EPUB/PDF when the
|
||||
optional ``firecrawl-anydoc`` package (imports as ``anydoc``) is installed. Malformed
|
||||
documents raise :class:`ExtractionError`; callers fall back to text/binary handling."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import functools
|
||||
import importlib
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import posixpath
|
||||
@@ -29,12 +27,10 @@ __all__ = ["EXTRACTABLE_EXTENSIONS", "ExtractionError", "extract_document_bytes"
|
||||
"extract_document_text", "is_extractable_document"]
|
||||
|
||||
EXTRACTABLE_EXTENSIONS = frozenset({".ipynb", ".docx", ".xlsx"})
|
||||
# Formats handled only when the optional anydoc converter is installed.
|
||||
ANYDOC_EXTENSIONS = frozenset({
|
||||
".doc", ".docm", ".ppt", ".pps", ".pot", ".pptx", ".pptm", ".ppsx", ".ppsm",
|
||||
".xls", ".xlsm", ".xlsb", ".odt", ".ods", ".odp", ".rtf", ".epub", ".pdf"})
|
||||
# anydoc loads whole files with no streaming and the read_file char budget only applies
|
||||
# after conversion — cap the input size.
|
||||
# anydoc loads whole files (no streaming); read_file's char budget applies only post-conversion.
|
||||
MAX_ANYDOC_BYTES = 50 * 1024 * 1024
|
||||
MAX_DOCUMENT_BYTES = 50 * 1024 * 1024
|
||||
_MAX_XLSX_ROWS_PER_SHEET = 5000
|
||||
@@ -59,15 +55,15 @@ def _extension(path: str) -> str:
|
||||
_ANYDOC_UNSET = object()
|
||||
_anydoc_module: Any = _ANYDOC_UNSET
|
||||
_anydoc_lock = threading.Lock()
|
||||
# After a failed load, wait this long before retrying: the attempt can shell out
|
||||
# to pip, so retrying every call would hammer the network where install can't succeed.
|
||||
# Cooldown after a failed load: the attempt can shell out to pip, so retrying every call would
|
||||
# hammer the network where install can't succeed.
|
||||
ANYDOC_RETRY_SECONDS = 300.0
|
||||
_anydoc_failed_at: Optional[float] = None
|
||||
|
||||
|
||||
def _anydoc() -> Optional[Any]:
|
||||
"""Lazily import the optional anydoc converter; None when unavailable. A failed load is
|
||||
retried after ANYDOC_RETRY_SECONDS so one transient pip/network blip does not stick."""
|
||||
"""Lazily import the optional anydoc converter (None when unavailable; failures retried after
|
||||
ANYDOC_RETRY_SECONDS so one transient pip/network blip does not stick)."""
|
||||
global _anydoc_module, _anydoc_failed_at
|
||||
if _anydoc_module is not _ANYDOC_UNSET:
|
||||
return _anydoc_module
|
||||
@@ -79,8 +75,7 @@ def _anydoc() -> Optional[Any]:
|
||||
return None
|
||||
try:
|
||||
from tools.lazy_deps import ensure as _lazy_ensure
|
||||
# prompt=False: read_file must never block on an install prompt.
|
||||
_lazy_ensure("tool.doc_extract", prompt=False)
|
||||
_lazy_ensure("tool.doc_extract", prompt=False) # read_file must never block on a prompt
|
||||
_anydoc_module = importlib.import_module("anydoc")
|
||||
except Exception: # install failure, ImportError or a broken native binding
|
||||
_anydoc_failed_at = time.monotonic()
|
||||
@@ -101,23 +96,19 @@ def _check_size(size: int, limit: int) -> None:
|
||||
@contextlib.contextmanager
|
||||
def _temp_copy(data: bytes, suffix: str) -> Iterator[str]:
|
||||
"""Materialize backend bytes in a private host temp file; removed even when parsing fails."""
|
||||
temp_path = ""
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh:
|
||||
fh.write(data)
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh:
|
||||
fh.write(data)
|
||||
temp_path = fh.name
|
||||
yield temp_path
|
||||
yield fh.name
|
||||
finally:
|
||||
if temp_path:
|
||||
with contextlib.suppress(OSError):
|
||||
os.unlink(temp_path)
|
||||
with contextlib.suppress(OSError):
|
||||
os.unlink(fh.name)
|
||||
|
||||
|
||||
def extract_document_text(path: str) -> str:
|
||||
ext = _extension(path)
|
||||
extractor = _STDLIB_EXTRACTORS.get(ext)
|
||||
if extractor is not None:
|
||||
return extractor(path)
|
||||
if ext in _STDLIB_EXTRACTORS:
|
||||
return _STDLIB_EXTRACTORS[ext](path)
|
||||
if ext in ANYDOC_EXTENSIONS:
|
||||
return _extract_anydoc(path)
|
||||
raise ExtractionError(f"Unsupported document type: {path!r}")
|
||||
@@ -131,14 +122,12 @@ def extract_document_bytes(data: bytes, path: str) -> str:
|
||||
return _extract_anydoc_bytes(data, path)
|
||||
if ext not in EXTRACTABLE_EXTENSIONS:
|
||||
raise ExtractionError(f"Unsupported document type: {path!r}")
|
||||
# The stdlib extractors are path-oriented.
|
||||
with _temp_copy(data, ext) as temp_path:
|
||||
return extract_document_text(temp_path)
|
||||
with _temp_copy(data, ext) as temp_path: # the stdlib extractors are path-oriented
|
||||
return _STDLIB_EXTRACTORS[ext](temp_path)
|
||||
|
||||
|
||||
def _anydoc_missing_error(path: str) -> str:
|
||||
"""Teaching error for anydoc-gated formats (deliberately absent from the schema so only
|
||||
sessions that hit one pay for it)."""
|
||||
"""Teaching text for anydoc-gated formats (not in the schema: only sessions hitting one pay)."""
|
||||
return (
|
||||
f"Cannot convert {path!r}: this format needs the optional anydoc "
|
||||
"converter, which is not installed (install blocked or first "
|
||||
@@ -149,10 +138,10 @@ def _anydoc_missing_error(path: str) -> str:
|
||||
|
||||
|
||||
def _hosted_ocr_config() -> tuple:
|
||||
"""Resolve hosted-OCR settings: (enabled, api_key, api_url). Never raises; no network.
|
||||
Maintainer decision: the ONLY route is a direct ``FIRECRAWL_API_KEY`` (anydoc defaults the
|
||||
api_url); the Nous gateway is NOT used — its Parse proxy live-probed broken (revisit when
|
||||
it grows Parse support). ``file_tools.hosted_ocr: false`` disables even with a key."""
|
||||
"""(enabled, api_key, api_url); never raises, no network. Maintainer decision: the ONLY route
|
||||
is a direct ``FIRECRAWL_API_KEY`` (anydoc defaults api_url); the Nous gateway's Parse proxy
|
||||
live-probed broken, so it is NOT used. ``file_tools.hosted_ocr: false`` disables even with a
|
||||
key."""
|
||||
api_key = os.environ.get("FIRECRAWL_API_KEY") or None
|
||||
enabled = api_key is not None
|
||||
with contextlib.suppress(Exception):
|
||||
@@ -164,35 +153,30 @@ def _hosted_ocr_config() -> tuple:
|
||||
|
||||
|
||||
def hosted_ocr_available() -> bool:
|
||||
"""Public probe for read_file's schema line; a key failing at conversion time lands in
|
||||
the NEEDS-OCR warning instead."""
|
||||
"""Probe for read_file's schema line; a key failing at conversion time surfaces in NEEDS-OCR."""
|
||||
return _hosted_ocr_config()[0]
|
||||
|
||||
|
||||
def _needs_ocr_warning(path: str, pages, hosted_error: str = "") -> str:
|
||||
"""Result text when anydoc raises NeedsOcrError and hosted OCR is off/failed. Hints at
|
||||
CHECKING for an OCR skill (never names one) and never advertises the hosted_ocr knob."""
|
||||
"""NeedsOcrError result when hosted OCR is off/failed; hints at CHECKING for an OCR skill
|
||||
(never names one) and never advertises the hosted_ocr knob."""
|
||||
page_list = ", ".join(str(p) for p in pages) if pages else "unknown"
|
||||
msg = (
|
||||
hosted = f"Hosted OCR was attempted and failed ({hosted_error}). " if hosted_error else ""
|
||||
return (
|
||||
f"[NEEDS OCR: pages {page_list} of this PDF are scanned images "
|
||||
"with no text layer — their content is MISSING below. ")
|
||||
if hosted_error:
|
||||
msg += f"Hosted OCR was attempted and failed ({hosted_error}). "
|
||||
msg += (
|
||||
f"with no text layer — their content is MISSING below. {hosted}"
|
||||
"If the missing pages matter: render just those pages with "
|
||||
f"`pdftoppm -jpeg -r 150 -f <first> -l <last> '{path}' /tmp/page` "
|
||||
"and inspect via vision_analyze, or check whether an OCR skill is "
|
||||
"available (skills_list).")
|
||||
return msg + "]\n"
|
||||
"available (skills_list).]\n")
|
||||
|
||||
|
||||
def _finalize_anydoc_text(text: Any, path: str, pdf_note: Callable[[], str]) -> str:
|
||||
"""Normalize converter output and, for PDFs, PREPEND the coverage note (read_file
|
||||
paginates: a footer may never be fetched). Covers PARTIAL gaps without NeedsOcrError."""
|
||||
"""Normalize converter output; PDFs get the coverage note PREPENDED (read_file paginates, so a
|
||||
footer may never be fetched) — this covers PARTIAL scan gaps that raise no NeedsOcrError."""
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
raise ExtractionError("Document contains no extractable text")
|
||||
note = pdf_note() if Path(path).suffix.lower() == ".pdf" else ""
|
||||
return (note or "") + text.rstrip("\n") + "\n"
|
||||
return (pdf_note() if Path(path).suffix.lower() == ".pdf" else "") + text.rstrip("\n") + "\n"
|
||||
|
||||
|
||||
def _ocr_scanned_pdf(mod: Any, path: str, exc: BaseException) -> str:
|
||||
@@ -206,40 +190,33 @@ def _ocr_scanned_pdf(mod: Any, path: str, exc: BaseException) -> str:
|
||||
return mod.to_markdown(path, ocr="hosted", **extra).rstrip("\n") + "\n"
|
||||
except Exception as hosted_exc: # noqa: BLE001
|
||||
hosted_error = f"{type(hosted_exc).__name__}: {hosted_exc}"
|
||||
# No route / disabled / hosted failed: whole doc is scans — the warning IS the result.
|
||||
return _needs_ocr_warning(path, pages, hosted_error)
|
||||
|
||||
|
||||
def _require_anydoc(path: str) -> Any:
|
||||
mod = _anydoc()
|
||||
if mod is None:
|
||||
raise ExtractionError(_anydoc_missing_error(path))
|
||||
return mod
|
||||
return _needs_ocr_warning(path, pages, hosted_error) # whole doc is scans: the warning IS it
|
||||
|
||||
|
||||
def _extract_anydoc(path: str) -> str:
|
||||
mod = _require_anydoc(path)
|
||||
try:
|
||||
size = os.path.getsize(path)
|
||||
except OSError as exc:
|
||||
raise ExtractionError(str(exc)) from exc
|
||||
_check_size(size, MAX_ANYDOC_BYTES)
|
||||
mod = _anydoc()
|
||||
if mod is None:
|
||||
raise ExtractionError(_anydoc_missing_error(path))
|
||||
try:
|
||||
_check_size(os.path.getsize(path), MAX_ANYDOC_BYTES)
|
||||
text = mod.to_markdown(path)
|
||||
except ExtractionError:
|
||||
raise
|
||||
except OSError as exc:
|
||||
raise ExtractionError(str(exc)) from exc
|
||||
except Exception as exc:
|
||||
needs_ocr = getattr(mod, "NeedsOcrError", None)
|
||||
if needs_ocr is not None and isinstance(exc, needs_ocr):
|
||||
return _ocr_scanned_pdf(mod, path, exc)
|
||||
# anydoc raises one ConvertError subclass per failure mode (Unsupported, Malformed,
|
||||
# Encrypted, ResourceLimit, MissingPart); all mean "no meaningful text".
|
||||
# Any ConvertError subclass (Unsupported/Malformed/Encrypted/...) = "no meaningful text".
|
||||
raise ExtractionError(f"{type(exc).__name__}: {exc}") from exc
|
||||
return _finalize_anydoc_text(text, path, lambda: _pdf_coverage_note(path))
|
||||
|
||||
|
||||
def _extract_anydoc_bytes(data: bytes, path: str) -> str:
|
||||
mod = _require_anydoc(path)
|
||||
mod = _anydoc()
|
||||
if mod is None:
|
||||
raise ExtractionError(_anydoc_missing_error(path))
|
||||
_check_size(len(data), MAX_ANYDOC_BYTES)
|
||||
try:
|
||||
text = mod.to_markdown_bytes(data)
|
||||
@@ -248,17 +225,13 @@ def _extract_anydoc_bytes(data: bytes, path: str) -> str:
|
||||
return _finalize_anydoc_text(text, path, lambda: _pdf_coverage_note_from_bytes(data, path))
|
||||
|
||||
|
||||
# ── Scanned-PDF coverage detection: text-layer extractors return nothing for scanned
|
||||
# pages, so a mostly-scanned PDF converts "successfully" into headers with empty bodies —
|
||||
# silent data loss. Count per-page text via pdftotext (form-feed separated) and warn.
|
||||
# ── Scanned-PDF coverage: text-layer extractors return nothing for scanned pages, so a mostly
|
||||
# scanned PDF converts "successfully" into silent data loss. Count per-page text via pdftotext.
|
||||
PDF_EMPTY_PAGE_CHARS = 20 # fewer extracted chars than this = empty page
|
||||
# Warn when empty pages reach both MIN_EMPTY and MIN_RATIO, or ABSOLUTE_EMPTY alone.
|
||||
PDF_COVERAGE_MIN_EMPTY = 2
|
||||
PDF_COVERAGE_MIN_RATIO = 0.2
|
||||
PDF_COVERAGE_ABSOLUTE_EMPTY = 10
|
||||
PDF_COVERAGE_MIN_EMPTY, PDF_COVERAGE_MIN_RATIO, PDF_COVERAGE_ABSOLUTE_EMPTY = 2, 0.2, 10
|
||||
PDF_PAGE_SCAN_TIMEOUT = 20.0
|
||||
# Cap the per-gap breakdown so alternating text/scan pages can't balloon the warning.
|
||||
PDF_GAP_MAP_MAX_ENTRIES = 20
|
||||
PDF_GAP_MAP_MAX_ENTRIES = 20 # cap so alternating text/scan pages can't balloon the warning
|
||||
_GAP_CONTEXT_CHARS = 60
|
||||
|
||||
|
||||
@@ -279,14 +252,10 @@ def _pdf_page_texts(path: str) -> Optional[list[str]]:
|
||||
|
||||
|
||||
def _gap_map(counts: list[int], texts: list[str], empty: list[int]) -> str:
|
||||
"""Per-gap breakdown, each empty range labeled with the last text seen before
|
||||
it (usually a section header), so the agent can pick WHICH gaps to OCR."""
|
||||
ranges: list[list[int]] = [] # sorted 1-based page numbers -> [start, end] runs
|
||||
for p in empty:
|
||||
if ranges and p == ranges[-1][1] + 1:
|
||||
ranges[-1][1] = p
|
||||
else:
|
||||
ranges.append([p, p])
|
||||
"""Per-gap breakdown labeled with the text before each gap, so the agent picks which to OCR."""
|
||||
# Sorted 1-based page numbers -> (start, end) runs; consecutive pages share ``page - index``.
|
||||
runs = [list(g) for _k, g in itertools.groupby(enumerate(empty), lambda e: e[1] - e[0])]
|
||||
ranges = [(run[0][1], run[-1][1]) for run in runs]
|
||||
lines: list[str] = []
|
||||
for a, b in ranges[:PDF_GAP_MAP_MAX_ENTRIES]:
|
||||
label = ""
|
||||
@@ -305,17 +274,17 @@ def _gap_map(counts: list[int], texts: list[str], empty: list[int]) -> str:
|
||||
|
||||
|
||||
def _pdf_coverage_note(path: str, display_path: Optional[str] = None) -> str:
|
||||
"""Warning header when many PDF pages produced no text, else ''. ``path`` is scanned
|
||||
(may be a host temp file); ``display_path`` is what the recovery command shows."""
|
||||
"""Warning header when many pages yielded no text, else ''. ``display_path`` (default ``path``,
|
||||
which may be a host temp file) is what the recovery command shows."""
|
||||
texts = _pdf_page_texts(path)
|
||||
if not texts or len(texts) < 2:
|
||||
return ""
|
||||
counts = [len(page.strip()) for page in texts]
|
||||
empty = [i + 1 for i, n in enumerate(counts) if n < PDF_EMPTY_PAGE_CHARS]
|
||||
total = len(counts)
|
||||
if len(empty) < PDF_COVERAGE_MIN_EMPTY or (
|
||||
len(empty) / total < PDF_COVERAGE_MIN_RATIO and len(empty) < PDF_COVERAGE_ABSOLUTE_EMPTY
|
||||
):
|
||||
n_empty = len(empty)
|
||||
enough = n_empty / total >= PDF_COVERAGE_MIN_RATIO or n_empty >= PDF_COVERAGE_ABSOLUTE_EMPTY
|
||||
if n_empty < PDF_COVERAGE_MIN_EMPTY or not enough:
|
||||
return ""
|
||||
shown = display_path or path
|
||||
return (
|
||||
@@ -335,13 +304,10 @@ def _pdf_coverage_note(path: str, display_path: Optional[str] = None) -> str:
|
||||
|
||||
|
||||
def _pdf_coverage_note_from_bytes(data: bytes, display_path: str) -> str:
|
||||
"""Coverage note for backend-transferred PDF bytes via a host temp copy (pdftotext is
|
||||
path-oriented); the recovery command still names ``display_path``."""
|
||||
try:
|
||||
with _temp_copy(data, ".pdf") as temp_path:
|
||||
return _pdf_coverage_note(temp_path, display_path=display_path)
|
||||
except OSError:
|
||||
return ""
|
||||
"""Coverage note for backend PDF bytes via a host temp copy (pdftotext needs a path)."""
|
||||
with contextlib.suppress(OSError), _temp_copy(data, ".pdf") as temp_path:
|
||||
return _pdf_coverage_note(temp_path, display_path=display_path)
|
||||
return ""
|
||||
|
||||
|
||||
def _joined(lines: list[str], empty_error: str) -> str:
|
||||
@@ -352,11 +318,10 @@ def _joined(lines: list[str], empty_error: str) -> str:
|
||||
|
||||
|
||||
def _source_text(source) -> str:
|
||||
if isinstance(source, str):
|
||||
return source
|
||||
"""Notebook source/text fields are a str or a list of str fragments."""
|
||||
if isinstance(source, list):
|
||||
return "".join(item for item in source if isinstance(item, str))
|
||||
return ""
|
||||
source = "".join(item for item in source if isinstance(item, str))
|
||||
return source if isinstance(source, str) else ""
|
||||
|
||||
|
||||
def _human_size(n_bytes: int) -> str:
|
||||
@@ -366,30 +331,24 @@ def _human_size(n_bytes: int) -> str:
|
||||
def _base64_bytes(payload: str) -> int:
|
||||
"""Approximate decoded size of a base64 payload (whitespace ignored)."""
|
||||
clean = re.sub(r"[^0-9+/=A-Za-z]", "", payload)
|
||||
padding = min(2, len(clean) - len(clean.rstrip("=")))
|
||||
return max(0, (len(clean) * 3) // 4 - padding)
|
||||
return max(0, (len(clean) * 3) // 4 - min(2, len(clean) - len(clean.rstrip("="))))
|
||||
|
||||
|
||||
def _clean_stream_text(text: str) -> str:
|
||||
"""Strip ANSI escapes; keep only the final ``\\r`` frame of each line (tqdm redraws)."""
|
||||
from tools.ansi_strip import strip_ansi
|
||||
lines = []
|
||||
for line in strip_ansi(text).replace("\r\n", "\n").split("\n"):
|
||||
frames = [frame for frame in line.split("\r") if frame]
|
||||
lines.append(frames[-1] if frames else "")
|
||||
return "\n".join(lines)
|
||||
return "\n".join(([f for f in line.split("\r") if f] or [""])[-1]
|
||||
for line in strip_ansi(text).replace("\r\n", "\n").split("\n"))
|
||||
|
||||
|
||||
# Per-output-block truncation so one runaway training log cannot flood the extraction.
|
||||
_MAX_OUTPUT_CHARS = 20_000
|
||||
_MAX_OUTPUT_CHARS = 20_000 # per code cell, so one runaway training log cannot flood the extraction
|
||||
# nbformat v3 stores mime data flat on the output dict under these keys.
|
||||
_V3_MIME_KEYS = (("png", "image/png"), ("jpeg", "image/jpeg"), ("svg", "image/svg+xml"), ("html", "text/html"))
|
||||
|
||||
|
||||
def _notebook_output_text(output: Any) -> str:
|
||||
"""Render one notebook output as compact text: stream text, tracebacks and textual
|
||||
results kept; token-heavy payloads (images, HTML, widgets) become sized placeholders.
|
||||
Handles nbformat v4 and legacy v3 (``pyout``/``pyerr``) shapes."""
|
||||
"""One notebook output as compact text: stream/traceback/textual results kept; token-heavy
|
||||
payloads (images, HTML, widgets) become sized placeholders. Handles v4 and legacy v3 shapes."""
|
||||
if not isinstance(output, dict):
|
||||
return ""
|
||||
otype = output.get("output_type")
|
||||
@@ -397,52 +356,41 @@ def _notebook_output_text(output: Any) -> str:
|
||||
body = _clean_stream_text(_source_text(output.get("text", "")))
|
||||
return body if body.strip() else ""
|
||||
if otype in {"error", "pyerr"}:
|
||||
traceback = output.get("traceback")
|
||||
tb_text = ""
|
||||
if isinstance(traceback, list):
|
||||
tb_text = _clean_stream_text(
|
||||
"\n".join(line for line in traceback if isinstance(line, str)))
|
||||
tb = output.get("traceback")
|
||||
tb_text = _clean_stream_text("\n".join(filter(lambda l: isinstance(l, str), tb))
|
||||
if isinstance(tb, list) else "")
|
||||
header = f"Error: {output.get('ename', '')}: {output.get('evalue', '')}".rstrip(": ")
|
||||
return f"{header}\n{tb_text}".rstrip()
|
||||
if otype not in {"execute_result", "display_data", "pyout"}:
|
||||
return ""
|
||||
|
||||
data = output.get("data")
|
||||
if not isinstance(data, dict): # legacy v3: mime payloads sit flat on the output dict
|
||||
data = {"text/plain": output["text"]} if isinstance(output.get("text"), (str, list)) else {}
|
||||
data.update((mime, output[k]) for k, mime in _V3_MIME_KEYS if k in output)
|
||||
if "application/vnd.jupyter.widget-view+json" in data:
|
||||
return "[interactive widget — omitted]"
|
||||
# Prefer readable text: models consume text/plain far better than markup.
|
||||
for mime in ("text/plain", "text/markdown"):
|
||||
if mime in data:
|
||||
body = _clean_stream_text(_source_text(data[mime]))
|
||||
if body.strip():
|
||||
return body
|
||||
for mime in ("text/plain", "text/markdown"): # models consume text far better than markup
|
||||
body = _clean_stream_text(_source_text(data[mime])) if mime in data else ""
|
||||
if body.strip():
|
||||
return body
|
||||
for mime, value in data.items():
|
||||
if isinstance(mime, str) and mime.startswith("image/"):
|
||||
size = _base64_bytes(_source_text(value))
|
||||
return f"[{mime} output — {_human_size(size)}, omitted]"
|
||||
return f"[{mime} output — {_human_size(_base64_bytes(_source_text(value)))}, omitted]"
|
||||
if "text/html" in data:
|
||||
html = _source_text(data["text/html"])
|
||||
return f"[text/html output — {len(html):,} chars, omitted]"
|
||||
mimes = ", ".join(str(m) for m in data) or "unknown"
|
||||
return f"[{mimes} output — omitted]"
|
||||
return f"[text/html output — {len(_source_text(data['text/html'])):,} chars, omitted]"
|
||||
return f"[{', '.join(str(m) for m in data) or 'unknown'} output — omitted]"
|
||||
|
||||
|
||||
def _notebook_outputs(cell: dict, jq_pointer: str = "", filename: str = "") -> str:
|
||||
outputs = cell.get("outputs")
|
||||
if not isinstance(outputs, list):
|
||||
return ""
|
||||
blocks = [text for text in (_notebook_output_text(o) for o in outputs) if text]
|
||||
if not blocks:
|
||||
return ""
|
||||
joined = "\n".join(blocks)
|
||||
if len(joined) > _MAX_OUTPUT_CHARS:
|
||||
omitted = len(joined) - _MAX_OUTPUT_CHARS
|
||||
hint = f" — full output: jq -r '{jq_pointer}' {filename}" if jq_pointer and filename else ""
|
||||
joined = joined[:_MAX_OUTPUT_CHARS] + f"\n… [{omitted:,} output chars truncated{hint}]"
|
||||
return joined
|
||||
joined = "\n".join(filter(None, map(_notebook_output_text, outputs)))
|
||||
if len(joined) <= _MAX_OUTPUT_CHARS:
|
||||
return joined
|
||||
hint = f" — full output: jq -r '{jq_pointer}' {filename}" if jq_pointer and filename else ""
|
||||
omitted = len(joined) - _MAX_OUTPUT_CHARS
|
||||
return joined[:_MAX_OUTPUT_CHARS] + f"\n… [{omitted:,} output chars truncated{hint}]"
|
||||
|
||||
|
||||
_CELL_LABELS = {"markdown": "Markdown", "code": "Code", "raw": "Raw"}
|
||||
@@ -459,7 +407,7 @@ def _extract_notebook(path: str) -> str:
|
||||
raw_cells = nb.get("cells")
|
||||
if isinstance(raw_cells, list):
|
||||
cells = [(f".cells[{i}].outputs", cell) for i, cell in enumerate(raw_cells)]
|
||||
else:
|
||||
else: # nbformat v3: cells live under worksheets
|
||||
cells = [
|
||||
(f".worksheets[{wi}].cells[{ci}].outputs", cell)
|
||||
for wi, ws in enumerate(nb.get("worksheets", [])) if isinstance(ws, dict)
|
||||
@@ -470,18 +418,16 @@ def _extract_notebook(path: str) -> str:
|
||||
counts = dict.fromkeys(_CELL_LABELS, 0)
|
||||
out: list[str] = []
|
||||
for jq_pointer, cell in cells:
|
||||
if not isinstance(cell, dict):
|
||||
continue
|
||||
typ = cell.get("cell_type")
|
||||
typ = cell.get("cell_type") if isinstance(cell, dict) else None
|
||||
if typ not in _CELL_LABELS:
|
||||
continue
|
||||
counts[typ] += 1
|
||||
suffix = f" {counts[typ]}" if typ != "raw" else ""
|
||||
out.extend((f"# ── {_CELL_LABELS[typ]} cell{suffix} ──", _source_text(cell.get("source", "")).rstrip("\n"), ""))
|
||||
if typ == "code":
|
||||
rendered = _notebook_outputs(cell, jq_pointer, nb_name)
|
||||
if rendered:
|
||||
out.extend((f"# ── Output (cell {counts[typ]}) ──", rendered.rstrip("\n"), ""))
|
||||
source = _source_text(cell.get("source", "")).rstrip("\n")
|
||||
out += [f"# ── {_CELL_LABELS[typ]} cell{suffix} ──", source, ""]
|
||||
rendered = _notebook_outputs(cell, jq_pointer, nb_name) if typ == "code" else ""
|
||||
if rendered:
|
||||
out += [f"# ── Output (cell {counts[typ]}) ──", rendered.rstrip("\n"), ""]
|
||||
return _joined(out, "Notebook contains no readable cells")
|
||||
|
||||
|
||||
@@ -491,24 +437,21 @@ def _open_zip(path: str, kind: str) -> Iterator[zipfile.ZipFile]:
|
||||
try:
|
||||
with zipfile.ZipFile(path) as zf:
|
||||
yield zf
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise ExtractionError(f"Not a valid {kind}: {exc}") from exc
|
||||
except OSError as exc:
|
||||
raise ExtractionError(str(exc)) from exc
|
||||
except (zipfile.BadZipFile, OSError) as exc:
|
||||
bad_zip = isinstance(exc, zipfile.BadZipFile)
|
||||
raise ExtractionError(f"Not a valid {kind}: {exc}" if bad_zip else str(exc)) from exc
|
||||
|
||||
|
||||
def _zip_xml(zf: zipfile.ZipFile, name: str, optional: bool = False) -> Any:
|
||||
"""Parse a package part; ``optional`` parts yield None when absent or malformed."""
|
||||
"""Parse a package part; ``optional`` parts yield an empty element when absent or malformed."""
|
||||
try:
|
||||
return ET.fromstring(zf.read(name))
|
||||
except KeyError as exc:
|
||||
except (KeyError, ET.ParseError) as exc:
|
||||
if optional:
|
||||
return None
|
||||
raise ExtractionError(f"Missing {name}") from exc
|
||||
except ET.ParseError as exc:
|
||||
if optional:
|
||||
return None
|
||||
raise ExtractionError(f"Malformed XML in {name}: {exc}") from exc
|
||||
return ET.Element("missing")
|
||||
raise ExtractionError(
|
||||
f"Missing {name}" if isinstance(exc, KeyError) else f"Malformed XML in {name}: {exc}"
|
||||
) from exc
|
||||
|
||||
|
||||
def _extract_docx(path: str) -> str:
|
||||
@@ -518,9 +461,9 @@ def _extract_docx(path: str) -> str:
|
||||
breaks = {f"{w}tab": "\t", f"{w}br": "\n", f"{w}cr": "\n"}
|
||||
lines: list[str] = []
|
||||
for para in root.iter(f"{w}p"):
|
||||
buf = [(node.text or "") if node.tag == f"{w}t" else breaks.get(node.tag, "")
|
||||
for node in para.iter()]
|
||||
lines.extend("".join(buf).split("\n"))
|
||||
text = "".join(
|
||||
(n.text or "") if n.tag == f"{w}t" else breaks.get(n.tag, "") for n in para.iter())
|
||||
lines.extend(text.split("\n"))
|
||||
return _joined(lines, "DOCX contains no extractable text")
|
||||
|
||||
|
||||
@@ -529,38 +472,27 @@ def _extract_xlsx(path: str) -> str:
|
||||
with _open_zip(path, "XLSX") as zf:
|
||||
names = set(zf.namelist())
|
||||
sst = _zip_xml(zf, "xl/sharedStrings.xml", optional=True)
|
||||
shared = [] if sst is None else [
|
||||
"".join(t.text or "" for t in item.iter(f"{s}t")) for item in sst.iter(f"{s}si")]
|
||||
shared = ["".join(t.text or "" for t in item.iter(f"{s}t")) for item in sst.iter(f"{s}si")]
|
||||
rels_root = _zip_xml(zf, "xl/_rels/workbook.xml.rels", optional=True)
|
||||
rels = {} if rels_root is None else {
|
||||
rel.get("Id", ""): rel.get("Target", "")
|
||||
for rel in rels_root.iter(f"{pr}Relationship") if rel.get("Id")}
|
||||
rels = {rel.get("Id", ""): rel.get("Target", "")
|
||||
for rel in rels_root.iter(f"{pr}Relationship") if rel.get("Id")}
|
||||
out: list[str] = []
|
||||
for sheet in _zip_xml(zf, "xl/workbook.xml").iter(f"{s}sheet"):
|
||||
if sheet.get("state", "visible") in {"hidden", "veryHidden"}:
|
||||
continue
|
||||
target = rels.get(sheet.get(f"{r}id", ""), "").lstrip("/")
|
||||
part = posixpath.normpath(target if target.startswith("xl/") else f"xl/{target}")
|
||||
if part not in names:
|
||||
if sheet.get("state", "visible") in {"hidden", "veryHidden"} or part not in names:
|
||||
continue
|
||||
try:
|
||||
with contextlib.suppress(ET.ParseError):
|
||||
rows = _sheet_rows(zf.read(part), shared)
|
||||
except ET.ParseError:
|
||||
continue
|
||||
out.append(f"# ── Sheet: {sheet.get('name', 'Sheet')} ──")
|
||||
out.extend("\t".join(row) for row in rows)
|
||||
if not rows:
|
||||
out.append("(empty)")
|
||||
out.append("")
|
||||
out += [f"# ── Sheet: {sheet.get('name', 'Sheet')} ──",
|
||||
*(["\t".join(row) for row in rows] or ["(empty)"]), ""]
|
||||
return _joined(out, "XLSX has no visible sheets with content")
|
||||
|
||||
|
||||
def _col_index(ref: str) -> int:
|
||||
idx = 0
|
||||
for ch in ref:
|
||||
if not ch.isalpha():
|
||||
break
|
||||
idx = idx * 26 + ord(ch.upper()) - ord("A") + 1
|
||||
"""0-based column of a cell ref: ``A1`` -> 0, ``AB7`` -> 27 (bijective base-26 letters)."""
|
||||
idx = functools.reduce(lambda acc, ch: acc * 26 + ord(ch.upper()) - ord("A") + 1,
|
||||
itertools.takewhile(str.isalpha, ref), 0)
|
||||
return max(idx - 1, 0)
|
||||
|
||||
|
||||
@@ -568,18 +500,15 @@ def _sheet_rows(xml_bytes: bytes, shared: list[str]) -> list[list[str]]:
|
||||
root = ET.fromstring(xml_bytes)
|
||||
s = f"{{{_NS_S}}}"
|
||||
rows: list[list[str]] = []
|
||||
for row in root.iter(f"{s}row"):
|
||||
if len(rows) >= _MAX_XLSX_ROWS_PER_SHEET:
|
||||
break
|
||||
for row in itertools.islice(root.iter(f"{s}row"), _MAX_XLSX_ROWS_PER_SHEET):
|
||||
cells: dict[int, str] = {}
|
||||
max_col = -1
|
||||
for cell in row.iter(f"{s}c"):
|
||||
col = _col_index(cell.get("r", "")) if cell.get("r") else max_col + 1
|
||||
if col >= _MAX_XLSX_COLS:
|
||||
continue
|
||||
cells[col] = _cell_value(cell, shared, s)
|
||||
max_col = max(max_col, col)
|
||||
rows.append([cells.get(i, "") for i in range(max_col + 1)] if max_col >= 0 else [])
|
||||
if col < _MAX_XLSX_COLS:
|
||||
cells[col] = _cell_value(cell, shared, s)
|
||||
max_col = max(max_col, col)
|
||||
rows.append([cells.get(i, "") for i in range(max_col + 1)])
|
||||
while rows and not any(value.strip() for value in rows[-1]):
|
||||
rows.pop()
|
||||
return rows
|
||||
|
||||
+97
-141
@@ -1,11 +1,7 @@
|
||||
"""Sanitize tool JSON schemas for broad LLM-backend compatibility.
|
||||
|
||||
Strict backends reject shapes OpenAI/Anthropic accept: llama.cpp's grammar converter fails
|
||||
on ``{"type": "object"}`` without ``properties``, bare-string schemas and ``type`` arrays;
|
||||
Anthropic rejects nullable ``anyOf`` at the top of ``input_schema``; Fireworks rejects
|
||||
``default`` beside ``$ref``; Codex rejects top-level combinators. This module walks the
|
||||
final schema tree on a deep copy and fixes only those shapes.
|
||||
"""
|
||||
"""Sanitize tool JSON schemas for strict LLM backends. llama.cpp's grammar converter fails on
|
||||
``{"type": "object"}`` without ``properties``, bare-string schemas and ``type`` arrays; Anthropic
|
||||
rejects nullable ``anyOf`` at the top of ``input_schema``; Fireworks rejects ``default`` beside
|
||||
``$ref``; Codex rejects top-level combinators. Walks a deep copy and fixes only those shapes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -16,15 +12,12 @@ from typing import Any, Callable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Anthropic (and Bedrock/Vertex/Azure fronting it) reject property keys not matching this;
|
||||
# one bad key anywhere in the tools array 400s the request (Cloudflare's MCP ships 61).
|
||||
# Anthropic (and Bedrock/Vertex/Azure fronting it) reject property keys not matching this; one bad
|
||||
# key anywhere in the tools array 400s the request (Cloudflare's MCP ships 61).
|
||||
_PROP_KEY_RE = re.compile(r"^[a-zA-Z0-9_.-]{1,64}$")
|
||||
_PROP_KEY_BAD_CHARS = re.compile(r"[^a-zA-Z0-9_.-]")
|
||||
|
||||
_UNION_KEYS = ("anyOf", "oneOf")
|
||||
# Outer-node metadata carried onto a union's replacement node.
|
||||
_UNION_META_KEYS = ("title", "description", "default", "examples")
|
||||
_UNION_META_KEYS = ("title", "description", "default", "examples") # copied onto replacements
|
||||
|
||||
|
||||
def _empty_object() -> dict:
|
||||
@@ -47,56 +40,45 @@ def sanitize_property_key(key: str) -> str:
|
||||
|
||||
def _rename_property_keys(props: dict, path: str) -> dict[str, str]:
|
||||
"""{original_key: conforming_key} for one properties dict (identity entries omitted).
|
||||
Deterministic (insertion order, numeric suffixes on collision) so the model-visible
|
||||
schema and the dispatch-time reverse map from the registry's original schema agree."""
|
||||
Deterministic (insertion order, numeric suffixes on collision) so the model-visible schema
|
||||
and the dispatch-time reverse map from the registry's original schema agree."""
|
||||
renames: dict[str, str] = {}
|
||||
taken = {k for k in props if _PROP_KEY_RE.match(k)}
|
||||
for key in props:
|
||||
if _PROP_KEY_RE.match(key):
|
||||
continue
|
||||
for key in (k for k in props if not _PROP_KEY_RE.match(k)):
|
||||
base = sanitize_property_key(key)
|
||||
candidate, i = base, 2
|
||||
while candidate in taken:
|
||||
suffix = f"_{i}"
|
||||
candidate = base[: 64 - len(suffix)] + suffix
|
||||
i += 1
|
||||
candidate, i = base[: 64 - len(f"_{i}")] + f"_{i}", i + 1
|
||||
taken.add(candidate)
|
||||
renames[key] = candidate
|
||||
logger.debug(
|
||||
"schema_sanitizer[%s]: renamed property key %r -> %r "
|
||||
"(provider key-pattern compat)", path, key, candidate)
|
||||
logger.debug("schema_sanitizer[%s]: renamed property key %r -> %r "
|
||||
"(provider key-pattern compat)", path, key, candidate)
|
||||
return renames
|
||||
|
||||
|
||||
def unrename_tool_args(params_schema: Any, args: Any) -> Any:
|
||||
"""Map sanitized property keys in model-emitted args back to wire names. ``params_schema``
|
||||
is the ORIGINAL registry schema; recurses into objects/array items; unknown keys pass."""
|
||||
if not isinstance(params_schema, dict) or not isinstance(args, dict):
|
||||
return args
|
||||
props = params_schema.get("properties")
|
||||
if not isinstance(props, dict):
|
||||
"""Map sanitized keys in model-emitted args back to wire names. ``params_schema`` is the
|
||||
ORIGINAL registry schema; recurses into objects/array items; unknown keys pass through."""
|
||||
props = params_schema.get("properties") if isinstance(params_schema, dict) else None
|
||||
if not isinstance(props, dict) or not isinstance(args, dict):
|
||||
return args
|
||||
reverse = {v: k for k, v in _rename_property_keys(props, "<unrename>").items()}
|
||||
out = {}
|
||||
for key, value in args.items():
|
||||
orig = reverse.get(key, key)
|
||||
subschema = props.get(orig)
|
||||
if isinstance(subschema, dict):
|
||||
if isinstance(value, dict):
|
||||
value = unrename_tool_args(subschema, value)
|
||||
elif isinstance(value, list) and isinstance(subschema.get("items"), dict):
|
||||
value = [
|
||||
unrename_tool_args(subschema["items"], item) if isinstance(item, dict) else item
|
||||
for item in value]
|
||||
sub = props.get(orig) if isinstance(props.get(orig), dict) else {}
|
||||
if isinstance(value, dict) and sub:
|
||||
value = unrename_tool_args(sub, value)
|
||||
elif isinstance(value, list) and isinstance(sub.get("items"), dict):
|
||||
value = [unrename_tool_args(sub["items"], item) if isinstance(item, dict) else item
|
||||
for item in value]
|
||||
out[orig] = value
|
||||
return out
|
||||
|
||||
|
||||
def sanitize_tool_schemas(tools: list[dict]) -> list[dict]:
|
||||
"""Deep-copied ``tools`` (OpenAI format) with sanitized parameter schemas; safe to mutate."""
|
||||
if not tools:
|
||||
return tools
|
||||
return [_sanitize_single_tool(tool) for tool in tools]
|
||||
return [_sanitize_single_tool(tool) for tool in tools] if tools else tools
|
||||
|
||||
|
||||
def _sanitize_single_tool(tool: dict) -> dict:
|
||||
@@ -110,9 +92,7 @@ def _sanitize_single_tool(tool: dict) -> dict:
|
||||
return out
|
||||
name = fn.get("name", "<tool>")
|
||||
top = _sanitize_node(params, path=name)
|
||||
# Guarantee the top level is an object with properties.
|
||||
if not isinstance(top, dict):
|
||||
top = {}
|
||||
top = top if isinstance(top, dict) else {} # guarantee an object with properties on top
|
||||
top["type"] = "object"
|
||||
if not isinstance(top.get("properties"), dict):
|
||||
top["properties"] = {}
|
||||
@@ -124,16 +104,14 @@ def _sanitize_single_tool(tool: dict) -> dict:
|
||||
return out
|
||||
|
||||
|
||||
# Sibling keywords strict JSON Schema validators reject alongside ``$ref``.
|
||||
_REF_FORBIDDEN_SIBLINGS = frozenset({"default"})
|
||||
_REF_FORBIDDEN_SIBLINGS = frozenset({"default"}) # strict validators reject these beside ``$ref``
|
||||
|
||||
|
||||
def _strip_ref_siblings(node: Any) -> Any:
|
||||
"""Recursively drop forbidden siblings of ``$ref`` (Fireworks rejects ``default`` there)."""
|
||||
def strip(out: dict) -> dict:
|
||||
if "$ref" in out:
|
||||
for key in _REF_FORBIDDEN_SIBLINGS:
|
||||
out.pop(key, None)
|
||||
for key in _REF_FORBIDDEN_SIBLINGS if "$ref" in out else ():
|
||||
out.pop(key, None)
|
||||
return out
|
||||
return _rewrite(node, strip)
|
||||
|
||||
@@ -143,18 +121,15 @@ _TOP_LEVEL_FORBIDDEN_KEYS = ("allOf", "anyOf", "oneOf", "enum", "not")
|
||||
|
||||
def _strip_top_level_combinators(params: dict, *, path: str = "<tool>") -> dict:
|
||||
"""Drop combinators from the TOP level only (Codex rejects them there). They are usually
|
||||
conditional-required hints, so validity is unchanged (handlers re-validate); nested
|
||||
combinators are preserved."""
|
||||
conditional-required hints, so validity is unchanged (handlers re-validate); nested ones
|
||||
stay."""
|
||||
if not isinstance(params, dict):
|
||||
return params
|
||||
out = dict(params)
|
||||
for key in _TOP_LEVEL_FORBIDDEN_KEYS:
|
||||
if key in out:
|
||||
logger.debug(
|
||||
"schema_sanitizer[%s]: stripped top-level %r combinator "
|
||||
"from tool parameters (strict-backend compat)",
|
||||
path, key)
|
||||
out.pop(key, None)
|
||||
for key in [k for k in _TOP_LEVEL_FORBIDDEN_KEYS if k in out]:
|
||||
logger.debug("schema_sanitizer[%s]: stripped top-level %r combinator "
|
||||
"from tool parameters (strict-backend compat)", path, key)
|
||||
del out[key]
|
||||
return out
|
||||
|
||||
|
||||
@@ -163,20 +138,18 @@ def _is_null_branch(item: Any) -> bool:
|
||||
|
||||
|
||||
def _carry_union_meta(outer: dict, replacement: dict, *, skip_default_on_ref: bool) -> None:
|
||||
"""Copy outer-union metadata onto *replacement* where absent."""
|
||||
"""Copy outer-union metadata onto *replacement* where absent (``default`` is illegal beside
|
||||
``$ref`` on strict backends, hence ``skip_default_on_ref``)."""
|
||||
for meta_key in _UNION_META_KEYS:
|
||||
if meta_key in outer and meta_key not in replacement:
|
||||
# ``default`` is illegal alongside ``$ref`` on strict backends.
|
||||
if skip_default_on_ref and meta_key == "default" and "$ref" in replacement:
|
||||
continue
|
||||
if meta_key in outer and meta_key not in replacement and not (
|
||||
skip_default_on_ref and meta_key == "default" and "$ref" in replacement):
|
||||
replacement[meta_key] = outer[meta_key]
|
||||
|
||||
|
||||
def strip_nullable_unions(schema: Any, *, keep_nullable_hint: bool = True) -> Any:
|
||||
"""Collapse ``anyOf``/``oneOf`` nullable unions to the single non-null branch. MCP/Pydantic
|
||||
optional fields arrive as ``{"anyOf": [{"type": "string"}, {"type": "null"}]}``; Anthropic
|
||||
rejects the null branch and optionality is already in the parent's ``required``. Only
|
||||
collapses when a null branch was dropped AND exactly one non-null branch survives.
|
||||
"""Collapse ``anyOf``/``oneOf`` nullable unions (MCP/Pydantic optional fields) to the single
|
||||
non-null branch: Anthropic rejects the null branch and optionality already lives in the parent's
|
||||
``required``. Only when a null branch was dropped AND exactly one non-null branch survives.
|
||||
``keep_nullable_hint`` sets ``nullable: true`` for runtime ``"null"`` → ``None`` coercion."""
|
||||
def collapse(stripped: dict) -> Any:
|
||||
for key in _UNION_KEYS:
|
||||
@@ -189,7 +162,7 @@ def strip_nullable_unions(schema: Any, *, keep_nullable_hint: bool = True) -> An
|
||||
if keep_nullable_hint:
|
||||
replacement.setdefault("nullable", True)
|
||||
_carry_union_meta(stripped, replacement, skip_default_on_ref=True)
|
||||
return _rewrite(replacement, collapse)
|
||||
return _rewrite(replacement, collapse) # the survivor may itself be a union
|
||||
return stripped
|
||||
return _rewrite(schema, collapse)
|
||||
|
||||
@@ -199,24 +172,23 @@ _CONST_PRIMITIVE_TYPES: dict[type, str] = {
|
||||
|
||||
|
||||
def _const_branch_type(branch: Any) -> str | None:
|
||||
"""JSON-Schema primitive type of a pure ``const`` branch, else None: a primitive ``const``
|
||||
whose declared ``type`` (if any) matches; only ``title``/``description`` may accompany it."""
|
||||
"""Primitive JSON-Schema type of a pure ``const`` branch (declared ``type``, if any, must match;
|
||||
only ``title``/``description`` may accompany it), else None."""
|
||||
if not isinstance(branch, dict) or "const" not in branch \
|
||||
or set(branch) - {"const", "type", "title", "description"}:
|
||||
return None
|
||||
# ``type(value)`` lookup (not isinstance): bool is a subclass of int.
|
||||
json_type = _CONST_PRIMITIVE_TYPES.get(type(branch["const"]))
|
||||
return json_type if json_type is not None and branch.get("type") in (None, json_type) else None
|
||||
return json_type if branch.get("type") in (None, json_type) else None
|
||||
|
||||
|
||||
def collapse_const_unions(schema: Any) -> Any:
|
||||
"""Collapse ``anyOf``/``oneOf`` unions of same-typed consts to ``enum`` (ported from
|
||||
block/goose ``tool_schema_normalize.rs``, Apache-2.0). Rust/TS MCP servers emit
|
||||
``{"anyOf": [{"const": "red"}, {"const": "green"}]}``, which strict backends mishandle.
|
||||
Applies only when EVERY non-null branch is a pure ``const`` of one primitive type
|
||||
(``bool`` never merges with ``integer``); one ``{"type": "null"}`` branch is tolerated
|
||||
and recorded as ``nullable: true``. Branch order is kept; outer metadata carried over;
|
||||
input never mutated."""
|
||||
"""Collapse ``anyOf``/``oneOf`` unions of same-typed consts (Rust/TS MCP servers emit
|
||||
``{"anyOf": [{"const": "red"}, {"const": "green"}]}``) to ``enum``; ported from block/goose
|
||||
``tool_schema_normalize.rs`` (Apache-2.0). Only when EVERY non-null branch is a pure ``const``
|
||||
of one primitive type (``bool`` never merges with ``integer``); one ``{"type": "null"}`` branch
|
||||
is tolerated as ``nullable: true``. Branch order kept; outer metadata carried; input never
|
||||
mutated."""
|
||||
def collapse(out: dict) -> Any:
|
||||
for key in _UNION_KEYS:
|
||||
variants = out.get(key)
|
||||
@@ -240,52 +212,48 @@ def collapse_const_unions(schema: Any) -> Any:
|
||||
|
||||
|
||||
_BARE_TYPE_NAMES = frozenset({"object", "string", "number", "integer", "boolean", "array", "null"})
|
||||
# Keys whose values are NOT schemas (recursing would treat "path" as a bare-string schema).
|
||||
# Values that are NOT schemas (recursing would treat a required name like "path" as a bare schema).
|
||||
_NON_SCHEMA_LIST_KEYS = frozenset({"required", "enum", "examples", "dependentRequired"})
|
||||
|
||||
|
||||
def _normalize_type_array(value: list, out: dict) -> None:
|
||||
"""Normalize a ``type: [...]`` array into *out* (llama.cpp and Gemini-via-OpenAI reject
|
||||
arrays). Per AI-SDK: one non-null type → ``type: X`` (+ ``nullable`` if ``null`` present);
|
||||
several → ``anyOf`` of single-type schemas so EVERY branch survives; none → ``null`` or
|
||||
the object fallback. Ported from anomalyco/opencode#31877."""
|
||||
"""Normalize a ``type: [...]`` array into *out* (llama.cpp and Gemini-via-OpenAI reject arrays).
|
||||
Per AI-SDK: one non-null type → ``type: X`` (+ ``nullable`` if ``null`` present); several →
|
||||
``anyOf`` of single-type schemas so EVERY branch survives; none → ``null``/object fallback."""
|
||||
has_null = "null" in value
|
||||
non_null = [t for t in value if isinstance(t, str) and t != "null"]
|
||||
if len(non_null) == 1:
|
||||
out["type"] = non_null[0]
|
||||
elif len(non_null) >= 2:
|
||||
out["anyOf"] = [{"type": t} for t in non_null]
|
||||
else:
|
||||
if not non_null:
|
||||
out["type"] = "null" if has_null else "object"
|
||||
return
|
||||
if len(non_null) == 1:
|
||||
out["type"] = non_null[0]
|
||||
else:
|
||||
out["anyOf"] = [{"type": t} for t in non_null]
|
||||
if has_null:
|
||||
out.setdefault("nullable", True)
|
||||
|
||||
|
||||
def _sanitize_node(node: Any, path: str) -> Any:
|
||||
"""Recursively sanitize a JSON-Schema fragment: bare-string schemas become ``{"type":
|
||||
<value>}`` (unknown strings → permissive object); object nodes gain ``properties: {}``;
|
||||
``type`` arrays are normalized; property keys are renamed to the provider-safe pattern
|
||||
and ``required`` follows, with entries missing from ``properties`` pruned."""
|
||||
"""Recursively sanitize a JSON-Schema fragment: bare-string schemas → ``{"type": <value>}``
|
||||
(unknown strings → permissive object); object nodes gain ``properties: {}``; ``type`` arrays
|
||||
are normalized; property keys are renamed to the provider-safe pattern and ``required``
|
||||
follows, with entries missing from ``properties`` pruned."""
|
||||
if isinstance(node, str):
|
||||
if node in _BARE_TYPE_NAMES:
|
||||
logger.debug(
|
||||
"schema_sanitizer[%s]: replacing bare-string schema %r with {'type': %r}",
|
||||
path, node, node)
|
||||
logger.debug("schema_sanitizer[%s]: replacing bare-string schema %r with {'type': %r}",
|
||||
path, node, node)
|
||||
return _empty_object() if node == "object" else {"type": node}
|
||||
logger.debug(
|
||||
"schema_sanitizer[%s]: replacing non-schema string %r "
|
||||
"with empty object schema", path, node)
|
||||
logger.debug("schema_sanitizer[%s]: replacing non-schema string %r "
|
||||
"with empty object schema", path, node)
|
||||
return _empty_object()
|
||||
if isinstance(node, list):
|
||||
return [_sanitize_node(item, f"{path}[{i}]") for i, item in enumerate(node)]
|
||||
if not isinstance(node, dict):
|
||||
return node
|
||||
|
||||
# Renames computed up front so ``required`` remaps even when it precedes ``properties``.
|
||||
prop_renames: dict[str, str] = {}
|
||||
if isinstance(node.get("properties"), dict):
|
||||
prop_renames = _rename_property_keys(node["properties"], f"{path}.properties")
|
||||
props_in = node.get("properties")
|
||||
prop_renames = (_rename_property_keys(props_in, f"{path}.properties")
|
||||
if isinstance(props_in, dict) else {})
|
||||
out: dict = {}
|
||||
for key, value in node.items():
|
||||
if key == "type" and isinstance(value, list):
|
||||
@@ -293,64 +261,55 @@ def _sanitize_node(node: Any, path: str) -> Any:
|
||||
elif key in {"properties", "$defs", "definitions"} and isinstance(value, dict):
|
||||
renames = prop_renames if key == "properties" else {}
|
||||
out[key] = {
|
||||
renames.get(sub_k, sub_k): _sanitize_node(sub_v, f"{path}.{key}.{renames.get(sub_k, sub_k)}")
|
||||
for sub_k, sub_v in value.items()}
|
||||
renames.get(k, k): _sanitize_node(v, f"{path}.{key}.{renames.get(k, k)}")
|
||||
for k, v in value.items()}
|
||||
elif key in {"items", "additionalProperties"}:
|
||||
# Bool ``additionalProperties`` is valid; bool ``items`` is non-standard but preserved.
|
||||
out[key] = value if isinstance(value, bool) else _sanitize_node(value, f"{path}.{key}")
|
||||
elif key in {"anyOf", "oneOf", "allOf"} and isinstance(value, list):
|
||||
out[key] = [_sanitize_node(item, f"{path}.{key}[{i}]") for i, item in enumerate(value)]
|
||||
elif key in _NON_SCHEMA_LIST_KEYS:
|
||||
if key == "required" and prop_renames and isinstance(value, list):
|
||||
out[key] = [prop_renames.get(r, r) if isinstance(r, str) else r for r in value]
|
||||
else:
|
||||
out[key] = copy.deepcopy(value) if isinstance(value, (list, dict)) else value
|
||||
else:
|
||||
else: # anyOf/oneOf/allOf and any other nested schema recurse (lists index the path)
|
||||
out[key] = _sanitize_node(value, f"{path}.{key}") if isinstance(value, (dict, list)) else value
|
||||
if out.get("type") == "object":
|
||||
if not isinstance(out.get("properties"), dict):
|
||||
out["properties"] = {}
|
||||
if isinstance(out.get("required"), list):
|
||||
props = out.get("properties") or {}
|
||||
valid = [r for r in out["required"] if isinstance(r, str) and r in props]
|
||||
if not valid:
|
||||
out.pop("required", None)
|
||||
elif len(valid) != len(out["required"]):
|
||||
valid = [r for r in out["required"] if isinstance(r, str) and r in out["properties"]]
|
||||
if valid:
|
||||
out["required"] = valid
|
||||
else:
|
||||
del out["required"]
|
||||
return out
|
||||
|
||||
|
||||
# ---- Reactive strips — only invoked after a backend rejects a schema -----------------------
|
||||
# ---- Reactive strips — only invoked after a backend rejects a schema ----
|
||||
_STRIP_ON_RECOVERY_KEYS = frozenset({"pattern", "format"})
|
||||
_SCHEMA_MARKERS = frozenset({"type", "anyOf", "oneOf", "allOf"}) # a node with one IS a schema
|
||||
|
||||
|
||||
def _dict_nodes(node: Any):
|
||||
"""Pre-order walk yielding every dict node; each is yielded before its values are visited,
|
||||
so a consumer may mutate it in place."""
|
||||
"""Pre-order walk over every dict node (yielded before its values, so it may be mutated)."""
|
||||
if isinstance(node, dict):
|
||||
yield node
|
||||
for v in node.values():
|
||||
yield from _dict_nodes(v)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
yield from _dict_nodes(item)
|
||||
children = node.values() if isinstance(node, dict) else node if isinstance(node, list) else ()
|
||||
for child in children:
|
||||
yield from _dict_nodes(child)
|
||||
|
||||
|
||||
def _reactive_strip(
|
||||
tools: list[dict], strip_node: Callable[[dict], int], log_msg: str) -> tuple[list[dict], int]:
|
||||
"""Apply *strip_node* (returns keywords removed) to every dict node of each tool's
|
||||
parameters, in place. Handles OpenAI (``{"function": {"parameters": ..}}``) and Responses
|
||||
(``{"name": .., "parameters": ..}``) formats. Returns ``(tools, stripped_count)``."""
|
||||
if not tools:
|
||||
return tools, 0
|
||||
"""Apply *strip_node* (-> keywords removed) to every dict node of each tool's parameters, in
|
||||
place; OpenAI (``{"function": {"parameters"}}``) and Responses (``{"parameters"}``) formats."""
|
||||
stripped = 0
|
||||
for tool in tools:
|
||||
for tool in tools or ():
|
||||
if not isinstance(tool, dict):
|
||||
continue
|
||||
fn = tool.get("function")
|
||||
params = fn.get("parameters") if isinstance(fn, dict) else None
|
||||
if not isinstance(params, dict):
|
||||
params = tool.get("parameters")
|
||||
params = params if isinstance(params, dict) else tool.get("parameters")
|
||||
if isinstance(params, dict):
|
||||
stripped += sum(strip_node(node) for node in _dict_nodes(params))
|
||||
if stripped:
|
||||
@@ -359,16 +318,14 @@ def _reactive_strip(
|
||||
|
||||
|
||||
def strip_pattern_and_format(tools: list[dict]) -> tuple[list[dict], int]:
|
||||
"""Strip ``pattern``/``format`` keywords from tool schemas, in place. Reactive: only after
|
||||
llama.cpp's grammar converter rejected a schema (its regex engine is a small ECMAScript
|
||||
subset), since cloud providers use these as prompting hints. Only strips beside
|
||||
``type``/combinators, so a property literally *named* ``pattern`` is untouched."""
|
||||
"""Strip ``pattern``/``format`` in place — reactive, only after llama.cpp's grammar converter
|
||||
rejected a schema (its regex engine is a small ECMAScript subset); cloud providers use these as
|
||||
prompting hints. Only beside ``type``/combinators, so a property *named* ``pattern`` stays."""
|
||||
def _strip(node: dict) -> int:
|
||||
if not ("type" in node or "anyOf" in node or "oneOf" in node or "allOf" in node):
|
||||
return 0
|
||||
hits = [k for k in node if k in _STRIP_ON_RECOVERY_KEYS]
|
||||
is_schema = bool(node.keys() & _SCHEMA_MARKERS)
|
||||
hits = [k for k in node if k in _STRIP_ON_RECOVERY_KEYS] if is_schema else []
|
||||
for k in hits:
|
||||
node.pop(k, None)
|
||||
del node[k]
|
||||
return len(hits)
|
||||
return _reactive_strip(
|
||||
tools, _strip,
|
||||
@@ -377,13 +334,12 @@ def strip_pattern_and_format(tools: list[dict]) -> tuple[list[dict], int]:
|
||||
|
||||
|
||||
def strip_slash_enum(tools: list[dict]) -> tuple[list[dict], int]:
|
||||
"""Strip ``enum`` keywords whose string values contain ``/``, in place. xAI compiles
|
||||
schemas to a grammar that rejects ``/`` in enum values (HTTP 400 before any token) —
|
||||
typically MCP enums of HuggingFace model IDs. The constraint is a prompting hint only."""
|
||||
"""Strip ``enum`` keywords whose string values contain ``/``, in place: xAI's grammar compiler
|
||||
rejects them (HTTP 400 before any token) — typically MCP enums of HuggingFace model IDs."""
|
||||
def _strip(node: dict) -> int:
|
||||
enum_val = node.get("enum")
|
||||
if isinstance(enum_val, list) and any(isinstance(v, str) and "/" in v for v in enum_val):
|
||||
node.pop("enum", None)
|
||||
del node["enum"]
|
||||
return 1
|
||||
return 0
|
||||
return _reactive_strip(
|
||||
|
||||
+112
-164
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
@@ -11,9 +12,7 @@ from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from tools.approval import (
|
||||
_bash_exec_payload,
|
||||
_deobfuscate_shell_word_for_detection,
|
||||
_iter_shell_command_starts,
|
||||
_bash_exec_payload, _deobfuscate_shell_word_for_detection, _iter_shell_command_starts,
|
||||
_read_shell_word)
|
||||
|
||||
# bisect drives repeated checkouts of the running root — the exact skew hazard guarded here.
|
||||
@@ -24,8 +23,7 @@ _WORKTREE_TARGET_ACTIONS = frozenset({"move", "remove"})
|
||||
_STASH_SAFE_ACTIONS = frozenset({"list", "show", "create", "store", "drop", "clear"})
|
||||
_RESET_WORKTREE_MODES = frozenset({"--hard", "--merge", "--keep"})
|
||||
# `reset`/`stash`/`clean`/`restore` reach this set only in their SAFE forms (_mutates_worktree
|
||||
# classifies the dangerous forms first); listing them just skips a pointless
|
||||
# `git config --get alias.<sub>` subprocess for `stash list`, `reset --soft`, `clean -n`.
|
||||
# runs first); listing them skips a pointless `git config --get alias.<sub>` subprocess.
|
||||
_KNOWN_GIT_BUILTINS = frozenset({
|
||||
"add", "am", "apply", "blame", "branch", "bundle", "cat-file", "clean", "clone", "commit",
|
||||
"config", "describe", "diff", "fetch", "format-patch", "grep", "help", "init", "log",
|
||||
@@ -64,36 +62,31 @@ class _Heredoc:
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ShellContext:
|
||||
class _ShellContext: # one `(` / `$(` / backtick nesting level and its live quote state
|
||||
kind: str
|
||||
opener: int
|
||||
quote: str | None = None
|
||||
|
||||
|
||||
def get_running_source_root() -> Path | None:
|
||||
"""Return the source checkout backing this process, if there is one."""
|
||||
try:
|
||||
"""The source checkout backing this process, if there is one."""
|
||||
with contextlib.suppress(OSError, RuntimeError):
|
||||
root = Path(__file__).resolve().parent.parent
|
||||
except (OSError, RuntimeError):
|
||||
return None
|
||||
return root if (root / ".git").exists() else None
|
||||
return root if (root / ".git").exists() else None
|
||||
return None
|
||||
|
||||
|
||||
def _resolve(path_str: str, base: Path) -> Path:
|
||||
path = Path(os.path.expanduser(path_str))
|
||||
if not path.is_absolute():
|
||||
path = base / path
|
||||
try:
|
||||
path = base / Path(os.path.expanduser(path_str)) # ``/`` keeps an absolute right operand
|
||||
with contextlib.suppress(OSError, RuntimeError, ValueError):
|
||||
return path.resolve()
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
return path
|
||||
return path
|
||||
|
||||
|
||||
def _is_within(path: Path, root: Path) -> bool:
|
||||
try:
|
||||
with contextlib.suppress(OSError, RuntimeError, ValueError):
|
||||
return path == root or path.is_relative_to(root)
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def _executable_name(value: str) -> str:
|
||||
@@ -101,6 +94,7 @@ def _executable_name(value: str) -> str:
|
||||
|
||||
|
||||
def _shell_words_at(command: str, start: int) -> list[str]:
|
||||
"""Deobfuscated words of the simple command at ``start`` (stops at a newline; max 64)."""
|
||||
words: list[str] = []
|
||||
cursor = start
|
||||
for _ in range(64):
|
||||
@@ -116,13 +110,10 @@ def _consume_options(
|
||||
words: list[str], start: int, options_with_arg: frozenset[str] = _NO_OPTIONS) -> int:
|
||||
"""Index of the first positional at/after ``start`` (``--`` ends options)."""
|
||||
index = start
|
||||
while index < len(words):
|
||||
option = words[index]
|
||||
if option == "--":
|
||||
while index < len(words) and words[index].startswith("-") and words[index] != "-":
|
||||
if words[index] == "--":
|
||||
return index + 1
|
||||
if not option.startswith("-") or option == "-":
|
||||
break
|
||||
index += 2 if "=" not in option and option in options_with_arg else 1
|
||||
index += 2 if "=" not in words[index] and words[index] in options_with_arg else 1
|
||||
return index
|
||||
|
||||
|
||||
@@ -140,9 +131,8 @@ def _command_parts(words: list[str]) -> tuple[dict[str, str], str | None, list[s
|
||||
wrapper_options = _WRAPPER_OPTIONS_WITH_ARG.get(executable)
|
||||
if wrapper_options is None:
|
||||
return env, words[index], words[index + 1 :]
|
||||
# `command -v/-V` only reports; nothing runs.
|
||||
if executable == "command" and words[index + 1 : index + 2] in (["-v"], ["-V"]):
|
||||
return env, None, []
|
||||
break # `command -v/-V` only reports; nothing runs
|
||||
index = _consume_options(words, index + 1, wrapper_options)
|
||||
return env, None, []
|
||||
|
||||
@@ -157,15 +147,14 @@ def _scope_keys(command: str, starts: list[int]) -> dict[int, tuple[int, ...]]:
|
||||
context = contexts[-1]
|
||||
quote = context.quote
|
||||
char = command[cursor]
|
||||
nested = len(contexts) > 1
|
||||
if quote == "'":
|
||||
if char == "'":
|
||||
context.quote = None
|
||||
closes = quote is None and len(contexts) > 1 # an unquoted closer may pop a scope
|
||||
if quote is not None and char == quote:
|
||||
context.quote = None
|
||||
elif quote == "'":
|
||||
pass # single quotes: no escapes, no substitutions
|
||||
elif char == "\\" and cursor + 1 < start:
|
||||
cursor += 1
|
||||
elif quote == '"' and char == '"':
|
||||
context.quote = None
|
||||
elif quote is None and char in {"'", '"'}:
|
||||
elif quote is None and char in "'\"":
|
||||
context.quote = char
|
||||
# Unquoted or inside double quotes: substitutions still open scopes.
|
||||
elif command.startswith("$(", cursor):
|
||||
@@ -173,28 +162,27 @@ def _scope_keys(command: str, starts: list[int]) -> dict[int, tuple[int, ...]]:
|
||||
cursor += 1
|
||||
elif quote is None and char == "(":
|
||||
contexts.append(_ShellContext("(", cursor))
|
||||
elif quote is None and char == ")" and nested and contexts[-1].kind in {"(", "$("}:
|
||||
elif (char == ")" and closes and context.kind in {"(", "$("}) or (
|
||||
char == "`" and closes and context.kind == "`"):
|
||||
contexts.pop()
|
||||
elif char == "`":
|
||||
if quote is None and nested and contexts[-1].kind == "`":
|
||||
contexts.pop()
|
||||
else:
|
||||
contexts.append(_ShellContext("`", cursor))
|
||||
contexts.append(_ShellContext("`", cursor))
|
||||
cursor += 1
|
||||
scopes[start] = tuple(item.opener for item in contexts[1:])
|
||||
return scopes
|
||||
|
||||
|
||||
def _operator_before(command: str, start: int) -> str | None:
|
||||
"""The list/grouping operator (or newline) immediately preceding a command start."""
|
||||
head = command[:start].rstrip()
|
||||
if head[-2:] in {"&&", "||"}:
|
||||
return head[-2:]
|
||||
if head[-1:] in {";", "|", "&", "(", "{"}:
|
||||
return head[-1:]
|
||||
for tail in (head[-2:], head[-1:]):
|
||||
if tail in {"&&", "||", ";", "|", "&", "(", "{"}:
|
||||
return tail
|
||||
return "\n" if "\n" in command[len(head):start] else None
|
||||
|
||||
|
||||
def _cd_target(executable: str, args: list[str], cwd: Path) -> Path | None:
|
||||
"""Directory a ``cd``/``pushd`` would land in (existing dirs only), else None."""
|
||||
if _executable_name(executable) not in {"cd", "pushd"}:
|
||||
return None
|
||||
index = _consume_options(args, 0)
|
||||
@@ -205,17 +193,15 @@ def _cd_target(executable: str, args: list[str], cwd: Path) -> Path | None:
|
||||
|
||||
|
||||
def _shell_script_arg(args: list[str]) -> str | None:
|
||||
"""Return the script string owned by a shell's ``-c``, if present. approval.py's
|
||||
``_bash_exec_payload`` parses bash's real option grammar (``-o pipefail -c '<script>'``
|
||||
hides ``-c`` behind an operand); when it finds no ``-c``, fall back to a permissive
|
||||
positional scan, since zsh/dash/ksh option letters (``zsh -yc``) fall outside bash's
|
||||
alphabet and would otherwise make this block-guard fail open."""
|
||||
"""Script string owned by a shell's ``-c``, if present. ``_bash_exec_payload`` parses bash's
|
||||
real option grammar (``-o pipefail -c '<script>'``); when it finds no ``-c``, fall back to a
|
||||
permissive scan: zsh/dash/ksh letters (``zsh -yc``) would otherwise fail this guard open."""
|
||||
has_c, payload = _bash_exec_payload(args)
|
||||
if has_c:
|
||||
return payload
|
||||
for index, arg in enumerate(args):
|
||||
if arg == "--" or not arg.startswith("-"):
|
||||
break
|
||||
return None
|
||||
if "c" in arg[1:]:
|
||||
return args[index + 1] if index + 1 < len(args) else None
|
||||
return None
|
||||
@@ -233,7 +219,7 @@ def _heredoc_specs(line: str) -> list[_Heredoc]:
|
||||
index += 1 # skip the escaped character too
|
||||
elif char == quote:
|
||||
quote = None
|
||||
elif char in {"'", '"'}:
|
||||
elif char in "'\"":
|
||||
quote = char
|
||||
if quote or not line.startswith("<<", index) or line.startswith("<<<", index):
|
||||
index += 1
|
||||
@@ -241,52 +227,47 @@ def _heredoc_specs(line: str) -> list[_Heredoc]:
|
||||
opener = _HEREDOC_OPENER_RE.match(line, index)
|
||||
if opener is None: # unterminated quoted delimiter: give up on this line
|
||||
break
|
||||
operator_at, index = index, opener.end()
|
||||
strip_tabs = bool(opener.group("dash"))
|
||||
header, index = line[:index], opener.end()
|
||||
delimiter = opener.group("quoted") if opener.group("q") else opener.group("bare")
|
||||
if not delimiter:
|
||||
continue
|
||||
header = line[:operator_at]
|
||||
starts = list(_iter_shell_command_starts(header))
|
||||
words = _shell_words_at(header, starts[-1]) if starts else []
|
||||
_, executable, args = _command_parts(words)
|
||||
_, executable, args = _command_parts(_shell_words_at(header, starts[-1]) if starts else [])
|
||||
# A bare shell (no -c script, no script operand) executes the body itself.
|
||||
execute_as_shell = bool(
|
||||
executable
|
||||
and _executable_name(executable) in _SHELL_EXECUTABLES
|
||||
executable and _executable_name(executable) in _SHELL_EXECUTABLES
|
||||
and _shell_script_arg(args) is None
|
||||
and not any(arg and not arg.startswith("-") for arg in args))
|
||||
specs.append(_Heredoc(delimiter, strip_tabs, execute_as_shell))
|
||||
specs.append(_Heredoc(delimiter, bool(opener.group("dash")), execute_as_shell))
|
||||
return specs
|
||||
|
||||
|
||||
def _mask_heredocs(command: str) -> tuple[str, list[str]]:
|
||||
"""Blank heredoc bodies; return (masked command, bodies a bare shell would execute).
|
||||
Unterminated heredocs run to end of input and are still reported."""
|
||||
"""Blank heredoc bodies -> (masked command, bodies a bare shell would execute). Unterminated
|
||||
heredocs run to end of input and are still reported."""
|
||||
output: list[str] = []
|
||||
pending: list[_Heredoc] = []
|
||||
finished: list[_Heredoc] = []
|
||||
for line in command.splitlines(keepends=True):
|
||||
if pending:
|
||||
current = pending[0]
|
||||
candidate = line.rstrip("\r\n")
|
||||
if current.strip_tabs:
|
||||
candidate = candidate.lstrip("\t")
|
||||
if candidate == current.delimiter:
|
||||
finished.append(pending.pop(0))
|
||||
else:
|
||||
current.body.append(line)
|
||||
output.append("".join(char if char in {"\r", "\n"} else " " for char in line))
|
||||
if not pending:
|
||||
output.append(line)
|
||||
pending.extend(_heredoc_specs(line))
|
||||
continue
|
||||
output.append(line)
|
||||
pending.extend(_heredoc_specs(line))
|
||||
current = pending[0]
|
||||
candidate = line.rstrip("\r\n")
|
||||
if (candidate.lstrip("\t") if current.strip_tabs else candidate) == current.delimiter:
|
||||
finished.append(pending.pop(0))
|
||||
else:
|
||||
current.body.append(line)
|
||||
output.append(re.sub(r"[^\r\n]", " ", line))
|
||||
shell_scripts = ["".join(spec.body) for spec in finished + pending if spec.execute_as_shell]
|
||||
return "".join(output), shell_scripts
|
||||
|
||||
|
||||
def _record_alias(config: str, aliases: dict[str, str]) -> None:
|
||||
"""Record an inline ``-c alias.<name>=<value>`` git config override."""
|
||||
if config.lower().startswith("alias.") and "=" in config:
|
||||
key, value = config.split("=", 1)
|
||||
key, sep, value = config.partition("=")
|
||||
if sep and key.lower().startswith("alias."):
|
||||
aliases[key[6:].lower()] = value
|
||||
|
||||
|
||||
@@ -305,88 +286,70 @@ def _git_target_and_subcommand(
|
||||
break
|
||||
if not arg.startswith("-"):
|
||||
break
|
||||
# Separate-argument form (`-C dir`) vs attached form (`-Cdir`, `--work-tree=dir`, `-cK=V`).
|
||||
if arg in _GIT_GLOBAL_OPTIONS_WITH_ARG:
|
||||
if index + 1 < len(args):
|
||||
value = args[index + 1]
|
||||
if arg == "-C":
|
||||
target = _resolve(value, target)
|
||||
elif arg == "--work-tree":
|
||||
work_tree = value
|
||||
elif arg == "-c":
|
||||
_record_alias(value, aliases)
|
||||
option, value = arg, args[index + 1] if index + 1 < len(args) else None
|
||||
index += 2
|
||||
continue
|
||||
if arg.startswith("-C") and len(arg) > 2:
|
||||
target = _resolve(arg[2:], target)
|
||||
elif arg.startswith("--work-tree="):
|
||||
work_tree = arg.split("=", 1)[1]
|
||||
elif arg.startswith("-calias."):
|
||||
_record_alias(arg[2:], aliases)
|
||||
index += 1
|
||||
else:
|
||||
option, value = next(((o, arg[len(o):]) for o in ("-C", "--work-tree=", "-c")
|
||||
if arg.startswith(o) and len(arg) > len(o)), (None, None))
|
||||
index += 1
|
||||
if option == "-C" and value is not None:
|
||||
target = _resolve(value, target)
|
||||
elif option and option.startswith("--work-tree") and value is not None:
|
||||
work_tree = value
|
||||
elif option == "-c" and value is not None:
|
||||
_record_alias(value, aliases) # only alias.* overrides matter; others are ignored
|
||||
explicit_work_tree = work_tree or env.get("GIT_WORK_TREE")
|
||||
if explicit_work_tree:
|
||||
target = _resolve(explicit_work_tree, target)
|
||||
return target, args[index].lower() if index < len(args) else None, args[index + 1 :], aliases
|
||||
|
||||
|
||||
def _has_short_flag(arg: str, letter: str) -> bool:
|
||||
return arg.startswith("-") and letter in arg[1:]
|
||||
|
||||
|
||||
def _reset_mutates(args: list[str]) -> bool:
|
||||
return any(arg in _RESET_WORKTREE_MODES or _RESET_HARD_RE.fullmatch(arg) for arg in args)
|
||||
|
||||
|
||||
def _stash_mutates(args: list[str]) -> bool:
|
||||
action = next((arg for arg in args if not arg.startswith("-")), "push")
|
||||
return action not in _STASH_SAFE_ACTIONS
|
||||
|
||||
|
||||
def _clean_mutates(args: list[str]) -> bool:
|
||||
return not any(
|
||||
arg == "--dry-run" or (not arg.startswith("--") and _has_short_flag(arg, "n"))
|
||||
def _has_flag(args: list[str], long: str, letter: str, short_only: bool = False) -> bool:
|
||||
"""``--long`` present, or ``letter`` inside a dash-prefixed arg (``-fdn``; ``short_only``
|
||||
additionally excludes ``--`` args from the letter scan)."""
|
||||
return any(
|
||||
arg == long or (arg.startswith("-") and letter in arg[1:]
|
||||
and not (short_only and arg.startswith("--")))
|
||||
for arg in args)
|
||||
|
||||
|
||||
def _restore_mutates(args: list[str]) -> bool:
|
||||
staged = any(arg == "--staged" or _has_short_flag(arg, "S") for arg in args)
|
||||
worktree = any(arg == "--worktree" or _has_short_flag(arg, "W") for arg in args)
|
||||
return worktree or not staged
|
||||
|
||||
|
||||
# Subcommands whose worktree impact depends on their arguments.
|
||||
# Subcommands whose worktree impact depends on their arguments -> predicate(args).
|
||||
_CONDITIONAL_MUTATIONS: dict[str, Callable[[list[str]], bool]] = {
|
||||
"reset": _reset_mutates,
|
||||
"stash": _stash_mutates,
|
||||
"clean": _clean_mutates,
|
||||
"restore": _restore_mutates}
|
||||
"reset": lambda args: any(
|
||||
arg in _RESET_WORKTREE_MODES or _RESET_HARD_RE.fullmatch(arg) for arg in args),
|
||||
"stash": lambda args: next(
|
||||
(arg for arg in args if not arg.startswith("-")), "push") not in _STASH_SAFE_ACTIONS,
|
||||
"clean": lambda args: not _has_flag(args, "--dry-run", "n", short_only=True),
|
||||
# `restore` touches the worktree unless ONLY --staged was requested.
|
||||
"restore": lambda args: (_has_flag(args, "--worktree", "W")
|
||||
or not _has_flag(args, "--staged", "S"))}
|
||||
|
||||
|
||||
def _mutates_worktree(subcommand: str, args: list[str]) -> bool:
|
||||
check = _CONDITIONAL_MUTATIONS.get(subcommand)
|
||||
return check(args) if check is not None else subcommand in _WORKTREE_MUTATIONS
|
||||
check = _CONDITIONAL_MUTATIONS.get(subcommand, lambda _args: subcommand in _WORKTREE_MUTATIONS)
|
||||
return check(args)
|
||||
|
||||
|
||||
def _inspect_git_worktree(args: list[str], cwd: Path, root: Path) -> str | None:
|
||||
"""Block `worktree remove|move` aimed at the running root, from any directory."""
|
||||
action_index = _consume_options(args, 0)
|
||||
action = args[action_index].lower() if action_index < len(args) else None
|
||||
if action not in _WORKTREE_TARGET_ACTIONS:
|
||||
return None
|
||||
target_index = _consume_options(args, action_index + 1)
|
||||
if target_index < len(args) and _resolve(args[target_index], cwd) == root:
|
||||
if (action in _WORKTREE_TARGET_ACTIONS and target_index < len(args)
|
||||
and _resolve(args[target_index], cwd) == root):
|
||||
return f"git worktree {action}"
|
||||
return None
|
||||
|
||||
|
||||
def _read_git_alias(executable: str, target: Path, alias: str) -> str | None:
|
||||
try:
|
||||
with contextlib.suppress(OSError, subprocess.SubprocessError):
|
||||
result = subprocess.run(
|
||||
[executable, "-C", str(target), "config", "--get", f"alias.{alias}"],
|
||||
capture_output=True, text=True, timeout=1, check=False)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
return (result.stdout.strip() or None) if result.returncode == 0 else None
|
||||
return (result.stdout.strip() or None) if result.returncode == 0 else None
|
||||
return None
|
||||
|
||||
|
||||
def _inspect_git(
|
||||
@@ -396,8 +359,7 @@ def _inspect_git(
|
||||
args, current_dir, env)
|
||||
if subcommand is None:
|
||||
return None
|
||||
# `worktree` names its victim as an argument, so the cwd check does not apply.
|
||||
if subcommand == "worktree":
|
||||
if subcommand == "worktree": # names its victim as an argument: the cwd check does not apply
|
||||
return _inspect_git_worktree(sub_args, target, root)
|
||||
if not _is_within(target, root):
|
||||
return None
|
||||
@@ -405,18 +367,16 @@ def _inspect_git(
|
||||
return f"git {subcommand}"
|
||||
if subcommand in _KNOWN_GIT_BUILTINS or depth >= _MAX_RECURSION:
|
||||
return None
|
||||
alias = inline_aliases.get(subcommand)
|
||||
if alias is None:
|
||||
alias = _read_git_alias(executable, target, subcommand)
|
||||
alias = (inline_aliases[subcommand] if subcommand in inline_aliases
|
||||
else _read_git_alias(executable, target, subcommand))
|
||||
if not alias:
|
||||
return None
|
||||
if alias.startswith("!"):
|
||||
if alias.startswith("!"): # shell alias: scan it as a command
|
||||
return _find_mutation(alias[1:], target, root, depth + 1)
|
||||
try:
|
||||
with contextlib.suppress(ValueError):
|
||||
alias_args = shlex.split(alias, posix=True)
|
||||
except ValueError:
|
||||
return None
|
||||
return _inspect_git(executable, [*alias_args, *sub_args], target, {}, root, depth + 1)
|
||||
return _inspect_git(executable, [*alias_args, *sub_args], target, {}, root, depth + 1)
|
||||
return None
|
||||
|
||||
|
||||
def _inspect_github_cli(
|
||||
@@ -425,9 +385,8 @@ def _inspect_github_cli(
|
||||
if not _is_within(current_dir, root):
|
||||
return None
|
||||
index = _consume_options(args, 0, frozenset({"-R", "--repo", "--hostname"}))
|
||||
if args[index : index + 2] == ["pr", "checkout"]:
|
||||
return f"{_executable_name(executable)} pr checkout"
|
||||
return None
|
||||
is_checkout = args[index : index + 2] == ["pr", "checkout"]
|
||||
return f"{_executable_name(executable)} pr checkout" if is_checkout else None
|
||||
|
||||
|
||||
def _inspect_shell(
|
||||
@@ -439,9 +398,7 @@ def _inspect_shell(
|
||||
|
||||
# executable name -> inspector(executable, args, current_dir, env, root, depth)
|
||||
_INSPECTORS: dict[str, Callable[..., str | None]] = {
|
||||
"git": _inspect_git,
|
||||
"gh": _inspect_github_cli,
|
||||
"hub": _inspect_github_cli,
|
||||
"git": _inspect_git, "gh": _inspect_github_cli, "hub": _inspect_github_cli,
|
||||
**{shell: _inspect_shell for shell in _SHELL_EXECUTABLES}}
|
||||
|
||||
|
||||
@@ -451,8 +408,7 @@ def _find_mutation(command: str, cwd: Path, root: Path, depth: int = 0) -> str |
|
||||
return None
|
||||
masked_command, heredoc_scripts = _mask_heredocs(command)
|
||||
for script in heredoc_scripts:
|
||||
operation = _find_mutation(script, cwd, root, depth + 1)
|
||||
if operation:
|
||||
if operation := _find_mutation(script, cwd, root, depth + 1):
|
||||
return operation
|
||||
starts = sorted(set(_iter_shell_command_starts(masked_command)))
|
||||
scopes = _scope_keys(masked_command, starts)
|
||||
@@ -462,50 +418,42 @@ def _find_mutation(command: str, cwd: Path, root: Path, depth: int = 0) -> str |
|
||||
for start in starts:
|
||||
scope = scopes[start]
|
||||
cwd_by_scope.setdefault(scope, cwd_by_scope.get(scope[:-1], cwd))
|
||||
operator = _operator_before(masked_command, start)
|
||||
pending = pending_cd.pop(scope, None)
|
||||
if pending is not None and operator in {"&&", ";", "\n"}:
|
||||
if pending is not None and _operator_before(masked_command, start) in {"&&", ";", "\n"}:
|
||||
cwd_by_scope[scope] = pending
|
||||
env, executable, args = _command_parts(_shell_words_at(masked_command, start))
|
||||
if executable is None:
|
||||
continue
|
||||
current_dir = cwd_by_scope[scope]
|
||||
cd_target = _cd_target(executable, args, current_dir)
|
||||
if cd_target is not None:
|
||||
if (cd_target := _cd_target(executable, args, current_dir)) is not None:
|
||||
pending_cd[scope] = cd_target
|
||||
continue
|
||||
inspect = _INSPECTORS.get(_executable_name(executable))
|
||||
if inspect is not None:
|
||||
operation = inspect(executable, args, current_dir, env, root, depth)
|
||||
if operation:
|
||||
return operation
|
||||
elif (inspect := _INSPECTORS.get(_executable_name(executable))) and (
|
||||
operation := inspect(executable, args, current_dir, env, root, depth)):
|
||||
return operation
|
||||
return None
|
||||
|
||||
|
||||
def guard_active() -> bool:
|
||||
"""Whether the self-repo git guard applies on this platform. Windows-only: NTFS locks
|
||||
loaded .py/.pyd files, so overwriting the live checkout can corrupt the running
|
||||
process. On POSIX open handles keep the old inode alive; the mixed-module hazard is
|
||||
limited to later lazy imports — not worth blocking every git workflow for."""
|
||||
"""Windows-only: NTFS locks loaded .py/.pyd files, so overwriting the live checkout can
|
||||
corrupt the running process. On POSIX open handles keep the old inode alive; the mixed-module
|
||||
hazard is limited to later lazy imports — not worth blocking every git workflow for."""
|
||||
return os.name == "nt"
|
||||
|
||||
|
||||
def detect_self_repo_git_mutation(
|
||||
command: str, cwd: str | None, source_root: Path | None = None) -> tuple[bool, str | None]:
|
||||
"""Return whether a command would rewrite the live source checkout."""
|
||||
"""-> (blocked, message): whether a command would rewrite the live source checkout."""
|
||||
root = source_root if source_root is not None else get_running_source_root()
|
||||
if root is None or not command:
|
||||
return False, None
|
||||
root = _resolve(str(root), Path("/"))
|
||||
operation = _find_mutation(command, _resolve(cwd, Path("/")) if cwd else Path("/"), root)
|
||||
operation = _find_mutation(command, _resolve(cwd or "/", Path("/")), root)
|
||||
return (True, _block_message(operation, root)) if operation is not None else (False, None)
|
||||
|
||||
|
||||
def _block_message(operation: str, root: Path) -> str:
|
||||
# Suggest a disk-backed scratch dir: /tmp is usually tmpfs (see message).
|
||||
hermes_home = os.environ.get("HERMES_HOME", "").strip()
|
||||
home = Path(hermes_home).expanduser() if hermes_home else Path.home() / ".hermes"
|
||||
scratch = home / "scratch"
|
||||
scratch = (Path(hermes_home).expanduser() if hermes_home else Path.home() / ".hermes") / "scratch"
|
||||
return (
|
||||
f"Blocked: `{operation}` would rewrite Hermes's live source checkout "
|
||||
f"({root}) and can mix module versions in this running process. "
|
||||
|
||||
+62
-111
@@ -1,19 +1,18 @@
|
||||
"""Conservative heredoc masking for shell-command scanners (terminal '&' guard, blocked-command
|
||||
checks, cron lifecycle_guard) that false-positive on heredoc *bodies*. Stripping every body is
|
||||
unsafe the other way (a fake ``<<`` in quotes can swallow a real operator; unquoted bodies
|
||||
expand; ``bash <<'EOF'`` executes), so a body is masked ONLY when every delimiter is quoted,
|
||||
every heredoc has an exact terminator line, the opener is a single command (no ``;|&``,
|
||||
``$(...)``, backticks or process substitution) and the consumer is an allowlisted non-shell
|
||||
interpreter. Otherwise the command is returned untouched: a false positive is acceptable,
|
||||
hiding shell syntax from a guard is not. Masked bodies keep their newline count (re.MULTILINE)."""
|
||||
"""Conservative heredoc masking for shell-command scanners ('&' guard, blocked-command checks,
|
||||
cron lifecycle_guard) that false-positive on heredoc *bodies*. Stripping every body is unsafe the
|
||||
other way (a fake ``<<`` in quotes can swallow an operator; unquoted bodies expand; ``bash <<'EOF'``
|
||||
executes), so a body is masked ONLY when every delimiter is quoted, every heredoc has an exact
|
||||
terminator line, the opener is a single command (no ``;|&``, ``$(...)``, backticks, process
|
||||
substitution) and the consumer is an allowlisted non-shell interpreter. Otherwise the command is
|
||||
returned untouched: a false positive is acceptable, hiding shell syntax from a guard is not.
|
||||
Masked bodies keep their newline count (re.MULTILINE)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# Non-shell interpreters whose quoted heredoc bodies are program text/data for
|
||||
# THAT interpreter. Optional VAR=... assignments, ``env`` and a path prefix are
|
||||
# allowed. Deliberately narrow: anything unmatched keeps its body visible.
|
||||
# Non-shell interpreters whose quoted heredoc bodies are data for THAT interpreter; optional
|
||||
# VAR=... assignments, ``env`` and a path prefix allowed. Narrow on purpose: unmatched = visible.
|
||||
_INERT_HEREDOC_CONSUMER_RE = re.compile(
|
||||
r"^\s*(?:[A-Z_][A-Z0-9_]*=\S+\s+)*(?:env\s+)?(?:[A-Za-z0-9_./-]+/)?"
|
||||
r"(?:python(?:3(?:\.\d+)*)?|osascript|cat)(?=\s|$)",
|
||||
@@ -24,12 +23,9 @@ def _span_end(command: str, cursor: int, closer: str) -> int:
|
||||
"""Index just past the backslash-aware span opened at ``cursor``."""
|
||||
end = cursor + 1
|
||||
while end < len(command):
|
||||
if command[end] == "\\" and end + 1 < len(command):
|
||||
end += 2
|
||||
continue
|
||||
if command[end] == closer:
|
||||
return end + 1
|
||||
end += 1
|
||||
end += 2 if command[end] == "\\" and end + 1 < len(command) else 1
|
||||
return end
|
||||
|
||||
|
||||
@@ -39,20 +35,15 @@ def _mask_simple_quotes(command: str) -> str:
|
||||
cursor = 0
|
||||
while cursor < len(command):
|
||||
char = command[cursor]
|
||||
if char == "'":
|
||||
closing = command.find("'", cursor + 1)
|
||||
if closing == -1:
|
||||
result.append(command[cursor:])
|
||||
break
|
||||
result.append("''")
|
||||
cursor = closing + 1
|
||||
elif char == '"':
|
||||
end = _span_end(command, cursor, '"')
|
||||
if not command[cursor:end].endswith('"'):
|
||||
result.append(command[cursor:])
|
||||
break
|
||||
if char in "'\"": # single quotes have no escapes; double quotes are backslash-aware
|
||||
end = (command.find("'", cursor + 1) + 1 if char == "'"
|
||||
else _span_end(command, cursor, '"'))
|
||||
segment = command[cursor:end]
|
||||
result.append(segment if "$(" in segment or "`" in segment else '""')
|
||||
if not segment.endswith(char):
|
||||
result.append(command[cursor:])
|
||||
break
|
||||
keep = char == '"' and ("$(" in segment or "`" in segment)
|
||||
result.append(segment if keep else char * 2)
|
||||
cursor = end
|
||||
elif char == "`":
|
||||
end = _span_end(command, cursor, "`")
|
||||
@@ -68,118 +59,88 @@ def _parse_heredoc_operator(command: str, index: int):
|
||||
"""Parse one ``<<`` opener -> ``(end_index, delimiter, strip_tabs, quoted)`` or None."""
|
||||
if not command.startswith("<<", index) or command.startswith("<<<", index):
|
||||
return None
|
||||
|
||||
cursor = index + 2
|
||||
strip_tabs = cursor < len(command) and command[cursor] == "-"
|
||||
if strip_tabs:
|
||||
cursor += 1
|
||||
strip_tabs = command.startswith("-", index + 2)
|
||||
cursor = index + 3 if strip_tabs else index + 2
|
||||
while cursor < len(command) and command[cursor] in " \t":
|
||||
cursor += 1
|
||||
if cursor >= len(command) or command[cursor] in "\r\n":
|
||||
return None
|
||||
|
||||
delimiter: list[str] = []
|
||||
quoted = False
|
||||
while cursor < len(command):
|
||||
while cursor < len(command) and not (command[cursor].isspace() or command[cursor] in ";&|<>()"):
|
||||
char = command[cursor]
|
||||
if char.isspace() or char in ";&|<>()":
|
||||
break
|
||||
if char == "\\":
|
||||
if char == "\\": # backslash-escaped char: quoted, literal
|
||||
if cursor + 1 >= len(command) or command[cursor + 1] in "\r\n":
|
||||
return None
|
||||
quoted = True
|
||||
delimiter.append(command[cursor + 1])
|
||||
cursor += 2
|
||||
continue
|
||||
if char in "'\"":
|
||||
elif char in "'\"":
|
||||
quoted = True
|
||||
quote = char
|
||||
cursor += 1
|
||||
while cursor < len(command) and command[cursor] != quote:
|
||||
while cursor < len(command) and command[cursor] != char:
|
||||
current = command[cursor]
|
||||
if current in "\r\n":
|
||||
return None
|
||||
if quote == '"' and current == "\\":
|
||||
if char == '"' and current == "\\":
|
||||
if cursor + 1 >= len(command):
|
||||
return None
|
||||
following = command[cursor + 1]
|
||||
if following in {"$", "`", '"', "\\", "\n"}:
|
||||
delimiter.append(following)
|
||||
cursor += 2
|
||||
continue
|
||||
# In double quotes, backslash is literal before other chars.
|
||||
if command[cursor + 1] in '$`"\\\n': # else backslash is literal in dquotes
|
||||
cursor += 1
|
||||
current = command[cursor]
|
||||
delimiter.append(current)
|
||||
cursor += 1
|
||||
if cursor >= len(command):
|
||||
if cursor >= len(command): # unterminated quote
|
||||
return None
|
||||
cursor += 1
|
||||
continue
|
||||
delimiter.append(char)
|
||||
cursor += 1
|
||||
|
||||
else:
|
||||
delimiter.append(char)
|
||||
cursor += 1
|
||||
if not delimiter and not quoted:
|
||||
return None
|
||||
return cursor, "".join(delimiter), strip_tabs, quoted
|
||||
|
||||
|
||||
def _scan_heredoc_command_unit(command: str, start: int):
|
||||
"""Scan one logical command -> ``(end, specs, unknown_operator, has_list_operator)``:
|
||||
an unparseable ``<<`` (caller must fail closed) / unquoted ``;|&`` on the opener."""
|
||||
"""Scan one logical command -> ``(end, specs, unknown_operator, has_list_operator)``: an
|
||||
unparseable ``<<`` (caller must fail closed) / an unquoted ``;|&`` on the opener line."""
|
||||
cursor = start
|
||||
quote = None
|
||||
comment = False
|
||||
specs = []
|
||||
unknown_operator = False
|
||||
has_list_operator = False
|
||||
|
||||
while cursor < len(command):
|
||||
char = command[cursor]
|
||||
if comment:
|
||||
if char == "\n":
|
||||
return cursor, specs, unknown_operator, has_list_operator
|
||||
cursor += 1
|
||||
continue
|
||||
if quote is not None:
|
||||
if quote in {'"', "`"} and char == "\\" and cursor + 1 < len(command):
|
||||
cursor += 2
|
||||
continue
|
||||
if char == "\n" and (comment or quote is None):
|
||||
break
|
||||
# Backslash escapes (incl. line continuations) outside single quotes skip the next char.
|
||||
escaped = char == "\\" and quote != "'" and not comment and cursor + 1 < len(command)
|
||||
if comment or quote is not None or escaped:
|
||||
if char == quote:
|
||||
quote = None
|
||||
cursor += 1
|
||||
continue
|
||||
if char == "\\" and cursor + 1 < len(command):
|
||||
# Includes line continuations: the logical command keeps going.
|
||||
cursor += 2
|
||||
continue
|
||||
if char in "'\"`":
|
||||
cursor += 2 if escaped else 1
|
||||
elif char in "'\"`":
|
||||
quote = char
|
||||
cursor += 1
|
||||
continue
|
||||
if char == "#":
|
||||
previous = command[cursor - 1] if cursor > start else ""
|
||||
if cursor == start or previous.isspace() or previous in ";&|()":
|
||||
comment = True
|
||||
cursor += 1
|
||||
continue
|
||||
if char == "\n":
|
||||
return cursor, specs, unknown_operator, has_list_operator
|
||||
if command.startswith("<<<", cursor):
|
||||
elif char == "#" and (cursor == start or command[cursor - 1].isspace()
|
||||
or command[cursor - 1] in ";&|()"):
|
||||
comment = True
|
||||
cursor += 1
|
||||
elif command.startswith("<<<", cursor):
|
||||
cursor += 3
|
||||
continue
|
||||
if command.startswith("<<", cursor):
|
||||
elif command.startswith("<<", cursor):
|
||||
parsed = _parse_heredoc_operator(command, cursor)
|
||||
if parsed is None:
|
||||
unknown_operator = True
|
||||
cursor += 2
|
||||
continue
|
||||
cursor, delimiter, strip_tabs, quoted = parsed
|
||||
specs.append((delimiter, strip_tabs, quoted))
|
||||
continue
|
||||
if char in ";|&":
|
||||
has_list_operator = True
|
||||
cursor += 1
|
||||
|
||||
return len(command), specs, unknown_operator, has_list_operator
|
||||
else:
|
||||
cursor, delimiter, strip_tabs, quoted = parsed
|
||||
specs.append((delimiter, strip_tabs, quoted))
|
||||
else:
|
||||
has_list_operator = has_list_operator or char in ";|&"
|
||||
cursor += 1
|
||||
return cursor, specs, unknown_operator, has_list_operator
|
||||
|
||||
|
||||
def _find_heredoc_close(
|
||||
@@ -200,14 +161,12 @@ def _find_heredoc_close(
|
||||
|
||||
def strip_inert_heredoc_bodies(command: str) -> str:
|
||||
"""Mask heredoc bodies that are provably inert data (see module docstring)."""
|
||||
# Runs on every terminal call: skip the state machine when no '<<' exists,
|
||||
# and stop scanning once past the last '<<'.
|
||||
# Runs on every terminal call: skip the state machine when no '<<' exists; stop past the last.
|
||||
if "<<" not in command:
|
||||
return command
|
||||
last_opener_index = command.rfind("<<")
|
||||
ranges: list[tuple[int, int]] = []
|
||||
command_start = 0
|
||||
|
||||
while command_start <= last_opener_index:
|
||||
command_end, specs, unknown_operator, has_list_operator = (
|
||||
_scan_heredoc_command_unit(command, command_start))
|
||||
@@ -219,9 +178,7 @@ def strip_inert_heredoc_bodies(command: str) -> str:
|
||||
command_start = command_end + 1
|
||||
continue
|
||||
if command_end >= len(command):
|
||||
# Opener with no body line: unterminated — leave visible.
|
||||
return command
|
||||
|
||||
return command # opener with no body line: unterminated — leave visible
|
||||
body_cursor = command_end + 1
|
||||
body_ranges: list[tuple[int, int]] = []
|
||||
for delimiter, strip_tabs, _quoted in specs:
|
||||
@@ -230,22 +187,16 @@ def strip_inert_heredoc_bodies(command: str) -> str:
|
||||
return command # unterminated
|
||||
body_ranges.append((body_cursor, close_end))
|
||||
body_cursor = close_end
|
||||
|
||||
if all(quoted for _delimiter, _strip_tabs, quoted in specs) and not has_list_operator:
|
||||
masked_opener = _mask_simple_quotes(command[command_start:command_end])
|
||||
nested_scope = any(m in masked_opener for m in ("$(", "`", "<(", ">("))
|
||||
if not nested_scope and _INERT_HEREDOC_CONSUMER_RE.search(masked_opener):
|
||||
if (not any(m in masked_opener for m in ("$(", "`", "<(", ">("))
|
||||
and _INERT_HEREDOC_CONSUMER_RE.search(masked_opener)):
|
||||
ranges.extend(body_ranges)
|
||||
command_start = body_cursor
|
||||
|
||||
if not ranges:
|
||||
return command
|
||||
# Single-pass rebuild: ranges are sorted and non-overlapping.
|
||||
# Single-pass rebuild (ranges are sorted and non-overlapping), bodies -> their newlines only.
|
||||
parts: list[str] = []
|
||||
previous = 0
|
||||
for start, end in ranges:
|
||||
parts.append(command[previous:start])
|
||||
parts.append("\n" * command.count("\n", start, end))
|
||||
parts += [command[previous:start], "\n" * command.count("\n", start, end)]
|
||||
previous = end
|
||||
parts.append(command[previous:])
|
||||
return "".join(parts)
|
||||
return "".join(parts) + command[previous:]
|
||||
|
||||
Reference in New Issue
Block a user