From f88c3285021dbf526bb490f48b6b8ff5d6acd08c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Thu, 3 Sep 2026 01:41:16 -0700 Subject: [PATCH] =?UTF-8?q?refactor(tools):=20wave-2=20group=20M=20?= =?UTF-8?q?=E2=80=94=20read=5Fextract/patch=5Fparser/shell=5Fheredoc/self?= =?UTF-8?q?=5Frepo=5Fguard/schema=5Fsanitizer=20-12%=20LOC=20(68%=20code)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - patch_parser: shared _fail/_written/_unified_diff apply helpers, single PatchResult build, flattened apply loop, merged inert/no-op hunk skips, overlay _remove, dropped never-set PatchOperation.content - self_repo_guard: conditional git-mutation predicates as a dispatch table over one _has_flag, unified git global-option/alias parse, suppress() for resolve/alias/shlex, folded scope and heredoc scanners - shell_heredoc: unit scanner and operator parser as single if/elif ladders, unified quote masking - read_extract: groupby gap runs, single try for anydoc, islice/suppress xlsx loops, folded notebook/docx render loops, _zip_xml optional -> empty element - schema_sanitizer: folded rename/type-array/recovery-strip loops, _dict_nodes/_carry_union_meta - docstrings/comments compacted by hand (every WHY kept) Verified: tool-schema dump byte-identical (SCHEMA-OK); golden corpus over pure functions and every registry schema + edge cases byte-identical vs 113f046 base (GOLDEN-OK); 95 test files 2517 passed (2 pre-existing reds in test_web_tools_config also fail on base). --- tools/patch_parser.py | 292 +++++++++++++++-------------------- tools/read_extract.py | 317 +++++++++++++++----------------------- tools/schema_sanitizer.py | 238 ++++++++++++---------------- tools/self_repo_guard.py | 276 ++++++++++++++------------------- tools/shell_heredoc.py | 173 ++++++++------------- 5 files changed, 515 insertions(+), 781 deletions(-) diff --git a/tools/patch_parser.py b/tools/patch_parser.py index 942941c6a1..74ce9fe331 100644 --- a/tools/patch_parser.py +++ b/tools/patch_parser.py @@ -1,14 +1,9 @@ -#!/usr/bin/env python3 -"""V4A patch format parser and applier (format used by codex, cline, etc.). - - *** Begin Patch / *** End Patch wrap the operations: - *** Update File: p.py then hunks: ``@@ hint @@``, `` ctx``, ``-old``, ``+new`` - *** Add File: n.py then ``+`` lines; *** Delete File: o.py; *** Move File: a -> b - - operations, error = parse_v4a_patch(patch_content) - result = apply_v4a_operations(operations, file_ops) -""" +"""V4A patch parser/applier (codex, cline). ``*** Begin Patch``/``*** End Patch`` wrap ops: +``*** Update File: p`` + hunks (``@@ hint @@``, `` ctx``, ``-old``, ``+new``); ``*** Add File: n`` ++ ``+`` lines; ``*** Delete File: o``; ``*** Move File: a -> b``. Entry points: +``parse_v4a_patch(text) -> (ops, error)`` and ``apply_v4a_operations(ops, file_ops)``.""" +import contextlib import difflib import inspect import re @@ -42,7 +37,6 @@ class PatchOperation: file_path: str new_path: Optional[str] = None # MOVE only hunks: List[Hunk] = field(default_factory=list) - content: Optional[str] = None # ADD only # Markers must occupy the whole line at column 0 so content lines that merely @@ -58,12 +52,9 @@ _HINT_RE = re.compile(r'@@\s*(.+?)\s*@@') def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[str]]: - """Parse a V4A patch -> ``(operations, None)`` (``[]`` for an empty patch is not an - error) or ``([], "Parse error: ...")`` for malformed operations.""" - # Tolerate CRLF bodies: a stray ``\r`` would otherwise end up in every - # HunkLine.content and defeat the anchored Begin/End markers. + """-> ``(operations, None)`` (empty patch = ``[]``, no error) or ``([], "Parse error: …")``.""" + # Tolerate CRLF: a stray ``\r`` would land in every HunkLine.content and defeat the markers. lines = [ln[:-1] if ln.endswith('\r') else ln for ln in patch_content.split('\n')] - start_idx = -1 # parse from the top when no Begin marker is present end_idx = len(lines) for i, line in enumerate(lines): @@ -72,7 +63,6 @@ def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[ elif _END_MARKER.match(line): end_idx = i break - operations: List[PatchOperation] = [] current_op: Optional[PatchOperation] = None current_hunk: Optional[Hunk] = None @@ -87,8 +77,7 @@ def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[ operations.append(current_op) for line in lines[start_idx + 1:end_idx]: - op_match = next( - ((kind, m) for kind, rx in _OP_MARKERS if (m := rx.match(line))), None) + op_match = next(((kind, m) for kind, rx in _OP_MARKERS if (m := rx.match(line))), None) if op_match: kind, m = op_match _flush() @@ -96,8 +85,8 @@ def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[ operation=kind, file_path=m.group(1).strip(), new_path=m.group(2).strip() if kind is OperationType.MOVE else None) - # UPDATE hunks start lazily (at '@@' or the first hunk line); ADD - # collects all '+' lines into one hunk; DELETE/MOVE are complete. + # UPDATE hunks start lazily ('@@' or first hunk line); ADD collects all '+' lines + # into one hunk; DELETE/MOVE are complete. current_hunk = Hunk() if kind is OperationType.ADD else None if kind in (OperationType.DELETE, OperationType.MOVE): operations.append(current_op) @@ -115,18 +104,16 @@ def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[ elif line[0] != '\\': # "\ No newline at end of file" marker is skipped current_hunk.lines.append(HunkLine(' ', line)) # implicit context line _flush() - parse_errors: List[str] = [] for op in operations: if not op.file_path: parse_errors.append("Operation with empty file path") - if op.operation == OperationType.UPDATE and not op.hunks: + if op.operation is OperationType.UPDATE and not op.hunks: parse_errors.append(f"UPDATE {op.file_path!r}: no hunks found") - if op.operation == OperationType.MOVE and not op.new_path: - parse_errors.append(f"MOVE {op.file_path!r}: missing destination path (expected 'src -> dst')") - if parse_errors: - return [], "Parse error: " + "; ".join(parse_errors) - return operations, None + if op.operation is OperationType.MOVE and not op.new_path: + parse_errors.append( + f"MOVE {op.file_path!r}: missing destination path (expected 'src -> dst')") + return ([], "Parse error: " + "; ".join(parse_errors)) if parse_errors else (operations, None) def _count_occurrences(text: str, pattern: str) -> int: @@ -136,66 +123,57 @@ def _count_occurrences(text: str, pattern: str) -> int: def _split_hunk(hunk: Hunk) -> Tuple[List[str], List[str]]: """``(search_lines, replace_lines)``: context+removed vs context+added.""" - search = [l.content for l in hunk.lines if l.prefix in {' ', '-'}] - replace = [l.content for l in hunk.lines if l.prefix in {' ', '+'}] - return search, replace + return ([l.content for l in hunk.lines if l.prefix != '+'], + [l.content for l in hunk.lines if l.prefix != '-']) def _no_match_hint(error: Optional[str], search_pattern: str, content: str) -> str: """Best-effort 'Did you mean...' suffix; never lets a hint failure mask the real error.""" - try: + with contextlib.suppress(Exception): from tools.fuzzy_match import format_no_match_hint return format_no_match_hint(error, 0, search_pattern, content) - except Exception: - return "" + return "" def _hint_ambiguity(content: str, hint: str, tail: str = "") -> Tuple[int, str]: """(occurrences, error) for an addition-only hunk's context hint; error is '' when unique.""" - occurrences = _count_occurrences(content, hint) - if occurrences > 1: - return occurrences, (f"context hint '{hint}' is ambiguous " - f"({occurrences} occurrences){tail}") - return occurrences, "" + n = _count_occurrences(content, hint) + return n, f"context hint '{hint}' is ambiguous ({n} occurrences){tail}" if n > 1 else "" def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> List[str]: - """Dry-run every operation; return error strings (empty list = safe to apply). UPDATE - hunks are simulated in order so later hunks see post-earlier-hunk content, as apply will.""" + """Dry-run every operation -> error strings (empty = safe). UPDATE hunks are simulated in + order so later hunks see post-earlier-hunk content, exactly as apply will.""" from tools.fuzzy_match import fuzzy_find_and_replace, is_already_applied - errors: List[str] = [] real_change_count = 0 - - # Virtual overlay so inter-op state validates (e.g. a MOVE creating the destination - # a later UPDATE targets): path -> content from an earlier op; paths MOVE/DELETE removed. + # Overlay so inter-op state validates (a MOVE creating the path a later UPDATE targets). pending_content: dict = {} removed_paths: set = set() - def _read(path: str): - if path in removed_paths and path not in pending_content: - return None, "file not found" + def _read(path: str) -> Tuple[Optional[str], Optional[str]]: if path in pending_content: return pending_content[path], None + if path in removed_paths: + return None, "file not found" r = file_ops.read_file_raw(path) return (None, r.error) if r.error else (r.content, None) def _validate_update(op: PatchOperation) -> None: nonlocal real_change_count - content, read_err = _read(op.file_path) + simulated, read_err = _read(op.file_path) if read_err: errors.append(f"{op.file_path}: {read_err}") return - simulated = content for hunk_index, hunk in enumerate(op.hunks, start=1): search_lines, replace_lines = _split_hunk(hunk) - if not any(l.prefix in '-+' for l in hunk.lines): - # Inert anchor hunk (context only) — models emit these - # between real changes; ignore without failing the patch. + if search_lines == replace_lines: + # Context-only anchor hunks (models emit these between changes) are inert; identical + # -/+ lines are skipped by apply as a no-op — neither may fail validation. + real_change_count += any(l.prefix in '-+' for l in hunk.lines) continue real_change_count += 1 - if not search_lines: - # Addition-only hunk: the context hint must be unique. + if not search_lines: # addition-only: the context hint must be unique if hunk.context_hint: occurrences, ambiguous = _hint_ambiguity(simulated, hunk.context_hint) if occurrences == 0: @@ -204,19 +182,13 @@ def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> Lis elif ambiguous: errors.append(f"{op.file_path}: addition-only hunk {ambiguous}") continue - search_pattern = '\n'.join(search_lines) - replacement = '\n'.join(replace_lines) - if search_lines == replace_lines: - # Identical -/+ lines: apply skips it as a no-op, so - # validation must not reject it with the identical-strings error. - continue + search_pattern, replacement = '\n'.join(search_lines), '\n'.join(replace_lines) new_simulated, count, _strategy, match_error = fuzzy_find_and_replace( simulated, search_pattern, replacement, replace_all=False) if count: simulated = new_simulated elif not is_already_applied(simulated or "", search_pattern, replacement): - # Already-applied hunks (edit landed in a prior call) are no-ops so - # multi-hunk patches don't fail wholesale; apply performs the same skip. + # Already-applied hunks are no-ops (apply performs the same skip). label = f"'{hunk.context_hint}'" if hunk.context_hint else "(no hint)" errors.append( f"{op.file_path}: hunk {hunk_index} {label} not found" @@ -224,18 +196,20 @@ def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> Lis + _no_match_hint(match_error, search_pattern, simulated)) pending_content[op.file_path] = simulated + def _remove(path: str) -> None: + removed_paths.add(path) + pending_content.pop(path, None) + for op in operations: if op.operation == OperationType.UPDATE: _validate_update(op) continue real_change_count += 1 if op.operation == OperationType.DELETE: - _content, read_err = _read(op.file_path) - if read_err: + if _read(op.file_path)[1]: errors.append(f"{op.file_path}: file not found for deletion") else: - removed_paths.add(op.file_path) - pending_content.pop(op.file_path, None) + _remove(op.file_path) elif op.operation == OperationType.MOVE: if not op.new_path: errors.append(f"{op.file_path}: MOVE operation missing destination path") @@ -243,16 +217,12 @@ def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> Lis src_content, src_err = _read(op.file_path) if src_err: errors.append(f"{op.file_path}: source file not found for move") - _dst, dst_err = _read(op.new_path) - if not dst_err: + if not _read(op.new_path)[1]: errors.append(f"{op.new_path}: destination already exists — move would overwrite") - # Only a cleanly-validated move updates the overlay. - if not src_err and dst_err: + elif not src_err: # only a cleanly-validated move updates the overlay pending_content[op.new_path] = src_content if src_content is not None else "" - pending_content.pop(op.file_path, None) - removed_paths.add(op.file_path) - # ADD: parent directory creation handled by write_file; no pre-check needed. - + _remove(op.file_path) + # ADD: write_file creates parent directories; no pre-check needed. if not errors and real_change_count == 0: errors.append("Patch contains no changes (only context lines were provided)") return errors @@ -262,104 +232,101 @@ def _validate_operations(operations: List[PatchOperation], file_ops: Any) -> Lis ApplyResult = Tuple[bool, str, Optional[str], Optional[dict]] +def _fail(error: str) -> ApplyResult: + return False, error, None, None + + +def _written(result: Any, diff: str) -> ApplyResult: + """Outcome of a write: its error, else success with LSP/lint propagated from the WriteResult.""" + if result.error: + return _fail(result.error) + return True, diff, getattr(result, "lsp_diagnostics", None), getattr(result, "lint", None) + + +def _unified_diff(path: str, old: str, new: Optional[str]) -> str: + """Unified diff ``a/path`` -> ``b/path`` (``new=None`` = deletion, ``/dev/null``).""" + return ''.join(difflib.unified_diff( + old.splitlines(keepends=True), [] if new is None else new.splitlines(keepends=True), + fromfile=f"a/{path}", tofile="/dev/null" if new is None else f"b/{path}")) + + def apply_v4a_operations(operations: List[PatchOperation], file_ops: Any) -> 'PatchResult': - """Validate all operations, then apply them (two-phase, atomic on validation failure). - A phase-2 failure (validate/apply race) is reported with a ``git diff`` note since state - may be inconsistent. ``file_ops`` needs read_file_raw/write_file/delete_file/move_file.""" + """Two-phase: validate everything, then apply (atomic on validation failure). A phase-2 + failure (validate/apply race) carries a ``git diff`` note since state may be inconsistent. + ``file_ops`` needs read_file_raw/write_file/delete_file/move_file.""" from tools.file_operations import PatchResult # avoid circular import - validation_errors = _validate_operations(operations, file_ops) - if validation_errors: + def _bullets(errs: List[str]) -> str: + return "\n".join(f" • {e}" for e in errs) + + if errors := _validate_operations(operations, file_ops): return PatchResult( success=False, - error="Patch validation failed (no files were modified):\n" - + "\n".join(f" • {e}" for e in validation_errors)) - + error="Patch validation failed (no files were modified):\n" + _bullets(errors)) files: Dict[str, List[str]] = {"created": [], "deleted": [], "modified": []} all_diffs: List[str] = [] - # V4A bypasses the WriteResult/PatchResult plumbing that write_file uses, - # so LSP diagnostics and lint must be propagated explicitly per file. + # V4A bypasses write_file's WriteResult plumbing: LSP diagnostics and lint propagate per file. lsp_blocks: List[str] = [] - errors: List[str] = [] lint_results: Dict[str, dict] = {} - for op in operations: + handler, verb, bucket = _APPLY_DISPATCH[op.operation] try: - handler, verb, bucket = _APPLY_DISPATCH[op.operation] ok, payload, lsp, lint = handler(op, file_ops) - if not ok: - errors.append(f"Failed to {verb} {op.file_path}: {payload}") - continue - label = op.file_path - if op.operation is OperationType.MOVE: - label = f"{op.file_path} -> {op.new_path}" - files[bucket].append(label) - all_diffs.append(payload) - if lsp: - lsp_blocks.append(lsp) - if lint: - lint_results[op.file_path] = lint except Exception as e: - errors.append(f"Error processing {op.file_path}: {str(e)}") - - # Each LSP block carries its own header, so plain - # concatenation keeps per-file attribution. - result_kwargs = dict( + ok, payload = None, str(e) + if not ok: + prefix = f"Failed to {verb}" if ok is False else "Error processing" + errors.append(f"{prefix} {op.file_path}: {payload}") + continue + is_move = op.operation is OperationType.MOVE + files[bucket].append(f"{op.file_path} -> {op.new_path}" if is_move else op.file_path) + all_diffs.append(payload) + if lsp: + lsp_blocks.append(lsp) + if lint: + lint_results[op.file_path] = lint + # Each LSP block carries its own header; joining keeps attribution. + return PatchResult( + success=not errors, + error=("Apply phase failed (state may be inconsistent — run `git diff` to assess):\n" + + _bullets(errors)) if errors else None, diff='\n'.join(all_diffs), files_modified=files["modified"], files_created=files["created"], files_deleted=files["deleted"], - lint=lint_results if lint_results else None, - lsp_diagnostics="\n\n".join(lsp_blocks) if lsp_blocks else None) - if errors: - return PatchResult( - success=False, - error="Apply phase failed (state may be inconsistent — run `git diff` to assess):\n" - + "\n".join(f" • {e}" for e in errors), - **result_kwargs) - return PatchResult(success=True, **result_kwargs) + lint=lint_results or None, lsp_diagnostics="\n\n".join(lsp_blocks) or None) def _write_file_accepts_pre_content(file_ops: Any) -> bool: - """True when ``file_ops.write_file`` accepts ``pre_content``. Decided from the signature, - not by catching TypeError around the call, so a TypeError raised *inside* a capable - write_file propagates instead of triggering a duplicate write.""" - try: + """Whether ``file_ops.write_file`` accepts ``pre_content`` — read from the signature, not by + catching TypeError around the call, so a TypeError raised *inside* it can't double-write.""" + with contextlib.suppress(TypeError, ValueError): params = inspect.signature(file_ops.write_file).parameters - except (TypeError, ValueError): - return False - return "pre_content" in params or any( - p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()) + return "pre_content" in params or any( + p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()) + return False def _apply_add(op: PatchOperation, file_ops: Any) -> ApplyResult: """Create a file from the hunks' '+' lines.""" content_lines = [line.content for hunk in op.hunks for line in hunk.lines if line.prefix == '+'] result = file_ops.write_file(op.file_path, '\n'.join(content_lines)) - if result.error: - return False, result.error, None, None diff = f"--- /dev/null\n+++ b/{op.file_path}\n" + '\n'.join(f"+{line}" for line in content_lines) - return True, diff, getattr(result, "lsp_diagnostics", None), getattr(result, "lint", None) + return _written(result, diff) def _apply_delete(op: PatchOperation, file_ops: Any) -> ApplyResult: """Delete a file, producing a real unified diff of the removed content.""" - # Validation already confirmed existence; the re-read guards against races. - read_result = file_ops.read_file_raw(op.file_path) + read_result = file_ops.read_file_raw(op.file_path) # re-read guards validate/apply races if read_result.error: - return False, f"Cannot delete {op.file_path}: file not found", None, None + return _fail(f"Cannot delete {op.file_path}: file not found") result = file_ops.delete_file(op.file_path) - if result.error: - return False, result.error, None, None - diff = ''.join(difflib.unified_diff( - read_result.content.splitlines(keepends=True), [], - fromfile=f"a/{op.file_path}", tofile="/dev/null")) - return True, diff or f"# Deleted: {op.file_path}", None, None + diff = _unified_diff(op.file_path, read_result.content, None) or f"# Deleted: {op.file_path}" + return _fail(result.error) if result.error else (True, diff, None, None) def _apply_move(op: PatchOperation, file_ops: Any) -> ApplyResult: result = file_ops.move_file(op.file_path, op.new_path) - if result.error: - return False, result.error, None, None - return True, f"# Moved: {op.file_path} -> {op.new_path}", None, None + return _fail(result.error) if result.error else ( + True, f"# Moved: {op.file_path} -> {op.new_path}", None, None) def _insert_addition_only(new_content: str, hunk: Hunk, insert_text: str) -> Tuple[Optional[str], Optional[str]]: @@ -371,71 +338,54 @@ def _insert_addition_only(new_content: str, hunk: Hunk, insert_text: str) -> Tup return None, f"Addition-only hunk: {ambiguous}" if occurrences == 1: eol = new_content.find('\n', new_content.find(hunk.context_hint)) - if eol != -1: - return new_content[:eol + 1] + insert_text + '\n' + new_content[eol + 1:], None - return new_content + '\n' + insert_text, None - # Hint not found — append at end as a safe fallback. + if eol == -1: + return new_content + '\n' + insert_text, None + return new_content[:eol + 1] + insert_text + '\n' + new_content[eol + 1:], None + # No hint / hint not found — append at end as a safe fallback. return new_content.rstrip('\n') + '\n' + insert_text + '\n', None def _apply_update(op: PatchOperation, file_ops: Any) -> ApplyResult: """Apply each hunk via fuzzy replace, then write once.""" from tools.fuzzy_match import fuzzy_find_and_replace, is_already_applied - - # Raw read: no line-number prefixes or per-line truncation. - read_result = file_ops.read_file_raw(op.file_path) + read_result = file_ops.read_file_raw(op.file_path) # raw: no line numbers / truncation if read_result.error: - return False, f"Cannot read file: {read_result.error}", None, None - current_content = read_result.content - new_content = current_content - + return _fail(f"Cannot read file: {read_result.error}") + current_content = new_content = read_result.content for hunk in op.hunks: search_lines, replace_lines = _split_hunk(hunk) if search_lines and search_lines == replace_lines: continue + search_pattern, replacement = '\n'.join(search_lines), '\n'.join(replace_lines) if not search_lines: - new_content, err = _insert_addition_only(new_content, hunk, '\n'.join(replace_lines)) + new_content, err = _insert_addition_only(new_content, hunk, replacement) if err: - return False, err, None, None + return _fail(err) continue - - search_pattern = '\n'.join(search_lines) - replacement = '\n'.join(replace_lines) new_content, count, _strategy, error = fuzzy_find_and_replace( new_content, search_pattern, replacement, replace_all=False) if not (error and count == 0): continue - # Retry inside a window around the context hint, if any. hint_pos = new_content.find(hunk.context_hint) if hunk.context_hint else -1 if hint_pos != -1: window_start = max(0, hint_pos - 500) window_end = min(len(new_content), hint_pos + 2000) window_new, count, _strategy, error = fuzzy_find_and_replace( - new_content[window_start:window_end], search_pattern, replacement, replace_all=False - ) + new_content[window_start:window_end], search_pattern, replacement, replace_all=False) if count > 0: new_content = new_content[:window_start] + window_new + new_content[window_end:] error = None if error: - # Mirror the validation-phase already-applied skip, or the two - # phases disagree and the whole patch fails here. + # Mirror validation's already-applied skip, else the two phases disagree and fail here. if is_already_applied(new_content, search_pattern, replacement): continue - return False, f"Could not apply hunk: {error}" + _no_match_hint(error, search_pattern, new_content), None, None - + hint = _no_match_hint(error, search_pattern, new_content) + return _fail(f"Could not apply hunk: {error}" + hint) # Pass pre_content to skip a redundant re-read inside write_file when supported. - if _write_file_accepts_pre_content(file_ops): - write_result = file_ops.write_file(op.file_path, new_content, pre_content=current_content) - else: - write_result = file_ops.write_file(op.file_path, new_content) - if write_result.error: - return False, write_result.error, None, None - - diff = ''.join(difflib.unified_diff( - current_content.splitlines(keepends=True), new_content.splitlines(keepends=True), - fromfile=f"a/{op.file_path}", tofile=f"b/{op.file_path}")) - return True, diff, getattr(write_result, "lsp_diagnostics", None), getattr(write_result, "lint", None) + extra = {"pre_content": current_content} if _write_file_accepts_pre_content(file_ops) else {} + write_result = file_ops.write_file(op.file_path, new_content, **extra) + return _written(write_result, _unified_diff(op.file_path, current_content, new_content)) # operation -> (handler, verb for error text, files_* bucket) diff --git a/tools/read_extract.py b/tools/read_extract.py index 23a4299af2..a96e5431b3 100644 --- a/tools/read_extract.py +++ b/tools/read_extract.py @@ -1,16 +1,14 @@ -"""Stdlib document-to-text extraction for ``read_file``. - -Jupyter, DOCX and XLSX need no dependencies. The optional ``firecrawl-anydoc`` package -(imports as ``anydoc``) widens coverage to legacy Office, OpenDocument, RTF, EPUB and PDF. -The stdlib extractors stay authoritative for their three formats so behavior is identical -with or without anydoc. Malformed documents raise :class:`ExtractionError`; callers then -fall back to normal text/binary handling. -""" +"""Document-to-text extraction for ``read_file``: stdlib Jupyter/DOCX/XLSX (always +authoritative for those three), plus legacy Office/OpenDocument/RTF/EPUB/PDF when the +optional ``firecrawl-anydoc`` package (imports as ``anydoc``) is installed. Malformed +documents raise :class:`ExtractionError`; callers fall back to text/binary handling.""" from __future__ import annotations import contextlib +import functools import importlib +import itertools import json import os import posixpath @@ -29,12 +27,10 @@ __all__ = ["EXTRACTABLE_EXTENSIONS", "ExtractionError", "extract_document_bytes" "extract_document_text", "is_extractable_document"] EXTRACTABLE_EXTENSIONS = frozenset({".ipynb", ".docx", ".xlsx"}) -# Formats handled only when the optional anydoc converter is installed. ANYDOC_EXTENSIONS = frozenset({ ".doc", ".docm", ".ppt", ".pps", ".pot", ".pptx", ".pptm", ".ppsx", ".ppsm", ".xls", ".xlsm", ".xlsb", ".odt", ".ods", ".odp", ".rtf", ".epub", ".pdf"}) -# anydoc loads whole files with no streaming and the read_file char budget only applies -# after conversion — cap the input size. +# anydoc loads whole files (no streaming); read_file's char budget applies only post-conversion. MAX_ANYDOC_BYTES = 50 * 1024 * 1024 MAX_DOCUMENT_BYTES = 50 * 1024 * 1024 _MAX_XLSX_ROWS_PER_SHEET = 5000 @@ -59,15 +55,15 @@ def _extension(path: str) -> str: _ANYDOC_UNSET = object() _anydoc_module: Any = _ANYDOC_UNSET _anydoc_lock = threading.Lock() -# After a failed load, wait this long before retrying: the attempt can shell out -# to pip, so retrying every call would hammer the network where install can't succeed. +# Cooldown after a failed load: the attempt can shell out to pip, so retrying every call would +# hammer the network where install can't succeed. ANYDOC_RETRY_SECONDS = 300.0 _anydoc_failed_at: Optional[float] = None def _anydoc() -> Optional[Any]: - """Lazily import the optional anydoc converter; None when unavailable. A failed load is - retried after ANYDOC_RETRY_SECONDS so one transient pip/network blip does not stick.""" + """Lazily import the optional anydoc converter (None when unavailable; failures retried after + ANYDOC_RETRY_SECONDS so one transient pip/network blip does not stick).""" global _anydoc_module, _anydoc_failed_at if _anydoc_module is not _ANYDOC_UNSET: return _anydoc_module @@ -79,8 +75,7 @@ def _anydoc() -> Optional[Any]: return None try: from tools.lazy_deps import ensure as _lazy_ensure - # prompt=False: read_file must never block on an install prompt. - _lazy_ensure("tool.doc_extract", prompt=False) + _lazy_ensure("tool.doc_extract", prompt=False) # read_file must never block on a prompt _anydoc_module = importlib.import_module("anydoc") except Exception: # install failure, ImportError or a broken native binding _anydoc_failed_at = time.monotonic() @@ -101,23 +96,19 @@ def _check_size(size: int, limit: int) -> None: @contextlib.contextmanager def _temp_copy(data: bytes, suffix: str) -> Iterator[str]: """Materialize backend bytes in a private host temp file; removed even when parsing fails.""" - temp_path = "" + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh: + fh.write(data) try: - with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh: - fh.write(data) - temp_path = fh.name - yield temp_path + yield fh.name finally: - if temp_path: - with contextlib.suppress(OSError): - os.unlink(temp_path) + with contextlib.suppress(OSError): + os.unlink(fh.name) def extract_document_text(path: str) -> str: ext = _extension(path) - extractor = _STDLIB_EXTRACTORS.get(ext) - if extractor is not None: - return extractor(path) + if ext in _STDLIB_EXTRACTORS: + return _STDLIB_EXTRACTORS[ext](path) if ext in ANYDOC_EXTENSIONS: return _extract_anydoc(path) raise ExtractionError(f"Unsupported document type: {path!r}") @@ -131,14 +122,12 @@ def extract_document_bytes(data: bytes, path: str) -> str: return _extract_anydoc_bytes(data, path) if ext not in EXTRACTABLE_EXTENSIONS: raise ExtractionError(f"Unsupported document type: {path!r}") - # The stdlib extractors are path-oriented. - with _temp_copy(data, ext) as temp_path: - return extract_document_text(temp_path) + with _temp_copy(data, ext) as temp_path: # the stdlib extractors are path-oriented + return _STDLIB_EXTRACTORS[ext](temp_path) def _anydoc_missing_error(path: str) -> str: - """Teaching error for anydoc-gated formats (deliberately absent from the schema so only - sessions that hit one pay for it).""" + """Teaching text for anydoc-gated formats (not in the schema: only sessions hitting one pay).""" return ( f"Cannot convert {path!r}: this format needs the optional anydoc " "converter, which is not installed (install blocked or first " @@ -149,10 +138,10 @@ def _anydoc_missing_error(path: str) -> str: def _hosted_ocr_config() -> tuple: - """Resolve hosted-OCR settings: (enabled, api_key, api_url). Never raises; no network. - Maintainer decision: the ONLY route is a direct ``FIRECRAWL_API_KEY`` (anydoc defaults the - api_url); the Nous gateway is NOT used — its Parse proxy live-probed broken (revisit when - it grows Parse support). ``file_tools.hosted_ocr: false`` disables even with a key.""" + """(enabled, api_key, api_url); never raises, no network. Maintainer decision: the ONLY route + is a direct ``FIRECRAWL_API_KEY`` (anydoc defaults api_url); the Nous gateway's Parse proxy + live-probed broken, so it is NOT used. ``file_tools.hosted_ocr: false`` disables even with a + key.""" api_key = os.environ.get("FIRECRAWL_API_KEY") or None enabled = api_key is not None with contextlib.suppress(Exception): @@ -164,35 +153,30 @@ def _hosted_ocr_config() -> tuple: def hosted_ocr_available() -> bool: - """Public probe for read_file's schema line; a key failing at conversion time lands in - the NEEDS-OCR warning instead.""" + """Probe for read_file's schema line; a key failing at conversion time surfaces in NEEDS-OCR.""" return _hosted_ocr_config()[0] def _needs_ocr_warning(path: str, pages, hosted_error: str = "") -> str: - """Result text when anydoc raises NeedsOcrError and hosted OCR is off/failed. Hints at - CHECKING for an OCR skill (never names one) and never advertises the hosted_ocr knob.""" + """NeedsOcrError result when hosted OCR is off/failed; hints at CHECKING for an OCR skill + (never names one) and never advertises the hosted_ocr knob.""" page_list = ", ".join(str(p) for p in pages) if pages else "unknown" - msg = ( + hosted = f"Hosted OCR was attempted and failed ({hosted_error}). " if hosted_error else "" + return ( f"[NEEDS OCR: pages {page_list} of this PDF are scanned images " - "with no text layer — their content is MISSING below. ") - if hosted_error: - msg += f"Hosted OCR was attempted and failed ({hosted_error}). " - msg += ( + f"with no text layer — their content is MISSING below. {hosted}" "If the missing pages matter: render just those pages with " f"`pdftoppm -jpeg -r 150 -f -l '{path}' /tmp/page` " "and inspect via vision_analyze, or check whether an OCR skill is " - "available (skills_list).") - return msg + "]\n" + "available (skills_list).]\n") def _finalize_anydoc_text(text: Any, path: str, pdf_note: Callable[[], str]) -> str: - """Normalize converter output and, for PDFs, PREPEND the coverage note (read_file - paginates: a footer may never be fetched). Covers PARTIAL gaps without NeedsOcrError.""" + """Normalize converter output; PDFs get the coverage note PREPENDED (read_file paginates, so a + footer may never be fetched) — this covers PARTIAL scan gaps that raise no NeedsOcrError.""" if not isinstance(text, str) or not text.strip(): raise ExtractionError("Document contains no extractable text") - note = pdf_note() if Path(path).suffix.lower() == ".pdf" else "" - return (note or "") + text.rstrip("\n") + "\n" + return (pdf_note() if Path(path).suffix.lower() == ".pdf" else "") + text.rstrip("\n") + "\n" def _ocr_scanned_pdf(mod: Any, path: str, exc: BaseException) -> str: @@ -206,40 +190,33 @@ def _ocr_scanned_pdf(mod: Any, path: str, exc: BaseException) -> str: return mod.to_markdown(path, ocr="hosted", **extra).rstrip("\n") + "\n" except Exception as hosted_exc: # noqa: BLE001 hosted_error = f"{type(hosted_exc).__name__}: {hosted_exc}" - # No route / disabled / hosted failed: whole doc is scans — the warning IS the result. - return _needs_ocr_warning(path, pages, hosted_error) - - -def _require_anydoc(path: str) -> Any: - mod = _anydoc() - if mod is None: - raise ExtractionError(_anydoc_missing_error(path)) - return mod + return _needs_ocr_warning(path, pages, hosted_error) # whole doc is scans: the warning IS it def _extract_anydoc(path: str) -> str: - mod = _require_anydoc(path) - try: - size = os.path.getsize(path) - except OSError as exc: - raise ExtractionError(str(exc)) from exc - _check_size(size, MAX_ANYDOC_BYTES) + mod = _anydoc() + if mod is None: + raise ExtractionError(_anydoc_missing_error(path)) try: + _check_size(os.path.getsize(path), MAX_ANYDOC_BYTES) text = mod.to_markdown(path) + except ExtractionError: + raise except OSError as exc: raise ExtractionError(str(exc)) from exc except Exception as exc: needs_ocr = getattr(mod, "NeedsOcrError", None) if needs_ocr is not None and isinstance(exc, needs_ocr): return _ocr_scanned_pdf(mod, path, exc) - # anydoc raises one ConvertError subclass per failure mode (Unsupported, Malformed, - # Encrypted, ResourceLimit, MissingPart); all mean "no meaningful text". + # Any ConvertError subclass (Unsupported/Malformed/Encrypted/...) = "no meaningful text". raise ExtractionError(f"{type(exc).__name__}: {exc}") from exc return _finalize_anydoc_text(text, path, lambda: _pdf_coverage_note(path)) def _extract_anydoc_bytes(data: bytes, path: str) -> str: - mod = _require_anydoc(path) + mod = _anydoc() + if mod is None: + raise ExtractionError(_anydoc_missing_error(path)) _check_size(len(data), MAX_ANYDOC_BYTES) try: text = mod.to_markdown_bytes(data) @@ -248,17 +225,13 @@ def _extract_anydoc_bytes(data: bytes, path: str) -> str: return _finalize_anydoc_text(text, path, lambda: _pdf_coverage_note_from_bytes(data, path)) -# ── Scanned-PDF coverage detection: text-layer extractors return nothing for scanned -# pages, so a mostly-scanned PDF converts "successfully" into headers with empty bodies — -# silent data loss. Count per-page text via pdftotext (form-feed separated) and warn. +# ── Scanned-PDF coverage: text-layer extractors return nothing for scanned pages, so a mostly +# scanned PDF converts "successfully" into silent data loss. Count per-page text via pdftotext. PDF_EMPTY_PAGE_CHARS = 20 # fewer extracted chars than this = empty page # Warn when empty pages reach both MIN_EMPTY and MIN_RATIO, or ABSOLUTE_EMPTY alone. -PDF_COVERAGE_MIN_EMPTY = 2 -PDF_COVERAGE_MIN_RATIO = 0.2 -PDF_COVERAGE_ABSOLUTE_EMPTY = 10 +PDF_COVERAGE_MIN_EMPTY, PDF_COVERAGE_MIN_RATIO, PDF_COVERAGE_ABSOLUTE_EMPTY = 2, 0.2, 10 PDF_PAGE_SCAN_TIMEOUT = 20.0 -# Cap the per-gap breakdown so alternating text/scan pages can't balloon the warning. -PDF_GAP_MAP_MAX_ENTRIES = 20 +PDF_GAP_MAP_MAX_ENTRIES = 20 # cap so alternating text/scan pages can't balloon the warning _GAP_CONTEXT_CHARS = 60 @@ -279,14 +252,10 @@ def _pdf_page_texts(path: str) -> Optional[list[str]]: def _gap_map(counts: list[int], texts: list[str], empty: list[int]) -> str: - """Per-gap breakdown, each empty range labeled with the last text seen before - it (usually a section header), so the agent can pick WHICH gaps to OCR.""" - ranges: list[list[int]] = [] # sorted 1-based page numbers -> [start, end] runs - for p in empty: - if ranges and p == ranges[-1][1] + 1: - ranges[-1][1] = p - else: - ranges.append([p, p]) + """Per-gap breakdown labeled with the text before each gap, so the agent picks which to OCR.""" + # Sorted 1-based page numbers -> (start, end) runs; consecutive pages share ``page - index``. + runs = [list(g) for _k, g in itertools.groupby(enumerate(empty), lambda e: e[1] - e[0])] + ranges = [(run[0][1], run[-1][1]) for run in runs] lines: list[str] = [] for a, b in ranges[:PDF_GAP_MAP_MAX_ENTRIES]: label = "" @@ -305,17 +274,17 @@ def _gap_map(counts: list[int], texts: list[str], empty: list[int]) -> str: def _pdf_coverage_note(path: str, display_path: Optional[str] = None) -> str: - """Warning header when many PDF pages produced no text, else ''. ``path`` is scanned - (may be a host temp file); ``display_path`` is what the recovery command shows.""" + """Warning header when many pages yielded no text, else ''. ``display_path`` (default ``path``, + which may be a host temp file) is what the recovery command shows.""" texts = _pdf_page_texts(path) if not texts or len(texts) < 2: return "" counts = [len(page.strip()) for page in texts] empty = [i + 1 for i, n in enumerate(counts) if n < PDF_EMPTY_PAGE_CHARS] total = len(counts) - if len(empty) < PDF_COVERAGE_MIN_EMPTY or ( - len(empty) / total < PDF_COVERAGE_MIN_RATIO and len(empty) < PDF_COVERAGE_ABSOLUTE_EMPTY - ): + n_empty = len(empty) + enough = n_empty / total >= PDF_COVERAGE_MIN_RATIO or n_empty >= PDF_COVERAGE_ABSOLUTE_EMPTY + if n_empty < PDF_COVERAGE_MIN_EMPTY or not enough: return "" shown = display_path or path return ( @@ -335,13 +304,10 @@ def _pdf_coverage_note(path: str, display_path: Optional[str] = None) -> str: def _pdf_coverage_note_from_bytes(data: bytes, display_path: str) -> str: - """Coverage note for backend-transferred PDF bytes via a host temp copy (pdftotext is - path-oriented); the recovery command still names ``display_path``.""" - try: - with _temp_copy(data, ".pdf") as temp_path: - return _pdf_coverage_note(temp_path, display_path=display_path) - except OSError: - return "" + """Coverage note for backend PDF bytes via a host temp copy (pdftotext needs a path).""" + with contextlib.suppress(OSError), _temp_copy(data, ".pdf") as temp_path: + return _pdf_coverage_note(temp_path, display_path=display_path) + return "" def _joined(lines: list[str], empty_error: str) -> str: @@ -352,11 +318,10 @@ def _joined(lines: list[str], empty_error: str) -> str: def _source_text(source) -> str: - if isinstance(source, str): - return source + """Notebook source/text fields are a str or a list of str fragments.""" if isinstance(source, list): - return "".join(item for item in source if isinstance(item, str)) - return "" + source = "".join(item for item in source if isinstance(item, str)) + return source if isinstance(source, str) else "" def _human_size(n_bytes: int) -> str: @@ -366,30 +331,24 @@ def _human_size(n_bytes: int) -> str: def _base64_bytes(payload: str) -> int: """Approximate decoded size of a base64 payload (whitespace ignored).""" clean = re.sub(r"[^0-9+/=A-Za-z]", "", payload) - padding = min(2, len(clean) - len(clean.rstrip("="))) - return max(0, (len(clean) * 3) // 4 - padding) + return max(0, (len(clean) * 3) // 4 - min(2, len(clean) - len(clean.rstrip("=")))) def _clean_stream_text(text: str) -> str: """Strip ANSI escapes; keep only the final ``\\r`` frame of each line (tqdm redraws).""" from tools.ansi_strip import strip_ansi - lines = [] - for line in strip_ansi(text).replace("\r\n", "\n").split("\n"): - frames = [frame for frame in line.split("\r") if frame] - lines.append(frames[-1] if frames else "") - return "\n".join(lines) + return "\n".join(([f for f in line.split("\r") if f] or [""])[-1] + for line in strip_ansi(text).replace("\r\n", "\n").split("\n")) -# Per-output-block truncation so one runaway training log cannot flood the extraction. -_MAX_OUTPUT_CHARS = 20_000 +_MAX_OUTPUT_CHARS = 20_000 # per code cell, so one runaway training log cannot flood the extraction # nbformat v3 stores mime data flat on the output dict under these keys. _V3_MIME_KEYS = (("png", "image/png"), ("jpeg", "image/jpeg"), ("svg", "image/svg+xml"), ("html", "text/html")) def _notebook_output_text(output: Any) -> str: - """Render one notebook output as compact text: stream text, tracebacks and textual - results kept; token-heavy payloads (images, HTML, widgets) become sized placeholders. - Handles nbformat v4 and legacy v3 (``pyout``/``pyerr``) shapes.""" + """One notebook output as compact text: stream/traceback/textual results kept; token-heavy + payloads (images, HTML, widgets) become sized placeholders. Handles v4 and legacy v3 shapes.""" if not isinstance(output, dict): return "" otype = output.get("output_type") @@ -397,52 +356,41 @@ def _notebook_output_text(output: Any) -> str: body = _clean_stream_text(_source_text(output.get("text", ""))) return body if body.strip() else "" if otype in {"error", "pyerr"}: - traceback = output.get("traceback") - tb_text = "" - if isinstance(traceback, list): - tb_text = _clean_stream_text( - "\n".join(line for line in traceback if isinstance(line, str))) + tb = output.get("traceback") + tb_text = _clean_stream_text("\n".join(filter(lambda l: isinstance(l, str), tb)) + if isinstance(tb, list) else "") header = f"Error: {output.get('ename', '')}: {output.get('evalue', '')}".rstrip(": ") return f"{header}\n{tb_text}".rstrip() if otype not in {"execute_result", "display_data", "pyout"}: return "" - data = output.get("data") if not isinstance(data, dict): # legacy v3: mime payloads sit flat on the output dict data = {"text/plain": output["text"]} if isinstance(output.get("text"), (str, list)) else {} data.update((mime, output[k]) for k, mime in _V3_MIME_KEYS if k in output) if "application/vnd.jupyter.widget-view+json" in data: return "[interactive widget — omitted]" - # Prefer readable text: models consume text/plain far better than markup. - for mime in ("text/plain", "text/markdown"): - if mime in data: - body = _clean_stream_text(_source_text(data[mime])) - if body.strip(): - return body + for mime in ("text/plain", "text/markdown"): # models consume text far better than markup + body = _clean_stream_text(_source_text(data[mime])) if mime in data else "" + if body.strip(): + return body for mime, value in data.items(): if isinstance(mime, str) and mime.startswith("image/"): - size = _base64_bytes(_source_text(value)) - return f"[{mime} output — {_human_size(size)}, omitted]" + return f"[{mime} output — {_human_size(_base64_bytes(_source_text(value)))}, omitted]" if "text/html" in data: - html = _source_text(data["text/html"]) - return f"[text/html output — {len(html):,} chars, omitted]" - mimes = ", ".join(str(m) for m in data) or "unknown" - return f"[{mimes} output — omitted]" + return f"[text/html output — {len(_source_text(data['text/html'])):,} chars, omitted]" + return f"[{', '.join(str(m) for m in data) or 'unknown'} output — omitted]" def _notebook_outputs(cell: dict, jq_pointer: str = "", filename: str = "") -> str: outputs = cell.get("outputs") if not isinstance(outputs, list): return "" - blocks = [text for text in (_notebook_output_text(o) for o in outputs) if text] - if not blocks: - return "" - joined = "\n".join(blocks) - if len(joined) > _MAX_OUTPUT_CHARS: - omitted = len(joined) - _MAX_OUTPUT_CHARS - hint = f" — full output: jq -r '{jq_pointer}' {filename}" if jq_pointer and filename else "" - joined = joined[:_MAX_OUTPUT_CHARS] + f"\n… [{omitted:,} output chars truncated{hint}]" - return joined + joined = "\n".join(filter(None, map(_notebook_output_text, outputs))) + if len(joined) <= _MAX_OUTPUT_CHARS: + return joined + hint = f" — full output: jq -r '{jq_pointer}' {filename}" if jq_pointer and filename else "" + omitted = len(joined) - _MAX_OUTPUT_CHARS + return joined[:_MAX_OUTPUT_CHARS] + f"\n… [{omitted:,} output chars truncated{hint}]" _CELL_LABELS = {"markdown": "Markdown", "code": "Code", "raw": "Raw"} @@ -459,7 +407,7 @@ def _extract_notebook(path: str) -> str: raw_cells = nb.get("cells") if isinstance(raw_cells, list): cells = [(f".cells[{i}].outputs", cell) for i, cell in enumerate(raw_cells)] - else: + else: # nbformat v3: cells live under worksheets cells = [ (f".worksheets[{wi}].cells[{ci}].outputs", cell) for wi, ws in enumerate(nb.get("worksheets", [])) if isinstance(ws, dict) @@ -470,18 +418,16 @@ def _extract_notebook(path: str) -> str: counts = dict.fromkeys(_CELL_LABELS, 0) out: list[str] = [] for jq_pointer, cell in cells: - if not isinstance(cell, dict): - continue - typ = cell.get("cell_type") + typ = cell.get("cell_type") if isinstance(cell, dict) else None if typ not in _CELL_LABELS: continue counts[typ] += 1 suffix = f" {counts[typ]}" if typ != "raw" else "" - out.extend((f"# ── {_CELL_LABELS[typ]} cell{suffix} ──", _source_text(cell.get("source", "")).rstrip("\n"), "")) - if typ == "code": - rendered = _notebook_outputs(cell, jq_pointer, nb_name) - if rendered: - out.extend((f"# ── Output (cell {counts[typ]}) ──", rendered.rstrip("\n"), "")) + source = _source_text(cell.get("source", "")).rstrip("\n") + out += [f"# ── {_CELL_LABELS[typ]} cell{suffix} ──", source, ""] + rendered = _notebook_outputs(cell, jq_pointer, nb_name) if typ == "code" else "" + if rendered: + out += [f"# ── Output (cell {counts[typ]}) ──", rendered.rstrip("\n"), ""] return _joined(out, "Notebook contains no readable cells") @@ -491,24 +437,21 @@ def _open_zip(path: str, kind: str) -> Iterator[zipfile.ZipFile]: try: with zipfile.ZipFile(path) as zf: yield zf - except zipfile.BadZipFile as exc: - raise ExtractionError(f"Not a valid {kind}: {exc}") from exc - except OSError as exc: - raise ExtractionError(str(exc)) from exc + except (zipfile.BadZipFile, OSError) as exc: + bad_zip = isinstance(exc, zipfile.BadZipFile) + raise ExtractionError(f"Not a valid {kind}: {exc}" if bad_zip else str(exc)) from exc def _zip_xml(zf: zipfile.ZipFile, name: str, optional: bool = False) -> Any: - """Parse a package part; ``optional`` parts yield None when absent or malformed.""" + """Parse a package part; ``optional`` parts yield an empty element when absent or malformed.""" try: return ET.fromstring(zf.read(name)) - except KeyError as exc: + except (KeyError, ET.ParseError) as exc: if optional: - return None - raise ExtractionError(f"Missing {name}") from exc - except ET.ParseError as exc: - if optional: - return None - raise ExtractionError(f"Malformed XML in {name}: {exc}") from exc + return ET.Element("missing") + raise ExtractionError( + f"Missing {name}" if isinstance(exc, KeyError) else f"Malformed XML in {name}: {exc}" + ) from exc def _extract_docx(path: str) -> str: @@ -518,9 +461,9 @@ def _extract_docx(path: str) -> str: breaks = {f"{w}tab": "\t", f"{w}br": "\n", f"{w}cr": "\n"} lines: list[str] = [] for para in root.iter(f"{w}p"): - buf = [(node.text or "") if node.tag == f"{w}t" else breaks.get(node.tag, "") - for node in para.iter()] - lines.extend("".join(buf).split("\n")) + text = "".join( + (n.text or "") if n.tag == f"{w}t" else breaks.get(n.tag, "") for n in para.iter()) + lines.extend(text.split("\n")) return _joined(lines, "DOCX contains no extractable text") @@ -529,38 +472,27 @@ def _extract_xlsx(path: str) -> str: with _open_zip(path, "XLSX") as zf: names = set(zf.namelist()) sst = _zip_xml(zf, "xl/sharedStrings.xml", optional=True) - shared = [] if sst is None else [ - "".join(t.text or "" for t in item.iter(f"{s}t")) for item in sst.iter(f"{s}si")] + shared = ["".join(t.text or "" for t in item.iter(f"{s}t")) for item in sst.iter(f"{s}si")] rels_root = _zip_xml(zf, "xl/_rels/workbook.xml.rels", optional=True) - rels = {} if rels_root is None else { - rel.get("Id", ""): rel.get("Target", "") - for rel in rels_root.iter(f"{pr}Relationship") if rel.get("Id")} + rels = {rel.get("Id", ""): rel.get("Target", "") + for rel in rels_root.iter(f"{pr}Relationship") if rel.get("Id")} out: list[str] = [] for sheet in _zip_xml(zf, "xl/workbook.xml").iter(f"{s}sheet"): - if sheet.get("state", "visible") in {"hidden", "veryHidden"}: - continue target = rels.get(sheet.get(f"{r}id", ""), "").lstrip("/") part = posixpath.normpath(target if target.startswith("xl/") else f"xl/{target}") - if part not in names: + if sheet.get("state", "visible") in {"hidden", "veryHidden"} or part not in names: continue - try: + with contextlib.suppress(ET.ParseError): rows = _sheet_rows(zf.read(part), shared) - except ET.ParseError: - continue - out.append(f"# ── Sheet: {sheet.get('name', 'Sheet')} ──") - out.extend("\t".join(row) for row in rows) - if not rows: - out.append("(empty)") - out.append("") + out += [f"# ── Sheet: {sheet.get('name', 'Sheet')} ──", + *(["\t".join(row) for row in rows] or ["(empty)"]), ""] return _joined(out, "XLSX has no visible sheets with content") def _col_index(ref: str) -> int: - idx = 0 - for ch in ref: - if not ch.isalpha(): - break - idx = idx * 26 + ord(ch.upper()) - ord("A") + 1 + """0-based column of a cell ref: ``A1`` -> 0, ``AB7`` -> 27 (bijective base-26 letters).""" + idx = functools.reduce(lambda acc, ch: acc * 26 + ord(ch.upper()) - ord("A") + 1, + itertools.takewhile(str.isalpha, ref), 0) return max(idx - 1, 0) @@ -568,18 +500,15 @@ def _sheet_rows(xml_bytes: bytes, shared: list[str]) -> list[list[str]]: root = ET.fromstring(xml_bytes) s = f"{{{_NS_S}}}" rows: list[list[str]] = [] - for row in root.iter(f"{s}row"): - if len(rows) >= _MAX_XLSX_ROWS_PER_SHEET: - break + for row in itertools.islice(root.iter(f"{s}row"), _MAX_XLSX_ROWS_PER_SHEET): cells: dict[int, str] = {} max_col = -1 for cell in row.iter(f"{s}c"): col = _col_index(cell.get("r", "")) if cell.get("r") else max_col + 1 - if col >= _MAX_XLSX_COLS: - continue - cells[col] = _cell_value(cell, shared, s) - max_col = max(max_col, col) - rows.append([cells.get(i, "") for i in range(max_col + 1)] if max_col >= 0 else []) + if col < _MAX_XLSX_COLS: + cells[col] = _cell_value(cell, shared, s) + max_col = max(max_col, col) + rows.append([cells.get(i, "") for i in range(max_col + 1)]) while rows and not any(value.strip() for value in rows[-1]): rows.pop() return rows diff --git a/tools/schema_sanitizer.py b/tools/schema_sanitizer.py index 39a3d6c937..c673998f28 100644 --- a/tools/schema_sanitizer.py +++ b/tools/schema_sanitizer.py @@ -1,11 +1,7 @@ -"""Sanitize tool JSON schemas for broad LLM-backend compatibility. - -Strict backends reject shapes OpenAI/Anthropic accept: llama.cpp's grammar converter fails -on ``{"type": "object"}`` without ``properties``, bare-string schemas and ``type`` arrays; -Anthropic rejects nullable ``anyOf`` at the top of ``input_schema``; Fireworks rejects -``default`` beside ``$ref``; Codex rejects top-level combinators. This module walks the -final schema tree on a deep copy and fixes only those shapes. -""" +"""Sanitize tool JSON schemas for strict LLM backends. llama.cpp's grammar converter fails on +``{"type": "object"}`` without ``properties``, bare-string schemas and ``type`` arrays; Anthropic +rejects nullable ``anyOf`` at the top of ``input_schema``; Fireworks rejects ``default`` beside +``$ref``; Codex rejects top-level combinators. Walks a deep copy and fixes only those shapes.""" from __future__ import annotations @@ -16,15 +12,12 @@ from typing import Any, Callable logger = logging.getLogger(__name__) - -# Anthropic (and Bedrock/Vertex/Azure fronting it) reject property keys not matching this; -# one bad key anywhere in the tools array 400s the request (Cloudflare's MCP ships 61). +# Anthropic (and Bedrock/Vertex/Azure fronting it) reject property keys not matching this; one bad +# key anywhere in the tools array 400s the request (Cloudflare's MCP ships 61). _PROP_KEY_RE = re.compile(r"^[a-zA-Z0-9_.-]{1,64}$") _PROP_KEY_BAD_CHARS = re.compile(r"[^a-zA-Z0-9_.-]") - _UNION_KEYS = ("anyOf", "oneOf") -# Outer-node metadata carried onto a union's replacement node. -_UNION_META_KEYS = ("title", "description", "default", "examples") +_UNION_META_KEYS = ("title", "description", "default", "examples") # copied onto replacements def _empty_object() -> dict: @@ -47,56 +40,45 @@ def sanitize_property_key(key: str) -> str: def _rename_property_keys(props: dict, path: str) -> dict[str, str]: """{original_key: conforming_key} for one properties dict (identity entries omitted). - Deterministic (insertion order, numeric suffixes on collision) so the model-visible - schema and the dispatch-time reverse map from the registry's original schema agree.""" + Deterministic (insertion order, numeric suffixes on collision) so the model-visible schema + and the dispatch-time reverse map from the registry's original schema agree.""" renames: dict[str, str] = {} taken = {k for k in props if _PROP_KEY_RE.match(k)} - for key in props: - if _PROP_KEY_RE.match(key): - continue + for key in (k for k in props if not _PROP_KEY_RE.match(k)): base = sanitize_property_key(key) candidate, i = base, 2 while candidate in taken: - suffix = f"_{i}" - candidate = base[: 64 - len(suffix)] + suffix - i += 1 + candidate, i = base[: 64 - len(f"_{i}")] + f"_{i}", i + 1 taken.add(candidate) renames[key] = candidate - logger.debug( - "schema_sanitizer[%s]: renamed property key %r -> %r " - "(provider key-pattern compat)", path, key, candidate) + logger.debug("schema_sanitizer[%s]: renamed property key %r -> %r " + "(provider key-pattern compat)", path, key, candidate) return renames def unrename_tool_args(params_schema: Any, args: Any) -> Any: - """Map sanitized property keys in model-emitted args back to wire names. ``params_schema`` - is the ORIGINAL registry schema; recurses into objects/array items; unknown keys pass.""" - if not isinstance(params_schema, dict) or not isinstance(args, dict): - return args - props = params_schema.get("properties") - if not isinstance(props, dict): + """Map sanitized keys in model-emitted args back to wire names. ``params_schema`` is the + ORIGINAL registry schema; recurses into objects/array items; unknown keys pass through.""" + props = params_schema.get("properties") if isinstance(params_schema, dict) else None + if not isinstance(props, dict) or not isinstance(args, dict): return args reverse = {v: k for k, v in _rename_property_keys(props, "").items()} out = {} for key, value in args.items(): orig = reverse.get(key, key) - subschema = props.get(orig) - if isinstance(subschema, dict): - if isinstance(value, dict): - value = unrename_tool_args(subschema, value) - elif isinstance(value, list) and isinstance(subschema.get("items"), dict): - value = [ - unrename_tool_args(subschema["items"], item) if isinstance(item, dict) else item - for item in value] + sub = props.get(orig) if isinstance(props.get(orig), dict) else {} + if isinstance(value, dict) and sub: + value = unrename_tool_args(sub, value) + elif isinstance(value, list) and isinstance(sub.get("items"), dict): + value = [unrename_tool_args(sub["items"], item) if isinstance(item, dict) else item + for item in value] out[orig] = value return out def sanitize_tool_schemas(tools: list[dict]) -> list[dict]: """Deep-copied ``tools`` (OpenAI format) with sanitized parameter schemas; safe to mutate.""" - if not tools: - return tools - return [_sanitize_single_tool(tool) for tool in tools] + return [_sanitize_single_tool(tool) for tool in tools] if tools else tools def _sanitize_single_tool(tool: dict) -> dict: @@ -110,9 +92,7 @@ def _sanitize_single_tool(tool: dict) -> dict: return out name = fn.get("name", "") top = _sanitize_node(params, path=name) - # Guarantee the top level is an object with properties. - if not isinstance(top, dict): - top = {} + top = top if isinstance(top, dict) else {} # guarantee an object with properties on top top["type"] = "object" if not isinstance(top.get("properties"), dict): top["properties"] = {} @@ -124,16 +104,14 @@ def _sanitize_single_tool(tool: dict) -> dict: return out -# Sibling keywords strict JSON Schema validators reject alongside ``$ref``. -_REF_FORBIDDEN_SIBLINGS = frozenset({"default"}) +_REF_FORBIDDEN_SIBLINGS = frozenset({"default"}) # strict validators reject these beside ``$ref`` def _strip_ref_siblings(node: Any) -> Any: """Recursively drop forbidden siblings of ``$ref`` (Fireworks rejects ``default`` there).""" def strip(out: dict) -> dict: - if "$ref" in out: - for key in _REF_FORBIDDEN_SIBLINGS: - out.pop(key, None) + for key in _REF_FORBIDDEN_SIBLINGS if "$ref" in out else (): + out.pop(key, None) return out return _rewrite(node, strip) @@ -143,18 +121,15 @@ _TOP_LEVEL_FORBIDDEN_KEYS = ("allOf", "anyOf", "oneOf", "enum", "not") def _strip_top_level_combinators(params: dict, *, path: str = "") -> dict: """Drop combinators from the TOP level only (Codex rejects them there). They are usually - conditional-required hints, so validity is unchanged (handlers re-validate); nested - combinators are preserved.""" + conditional-required hints, so validity is unchanged (handlers re-validate); nested ones + stay.""" if not isinstance(params, dict): return params out = dict(params) - for key in _TOP_LEVEL_FORBIDDEN_KEYS: - if key in out: - logger.debug( - "schema_sanitizer[%s]: stripped top-level %r combinator " - "from tool parameters (strict-backend compat)", - path, key) - out.pop(key, None) + for key in [k for k in _TOP_LEVEL_FORBIDDEN_KEYS if k in out]: + logger.debug("schema_sanitizer[%s]: stripped top-level %r combinator " + "from tool parameters (strict-backend compat)", path, key) + del out[key] return out @@ -163,20 +138,18 @@ def _is_null_branch(item: Any) -> bool: def _carry_union_meta(outer: dict, replacement: dict, *, skip_default_on_ref: bool) -> None: - """Copy outer-union metadata onto *replacement* where absent.""" + """Copy outer-union metadata onto *replacement* where absent (``default`` is illegal beside + ``$ref`` on strict backends, hence ``skip_default_on_ref``).""" for meta_key in _UNION_META_KEYS: - if meta_key in outer and meta_key not in replacement: - # ``default`` is illegal alongside ``$ref`` on strict backends. - if skip_default_on_ref and meta_key == "default" and "$ref" in replacement: - continue + if meta_key in outer and meta_key not in replacement and not ( + skip_default_on_ref and meta_key == "default" and "$ref" in replacement): replacement[meta_key] = outer[meta_key] def strip_nullable_unions(schema: Any, *, keep_nullable_hint: bool = True) -> Any: - """Collapse ``anyOf``/``oneOf`` nullable unions to the single non-null branch. MCP/Pydantic - optional fields arrive as ``{"anyOf": [{"type": "string"}, {"type": "null"}]}``; Anthropic - rejects the null branch and optionality is already in the parent's ``required``. Only - collapses when a null branch was dropped AND exactly one non-null branch survives. + """Collapse ``anyOf``/``oneOf`` nullable unions (MCP/Pydantic optional fields) to the single + non-null branch: Anthropic rejects the null branch and optionality already lives in the parent's + ``required``. Only when a null branch was dropped AND exactly one non-null branch survives. ``keep_nullable_hint`` sets ``nullable: true`` for runtime ``"null"`` → ``None`` coercion.""" def collapse(stripped: dict) -> Any: for key in _UNION_KEYS: @@ -189,7 +162,7 @@ def strip_nullable_unions(schema: Any, *, keep_nullable_hint: bool = True) -> An if keep_nullable_hint: replacement.setdefault("nullable", True) _carry_union_meta(stripped, replacement, skip_default_on_ref=True) - return _rewrite(replacement, collapse) + return _rewrite(replacement, collapse) # the survivor may itself be a union return stripped return _rewrite(schema, collapse) @@ -199,24 +172,23 @@ _CONST_PRIMITIVE_TYPES: dict[type, str] = { def _const_branch_type(branch: Any) -> str | None: - """JSON-Schema primitive type of a pure ``const`` branch, else None: a primitive ``const`` - whose declared ``type`` (if any) matches; only ``title``/``description`` may accompany it.""" + """Primitive JSON-Schema type of a pure ``const`` branch (declared ``type``, if any, must match; + only ``title``/``description`` may accompany it), else None.""" if not isinstance(branch, dict) or "const" not in branch \ or set(branch) - {"const", "type", "title", "description"}: return None # ``type(value)`` lookup (not isinstance): bool is a subclass of int. json_type = _CONST_PRIMITIVE_TYPES.get(type(branch["const"])) - return json_type if json_type is not None and branch.get("type") in (None, json_type) else None + return json_type if branch.get("type") in (None, json_type) else None def collapse_const_unions(schema: Any) -> Any: - """Collapse ``anyOf``/``oneOf`` unions of same-typed consts to ``enum`` (ported from - block/goose ``tool_schema_normalize.rs``, Apache-2.0). Rust/TS MCP servers emit - ``{"anyOf": [{"const": "red"}, {"const": "green"}]}``, which strict backends mishandle. - Applies only when EVERY non-null branch is a pure ``const`` of one primitive type - (``bool`` never merges with ``integer``); one ``{"type": "null"}`` branch is tolerated - and recorded as ``nullable: true``. Branch order is kept; outer metadata carried over; - input never mutated.""" + """Collapse ``anyOf``/``oneOf`` unions of same-typed consts (Rust/TS MCP servers emit + ``{"anyOf": [{"const": "red"}, {"const": "green"}]}``) to ``enum``; ported from block/goose + ``tool_schema_normalize.rs`` (Apache-2.0). Only when EVERY non-null branch is a pure ``const`` + of one primitive type (``bool`` never merges with ``integer``); one ``{"type": "null"}`` branch + is tolerated as ``nullable: true``. Branch order kept; outer metadata carried; input never + mutated.""" def collapse(out: dict) -> Any: for key in _UNION_KEYS: variants = out.get(key) @@ -240,52 +212,48 @@ def collapse_const_unions(schema: Any) -> Any: _BARE_TYPE_NAMES = frozenset({"object", "string", "number", "integer", "boolean", "array", "null"}) -# Keys whose values are NOT schemas (recursing would treat "path" as a bare-string schema). +# Values that are NOT schemas (recursing would treat a required name like "path" as a bare schema). _NON_SCHEMA_LIST_KEYS = frozenset({"required", "enum", "examples", "dependentRequired"}) def _normalize_type_array(value: list, out: dict) -> None: - """Normalize a ``type: [...]`` array into *out* (llama.cpp and Gemini-via-OpenAI reject - arrays). Per AI-SDK: one non-null type → ``type: X`` (+ ``nullable`` if ``null`` present); - several → ``anyOf`` of single-type schemas so EVERY branch survives; none → ``null`` or - the object fallback. Ported from anomalyco/opencode#31877.""" + """Normalize a ``type: [...]`` array into *out* (llama.cpp and Gemini-via-OpenAI reject arrays). + Per AI-SDK: one non-null type → ``type: X`` (+ ``nullable`` if ``null`` present); several → + ``anyOf`` of single-type schemas so EVERY branch survives; none → ``null``/object fallback.""" has_null = "null" in value non_null = [t for t in value if isinstance(t, str) and t != "null"] - if len(non_null) == 1: - out["type"] = non_null[0] - elif len(non_null) >= 2: - out["anyOf"] = [{"type": t} for t in non_null] - else: + if not non_null: out["type"] = "null" if has_null else "object" return + if len(non_null) == 1: + out["type"] = non_null[0] + else: + out["anyOf"] = [{"type": t} for t in non_null] if has_null: out.setdefault("nullable", True) def _sanitize_node(node: Any, path: str) -> Any: - """Recursively sanitize a JSON-Schema fragment: bare-string schemas become ``{"type": - }`` (unknown strings → permissive object); object nodes gain ``properties: {}``; - ``type`` arrays are normalized; property keys are renamed to the provider-safe pattern - and ``required`` follows, with entries missing from ``properties`` pruned.""" + """Recursively sanitize a JSON-Schema fragment: bare-string schemas → ``{"type": }`` + (unknown strings → permissive object); object nodes gain ``properties: {}``; ``type`` arrays + are normalized; property keys are renamed to the provider-safe pattern and ``required`` + follows, with entries missing from ``properties`` pruned.""" if isinstance(node, str): if node in _BARE_TYPE_NAMES: - logger.debug( - "schema_sanitizer[%s]: replacing bare-string schema %r with {'type': %r}", - path, node, node) + logger.debug("schema_sanitizer[%s]: replacing bare-string schema %r with {'type': %r}", + path, node, node) return _empty_object() if node == "object" else {"type": node} - logger.debug( - "schema_sanitizer[%s]: replacing non-schema string %r " - "with empty object schema", path, node) + logger.debug("schema_sanitizer[%s]: replacing non-schema string %r " + "with empty object schema", path, node) return _empty_object() if isinstance(node, list): return [_sanitize_node(item, f"{path}[{i}]") for i, item in enumerate(node)] if not isinstance(node, dict): return node - # Renames computed up front so ``required`` remaps even when it precedes ``properties``. - prop_renames: dict[str, str] = {} - if isinstance(node.get("properties"), dict): - prop_renames = _rename_property_keys(node["properties"], f"{path}.properties") + props_in = node.get("properties") + prop_renames = (_rename_property_keys(props_in, f"{path}.properties") + if isinstance(props_in, dict) else {}) out: dict = {} for key, value in node.items(): if key == "type" and isinstance(value, list): @@ -293,64 +261,55 @@ def _sanitize_node(node: Any, path: str) -> Any: elif key in {"properties", "$defs", "definitions"} and isinstance(value, dict): renames = prop_renames if key == "properties" else {} out[key] = { - renames.get(sub_k, sub_k): _sanitize_node(sub_v, f"{path}.{key}.{renames.get(sub_k, sub_k)}") - for sub_k, sub_v in value.items()} + renames.get(k, k): _sanitize_node(v, f"{path}.{key}.{renames.get(k, k)}") + for k, v in value.items()} elif key in {"items", "additionalProperties"}: # Bool ``additionalProperties`` is valid; bool ``items`` is non-standard but preserved. out[key] = value if isinstance(value, bool) else _sanitize_node(value, f"{path}.{key}") - elif key in {"anyOf", "oneOf", "allOf"} and isinstance(value, list): - out[key] = [_sanitize_node(item, f"{path}.{key}[{i}]") for i, item in enumerate(value)] elif key in _NON_SCHEMA_LIST_KEYS: if key == "required" and prop_renames and isinstance(value, list): out[key] = [prop_renames.get(r, r) if isinstance(r, str) else r for r in value] else: out[key] = copy.deepcopy(value) if isinstance(value, (list, dict)) else value - else: + else: # anyOf/oneOf/allOf and any other nested schema recurse (lists index the path) out[key] = _sanitize_node(value, f"{path}.{key}") if isinstance(value, (dict, list)) else value if out.get("type") == "object": if not isinstance(out.get("properties"), dict): out["properties"] = {} if isinstance(out.get("required"), list): - props = out.get("properties") or {} - valid = [r for r in out["required"] if isinstance(r, str) and r in props] - if not valid: - out.pop("required", None) - elif len(valid) != len(out["required"]): + valid = [r for r in out["required"] if isinstance(r, str) and r in out["properties"]] + if valid: out["required"] = valid + else: + del out["required"] return out -# ---- Reactive strips — only invoked after a backend rejects a schema ----------------------- +# ---- Reactive strips — only invoked after a backend rejects a schema ---- _STRIP_ON_RECOVERY_KEYS = frozenset({"pattern", "format"}) +_SCHEMA_MARKERS = frozenset({"type", "anyOf", "oneOf", "allOf"}) # a node with one IS a schema def _dict_nodes(node: Any): - """Pre-order walk yielding every dict node; each is yielded before its values are visited, - so a consumer may mutate it in place.""" + """Pre-order walk over every dict node (yielded before its values, so it may be mutated).""" if isinstance(node, dict): yield node - for v in node.values(): - yield from _dict_nodes(v) - elif isinstance(node, list): - for item in node: - yield from _dict_nodes(item) + children = node.values() if isinstance(node, dict) else node if isinstance(node, list) else () + for child in children: + yield from _dict_nodes(child) def _reactive_strip( tools: list[dict], strip_node: Callable[[dict], int], log_msg: str) -> tuple[list[dict], int]: - """Apply *strip_node* (returns keywords removed) to every dict node of each tool's - parameters, in place. Handles OpenAI (``{"function": {"parameters": ..}}``) and Responses - (``{"name": .., "parameters": ..}``) formats. Returns ``(tools, stripped_count)``.""" - if not tools: - return tools, 0 + """Apply *strip_node* (-> keywords removed) to every dict node of each tool's parameters, in + place; OpenAI (``{"function": {"parameters"}}``) and Responses (``{"parameters"}``) formats.""" stripped = 0 - for tool in tools: + for tool in tools or (): if not isinstance(tool, dict): continue fn = tool.get("function") params = fn.get("parameters") if isinstance(fn, dict) else None - if not isinstance(params, dict): - params = tool.get("parameters") + params = params if isinstance(params, dict) else tool.get("parameters") if isinstance(params, dict): stripped += sum(strip_node(node) for node in _dict_nodes(params)) if stripped: @@ -359,16 +318,14 @@ def _reactive_strip( def strip_pattern_and_format(tools: list[dict]) -> tuple[list[dict], int]: - """Strip ``pattern``/``format`` keywords from tool schemas, in place. Reactive: only after - llama.cpp's grammar converter rejected a schema (its regex engine is a small ECMAScript - subset), since cloud providers use these as prompting hints. Only strips beside - ``type``/combinators, so a property literally *named* ``pattern`` is untouched.""" + """Strip ``pattern``/``format`` in place — reactive, only after llama.cpp's grammar converter + rejected a schema (its regex engine is a small ECMAScript subset); cloud providers use these as + prompting hints. Only beside ``type``/combinators, so a property *named* ``pattern`` stays.""" def _strip(node: dict) -> int: - if not ("type" in node or "anyOf" in node or "oneOf" in node or "allOf" in node): - return 0 - hits = [k for k in node if k in _STRIP_ON_RECOVERY_KEYS] + is_schema = bool(node.keys() & _SCHEMA_MARKERS) + hits = [k for k in node if k in _STRIP_ON_RECOVERY_KEYS] if is_schema else [] for k in hits: - node.pop(k, None) + del node[k] return len(hits) return _reactive_strip( tools, _strip, @@ -377,13 +334,12 @@ def strip_pattern_and_format(tools: list[dict]) -> tuple[list[dict], int]: def strip_slash_enum(tools: list[dict]) -> tuple[list[dict], int]: - """Strip ``enum`` keywords whose string values contain ``/``, in place. xAI compiles - schemas to a grammar that rejects ``/`` in enum values (HTTP 400 before any token) — - typically MCP enums of HuggingFace model IDs. The constraint is a prompting hint only.""" + """Strip ``enum`` keywords whose string values contain ``/``, in place: xAI's grammar compiler + rejects them (HTTP 400 before any token) — typically MCP enums of HuggingFace model IDs.""" def _strip(node: dict) -> int: enum_val = node.get("enum") if isinstance(enum_val, list) and any(isinstance(v, str) and "/" in v for v in enum_val): - node.pop("enum", None) + del node["enum"] return 1 return 0 return _reactive_strip( diff --git a/tools/self_repo_guard.py b/tools/self_repo_guard.py index 32b72a1fba..a97bb2ffb0 100644 --- a/tools/self_repo_guard.py +++ b/tools/self_repo_guard.py @@ -2,6 +2,7 @@ from __future__ import annotations +import contextlib import os import re import shlex @@ -11,9 +12,7 @@ from pathlib import Path from typing import Callable from tools.approval import ( - _bash_exec_payload, - _deobfuscate_shell_word_for_detection, - _iter_shell_command_starts, + _bash_exec_payload, _deobfuscate_shell_word_for_detection, _iter_shell_command_starts, _read_shell_word) # bisect drives repeated checkouts of the running root — the exact skew hazard guarded here. @@ -24,8 +23,7 @@ _WORKTREE_TARGET_ACTIONS = frozenset({"move", "remove"}) _STASH_SAFE_ACTIONS = frozenset({"list", "show", "create", "store", "drop", "clear"}) _RESET_WORKTREE_MODES = frozenset({"--hard", "--merge", "--keep"}) # `reset`/`stash`/`clean`/`restore` reach this set only in their SAFE forms (_mutates_worktree -# classifies the dangerous forms first); listing them just skips a pointless -# `git config --get alias.` subprocess for `stash list`, `reset --soft`, `clean -n`. +# runs first); listing them skips a pointless `git config --get alias.` subprocess. _KNOWN_GIT_BUILTINS = frozenset({ "add", "am", "apply", "blame", "branch", "bundle", "cat-file", "clean", "clone", "commit", "config", "describe", "diff", "fetch", "format-patch", "grep", "help", "init", "log", @@ -64,36 +62,31 @@ class _Heredoc: @dataclass -class _ShellContext: +class _ShellContext: # one `(` / `$(` / backtick nesting level and its live quote state kind: str opener: int quote: str | None = None def get_running_source_root() -> Path | None: - """Return the source checkout backing this process, if there is one.""" - try: + """The source checkout backing this process, if there is one.""" + with contextlib.suppress(OSError, RuntimeError): root = Path(__file__).resolve().parent.parent - except (OSError, RuntimeError): - return None - return root if (root / ".git").exists() else None + return root if (root / ".git").exists() else None + return None def _resolve(path_str: str, base: Path) -> Path: - path = Path(os.path.expanduser(path_str)) - if not path.is_absolute(): - path = base / path - try: + path = base / Path(os.path.expanduser(path_str)) # ``/`` keeps an absolute right operand + with contextlib.suppress(OSError, RuntimeError, ValueError): return path.resolve() - except (OSError, RuntimeError, ValueError): - return path + return path def _is_within(path: Path, root: Path) -> bool: - try: + with contextlib.suppress(OSError, RuntimeError, ValueError): return path == root or path.is_relative_to(root) - except (OSError, RuntimeError, ValueError): - return False + return False def _executable_name(value: str) -> str: @@ -101,6 +94,7 @@ def _executable_name(value: str) -> str: def _shell_words_at(command: str, start: int) -> list[str]: + """Deobfuscated words of the simple command at ``start`` (stops at a newline; max 64).""" words: list[str] = [] cursor = start for _ in range(64): @@ -116,13 +110,10 @@ def _consume_options( words: list[str], start: int, options_with_arg: frozenset[str] = _NO_OPTIONS) -> int: """Index of the first positional at/after ``start`` (``--`` ends options).""" index = start - while index < len(words): - option = words[index] - if option == "--": + while index < len(words) and words[index].startswith("-") and words[index] != "-": + if words[index] == "--": return index + 1 - if not option.startswith("-") or option == "-": - break - index += 2 if "=" not in option and option in options_with_arg else 1 + index += 2 if "=" not in words[index] and words[index] in options_with_arg else 1 return index @@ -140,9 +131,8 @@ def _command_parts(words: list[str]) -> tuple[dict[str, str], str | None, list[s wrapper_options = _WRAPPER_OPTIONS_WITH_ARG.get(executable) if wrapper_options is None: return env, words[index], words[index + 1 :] - # `command -v/-V` only reports; nothing runs. if executable == "command" and words[index + 1 : index + 2] in (["-v"], ["-V"]): - return env, None, [] + break # `command -v/-V` only reports; nothing runs index = _consume_options(words, index + 1, wrapper_options) return env, None, [] @@ -157,15 +147,14 @@ def _scope_keys(command: str, starts: list[int]) -> dict[int, tuple[int, ...]]: context = contexts[-1] quote = context.quote char = command[cursor] - nested = len(contexts) > 1 - if quote == "'": - if char == "'": - context.quote = None + closes = quote is None and len(contexts) > 1 # an unquoted closer may pop a scope + if quote is not None and char == quote: + context.quote = None + elif quote == "'": + pass # single quotes: no escapes, no substitutions elif char == "\\" and cursor + 1 < start: cursor += 1 - elif quote == '"' and char == '"': - context.quote = None - elif quote is None and char in {"'", '"'}: + elif quote is None and char in "'\"": context.quote = char # Unquoted or inside double quotes: substitutions still open scopes. elif command.startswith("$(", cursor): @@ -173,28 +162,27 @@ def _scope_keys(command: str, starts: list[int]) -> dict[int, tuple[int, ...]]: cursor += 1 elif quote is None and char == "(": contexts.append(_ShellContext("(", cursor)) - elif quote is None and char == ")" and nested and contexts[-1].kind in {"(", "$("}: + elif (char == ")" and closes and context.kind in {"(", "$("}) or ( + char == "`" and closes and context.kind == "`"): contexts.pop() elif char == "`": - if quote is None and nested and contexts[-1].kind == "`": - contexts.pop() - else: - contexts.append(_ShellContext("`", cursor)) + contexts.append(_ShellContext("`", cursor)) cursor += 1 scopes[start] = tuple(item.opener for item in contexts[1:]) return scopes def _operator_before(command: str, start: int) -> str | None: + """The list/grouping operator (or newline) immediately preceding a command start.""" head = command[:start].rstrip() - if head[-2:] in {"&&", "||"}: - return head[-2:] - if head[-1:] in {";", "|", "&", "(", "{"}: - return head[-1:] + for tail in (head[-2:], head[-1:]): + if tail in {"&&", "||", ";", "|", "&", "(", "{"}: + return tail return "\n" if "\n" in command[len(head):start] else None def _cd_target(executable: str, args: list[str], cwd: Path) -> Path | None: + """Directory a ``cd``/``pushd`` would land in (existing dirs only), else None.""" if _executable_name(executable) not in {"cd", "pushd"}: return None index = _consume_options(args, 0) @@ -205,17 +193,15 @@ def _cd_target(executable: str, args: list[str], cwd: Path) -> Path | None: def _shell_script_arg(args: list[str]) -> str | None: - """Return the script string owned by a shell's ``-c``, if present. approval.py's - ``_bash_exec_payload`` parses bash's real option grammar (``-o pipefail -c '