fix(web_extract): bound stored full-text size + give concrete read_file offset

Two robustness gaps from the #54843 truncate-store path:

- _store_full_text wrote the full clean page to cache/web with no upper
  bound (path.write_text(content)); a multi-MB page → unbounded per-extract
  disk write. Cap at MAX_STORED_TEXT_CHARS (2MB, the pre-truncate-store
  refusal ceiling) with a marker when capped.
- The truncation footer told the model 'read_file ... offset=<line>' — a
  literal placeholder it had to guess. Compute the real starting line of the
  omitted middle (head line count + 1) so the first read_file lands in the gap.
This commit is contained in:
kshitijk4poor
2026-06-30 03:06:27 +05:30
committed by Teknium
parent c1b9de73f5
commit c9269fbfb6
2 changed files with 79 additions and 1 deletions
+25 -1
View File
@@ -307,6 +307,15 @@ def _web_requires_env() -> list[str]:
# Override via web.extract_char_limit in config.yaml.
DEFAULT_EXTRACT_CHAR_LIMIT = 15000
# Hard ceiling on the full-text file written to cache/web. The truncate-store
# path otherwise calls path.write_text(content) with no upper bound, so a
# multi-MB page (some backends return very large markdown) writes unbounded
# bytes to disk on every extract. Cap the stored copy; the model only ever
# sees char_limit anyway, and a 2MB page is already far more than any single
# read_file paging session needs. Mirrors the pre-truncate-store era's 2MB
# refusal ceiling, but stores (capped) instead of refusing.
MAX_STORED_TEXT_CHARS = 2_000_000
_debug = DebugSession("web_tools", env_var="WEB_TOOLS_DEBUG")
@@ -374,6 +383,15 @@ def _store_full_text(url: str, content: str) -> Optional[str]:
slug = re.sub(r"[^A-Za-z0-9._-]", "-", host)[:60].strip("-") or "page"
digest = hashlib.sha256(url.encode("utf-8")).hexdigest()[:10]
path = cache_dir / f"{slug}-{digest}.md"
# Bound the stored copy so a pathologically large page can't write
# unbounded bytes to disk. If capped, append a marker so a reader of
# the file knows it isn't the literal complete page.
if len(content) > MAX_STORED_TEXT_CHARS:
content = (
content[:MAX_STORED_TEXT_CHARS]
+ f"\n\n[... stored copy truncated at {MAX_STORED_TEXT_CHARS:,} chars "
f"of {len(content):,}; re-extract a more specific URL for the rest ...]"
)
path.write_text(content, encoding="utf-8")
return str(path)
except Exception as exc: # noqa: BLE001
@@ -422,10 +440,16 @@ def _truncate_with_footer(
f"of {total:,} total clean characters.",
]
if stored_path:
# The omitted middle begins right after the head we're showing. Give
# the model a concrete starting line (head line count + 1) so its first
# read_file lands in the gap instead of guessing <line>. read_file is
# 1-indexed; +1 moves past the last head line we already showed.
middle_start_line = head.count("\n") + 2
footer_lines.append(f"Full text saved to: {stored_path}")
footer_lines.append(
f'To read the omitted middle: read_file path="{stored_path}" '
f"offset=<line> limit=<n> (the file is the complete page)."
f"offset={middle_start_line} limit=200 (the file is the complete page; "
f"raise/lower offset to page through it)."
)
else:
footer_lines.append(