From d61cff60e3c474b57d6d4486b7aebd0158ac3ceb Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:03:08 -0700 Subject: [PATCH] fix(context): subdirectory AGENTS.md hints keep head+tail and warn when over the ceiling; ceiling 8k -> 32k MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The on-demand subdirectory hint loader (agent/subdirectory_hints.py, #5291) tail-chopped anything past 8,000 chars with a bare marker and no log line. apps/desktop/AGENTS.md (11k) has been arriving cut off in every Desktop-area session since it was written, and nobody could tell. Compared with the field: Codex caps its whole instruction chain at 32 KiB (project_doc_max_bytes) and documents it; Claude Code and Cursor apply no cap to nested files; OpenCode has no nested discovery at all. Our on-demand + cache-safe + ancestor-walk design is the strongest of the four; only the constant and the silent cut were wrong. Now: ceiling 32,000 chars (Codex's number, a guard against a stray huge CLAUDE.md in a vendored tree, not a target), and truncation goes through prompt_builder._truncate_content — head 70% + tail 20%, a marker naming the file to read_file for the rest, and a WARNING in the log. Area AGENTS.md files should stay around 8k anyway: the text lands in a tool result on the first touch of the directory. Tests: oversized hint keeps head+tail, names the path, and logs; a 12k area file (over the old cap, under the new) arrives intact. --- agent/subdirectory_hints.py | 12 ++++++---- tests/agent/test_subdirectory_hints.py | 33 +++++++++++++++++++------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/agent/subdirectory_hints.py b/agent/subdirectory_hints.py index b310783244..68512719cb 100644 --- a/agent/subdirectory_hints.py +++ b/agent/subdirectory_hints.py @@ -11,14 +11,19 @@ import shlex from pathlib import Path from typing import Dict, Any, Optional, Set -from agent.prompt_builder import _read_text_with_timeout, _scan_context_content +from agent.prompt_builder import _read_text_with_timeout, _scan_context_content, _truncate_content from agent.search_policy import SEARCH_PRUNE_DIR_NAMES logger = logging.getLogger(__name__) # Same filenames as prompt_builder.py, in priority order (first match wins per dir). _HINT_FILENAMES = ["AGENTS.override.md", "AGENTS.md", "agents.md", "CLAUDE.md", "claude.md", ".cursorrules"] -_MAX_HINT_CHARS = 8_000 +# Per-file ceiling for on-demand subdirectory hints. 32 KiB matches Codex's `project_doc_max_bytes` default +# (Claude Code and Cursor apply none); it is a guard against a stray huge CLAUDE.md in a vendored tree, not a +# target — keep area AGENTS.md files well under it (~8k) because this text lands in a tool result on the first +# touch of that directory. Over the ceiling: head+tail kept, marker with the path so the agent can read_file it, +# and a WARNING in the log (the old 8k silent tail-chop cut apps/desktop/AGENTS.md for months unnoticed). +_MAX_HINT_CHARS = 32_000 _PATH_ARG_KEYS = {"path", "file_path", "workdir"} _COMMAND_TOOLS = {"terminal"} _MAX_ANCESTOR_WALK = 5 # ancestor levels walked per path — bounds deep-path scans @@ -171,9 +176,8 @@ class SubdirectoryHintTracker: self._loaded_digests.add(digest) # Same security scan as startup context loading. content = _scan_context_content(content, filename) - if len(content) > _MAX_HINT_CHARS: - content = content[:_MAX_HINT_CHARS] + f"\n\n[...truncated {filename}: {len(content):,} chars total]" rel_path = self._display_path(hint_path) + content = _truncate_content(content, filename, max_chars=_MAX_HINT_CHARS, read_path=rel_path) logger.debug("Loaded subdirectory hints from %s: %s", directory, [rel_path]) return f"[Subdirectory context discovered: {rel_path}]\n{content}" # first match wins per directory except Exception as exc: diff --git a/tests/agent/test_subdirectory_hints.py b/tests/agent/test_subdirectory_hints.py index 8972027adc..3709a3792b 100644 --- a/tests/agent/test_subdirectory_hints.py +++ b/tests/agent/test_subdirectory_hints.py @@ -96,20 +96,35 @@ class TestSubdirectoryHintTracker: - def test_truncation_of_large_hints(self, tmp_path): - """Hint files over the limit are truncated.""" + def test_truncation_of_large_hints(self, tmp_path, caplog): + """Over the ceiling: head AND tail survive, the marker names the file to read_file, and it is logged + (the old silent tail-chop hid a truncated apps/desktop/AGENTS.md for months).""" + import logging + from agent import subdirectory_hints as sh sub = tmp_path / "bigdir" sub.mkdir() - (sub / "AGENTS.md").write_text("x" * 20_000) + body = "HEAD-MARKER " + ("x" * (sh._MAX_HINT_CHARS + 5_000)) + " TAIL-MARKER" + (sub / "AGENTS.md").write_text(body) tracker = SubdirectoryHintTracker(working_dir=str(tmp_path)) - result = tracker.check_tool_call( - "read_file", {"path": str(sub / "file.py")} - ) + with caplog.at_level(logging.WARNING, logger="agent.prompt_builder"): + result = tracker.check_tool_call("read_file", {"path": str(sub / "file.py")}) assert result is not None - assert "truncated" in result.lower() - # Should be capped - assert len(result) < 20_000 + assert "HEAD-MARKER" in result and "TAIL-MARKER" in result + assert "truncated AGENTS.md" in result and "bigdir/AGENTS.md" in result + assert len(result) < len(body) + assert any("TRUNCATED" in r.message and "AGENTS.md" in r.message for r in caplog.records) + + def test_area_file_under_ceiling_is_delivered_whole(self, tmp_path): + """An area AGENTS.md sized like ours (well under the ceiling) arrives intact — no marker.""" + sub = tmp_path / "gateway" + sub.mkdir() + body = "# Gateway rules\n" + ("- rule\n" * 1500) # ~12k chars: over the OLD 8k cap, under the new one + (sub / "AGENTS.md").write_text(body) + tracker = SubdirectoryHintTracker(working_dir=str(tmp_path)) + result = tracker.check_tool_call("read_file", {"path": str(sub / "run.py")}) + assert result is not None and "truncated" not in result.lower() + assert result.endswith(body.strip()) def test_empty_args(self, project): """Empty args should not crash."""