From 33242d5ee0f65945abaacae557027d63fa0a6b09 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:57:20 -0700 Subject: [PATCH] feat(evals): compaction recall eval harness Measures recall accuracy vs tokens retained across compaction policies. Real transcripts in, LLM-generated recall exam from the summarized region, per-policy answer+judge passes, scorecard out. --- evals/compaction/README.md | 49 ++++++++ evals/compaction/fixtures.py | 79 ++++++++++++ evals/compaction/policies.py | 48 ++++++++ evals/compaction/report.py | 43 +++++++ evals/compaction/runner.py | 228 +++++++++++++++++++++++++++++++++++ 5 files changed, 447 insertions(+) create mode 100644 evals/compaction/README.md create mode 100644 evals/compaction/fixtures.py create mode 100644 evals/compaction/policies.py create mode 100644 evals/compaction/report.py create mode 100644 evals/compaction/runner.py diff --git a/evals/compaction/README.md b/evals/compaction/README.md new file mode 100644 index 0000000000..c4c77ef73f --- /dev/null +++ b/evals/compaction/README.md @@ -0,0 +1,49 @@ +# Compaction Eval Harness + +Measures what context compaction actually costs in *recall*, not just tokens. + +## What it does + +1. Takes a real long transcript (JSON: `{"messages": [...]}`, chat format). +2. Generates a bank of factual recall questions from the region that + compaction will summarize away (cached per transcript for reproducibility). +3. Runs the transcript through `ContextCompressor.compress()` under each + policy in the matrix (current default, aggressive tail, codex-style, ...). +4. For each policy, asks a fresh LLM the recall questions with ONLY the + post-compaction context, and judges answers against gold. +5. Emits a scorecard: recall accuracy vs tokens retained, per policy. + +## Usage + +```bash +# from repo root, venv active +python evals/compaction/runner.py \ + --transcript /path/to/lineage.json \ + --policies current,aggressive,floor10k \ + --questions 15 \ + --out evals/compaction/results/run1 +python evals/compaction/report.py evals/compaction/results/run1 +``` + +Transcripts are NOT committed (they contain real session data). Point +`--transcript` at a local file. See `fixtures.py` for the expected shape and +a synthetic-transcript generator used by CI smoke tests. + +## Policies + +Defined in `policies.py`. Each policy maps to `ContextCompressor` constructor +kwargs plus optional attribute overrides applied post-construction (e.g. +`tail_token_budget`). Add new policies there — the runner picks them up by +name. + +## Notes + +- Question generation and judging use `agent.auxiliary_client.call_llm` + (same transport the compressor uses), so the harness needs a configured + provider. Costs real tokens: ~(policies x questions) answer calls plus + one generation and one judge pass. +- Accuracy is judged 2/1/0 (correct / partial / wrong); the scorecard + reports normalized percent. The judge sees gold answers, the answerer + does not. +- `--also-uncompacted` adds a control arm that answers from the full + original transcript — the recall ceiling. diff --git a/evals/compaction/fixtures.py b/evals/compaction/fixtures.py new file mode 100644 index 0000000000..7c5b4e5a59 --- /dev/null +++ b/evals/compaction/fixtures.py @@ -0,0 +1,79 @@ +"""Transcript fixtures for the compaction eval harness. + +Real transcripts are supplied by path (never committed). This module loads +them, estimates tokens the same way the harness scores them, and can generate +a small synthetic transcript so CI smoke tests run without real data. +""" +from __future__ import annotations + +import json +import random +from typing import Any, Dict, List + + +def estimate_tokens(msg: Dict[str, Any]) -> int: + """Chars/4 estimate, matching the harness's scoring convention.""" + total = len(msg.get("content") or "") if isinstance(msg.get("content"), str) else 0 + tc = msg.get("tool_calls") + if tc: + total += len(json.dumps(tc, default=str)) + return total // 4 + + +def total_tokens(messages: List[Dict[str, Any]]) -> int: + return sum(estimate_tokens(m) for m in messages) + + +def load_transcript(path: str, cap_tokens: int | None = None) -> List[Dict[str, Any]]: + """Load a transcript JSON ({"messages": [...]}) and optionally cap it. + + The cap takes the chronological prefix, then drops trailing assistant + tool_calls whose results were cut off so the input is well-formed. + """ + data = json.load(open(path)) + msgs = data["messages"] if isinstance(data, dict) else data + if cap_tokens is None: + return msgs + prefix: List[Dict[str, Any]] = [] + running = 0 + for m in msgs: + t = estimate_tokens(m) + if running + t > cap_tokens and len(prefix) > 10: + break + prefix.append(m) + running += t + while prefix and prefix[-1].get("tool_calls"): + prefix.pop() + return prefix + + +def synthetic_transcript(n_turns: int = 60, seed: int = 7) -> List[Dict[str, Any]]: + """Deterministic fake transcript with plantable facts for smoke tests. + + Every 10th turn plants a distinctive fact ("The deploy code for region + N is XYZ") so smoke tests can assert recall mechanics without an LLM. + """ + rng = random.Random(seed) + msgs: List[Dict[str, Any]] = [ + {"role": "system", "content": "You are a test agent."}, + {"role": "user", "content": "Work through the checklist and remember the codes."}, + ] + for i in range(n_turns): + fact = "" + if i % 10 == 0: + fact = f" The deploy code for region {i // 10} is Z{rng.randint(1000, 9999)}." + msgs.append({ + "role": "assistant", + "content": f"Working on step {i}.{fact}", + "tool_calls": [{ + "id": f"c{i}", + "function": {"name": "terminal", "arguments": json.dumps({"command": f"echo step {i}"})}, + }], + }) + msgs.append({ + "role": "tool", + "tool_call_id": f"c{i}", + "content": ("step output " * 200) + f"result-{i}", + }) + msgs.append({"role": "assistant", "content": "Checklist complete."}) + return msgs diff --git a/evals/compaction/policies.py b/evals/compaction/policies.py new file mode 100644 index 0000000000..e360d68b9b --- /dev/null +++ b/evals/compaction/policies.py @@ -0,0 +1,48 @@ +"""Compaction policy matrix. + +Each policy is a name -> spec mapping. A spec has: + ctor: extra kwargs for ContextCompressor(...) + attrs: attribute overrides applied after construction (lets us pin + tail_token_budget and other derived values without touching the + class) +The runner constructs one compressor per policy and calls +compress(force=True) with the transcript's estimated tokens. +""" +from __future__ import annotations + +from typing import Any, Dict + +# Window we evaluate against (fable-5 class model). +EVAL_MODEL = "anthropic/claude-fable-5" +EVAL_WINDOW = 1_000_000 + +POLICIES: Dict[str, Dict[str, Any]] = { + # Shipping behavior, untouched. + "current": { + "ctor": {}, + "attrs": {}, + }, + # Proposed: tail = max(10K, 0.025% ... interpreted as 2.5% of window) + # capped hard at 25K on a 1M model. protect_last_n stays for message-count + # floor semantics. + "tail25k": { + "ctor": {}, + "attrs": {"tail_token_budget": 25_000}, + }, + # Hard floor variant: minimum viable tail. + "tail10k": { + "ctor": {}, + "attrs": {"tail_token_budget": 10_000}, + }, + # Codex posture: nearly no tail; summary carries everything. + "codex_style": { + "ctor": {"protect_last_n": 3}, + "attrs": {"tail_token_budget": 2_000}, + }, +} + + +def apply_policy(compressor, spec: Dict[str, Any]): + for key, value in (spec.get("attrs") or {}).items(): + setattr(compressor, key, value) + return compressor diff --git a/evals/compaction/report.py b/evals/compaction/report.py new file mode 100644 index 0000000000..c80efb788a --- /dev/null +++ b/evals/compaction/report.py @@ -0,0 +1,43 @@ +"""Render a compaction-eval scorecard as a terminal table + markdown. + +Usage: python evals/compaction/report.py +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + + +def main(): + out_dir = Path(sys.argv[1]) + card = json.loads((out_dir / "scorecard.json").read_text()) + card.sort(key=lambda s: -s["recall_pct"]) + + rows = [] + for s in card: + before = s.get("before_tokens", 0) + after = s.get("after_tokens", 0) + kept = f"{100 * after / before:.1f}%" if before else "?" + rows.append(( + s["policy"], f"{s['recall_pct']}%", f"{before:,}", f"{after:,}", kept, + str(s.get("compress_seconds", "-")), + )) + + headers = ("policy", "recall", "tokens before", "tokens after", "kept", "sec") + widths = [max(len(headers[i]), *(len(r[i]) for r in rows)) for i in range(len(headers))] + line = " ".join(h.ljust(widths[i]) for i, h in enumerate(headers)) + print(line) + print("-" * len(line)) + for r in rows: + print(" ".join(str(r[i]).ljust(widths[i]) for i in range(len(headers)))) + + md = ["| " + " | ".join(headers) + " |", "|" + "|".join("---" for _ in headers) + "|"] + for r in rows: + md.append("| " + " | ".join(r) + " |") + (out_dir / "scorecard.md").write_text("\n".join(md) + "\n") + print(f"\nmarkdown -> {out_dir}/scorecard.md") + + +if __name__ == "__main__": + main() diff --git a/evals/compaction/runner.py b/evals/compaction/runner.py new file mode 100644 index 0000000000..7da134e049 --- /dev/null +++ b/evals/compaction/runner.py @@ -0,0 +1,228 @@ +"""Compaction eval runner. + +Pipeline per transcript: + 1. Load + cap the transcript. + 2. Generate (or load cached) recall questions from the region that will be + summarized away under the CURRENT policy (the most conservative boundary: + anything the current policy summarizes is fair game for every policy). + 3. For each policy: compress, then answer each question with ONLY the + compressed context, using a single LLM call per question. + 4. Judge answers against gold with an LLM judge (sees gold; answerer + does not). + 5. Write per-policy results JSON for report.py. + +Run from repo root with the project venv (needs a configured provider). +""" +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import re +import sys +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) + +from evals.compaction.fixtures import ( # noqa: E402 + estimate_tokens, + load_transcript, + total_tokens, +) +from evals.compaction.policies import EVAL_MODEL, POLICIES, apply_policy # noqa: E402 + +QUESTION_PROMPT = """You are building a factual recall exam from an AI-agent work session transcript. + +Write {n} questions that test SPECIFIC, VERIFIABLE facts from the transcript below: identifiers (PR numbers, file paths, error messages, commit subjects), decisions and their reasons, user instructions, and outcomes. Rules: +- Every answer must appear literally in the transcript. +- No questions about the system prompt or generic behavior. +- Spread questions across the WHOLE span (early, middle, late). +- Prefer facts that matter for continuing the work (what was decided, what failed, what the user asked for). + +Return STRICT JSON: a list of {{"q": "...", "gold": "...", "where": ""}}. + +TRANSCRIPT: +{transcript} +""" + +ANSWER_PROMPT = """You are an AI agent resuming a work session. Below is your CURRENT conversation context (it may include a compaction summary of earlier work). Answer the question using ONLY this context. If the context does not contain the answer, say exactly "NOT IN CONTEXT" and give your best guess after a semicolon. + +CONTEXT: +{context} + +QUESTION: {question} + +Answer in one or two sentences.""" + +JUDGE_PROMPT = """Score this answer against the gold answer. Reply with STRICT JSON: {{"score": 2|1|0, "why": "..."}}. +2 = factually matches gold (wording may differ) +1 = partially correct or hedged-but-right ("NOT IN CONTEXT; guess X" where X is right scores 1) +0 = wrong, or "NOT IN CONTEXT" with a wrong/no guess + +QUESTION: {question} +GOLD: {gold} +ANSWER: {answer}""" + + +def _call(prompt: str, max_tokens: int = 2000) -> str: + from agent.auxiliary_client import call_llm + + resp = call_llm( + messages=[{"role": "user", "content": prompt}], + task="compression", + max_tokens=max_tokens, + ) + if hasattr(resp, "choices"): + return resp.choices[0].message.content or "" + return str(resp) + + +def _extract_json(text: str): + m = re.search(r"```(?:json)?\s*(.*?)```", text, re.S) + if m: + text = m.group(1) + start = min([i for i in (text.find("["), text.find("{")) if i >= 0], default=0) + return json.loads(text[start:]) + + +def serialize_for_exam(messages, char_cap: int = 600_000) -> str: + parts = [] + for m in messages: + role = m.get("role") + c = m.get("content") + if not isinstance(c, str) or not c: + continue + if role == "system": + continue + parts.append(f"[{role}] {c}") + text = "\n\n".join(parts) + if len(text) > char_cap: + half = char_cap // 2 + text = text[:half] + "\n\n...[middle elided for exam generation]...\n\n" + text[-half:] + return text + + +def summarized_region(compressor_module, messages): + """The middle region the current policy would summarize: everything + between the protected head and the tail cut. Questions come from here.""" + from agent.context_compressor import ContextCompressor + + comp = ContextCompressor(model=EVAL_MODEL, quiet_mode=True) + head_end = comp.protect_first_n + tail_start = comp._find_tail_cut_by_tokens(messages, head_end) + return messages[head_end:tail_start] + + +def generate_questions(messages, n: int, cache_path: Path) -> list: + if cache_path.exists(): + return json.loads(cache_path.read_text()) + import agent.context_compressor as cc + + region = summarized_region(cc, messages) + text = serialize_for_exam(region) + raw = _call(QUESTION_PROMPT.format(n=n, transcript=text), max_tokens=4000) + questions = _extract_json(raw)[:n] + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text(json.dumps(questions, indent=1)) + return questions + + +def run_policy(name: str, spec: dict, messages, questions, out_dir: Path) -> dict: + from agent.context_compressor import ContextCompressor + + before = copy.deepcopy(messages) + comp = apply_policy(ContextCompressor(model=EVAL_MODEL, quiet_mode=True), spec) + for key, value in (spec.get("ctor") or {}).items(): + setattr(comp, key, value) + t0 = time.time() + compressed = comp.compress(copy.deepcopy(messages), current_tokens=total_tokens(messages), force=True) + elapsed = time.time() - t0 + + context_text = serialize_for_exam(compressed, char_cap=700_000) + results = [] + for qa in questions: + answer = _call(ANSWER_PROMPT.format(context=context_text, question=qa["q"]), max_tokens=400) + verdict_raw = _call(JUDGE_PROMPT.format(question=qa["q"], gold=qa["gold"], answer=answer), max_tokens=300) + try: + verdict = _extract_json(verdict_raw) + except Exception: + verdict = {"score": 0, "why": f"judge parse failure: {verdict_raw[:100]}"} + results.append({"q": qa["q"], "gold": qa["gold"], "answer": answer, **verdict}) + + scored = [r["score"] for r in results] + summary = { + "policy": name, + "before_tokens": total_tokens(before), + "after_tokens": total_tokens(compressed), + "after_msgs": len(compressed), + "compress_seconds": round(elapsed, 1), + "recall_pct": round(100 * sum(scored) / (2 * len(scored)), 1) if scored else 0.0, + "scores": scored, + "summary_error": getattr(comp, "_last_summary_error", None), + } + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / f"{name}.json").write_text(json.dumps({"summary": summary, "results": results}, indent=1)) + return summary + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--transcript", required=True) + ap.add_argument("--cap-tokens", type=int, default=500_000) + ap.add_argument("--policies", default="current,tail25k,codex_style") + ap.add_argument("--questions", type=int, default=15) + ap.add_argument("--out", required=True) + ap.add_argument("--also-uncompacted", action="store_true") + args = ap.parse_args() + + messages = load_transcript(args.transcript, cap_tokens=args.cap_tokens) + out_dir = Path(args.out) + tid = hashlib.md5(args.transcript.encode()).hexdigest()[:10] + qcache = out_dir / f"questions-{tid}.json" + questions = generate_questions(messages, args.questions, qcache) + print(f"{len(questions)} questions ready ({qcache})") + + summaries = [] + if args.also_uncompacted: + spec = {"ctor": {}, "attrs": {"tail_token_budget": 10**9}} + # control: no compression at all — answer from the full transcript + context_text = serialize_for_exam(messages, char_cap=900_000) + results = [] + for qa in questions: + answer = _call(ANSWER_PROMPT.format(context=context_text, question=qa["q"]), max_tokens=400) + verdict_raw = _call(JUDGE_PROMPT.format(question=qa["q"], gold=qa["gold"], answer=answer), max_tokens=300) + try: + verdict = _extract_json(verdict_raw) + except Exception: + verdict = {"score": 0, "why": "judge parse failure"} + results.append({"q": qa["q"], **verdict, "answer": answer}) + scored = [r["score"] for r in results] + ctl = { + "policy": "uncompacted_control", + "before_tokens": total_tokens(messages), + "after_tokens": total_tokens(messages), + "recall_pct": round(100 * sum(scored) / (2 * len(scored)), 1), + "scores": scored, + } + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "uncompacted_control.json").write_text(json.dumps({"summary": ctl, "results": results}, indent=1)) + summaries.append(ctl) + print(json.dumps(ctl, indent=1)) + + for name in args.policies.split(","): + name = name.strip() + if name not in POLICIES: + print(f"unknown policy {name}, skipping"); continue + s = run_policy(name, POLICIES[name], messages, questions, out_dir) + summaries.append(s) + print(json.dumps(s, indent=1)) + + (out_dir / "scorecard.json").write_text(json.dumps(summaries, indent=1)) + print(f"\nscorecard -> {out_dir}/scorecard.json") + + +if __name__ == "__main__": + main()