From 43c79cd84a7dcba6172d06f56f5af2f6fb08af4f Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 25 Jul 2026 19:00:24 -0700 Subject: [PATCH] feat(skills): add grounded-citations skill for verifiable sourcing Answers and written deliverables that rest on retrieved information now get inline numbered citations plus a mechanically-rendered Sources list, with a persistent ledger that makes a hallucinated citation detectable. - skills/research/grounded-citations/scripts/sources.py: stdlib citation ledger (add/ingest/list/render/verify) at $HERMES_HOME/cache/citations/ledger.json, profile-aware, O_EXCL-locked so parallel subagents sharing a ledger can't collide on ids - SKILL.md: cite-while-drafting procedure, register-at-retrieval rule, pitfalls, verification gate - references/citation-formats.md: per-target placement (markdown, LaTeX/PDF, docx, pptx, xlsx, wiki, BibTeX handoff to research-paper-writing) - references/grounding-rationale.md: why numbered ids (ALCE 2305.14627, WebGPT 2112.09332, Perplexity marker conventions), and how this relates to the in-process registry in PR #44833 - tests/skills/test_grounded_citations_skill.py: 30 tests --- skills/research/grounded-citations/SKILL.md | 153 +++++ .../references/citation-formats.md | 65 +++ .../references/grounding-rationale.md | 64 +++ .../scripts/_hermes_home.py | 23 + .../grounded-citations/scripts/sources.py | 524 ++++++++++++++++++ tests/skills/test_grounded_citations_skill.py | 339 +++++++++++ website/docs/reference/skills-catalog.md | 1 + .../research/research-grounded-citations.md | 170 ++++++ website/sidebars.ts | 1 + 9 files changed, 1340 insertions(+) create mode 100644 skills/research/grounded-citations/SKILL.md create mode 100644 skills/research/grounded-citations/references/citation-formats.md create mode 100644 skills/research/grounded-citations/references/grounding-rationale.md create mode 100644 skills/research/grounded-citations/scripts/_hermes_home.py create mode 100644 skills/research/grounded-citations/scripts/sources.py create mode 100644 tests/skills/test_grounded_citations_skill.py create mode 100644 website/docs/user-guide/skills/bundled/research/research-grounded-citations.md diff --git a/skills/research/grounded-citations/SKILL.md b/skills/research/grounded-citations/SKILL.md new file mode 100644 index 0000000000..a73c901ff5 --- /dev/null +++ b/skills/research/grounded-citations/SKILL.md @@ -0,0 +1,153 @@ +--- +name: grounded-citations +description: "Ground answers and documents in cited, verifiable sources." +version: 1.0.0 +author: Hermes Agent + Teknium +license: MIT +platforms: [linux, macos, windows] +metadata: + hermes: + tags: [Research, Citations, Grounding, Sources, Web, Reports] + category: research + related_skills: [research-paper-writing, arxiv, ocr-and-documents] +--- + +# Grounded Citations + +Every claim taken from an outside source gets an inline numbered citation and a +`Sources:` list, Perplexity-style. A ledger script owns the `url → [n]` mapping +so the numbers and URLs come from retrieval, never from memory — the model only +ever emits small integers it was handed. + +This skill covers answers in chat, written documents (markdown, PDF, docx, +slides), and research reports. It does not cover academic BibTeX pipelines — +for conference papers use the `research-paper-writing` skill, which this skill +feeds (see `references/citation-formats.md`). + +## When to Use + +Use whenever an answer or artifact rests on information you fetched rather than +knew: + +- Research, comparisons, news summaries, "what is the current state of X" +- Any deliverable you write to disk that quotes, paraphrases, or reports + outside facts — reports, briefs, docs, decks, wiki pages +- Fact-finding where the user will want to check your work +- Multi-source synthesis where conflicting sources must be attributed + +Skip inline citations when the retrieval is incidental to another task — a +quick syntax/version lookup mid-coding, casual conversation, creative writing. +Mention a URL only if the user would plausibly want the link. + +## Prerequisites + +None beyond the standard toolset. `scripts/sources.py` is stdlib-only Python 3. +Retrieval comes from whatever is configured: `web_search`, `web_extract`, +`browser_navigate`, or `terminal` (curl, CLIs). + +Ledger location: `$HERMES_HOME/cache/citations/ledger.json` (profile-aware). +Override per task with `--ledger ` or `HERMES_CITATION_LEDGER`. + +## How to Run + +```bash +S=~/.hermes/skills/research/grounded-citations/scripts/sources.py + +python3 "$S" reset # start a clean ledger +python3 "$S" add https://example.com/a --title "A" # prints: [1] +python3 "$S" add https://example.com/b --title "B" # prints: [2] +python3 "$S" list # ledger table +python3 "$S" render # Sources: block +python3 "$S" verify draft.md # catch bad citations +``` + +`add` is idempotent and URL-normalized: the same page always returns the same +id within a ledger, so ids stay stable across many search/extract rounds. + +## Quick Reference + +| Action | Command | +|---|---| +| Fresh ledger for a new task | `sources.py reset` | +| Register a source, get its id | `sources.py add [--title T]` | +| Register several at once | `sources.py add ...` | +| Register from JSON tool output | `sources.py ingest results.json` | +| Show ledger | `sources.py list [--json]` | +| Render the Sources block | `sources.py render [--style markdown\|plain\|footnotes\|bibtex] [--only 1,3]` | +| Render only what a draft cites | `sources.py render --cited-in draft.md` | +| Check a draft's citations | `sources.py verify draft.md [--strict] [--min-coverage 0.6]` | + +## Procedure + +① **Reset the ledger** at the start of a task that will produce a grounded +answer or document. Skip the reset when continuing work whose ids are already +in a draft — reusing the ledger keeps the numbering stable. + +② **Register every source at retrieval time.** After each `web_search` / +`web_extract` / `browser_navigate` / fetch, pass the URLs to `sources.py add` +(or pipe the raw JSON through `sources.py ingest`). Do this *before* writing +prose. Registering later, from memory, is the failure mode this skill exists to +prevent. + +③ **Write cite-while-drafting.** Place the bracketed id(s) immediately after +each sentence the source supports: + +``` +Ice floats because it is less dense than liquid water.[1][2] +``` + +- No space before the bracket; each id in its own brackets. +- Max 3 ids per sentence. Cite per sentence, not one dump at the end. +- Only ids the ledger returned. Never invent an id or a URL. +- Claims from your own knowledge get no citation. +- Conflicting sources: present both readings, each with its own id. +- Quote exact figures, dates, and names as the source states them; flag gaps + explicitly ("no source found for X") instead of smoothing them over. + +④ **Append the Sources block** with `sources.py render --cited-in ` so +the id → URL mapping is generated mechanically from the ledger, not retyped. +For non-markdown targets pick the matching `--style` and follow +`references/citation-formats.md` for placement (footnotes in docx, endnotes in +PDF/LaTeX, a Sources slide in decks, per-page source lists in wiki output). + +⑤ **Verify before delivering** — `sources.py verify ` exits non-zero on +unknown ids, on a Sources block that disagrees with the ledger, or (with +`--min-coverage`) on prose that is too thinly cited. Fix and re-run. + +⑥ **Chat answers** follow the same steps with the draft in your reply: register +sources, cite inline, end with the rendered `Sources:` list. For a short answer +you may render the block from `sources.py render --only ` instead of +writing to a file. + +## Pitfalls + +- **Registering after writing.** The ledger must be populated from tool output, + not reconstructed from the draft — that reintroduces exactly the hallucinated + -URL risk the numbering removes. +- **Renumbering mid-task.** Never hand-edit ids in a draft. Ids are ledger + identities; if a draft cites `[4]`, `[4]` must stay that source. Run `reset` + only between tasks. +- **Retyping URLs into the Sources block.** Always `render`. A hand-typed URL + is an unverified claim. +- **Citing a search snippet as if you read the page.** A `web_search` + description supports only what it literally says. Cite the extracted page + when the claim needs the body — `web_extract` it first. +- **Over-citing.** Three ids on a sentence is the ceiling; a citation on every + clause makes text unreadable and hides which source carries the load. +- **Citing the ledger in code/config artifacts.** Source comments belong in + prose deliverables and doc headers, not inside generated code. +- **Parallel subagents.** Each subagent has its own working directory; point + them all at one ledger with `--ledger` (or `HERMES_CITATION_LEDGER`) if their + outputs get merged, otherwise their ids will collide. + +## Verification + +```bash +python3 "$S" verify report.md --strict --min-coverage 0.5 +``` + +Green means: every `[n]` in the draft exists in the ledger, the Sources block +lists exactly the cited ids with the ledger's URLs, and the cited share of +source-bearing sentences meets the threshold. Read the warnings even when the +exit code is 0 — uncited registered sources usually mean a claim lost its +attribution during editing. diff --git a/skills/research/grounded-citations/references/citation-formats.md b/skills/research/grounded-citations/references/citation-formats.md new file mode 100644 index 0000000000..d83e270bf0 --- /dev/null +++ b/skills/research/grounded-citations/references/citation-formats.md @@ -0,0 +1,65 @@ +# Citation formats per output target + +The ledger is format-agnostic: `sources.py render --style ...` emits the block, +this file says where it goes and what the inline marker looks like. + +## Markdown / chat answers + +Inline `[n]` immediately after the sentence. Block at the end: + +``` +## Sources + +[1] https://example.com/a — Page title +[2] https://example.com/b +``` + +`--style plain` gives a bare `Sources:` header for chat replies where a +markdown heading would be noise. + +## PDF via LaTeX (`latex-pdf-report` skill) + +Use `--style footnotes` and map each id to `\footnote{}` at first use, or keep +numeric markers and emit an endnotes section. For a bibliography-shaped report, +`--style bibtex` writes `@misc` entries keyed `source`; cite them with +`\cite{source3}` and let BibTeX render the list. + +Do not mix: either numeric `[n]` + a Sources section, or `\cite{}` + BibTeX. +Two numbering systems in one document is worse than none. + +## Word (.docx, `docx` skill) + +Real footnotes are preferred over inline brackets in prose documents intended +for human editing — reviewers expect Word footnotes. Keep the ledger ids as the +footnote numbers so `verify` still works on a markdown source-of-truth, and +generate the .docx from that markdown. + +## Slides (.pptx, `powerpoint` skill) + +Inline `[n]` in the bullet, one "Sources" slide at the end rendered with +`--style plain`. Never put a URL in a body bullet — it wrecks the layout and +can't be clicked in a projected deck. + +## Spreadsheets (.xlsx) + +Add a `source` column holding the id, plus a `Sources` sheet built from +`render --style plain`. Do not paste URLs into data cells. + +## Wiki / multi-page output (`llm-wiki`, Obsidian) + +Per-page Sources block, ids shared across pages from one ledger. Because ids +are ledger identities, `[7]` means the same page everywhere in the wiki — that +consistency is the reason not to reset the ledger between pages of one build. + +## Research papers + +Hand off to the `research-paper-writing` skill. Export with +`--style bibtex` into `references.bib`, then follow that skill's citation +verification (it greps `\cite{...}` against the .bib). The ledger's job ends at +producing verified URL entries; venue formatting is that skill's domain. + +## Code and config artifacts + +No citations inside generated code. If provenance matters, put it in the +commit message, the PR body, or a doc header — not in comments scattered +through source. diff --git a/skills/research/grounded-citations/references/grounding-rationale.md b/skills/research/grounded-citations/references/grounding-rationale.md new file mode 100644 index 0000000000..7c25d6b29a --- /dev/null +++ b/skills/research/grounded-citations/references/grounding-rationale.md @@ -0,0 +1,64 @@ +# Why numbered ledger ids (grounding research basis) + +Design notes for anyone changing the citation instructions or the ledger +mechanics. The wording in SKILL.md is not arbitrary. + +## The structural trick + +Hallucinated citations happen when a model reconstructs a URL from memory. If +the only thing the model has to emit is a small integer it was handed at +retrieval time, there is nothing to reconstruct — a wrong id is detectable +(it's not in the ledger) and a wrong URL is impossible (the model never types +one; `render` does). This is the property Perplexity's product relies on, and +it's why the ledger, not the prose, owns the URL. + +Consequence: **register at retrieval, render mechanically.** Any workflow that +lets the model type a URL into the Sources block gives the guarantee back. + +## Cite while writing, not after + +ALCE (arXiv:2305.14627) evaluates attribution for LLM answers and finds that +generating citations during composition, from numbered retrieved snippets, +produces materially better attribution than post-hoc citation insertion. +Post-hoc attribution invites the model to find a source that plausibly matches +a sentence it already wrote — which is exactly how a citation ends up +supporting something the page doesn't say. + +Hence: cite per supported sentence, in-line, as the sentence is written. Never +a citation dump at the end of a paragraph or document. + +## Verbatim quotes ground claims + +WebGPT (arXiv:2112.09332) collects verbatim quotes at browse time and composes +answers from them. The practical rule for this skill: when a claim carries a +figure, date, name, or quantity, take it from the source's own words rather +than paraphrasing from a summary of a summary. Each summarization hop is a +chance for a number to drift. + +## Formatting conventions + +From Perplexity's leaked system prompts (jujumilk3/leaked-system-prompts) — the +conventions are worth copying because they're the ones users have been trained +to read: + +- Marker directly after the terminal punctuation, no space: `water.[1]` +- Each id in its own brackets: `[1][2]`, not `[1, 2]` +- At most 3 ids per sentence — beyond that the citation stops identifying which + source carries the claim +- Never cite a source not actually consulted +- Query classes that shouldn't be cited (translation, creative writing, casual + chat) are exempted by instruction, not by a separate classifier + +Perplexity forbids raw URLs in the answer because its UI renders source cards. +Hermes has no such UI layer in chat or in a written file, so this skill renders +the id → URL list explicitly instead. + +## Related in-tree implementation + +`tools/web_tools.py` grew an in-process version of this idea (a `url -> [n]` +registry plus citation guidance attached to tool results) in PR #44833. That +path grounds ad-hoc web answers automatically when it lands. This skill is the +portable half: it works with any retrieval source (browser, curl, CLIs, local +PDFs) and it persists the ledger to disk so multi-turn, multi-file, and +multi-subagent work keeps stable ids. The two can coexist — the ledger is the +source of truth for anything written to a file. diff --git a/skills/research/grounded-citations/scripts/_hermes_home.py b/skills/research/grounded-citations/scripts/_hermes_home.py new file mode 100644 index 0000000000..45ed49d377 --- /dev/null +++ b/skills/research/grounded-citations/scripts/_hermes_home.py @@ -0,0 +1,23 @@ +"""Resolve HERMES_HOME for standalone skill scripts. + +Skill scripts may run outside the Hermes process (system Python, nix env, +CI) where ``hermes_constants`` is not importable. This module provides the +same ``get_hermes_home()`` contract without requiring it on ``sys.path``. + +When ``hermes_constants`` IS available it is used directly so profile +resolution and any future enhancements are picked up automatically. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +try: + from hermes_constants import get_hermes_home as get_hermes_home +except (ModuleNotFoundError, ImportError): + + def get_hermes_home() -> Path: + """Return the Hermes home directory (default: ``~/.hermes``).""" + val = os.environ.get("HERMES_HOME", "").strip() + return Path(val) if val else Path.home() / ".hermes" diff --git a/skills/research/grounded-citations/scripts/sources.py b/skills/research/grounded-citations/scripts/sources.py new file mode 100644 index 0000000000..a80e2dda0d --- /dev/null +++ b/skills/research/grounded-citations/scripts/sources.py @@ -0,0 +1,524 @@ +#!/usr/bin/env python3 +"""Citation ledger for grounded answers and documents. + +Owns the ``url -> [n]`` mapping used by the ``grounded-citations`` skill. +Ids are assigned at retrieval time and never change, so a draft's ``[3]`` +always resolves to the same page. The model only ever emits integers the +ledger handed it, which is what makes the citations verifiable. + +Subcommands +----------- + reset start a clean ledger + add URL [URL ...] register source(s), print their ids + ingest FILE|- register every url found in JSON tool output + list show the ledger + render render a Sources block + verify DRAFT check a draft's citations against the ledger + +Ledger path resolution (first wins): + --ledger PATH + $HERMES_CITATION_LEDGER + $HERMES_HOME/cache/citations/ledger.json +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import time +from pathlib import Path +from typing import Any, Iterable + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from _hermes_home import get_hermes_home # noqa: E402 + +SCHEMA_VERSION = 1 + +# A citation marker in prose: [12]. Markdown links ([text](url)) and +# reference-style labels are excluded by requiring digits only and no +# following "(" or ":". +_CITE_RE = re.compile(r"\[(\d{1,4})\](?![(:])") +_SOURCES_HEADER_RE = re.compile(r"^\s*(?:#{1,6}\s*)?(?:\*\*)?sources:?(?:\*\*)?\s*$", re.IGNORECASE) +_SOURCE_LINE_RE = re.compile(r"^\s*\[(\d{1,4})\]\s*[-–:]?\s*(\S+)") +_URL_IN_TEXT_RE = re.compile(r"https?://[^\s\"'<>)\]}]+") +_FENCE_RE = re.compile(r"^\s*(?:```|~~~)") + + +# --------------------------------------------------------------------------- +# Ledger I/O +# --------------------------------------------------------------------------- + + +def resolve_ledger_path(explicit: str | None = None) -> Path: + if explicit: + return Path(explicit).expanduser() + env = os.environ.get("HERMES_CITATION_LEDGER", "").strip() + if env: + return Path(env).expanduser() + return get_hermes_home() / "cache" / "citations" / "ledger.json" + + +def normalize_url(url: str) -> str: + """Canonicalize a URL for ledger identity. + + Strips the fragment and a trailing slash so ``/page``, ``/page/`` and + ``/page#section`` are one source. Query strings are significant and are + kept — they usually select different content. + """ + u = (url or "").strip() + if "#" in u: + u = u.split("#", 1)[0] + stripped = u.rstrip("/") + return stripped or u + + +def load_ledger(path: Path) -> dict[str, Any]: + if not path.exists(): + return {"version": SCHEMA_VERSION, "sources": []} + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + raise SystemExit(f"error: ledger at {path} is unreadable ({exc}); run `reset` to start over") + if not isinstance(data, dict) or not isinstance(data.get("sources"), list): + raise SystemExit(f"error: ledger at {path} has an unexpected shape; run `reset`") + return data + + +def save_ledger(path: Path, data: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + f".tmp{os.getpid()}") + tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + os.replace(tmp, path) + + +class _LedgerLock: + """Best-effort cross-process lock (O_EXCL lockfile, stdlib only). + + Parallel subagents can share one ledger via --ledger; without a lock two + concurrent ``add`` calls can assign the same id. Falls through after the + timeout rather than blocking a task forever — a stale lock must never + wedge the ledger. + """ + + def __init__(self, path: Path, timeout: float = 5.0) -> None: + self.lock_path = path.with_suffix(path.suffix + ".lock") + self.timeout = timeout + self.fd: int | None = None + + def __enter__(self) -> "_LedgerLock": + self.lock_path.parent.mkdir(parents=True, exist_ok=True) + deadline = time.monotonic() + self.timeout + while True: + try: + self.fd = os.open(str(self.lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY) + return self + except FileExistsError: + if time.monotonic() >= deadline: + # Assume a stale lock from a crashed run. + try: + self.lock_path.unlink() + except OSError: + return self + continue + time.sleep(0.05) + + def __exit__(self, *_exc: object) -> None: + if self.fd is not None: + try: + os.close(self.fd) + except OSError: + pass + try: + self.lock_path.unlink() + except OSError: + pass + + +# --------------------------------------------------------------------------- +# Core operations +# --------------------------------------------------------------------------- + + +def _by_url(sources: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + return {s["url"]: s for s in sources} + + +def add_sources( + path: Path, + urls: Iterable[str], + title: str | None = None, + accessed: str | None = None, +) -> list[dict[str, Any]]: + """Register urls, returning their ledger entries (existing or new).""" + urls = [u for u in (str(u).strip() for u in urls) if u] + if not urls: + return [] + with _LedgerLock(path): + data = load_ledger(path) + sources = data["sources"] + index = _by_url(sources) + out: list[dict[str, Any]] = [] + changed = False + for raw in urls: + key = normalize_url(raw) + existing = index.get(key) + if existing is not None: + if title and not existing.get("title"): + existing["title"] = title + changed = True + out.append(existing) + continue + entry = { + "id": len(sources) + 1, + "url": key, + "title": (title or "").strip(), + "accessed": accessed or time.strftime("%Y-%m-%d"), + } + sources.append(entry) + index[key] = entry + out.append(entry) + changed = True + if changed: + save_ledger(path, data) + return out + + +def urls_from_json(payload: Any) -> list[tuple[str, str]]: + """Walk arbitrary JSON tool output collecting (url, title) pairs. + + Handles web_search (``data.web[]``), web_extract (``results[]``) and any + other nesting, in document order, deduped. + """ + found: list[tuple[str, str]] = [] + seen: set[str] = set() + + def walk(node: Any) -> None: + if isinstance(node, dict): + url = node.get("url") or node.get("link") or node.get("source_url") + if isinstance(url, str) and url.startswith(("http://", "https://")): + key = normalize_url(url) + if key not in seen: + seen.add(key) + raw_title = node.get("title") or node.get("name") or "" + found.append((url, raw_title if isinstance(raw_title, str) else "")) + for value in node.values(): + walk(value) + elif isinstance(node, list): + for item in node: + walk(item) + + walk(payload) + return found + + +def render_sources( + sources: list[dict[str, Any]], + style: str = "markdown", + only: set[int] | None = None, +) -> str: + picked = [s for s in sources if only is None or s["id"] in only] + picked.sort(key=lambda s: s["id"]) + if not picked: + return "" + lines: list[str] = [] + if style == "bibtex": + for s in picked: + key = f"source{s['id']}" + title = s.get("title") or s["url"] + lines.append( + "@misc{%s,\n title = {%s},\n howpublished = {\\url{%s}},\n note = {Accessed %s}\n}" + % (key, title, s["url"], s.get("accessed", "")) + ) + return "\n".join(lines) + if style == "footnotes": + for s in picked: + title = s.get("title") + suffix = f" — {title}" if title else "" + lines.append(f"[^{s['id']}]: {s['url']}{suffix}") + return "\n".join(lines) + header = "Sources:" if style == "plain" else "## Sources" + lines.append(header) + if style != "plain": + lines.append("") + for s in picked: + title = s.get("title") + suffix = f" — {title}" if title else "" + lines.append(f"[{s['id']}] {s['url']}{suffix}") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Verification +# --------------------------------------------------------------------------- + + +def _split_draft(text: str) -> tuple[str, dict[int, str]]: + """Split a draft into (prose, sources_block_map). + + The sources block is everything after the last ``Sources`` header; its + ``[n] url`` lines are parsed out so they aren't counted as prose citations. + Fenced code blocks are dropped from prose. + """ + lines = text.splitlines() + header_idx = -1 + for i, line in enumerate(lines): + if _SOURCES_HEADER_RE.match(line): + header_idx = i + listed: dict[int, str] = {} + if header_idx >= 0: + for line in lines[header_idx + 1:]: + m = _SOURCE_LINE_RE.match(line) + if m: + url_match = _URL_IN_TEXT_RE.search(line) + listed[int(m.group(1))] = url_match.group(0) if url_match else m.group(2) + body_lines = lines[:header_idx] + else: + body_lines = lines + prose: list[str] = [] + in_fence = False + for line in body_lines: + if _FENCE_RE.match(line): + in_fence = not in_fence + continue + if not in_fence: + prose.append(line) + return "\n".join(prose), listed + + +def _sentences(prose: str) -> list[str]: + """Rough sentence split over prose lines, skipping headings and tables.""" + out: list[str] = [] + for line in prose.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#") or stripped.startswith("|"): + continue + if stripped.startswith(">"): + stripped = stripped.lstrip("> ").strip() + for part in re.split(r"(?<=[.!?])\s+", stripped): + part = part.strip() + if len(part.split()) >= 4: + out.append(part) + return out + + +def verify_draft( + draft_path: Path, + sources: list[dict[str, Any]], + strict: bool = False, + min_coverage: float | None = None, +) -> tuple[int, list[str], list[str]]: + """Return (exit_code, errors, warnings).""" + text = draft_path.read_text(encoding="utf-8") + prose, listed = _split_draft(text) + by_id = {s["id"]: s for s in sources} + + errors: list[str] = [] + warnings: list[str] = [] + + cited = [int(m) for m in _CITE_RE.findall(prose)] + cited_set = set(cited) + + unknown = sorted(i for i in cited_set if i not in by_id) + if unknown: + errors.append( + "citations not in the ledger (hallucinated or renumbered): " + + ", ".join(f"[{i}]" for i in unknown) + ) + + if cited_set and not listed: + errors.append("draft cites sources but has no `Sources:` block — run `render --cited-in`") + + missing_from_block = sorted(cited_set - set(listed)) if listed else [] + if missing_from_block: + errors.append( + "cited but absent from the Sources block: " + + ", ".join(f"[{i}]" for i in missing_from_block) + ) + + for sid, url in sorted(listed.items()): + entry = by_id.get(sid) + if entry is None: + errors.append(f"Sources block lists [{sid}], which is not in the ledger") + continue + if normalize_url(url) != entry["url"]: + errors.append( + f"Sources block URL for [{sid}] does not match the ledger " + f"(block: {url} / ledger: {entry['url']}) — re-run `render`" + ) + + extra_in_block = sorted(set(listed) - cited_set) + if extra_in_block: + warnings.append( + "listed in Sources but never cited inline: " + + ", ".join(f"[{i}]" for i in extra_in_block) + ) + + registered_uncited = sorted(set(by_id) - cited_set) + if registered_uncited: + warnings.append( + "registered in the ledger but not cited in this draft: " + + ", ".join(f"[{i}]" for i in registered_uncited) + ) + + sentences = _sentences(prose) + cited_sentences = [s for s in sentences if _CITE_RE.search(s)] + coverage = (len(cited_sentences) / len(sentences)) if sentences else 0.0 + if min_coverage is not None and sentences and coverage < min_coverage: + errors.append( + f"citation coverage {coverage:.0%} is below the required {min_coverage:.0%} " + f"({len(cited_sentences)}/{len(sentences)} sentences cited)" + ) + + over_cited = [s for s in sentences if len(_CITE_RE.findall(s)) > 3] + if over_cited: + warnings.append(f"{len(over_cited)} sentence(s) carry more than 3 citations") + + code = 1 if errors else (1 if (strict and warnings) else 0) + stats = ( + f"{len(sentences)} prose sentence(s), {len(cited_sentences)} cited " + f"({coverage:.0%}), {len(cited_set)} distinct source(s) cited, " + f"{len(by_id)} in ledger" + ) + warnings.insert(0, f"stats: {stats}") + return code, errors, warnings + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _parse_only(spec: str | None) -> set[int] | None: + if not spec: + return None + out: set[int] = set() + for chunk in spec.replace(" ", "").split(","): + if not chunk: + continue + if "-" in chunk: + lo, _, hi = chunk.partition("-") + out.update(range(int(lo), int(hi) + 1)) + else: + out.add(int(chunk)) + return out + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="sources.py", description="Citation ledger for grounded answers and documents." + ) + parser.add_argument("--ledger", help="ledger file path (overrides env / default)") + sub = parser.add_subparsers(dest="cmd", required=True) + + sub.add_parser("reset", help="start a clean ledger") + + p_add = sub.add_parser("add", help="register source url(s), print their ids") + p_add.add_argument("urls", nargs="+") + p_add.add_argument("--title", help="title for the source (single-url calls)") + p_add.add_argument("--accessed", help="access date (default: today)") + p_add.add_argument("--json", action="store_true", help="emit JSON instead of ids") + + p_ing = sub.add_parser("ingest", help="register every url in JSON tool output") + p_ing.add_argument("file", help="JSON file, or - for stdin") + + p_list = sub.add_parser("list", help="show the ledger") + p_list.add_argument("--json", action="store_true") + + p_render = sub.add_parser("render", help="render a Sources block") + p_render.add_argument( + "--style", default="markdown", choices=["markdown", "plain", "footnotes", "bibtex"] + ) + p_render.add_argument("--only", help="ids to include, e.g. 1,3,5-7") + p_render.add_argument("--cited-in", help="include only ids cited in this draft file") + + p_ver = sub.add_parser("verify", help="check a draft's citations against the ledger") + p_ver.add_argument("draft") + p_ver.add_argument("--strict", action="store_true", help="treat warnings as failures") + p_ver.add_argument("--min-coverage", type=float, help="required cited-sentence share, e.g. 0.5") + + args = parser.parse_args(argv) + path = resolve_ledger_path(args.ledger) + + if args.cmd == "reset": + with _LedgerLock(path): + save_ledger(path, {"version": SCHEMA_VERSION, "sources": []}) + print(f"ledger reset: {path}") + return 0 + + if args.cmd == "add": + title = args.title if len(args.urls) == 1 else None + entries = add_sources(path, args.urls, title=title, accessed=args.accessed) + if args.json: + print(json.dumps(entries, indent=2, ensure_ascii=False)) + else: + for e in entries: + print(f"[{e['id']}] {e['url']}") + return 0 + + if args.cmd == "ingest": + raw = sys.stdin.read() if args.file == "-" else Path(args.file).read_text(encoding="utf-8") + try: + payload = json.loads(raw) + except json.JSONDecodeError as exc: + print(f"error: input is not valid JSON ({exc})", file=sys.stderr) + return 2 + pairs = urls_from_json(payload) + if not pairs: + print("no urls found in input", file=sys.stderr) + return 1 + for url, title in pairs: + entry = add_sources(path, [url], title=title or None)[0] + print(f"[{entry['id']}] {entry['url']}") + return 0 + + data = load_ledger(path) + sources = sorted(data["sources"], key=lambda s: s["id"]) + + if args.cmd == "list": + if args.json: + print(json.dumps(sources, indent=2, ensure_ascii=False)) + elif not sources: + print(f"ledger is empty: {path}") + else: + for s in sources: + title = f" {s['title']}" if s.get("title") else "" + print(f"[{s['id']}] {s['url']}{title}") + return 0 + + if args.cmd == "render": + only = _parse_only(args.only) + if args.cited_in: + draft = Path(args.cited_in).read_text(encoding="utf-8") + prose, _ = _split_draft(draft) + cited = {int(m) for m in _CITE_RE.findall(prose)} + only = cited if only is None else (only & cited) + block = render_sources(sources, style=args.style, only=only) + if not block: + print("no sources to render", file=sys.stderr) + return 1 + print(block) + return 0 + + if args.cmd == "verify": + draft_path = Path(args.draft) + if not draft_path.is_file(): + print(f"error: no such draft: {draft_path}", file=sys.stderr) + return 2 + code, errors, warnings = verify_draft( + draft_path, sources, strict=args.strict, min_coverage=args.min_coverage + ) + for w in warnings: + print(f"warn: {w}") + for e in errors: + print(f"FAIL: {e}", file=sys.stderr) + print("citations OK" if code == 0 else "verification failed") + return code + + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/skills/test_grounded_citations_skill.py b/tests/skills/test_grounded_citations_skill.py new file mode 100644 index 0000000000..08f597e588 --- /dev/null +++ b/tests/skills/test_grounded_citations_skill.py @@ -0,0 +1,339 @@ +"""Tests for the grounded-citations bundled skill. + +Covers the SKILL.md authoring standards (frontmatter shape, ≤60-char +description) and the behavior of ``scripts/sources.py`` — the citation ledger +that assigns stable ``url -> [n]`` ids, renders Sources blocks, and verifies a +draft's citations. The verify path is the load-bearing piece: it is what +catches a hallucinated or renumbered citation before delivery. +""" + +from __future__ import annotations + +import importlib.util +import json +import re +from pathlib import Path + +import pytest +import yaml + +SKILL_DIR = Path(__file__).resolve().parents[2] / "skills" / "research" / "grounded-citations" +SCRIPT = SKILL_DIR / "scripts" / "sources.py" + + +@pytest.fixture(scope="module") +def frontmatter() -> dict: + src = (SKILL_DIR / "SKILL.md").read_text(encoding="utf-8") + m = re.search(r"^---\n(.*?)\n---", src, re.DOTALL) + assert m, "SKILL.md missing YAML frontmatter" + return yaml.safe_load(m.group(1)) + + +@pytest.fixture(scope="module") +def sources_mod(): + spec = importlib.util.spec_from_file_location("gc_sources", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def ledger(tmp_path: Path) -> Path: + return tmp_path / "ledger.json" + + +# --------------------------------------------------------------------------- +# Authoring standards +# --------------------------------------------------------------------------- + + +def test_skill_files_present() -> None: + assert (SKILL_DIR / "SKILL.md").is_file() + assert SCRIPT.is_file() + assert (SKILL_DIR / "references" / "citation-formats.md").is_file() + assert (SKILL_DIR / "references" / "grounding-rationale.md").is_file() + + +def test_description_within_limit(frontmatter: dict) -> None: + desc = frontmatter["description"] + assert len(desc) <= 60, f"description is {len(desc)} chars (limit 60): {desc!r}" + assert desc.endswith(".") + + +def test_required_frontmatter_fields(frontmatter: dict) -> None: + assert frontmatter["name"] == "grounded-citations" + for field in ("version", "author", "license", "platforms"): + assert frontmatter.get(field), f"missing frontmatter field: {field}" + assert frontmatter["metadata"]["hermes"]["category"] == "research" + + +def test_skill_body_has_modern_sections() -> None: + body = (SKILL_DIR / "SKILL.md").read_text(encoding="utf-8") + for heading in ( + "## When to Use", + "## Prerequisites", + "## How to Run", + "## Quick Reference", + "## Procedure", + "## Pitfalls", + "## Verification", + ): + assert heading in body, f"SKILL.md missing section: {heading}" + + +# --------------------------------------------------------------------------- +# Ledger identity +# --------------------------------------------------------------------------- + + +def test_ids_are_stable_and_sequential(sources_mod, ledger: Path) -> None: + first = sources_mod.add_sources(ledger, ["https://a.example"]) + second = sources_mod.add_sources(ledger, ["https://b.example"]) + assert (first[0]["id"], second[0]["id"]) == (1, 2) + again = sources_mod.add_sources(ledger, ["https://a.example"]) + assert again[0]["id"] == 1 + + +def test_url_normalization_collapses_fragment_and_trailing_slash(sources_mod, ledger: Path) -> None: + base = sources_mod.add_sources(ledger, ["https://x.example/page"])[0]["id"] + for variant in ("https://x.example/page/", "https://x.example/page#part"): + assert sources_mod.add_sources(ledger, [variant])[0]["id"] == base + + +def test_query_string_is_significant(sources_mod, ledger: Path) -> None: + a = sources_mod.add_sources(ledger, ["https://x.example/s?q=1"])[0]["id"] + b = sources_mod.add_sources(ledger, ["https://x.example/s?q=2"])[0]["id"] + assert a != b + + +def test_title_backfills_without_changing_id(sources_mod, ledger: Path) -> None: + first = sources_mod.add_sources(ledger, ["https://t.example"])[0] + assert first["title"] == "" + second = sources_mod.add_sources(ledger, ["https://t.example"], title="Later title")[0] + assert (second["id"], second["title"]) == (first["id"], "Later title") + + +def test_ingest_walks_search_and_extract_payloads(sources_mod) -> None: + payload = { + "data": {"web": [{"title": "One", "url": "https://n.example/1"}]}, + "results": [ + {"url": "https://n.example/1", "title": "One again"}, + {"url": "https://n.example/2", "title": "Two"}, + ], + } + pairs = sources_mod.urls_from_json(payload) + assert [u for u, _ in pairs] == ["https://n.example/1", "https://n.example/2"] + + +def test_ingest_ignores_non_http_values(sources_mod) -> None: + payload = {"url": "file:///etc/passwd", "nested": {"link": "mailto:a@b.c"}} + assert sources_mod.urls_from_json(payload) == [] + + +# --------------------------------------------------------------------------- +# Rendering +# --------------------------------------------------------------------------- + + +def _seed(sources_mod, ledger: Path) -> list[dict]: + sources_mod.add_sources(ledger, ["https://a.example"], title="Alpha") + sources_mod.add_sources(ledger, ["https://b.example"]) + sources_mod.add_sources(ledger, ["https://c.example"], title="Gamma") + return json.loads(ledger.read_text(encoding="utf-8"))["sources"] + + +def test_render_markdown_lists_ids_and_urls(sources_mod, ledger: Path) -> None: + block = sources_mod.render_sources(_seed(sources_mod, ledger)) + assert block.startswith("## Sources") + assert "[1] https://a.example — Alpha" in block + assert "[2] https://b.example" in block + + +def test_render_only_subset_and_ordering(sources_mod, ledger: Path) -> None: + block = sources_mod.render_sources(_seed(sources_mod, ledger), style="plain", only={3, 1}) + lines = [ln for ln in block.splitlines() if ln.startswith("[")] + assert lines[0].startswith("[1]") and lines[1].startswith("[3]") + assert not any(ln.startswith("[2]") for ln in lines) + + +def test_render_bibtex_keys_match_ids(sources_mod, ledger: Path) -> None: + block = sources_mod.render_sources(_seed(sources_mod, ledger), style="bibtex", only={1}) + assert "@misc{source1," in block + assert r"\url{https://a.example}" in block + + +def test_render_empty_selection_is_empty_string(sources_mod, ledger: Path) -> None: + assert sources_mod.render_sources(_seed(sources_mod, ledger), only=set()) == "" + + +# --------------------------------------------------------------------------- +# Verification — the guarantee +# --------------------------------------------------------------------------- + + +def _verify(sources_mod, ledger: Path, tmp_path: Path, text: str, **kw): + draft = tmp_path / "draft.md" + draft.write_text(text, encoding="utf-8") + sources = json.loads(ledger.read_text(encoding="utf-8"))["sources"] + return sources_mod.verify_draft(draft, sources, **kw) + + +def test_well_formed_draft_passes(sources_mod, ledger: Path, tmp_path: Path) -> None: + _seed(sources_mod, ledger) + text = ( + "Ice is less dense than liquid water and floats.[1][2]\n\n" + "Sources:\n[1] https://a.example\n[2] https://b.example\n" + ) + code, errors, _ = _verify(sources_mod, ledger, tmp_path, text) + assert (code, errors) == (0, []) + + +def test_unknown_citation_id_fails(sources_mod, ledger: Path, tmp_path: Path) -> None: + _seed(sources_mod, ledger) + text = "A claim with an invented source id here.[42]\n\nSources:\n[42] https://fake.example\n" + code, errors, _ = _verify(sources_mod, ledger, tmp_path, text) + assert code == 1 + # Must be flagged as an inline citation the ledger never issued — not merely + # as a stray Sources-block line, which is a separate (weaker) error. + assert any("hallucinated or renumbered" in e for e in errors), errors + + +def test_sources_block_url_must_match_ledger(sources_mod, ledger: Path, tmp_path: Path) -> None: + _seed(sources_mod, ledger) + text = "A real claim carrying a real id.[1]\n\nSources:\n[1] https://wrong.example\n" + code, errors, _ = _verify(sources_mod, ledger, tmp_path, text) + assert code == 1 + assert any("does not match the ledger" in e for e in errors) + + +def test_missing_sources_block_fails(sources_mod, ledger: Path, tmp_path: Path) -> None: + _seed(sources_mod, ledger) + code, errors, _ = _verify(sources_mod, ledger, tmp_path, "A real claim carrying an id.[1]\n") + assert code == 1 + assert any("no `Sources:` block" in e for e in errors) + + +def test_cited_but_absent_from_block_fails(sources_mod, ledger: Path, tmp_path: Path) -> None: + _seed(sources_mod, ledger) + text = ( + "First claim about the topic at hand.[1]\n" + "Second claim about the topic at hand.[2]\n\n" + "Sources:\n[1] https://a.example\n" + ) + code, errors, _ = _verify(sources_mod, ledger, tmp_path, text) + assert code == 1 + assert any("absent from the Sources block" in e for e in errors) + + +def test_brackets_inside_code_fences_are_not_citations(sources_mod, ledger: Path, tmp_path: Path) -> None: + _seed(sources_mod, ledger) + text = "Prose with no external claims in it.\n\n```python\nvalue = arr[42]\n```\n" + code, errors, _ = _verify(sources_mod, ledger, tmp_path, text) + assert (code, errors) == (0, []) + + +def test_markdown_links_are_not_citations(sources_mod, ledger: Path, tmp_path: Path) -> None: + _seed(sources_mod, ledger) + text = "See the [docs](https://x.example) for the full option list.\n" + code, errors, _ = _verify(sources_mod, ledger, tmp_path, text) + assert (code, errors) == (0, []) + + +def test_min_coverage_gate(sources_mod, ledger: Path, tmp_path: Path) -> None: + _seed(sources_mod, ledger) + text = ( + "A cited claim about the subject matter.[1]\n" + "An uncited claim about the subject matter.\n" + "Another uncited claim about the subject matter.\n" + "A third uncited claim about the subject matter.\n\n" + "Sources:\n[1] https://a.example\n" + ) + ok_code, _, _ = _verify(sources_mod, ledger, tmp_path, text) + assert ok_code == 0 + code, errors, _ = _verify(sources_mod, ledger, tmp_path, text, min_coverage=0.5) + assert code == 1 + assert any("coverage" in e for e in errors) + + +def test_over_citation_warns_without_failing(sources_mod, ledger: Path, tmp_path: Path) -> None: + sources_mod.add_sources( + ledger, [f"https://s{i}.example" for i in range(1, 5)] + ) + text = ( + "One sentence leaning on far too many sources at once.[1][2][3][4]\n\n" + "Sources:\n" + + "".join(f"[{i}] https://s{i}.example\n" for i in range(1, 5)) + ) + code, errors, warnings = _verify(sources_mod, ledger, tmp_path, text) + assert (code, errors) == (0, []) + assert any("more than 3 citations" in w for w in warnings) + + +def test_strict_mode_promotes_warnings_to_failure(sources_mod, ledger: Path, tmp_path: Path) -> None: + _seed(sources_mod, ledger) + text = "A cited claim about the subject.[1]\n\nSources:\n[1] https://a.example\n" + assert _verify(sources_mod, ledger, tmp_path, text)[0] == 0 + assert _verify(sources_mod, ledger, tmp_path, text, strict=True)[0] == 1 + + +# --------------------------------------------------------------------------- +# CLI surface +# --------------------------------------------------------------------------- + + +def test_cli_add_render_verify_roundtrip(sources_mod, tmp_path: Path, capsys) -> None: + ledger = tmp_path / "cli.json" + args = ["--ledger", str(ledger)] + assert sources_mod.main(args + ["add", "https://a.example", "https://b.example"]) == 0 + assert "[1] https://a.example" in capsys.readouterr().out + + draft = tmp_path / "d.md" + draft.write_text("Only the first source is used here.[1]\n", encoding="utf-8") + assert sources_mod.main(args + ["render", "--cited-in", str(draft)]) == 0 + block = capsys.readouterr().out + assert "[1] https://a.example" in block and "[2]" not in block + + with draft.open("a", encoding="utf-8") as fh: + fh.write("\n" + block) + assert sources_mod.main(args + ["verify", str(draft)]) == 0 + + +def test_cli_reset_empties_the_ledger(sources_mod, tmp_path: Path, capsys) -> None: + ledger = tmp_path / "r.json" + args = ["--ledger", str(ledger)] + sources_mod.main(args + ["add", "https://a.example"]) + capsys.readouterr() + assert sources_mod.main(args + ["reset"]) == 0 + capsys.readouterr() + assert sources_mod.main(args + ["add", "https://z.example"]) == 0 + assert "[1] https://z.example" in capsys.readouterr().out + + +def test_cli_verify_missing_draft_returns_2(sources_mod, tmp_path: Path) -> None: + ledger = tmp_path / "m.json" + code = sources_mod.main(["--ledger", str(ledger), "verify", str(tmp_path / "nope.md")]) + assert code == 2 + + +def test_cli_ledger_path_prefers_flag_over_env(sources_mod, tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("HERMES_CITATION_LEDGER", str(tmp_path / "env.json")) + flagged = tmp_path / "flag.json" + assert sources_mod.resolve_ledger_path(str(flagged)) == flagged + assert sources_mod.resolve_ledger_path(None) == tmp_path / "env.json" + + +def test_cli_ledger_path_defaults_under_hermes_home(sources_mod, tmp_path: Path, monkeypatch) -> None: + monkeypatch.delenv("HERMES_CITATION_LEDGER", raising=False) + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + path = sources_mod.resolve_ledger_path(None) + assert path.parts[-3:] == ("cache", "citations", "ledger.json") + assert str(tmp_path) in str(path) + + +def test_corrupt_ledger_raises_actionable_error(sources_mod, tmp_path: Path) -> None: + bad = tmp_path / "bad.json" + bad.write_text("not json", encoding="utf-8") + with pytest.raises(SystemExit) as exc: + sources_mod.load_ledger(bad) + assert "reset" in str(exc.value) diff --git a/website/docs/reference/skills-catalog.md b/website/docs/reference/skills-catalog.md index e826c20aae..9a06994454 100644 --- a/website/docs/reference/skills-catalog.md +++ b/website/docs/reference/skills-catalog.md @@ -115,6 +115,7 @@ If a skill is missing from this list but present in the repo, the catalog is reg |-------|-------------|------| | [`arxiv`](/docs/user-guide/skills/bundled/research/research-arxiv) | Search arXiv papers by keyword, author, category, or ID. | `research/arxiv` | | [`blogwatcher`](/docs/user-guide/skills/bundled/research/research-blogwatcher) | Monitor blogs and RSS/Atom feeds via blogwatcher-cli tool. | `research/blogwatcher` | +| [`grounded-citations`](/docs/user-guide/skills/bundled/research/research-grounded-citations) | Ground answers and documents in cited, verifiable sources. | `research/grounded-citations` | | [`llm-wiki`](/docs/user-guide/skills/bundled/research/research-llm-wiki) | Karpathy's LLM Wiki: build/query interlinked markdown KB. | `research/llm-wiki` | | [`polymarket`](/docs/user-guide/skills/bundled/research/research-polymarket) | Query Polymarket: markets, prices, orderbooks, history. | `research/polymarket` | | [`research-paper-writing`](/docs/user-guide/skills/bundled/research/research-research-paper-writing) | Write ML papers for NeurIPS/ICML/ICLR: design→submit. | `research/research-paper-writing` | diff --git a/website/docs/user-guide/skills/bundled/research/research-grounded-citations.md b/website/docs/user-guide/skills/bundled/research/research-grounded-citations.md new file mode 100644 index 0000000000..4bd8951d53 --- /dev/null +++ b/website/docs/user-guide/skills/bundled/research/research-grounded-citations.md @@ -0,0 +1,170 @@ +--- +title: "Grounded Citations — Ground answers and documents in cited, verifiable sources" +sidebar_label: "Grounded Citations" +description: "Ground answers and documents in cited, verifiable sources" +--- + +{/* This page is auto-generated from the skill's SKILL.md by website/scripts/generate-skill-docs.py. Edit the source SKILL.md, not this page. */} + +# Grounded Citations + +Ground answers and documents in cited, verifiable sources. + +## Skill metadata + +| | | +|---|---| +| Source | Bundled (installed by default) | +| Path | `skills/research/grounded-citations` | +| Version | `1.0.0` | +| Author | Hermes Agent + Teknium | +| License | MIT | +| Platforms | linux, macos, windows | +| Tags | `Research`, `Citations`, `Grounding`, `Sources`, `Web`, `Reports` | +| Related skills | [`research-paper-writing`](/docs/user-guide/skills/bundled/research/research-research-paper-writing), [`arxiv`](/docs/user-guide/skills/bundled/research/research-arxiv), [`ocr-and-documents`](/docs/user-guide/skills/bundled/productivity/productivity-ocr-and-documents) | + +## Reference: full SKILL.md + +:::info +The following is the complete skill definition that Hermes loads when this skill is triggered. This is what the agent sees as instructions when the skill is active. +::: + +# Grounded Citations + +Every claim taken from an outside source gets an inline numbered citation and a +`Sources:` list, Perplexity-style. A ledger script owns the `url → [n]` mapping +so the numbers and URLs come from retrieval, never from memory — the model only +ever emits small integers it was handed. + +This skill covers answers in chat, written documents (markdown, PDF, docx, +slides), and research reports. It does not cover academic BibTeX pipelines — +for conference papers use the `research-paper-writing` skill, which this skill +feeds (see `references/citation-formats.md`). + +## When to Use + +Use whenever an answer or artifact rests on information you fetched rather than +knew: + +- Research, comparisons, news summaries, "what is the current state of X" +- Any deliverable you write to disk that quotes, paraphrases, or reports + outside facts — reports, briefs, docs, decks, wiki pages +- Fact-finding where the user will want to check your work +- Multi-source synthesis where conflicting sources must be attributed + +Skip inline citations when the retrieval is incidental to another task — a +quick syntax/version lookup mid-coding, casual conversation, creative writing. +Mention a URL only if the user would plausibly want the link. + +## Prerequisites + +None beyond the standard toolset. `scripts/sources.py` is stdlib-only Python 3. +Retrieval comes from whatever is configured: `web_search`, `web_extract`, +`browser_navigate`, or `terminal` (curl, CLIs). + +Ledger location: `$HERMES_HOME/cache/citations/ledger.json` (profile-aware). +Override per task with `--ledger ` or `HERMES_CITATION_LEDGER`. + +## How to Run + +```bash +S=~/.hermes/skills/research/grounded-citations/scripts/sources.py + +python3 "$S" reset # start a clean ledger +python3 "$S" add https://example.com/a --title "A" # prints: [1] +python3 "$S" add https://example.com/b --title "B" # prints: [2] +python3 "$S" list # ledger table +python3 "$S" render # Sources: block +python3 "$S" verify draft.md # catch bad citations +``` + +`add` is idempotent and URL-normalized: the same page always returns the same +id within a ledger, so ids stay stable across many search/extract rounds. + +## Quick Reference + +| Action | Command | +|---|---| +| Fresh ledger for a new task | `sources.py reset` | +| Register a source, get its id | `sources.py add [--title T]` | +| Register several at once | `sources.py add ...` | +| Register from JSON tool output | `sources.py ingest results.json` | +| Show ledger | `sources.py list [--json]` | +| Render the Sources block | `sources.py render [--style markdown\|plain\|footnotes\|bibtex] [--only 1,3]` | +| Render only what a draft cites | `sources.py render --cited-in draft.md` | +| Check a draft's citations | `sources.py verify draft.md [--strict] [--min-coverage 0.6]` | + +## Procedure + +① **Reset the ledger** at the start of a task that will produce a grounded +answer or document. Skip the reset when continuing work whose ids are already +in a draft — reusing the ledger keeps the numbering stable. + +② **Register every source at retrieval time.** After each `web_search` / +`web_extract` / `browser_navigate` / fetch, pass the URLs to `sources.py add` +(or pipe the raw JSON through `sources.py ingest`). Do this *before* writing +prose. Registering later, from memory, is the failure mode this skill exists to +prevent. + +③ **Write cite-while-drafting.** Place the bracketed id(s) immediately after +each sentence the source supports: + +``` +Ice floats because it is less dense than liquid water.[1][2] +``` + +- No space before the bracket; each id in its own brackets. +- Max 3 ids per sentence. Cite per sentence, not one dump at the end. +- Only ids the ledger returned. Never invent an id or a URL. +- Claims from your own knowledge get no citation. +- Conflicting sources: present both readings, each with its own id. +- Quote exact figures, dates, and names as the source states them; flag gaps + explicitly ("no source found for X") instead of smoothing them over. + +④ **Append the Sources block** with `sources.py render --cited-in ` so +the id → URL mapping is generated mechanically from the ledger, not retyped. +For non-markdown targets pick the matching `--style` and follow +`references/citation-formats.md` for placement (footnotes in docx, endnotes in +PDF/LaTeX, a Sources slide in decks, per-page source lists in wiki output). + +⑤ **Verify before delivering** — `sources.py verify ` exits non-zero on +unknown ids, on a Sources block that disagrees with the ledger, or (with +`--min-coverage`) on prose that is too thinly cited. Fix and re-run. + +⑥ **Chat answers** follow the same steps with the draft in your reply: register +sources, cite inline, end with the rendered `Sources:` list. For a short answer +you may render the block from `sources.py render --only ` instead of +writing to a file. + +## Pitfalls + +- **Registering after writing.** The ledger must be populated from tool output, + not reconstructed from the draft — that reintroduces exactly the hallucinated + -URL risk the numbering removes. +- **Renumbering mid-task.** Never hand-edit ids in a draft. Ids are ledger + identities; if a draft cites `[4]`, `[4]` must stay that source. Run `reset` + only between tasks. +- **Retyping URLs into the Sources block.** Always `render`. A hand-typed URL + is an unverified claim. +- **Citing a search snippet as if you read the page.** A `web_search` + description supports only what it literally says. Cite the extracted page + when the claim needs the body — `web_extract` it first. +- **Over-citing.** Three ids on a sentence is the ceiling; a citation on every + clause makes text unreadable and hides which source carries the load. +- **Citing the ledger in code/config artifacts.** Source comments belong in + prose deliverables and doc headers, not inside generated code. +- **Parallel subagents.** Each subagent has its own working directory; point + them all at one ledger with `--ledger` (or `HERMES_CITATION_LEDGER`) if their + outputs get merged, otherwise their ids will collide. + +## Verification + +```bash +python3 "$S" verify report.md --strict --min-coverage 0.5 +``` + +Green means: every `[n]` in the draft exists in the ledger, the Sources block +lists exactly the cited ids with the ledger's URLs, and the cited share of +source-bearing sentences meets the threshold. Read the warnings even when the +exit code is 0 — uncited registered sources usually mean a claim lost its +attribution during editing. diff --git a/website/sidebars.ts b/website/sidebars.ts index 172dfc04bb..1af0efb48e 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -276,6 +276,7 @@ const sidebars: SidebarsConfig = { items: [ 'user-guide/skills/bundled/research/research-arxiv', 'user-guide/skills/bundled/research/research-blogwatcher', + 'user-guide/skills/bundled/research/research-grounded-citations', 'user-guide/skills/bundled/research/research-llm-wiki', 'user-guide/skills/bundled/research/research-polymarket', 'user-guide/skills/bundled/research/research-research-paper-writing',