diff --git a/datagen-config-examples/example_browser_tasks.jsonl b/datagen-config-examples/example_browser_tasks.jsonl deleted file mode 100644 index 04c2848c50..0000000000 --- a/datagen-config-examples/example_browser_tasks.jsonl +++ /dev/null @@ -1,5 +0,0 @@ -{"prompt": "Go to https://news.ycombinator.com and find the top 5 posts on the front page. For each post, get the title, URL, points, and number of comments. Return the results as a formatted summary."} -{"prompt": "Navigate to https://en.wikipedia.org/wiki/Hermes and extract the first paragraph of the article, the image caption, and the list of items in the infobox. Summarize what you find."} -{"prompt": "Go to https://github.com/trending and find the top 3 trending repositories today. For each repo, get the name, description, language, and star count. Write the results to a file called trending_repos.md."} -{"prompt": "Visit https://httpbin.org/forms/post and fill out the form with sample data (customer name: Jane Doe, size: Medium, topping: Bacon, delivery time: 12:00). Submit the form and report what the response page shows."} -{"prompt": "Navigate to https://books.toscrape.com, browse to the Travel category, find the highest-rated book, and extract its title, price, availability, and description."} diff --git a/datagen-config-examples/run_browser_tasks.sh b/datagen-config-examples/run_browser_tasks.sh deleted file mode 100755 index 1d62e3abba..0000000000 --- a/datagen-config-examples/run_browser_tasks.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -# ============================================================================= -# Example: Browser-Focused Data Generation -# ============================================================================= -# -# Generates tool-calling trajectories for browser automation tasks. -# The agent navigates websites, fills forms, extracts information, etc. -# -# Distribution: browser 97%, web 20%, vision 12%, terminal 15% -# -# Prerequisites: -# - OPENROUTER_API_KEY in ~/.hermes/.env -# - BROWSERBASE_API_KEY in ~/.hermes/.env (for browser tools) -# - A dataset JSONL file with one {"prompt": "..."} per line -# -# Usage: -# cd ~/.hermes/hermes-agent -# bash datagen-config-examples/run_browser_tasks.sh -# -# Output: data/browser_tasks_example/trajectories.jsonl -# ============================================================================= - -mkdir -p logs - -LOG_FILE="logs/browser_tasks_$(date +%Y%m%d_%H%M%S).log" -echo "📝 Logging to: $LOG_FILE" - -# Point to the example dataset in this directory -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" - -python batch_runner.py \ - --dataset_file="$SCRIPT_DIR/example_browser_tasks.jsonl" \ - --batch_size=5 \ - --run_name="browser_tasks_example" \ - --distribution="browser_tasks" \ - --model="anthropic/claude-sonnet-4" \ - --base_url="https://openrouter.ai/api/v1" \ - --num_workers=3 \ - --max_turns=30 \ - --ephemeral_system_prompt="You are an AI assistant with browser automation capabilities. Your primary task is to navigate and interact with web pages to accomplish user goals. - -IMPORTANT GUIDELINES: - -1. SEARCHING: Do NOT search directly on Google via the browser — they block automated searches. Use the web_search tool first to find URLs, then navigate to them with browser tools. - -2. COOKIE/PRIVACY DIALOGS: After navigating to a page, check for cookie consent or privacy popups. Dismiss them by clicking Accept/Close/OK before interacting with other elements. Take a fresh browser_snapshot afterward. - -3. HANDLING TIMEOUTS: If an action times out, the element may be blocked by an overlay. Take a new snapshot and look for dialogs to dismiss. If none, try an alternative approach or report the issue. - -4. GENERAL: Use browser tools to click, fill forms, and extract information. Use terminal for local file operations. Verify your actions and handle errors gracefully." \ - 2>&1 | tee "$LOG_FILE" - -echo "✅ Done. Log: $LOG_FILE" - -# ============================================================================= -# Common options you can add: -# -# --resume Resume from checkpoint if interrupted -# --verbose Enable detailed logging - -# --reasoning_disabled Disable model thinking/reasoning tokens -# --providers_allowed="anthropic,google" Restrict to specific providers -# --prefill_messages_file="configs/prefill.json" Few-shot priming -# ============================================================================= diff --git a/datagen-config-examples/trajectory_compression.yaml b/datagen-config-examples/trajectory_compression.yaml deleted file mode 100644 index c5b92a97c1..0000000000 --- a/datagen-config-examples/trajectory_compression.yaml +++ /dev/null @@ -1,101 +0,0 @@ -# Trajectory Compression Configuration -# -# Post-processes completed agent trajectories to fit within a target token budget. -# Compression preserves head/tail turns and summarizes middle content only as needed. - -# Tokenizer settings for accurate token counting -tokenizer: - # HuggingFace tokenizer name - name: "moonshotai/Kimi-K2-Thinking" - - # Trust remote code (required for some tokenizers) - trust_remote_code: true - -# Compression targets and behavior -compression: - # Target maximum tokens for compressed trajectory - target_max_tokens: 29000 - - # Target size for summary (in tokens) - # This is factored into calculations when determining what to compress - summary_target_tokens: 750 - -# Protected turns that should NEVER be compressed -protected_turns: - # Always protect the first system message (tool definitions) - first_system: true - - # Always protect the first human message (original request) - first_human: true - - # Always protect the first gpt message (initial response/tool_call) - first_gpt: true - - # Always protect the first tool response (result of first action) - first_tool: true - - # Always protect the last 2 complete turn pairs (gpt+tool or gpt only) - # This ensures the model's final actions and conclusions are preserved - last_n_turns: 4 - -# LLM settings for generating summaries (OpenRouter only) -summarization: - # Model to use for summarization (should be fast and cheap) - # Using OpenRouter model path format - model: "google/gemini-3-flash-preview" - - # OpenRouter API settings - base_url: "https://openrouter.ai/api/v1" - - # Environment variable containing OpenRouter API key - api_key_env: "OPENROUTER_API_KEY" - - # Temperature for summarization (lower = more deterministic) - temperature: 0.3 - - # Max retries for API failures - max_retries: 3 - - # Delay between retries (seconds) - retry_delay: 2 - -# Output settings -output: - # Add notice to system message about potential summarization - add_summary_notice: true - - # Text to append to system message - summary_notice_text: "\n\nSome of the conversation may be summarized to preserve context." - - # Output directory suffix (appended to input directory name) - output_suffix: "_compressed" - -# Processing settings -processing: - # Number of parallel workers for batch processing - num_workers: 4 - - # Maximum concurrent API calls for summarization (async parallelism) - max_concurrent_requests: 50 - - # Skip trajectories that are already under target length - skip_under_target: true - - # If true, save trajectories even if compression can't get under target - # (will compress as much as possible) - save_over_limit: true - - # Timeout per trajectory in seconds (skip if takes longer) - # Helps avoid hanging on problematic entries - per_trajectory_timeout: 300 # 5 minutes - -# Metrics to track -metrics: - # Log detailed compression statistics - enabled: true - - # Save per-trajectory metrics in output - per_trajectory: false - - # Metrics file name (saved in output directory) - output_file: "compression_metrics.json" diff --git a/datagen-config-examples/web_research.yaml b/datagen-config-examples/web_research.yaml deleted file mode 100644 index 6275dbed69..0000000000 --- a/datagen-config-examples/web_research.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# datagen-config-examples/web_research.yaml -# -# Batch data generation config for WebResearchEnv. -# Generates tool-calling trajectories for multi-step web research tasks. -# -# Usage: -# python batch_runner.py \ -# --config datagen-config-examples/web_research.yaml \ -# --run_name web_research_v1 - -environment: web-research - -# Toolsets available to the agent during data generation -toolsets: - - web - - file - -# How many parallel workers to use -num_workers: 4 - -# Questions per batch -batch_size: 20 - -# Total trajectories to generate (comment out to run full dataset) -max_items: 500 - -# Model to use for generation (override with --model flag) -model: openrouter/nousresearch/hermes-3-llama-3.1-405b - -# System prompt additions (ephemeral — not saved to trajectories) -ephemeral_system_prompt: | - You are a highly capable research agent. When asked a factual question, - always use web_search to find current, accurate information before answering. - Cite at least 2 sources. Be concise and accurate. - -# Output directory -output_dir: data/web_research_v1 - -# Trajectory compression settings (for fitting into training token budgets) -compression: - enabled: true - target_max_tokens: 16000 - -# Eval settings -eval_every: 100 # Run eval every N trajectories -eval_size: 25 # Number of held-out questions per eval run diff --git a/docs/ADR.md b/docs/ADR.md deleted file mode 100644 index cc9ecfb927..0000000000 --- a/docs/ADR.md +++ /dev/null @@ -1,77 +0,0 @@ -# Architecture Decision Records - -## 2026-07-13: Scope plugin manager state by Hermes home/profile (keyed cache) - -Status: Accepted - -Context: -Hermes supports multiple profiles via different Hermes home directories. -Homes are switched two ways in a running process: the `HERMES_HOME` -environment variable (single-profile CLI/gateway processes), and the -context-local `set_hermes_home_override()` (`hermes_constants.py`), which -the multiplexed gateway worker (`gateway/run.py`'s `_profile_scope`) and -subagent/embedded callers use to serve several profiles from one -long-lived process. The override is a `ContextVar` and deliberately does -**not** mutate `os.environ`, since that would leak one profile's home -into every other concurrent task in the same process. - -The plugin manager was a process-global single-slot singleton -(`_plugin_manager`). User-installed plugins are discovered from -`get_hermes_home() / "plugins"`, and context-engine plugins (e.g. -`hermes-lcm`) capture profile-scoped state — such as the LCM database -path — at registration time. A single-slot cache meant: - -1. Switching homes via `set_hermes_home_override()` was invisible to a - naive "did `HERMES_HOME` change" check, so the singleton silently kept - serving the first profile's manager to every other profile in the - process. -2. Even when a fresh `PluginManager` *was* created for a new home, plugin - modules are imported into `sys.modules` as `hermes_plugins.` by - `_load_directory_module`, and only that top-level module was ever - replaced. A same-slug plugin's *relative* imports - (`from . import state`) are cached separately under - `hermes_plugins..`, and Python's import machinery - resolves those from `sys.modules` first — so a profile switch could - silently keep serving a previous profile's already-imported submodule - code/state instead of re-executing the new profile's plugin. - -Decision: -- Replace the single-slot singleton with a cache keyed on the *resolved* - Hermes home path (`_plugin_managers_by_home: Dict[Path, PluginManager]`). - `get_plugin_manager()` resolves the current home via `get_hermes_home()` - (which itself already consults `get_hermes_home_override()` before - `os.environ`), so both the env-var and context-local override paths are - covered uniformly. -- `_plugin_manager` (the old single-slot name) is kept as a thin "last - manager returned" pointer purely for backward compatibility with - existing test code that does - `monkeypatch.setattr(plugins_mod, "_plugin_manager", some_manager)`. - When that name is monkeypatched to a manager the keyed cache doesn't - know about, `get_plugin_manager()` treats it as an explicit injection - and adopts it into the cache under the *current* resolved home, rather - than discarding it. -- Both `PluginManager._load_directory_module` (initial/`force=True` - reload within the same home) and the shared `_clear_plugin_submodules` - helper (profile switch / test teardown) evict `sys.modules[module_name]` - **and every name prefixed with `module_name + "."`** before a plugin - slug is (re-)imported, so relative-import submodules can never survive - a reload or a home switch. -- Test isolation (`tests/conftest.py`'s `_hermetic_environment` fixture) - calls a new `_reset_plugin_managers_for_tests()` helper that drops the - entire keyed cache and purges every plugin submodule from `sys.modules` - between tests, instead of only resetting the single-slot pointer. - -Consequences: -- Per-profile LCM instances (and any other context-engine plugin) use - their own `{home}/lcm.db` regardless of whether the profile switch went - through `HERMES_HOME` or `set_hermes_home_override()`. -- Plugin discovery remains cached within a profile for normal - performance, and re-entering a previously-seen profile reuses its - cached manager instead of rebuilding from scratch. -- Sequential *and* interleaved profile switching — in tests, the gateway - multiplexer worker, or embedded callers using the context-local - override — no longer leaks context-engine state, plugin module state, - or stale relative-import submodules across profiles. -- Regression coverage exercises the real production path - (`set_hermes_home_override()`) rather than only the env-var path, and - includes a dedicated relative-import leak test. diff --git a/docs/cron-doctor-spec.md b/docs/cron-doctor-spec.md deleted file mode 100644 index ac2595153a..0000000000 --- a/docs/cron-doctor-spec.md +++ /dev/null @@ -1,32 +0,0 @@ -# Cron Doctor Spec - -## Problem - -Scheduled jobs can silently degrade when a script is moved, a workdir disappears, -a provider run fails, or delivery starts failing. `hermes cron list` shows some of -this inline, but there is no compact read-only health check that can be run from a -terminal, cron job, or CI-style smoke check. - -## Goal - -Add `hermes cron doctor` as a read-only diagnostic command that summarizes cron -job health and exits non-zero when actionable issues are found. - -## Non-goals - -- Do not mutate jobs or auto-repair state. -- Do not start/stop the gateway. -- Do not inspect secrets or print credentials. - -## Acceptance criteria - -- `hermes cron doctor` returns `0` and prints a healthy message when active jobs - have no detected issues. -- It returns `1` and prints grouped job-level issues when any active job has: - - last run failure (`last_status` not `ok`), - - last delivery failure, - - no `next_run_at` while still active, - - `no_agent` enabled without a script, - - script path missing/outside `HERMES_HOME/scripts`, or - - configured workdir path missing. -- Parser, command dispatch, and focused tests cover the new subcommand. diff --git a/docs/design/kanban-dialogs/index.html b/docs/design/kanban-dialogs/index.html deleted file mode 100644 index f32f74f5a8..0000000000 --- a/docs/design/kanban-dialogs/index.html +++ /dev/null @@ -1,903 +0,0 @@ - - - - - -Hermes Kanban — Native Dialog Prototypes - - - - -
-

Hermes Kanban — Native Dialog Prototypes

-

- Four approaches to replacing window.confirm(), - window.prompt(), and window.alert() in - plugins/kanban/dashboard/dist/index.js. Click a page-level - trigger to fire the same flow in every variant simultaneously, or use the - per-variant buttons to fire flows unique to that variant. -

-
- - - - - - - -
-
- -
- - - - -
-
-

Variant A

-
Conservative
-

Direct 1:1 mapping to the host's ConfirmDialog. Single-line input. Minimal chrome.

-
-
-
Click a trigger to preview.
-
-
- Trade-off: Safest to ship — zero new components, zero new - patterns. Fails the GPT-OSS review on three points: no multi-line - summary, validation triggers a SECOND dialog instead of inline, no bulk - affordance. -
-
- - - - -
-
-

Variant B

-
Strong-fit (Pro's pick)
-

Textarea + contextual SVG icon + inline validation + pluralized copy. What the design brief recommends.

-
-
-
Click a trigger to preview.
-
-
- Trade-off: Best baseline, but Pro suggested either - a disabled-button or an inline error — GPT-OSS caught that both - are needed (button disabled AND error visible) for screen-reader users. - Bottom-sheet on mobile is the right move but still has keyboard edge cases. -
-
- - - - -
-
-

Variant B-refined

-
Synthesis (recommended)
-

All of B's improvements + GPT-OSS fixes: auto-focus, dual-validation, cancellable-spinner, per-task summaries in bulk.

-
-
-
Click a trigger to preview.
-
-
- Why this is the recommendation: Single contextual icon - (not four) keeps the title readable. Confirm button stays enabled until - textarea has content; the inline error appears on submit-attempted-empty - AND on blur if still empty. Cancel button stays clickable during the - PATCH (only the confirm shows the spinner) so users can abort slow - networks. Bulk-many shows an expandable list with per-task summary - fields. -
-
- - - - -
-
-

Variant C

-
Divergent: undo toast
-

Skip the modal entirely for non-destructive moves. Optimistic UI + 5s undo in a bottom-right toast.

-
-
-
Click a trigger to preview.
-
-
-
- Trade-off: Radical speedup for routine moves, but breaks - the required-summary flow (you can't optimistically "complete" a task - that's missing required schema data). Best used as a - complement to B-refined — undo toast for the safe moves, - modal for done/blocked/archive. -
-
- -
- -
-

Decision matrix — recommended pick: B-refined

- - - - - - - - - - - - - - - - - -
CapabilityABB-refinedC
Centered modal (Radix) ✓ ✓ ✓ ✗
Multi-line summary ✗ ✓ ✓ ✗
Single contextual icon ✗ 4 ✓ ✗
Inline validation (no second dialog)✗ ~ ✓ ✗
Disabled button + persistent error ✗ ~ ✓ ✗
Cancellable spinner during PATCH ✗ ✗ ✓ ✓
Bulk-many expandable list ✗ ✗ ✓ ✗
Auto-focus textarea + mobile scroll ✗ ~ ✓ ✗
Toast on success ✗ ✗ ✗ ✓
Toast on error ✗ ✗ opt. ✓
Survives the required-summary flow ✗ ✓ ✓ ✗
-
- - - - - \ No newline at end of file diff --git a/docs/design/profile-builder.md b/docs/design/profile-builder.md deleted file mode 100644 index 76649096d2..0000000000 --- a/docs/design/profile-builder.md +++ /dev/null @@ -1,146 +0,0 @@ -# Profile Builder — Dashboard-Native, Full-Featured Profile Creation - -Status: design proposal (not yet implemented) -Author: drafted for Teknium -Supersedes: PR #31781 (prompt_toolkit `hermes profile wizard`) - -## Why this, not the CLI wizard - -PR #31781 added a keyboard-driven `hermes profile wizard` in the terminal. -The decision is to **not** build the profile-creation experience in the CLI. -The dashboard already owns mature, separate pages for every element a profile -needs, and a profile is just a HERMES_HOME directory — so the dashboard is the -right home for a full-featured builder, and it can reuse everything that -already exists. - -A profile = a full `~/.hermes/profiles//` directory with its own: -- `config.yaml` — holds `model`/`provider`, `mcp_servers`, enabled skills -- `skills/` — physical SKILL.md files (built-in seed + optional + hub installs) -- `.env` — secrets -- `SOUL.md` / `USER.md` — identity - -So per-profile scoping of Model, MCPs, and Skills is **native** — no data-model -change needed. The gap is purely UX: creation today is a thin modal -(name + clone + model + description), and you can only compose skills/MCPs -*after* the profile exists, by visiting other pages and remembering to scope -them. - -## What already exists (reuse, don't rebuild) - -| Element | Existing page | Existing API | Profile-scopable? | -|---|---|---|---| -| Name / Description | ProfilesPage create modal | `POST /api/profiles` (`create_profile`) | yes (args) | -| Model + Provider | ModelsPage | `_write_profile_model(profile_dir, …)` | yes — HERMES_HOME override, already wired into create endpoint | -| MCPs | McpPage | `mcp_config._save_mcp_server` + `/api/mcp/catalog` | yes — wrap with HERMES_HOME override | -| Skills (built-in/optional) | SkillsPage | `GET /api/skills`, `/api/skills/toggle` | yes — config write | -| Skills (hub) | SkillsPage | `/api/skills/hub/search`, `/api/skills/hub/install` | **only via subprocess** — see seam #1 | - -## Two architectural seams found while grounding this design - -These are load-bearing — they change the implementation, not just the polish. - -### Seam #1 — hub-skill install cannot use the HERMES_HOME override - -`tools/skills_hub.py` binds `SKILLS_DIR = HERMES_HOME / "skills"` at **module -import time**. The context-local `set_hermes_home_override()` swap (which makes -`_write_profile_model` and the MCP write land in the target profile) does NOT -retroactively rebind that already-imported module global. So a data-layer wrap -of hub install would write into the dashboard's *own* active profile, not the -new one. - -The correct mechanism is the existing subprocess path: `_spawn_hermes_action` -runs `python -m hermes_cli.main `, and `_apply_profile_override()` -re-reads `sys.argv` at import in the fresh child. Prepend `-p `: - -```python -_spawn_hermes_action(["-p", profile, "skills", "install", identifier], "skills-install") -``` - -A fresh subprocess re-imports `skills_hub` with the profile's HERMES_HOME bound -from the start, so `SKILLS_DIR` resolves to `/skills/`. Correct by -construction. - -### Seam #2 — hub installs are async, so create cannot be fully atomic - -Built-in/optional skill enabling and MCP writes are **synchronous config ops** -and can be part of the create call. Hub installs are long-running git fetches -spawned detached (`_spawn_hermes_action` returns a PID immediately). So the -create flow is: - -1. `create_profile()` — make the dir (synchronous) -2. write model (synchronous, HERMES_HOME override) -3. write selected MCP servers (synchronous, HERMES_HOME override) -4. seed/enable selected built-in + optional skills (synchronous) -5. spawn `hermes -p skills install ` per hub skill (async, returns PIDs) - -Steps 1–4 commit before the response; step 5 returns a list of action PIDs the -UI polls (same pattern as today's SkillsPage hub install). The builder's -"Review → Create" returns `{ok, name, path, hub_installs: [{id, pid}]}` and the -final screen shows live install progress for the hub skills. - -## Proposed backend change (small, follows existing patterns) - -Extend `ProfileCreate` and the create endpoint — no new endpoints, no rewrite: - -```python -class ProfileCreate(BaseModel): - name: str - clone_from: Optional[str] = None - # Backward compatibility for older dashboard/desktop clients. - clone_from_default: bool = False - clone_all: bool = False - no_skills: bool = False - description: Optional[str] = None - provider: Optional[str] = None - model: Optional[str] = None - # NEW — all optional, all best-effort post-create (profile already exists) - mcp_servers: List[MCPServerCreate] = [] # synchronous, HERMES_HOME override - builtin_skills: List[str] = [] # synchronous enable/seed - hub_skills: List[str] = [] # async spawn, returns PIDs -``` - -The endpoint already does best-effort post-create steps (`seed_profile_skills`, -`_write_profile_model`). Add two more best-effort blocks (MCP write, hub-skill -spawn) in the same style — a failure in any of them must not 500 the create, -since the profile dir already exists and the user can fix it from the relevant -page afterward. Mirror `_write_profile_model`'s HERMES_HOME-override helper for -the MCP write (`_write_profile_mcp_servers(profile_dir, servers)`). - -## Proposed frontend — dedicated builder page `/profiles/new` - -A full page (not the cramped modal), stepped, each step reusing the existing -page's component + API, targeted at the new profile: - -``` -① Identity Name + Description (+ optional clone-from existing profile) -② Model Provider + model picker (reuse ModelsPage picker) -③ Skills Tabs: Built-in · Optional · Hub-search - multi-select; "Start from default bundle" preset button -④ MCPs Tabs: Catalog browse · Manual add (reuse McpPage form) -⑤ Review Blueprint preview → Create - → progress screen for async hub installs -``` - -Nothing writes to disk until ⑤. - -## Open product decisions (need Teknium) - -1. **Skills seeding default.** Fresh profiles auto-seed the default bundle - today. In the builder, should the skill step **replace** the bundle (pick - exactly what you want; offer a "start from default bundle" preset) or - **augment** it? Recommendation: replace + preset button. - -2. **Page vs richer modal.** Dedicated `/profiles/new` page (room to grow: - SOUL editing, multi-agent fleets later) vs a bigger create modal on - ProfilesPage. Recommendation: dedicated page — matches "full-featured / way - more options." - -## Verification plan (when built) - -- Backend E2E with isolated HERMES_HOME: POST a full create body - (name + model + 2 MCPs + 3 builtin skills + 1 hub skill), assert the new - profile dir has the model in config.yaml, both MCP servers in config.yaml, - the builtin skills enabled, and a spawned PID for the hub skill. Negative: - a bad MCP entry must not 500 the create. -- `cd web && npm run build` (no JS test suite in web/). -- Targeted: `pytest tests/ -k profile_create`. diff --git a/docs/hermes-kanban-v1-spec.pdf b/docs/hermes-kanban-v1-spec.pdf deleted file mode 100644 index c7899cd12a..0000000000 Binary files a/docs/hermes-kanban-v1-spec.pdf and /dev/null differ diff --git a/docs/profile-routing.md b/docs/profile-routing.md deleted file mode 100644 index 9b0237f5c6..0000000000 --- a/docs/profile-routing.md +++ /dev/null @@ -1,117 +0,0 @@ -# Profile-Based Routing for Inbound Messages - -> **Audience:** Gateway operators and contributors -> **Source files:** `gateway/profile_routing.py`, `gateway/run.py` (`_profile_name_for_source`), `gateway/platforms/base.py` (`build_source`), `gateway/config.py` -> **Related:** [Session Lifecycle](session-lifecycle.md), `docs/design/profile-builder.md` - -## Overview - -By default a single gateway run uses one profile (memory, persona, tools). **Profile-based -routing** lets one gateway instance serve **multiple isolated profiles**, selecting which -profile handles an inbound message based on *where the message came from* — the platform, -server (`guild_id`), channel (`chat_id`), and/or thread (`thread_id`). - -This is the inbound counterpart to multiplexing: instead of running N gateways, run one -gateway and route per-community / per-channel / per-thread to a dedicated profile. Each -profile keeps fully isolated state (`MEMORY.md`, `USER.md`, `SOUL.md`, sessions, tools). - -Routing is **platform-generic**: it works for Discord, Telegram, Feishu, Slack, and every -adapter — not just Discord. - -## Configuring routes - -Routes live under `profile_routes` in `config.yaml`. Both the top-level and the nested -`gateway.profile_routes` forms are accepted (the nested form is what -`hermes config set gateway.profile_routes ...` writes). - -```yaml -profile_routes: - # Route an entire Discord server (guild) to one profile. - - name: server-default - platform: discord - guild_id: "1234567890" - profile: server-profile - - # Override a specific channel within that server with a different profile. - - name: support-channel - platform: discord - guild_id: "1234567890" - chat_id: "9876543210" - profile: support-profile - - # Pin a Telegram group to a profile (Telegram has no guild_id — chat_id only). - - name: tg-group - platform: telegram - chat_id: "-1001234567890" - profile: tg-profile - - # Route a single Discord thread. - - name: standup-thread - platform: discord - guild_id: "1234567890" - chat_id: "9876543210" - thread_id: "1111111111" - profile: standup -``` - -### Fields - -| Field | Required | Description | -|---|---|---| -| `name` | yes | Human-readable route identifier (used in logs). | -| `platform` | yes | Adapter platform: `discord`, `telegram`, `feishu`, `slack`, … | -| `profile` | yes | Target profile name (must exist under `~/.hermes/profiles/`). | -| `guild_id` | no | Server/guild (Discord). | -| `chat_id` | no | Channel/group/DM id. | -| `thread_id` | no | Thread id within a channel. | -| `enabled` | no | Default `true`; set `false` to disable a route without removing it. | - -## Matching rules - -A route matches an inbound source when **every discriminator the route declares is satisfied** -(conjunctive / AND). A field the route leaves unset is ignored. - -- **`platform`** must equal the source platform exactly. -- **`thread_id`** (if set) must equal the source thread id. -- **`chat_id`** (if set) must match the source channel **or** its parent — a thread in a - channel matches the channel's route (hierarchical match for Discord forums/threads). -- **`guild_id`** (if set) must equal the source guild. - -> A route declaring **both** `guild_id` and `chat_id` requires both to hold. A channel match -> alone does not satisfy a guild constraint — this is intentional and tested. - -When multiple routes match, the **most specific** one wins. Specificity is additive: - -| Discriminator | Weight | -|---|---| -| `thread_id` | 8 | -| `chat_id` | 4 | -| `guild_id` | 2 | -| (platform only) | 0 | - -So a thread route (8) beats a channel route (4) beats a guild route (2) within the same server. -If no route matches, the message uses the default/active profile. - -## How it works at runtime - -1. An inbound message arrives at a platform adapter. -2. `BasePlatformAdapter.build_source` builds the `SessionSource` for the message. Every - adapter carries a back-reference to the running `GatewayRunner` - (`gateway_runner`, injected in `gateway/run.py`), so it asks the runner to resolve the - target profile via `_profile_name_for_source`. -3. `_profile_name_for_source` runs the configured routes through `match_profile_route` and - stamps `source.profile` with the winning route's profile (or leaves it unset). -4. Downstream, `_resolve_profile_home_for_source` chooses the profile home directory - (`source.profile` → active profile → `default`) and the session is scoped per-profile, so - each routed community gets isolated memory and conversation state. - -Because `gateway_runner` is injected for **all** adapters (declared on `BasePlatformAdapter`), -every platform goes through this path — not just Discord. - -## Relationship to multiplexing - -`profile_routes` requires `gateway.multiplex_profiles: true`. Multiplexing is what -activates the per-profile runtime scope (per-profile `HERMES_HOME`, secret scope, and -profile-namespaced session keys); routing is the decision layer that picks *which* -profile a given guild/channel/thread lands in. With multiplexing off, `profile_routes` -is ignored entirely — behavior is byte-identical to a single-profile gateway. diff --git a/docs/rca-ssl-cacert-post-git-pull.md b/docs/rca-ssl-cacert-post-git-pull.md deleted file mode 100644 index 505def8750..0000000000 --- a/docs/rca-ssl-cacert-post-git-pull.md +++ /dev/null @@ -1,54 +0,0 @@ -# RCA: SSL CA cert bundle corruption after `hermes update` - -**Status:** resolved by `fix(ssl): surface broken CA bundles before provider calls` -**Severity:** P2 — degrades the agent into opaque provider/client failures until the user repairs deps or CA configuration. - -## Summary - -A partial `hermes update`, interrupted venv repair, or stale CA-bundle environment variable can leave Python TLS configuration pointing at a missing, empty, or unloadable CA bundle. The first outbound HTTPS client creation or request can then fail with a raw `FileNotFoundError: [Errno 2] No such file or directory` or a low-level SSL error that does not name the broken CA path. - -## Root cause - -Hermes uses OpenAI/httpx and requests-based clients for provider calls, model metadata, gateway delivery, and web tools. Those clients inherit CA bundle settings from: - -- `HERMES_CA_BUNDLE` -- `SSL_CERT_FILE` -- `REQUESTS_CA_BUNDLE` -- `CURL_CA_BUNDLE` -- the bundled `certifi` package's `cacert.pem` - -When the venv is partially refreshed, or when one of those env vars points at a file that no longer exists, provider client construction can fail before Hermes has enough context to produce a useful message. - -## Fix - -`agent/ssl_guard.py` validates CA bundle configuration before the OpenAI-compatible provider client is created in `agent/agent_init.py`. It: - -1. Checks explicit CA bundle env vars and reports the exact broken variable/path, -2. Verifies `certifi` is importable, -3. Verifies `certifi.where()` points at an existing file of plausible size, -4. Builds an `ssl.SSLContext` from each checked bundle, -5. Raises a typed `SSLConfigurationError` with a repair hint before httpx/OpenAI can raise a raw low-level error. - -`hermes_cli doctor` exposes the same check under `SSL / CA Certificates`, so users can diagnose the problem without starting a model session. - -## Recovery - -When the guard fires during agent init, the user sees a message like: - -```text -Failed to initialize OpenAI client: SSL_CERT_FILE points to a missing CA bundle: C:\path\to\missing\cacert.pem -Repair: python -m pip install --force-reinstall certifi openai httpx -If you configured a custom corporate CA bundle, fix or unset the broken CA bundle environment variable. -``` - -For a normal corrupted Hermes venv, reinstall the affected client dependencies: - -```bash -python -m pip install --force-reinstall certifi openai httpx -``` - -For a custom/corporate CA setup, fix the env var so it points at a real PEM bundle, or unset it if Hermes should use the bundled `certifi` store. - -## Environment escape hatch - -Set `HERMES_SKIP_SSL_GUARD=1` to bypass the preflight check. This is intended only for sandboxed or managed-trust environments where the Python CA path looks unusual but downstream clients are known to work. diff --git a/docs/rfcs/2026-07-plugin-architecture-lessons-pi-opencode.md b/docs/rfcs/2026-07-plugin-architecture-lessons-pi-opencode.md deleted file mode 100644 index 7aaf126417..0000000000 --- a/docs/rfcs/2026-07-plugin-architecture-lessons-pi-opencode.md +++ /dev/null @@ -1,134 +0,0 @@ -# Research spike: plugin-architecture lessons from Pi and OpenCode - -**Issue:** #64180 · **Informs:** #64164 (event bus), #64161 (streaming hooks), #64162 (pluggable approval), #64165 (manifest v2), #64229 (lifecycle/ledger), #64230 (Plugin Doctor) - -**Method.** Both systems were read at source level (shallow clones pinned to a commit), not from docs sites alone: Pi (`badlogic/pi-mono`, now `earendil-works/pi`) at `eb79351` (v0.80.7, 2026-07-14) and OpenCode (`sst/opencode`, now `anomalyco/opencode`) at `c69abee` (v1.18.2, 2026-07-16). Claims below carry file:line references into those commits. Where something could not be found (ADRs, policies, timeouts), that absence was verified by search and is reported as a finding. Hermes ground rules from #64182 (additive-only, prompt-cache sacred, observer-first, fail-closed security) are treated as overriding constraints throughout — this report grades imported patterns against them, per the adapt-don't-copy rubric. - -**Headline.** The two systems are near-perfect opposites on the four design axes Hermes currently has open, which makes them a natural controlled experiment: - -| Axis | Pi | OpenCode | Hermes proposal on the table | -|---|---|---|---| -| Per-delta streaming hook | Yes — awaited inline, no timeout | Structurally absent (text-end only) | Observer per-delta + never-block contract (#64161) | -| Veto semantics | Typed per-event result vocabulary (`{block}`, `{cancel}`, `"handled"`) | Veto-by-throw (bug and policy denial indistinguishable) | TBD in #64162 | -| Guard-hook failure | Fail closed (`tool_call` only) | No runtime containment at all | Ground rule 4: fail closed on security-adjacent | -| Plugin event bus | Yes — 33 lines, **un-namespaced** channels | None (plugins observe core bus, cannot emit) | Namespaced `ctx.emit`/`ctx.subscribe` (#64164) | - -Neither system has hook timeouts. Both have shipped hang-class or drift-class failures because of it. That is the single strongest cross-cutting lesson for Hermes. - ---- - -## 1. Pi (badlogic/pi-mono → earendil-works/pi) - -Extensions are in-process TypeScript modules (loaded via jiti) receiving an `ExtensionAPI`; no separate process, no IPC, no manifest permissions. Pi explicitly rejected MCP as its extension mechanism (Zechner, ["What if you don't need MCP at all?"](https://mariozechner.at/posts/2025-11-02-what-if-you-dont-need-mcp/)). Notably, Pi's founding "minimal, no hooks" stance (Nov 2025) reversed into a 33-event extension system within ~7 months — extensibility demand won; the no-MCP stance held. - -### 1.1 Hook/event taxonomy - -33 event types (`src/core/extensions/types.ts:507-902`). The load-bearing design choice: **every mutating event gets its own typed emitter with its own result vocabulary**, rather than one generic middleware pipe — - -- `tool_call` → `{ block: true, reason }`; argument mutation in place, explicitly "no re-validation after your mutation" (`docs/extensions.md:742-765`) -- `session_before_*` → `{ cancel: true }`, first canceller wins, later handlers skipped -- `input` → `"handled"` (short-circuit) vs `"transform"` (chain) (`runner.ts:1148-1188`) -- `tool_result` → partial-patch accumulation across handlers (`runner.ts:835-883`) -- `before_agent_start` → system-prompt chaining with a live `ctx.getSystemPrompt()` reflecting earlier handlers (`runner.ts:1034-1098`) -- Observer events have **no return channel at all** — the observer/mutator split is enforced by which emitter the event flows through, not by convention. - -Dispatch is fully sequential async (`runner.ts:759-791`): every handler awaited one at a time; the only ordering rule is extension load order (project → global → CLI, `loader.ts:660-708`) then registration order. No priorities, no phases — and no community demand for them in ~8 months. Name collisions are handled per registry with explicit deterministic policies instead of dependency resolution: first-wins tools, suffixed duplicate commands (`name:2`), last-wins shortcuts with a warning plus an 18-key reserved denylist (`runner.ts:421-604`). - -Two ordering guarantees worth copying: extensions see events **before** the UI and **before** session persistence (`agent-session.ts:596-601`), and parallel sibling tool calls are "preflighted sequentially, then executed concurrently" (`docs/extensions.md:750`). - -### 1.2 Plugin-to-plugin interaction - -A 33-line shared event bus (`src/core/event-bus.ts`): `pi.events.emit/on` on **arbitrary string channels** — no namespacing, no declarations, no collision protection; per-handler try/catch and an unsubscribe closure. Everything richer (capability registries, dependencies) is deliberately absent; a community RFC to build richer coordination as an extension (pi#2715) was left to userland. - -### 1.3 Compatibility strategy - -No API versioning, no handshake, no deprecation annotations. In its place, three working practices: (a) loader-level alias shims that kept old imports resolving across the `@mariozechner`→`@earendil-works` package rename and a pi-ai API split, with pre-announced removal (`loader.ts:47-71`; `CHANGELOG.md:243`); (b) loud "Breaking Changes" changelog sections with **automatic migrations** (directory renames on startup, session format v2→v3 auto-migrated) for the big hooks+customTools→extensions unification (v0.35-0.37); (c) per-tool `prepareArguments()` shims for old argument shapes. - -Real breakage on record: pi#2860 — an internal session-management refactor made `pi.sendUserMessage()` after `ctx.newSession()` silently drop messages. The remediation pattern is distinctive: **stale-context poisoning** — after session replacement, every captured context getter throws a paragraph-long teaching error pointing at the safe `withSession` pattern (`runner.ts:514-527`; `docs/extensions.md:1223-1265` documents the footgun with unsafe-pattern code). Compat by making misuse loud, not by never changing. - -### 1.4 Failure isolation - -Contain errors, don't contain time: every handler call is individually try/caught and surfaced (red stack trace in chat UI), load failures skip only the broken extension, and a `pi -ne` no-extensions escape hatch exists. The **one deliberate exception**: `tool_call` has no internal catch — a guard-hook crash blocks the tool ("Extension failed, blocking execution"), converted into an error tool result the LLM sees; the loop survives, the guard fails closed (`runner.ts:885-906`; `agent-session.ts:454-467`; `agent-loop.ts:657-665`). - -**There are zero timeouts.** All handlers — including per-token `message_update` — are awaited inline in the agent event pipeline (`agent-session.ts:598, 728-734`). A hung extension freezes the agent; the changelog records hang-class fixes (pi#5687 background handles, pi#5115 shutdown drains). Mitigations offered are cooperative only (`ctx.signal` AbortSignal, Esc-abort). - -No sandbox, documented as a decision: "a partial in-process sandbox would be easy to misunderstand as a security boundary" (`docs/security.md:33-35`); the only gate is project-trust on load. - -### 1.5 Design-history record - -**Pi has no ADRs** — the Discord recollection that prompted this spike is not substantiated for Pi. Rationale lives in four places: "footgun" sections inside user docs (effectively inline ADRs), a 5,000-line issue-linked changelog (which records abandoned designs: the hooks/customTools split, slash-commands→prompt-templates rename, a fetch-override proxy abandoned for undici dispatchers), Zechner's blog, and issues themselves. - -### 1.6 Prompt/context construction - -Pi is the only surveyed system that treats **prompt-cache stability as part of the extension API contract**: extensions receive structured prompt inputs (`systemPromptOptions` — the same decomposed inputs Pi itself uses) rather than a final string; per-request context transforms operate on a `structuredClone` so session history is never corrupted; and cache-friendly dynamic tool loading (v0.80.6, pi#6474) activates tools additively via native provider deferred loading (Anthropic `defer_loading`) explicitly to avoid invalidating the prefix cache — with documented warnings about second-order invalidation through prompt-metadata changes (`docs/extensions.md:2254-2290`). - ---- - -## 2. OpenCode (sst/opencode → anomalyco/opencode) - -Server (Bun/Effect) + clients (TUI/desktop/web); plugins are npm packages or local TS files loaded in-process, with **two plugin kinds in one package** — `server` and `tui` entrypoints, hard rule one-or-the-other (`shared.ts:103-114, 293-295`). A v2 plugin API ships side-by-side with v1 (`/v2/effect`, `/v2/promise` subpaths), designed in a candid 516-line `PLAN.md`. - -### 2.1 Hook/event taxonomy - -One `Hooks` bag returned by an async factory (`packages/plugin/src/index.ts:74, 222-335`): ~16 mutating `(input, output)` hooks (mutate `output` in place; host reads it back) plus declarative registries (tools, auth, providers) and a single firehose `event` observer. The whole dispatch engine is ~13 lines (`plugin/index.ts:280-293`): sequential, awaited, load-order, later hooks see earlier mutations — ordering documented as a spec ("global config → project config → global plugin dir → project plugin dir"; v2: "plugin registration order, then transform registration order"). - -**Veto is throw.** The documented idiom for denying a tool call is `throw new Error("Do not read .env files")` (`plugins.mdx:247-257`) — policy denial and plugin bug are indistinguishable in every consumer downstream. - -**A typed-but-dead hook.** `permission.ask` — the only hook with real decision semantics (`output.status: "ask" | "deny" | "allow"`) — exists in the published types but has **no dispatch site anywhere in the tree**: a permission-subsystem rewrite orphaned it, types kept compiling, plugins silently no-op'd (oc#7006, open since Jan 2026). This is the sharpest single failure mode found in the whole spike. - -### 2.2 Plugin-to-plugin interaction - -No dependencies, no registry, no plugin-emitted events. Interaction is (a) blind composition through the shared mutable `output` (last-writer-wins per field) and (b) observing the core event bus. The 2026 core bus itself is heavyweight — event-sourced, durable (SQLite, per-aggregate sequences, idempotent replay), versioned wire types, and a **backpressure guard**: `allBounded` wraps a dropping queue and fails slow subscribers with `SubscriberOverflowError` (`packages/core/src/event.ts:152-164`). Plugins get the untyped tail of a three-tier bridge (Effect streams → global emitter → `event` hook), fired **fire-and-forget** — an async observer's rejection is an unhandled promise rejection invisible to host error routing (`plugin/index.ts:251-258`). - -### 2.3 Compatibility strategy - -Lockstep versioning (`opencode` = `@opencode-ai/plugin` = `@opencode-ai/sdk` = 1.18.2) plus one gate: npm plugins may declare `engines.opencode` semver ranges, checked at load (`shared.ts:194-205`) — but it is plugin-opt-in, and local file plugins skip it entirely. Three generations of module shape are loaded simultaneously; superseded packages are silently ignored via a hardcoded `DEPRECATED_PLUGIN_PACKAGES` list. - -The community-experience record is instructive: v1.14.42 — a **patch release** — removed the whole `api.command.*` TUI namespace with no deprecation cycle (oc#26557); the aftermath is visible in-tree as a deprecated shim added back *after* the outcry ("Legacy `api.command` API kept so v1 plugins can initialize. Remove in v2", `tui.ts:87-120`). No written deprecation policy exists anywhere. Shims-after-outcry is the de facto process. - -### 2.4 Failure isolation - -Strong at the edges, absent in the middle. Load is staged (`install | entry | compatibility | missing | load`) with per-plugin, per-stage containment and user-visible toasts (`loader.ts:82-93`; `plugin/index.ts:215-249`). Runtime hooks have **no catch and no timeout**: a throw in a tool hook fails that tool call (the sanctioned veto), a throw in chat/transform hooks aborts the turn (session errors, server survives), a **hang hangs the turn forever** — the v2 PLAN explicitly lists "Transform timeouts" under *Deferred Decisions* (`PLAN.md:507-510`). Boot re-entrancy burned them: a plugin calling the SDK client during its own init deadlocked startup (oc#7741). The official troubleshooting page's first advice is "start by disabling plugins." - -Two mature exceptions worth stealing: the streaming hot path is protected **structurally** — there is no per-delta hook at all; text hooks fire once at `text-end` (`processor.ts:512-524`) — and the TUI runtime has scope-tracked registrations (a `Proxy`-wrapped keymap API auto-records every registration per plugin, enabling clean live deactivate) plus a **hard 5s dispose budget** racing each cleanup against a timer (`runtime.ts:122-226, 388-468`). - -### 2.5 Design-history record - -No ADR system; the `PLAN.md` for v2 is the exception and is better than most ADR archives — it names v1's mistakes (the returned-hooks bag, finalizer-triggered special cases), specifies ordering as a contract, splits replayable *transforms* from live *hooks*, and — crucially — carries an honest **Deferred Decisions** section (typed error model, transform timeouts) rather than pretending closure. - -### 2.6 Prompt/context construction - -Plugins can touch every layer, but the deep layers are gated behind an `experimental.` prefix (`experimental.chat.system.transform`, `experimental.chat.messages.transform`, compaction prompt replacement) — a deliberate two-tier stability promise: interception at operation boundaries is stable, rewriting the context itself is not. No cache-stability discipline comparable to Pi's was found. - ---- - -## 3. Adopt / Adapt / Avoid for Hermes - -Grading against #64182's ground rules. "Validated" = the proposal already on the Hermes issue is independently confirmed by field evidence. - -| # | Lesson | Verdict | Maps to | Evidence | -|---|---|---|---|---| -| 1 | **Typed per-hook result vocabularies, not veto-by-throw.** Pi's `{block, reason}` / `{cancel}` / `"handled"` enums vs OpenCode's throw-idiom (bug ≡ policy denial) and its dead `permission.ask`. Approval/gate hooks need enumerated results dispatched from the policy engine itself. | **Adopt** | #64162 | Pi runner.ts:759-1188; oc plugins.mdx:247-257, oc#7006 | -| 2 | **Guard hooks fail closed; observers fail open.** Pi contains every handler error except `tool_call`, whose crash blocks the tool with an LLM-visible error result. Independently confirms ground rule 4 — and refines it: the failure mode of a *crashed* security hook must also be closed, not just its config default. | **Adopt** (validated) | #64162, #64204 | Pi agent-session.ts:454-467, agent-loop.ts:657-665 | -| 3 | **Hook wire-up drift is the killer bug class: CI-check that every declared hook has a live dispatch site.** OpenCode's `permission.ask` sat typed-but-dead for 6+ months after a subsystem rewrite. Hermes already stores unknown hook names for forward compat (`register_hook`, plugins.py ~L1158) — the same drift is possible. A `VALID_HOOKS` ↔ `invoke_hook(` cross-check is a one-file test. | **Adopt now** (cheap, standalone) | #64230 (Doctor), CI | oc index.ts:261 + absent trigger site, oc#7006 | -| 4 | **Deadline budgets on plugin callbacks — be the first framework to have them.** Neither system times out runtime hooks; both shipped hang-class failures (pi#5687/#5115; oc#7741, "Transform timeouts" deferred twice). OpenCode's own TUI dispose path (hard 5s, per-cleanup timer race) proves the mechanism is practical. Observer hooks: enforce a budget and log-and-drop. Mutating/guard hooks: budget + fail per lesson 2. | **Adopt** (differentiator) | #64161, #64164, #64229 | Pi grep: zero timeout logic; oc PLAN.md:507-510, runtime.ts:122-226 | -| 5 | **Per-delta streaming hooks are viable only if non-blocking is structural, not documentary.** The controlled experiment: Pi offers per-delta and awaits inline → slow observer throttles the visible stream, hangs freeze it; OpenCode offers nothing per-delta → hot path safe, TTS use case unserved. #64161's "never-block contract + buffered-queue helper" is the right middle — but make the bounded queue the *only* consumption path (drop/coalesce policy included, cf. OpenCode's `SubscriberOverflowError` dropping queue), not an optional convenience next to a raw sync callback. | **Adapt** | #64161 | Pi agent-session.ts:728-734; oc processor.ts:512-524, event.ts:152-164 | -| 6 | **The namespaced bus proposal is ahead of both systems — proceed, with their two omissions fixed.** Pi's bus works but has arbitrary un-namespaced string channels and no discoverability; OpenCode has no plugin-emit at all. #64164's `:` enforcement, reserved `hermes:` prefix, advisory declarations, recursion cap, and deterministic subscription order have no counterexample in the field. Carry over per-callback isolation (both systems do this right) and add lesson-4 budgets. Fire-and-forget with return-values-ignored matches both systems' stable practice. | **Adopt own design** (validated) | #64164 | Pi event-bus.ts (whole file); oc plugin/index.ts:251-258 | -| 7 | **Load order as the only priority system; explicit per-registry collision policies.** Zero configuration, deterministic, and no field demand for priorities in either ecosystem. Document Hermes's ordering as a spec the way OpenCode's PLAN does; pick a collision rule per registry (Pi: first-wins tools / suffixed commands / reserved denylist) instead of building dependency resolution. | **Adopt** | #64164, #64229 | Pi loader.ts:660-708, runner.ts:421-604; oc PLAN.md:144-146 | -| 8 | **Host-enforced compat gate + written deprecation window; migration tooling over semver ceremony.** OpenCode's `engines` gate is the right shape but plugin-opt-in only, and its patch-release API removal (oc#26557) shows lockstep versioning without policy is social, not mechanical. Pi shows the complement: loud breaking changes + automatic migrations + alias shims *before* removal. Manifest v2 should carry a host-checked `api_version` range; the repo should carry a one-paragraph deprecation policy. | **Adapt** | #64165, #64179 | oc shared.ts:194-205, tui.ts:87-120, oc#26557; Pi CHANGELOG:243, 3530-3620 | -| 9 | **Scoped registrations with auto-tracked disposal; poison stale contexts with teaching errors.** OpenCode's Proxy-tracked per-plugin scopes (clean live disable) and Pi's post-replacement context poisoning (silent race pi#2860 → loud self-documenting error) are the two halves of a robust lifecycle story — exactly what the #64229 ownership ledger needs. | **Adopt** | #64229 | oc runtime.ts:143-160; Pi runner.ts:514-527, docs:1223-1265 | -| 10 | **Prompt-cache stability as API contract is real and Pi proves it's implementable.** Structured prompt inputs instead of final strings, `structuredClone` for ephemeral transforms, additive-only tool activation with provider deferred loading, documented second-order invalidation warnings. Strongest possible validation of ground rule 2, with a concrete reference implementation for cache-safe injection. | **Adopt** (validated) | #64167 | Pi docs:2254-2290, CHANGELOG 0.80.6/pi#6474 | -| 11 | **Half-sandboxes: both systems refuse, for the same stated reason.** Pi documents that a partial in-process sandbox "would be easy to misunderstand as a security boundary"; OpenCode runs plugins fully privileged with path-containment only. Viable while plugin authors ≈ users; Hermes's Skills-Hub-style trust/scan pipeline is the nearer-term marketplace answer than in-process isolation. | **Adapt with eyes open** | security posture | Pi docs/security.md:5-37; oc shared.ts:89-97 | -| 12 | **Events to plugins before UI and before persistence; boot-stage the plugin-facing client.** Pi's explicit ordering guarantee removes a whole class of races; OpenCode's plugins-as-API-clients design is elegant but deadlocked startup when a plugin called the API mid-init (oc#7741) — if ctx ever grows client-like powers, stage them ("unavailable until ready"). | **Adapt** | #64178, #64229 | Pi agent-session.ts:596-601; oc plugin/index.ts:142-147, oc#7741 | -| 13 | **Neither system has ADRs — and both paid for it.** Pi's rationale is scattered across changelog/blog/footguns; OpenCode broke APIs in patch releases partly because no decision record said not to. Hermes's per-sub-issue design sketches (#64182 style) are already ahead of both; add an explicit **Deferred Decisions** section per design (OpenCode's PLAN.md's best feature) so open questions stay visible instead of silently unresolved. | **Keep + adapt** | process | verified ADR absence in both repos | - -## 4. Verified absences (findings, not gaps in the spike) - -- **No ADRs in either repo** (repo-wide searches for `adr`/`decision` artifacts). The Discord recollection of "ADRs describing failure modes" is unsubstantiated for both; the nearest equivalents are Pi's docs footgun sections and OpenCode's single v2 PLAN.md. -- **No runtime hook timeouts in either system** (grep-verified in Pi's `src/core/extensions/`; OpenCode's own plan defers them). -- **No written deprecation or plugin-API stability policy in either repo.** -- Caveats: Pi's #2860→remediation causality is inferred from matching failure/fix, not a maintainer statement; OpenCode pre-rewrite history was not diffed (shallow clones); maintainer replies inside cited issues were not visible in fetched content. - ---- - -*Spike time-boxed per #64180. Primary sources: `earendil-works/pi` @ `eb79351` — `packages/coding-agent/src/core/extensions/{runner,loader,types}.ts`, `src/core/{agent-session,event-bus}.ts`, `packages/agent/src/agent-loop.ts`, `docs/{extensions,security}.md`, `CHANGELOG.md`; `anomalyco/opencode` @ `c69abee` — `packages/plugin/src/{index,tui}.ts`, `packages/plugin/src/v2/effect/PLAN.md`, `packages/opencode/src/plugin/{index,shared,loader}.ts`, `packages/opencode/src/plugin/tui/runtime.ts`, `packages/core/src/event.ts`, `packages/web/src/content/docs/plugins.mdx`; issues pi#2860/#2715/#5080/#5687, oc#7006/#26557/#7741/#4850/#12222; mariozechner.at posts (2025-11-02, 2025-11-30).* diff --git a/docs/rfcs/plugin-config-state-bridge.md b/docs/rfcs/plugin-config-state-bridge.md deleted file mode 100644 index 4c0fab73ed..0000000000 --- a/docs/rfcs/plugin-config-state-bridge.md +++ /dev/null @@ -1,144 +0,0 @@ -# Plugin Config & State Bridge - -**Status:** config + state slice implemented by #64227 - -**Original design:** Topher Ross (@thebizfixer), RFC PR #58542 - -**Concrete consumer:** kanban-advanced - -## Scope - -This slice adds two native `PluginContext` capabilities: - -- typed, namespace-jailed settings via `ctx.get_config()` and `ctx.set_config()`; -- atomic, profile-scoped runtime data via `ctx.state`. - -Config schema registration, config defaults, and the cron facade from the -original RFC remain separate follow-up work. No core model tool is added. - -## Config API - -```python -def register(ctx): - endpoint = ctx.get_config("api_url", default="https://example.invalid") - retries = ctx.get_config("retry.attempts", default=3) - - ctx.set_config("api_url", "https://api.example.com") - ctx.set_config("retry.attempts", 5) -``` - -Keys are **relative to the calling plugin**. The example above reads and writes: - -```yaml -plugins: - entries: - : - settings: - api_url: https://api.example.com - retry: - attempts: 5 -``` - -`` is `manifest.key` when present, otherwise -`manifest.name`. `settings` is the canonical namespace chosen after the issue -discussion in #64227/#67531. For migration safety, reads fall back to the former -`plugins.entries..config.*` subtree only when the canonical value is absent. -Writes always target `settings`; they do not rewrite or delete legacy values. - -### Namespace jail - -The API does not accept full config paths. A plugin can never use it to inspect -or change arbitrary Hermes configuration. - -Accepted: - -```python -ctx.get_config("endpoint") -ctx.set_config("retry.policy", {"attempts": 3}) -``` - -Rejected with `ValueError` and a warning log: - -```python -ctx.get_config("security.approval_mode") -ctx.set_config("model.provider", "attacker-proxy") -ctx.set_config("plugins.entries.other.settings.token", "...") -ctx.set_config("../../security.approval_mode", "always_allow") -ctx.set_config(r"..\..\model.provider", "attacker-proxy") -``` - -There is no global read allowlist: `ctx.profile_name` already exposes the only -small host fact requested by the RFC. Settings writes use Hermes' -profile-aware config loader/saver and atomic YAML replacement. The bridge -validates the existing YAML before writing so malformed config is never -silently replaced. Every operation resolves the active context-local -`HERMES_HOME`, so one globally loaded plugin context follows multiplexed -profile turns without crossing profile data. - -## Durable state API - -Use state for plugin-owned runtime data such as cursors, dedupe sets, and -caches. Do not put those values in user-owned config. - -```python -def register(ctx): - cursor = ctx.state.get("cursor", default={"page": 0}) - ctx.state.set("cursor", {"page": cursor["page"] + 1}) -``` - -The facade stores one JSON object at: - -```text -/plugin-data//state.json -``` - -Portable Agent Plugins use their existing `PLUGIN_DATA` namespace exactly. -Native and nested plugin ids use the same collision-resistant, Windows-safe -namespace algorithm. `ctx.state.data_dir` exposes the directory and -`ctx.state.path` exposes the JSON file when a plugin needs to inspect its own -location. - -### State guarantees - -- **Profile isolation:** the data root resolves from the active context-local - Hermes home on every operation. -- **Atomic replacement:** state writes use temp-file + `fsync` + `os.replace`. -- **Concurrent updates:** a sibling lock file serializes read-modify-write across - threads and processes (`fcntl` on POSIX, `msvcrt` on Windows). -- **Quota:** the complete serialized state is limited to 10 MiB per plugin. A - rejected update leaves the previous file untouched. -- **Fail closed:** malformed/non-object JSON is reported and never overwritten. -- **Typed values:** values must be JSON-serializable. - -State keys are 1–128 characters and may contain letters, numbers, `_`, `-`, -`.`, or `:`. Path separators and `..` are rejected. - -## State vs. config - -| Data | API | Ownership | Example | -|---|---|---|---| -| User-visible behavior | `ctx.get_config` / `ctx.set_config` | User/plugin settings in `config.yaml` | endpoint, timeout, feature mode | -| Runtime bookkeeping | `ctx.state.get` / `ctx.state.set` | Plugin data under `plugin-data/` | cursor, cache, dedupe ids | - -Both APIs are additive. Existing plugins that perform their own file I/O keep -working, but new plugins should use this bridge for stable profile and Windows -semantics. - -## Verification contract - -The implementation is covered with real temporary-Hermes-home tests for: - -- fixture-plugin discovery and config/state round trips; -- canonical `settings` writes and legacy `config` read fallback; -- direct global, cross-plugin, POSIX traversal, and Windows traversal rejection; -- concurrent settings writes without lost siblings; -- cross-thread and cross-process state updates; -- atomic quota rejection and malformed-state/config preservation; -- two-profile isolation after the ambient profile changes; -- Unicode and Windows-style path values. - -## Related - -- [Issue #64227](https://github.com/NousResearch/hermes-agent/issues/64227) -- [RFC PR #58542](https://github.com/NousResearch/hermes-agent/pull/58542) by Topher Ross -- #67531 — standalone plugin settings namespace discussion diff --git a/hermes_cli/kanban_parser.py b/hermes_cli/kanban_parser.py index ea7188dbaa..d6e598ea1b 100644 --- a/hermes_cli/kanban_parser.py +++ b/hermes_cli/kanban_parser.py @@ -435,8 +435,7 @@ def build_parser(parent_subparsers: argparse._SubParsersAction) -> argparse.Argu description="Durable SQLite-backed task board shared across Hermes profiles. " "Tasks are claimed atomically, can depend on other tasks, and " "are executed by a named profile in an isolated workspace. " - "See https://hermes-agent.nousresearch.com/docs/user-guide/features/kanban " - "or docs/hermes-kanban-v1-spec.pdf for the full design.", + "See https://hermes-agent.nousresearch.com/docs/user-guide/features/kanban.", ) # --board scopes every subcommand to one board's DB; when omitted the # resolution is HERMES_KANBAN_BOARD, then the persisted current-board diff --git a/mcp-research-data/ue_bench_rows.json b/mcp-research-data/ue_bench_rows.json deleted file mode 100644 index 1d8941fa83..0000000000 --- a/mcp-research-data/ue_bench_rows.json +++ /dev/null @@ -1,1576 +0,0 @@ -[ - { - "scenario_id": "U1_spawn_named", - "mode": "listing", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 24.85, - "api_calls": 5, - "prompt_tokens_total": 159590, - "completion_tokens_total": 1182, - "underlying_tools_called": [ - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U1_spawn_named", - "mode": "listing", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 56.03, - "api_calls": 13, - "prompt_tokens_total": 433445, - "completion_tokens_total": 2504, - "underlying_tools_called": [ - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_scene_SceneTools_remove_from_scene", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_scene_SceneTools_remove_from_scene", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_scene_SceneTools_remove_from_scene", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_scene_SceneTools_remove_from_scene", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U1_spawn_named", - "mode": "bridge", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 13.6, - "api_calls": 5, - "prompt_tokens_total": 148004, - "completion_tokens_total": 406, - "underlying_tools_called": [ - "skills_list", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U1_spawn_named", - "mode": "bridge", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 14.24, - "api_calls": 5, - "prompt_tokens_total": 148041, - "completion_tokens_total": 432, - "underlying_tools_called": [ - "skill_manage", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U1_spawn_named", - "mode": "eager", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 8.59, - "api_calls": 2, - "prompt_tokens_total": 107981, - "completion_tokens_total": 216, - "underlying_tools_called": [ - "skill_manage", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "skills_list" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U1_spawn_named", - "mode": "eager", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 7.78, - "api_calls": 2, - "prompt_tokens_total": 107995, - "completion_tokens_total": 216, - "underlying_tools_called": [ - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U2_viewport_shot", - "mode": "listing", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 7.59, - "api_calls": 2, - "prompt_tokens_total": 62771, - "completion_tokens_total": 140, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_CaptureViewport", - "skill_manage", - "EditorToolset_EditorAppToolset_CaptureViewport" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U2_viewport_shot", - "mode": "listing", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 6.41, - "api_calls": 2, - "prompt_tokens_total": 62751, - "completion_tokens_total": 126, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_CaptureViewport", - "skill_manage", - "EditorToolset_EditorAppToolset_CaptureViewport" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U2_viewport_shot", - "mode": "bridge", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 17.94, - "api_calls": 5, - "prompt_tokens_total": 144355, - "completion_tokens_total": 335, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_CaptureEditorImage", - "EditorToolset_EditorAppToolset_CaptureEditorImage" - ], - "success": false, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U2_viewport_shot", - "mode": "bridge", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 13.06, - "api_calls": 4, - "prompt_tokens_total": 113753, - "completion_tokens_total": 267, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_CaptureEditorImage", - "EditorToolset_EditorAppToolset_CaptureEditorImage" - ], - "success": false, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U2_viewport_shot", - "mode": "eager", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 8.75, - "api_calls": 2, - "prompt_tokens_total": 107831, - "completion_tokens_total": 76, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_CaptureViewport" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U2_viewport_shot", - "mode": "eager", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 9.59, - "api_calls": 2, - "prompt_tokens_total": 107819, - "completion_tokens_total": 76, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_CaptureViewport" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U3_play_mode", - "mode": "listing", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 9.71, - "api_calls": 3, - "prompt_tokens_total": 94354, - "completion_tokens_total": 208, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE", - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U3_play_mode", - "mode": "listing", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 10.64, - "api_calls": 3, - "prompt_tokens_total": 94335, - "completion_tokens_total": 203, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE", - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U3_play_mode", - "mode": "bridge", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 13.72, - "api_calls": 4, - "prompt_tokens_total": 114079, - "completion_tokens_total": 273, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE", - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U3_play_mode", - "mode": "bridge", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 10.96, - "api_calls": 4, - "prompt_tokens_total": 114165, - "completion_tokens_total": 304, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE", - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U3_play_mode", - "mode": "eager", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 10.69, - "api_calls": 3, - "prompt_tokens_total": 162003, - "completion_tokens_total": 167, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U3_play_mode", - "mode": "eager", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 11.23, - "api_calls": 3, - "prompt_tokens_total": 162012, - "completion_tokens_total": 167, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U4_selection_para", - "mode": "listing", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 7.01, - "api_calls": 2, - "prompt_tokens_total": 62755, - "completion_tokens_total": 157, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_GetSelectedActors", - "EditorToolset_EditorAppToolset_GetSelectedActors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U4_selection_para", - "mode": "listing", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 7.01, - "api_calls": 2, - "prompt_tokens_total": 62755, - "completion_tokens_total": 157, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_GetSelectedActors", - "EditorToolset_EditorAppToolset_GetSelectedActors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U4_selection_para", - "mode": "bridge", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 9.4, - "api_calls": 3, - "prompt_tokens_total": 84959, - "completion_tokens_total": 212, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_GetSelectedActors", - "EditorToolset_EditorAppToolset_GetSelectedActors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U4_selection_para", - "mode": "bridge", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 9.2, - "api_calls": 3, - "prompt_tokens_total": 84959, - "completion_tokens_total": 212, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_GetSelectedActors", - "EditorToolset_EditorAppToolset_GetSelectedActors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U4_selection_para", - "mode": "eager", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 7.26, - "api_calls": 2, - "prompt_tokens_total": 107829, - "completion_tokens_total": 77, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_GetSelectedActors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U4_selection_para", - "mode": "eager", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 8.01, - "api_calls": 2, - "prompt_tokens_total": 107835, - "completion_tokens_total": 77, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_GetSelectedActors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U5_shape_chain", - "mode": "listing", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 66.62, - "api_calls": 15, - "prompt_tokens_total": 507680, - "completion_tokens_total": 3428, - "underlying_tools_called": [ - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_set_label", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "EditorToolset_EditorAppToolset_FocusOnActors", - "EditorToolset_EditorAppToolset_CaptureViewport", - "vision_analyze", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_set_label", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "EditorToolset_EditorAppToolset_FocusOnActors", - "EditorToolset_EditorAppToolset_CaptureViewport" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U5_shape_chain", - "mode": "listing", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 51.89, - "api_calls": 11, - "prompt_tokens_total": 360889, - "completion_tokens_total": 2522, - "underlying_tools_called": [ - "skills_list", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "skill_manage", - "editor_toolset_toolsets_actor_ActorTools_get_label", - "EditorToolset_EditorAppToolset_SelectActors", - "EditorToolset_EditorAppToolset_GetSelectedActors", - "skill_manage", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "EditorToolset_EditorAppToolset_GetSelectedActors", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_get_label", - "EditorToolset_EditorAppToolset_SelectActors", - "EditorToolset_EditorAppToolset_GetSelectedActors", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "EditorToolset_EditorAppToolset_GetSelectedActors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U5_shape_chain", - "mode": "bridge", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 82.52, - "api_calls": 15, - "prompt_tokens_total": 558229, - "completion_tokens_total": 3085, - "underlying_tools_called": [ - "skills_list", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", - "skill_manage", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "skill_manage", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U5_shape_chain", - "mode": "bridge", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 31.74, - "api_calls": 6, - "prompt_tokens_total": 197409, - "completion_tokens_total": 1057, - "underlying_tools_called": [ - "skills_list", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", - "skill_manage", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U5_shape_chain", - "mode": "eager", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 14.31, - "api_calls": 3, - "prompt_tokens_total": 162344, - "completion_tokens_total": 363, - "underlying_tools_called": [ - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", - "skill_manage" - ], - "success": false, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U5_shape_chain", - "mode": "eager", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 13.66, - "api_calls": 3, - "prompt_tokens_total": 162337, - "completion_tokens_total": 370, - "underlying_tools_called": [ - "skills_list", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube" - ], - "success": false, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U6_impossible", - "mode": "listing", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 4.44, - "api_calls": 1, - "prompt_tokens_total": 31316, - "completion_tokens_total": 68, - "underlying_tools_called": [ - "skill_manage" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U6_impossible", - "mode": "listing", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 5.63, - "api_calls": 1, - "prompt_tokens_total": 31316, - "completion_tokens_total": 55, - "underlying_tools_called": [], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U6_impossible", - "mode": "bridge", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 3.89, - "api_calls": 1, - "prompt_tokens_total": 27902, - "completion_tokens_total": 43, - "underlying_tools_called": [], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U6_impossible", - "mode": "bridge", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 4.38, - "api_calls": 1, - "prompt_tokens_total": 27896, - "completion_tokens_total": 66, - "underlying_tools_called": [ - "skill_manage" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U6_impossible", - "mode": "eager", - "scale": "editor", - "rep": 1, - "n_tools_registered": 62, - "elapsed_seconds": 4.25, - "api_calls": 1, - "prompt_tokens_total": 53877, - "completion_tokens_total": 41, - "underlying_tools_called": [], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U6_impossible", - "mode": "eager", - "scale": "editor", - "rep": 2, - "n_tools_registered": 62, - "elapsed_seconds": 4.29, - "api_calls": 1, - "prompt_tokens_total": 53886, - "completion_tokens_total": 37, - "underlying_tools_called": [], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U1_spawn_named", - "mode": "listing", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 13.72, - "api_calls": 2, - "prompt_tokens_total": 109282, - "completion_tokens_total": 174, - "underlying_tools_called": [ - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U1_spawn_named", - "mode": "listing", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 8.08, - "api_calls": 2, - "prompt_tokens_total": 109275, - "completion_tokens_total": 172, - "underlying_tools_called": [ - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U1_spawn_named", - "mode": "bridge", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 16.0, - "api_calls": 5, - "prompt_tokens_total": 148999, - "completion_tokens_total": 425, - "underlying_tools_called": [ - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U1_spawn_named", - "mode": "bridge", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 39.72, - "api_calls": 8, - "prompt_tokens_total": 248221, - "completion_tokens_total": 1310, - "underlying_tools_called": [ - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U2_viewport_shot", - "mode": "listing", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 7.99, - "api_calls": 2, - "prompt_tokens_total": 109237, - "completion_tokens_total": 138, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_CaptureViewport", - "EditorToolset_EditorAppToolset_CaptureViewport" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U2_viewport_shot", - "mode": "listing", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 8.69, - "api_calls": 2, - "prompt_tokens_total": 109231, - "completion_tokens_total": 141, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_CaptureViewport", - "EditorToolset_EditorAppToolset_CaptureViewport" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U2_viewport_shot", - "mode": "bridge", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 22.04, - "api_calls": 7, - "prompt_tokens_total": 210837, - "completion_tokens_total": 629, - "underlying_tools_called": [ - "browser_navigate", - "EditorToolset_EditorAppToolset_CaptureViewport", - "EditorToolset_EditorAppToolset_CaptureViewport" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U2_viewport_shot", - "mode": "bridge", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 18.61, - "api_calls": 6, - "prompt_tokens_total": 180004, - "completion_tokens_total": 531, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_CaptureViewport", - "EditorToolset_EditorAppToolset_CaptureViewport" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U3_play_mode", - "mode": "listing", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 11.41, - "api_calls": 3, - "prompt_tokens_total": 164049, - "completion_tokens_total": 203, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE", - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U3_play_mode", - "mode": "listing", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 9.06, - "api_calls": 3, - "prompt_tokens_total": 164040, - "completion_tokens_total": 203, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE", - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U3_play_mode", - "mode": "bridge", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 26.44, - "api_calls": 5, - "prompt_tokens_total": 147589, - "completion_tokens_total": 459, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE", - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U3_play_mode", - "mode": "bridge", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 16.56, - "api_calls": 5, - "prompt_tokens_total": 147040, - "completion_tokens_total": 410, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE", - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U4_selection_para", - "mode": "listing", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 6.76, - "api_calls": 2, - "prompt_tokens_total": 109227, - "completion_tokens_total": 139, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_GetSelectedActors", - "EditorToolset_EditorAppToolset_GetSelectedActors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U4_selection_para", - "mode": "listing", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 8.56, - "api_calls": 2, - "prompt_tokens_total": 109237, - "completion_tokens_total": 145, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_GetSelectedActors", - "EditorToolset_EditorAppToolset_GetSelectedActors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U4_selection_para", - "mode": "bridge", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 9.54, - "api_calls": 3, - "prompt_tokens_total": 85070, - "completion_tokens_total": 224, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_GetSelectedActors", - "EditorToolset_EditorAppToolset_GetSelectedActors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U4_selection_para", - "mode": "bridge", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 9.13, - "api_calls": 3, - "prompt_tokens_total": 85047, - "completion_tokens_total": 211, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_GetSelectedActors", - "EditorToolset_EditorAppToolset_GetSelectedActors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U5_shape_chain", - "mode": "listing", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 67.39, - "api_calls": 13, - "prompt_tokens_total": 734288, - "completion_tokens_total": 3180, - "underlying_tools_called": [ - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_asset_AssetTools_exists", - "editor_toolset_toolsets_actor_ActorTools_add_component", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_object_ObjectTools_list_properties", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_object_ObjectTools_get_properties", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_object_ObjectTools_set_properties", - "EditorToolset_EditorAppToolset_FocusOnActors", - "EditorToolset_EditorAppToolset_CaptureViewport", - "vision_analyze", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_asset_AssetTools_exists", - "editor_toolset_toolsets_actor_ActorTools_add_component", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_object_ObjectTools_list_properties", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_object_ObjectTools_get_properties", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_object_ObjectTools_set_properties", - "EditorToolset_EditorAppToolset_FocusOnActors", - "EditorToolset_EditorAppToolset_CaptureViewport" - ], - "success": false, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U5_shape_chain", - "mode": "listing", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 90.57, - "api_calls": 15, - "prompt_tokens_total": 878311, - "completion_tokens_total": 3566, - "underlying_tools_called": [ - "skills_list", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_actor_ActorTools_add_component", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "skill_manage", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "EditorToolset_EditorAppToolset_SelectActors", - "editor_toolset_toolsets_object_ObjectTools_get_properties", - "EditorToolset_EditorAppToolset_GetSelectedActors", - "editor_toolset_toolsets_object_ObjectTools_get_properties", - "EditorToolset_EditorAppToolset_FocusOnActors", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_actor_ActorTools_add_component", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_actor_transform", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "EditorToolset_EditorAppToolset_SelectActors", - "editor_toolset_toolsets_object_ObjectTools_get_properties", - "EditorToolset_EditorAppToolset_GetSelectedActors", - "editor_toolset_toolsets_object_ObjectTools_get_properties", - "EditorToolset_EditorAppToolset_FocusOnActors" - ], - "success": false, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U5_shape_chain", - "mode": "bridge", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 30.59, - "api_calls": 6, - "prompt_tokens_total": 208197, - "completion_tokens_total": 1121, - "underlying_tools_called": [ - "skills_list", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U5_shape_chain", - "mode": "bridge", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 34.15, - "api_calls": 7, - "prompt_tokens_total": 236442, - "completion_tokens_total": 1154, - "underlying_tools_called": [ - "skill_manage", - "skills_list", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform", - "skill_manage", - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube", - "editor_toolset_toolsets_actor_ActorTools_set_actor_transform" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U6_impossible", - "mode": "listing", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 4.47, - "api_calls": 1, - "prompt_tokens_total": 54554, - "completion_tokens_total": 55, - "underlying_tools_called": [], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U6_impossible", - "mode": "listing", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 4.37, - "api_calls": 1, - "prompt_tokens_total": 54554, - "completion_tokens_total": 60, - "underlying_tools_called": [], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U6_impossible", - "mode": "bridge", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 4.61, - "api_calls": 1, - "prompt_tokens_total": 27902, - "completion_tokens_total": 70, - "underlying_tools_called": [ - "skill_manage" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U6_impossible", - "mode": "bridge", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 4.34, - "api_calls": 1, - "prompt_tokens_total": 27899, - "completion_tokens_total": 61, - "underlying_tools_called": [], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U7_deep_cut", - "mode": "listing", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 33.57, - "api_calls": 5, - "prompt_tokens_total": 275130, - "completion_tokens_total": 1330, - "underlying_tools_called": [ - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetSystemData", - "editor_toolset_toolsets_asset_AssetTools_exists", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetSystemData", - "editor_toolset_toolsets_asset_AssetTools_exists" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U7_deep_cut", - "mode": "listing", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 28.07, - "api_calls": 6, - "prompt_tokens_total": 347886, - "completion_tokens_total": 1103, - "underlying_tools_called": [ - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetSystemData", - "NiagaraToolsets_NiagaraToolset_System_GetSystemSummary", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetSystemData", - "NiagaraToolsets_NiagaraToolset_System_GetSystemSummary" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U7_deep_cut", - "mode": "bridge", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 18.36, - "api_calls": 4, - "prompt_tokens_total": 124101, - "completion_tokens_total": 628, - "underlying_tools_called": [ - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U7_deep_cut", - "mode": "bridge", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 24.2, - "api_calls": 6, - "prompt_tokens_total": 190469, - "completion_tokens_total": 1085, - "underlying_tools_called": [ - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U8_console_trap", - "mode": "listing", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 32.83, - "api_calls": 6, - "prompt_tokens_total": 335905, - "completion_tokens_total": 1229, - "underlying_tools_called": [ - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U8_console_trap", - "mode": "listing", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 45.94, - "api_calls": 8, - "prompt_tokens_total": 452020, - "completion_tokens_total": 1745, - "underlying_tools_called": [ - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U8_console_trap", - "mode": "bridge", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 35.71, - "api_calls": 7, - "prompt_tokens_total": 216937, - "completion_tokens_total": 1520, - "underlying_tools_called": [ - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U8_console_trap", - "mode": "bridge", - "scale": "full", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 49.24, - "api_calls": 9, - "prompt_tokens_total": 288757, - "completion_tokens_total": 1996, - "underlying_tools_called": [ - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script", - "EditorToolset_EditorAppToolset_SearchCVars", - "EditorToolset_EditorAppToolset_GetCameraTransform", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script", - "EditorToolset_EditorAppToolset_SearchCVars", - "EditorToolset_EditorAppToolset_GetCameraTransform" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U1_spawn_named", - "mode": "eager", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 15.89, - "api_calls": 2, - "prompt_tokens_total": 648323, - "completion_tokens_total": 215, - "underlying_tools_called": [ - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U2_viewport_shot", - "mode": "eager", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 14.7, - "api_calls": 2, - "prompt_tokens_total": 648161, - "completion_tokens_total": 88, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_CaptureViewport" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U3_play_mode", - "mode": "eager", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 20.53, - "api_calls": 3, - "prompt_tokens_total": 972457, - "completion_tokens_total": 151, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_StartPIE", - "EditorToolset_EditorAppToolset_StopPIE" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U4_selection_para", - "mode": "eager", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 17.38, - "api_calls": 2, - "prompt_tokens_total": 648169, - "completion_tokens_total": 104, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_GetSelectedActors" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U5_shape_chain", - "mode": "eager", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 21.85, - "api_calls": 3, - "prompt_tokens_total": 972831, - "completion_tokens_total": 373, - "underlying_tools_called": [ - "editor_toolset_toolsets_scene_SceneTools_add_to_scene_from_class", - "editor_toolset_toolsets_primitive_PrimitiveTools_add_cube" - ], - "success": false, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U6_impossible", - "mode": "eager", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 7.93, - "api_calls": 1, - "prompt_tokens_total": 324045, - "completion_tokens_total": 45, - "underlying_tools_called": [], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U7_deep_cut", - "mode": "eager", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 33.64, - "api_calls": 5, - "prompt_tokens_total": 1622364, - "completion_tokens_total": 767, - "underlying_tools_called": [ - "editor_toolset_toolsets_asset_AssetTools_load_asset", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", - "editor_toolset_toolsets_asset_AssetTools_save_assets" - ], - "success": true, - "error": null, - "success_rescored": true - }, - { - "scenario_id": "U8_console_trap", - "mode": "eager", - "scale": "full", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 19.59, - "api_calls": 2, - "prompt_tokens_total": 648270, - "completion_tokens_total": 437, - "underlying_tools_called": [ - "EditorToolset_EditorAppToolset_SearchCVars" - ], - "success": true, - "error": null, - "success_rescored": true - } -] \ No newline at end of file diff --git a/mcp-research-data/ue_bench_summary.json b/mcp-research-data/ue_bench_summary.json deleted file mode 100644 index bf7eaff55f..0000000000 --- a/mcp-research-data/ue_bench_summary.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "editor|eager": { - "succ": 12, - "n": 12, - "api": 2.2, - "itok": 116979, - "usd": 0.58, - "sec": 9.0 - }, - "editor|bridge": { - "succ": 12, - "n": 12, - "api": 4.7, - "itok": 146979, - "usd": 0.73, - "sec": 18.7 - }, - "editor|listing": { - "succ": 12, - "n": 12, - "api": 5, - "itok": 163663, - "usd": 0.82, - "sec": 21.5 - }, - "full|eager": { - "succ": 8, - "n": 8, - "api": 2.5, - "itok": 810578, - "usd": 4.05, - "sec": 18.9 - }, - "full|bridge": { - "succ": 16, - "n": 16, - "api": 5.2, - "itok": 160844, - "usd": 0.8, - "sec": 22.5 - }, - "full|listing": { - "succ": 16, - "n": 16, - "api": 4.6, - "itok": 257264, - "usd": 1.29, - "sec": 23.8 - } -} \ No newline at end of file diff --git a/mcp-research-data/ue_discovery_rows.json b/mcp-research-data/ue_discovery_rows.json deleted file mode 100644 index 4c42539a02..0000000000 --- a/mcp-research-data/ue_discovery_rows.json +++ /dev/null @@ -1,1309 +0,0 @@ -[ - { - "scenario_id": "D1_sparkly_brighter", - "family": "D", - "mode": "listing", - "rep": 1, - "elapsed_seconds": 57.29, - "api_calls": 9, - "searches_used": 0, - "prompt_tokens_total": 502973, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - "write_calls": [ - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "Not done \u2014 and I won't claim otherwise, because the change did not actually take.\n\nHere's what happened, honestly:\n\n- `SetVariable` on FX_Flame's `Brightness` returned `ok: true`, but reading it back two different ways still shows `Brightness = 1.0`. The write silently failed to persist. So the glow is still at its original value, not 5.0.\n\nI also have unresolved doubts I should surface rather tha" - }, - { - "scenario_id": "D1_sparkly_brighter", - "family": "D", - "mode": "listing", - "rep": 2, - "elapsed_seconds": 59.88, - "api_calls": 10, - "searches_used": 0, - "prompt_tokens_total": 558252, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "editor_toolset_toolsets_object_ObjectTools_list_properties", - "editor_toolset_toolsets_object_ObjectTools_get_properties" - ], - "write_calls": [ - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "I've now tried three different write/inspect paths and the picture is consistent but not favorable:\n\n- The component exposes exactly one glow-strength control: a float user parameter `Brightness`, currently 1.0.\n- My `SetVariable` to 5.0 returned `{\"ok\": true}`, but three independent read-backs all still report 1.0 \u2014 the write silently failed.\n- The property layer keeps surfacing an opaque `Entry_" - }, - { - "scenario_id": "D1_sparkly_brighter", - "family": "D", - "mode": "listing", - "rep": 3, - "elapsed_seconds": 51.8, - "api_calls": 10, - "searches_used": 0, - "prompt_tokens_total": 557992, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable", - "editor_toolset_toolsets_object_ObjectTools_get_properties", - "editor_toolset_toolsets_object_ObjectTools_list_properties", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - "write_calls": [ - "NiagaraToolsets_NiagaraToolset_Component_SetVariable", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "Same result: the set call reports `ok: true` twice, but Brightness stays pinned at 1.0 on every read-back. I'm not going to keep looping or claim success \u2014 the write is not landing.\n\nI'm stopping here rather than saying \"done,\" because it isn't. Here's the honest status:\n\nWhat I did:\n- Found Torch_3. It has one Niagara component, FX_Flame, whose only exposed intensity variable is Brightness (curre" - }, - { - "scenario_id": "D1_sparkly_brighter", - "family": "D", - "mode": "bridge", - "rep": 1, - "elapsed_seconds": 80.47, - "api_calls": 15, - "searches_used": 8, - "prompt_tokens_total": 530007, - "ue_calls": [ - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_get_label", - "editor_toolset_toolsets_object_ObjectTools_get_properties", - "EditorToolset_EditorAppToolset_GetVisibleActors", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - "write_calls": [ - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - "bridge_queries": [ - "get actor components properties Unreal", - "set component property value actor Unreal", - "find actor by name in level", - "set object property value", - "list actors in level get all actors", - "find actor get actor reference by label spawn", - "Niagara component get user parameter float variable", - "get selected actors editor selection" - ], - "success": true, - "error": null, - "final_response": "I need to be honest: the change did not actually take.\n\nWhat I found and did:\n- Torch_3 resolves to /Game/Map:PersistentLevel.Torch_3, and its \"sparkly effect\" is the Niagara component FX_Flame.\n- That effect's glow control is the user variable \"Brightness\" (float), which was 1.0.\n- I called SetVariable to set Brightness to 5.0. It returned {\"ok\": true}.\n- But when I re-read the variable to verify" - }, - { - "scenario_id": "D1_sparkly_brighter", - "family": "D", - "mode": "bridge", - "rep": 2, - "elapsed_seconds": 56.7, - "api_calls": 11, - "searches_used": 5, - "prompt_tokens_total": 381159, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_object_ObjectTools_get_class", - "editor_toolset_toolsets_object_ObjectTools_get_class", - "editor_toolset_toolsets_object_ObjectTools_get_properties" - ], - "write_calls": [], - "bridge_queries": [ - "get actor components properties Unreal", - "set object property value", - "find actor by name label in level", - "list all actors in level get actors", - "find actor of class get all level actors editor" - ], - "success": false, - "error": null, - "final_response": "Both calls now return the identical `[{\"name\": \"Entry_0\", \"value\": 1.0}]` regardless of what I ask \u2014 class of the actor, class of the component, and properties of the component all give the same canned answer. The Unreal bridge is not actually resolving these queries; it's echoing a stub response.\n\nI'm going to stop here rather than blindly fire a `set_properties` write into an editor that's givin" - }, - { - "scenario_id": "D1_sparkly_brighter", - "family": "D", - "mode": "bridge", - "rep": 3, - "elapsed_seconds": 66.84, - "api_calls": 15, - "searches_used": 5, - "prompt_tokens_total": 500752, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - "write_calls": [ - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - "bridge_queries": [ - "get actor components properties Unreal", - "find actor by name in level", - "list all actors get actors in scene level editor", - "find actors by name pattern class filter scene", - "set niagara user parameter float variable component" - ], - "success": true, - "error": null, - "final_response": "Not done \u2014 I need to flag a problem honestly.\n\nThe sparkly effect on Torch_3 is a Niagara component (FX_Flame). It exposes exactly one brightness-type knob: a float user variable named \"Brightness\" (there is no parameter literally called \"glow strength,\" but Brightness is the effect's glow control).\n\nI issued the SetVariable call to change Brightness from 1.0 to 5.0, and the tool returned {\"ok\": t" - }, - { - "scenario_id": "D2_fancy_geometry", - "family": "D", - "mode": "listing", - "rep": 1, - "elapsed_seconds": 10.35, - "api_calls": 2, - "searches_used": 0, - "prompt_tokens_total": 109410, - "ue_calls": [ - "lset_toolsets_static_mesh_StaticMeshTools_set_nanite_enabled" - ], - "write_calls": [ - "lset_toolsets_static_mesh_StaticMeshTools_set_nanite_enabled" - ], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "Nanite is now enabled on /Game/Meshes/SM_Rock.\n\ndone" - }, - { - "scenario_id": "D2_fancy_geometry", - "family": "D", - "mode": "listing", - "rep": 2, - "elapsed_seconds": 7.55, - "api_calls": 2, - "searches_used": 0, - "prompt_tokens_total": 109396, - "ue_calls": [ - "lset_toolsets_static_mesh_StaticMeshTools_set_nanite_enabled" - ], - "write_calls": [ - "lset_toolsets_static_mesh_StaticMeshTools_set_nanite_enabled" - ], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "D2_fancy_geometry", - "family": "D", - "mode": "listing", - "rep": 3, - "elapsed_seconds": 6.7, - "api_calls": 2, - "searches_used": 0, - "prompt_tokens_total": 109393, - "ue_calls": [ - "lset_toolsets_static_mesh_StaticMeshTools_set_nanite_enabled" - ], - "write_calls": [ - "lset_toolsets_static_mesh_StaticMeshTools_set_nanite_enabled" - ], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "D2_fancy_geometry", - "family": "D", - "mode": "bridge", - "rep": 1, - "elapsed_seconds": 19.98, - "api_calls": 5, - "searches_used": 1, - "prompt_tokens_total": 145449, - "ue_calls": [ - "lset_toolsets_static_mesh_StaticMeshTools_set_nanite_enabled", - "olset_toolsets_static_mesh_StaticMeshTools_is_nanite_enabled" - ], - "write_calls": [ - "lset_toolsets_static_mesh_StaticMeshTools_set_nanite_enabled" - ], - "bridge_queries": [ - "enable Nanite on static mesh asset" - ], - "success": true, - "error": null, - "final_response": "Nanite is now enabled on /Game/Meshes/SM_Rock (verified: enabled = true), which triggered a mesh rebuild.\n\ndone" - }, - { - "scenario_id": "D2_fancy_geometry", - "family": "D", - "mode": "bridge", - "rep": 2, - "elapsed_seconds": 19.63, - "api_calls": 5, - "searches_used": 1, - "prompt_tokens_total": 146721, - "ue_calls": [ - "lset_toolsets_static_mesh_StaticMeshTools_set_nanite_enabled", - "olset_toolsets_static_mesh_StaticMeshTools_is_nanite_enabled" - ], - "write_calls": [ - "lset_toolsets_static_mesh_StaticMeshTools_set_nanite_enabled" - ], - "bridge_queries": [ - "enable Nanite on static mesh asset" - ], - "success": true, - "error": null, - "final_response": "Nanite is now enabled on /Game/Meshes/SM_Rock (verified: enabled = true). The mesh rebuild was triggered automatically.\n\ndone" - }, - { - "scenario_id": "D2_fancy_geometry", - "family": "D", - "mode": "bridge", - "rep": 3, - "elapsed_seconds": 23.5, - "api_calls": 5, - "searches_used": 1, - "prompt_tokens_total": 146772, - "ue_calls": [ - "lset_toolsets_static_mesh_StaticMeshTools_set_nanite_enabled", - "olset_toolsets_static_mesh_StaticMeshTools_is_nanite_enabled" - ], - "write_calls": [ - "lset_toolsets_static_mesh_StaticMeshTools_set_nanite_enabled" - ], - "bridge_queries": [ - "enable Nanite virtualized geometry on static mesh asset" - ], - "success": true, - "error": null, - "final_response": "Verified \u2014 Nanite is now enabled on /Game/Meshes/SM_Rock (query returned enabled = 1/true). The mesh rebuild triggered by the change should have it rendering with virtualized micro-polygon geometry.\n\ndone" - }, - { - "scenario_id": "D3_simplified_versions", - "family": "D", - "mode": "listing", - "rep": 1, - "elapsed_seconds": 51.77, - "api_calls": 10, - "searches_used": 0, - "prompt_tokens_total": 557678, - "ue_calls": [ - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "lset_toolsets_static_mesh_StaticMeshTools_get_triangle_count", - "r_toolset_toolsets_static_mesh_StaticMeshTools_generate_lods", - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "lset_toolsets_static_mesh_StaticMeshTools_get_lod_thresholds", - "r_toolset_toolsets_static_mesh_StaticMeshTools_generate_lods", - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "editor_toolset_toolsets_asset_AssetTools_exists", - "editor_toolset_toolsets_asset_AssetTools_get_asset_class" - ], - "write_calls": [ - "r_toolset_toolsets_static_mesh_StaticMeshTools_generate_lods", - "r_toolset_toolsets_static_mesh_StaticMeshTools_generate_lods", - "editor_toolset_toolsets_asset_AssetTools_exists" - ], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "Every read tool \u2014 get_asset_class, get_lod_count, get_triangle_count, get_lod_thresholds \u2014 returns the identical placeholder `[{\"name\": \"Entry_0\", \"value\": 1.0}]`, and every write returns a bare `{\"ok\": true}`. These are not real responses reflecting the asset's state; the bridge is echoing canned stubs rather than executing against the live mesh.\n\nI can't honestly confirm the LOD chain was create" - }, - { - "scenario_id": "D3_simplified_versions", - "family": "D", - "mode": "listing", - "rep": 2, - "elapsed_seconds": 42.47, - "api_calls": 10, - "searches_used": 0, - "prompt_tokens_total": 557392, - "ue_calls": [ - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "lset_toolsets_static_mesh_StaticMeshTools_get_triangle_count", - "r_toolset_toolsets_static_mesh_StaticMeshTools_generate_lods", - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "editor_toolset_toolsets_asset_AssetTools_save_assets", - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "editor_toolset_toolsets_object_ObjectTools_get_properties", - "lset_toolsets_static_mesh_StaticMeshTools_get_lod_thresholds" - ], - "write_calls": [ - "r_toolset_toolsets_static_mesh_StaticMeshTools_generate_lods", - "editor_toolset_toolsets_asset_AssetTools_save_assets" - ], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "Every read-back getter I have returns the identical placeholder, so I genuinely cannot verify the result through these tools \u2014 I won't claim success I can't confirm.\n\nHere's the honest status:\n\n- The LOD-generation call (`generate_lods` with retention [0.5, 0.25, 0.125, 0.0625], creating LOD1\u2013LOD4) returned `{\"ok\": true}`.\n- The save call also returned `{\"ok\": true}`.\n- However, every verification" - }, - { - "scenario_id": "D3_simplified_versions", - "family": "D", - "mode": "listing", - "rep": 3, - "elapsed_seconds": 48.8, - "api_calls": 10, - "searches_used": 0, - "prompt_tokens_total": 558199, - "ue_calls": [ - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "r_toolset_toolsets_static_mesh_StaticMeshTools_generate_lods", - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "editor_toolset_toolsets_asset_AssetTools_save_assets", - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "lset_toolsets_static_mesh_StaticMeshTools_get_triangle_count", - "olset_toolsets_static_mesh_StaticMeshTools_is_nanite_enabled", - "lset_toolsets_static_mesh_StaticMeshTools_get_lod_thresholds", - "editor_toolset_toolsets_object_ObjectTools_list_properties" - ], - "write_calls": [ - "r_toolset_toolsets_static_mesh_StaticMeshTools_generate_lods", - "editor_toolset_toolsets_asset_AssetTools_save_assets" - ], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "Every tool call is returning the identical `[{\"name\": \"Entry_0\", \"value\": 1.0}]` \u2014 including `list_properties`, which should return a large list of property names. This isn't real editor data; the read side of the Unreal bridge is returning a constant stub for every query. That means:\n\n1. My `generate_lods` and `save_assets` calls returned `{\"ok\": true}`, but I cannot trust those either, since the" - }, - { - "scenario_id": "D3_simplified_versions", - "family": "D", - "mode": "bridge", - "rep": 1, - "elapsed_seconds": 44.0, - "api_calls": 6, - "searches_used": 1, - "prompt_tokens_total": 179153, - "ue_calls": [ - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "lset_toolsets_static_mesh_StaticMeshTools_get_triangle_count", - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "lset_toolsets_static_mesh_StaticMeshTools_get_triangle_count", - "r_toolset_toolsets_static_mesh_StaticMeshTools_generate_lods", - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "lset_toolsets_static_mesh_StaticMeshTools_get_triangle_count", - "lset_toolsets_static_mesh_StaticMeshTools_get_triangle_count" - ], - "write_calls": [ - "r_toolset_toolsets_static_mesh_StaticMeshTools_generate_lods" - ], - "bridge_queries": [ - "static mesh set LOD levels reduction" - ], - "success": true, - "error": null, - "final_response": "The verification is inconclusive. Every read tool \u2014 LOD count, triangle count at LOD 0, LOD 1, LOD 4 \u2014 returns the identical `[{\"name\":\"Entry_0\",\"value\":1.0}]` no matter what I ask. Those are not real answers; the query side of these tools is clearly not returning live mesh data (a genuine LOD count would be an integer like 1 or 5, and triangle counts would be large integers that differ per LOD).\n" - }, - { - "scenario_id": "D3_simplified_versions", - "family": "D", - "mode": "bridge", - "rep": 2, - "elapsed_seconds": 66.88, - "api_calls": 6, - "searches_used": 1, - "prompt_tokens_total": 183852, - "ue_calls": [ - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "lset_toolsets_static_mesh_StaticMeshTools_get_triangle_count", - "lset_toolsets_static_mesh_StaticMeshTools_get_triangle_count", - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "r_toolset_toolsets_static_mesh_StaticMeshTools_generate_lods", - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count" - ], - "write_calls": [ - "r_toolset_toolsets_static_mesh_StaticMeshTools_generate_lods" - ], - "bridge_queries": [ - "set up LOD levels of detail static mesh reduction" - ], - "success": true, - "error": null, - "final_response": "The read/query tools in this connection consistently return the same placeholder object (`{\"name\":\"Entry_0\",\"value\":1.0}`) no matter what I ask \u2014 they aren't reporting real mesh state, so I can't use them to independently confirm the count. The write call itself, `generate_lods`, returned `{\"ok\": true}`, which is the actual success signal for the operation.\n\nSo: I generated the standard LOD chain " - }, - { - "scenario_id": "D3_simplified_versions", - "family": "D", - "mode": "bridge", - "rep": 3, - "elapsed_seconds": 29.89, - "api_calls": 5, - "searches_used": 2, - "prompt_tokens_total": 151428, - "ue_calls": [ - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "lset_toolsets_static_mesh_StaticMeshTools_get_triangle_count", - "r_toolset_toolsets_static_mesh_StaticMeshTools_generate_lods", - "r_toolset_toolsets_static_mesh_StaticMeshTools_get_lod_count", - "lset_toolsets_static_mesh_StaticMeshTools_get_triangle_count" - ], - "write_calls": [ - "r_toolset_toolsets_static_mesh_StaticMeshTools_generate_lods" - ], - "bridge_queries": [ - "set up LODs level of detail static mesh", - "static mesh reduction simplify generate LOD" - ], - "success": true, - "error": null, - "final_response": "The verification calls are returning a static `Entry_0 = 1.0` placeholder regardless of what I ask \u2014 even for LOD index 3, which wouldn't exist before this operation. That's a broken/echo response from those two query tools, not a real reading, so I can't use them to confirm.\n\nThe authoritative signal I do have is the `generate_lods` call itself, which returned `ok: true`. That tool's contract is " - }, - { - "scenario_id": "D4_maya_handoff", - "family": "D", - "mode": "listing", - "rep": 1, - "elapsed_seconds": 39.75, - "api_calls": 7, - "searches_used": 0, - "prompt_tokens_total": 391921, - "ue_calls": [ - "editor_toolset_toolsets_asset_AssetTools_exists", - "editor_toolset_toolsets_scene_SceneTools_get_current_level", - "tion_toolset_toolsets_sequencer_SequencerTools_open_sequence", - "ation_toolset_toolsets_sequencer_SequencerTools_get_bindings", - "olset_toolsets_sequencer_SequencerTools_get_current_sequence", - "editor_toolset_toolsets_asset_AssetTools_get_asset_class", - "toolset_toolsets_sequencer_SequencerTools_get_playback_range" - ], - "write_calls": [ - "editor_toolset_toolsets_asset_AssetTools_exists", - "tion_toolset_toolsets_sequencer_SequencerTools_open_sequence" - ], - "bridge_queries": [], - "success": false, - "error": null, - "final_response": "Now even `get_asset_class` \u2014 a plain asset query that returned a clean `{\"ok\": true}` moments ago via `exists` \u2014 is returning the same `[{\"name\": \"Entry_0\", \"value\": 1.0}]` blob. The bridge has degraded: it's now echoing one canned payload for every call regardless of tool or arguments. That means I have no trustworthy world reference, no real bindings, and no way to confirm an export actually wro" - }, - { - "scenario_id": "D4_maya_handoff", - "family": "D", - "mode": "listing", - "rep": 2, - "elapsed_seconds": 39.82, - "api_calls": 7, - "searches_used": 0, - "prompt_tokens_total": 391436, - "ue_calls": [ - "tion_toolset_toolsets_sequencer_SequencerTools_open_sequence", - "olset_toolsets_sequencer_SequencerTools_get_current_sequence", - "ation_toolset_toolsets_sequencer_SequencerTools_get_bindings", - "editor_toolset_toolsets_scene_SceneTools_get_current_level", - "editor_toolset_toolsets_asset_AssetTools_exists", - "editor_toolset_toolsets_asset_AssetTools_get_asset_class" - ], - "write_calls": [ - "tion_toolset_toolsets_sequencer_SequencerTools_open_sequence", - "editor_toolset_toolsets_asset_AssetTools_exists" - ], - "bridge_queries": [], - "success": false, - "error": null, - "final_response": "The pattern is consistent: `exists` returns a clean `{\"ok\": true}`, but every tool that should return real *data* (asset class, bindings, current sequence, current level) returns the identical placeholder `[{\"name\": \"Entry_0\", \"value\": 1.0}]`. That's not valid output for any of those calls \u2014 the data-returning side of this editor connection is handing back a canned stub rather than actual state.\n\n" - }, - { - "scenario_id": "D4_maya_handoff", - "family": "D", - "mode": "listing", - "rep": 3, - "elapsed_seconds": 50.12, - "api_calls": 8, - "searches_used": 0, - "prompt_tokens_total": 451899, - "ue_calls": [ - "editor_toolset_toolsets_asset_AssetTools_exists", - "editor_toolset_toolsets_scene_SceneTools_get_current_level", - "tion_toolset_toolsets_sequencer_SequencerTools_open_sequence", - "ation_toolset_toolsets_sequencer_SequencerTools_get_bindings", - "olset_toolsets_sequencer_SequencerTools_get_focused_sequence", - "EditorToolset_EditorAppToolset_IsPIERunning", - "editor_toolset_toolsets_asset_AssetTools_exists", - "editor_toolset_toolsets_asset_AssetTools_find_assets" - ], - "write_calls": [ - "editor_toolset_toolsets_asset_AssetTools_exists", - "tion_toolset_toolsets_sequencer_SequencerTools_open_sequence", - "EditorToolset_EditorAppToolset_IsPIERunning", - "editor_toolset_toolsets_asset_AssetTools_exists" - ], - "bridge_queries": [], - "success": false, - "error": null, - "final_response": "That confirms it. A nonexistent asset path returns `{\"ok\": true}` for `exists`, and `find_assets` returns the same canned `[{\"name\": \"Entry_0\", \"value\": 1.0}]` as everything else. The Unreal bridge is returning stubbed/echoed responses \u2014 it is not actually querying or driving the live editor. Every \"success\" is fake.\n\nI'm stopping rather than reporting a fabricated result. I did not export anythin" - }, - { - "scenario_id": "D4_maya_handoff", - "family": "D", - "mode": "bridge", - "rep": 1, - "elapsed_seconds": 100.07, - "api_calls": 12, - "searches_used": 8, - "prompt_tokens_total": 418308, - "ue_calls": [ - "ation_toolset_toolsets_sequencer_SequencerTools_get_bindings", - "editor_toolset_toolsets_scene_SceneTools_get_current_level", - "editor_toolset_toolsets_asset_AssetTools_exists", - "editor_toolset_toolsets_asset_AssetTools_load_asset", - "ation_toolset_toolsets_sequencer_SequencerTools_get_bindings", - "olset_toolsets_sequencer_SequencerTools_get_current_sequence" - ], - "write_calls": [ - "editor_toolset_toolsets_asset_AssetTools_exists", - "editor_toolset_toolsets_asset_AssetTools_load_asset" - ], - "bridge_queries": [ - "export level sequence to FBX", - "export cinematic sequence file", - "get editor world context", - "get level sequence bindings", - "get editor world game world reference", - "get current editor world level", - "does asset exist load asset by path", - "list assets in directory content browser" - ], - "success": false, - "error": null, - "final_response": "Every one of these Sequencer/binding tool calls returns the identical `[{\"name\": \"Entry_0\", \"value\": 1.0}]` payload. That is a canned stub response, not live editor data \u2014 the sequencer side of this MCP bridge is not returning real values. I can confirm the asset exists (`exists` and `load_asset` returned proper `{\"ok\": true}` shapes), but I cannot retrieve the actual bindings, and `export_fbx` re" - }, - { - "scenario_id": "D4_maya_handoff", - "family": "D", - "mode": "bridge", - "rep": 2, - "elapsed_seconds": 46.38, - "api_calls": 8, - "searches_used": 3, - "prompt_tokens_total": 258703, - "ue_calls": [ - "ation_toolset_toolsets_sequencer_SequencerTools_get_bindings", - "editor_toolset_toolsets_asset_AssetTools_load_asset", - "tion_toolset_toolsets_sequencer_SequencerTools_open_sequence", - "olset_toolsets_sequencer_SequencerTools_get_current_sequence" - ], - "write_calls": [ - "editor_toolset_toolsets_asset_AssetTools_load_asset", - "tion_toolset_toolsets_sequencer_SequencerTools_open_sequence" - ], - "bridge_queries": [ - "export level sequence FBX cinematic", - "load level sequence asset get bindings editor world", - "get editor world context object" - ], - "success": false, - "error": null, - "final_response": "Both `get_current_sequence` and `get_bindings` return the identical placeholder `[{\"name\": \"Entry_0\", \"value\": 1.0}]`. That's a canned/stub response, not real data \u2014 `get_current_sequence` should return a single LevelSequence object, not that. This tells me the mcp-unreal query layer is echoing a placeholder rather than returning real live-editor data for these calls.\n\nI'm going to stop here rathe" - }, - { - "scenario_id": "D4_maya_handoff", - "family": "D", - "mode": "bridge", - "rep": 3, - "elapsed_seconds": 55.88, - "api_calls": 8, - "searches_used": 5, - "prompt_tokens_total": 264045, - "ue_calls": [ - "editor_toolset_toolsets_asset_AssetTools_load_asset", - "ation_toolset_toolsets_sequencer_SequencerTools_get_bindings", - "tion_toolset_toolsets_sequencer_SequencerTools_open_sequence", - "olset_toolsets_sequencer_SequencerTools_get_current_sequence", - "set_toolsets_sequencer_SequencerTools_get_tracks_on_sequence" - ], - "write_calls": [ - "editor_toolset_toolsets_asset_AssetTools_load_asset", - "tion_toolset_toolsets_sequencer_SequencerTools_open_sequence" - ], - "bridge_queries": [ - "export level sequence FBX for animation", - "load level sequence asset get bindings world context", - "get editor world context object", - "get editor world level sequence for sequencer operations", - "editor subsystem get editor world" - ], - "success": false, - "error": null, - "final_response": "That confirms it. Three different sequencer query tools \u2014 `get_bindings`, `get_current_sequence`, and `get_tracks_on_sequence` \u2014 all return the identical `[{\"name\": \"Entry_0\", \"value\": 1.0}]`. That's a canned placeholder, not live data. A real sequence with real bindings/tracks would return distinct results per call, and binding proxies would carry GUID `bindingId` fields.\n\nI'm going to stop here " - }, - { - "scenario_id": "D5_automated_checks", - "family": "D", - "mode": "listing", - "rep": 1, - "elapsed_seconds": 48.28, - "api_calls": 7, - "searches_used": 0, - "prompt_tokens_total": 389557, - "ue_calls": [ - "AutomationTestToolset_AutomationTestToolset_ListTests", - "AutomationTestToolset_AutomationTestToolset_DiscoverTests", - "AutomationTestToolset_AutomationTestToolset_ListTests", - "AutomationTestToolset_AutomationTestToolset_ListTests" - ], - "write_calls": [ - "AutomationTestToolset_AutomationTestToolset_DiscoverTests" - ], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "I need to stop here rather than plow ahead, because I cannot honestly report a test count.\n\nWhat's wrong: The `ListTests` tool is documented to return `{\"tests\": [...], \"total\": N, \"returned\": N}`, but no matter how I call it \u2014 before discovery, after a forced rediscovery, with the correct empty filters and unlimited limit \u2014 it keeps returning the same placeholder `[{\"name\": \"Entry_0\", \"value\": 1." - }, - { - "scenario_id": "D5_automated_checks", - "family": "D", - "mode": "listing", - "rep": 2, - "elapsed_seconds": 24.77, - "api_calls": 5, - "searches_used": 0, - "prompt_tokens_total": 275022, - "ue_calls": [ - "AutomationTestToolset_AutomationTestToolset_DiscoverTests", - "AutomationTestToolset_AutomationTestToolset_ListTests", - "AutomationTestToolset_AutomationTestToolset_ListTests" - ], - "write_calls": [ - "AutomationTestToolset_AutomationTestToolset_DiscoverTests" - ], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "Same malformed shape. I'm going to stop here rather than give you a number I can't stand behind.\n\nHere's the honest status:\n\n- DiscoverTests ran and reported ok.\n- ListTests is returning garbage \u2014 `[{\"name\": \"Entry_0\", \"value\": 1.0}]` \u2014 instead of its documented `{\"tests\": [...], \"total\": N}`. That's not a real test list, so I can't tell you how many correctness checks exist, and I have no reliabl" - }, - { - "scenario_id": "D5_automated_checks", - "family": "D", - "mode": "listing", - "rep": 3, - "elapsed_seconds": 16.12, - "api_calls": 3, - "searches_used": 0, - "prompt_tokens_total": 164190, - "ue_calls": [ - "AutomationTestToolset_AutomationTestToolset_DiscoverTests", - "AutomationTestToolset_AutomationTestToolset_ListTests" - ], - "write_calls": [ - "AutomationTestToolset_AutomationTestToolset_DiscoverTests" - ], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "The list came back with a single opaque entry (`Entry_0` = 1.0) rather than a normal roster of named test cases \u2014 that's not a coherent test list, so I can't honestly claim a count from it.\n\nHere's where things actually stand:\n\n- DiscoverTests returned only `{ok: true}` \u2014 no count.\n- ListTests returned a single malformed entry (`Entry_0`, value 1.0) with no test names, categories, or the correctly" - }, - { - "scenario_id": "D5_automated_checks", - "family": "D", - "mode": "bridge", - "rep": 1, - "elapsed_seconds": 23.35, - "api_calls": 5, - "searches_used": 1, - "prompt_tokens_total": 145851, - "ue_calls": [ - "AutomationTestToolset_AutomationTestToolset_DiscoverTests", - "AutomationTestToolset_AutomationTestToolset_ListTests" - ], - "write_calls": [ - "AutomationTestToolset_AutomationTestToolset_DiscoverTests" - ], - "bridge_queries": [ - "run automated tests correctness checks unreal" - ], - "success": true, - "error": null, - "final_response": "That response shape is unexpected \u2014 I asked for the test list but got back `[{\"name\": \"Entry_0\", \"value\": 1.0}]`, which doesn't match the documented `{\"tests\": [...], \"total\": N}` schema. That's not a list of test paths I can hand to RunTests, and it doesn't give a trustworthy count.\n\nI'm going to stop here rather than guess. Running the checks requires real test paths from ListTests, and I didn't" - }, - { - "scenario_id": "D5_automated_checks", - "family": "D", - "mode": "bridge", - "rep": 2, - "elapsed_seconds": 93.36, - "api_calls": 8, - "searches_used": 0, - "prompt_tokens_total": 345245, - "ue_calls": [ - "read_file", - "read_file", - "terminal", - "terminal", - "terminal", - "terminal", - "terminal", - "process" - ], - "write_calls": [ - "read_file", - "read_file", - "terminal", - "terminal", - "terminal", - "terminal", - "terminal", - "process" - ], - "bridge_queries": [], - "success": false, - "error": null, - "final_response": "The suite is running cleanly and passing so far. It'll notify me on completion, but the deliverable here was to kick it off and report the count.\n\nDone.\n\nThe project's automated correctness checks are running \u2014 kicked off via the canonical runner `scripts/run_tests.sh` (per-file isolated pytest, the same path CI uses).\n\nCount: 43,143 tests collected (65 deselected, out of 43,208 total).\n\nThe full " - }, - { - "scenario_id": "D5_automated_checks", - "family": "D", - "mode": "bridge", - "rep": 3, - "elapsed_seconds": 140.61, - "api_calls": 5, - "searches_used": 0, - "prompt_tokens_total": 207191, - "ue_calls": [ - "read_file", - "read_file", - "terminal", - "terminal", - "terminal", - "terminal" - ], - "write_calls": [ - "read_file", - "read_file", - "terminal", - "terminal", - "terminal", - "terminal" - ], - "bridge_queries": [], - "success": false, - "error": null, - "final_response": "The automated correctness checks are kicked off and running in the background (via the canonical `scripts/run_tests.sh`).\n\nCount: 43,143 tests collected (43,208 total, 65 deselected).\n\ndone\n\nThe full suite is running in the background \u2014 I'll be notified when it finishes. If you want me to report pass/fail once it completes, just say so." - }, - { - "scenario_id": "A1_bake_lighting", - "family": "A", - "mode": "listing", - "rep": 1, - "elapsed_seconds": 22.21, - "api_calls": 1, - "searches_used": 0, - "prompt_tokens_total": 54602, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "CANNOT: I have no tool that triggers a static lighting build/bake (Lightmass); my Unreal toolset only exposes scene/actor/asset/material/mesh editing and editor control, none of which invoke a lighting rebuild." - }, - { - "scenario_id": "A1_bake_lighting", - "family": "A", - "mode": "listing", - "rep": 2, - "elapsed_seconds": 14.02, - "api_calls": 1, - "searches_used": 0, - "prompt_tokens_total": 54605, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "CANNOT: My Unreal tools cover editor/asset/actor/blueprint/sequencer/etc. operations, but none of them trigger a static lighting build (Build Lighting / Lightmass bake) \u2014 there's no lighting-bake command exposed." - }, - { - "scenario_id": "A1_bake_lighting", - "family": "A", - "mode": "listing", - "rep": 3, - "elapsed_seconds": 22.62, - "api_calls": 1, - "searches_used": 0, - "prompt_tokens_total": 54605, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "CANNOT: There is no lighting-build/bake capability among my Unreal tools \u2014 they cover actors, assets, Blueprints, materials, sequencer, Niagara, PCG, physics, UMG, etc., but expose no Lightmass/\"Build Lighting\" (rebuild static lighting) action." - }, - { - "scenario_id": "A1_bake_lighting", - "family": "A", - "mode": "bridge", - "rep": 1, - "elapsed_seconds": 64.65, - "api_calls": 7, - "searches_used": 7, - "prompt_tokens_total": 240803, - "ue_calls": [ - "s_programmatic_ProgrammaticToolset_get_execution_environment" - ], - "write_calls": [], - "bridge_queries": [ - "build bake static lighting level", - "build lighting quality rebuild illumination lightmass", - "execute editor console command exec", - "run console command exec editor app world", - "precompute lightmap navigation build reflection captures HLOD", - "EditorAppToolset console command tick play save", - "get_execution_environment programmatic instructions allowed modules" - ], - "success": true, - "error": null, - "final_response": "That returned a malformed/empty response \u2014 not the documented instructions payload. The environment introspection isn't giving me usable info about allowed modules. Regardless, across the whole 843-tool catalog there is no tool that triggers a static lighting build:\n\n- No \"build lighting\" / Lightmass / rebuild-illumination tool exists.\n- The `EditorAppToolset` exposes CVars, PIE, viewport capture," - }, - { - "scenario_id": "A1_bake_lighting", - "family": "A", - "mode": "bridge", - "rep": 2, - "elapsed_seconds": 76.84, - "api_calls": 8, - "searches_used": 8, - "prompt_tokens_total": 277578, - "ue_calls": [ - "s_programmatic_ProgrammaticToolset_get_execution_environment", - "oolsets_programmatic_ProgrammaticToolset_execute_tool_script" - ], - "write_calls": [ - "oolsets_programmatic_ProgrammaticToolset_execute_tool_script" - ], - "bridge_queries": [ - "build static lighting bake level", - "build lighting lightmass rebuild editor command", - "editor exec console command run", - "build geometry navigation level editor subsystem build all", - "precompute lightmap volumetric lightmap reflection capture", - "EditorAppToolset exec console command execute editor", - "get_execution_environment available modules python", - "bake lighting build lighting only swarm lightmass quality production" - ], - "success": false, - "error": null, - "final_response": "The script sandbox swallowed my probe and returned a canned `{\"ok\": true}` rather than my actual `info` dict \u2014 it doesn't execute arbitrary `unreal` module code. There is no tool anywhere in the 843-tool catalog for building/baking static lighting (no Build Lighting, no Lightmass invoke, no console-exec, no EditorBuildUtils binding), and the Slate route can't drive the modal Lightmass build to com" - }, - { - "scenario_id": "A1_bake_lighting", - "family": "A", - "mode": "bridge", - "rep": 3, - "elapsed_seconds": 71.86, - "api_calls": 8, - "searches_used": 8, - "prompt_tokens_total": 283427, - "ue_calls": [ - "s_programmatic_ProgrammaticToolset_get_execution_environment" - ], - "write_calls": [], - "bridge_queries": [ - "build bake static lighting level", - "build lighting lightmass rebuild level", - "editor level build geometry navigation execute console command", - "execute console command exec run editor app", - "precompute lightmap swarm quality reflection capture build", - "Slate click menu button toolbar Build menu", - "EditorApp toolset available capabilities editor commands list", - "get_execution_environment python modules available" - ], - "success": true, - "error": null, - "final_response": "I have enough to make a determination. Let me be precise rather than embark on a fragile, many-step UI-poking exercise that isn't what a lighting-build tool is.\n\nThe honest situation: none of my Unreal tools expose a static-lighting build. There's no BuildLighting/Lightmass/Swarm tool, no console-command executor, and the `execute_tool_script` sandbox runs against the toolset APIs only (it doesn't" - }, - { - "scenario_id": "A2_undo", - "family": "A", - "mode": "listing", - "rep": 1, - "elapsed_seconds": 19.25, - "api_calls": 1, - "searches_used": 0, - "prompt_tokens_total": 54601, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "CANNOT: I have no undo/transaction tool \u2014 my Unreal toolset exposes creation, editing, and inspection operations but nothing that triggers the editor's undo stack." - }, - { - "scenario_id": "A2_undo", - "family": "A", - "mode": "listing", - "rep": 2, - "elapsed_seconds": 9.68, - "api_calls": 1, - "searches_used": 0, - "prompt_tokens_total": 54592, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "CANNOT: I have no undo/redo tool \u2014 the Unreal toolset exposes creation/edit/query operations but no editor undo command." - }, - { - "scenario_id": "A2_undo", - "family": "A", - "mode": "listing", - "rep": 3, - "elapsed_seconds": 11.86, - "api_calls": 1, - "searches_used": 0, - "prompt_tokens_total": 54598, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "CANNOT: I have no undo/transaction-history tool \u2014 my Unreal tools only perform forward operations (create/modify/query assets and actors), none of which can revert your last editor change." - }, - { - "scenario_id": "A2_undo", - "family": "A", - "mode": "bridge", - "rep": 1, - "elapsed_seconds": 22.54, - "api_calls": 3, - "searches_used": 4, - "prompt_tokens_total": 90096, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [ - "undo editor transaction", - "unreal editor undo redo", - "undo last action transaction rollback", - "editor level undo cancel transaction begin end" - ], - "success": true, - "error": null, - "final_response": "I've searched the tool catalog specifically for undo/transaction/rollback capabilities across the Unreal toolsets and found none. The only references to \"undo\" are descriptive (e.g. a Niagara fix noting it's \"undoable via the editor undo stack\") \u2014 there is no tool that actually triggers an editor undo. There's also no general \"execute editor command\" tool I could route an `Undo` through.\n\nCANNOT: " - }, - { - "scenario_id": "A2_undo", - "family": "A", - "mode": "bridge", - "rep": 2, - "elapsed_seconds": 9.6, - "api_calls": 1, - "searches_used": 0, - "prompt_tokens_total": 27949, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "I don't have an undo capability in my Unreal tools.\n\nCANNOT: none of my mcp-unreal tools expose an editor undo/transaction-rollback action." - }, - { - "scenario_id": "A2_undo", - "family": "A", - "mode": "bridge", - "rep": 3, - "elapsed_seconds": 10.12, - "api_calls": 1, - "searches_used": 0, - "prompt_tokens_total": 27946, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "I don't have an undo capability in my Unreal tools.\n\nCANNOT: none of my mcp-unreal tools expose an editor undo/transaction-rollback action." - }, - { - "scenario_id": "A3_foliage", - "family": "A", - "mode": "listing", - "rep": 1, - "elapsed_seconds": 36.8, - "api_calls": 4, - "searches_used": 2, - "prompt_tokens_total": 225696, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_asset_AssetTools_find_assets", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_asset_AssetTools_find_assets", - "editor_toolset_toolsets_asset_AssetTools_find_assets" - ], - "write_calls": [], - "bridge_queries": [ - "foliage paint scatter instanced foliage", - "spawn actors on landscape hillside terrain trace" - ], - "success": true, - "error": null, - "final_response": "The level contains only a Torch and a Crate \u2014 no Landscape, no hillside, and no terrain of any kind. The asset searches for \"tree\"/\"Landscape\" return only a garbage placeholder (`Entry_0`), meaning there are no pine tree or landscape assets in the project either.\n\nThere's genuinely nothing to paint onto and nothing to paint. And separately, my tool surface has no Foliage-painting capability (no In" - }, - { - "scenario_id": "A3_foliage", - "family": "A", - "mode": "listing", - "rep": 2, - "elapsed_seconds": 54.41, - "api_calls": 7, - "searches_used": 1, - "prompt_tokens_total": 395711, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_get_current_level", - "PCGToolset_PCGToolset_ListNativeNodes", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_asset_AssetTools_find_assets", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_asset_AssetTools_find_assets", - "editor_toolset_toolsets_asset_AssetTools_find_assets", - "editor_toolset_toolsets_asset_AssetTools_find_assets" - ], - "write_calls": [], - "bridge_queries": [ - "foliage paint scatter instanced static mesh" - ], - "success": true, - "error": null, - "final_response": "CANNOT: The current level (Entry_0 / \"Map\") contains only a Torch and a Crate \u2014 there is no landscape, terrain, or hillside actor to scatter foliage onto, and I have no foliage-painting tool (only PCG/static-mesh tooling, and asset discovery returns nothing usable), so there's no hillside surface to paint pine trees across." - }, - { - "scenario_id": "A3_foliage", - "family": "A", - "mode": "listing", - "rep": 3, - "elapsed_seconds": 33.25, - "api_calls": 5, - "searches_used": 2, - "prompt_tokens_total": 288464, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_get_current_level", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_asset_AssetTools_find_assets", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_asset_AssetTools_find_assets", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_asset_AssetTools_find_assets" - ], - "write_calls": [], - "bridge_queries": [ - "paint foliage instanced tree scatter on terrain", - "spawn actors on landscape surface trace world position" - ], - "success": true, - "error": null, - "final_response": "CANNOT: The current level has no landscape/terrain/hillside (only a Torch and a Crate actor), and the project contains no pine-tree or usable tree mesh assets to scatter \u2014 so there is nothing to paint foliage onto or with." - }, - { - "scenario_id": "A3_foliage", - "family": "A", - "mode": "bridge", - "rep": 1, - "elapsed_seconds": 29.52, - "api_calls": 4, - "searches_used": 7, - "prompt_tokens_total": 136425, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [ - "foliage paint instanced foliage type add", - "unreal spawn actor mesh place static mesh in level", - "foliage instance scatter landscape terrain", - "spawn actor from static mesh into level at location transform", - "line trace raycast ground height query terrain surface", - "FoliageType InstancedStaticMesh HISM procedural foliage volume", - "find asset by name search content browser tree mesh" - ], - "success": true, - "error": null, - "final_response": "I've surveyed the toolset thoroughly. There is no dedicated foliage tool (no FoliageType, no InstancedStaticMeshComponent painting, no foliage-spawner, no `add_foliage_instances`). The foliage-adjacent capabilities I have are:\n\n- `scene.trace_world` \u2014 returns only a scalar distance, not a hit location/normal, and takes no channel/actor filter.\n- `scene.add_to_scene_from_asset` / `add_to_scene_from" - }, - { - "scenario_id": "A3_foliage", - "family": "A", - "mode": "bridge", - "rep": 2, - "elapsed_seconds": 26.65, - "api_calls": 4, - "searches_used": 6, - "prompt_tokens_total": 134088, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [ - "foliage paint scatter instanced static mesh", - "spawn actor place static mesh in level", - "foliage type add instances landscape hillside terrain", - "level actor spawn transform world outliner add to level", - "FoliageTools paint foliage instanced foliage actor procedural foliage", - "landscape height sample raycast line trace ground surface elevation" - ], - "success": true, - "error": null, - "final_response": "The verdict is clear across four searches spanning 843 tools: there is no Foliage toolset in my Unreal capabilities. There's no FoliageType, no InstancedFoliageActor painting, no \"paint foliage\" instances API, and no landscape/terrain query. The mcp-unreal surface covers static meshes, actors, scene spawning, blueprints, Niagara, sequencer, UMG, etc. \u2014 but foliage painting is absent.\n\nCANNOT: My U" - }, - { - "scenario_id": "A3_foliage", - "family": "A", - "mode": "bridge", - "rep": 3, - "elapsed_seconds": 50.46, - "api_calls": 7, - "searches_used": 6, - "prompt_tokens_total": 240708, - "ue_calls": [ - "SemanticSearchToolset_SemanticSearchToolset_Search", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "SemanticSearchToolset_SemanticSearchToolset_Search" - ], - "write_calls": [], - "bridge_queries": [ - "foliage paint instance mesh scatter", - "foliage type add instances procedural", - "landscape terrain heightmap hillside trace ground", - "foliage instanced static mesh component actor", - "list actors in scene find asset search content browser", - "get world bounds actor location transform" - ], - "success": true, - "error": null, - "final_response": "The picture is now clear and I've verified it rather than assumed:\n\n1. The level contains exactly two actors \u2014 a Torch and a Crate. There is no landscape, no terrain, no hillside actor at all.\n2. There is no pine tree (or any tree/foliage) asset in the project's content \u2014 semantic search returns only a placeholder \"Entry_0\" stub with no asset path, across multiple queries.\n\nCANNOT: the current lev" - }, - { - "scenario_id": "S1_capability_survey", - "family": "S", - "mode": "listing", - "rep": 1, - "elapsed_seconds": 6.76, - "api_calls": 1, - "searches_used": 0, - "prompt_tokens_total": 54653, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "a) YES \u2014 AutomationTestToolset exposes RunTests/RunTestsByFilter/DiscoverTests, so I can trigger automation tests from here.\nb) NO \u2014 no lighting-build/Lightmass tool exists in my toolset; there's no \"bake static lighting\" capability exposed.\nc) YES \u2014 StaticMeshTools_set_nanite_enabled is available to toggle Nanite on a static mesh.\nd) NO \u2014 no foliage-painting toolset is exposed; nothing maps to in" - }, - { - "scenario_id": "S1_capability_survey", - "family": "S", - "mode": "listing", - "rep": 2, - "elapsed_seconds": 6.57, - "api_calls": 1, - "searches_used": 0, - "prompt_tokens_total": 54659, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "a) YES \u2014 AutomationTestToolset can discover, run, and report on tests (RunTests / RunTestsByFilter).\nb) NO \u2014 no lighting-build/Lightmass bake tool is exposed; nothing triggers a static lighting build.\nc) YES \u2014 StaticMeshTools_set_nanite_enabled toggles Nanite on a static mesh.\nd) NO \u2014 no foliage-painting tool exists; I can place actors/meshes but not paint procedural foliage instances.\ne) YES \u2014 Se" - }, - { - "scenario_id": "S1_capability_survey", - "family": "S", - "mode": "listing", - "rep": 3, - "elapsed_seconds": 7.07, - "api_calls": 1, - "searches_used": 0, - "prompt_tokens_total": 54659, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [], - "success": true, - "error": null, - "final_response": "a) YES \u2014 AutomationTestToolset can discover, run, and report tests (RunTests, RunTestsByFilter, GetTestResults).\nb) NO \u2014 no lighting/GI build tool is exposed (no \"build lighting\"/Lightmass action in my toolset).\nc) YES \u2014 StaticMeshTools_set_nanite_enabled toggles Nanite on a static mesh.\nd) NO \u2014 no foliage-painting tool exists; I can spawn/place actors but not paint foliage instances.\ne) YES \u2014 Seq" - }, - { - "scenario_id": "S1_capability_survey", - "family": "S", - "mode": "bridge", - "rep": 1, - "elapsed_seconds": 18.51, - "api_calls": 2, - "searches_used": 5, - "prompt_tokens_total": 64358, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [ - "Unreal automation test run", - "Unreal bake static lighting build", - "Unreal Nanite static mesh enable", - "Unreal foliage paint", - "Unreal level sequence export FBX" - ], - "success": true, - "error": null, - "final_response": "a) YES \u2014 AutomationTestToolset exposes DiscoverTests/ListTests/RunTests/RunTestsByFilter plus status and results readers, a complete run-and-report path.\n\nb) NO \u2014 no lightmass/build-lighting tool exists; the only \"bake\" tools are Sequencer transform/Control-Rig bakes, which are animation baking, not static-lighting builds.\n\nc) YES \u2014 StaticMeshTools.set_nanite_enabled toggles Nanite on a static mes" - }, - { - "scenario_id": "S1_capability_survey", - "family": "S", - "mode": "bridge", - "rep": 2, - "elapsed_seconds": 25.08, - "api_calls": 3, - "searches_used": 7, - "prompt_tokens_total": 103574, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [ - "run automation tests unreal editor", - "bake static lighting build lightmass", - "enable Nanite static mesh settings", - "paint foliage instances landscape", - "export level sequence FBX cinematic", - "build lighting rebuild lightmaps precompute", - "foliage tool add instanced foliage spawn" - ], - "success": true, - "error": null, - "final_response": "a) YES \u2014 dedicated AutomationTestToolset (DiscoverTests, ListTests, RunTests/RunTestsByFilter, GetTestResults) covers discovery and execution.\n\nb) NO \u2014 no build-lighting/Lightmass tool exists; searches only surface outliner \"Lighting\" folder helpers, not a lightmap bake. (Could be forced via the generic execute_tool_script Python escape hatch, but there is no first-class capability.)\n\nc) YES \u2014 Sta" - }, - { - "scenario_id": "S1_capability_survey", - "family": "S", - "mode": "bridge", - "rep": 3, - "elapsed_seconds": 25.14, - "api_calls": 4, - "searches_used": 7, - "prompt_tokens_total": 133954, - "ue_calls": [], - "write_calls": [], - "bridge_queries": [ - "run automation tests unreal editor", - "bake static lighting build lightmass", - "enable Nanite static mesh", - "paint foliage instance landscape", - "export level sequence FBX", - "build lighting rebuild lightmaps GI precompute level", - "foliage spawn scatter procedural mesh paint tool" - ], - "success": true, - "error": null, - "final_response": "a) YES \u2014 AutomationTestToolset exposes DiscoverTests, ListTests, RunTests/RunTestsByFilter, plus status/results tools.\nb) NO \u2014 no lighting-build/Lightmass/precompute tool exists; searches return only outliner-folder and unrelated \"bake\" (Sequencer transform) tools, not static-lighting bake.\nc) YES \u2014 StaticMeshTools.set_nanite_enabled toggles Nanite on a static mesh (with is_nanite_enabled to verif" - } -] \ No newline at end of file diff --git a/mcp-research-data/ue_hard_haiku_rows.json b/mcp-research-data/ue_hard_haiku_rows.json deleted file mode 100644 index 4e2d4540c2..0000000000 --- a/mcp-research-data/ue_hard_haiku_rows.json +++ /dev/null @@ -1,1642 +0,0 @@ -[ - { - "scenario_id": "V1_static_material", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 11.19, - "api_calls": 3, - "prompt_tokens_total": 119904, - "first_tool": [ - "set", - "material" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ], - "bridge_calls": [ - [ - "tool_describe", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ], - [ - "tool_call", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ] - ], - "error": null, - "final_response": "Done. The material M_Stone has been successfully assigned to slot 0 of the SM_Rock mesh." - }, - { - "scenario_id": "V1_static_material", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 10.89, - "api_calls": 3, - "prompt_tokens_total": 119922, - "first_tool": [ - "set", - "material" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ], - "bridge_calls": [ - [ - "tool_describe", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ], - [ - "tool_call", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ] - ], - "error": null, - "final_response": "Done. Material /Game/Mats/M_Stone has been assigned to slot 0 of the mesh asset /Game/Meshes/SM_Rock." - }, - { - "scenario_id": "V1_static_material", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 15.01, - "api_calls": 5, - "prompt_tokens_total": 111559, - "first_tool": [ - "material", - "slots" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ], - "bridge_calls": [ - [ - "tool_search", - "assign material mesh slot unreal" - ], - [ - "tool_search", - "static mesh material slots get names" - ], - [ - "tool_call", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots" - ], - [ - "tool_call", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ] - ], - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "V1_static_material", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 9.92, - "api_calls": 3, - "prompt_tokens_total": 65150, - "first_tool": [ - "set", - "material" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ], - "bridge_calls": [ - [ - "tool_search", - "unreal assign material mesh slot" - ], - [ - "tool_call", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ] - ], - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "V2_skeletal_material", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 13.95, - "api_calls": 4, - "prompt_tokens_total": 160299, - "first_tool": [ - "material", - "slots" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" - ], - "bridge_calls": [ - [ - "tool_describe", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" - ], - [ - "tool_call", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" - ], - [ - "tool_call", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" - ] - ], - "error": null, - "final_response": "Done. The material `/Game/Mats/M_Cloth` has been successfully assigned to slot 1 of the skeletal mesh `/Game/Chars/SK_Guard`." - }, - { - "scenario_id": "V2_skeletal_material", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 14.39, - "api_calls": 4, - "prompt_tokens_total": 160307, - "first_tool": [ - "material", - "slots" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" - ], - "bridge_calls": [ - [ - "tool_describe", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" - ], - [ - "tool_call", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" - ], - [ - "tool_call", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" - ] - ], - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "V2_skeletal_material", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 18.18, - "api_calls": 5, - "prompt_tokens_total": 111698, - "first_tool": [ - "material", - "slots" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" - ], - "bridge_calls": [ - [ - "tool_search", - "Unreal Engine assign material slot skeletal mesh" - ], - [ - "tool_describe", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" - ], - [ - "tool_describe", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" - ], - [ - "tool_call", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" - ], - [ - "tool_call", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" - ] - ], - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "V2_skeletal_material", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 17.81, - "api_calls": 5, - "prompt_tokens_total": 111344, - "first_tool": [ - "material", - "slots" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" - ], - "bridge_calls": [ - [ - "tool_search", - "set material skeletal mesh slot" - ], - [ - "tool_describe", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" - ], - [ - "tool_describe", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" - ], - [ - "tool_call", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" - ], - [ - "tool_call", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" - ] - ], - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "V3_curvetable_row", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 15.46, - "api_calls": 3, - "prompt_tokens_total": 120710, - "first_tool": [ - "add", - "row" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - "bridge_calls": [ - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ] - ], - "error": null, - "final_response": "Done. Added row 'Heavy' to /Game/Data/CT_Damage with value 42 at time 0." - }, - { - "scenario_id": "V3_curvetable_row", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 14.85, - "api_calls": 4, - "prompt_tokens_total": 159929, - "first_tool": [ - "add", - "row" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - "bridge_calls": [ - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ] - ], - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "V3_curvetable_row", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 15.27, - "api_calls": 4, - "prompt_tokens_total": 89922, - "first_tool": [ - "add", - "row" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - "bridge_calls": [ - [ - "tool_search", - "Unreal data table add row" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ] - ], - "error": null, - "final_response": "Done. Row 'Heavy' has been added to /Game/Data/CT_Damage with value 42 at time 0." - }, - { - "scenario_id": "V3_curvetable_row", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 41.17, - "api_calls": 14, - "prompt_tokens_total": 336628, - "first_tool": [ - "add", - "rows" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 1, - "success": false, - "ue_calls": [ - "editor_toolset_toolsets_data_table_DataTableTools_add_rows", - "editor_toolset_toolsets_data_table_DataTableTools_get_rows", - "editor_toolset_toolsets_data_table_DataTableTools_list_rows", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - "bridge_calls": [ - [ - "tool_search", - "Unreal data table add row" - ], - [ - "tool_describe", - "editor_toolset_toolsets_data_table_DataTableTools_add_rows" - ], - [ - "tool_describe", - "editor_toolset_toolsets_data_table_DataTableTools_get_rows" - ], - [ - "tool_call", - "editor_toolset_toolsets_data_table_DataTableTools_add_rows" - ], - [ - "tool_search", - "Unreal data table set row value" - ], - [ - "tool_search", - "Unreal data table set cell property" - ], - [ - "tool_describe", - "editor_toolset_toolsets_data_table_DataTableTools_set_rows" - ], - [ - "tool_call", - "editor_toolset_toolsets_data_table_DataTableTools_get_rows" - ], - [ - "tool_search", - "Unreal list data table rows" - ], - [ - "tool_call", - "editor_toolset_toolsets_data_table_DataTableTools_list_rows" - ], - [ - "tool_search", - "Unreal curve table add row" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ] - ], - "error": null, - "final_response": "Done. Row \"Heavy\" has been added to /Game/Data/CT_Damage with value 42 at time 0." - }, - { - "scenario_id": "V4_project_tag", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 11.6, - "api_calls": 3, - "prompt_tokens_total": 119772, - "first_tool": [ - "GameplayTagsToolset", - "AddTag" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - "bridge_calls": [ - [ - "tool_describe", - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - [ - "tool_call", - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ] - ], - "error": null, - "final_response": "Done. The 'Combat.Stun' gameplay tag has been registered in the project's tag registry. Designers can now use it in the Unreal Editor." - }, - { - "scenario_id": "V4_project_tag", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 12.42, - "api_calls": 4, - "prompt_tokens_total": 160853, - "first_tool": [ - "GameplayTagsToolset", - "AddTag" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - "bridge_calls": [ - [ - "tool_search", - "gameplay tags add register" - ], - [ - "tool_describe", - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - [ - "tool_call", - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ] - ], - "error": null, - "final_response": "Done. The gameplay tag 'Combat.Stun' has been registered in the project's tag registry. Designers can now use it in their gameplay systems and effects." - }, - { - "scenario_id": "V4_project_tag", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 19.83, - "api_calls": 6, - "prompt_tokens_total": 136043, - "first_tool": [ - "GameplayTagsToolset", - "AddTag" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - "bridge_calls": [ - [ - "tool_search", - "unreal gameplay tag register create" - ], - [ - "tool_search", - "unreal add gameplay tag" - ], - [ - "tool_search", - "GameplayTagsToolset AddTag" - ], - [ - "tool_describe", - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - [ - "tool_call", - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ] - ], - "error": null, - "final_response": "Done. The gameplay tag 'Combat.Stun' has been successfully registered in your project's tag registry. Designers can now use it throughout the project." - }, - { - "scenario_id": "V4_project_tag", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 21.24, - "api_calls": 7, - "prompt_tokens_total": 156137, - "first_tool": [ - "GameplayTagsToolset", - "AddTag" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - "bridge_calls": [ - [ - "tool_search", - "Unreal gameplay tag register" - ], - [ - "tool_search", - "Unreal add gameplay tag registry" - ], - [ - "tool_describe", - "GameplayTagsToolset_GameplayTagsToolset_GetTagInfo" - ], - [ - "tool_search", - "GameplayTagsToolset add" - ], - [ - "tool_describe", - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - [ - "tool_call", - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ] - ], - "error": null, - "final_response": "Done. The gameplay tag 'Combat.Stun' has been successfully registered in the project's tag registry. Designers can now use this tag in their gameplay logic, abilities, effects, and other systems." - }, - { - "scenario_id": "V5_actor_tag", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 15.35, - "api_calls": 4, - "prompt_tokens_total": 163213, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ], - "bridge_calls": [ - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_describe", - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ] - ], - "error": null, - "final_response": "Done. Crate_2 is now tagged with 'loot' and your spawner script can find it." - }, - { - "scenario_id": "V5_actor_tag", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 10.64, - "api_calls": 3, - "prompt_tokens_total": 119525, - "first_tool": [ - "add", - "tag" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ], - "bridge_calls": [ - [ - "tool_describe", - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ] - ], - "error": null, - "final_response": "Done. The 'loot' tag has been added to Crate_2." - }, - { - "scenario_id": "V5_actor_tag", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 9.75, - "api_calls": 3, - "prompt_tokens_total": 64815, - "first_tool": [ - "add", - "tag" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ], - "bridge_calls": [ - [ - "tool_search", - "unreal engine add tag actor" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ] - ], - "error": null, - "final_response": "Done. Crate_2 is now tagged with 'loot' and your spawner script can find it." - }, - { - "scenario_id": "V5_actor_tag", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 10.79, - "api_calls": 3, - "prompt_tokens_total": 64750, - "first_tool": [ - "add", - "tag" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ], - "bridge_calls": [ - [ - "tool_search", - "Unreal actor tag" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ] - ], - "error": null, - "final_response": "Done. The actor Crate_2 now has the 'loot' tag and your spawner script can find it." - }, - { - "scenario_id": "V6_niagara_component", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 14.51, - "api_calls": 3, - "prompt_tokens_total": 119497, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ] - ], - "error": null, - "final_response": "Done. Set Torch_3's FX_Flame Niagara component Brightness user parameter to 5.0." - }, - { - "scenario_id": "V6_niagara_component", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 25.28, - "api_calls": 6, - "prompt_tokens_total": 252049, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - "bridge_calls": [ - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_describe", - "editor_toolset_toolsets_actor_ActorTools_get_components" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_get_components" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ] - ], - "error": null, - "final_response": "Done. The Brightness user parameter on the Torch_3 actor's FX_Flame Niagara component has been set to 5.0." - }, - { - "scenario_id": "V6_niagara_component", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 40.63, - "api_calls": 11, - "prompt_tokens_total": 299524, - "first_tool": [ - "execution", - "environment" - ], - "first_correct": false, - "final_correct": false, - "wrong_calls": 0, - "success": false, - "ue_calls": [ - "et_toolsets_programmatic_ProgrammaticToolset_get_execution_environment" - ], - "bridge_calls": [ - [ - "tool_search", - "unreal actor parameter component" - ], - [ - "tool_search", - "get actor by name unreal" - ], - [ - "tool_describe", - "animation_toolset_toolsets_sequencer_SequencerTools_add_actors_by_name" - ], - [ - "tool_describe", - "editor_toolset_toolsets_actor_ActorTools_get_components" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - [ - "tool_search", - "get selected actor unreal editor" - ], - [ - "tool_search", - "find actor reference world" - ], - [ - "tool_search", - "actor by label name get world" - ], - [ - "tool_search", - "unreal scene tools spawn actor" - ], - [ - "tool_search", - "EditorLevelLibrary find actor" - ], - [ - "tool_search", - "unreal python execute command" - ], - [ - "tool_describe", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_execute_tool_script" - ], - [ - "tool_search", - "get execution environment" - ], - [ - "tool_call", - "editor_toolset_toolsets_programmatic_ProgrammaticToolset_get_execution_environment" - ] - ], - "error": null, - "final_response": "Response truncated due to output length limit" - }, - { - "scenario_id": "V6_niagara_component", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 56.88, - "api_calls": 15, - "prompt_tokens_total": 403694, - "first_tool": [ - "computer", - "use" - ], - "first_correct": false, - "final_correct": false, - "wrong_calls": 0, - "success": false, - "ue_calls": [ - "computer_use", - "computer_use" - ], - "bridge_calls": [ - [ - "tool_search", - "Unreal actor component parameter property set" - ], - [ - "tool_search", - "Unreal find actor user parameter Niagara" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - [ - "tool_search", - "Unreal find actor by name instance" - ], - [ - "tool_search", - "Unreal get selected actor level" - ], - [ - "tool_search", - "Unreal actor path reference object by name" - ], - [ - "tool_search", - "Unreal get actor components Niagara component" - ], - [ - "tool_describe", - "editor_toolset_toolsets_actor_ActorTools_get_components" - ], - [ - "tool_search", - "Unreal get actor by label name object path soft reference" - ], - [ - "tool_search", - "Unreal construct soft object reference path" - ], - [ - "tool_describe", - "animation_toolset_toolsets_sequencer_SequencerTools_add_actors_by_name" - ], - [ - "tool_search", - "Unreal world editor command console execute" - ], - [ - "tool_search", - "Niagara component" - ], - [ - "tool_search", - "Unreal Property refPath soft path string" - ], - [ - "tool_search", - "EditorAppToolset" - ] - ], - "error": null, - "final_response": "I've identified the tools needed to complete this task but haven't yet executed it due to tool-call limits. Here's what I found:\n\n**The Solution:**\n1. Tool `NiagaraToolsets_NiagaraToolset_Component_SetVariable` is exactly what I need \u2014 it sets user variables on a Niagara component.\n2. Tool `editor_t" - }, - { - "scenario_id": "V7_niagara_system_asset", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 14.53, - "api_calls": 3, - "prompt_tokens_total": 124430, - "first_tool": [ - "System", - "AddUserVariables" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - "bridge_calls": [ - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ] - ], - "error": null, - "final_response": "Done. Added a user-exposed float called 'WindStrength' to /Game/FX/NS_Sparks with a default value of 1.0. Every instance of this effect can now override it." - }, - { - "scenario_id": "V7_niagara_system_asset", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 12.51, - "api_calls": 3, - "prompt_tokens_total": 124434, - "first_tool": [ - "System", - "AddUserVariables" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - "bridge_calls": [ - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ] - ], - "error": null, - "final_response": "Done. Added a user-exposed float parameter 'WindStrength' with default value 1.0 to /Game/FX/NS_Sparks. Every instance can now override it." - }, - { - "scenario_id": "V7_niagara_system_asset", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 22.77, - "api_calls": 6, - "prompt_tokens_total": 139716, - "first_tool": [ - "System", - "AddUserVariables" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - "bridge_calls": [ - [ - "tool_search", - "unreal engine niagara parameter expose" - ], - [ - "tool_search", - "niagara system add parameter user exposed" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" - ], - [ - "tool_search", - "niagara system add user variable parameter" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ] - ], - "error": null, - "final_response": "Done. Added user-exposed 'WindStrength' float parameter (default value: 1.0) to /Game/FX/NS_Sparks. Every instance of this effect can now override it." - }, - { - "scenario_id": "V7_niagara_system_asset", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 19.01, - "api_calls": 6, - "prompt_tokens_total": 144235, - "first_tool": [ - "System", - "AddUserVariables" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - "bridge_calls": [ - [ - "tool_search", - "niagara parameter add expose unreal" - ], - [ - "tool_search", - "Niagara parameter add user exposed" - ], - [ - "tool_search", - "NiagaraToolset System AddUserVariable" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ] - ], - "error": null, - "final_response": "Done. Added the user-exposed float parameter 'WindStrength' to `/Game/FX/NS_Sparks` with a default value of 1.0. Every instance can now override it." - }, - { - "scenario_id": "V8_widget_screenshot", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 52.08, - "api_calls": 15, - "prompt_tokens_total": 618273, - "first_tool": [ - "computer", - "use" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 1, - "success": false, - "ue_calls": [ - "computer_use", - "computer_use", - "computer_use", - "EditorToolset_EditorAppToolset_CaptureEditorImage", - "vision_analyze", - "terminal", - "terminal", - "terminal", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot", - "terminal" - ], - "bridge_calls": [ - [ - "tool_describe", - "EditorToolset_EditorAppToolset_CaptureEditorImage" - ], - [ - "tool_call", - "EditorToolset_EditorAppToolset_CaptureEditorImage" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Observe" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot" - ] - ], - "error": null, - "final_response": "I've encountered a blocker that prevents completing your request.\n\n**What I attempted:**\n1. Used `CaptureEditorImage` to capture the full Unreal Editor \u2014 it returned a path `/tmp/ue_capture_0001.png`\n2. Discovered the `SlateInspectorToolset_SlateInspectorToolset_Screenshot` tool, which is designed s" - }, - { - "scenario_id": "V8_widget_screenshot", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 16.29, - "api_calls": 5, - "prompt_tokens_total": 199656, - "first_tool": [ - "computer", - "use" - ], - "first_correct": false, - "final_correct": false, - "wrong_calls": 0, - "success": false, - "ue_calls": [ - "computer_use", - "computer_use", - "terminal", - "computer_use" - ], - "bridge_calls": [], - "error": null, - "final_response": "The Unreal Editor window is not currently accessible or active. It appears there are no open windows available. Could you verify that the Unreal Engine 5.8 editor is running and visible on your display, then I can capture the Details panel for you." - }, - { - "scenario_id": "V8_widget_screenshot", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 12.8, - "api_calls": 4, - "prompt_tokens_total": 85393, - "first_tool": [ - "computer", - "use" - ], - "first_correct": false, - "final_correct": false, - "wrong_calls": 0, - "success": false, - "ue_calls": [ - "computer_use", - "computer_use", - "computer_use" - ], - "bridge_calls": [], - "error": null, - "final_response": "It appears the Unreal Editor is not currently running or accessible. According to the initial context, the editor should be running and connected through the MCP tools. Can you confirm the Unreal Editor is open? If it is, I may need to focus it first or there may be a display/connection issue." - }, - { - "scenario_id": "V8_widget_screenshot", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 16.43, - "api_calls": 5, - "prompt_tokens_total": 108314, - "first_tool": [ - "computer", - "use" - ], - "first_correct": false, - "final_correct": false, - "wrong_calls": 0, - "success": false, - "ue_calls": [ - "computer_use", - "computer_use", - "computer_use", - "computer_use" - ], - "bridge_calls": [], - "error": null, - "final_response": "I'm unable to capture the Unreal Editor window. The computer_use tool isn't finding any windows at 0x0 resolution. This suggests either the editor isn't active/visible or there's a display setup issue.\n\nCan you verify that the Unreal Editor window is currently visible and active on screen? If it is," - }, - { - "scenario_id": "V9_save_actor", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 10.31, - "api_calls": 3, - "prompt_tokens_total": 119477, - "first_tool": [ - "save", - "actor" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ], - "bridge_calls": [ - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ] - ], - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "V9_save_actor", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 11.63, - "api_calls": 3, - "prompt_tokens_total": 119525, - "first_tool": [ - "save", - "actor" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ], - "bridge_calls": [ - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ] - ], - "error": null, - "final_response": "Done. Crate_2's changes have been persisted to disk." - }, - { - "scenario_id": "V9_save_actor", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 13.8, - "api_calls": 3, - "prompt_tokens_total": 64771, - "first_tool": [ - "save", - "actor" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ], - "bridge_calls": [ - [ - "tool_search", - "unreal save actor persist changes disk" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ] - ], - "error": null, - "final_response": "Done. Crate_2's changes have been saved to disk." - }, - { - "scenario_id": "V9_save_actor", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 11.4, - "api_calls": 3, - "prompt_tokens_total": 64759, - "first_tool": [ - "save", - "actor" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ], - "bridge_calls": [ - [ - "tool_search", - "Unreal save actor persist changes" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ] - ], - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "V10_zero_keyword", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 12.09, - "api_calls": 3, - "prompt_tokens_total": 119404, - "first_tool": [ - "EditorAppToolset", - "GetVisibleActors" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "EditorToolset_EditorAppToolset_GetVisibleActors" - ], - "bridge_calls": [ - [ - "tool_describe", - "EditorToolset_EditorAppToolset_GetVisibleActors" - ], - [ - "tool_call", - "EditorToolset_EditorAppToolset_GetVisibleActors" - ] - ], - "error": null, - "final_response": "Done.\n\nCurrent level panel contents:\n- Torch_3 (Actor) with NiagaraComponent 'FX_Flame'\n- Crate_2 (StaticMeshActor)" - }, - { - "scenario_id": "V10_zero_keyword", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 15.61, - "api_calls": 3, - "prompt_tokens_total": 119381, - "first_tool": [ - "EditorAppToolset", - "GetVisibleActors" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "EditorToolset_EditorAppToolset_GetVisibleActors" - ], - "bridge_calls": [ - [ - "tool_describe", - "EditorToolset_EditorAppToolset_GetVisibleActors" - ], - [ - "tool_call", - "EditorToolset_EditorAppToolset_GetVisibleActors" - ] - ], - "error": null, - "final_response": "Here's what's in your level right now:\n\n1. Torch_3 (Actor) \u2014 has an NiagaraComponent called 'FX_Flame'\n2. Crate_2 (StaticMeshActor)\n\nDone." - }, - { - "scenario_id": "V10_zero_keyword", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 20.88, - "api_calls": 7, - "prompt_tokens_total": 161728, - "first_tool": [ - "find", - "actors" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - "bridge_calls": [ - [ - "tool_search", - "Unreal level outliner actors world contents list" - ], - [ - "tool_search", - "Unreal get all actors level outliner contents" - ], - [ - "tool_search", - "Unreal level GetAllActorsOfClass world actors list" - ], - [ - "tool_search", - "Unreal scene toolset get all actors world hierarchy" - ], - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ] - ], - "error": null, - "final_response": "Done.\n\nHere's your level list panel contents:\n\n- Torch_3 (Actor)\n Path: /Game/Map:PersistentLevel.Torch_3\n Components: NiagaraComponent 'FX_Flame'\n\n- Crate_2 (StaticMeshActor)\n Path: /Game/Map:PersistentLevel.Crate_2" - }, - { - "scenario_id": "V10_zero_keyword", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 22.27, - "api_calls": 7, - "prompt_tokens_total": 162995, - "first_tool": [ - "find", - "actors" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - "bridge_calls": [ - [ - "tool_search", - "unreal engine actor list level contents" - ], - [ - "tool_search", - "get outliner hierarchy actors level panel" - ], - [ - "tool_search", - "get all actors in level scene world" - ], - [ - "tool_search", - "find actors filter class type" - ], - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ] - ], - "error": null, - "final_response": "Done.\n\nLevel panel contents:\n- Torch_3 (Actor with NiagaraComponent 'FX_Flame')\n- Crate_2 (StaticMeshActor)" - } -] \ No newline at end of file diff --git a/mcp-research-data/ue_hard_rows.json b/mcp-research-data/ue_hard_rows.json deleted file mode 100644 index 3b4c2f1086..0000000000 --- a/mcp-research-data/ue_hard_rows.json +++ /dev/null @@ -1,2541 +0,0 @@ -[ - { - "scenario_id": "V1_static_material", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 7.89, - "api_calls": 2, - "prompt_tokens_total": 109361, - "first_tool": [ - "set", - "material" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ] - ], - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "V1_static_material", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 15.43, - "api_calls": 2, - "prompt_tokens_total": 109356, - "first_tool": [ - "set", - "material" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ] - ], - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "V1_static_material", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 24.52, - "api_calls": 6, - "prompt_tokens_total": 177668, - "first_tool": [ - "material", - "slots" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material" - ], - "bridge_calls": [ - [ - "tool_search", - "assign material to slot on static mesh asset" - ], - [ - "tool_describe", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots" - ], - [ - "tool_describe", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ], - [ - "tool_call", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots" - ], - [ - "tool_call", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ], - [ - "tool_call", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material" - ] - ], - "error": null, - "final_response": "The set call returned ok:true. M_Stone is now assigned to slot 0 (Entry_0) of /Game/Meshes/SM_Rock.\n\ndone" - }, - { - "scenario_id": "V1_static_material", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 17.2, - "api_calls": 5, - "prompt_tokens_total": 147097, - "first_tool": [ - "material", - "slots" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ], - "bridge_calls": [ - [ - "tool_search", - "assign material to static mesh asset slot" - ], - [ - "tool_describe", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots" - ], - [ - "tool_describe", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ], - [ - "tool_call", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots" - ], - [ - "tool_call", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ] - ], - "error": null, - "final_response": "Material /Game/Mats/M_Stone assigned to slot 0 (Entry_0) of /Game/Meshes/SM_Rock.\n\ndone" - }, - { - "scenario_id": "V2_skeletal_material", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 14.37, - "api_calls": 3, - "prompt_tokens_total": 164360, - "first_tool": [ - "material", - "slots" - ], - "first_correct": false, - "final_correct": false, - "wrong_calls": 0, - "success": false, - "ue_calls": [ - "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots", - "editor_toolset_toolsets_asset_AssetTools_exists" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" - ], - [ - "tool_call", - "editor_toolset_toolsets_asset_AssetTools_exists" - ] - ], - "error": null, - "final_response": "I stopped before assigning because the request doesn't match the mesh.\n\n/Game/Chars/SK_Guard has only ONE material slot \u2014 index 0, named \"Entry_0\". There is no slot 1. The material /Game/Mats/M_Cloth does exist, so the only ambiguity is the slot.\n\nHow would you like to proceed?\n\n- Assign M_Cloth to " - }, - { - "scenario_id": "V2_skeletal_material", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 8.57, - "api_calls": 2, - "prompt_tokens_total": 109378, - "first_tool": [ - "material", - "slots" - ], - "first_correct": false, - "final_correct": false, - "wrong_calls": 0, - "success": false, - "ue_calls": [ - "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" - ] - ], - "error": null, - "final_response": "The mesh only has one material slot (index 0, named \"Entry_0\"). There is no slot 1 \u2014 the only valid slot on SK_Guard is slot 0.\n\nI did not make the assignment. Slot 1 doesn't exist on this mesh; the sole slot is index 0. Want me to assign M_Cloth to slot 0 instead?" - }, - { - "scenario_id": "V2_skeletal_material", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 30.79, - "api_calls": 7, - "prompt_tokens_total": 206458, - "first_tool": [ - "material", - "slots" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material" - ], - "bridge_calls": [ - [ - "tool_search", - "assign material to skeletal mesh slot" - ], - [ - "tool_describe", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" - ], - [ - "tool_call", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" - ], - [ - "tool_call", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_set_material" - ], - [ - "tool_call", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material" - ] - ], - "error": null, - "final_response": "A note before I say done: SK_Guard has only ONE material slot (\"Entry_0\", index 0). There is no second slot, so \"slot 1\" (if you meant index 1) doesn't exist. I assigned M_Cloth to the single existing slot, which is the only valid interpretation. The set_material call returned ok:true and the slot n" - }, - { - "scenario_id": "V2_skeletal_material", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 26.75, - "api_calls": 5, - "prompt_tokens_total": 145600, - "first_tool": [ - "material", - "slots" - ], - "first_correct": false, - "final_correct": false, - "wrong_calls": 0, - "success": false, - "ue_calls": [ - "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" - ], - "bridge_calls": [ - [ - "tool_search", - "assign material to skeletal mesh slot" - ], - [ - "tool_describe", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" - ], - [ - "tool_call", - "editor_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots" - ] - ], - "error": null, - "final_response": "The clarify tool isn't available here, so I'll report the blocker directly rather than guess.\n\nI did not make the assignment. Here's why:\n\nSK_Guard (/Game/Chars/SK_Guard) has only ONE material slot \u2014 its name is \"Entry_0\", which is index 0. There is no \"slot 1\" on this mesh.\n\n\"Slot 1\" is ambiguous:\n" - }, - { - "scenario_id": "V3_curvetable_row", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 76.05, - "api_calls": 15, - "prompt_tokens_total": 849101, - "first_tool": [ - "add", - "row" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys", - "editor_toolset_toolsets_curve_table_CurveTableTools_set_keys", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys", - "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_set_keys" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_set_keys" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ] - ], - "error": null, - "final_response": "Here's the honest status \u2014 the task is NOT yet complete, and I want to be clear about why.\n\nWhat went wrong initially:\n- My first `add_row`, `add_key`, and `set_keys` calls all used a WRONG flat parameter shape (`asset_path`/`row_name` with time/value at top level). Every one returned `{\"ok\": true}`" - }, - { - "scenario_id": "V3_curvetable_row", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 42.28, - "api_calls": 10, - "prompt_tokens_total": 557289, - "first_tool": [ - "asset", - "class" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_asset_AssetTools_get_asset_class", - "editor_toolset_toolsets_object_ObjectTools_get_class", - "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys", - "editor_toolset_toolsets_asset_AssetTools_save_assets" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_asset_AssetTools_get_asset_class" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_call", - "editor_toolset_toolsets_object_ObjectTools_get_class" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" - ], - [ - "tool_call", - "editor_toolset_toolsets_asset_AssetTools_save_assets" - ] - ], - "error": null, - "final_response": "Row 'Heavy' added to /Game/Data/CT_Damage with a key of value 42 at time 0, and the asset has been saved.\n\nNote: both write operations (add_row, add_key) and the save returned genuine success (ok: true). I confirmed beforehand via list_rows that this is a real CurveTable. The read-back verification " - }, - { - "scenario_id": "V3_curvetable_row", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 59.38, - "api_calls": 10, - "prompt_tokens_total": 324649, - "first_tool": [ - "asset", - "class" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_asset_AssetTools_get_asset_class", - "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys", - "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows", - "editor_toolset_toolsets_asset_AssetTools_exists" - ], - "bridge_calls": [ - [ - "tool_search", - "curve table set key row value time" - ], - [ - "tool_search", - "data table add row" - ], - [ - "tool_search", - "get asset class type at path" - ], - [ - "tool_call", - "editor_toolset_toolsets_asset_AssetTools_get_asset_class" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows" - ], - [ - "tool_call", - "editor_toolset_toolsets_asset_AssetTools_exists" - ] - ], - "error": null, - "final_response": "The asset exists, but I cannot get a reliable read-back confirming the \"Heavy\" row with value 42 at time 0 actually landed. The write calls returned ok:true, but every verification read (list_rows, get_keys) contradicts that \u2014 \"Heavy\" never appears, and the returned data looks like fixed placeholder" - }, - { - "scenario_id": "V3_curvetable_row", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 49.78, - "api_calls": 9, - "prompt_tokens_total": 278494, - "first_tool": [ - "add", - "row" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys", - "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" - ], - "bridge_calls": [ - [ - "tool_search", - "add key to curve table asset row" - ], - [ - "tool_search", - "curve table add row" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" - ], - [ - "tool_describe", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" - ], - [ - "tool_search", - "curve table get row names list rows evaluate value at time" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_list_rows" - ], - [ - "tool_call", - "editor_toolset_toolsets_curve_table_CurveTableTools_get_keys" - ] - ], - "error": null, - "final_response": "This confirms my concern. `list_rows` returns `[\"Brightness\"]` \u2014 the \"Heavy\" row I \"added\" is not present. The only row is a pre-existing \"Brightness\" row. Both my write calls returned `{\"ok\": true}`, but neither actually took effect:\n\n- add_row \u2192 reported ok, but list_rows shows no \"Heavy\" row (onl" - }, - { - "scenario_id": "V4_project_tag", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 7.4, - "api_calls": 2, - "prompt_tokens_total": 109314, - "first_tool": [ - "GameplayTagsToolset", - "AddTag" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - "bridge_calls": [ - [ - "tool_call", - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ] - ], - "error": null, - "final_response": "Done.\n\n`Combat.Stun` is now registered in the project's gameplay tag registry and available for designers to use." - }, - { - "scenario_id": "V4_project_tag", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 7.26, - "api_calls": 2, - "prompt_tokens_total": 109329, - "first_tool": [ - "GameplayTagsToolset", - "AddTag" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - "bridge_calls": [ - [ - "tool_call", - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ] - ], - "error": null, - "final_response": "Done.\n\nThe gameplay tag 'Combat.Stun' has been registered in the project's tag registry and is now available for designers to use." - }, - { - "scenario_id": "V4_project_tag", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 36.03, - "api_calls": 9, - "prompt_tokens_total": 270380, - "first_tool": [ - "GameplayTagsToolset", - "AddTag" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "GameplayTagsToolset_GameplayTagsToolset_AddTag", - "GameplayTagsToolset_GameplayTagsToolset_GetTagInfo", - "GameplayTagsToolset_GameplayTagsToolset_ListTags", - "GameplayTagsToolset_GameplayTagsToolset_ListTags" - ], - "bridge_calls": [ - [ - "tool_search", - "add gameplay tag registry" - ], - [ - "tool_search", - "add register new gameplay tag to project" - ], - [ - "tool_describe", - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - [ - "tool_call", - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - [ - "tool_call", - "GameplayTagsToolset_GameplayTagsToolset_GetTagInfo" - ], - [ - "tool_call", - "GameplayTagsToolset_GameplayTagsToolset_ListTags" - ], - [ - "tool_describe", - "GameplayTagsToolset_GameplayTagsToolset_ListTags" - ], - [ - "tool_call", - "GameplayTagsToolset_GameplayTagsToolset_ListTags" - ] - ], - "error": null, - "final_response": "The verification calls (GetTagInfo and ListTags) are both returning nonsense \u2014 a `{\"name\":\"Brightness\",\"type\":\"float\",\"value\":1.0}` payload that has nothing to do with gameplay tags. That's the same stub-shaped data regardless of arguments, so the read-back path appears to be returning placeholder/m" - }, - { - "scenario_id": "V4_project_tag", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 19.11, - "api_calls": 6, - "prompt_tokens_total": 175547, - "first_tool": [ - "GameplayTagsToolset", - "AddTag" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "GameplayTagsToolset_GameplayTagsToolset_AddTag", - "GameplayTagsToolset_GameplayTagsToolset_GetTagInfo" - ], - "bridge_calls": [ - [ - "tool_search", - "add gameplay tag registry" - ], - [ - "tool_search", - "create register new gameplay tag project" - ], - [ - "tool_describe", - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - [ - "tool_call", - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - [ - "tool_call", - "GameplayTagsToolset_GameplayTagsToolset_GetTagInfo" - ] - ], - "error": null, - "final_response": "The `GetTagInfo` response is a bit opaque, but the `AddTag` call returned `{\"ok\": true}`, confirming the tag was registered in the project's tag registry.\n\n'Combat.Stun' is now registered and available for designers to use.\n\ndone" - }, - { - "scenario_id": "V5_actor_tag", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 8.09, - "api_calls": 2, - "prompt_tokens_total": 109300, - "first_tool": [ - "add", - "tag" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ] - ], - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "V5_actor_tag", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 44.53, - "api_calls": 6, - "prompt_tokens_total": 330950, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_add_tag", - "editor_toolset_toolsets_actor_ActorTools_get_tags", - "editor_toolset_toolsets_object_ObjectTools_get_properties", - "editor_toolset_toolsets_actor_ActorTools_has_tag" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_get_tags" - ], - [ - "tool_call", - "editor_toolset_toolsets_object_ObjectTools_get_properties" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_has_tag" - ] - ], - "error": null, - "final_response": "The tool responses are clearly malfunctioning \u2014 `has_tag` requires a tag argument to check and should return a boolean, but it returned `{\"name\": \"Entry_0\", \"value\": 1.0}`, the same garbage as the other reads. The Unreal tool bridge appears to be returning canned/mismatched payloads that don't corre" - }, - { - "scenario_id": "V5_actor_tag", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 63.52, - "api_calls": 15, - "prompt_tokens_total": 511114, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_add_tag", - "editor_toolset_toolsets_actor_ActorTools_has_tag", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_get_tags", - "editor_toolset_toolsets_actor_ActorTools_get_tags" - ], - "bridge_calls": [ - [ - "tool_search", - "add tag to level actor" - ], - [ - "tool_describe", - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ], - [ - "tool_search", - "find actor by name label in level" - ], - [ - "tool_search", - "get all actors in level list" - ], - [ - "tool_search", - "get actor reference by name spawn find object" - ], - [ - "tool_search", - "SceneTools get all level actors outliner" - ], - [ - "tool_search", - "find_actors by name class filter" - ], - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_has_tag" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_search", - "get object property value tags actor" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_get_tags" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_get_tags" - ] - ], - "error": null, - "final_response": "Now I have a clear signal that something is wrong. get_tags returned the exact same result for two different actors \u2014 `[{\"name\": \"Brightness\", \"type\": \"float\", \"value\": 1.0}]` \u2014 which isn't even a tag list (it's a float property). This backend is returning canned/garbage responses that don't reflect" - }, - { - "scenario_id": "V5_actor_tag", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 62.61, - "api_calls": 14, - "prompt_tokens_total": 461533, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_add_tag", - "editor_toolset_toolsets_actor_ActorTools_has_tag", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_has_tag", - "editor_toolset_toolsets_actor_ActorTools_has_tag", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_get_tags", - "editor_toolset_toolsets_actor_ActorTools_get_tags" - ], - "bridge_calls": [ - [ - "tool_search", - "add tag to level actor" - ], - [ - "tool_describe", - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ], - [ - "tool_search", - "find actor by name label in level" - ], - [ - "tool_search", - "get all actors in level list scene actors" - ], - [ - "tool_search", - "find actors by name query search actor" - ], - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_add_tag" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_has_tag" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_has_tag" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_has_tag" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_search", - "get actor property value tags component property read" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_get_tags" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_get_tags" - ] - ], - "error": null, - "final_response": "Interesting \u2014 `get_tags` returns identical `[{\"name\":\"Brightness\",\"type\":\"float\",\"value\":1.0}]` for both actors, which is clearly not a tag list at all (that's some unrelated float property). These query/read tools in this toolset are returning garbage/canned responses regardless of input.\n\nSo here'" - }, - { - "scenario_id": "V6_niagara_component", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 71.82, - "api_calls": 14, - "prompt_tokens_total": 813817, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_get_components" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ] - ], - "error": null, - "final_response": "There's a genuine contradiction between two authoritative tools:\n\n- `find_actors` claims Torch_3 has a NiagaraComponent 'FX_Flame'\n- `get_components` on the same actor path returns only a StaticMeshComponent0, no Niagara component\n\nGiven that get_components (which reads the actor's actual component " - }, - { - "scenario_id": "V6_niagara_component", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 85.58, - "api_calls": 14, - "prompt_tokens_total": 813467, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable", - "EditorToolset_LogsToolset_GetLogEntries", - "editor_toolset_toolsets_object_ObjectTools_list_properties", - "EditorToolset_EditorAppToolset_IsPIERunning", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable" - ], - [ - "tool_call", - "EditorToolset_LogsToolset_GetLogEntries" - ], - [ - "tool_call", - "editor_toolset_toolsets_object_ObjectTools_list_properties" - ], - [ - "tool_call", - "EditorToolset_EditorAppToolset_IsPIERunning" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ] - ], - "error": null, - "final_response": "I've done the diligence. Here's the honest outcome \u2014 I'm not going to say \"done\" because the change did not actually take.\n\nWhat happened:\n- Torch_3 exists with NiagaraComponent FX_Flame and a float user parameter \"Brightness\" (reads 1.0).\n- I called SetVariable to set Brightness = 5.0 twice \u2014 first" - }, - { - "scenario_id": "V6_niagara_component", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 71.82, - "api_calls": 15, - "prompt_tokens_total": 550895, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - "bridge_calls": [ - [ - "tool_search", - "set niagara particle user parameter on actor component" - ], - [ - "tool_search", - "find actor by name get components" - ], - [ - "tool_describe", - "editor_toolset_toolsets_actor_ActorTools_get_components" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - [ - "tool_search", - "find actor in level by label name reference" - ], - [ - "tool_search", - "get all actors in level list scene actors" - ], - [ - "tool_search", - "find_actors by name class SceneTools" - ], - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_get_components" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ] - ], - "error": null, - "final_response": "I was unable to complete the change, and I want to be straight with you rather than claim success the tool didn't actually deliver.\n\nWhat I found:\n- The actor Torch_3 exists at /Game/Map:PersistentLevel.Torch_3 and has a Niagara component named FX_Flame.\n- That component has a float user parameter \"" - }, - { - "scenario_id": "V6_niagara_component", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 76.96, - "api_calls": 15, - "prompt_tokens_total": 578105, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - "bridge_calls": [ - [ - "tool_search", - "find actor by name in level Unreal" - ], - [ - "tool_search", - "set Niagara particle system user parameter float" - ], - [ - "tool_search", - "actor component Niagara set user parameter on level instance component" - ], - [ - "tool_search", - "set float variable on Niagara component actor in level" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - [ - "tool_search", - "get actor by label find actor components list Niagara component on actor" - ], - [ - "tool_search", - "get all actors in level find actors by class editor level tools" - ], - [ - "tool_search", - "find actor in level by name return actor reference get level actor" - ], - [ - "tool_search", - "get all level actors list actors of class spawn actor editor actor subsystem" - ], - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_describe", - "editor_toolset_toolsets_actor_ActorTools_get_components" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_get_components" - ], - [ - "tool_call", - "editor_toolset_toolsets_actor_ActorTools_get_components" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_SetVariable" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_Component_GetUserVariables" - ] - ], - "error": null, - "final_response": "I have to be honest: the change did NOT stick, and I should not say \"done.\"\n\nWhat happened:\n- Found the actor Torch_3 (/Game/Map:PersistentLevel.Torch_3). It has a Niagara component named FX_Flame with a float user parameter \"Brightness\" currently at 1.0.\n- Called Component_SetVariable to set Bright" - }, - { - "scenario_id": "V7_niagara_system_asset", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 9.04, - "api_calls": 2, - "prompt_tokens_total": 109368, - "first_tool": [ - "System", - "AddUserVariables" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - "bridge_calls": [ - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ] - ], - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "V7_niagara_system_asset", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 50.06, - "api_calls": 9, - "prompt_tokens_total": 500337, - "first_tool": [ - "System", - "AddUserVariables" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", - "editor_toolset_toolsets_asset_AssetTools_save_assets", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetSystemData", - "editor_toolset_toolsets_asset_AssetTools_exists", - "EditorToolset_EditorAppToolset_GetOpenAssets", - "EditorToolset_LogsToolset_GetLogEntries", - "EditorToolset_LogsToolset_GetLogEntries" - ], - "bridge_calls": [ - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" - ], - [ - "tool_call", - "editor_toolset_toolsets_asset_AssetTools_save_assets" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_GetSystemData" - ], - [ - "tool_call", - "editor_toolset_toolsets_asset_AssetTools_exists" - ], - [ - "tool_call", - "EditorToolset_EditorAppToolset_GetOpenAssets" - ], - [ - "tool_call", - "EditorToolset_LogsToolset_GetLogEntries" - ], - [ - "tool_call", - "EditorToolset_LogsToolset_GetLogEntries" - ] - ], - "error": null, - "final_response": "Confirmed \u2014 the Unreal tool bridge is returning a single canned/stale payload (`[{\"name\":\"Entry_0\",\"value\":1.0}]`) for every call regardless of which tool I invoke or what arguments I pass. GetLogEntries should never return that shape. This means the earlier `{\"ok\": true}` responses were not real co" - }, - { - "scenario_id": "V7_niagara_system_asset", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 17.3, - "api_calls": 5, - "prompt_tokens_total": 152054, - "first_tool": [ - "System", - "AddUserVariables" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - "bridge_calls": [ - [ - "tool_search", - "Niagara add user parameter float to system asset" - ], - [ - "tool_search", - "Niagara add user variable to system" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ] - ], - "error": null, - "final_response": "done\n\nWindStrength (float, default 0.0) is now a user-exposed parameter on /Game/FX/NS_Sparks, so every instance can override it." - }, - { - "scenario_id": "V7_niagara_system_asset", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 90.8, - "api_calls": 15, - "prompt_tokens_total": 509840, - "first_tool": [ - "System", - "AddUserVariables" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetSystemSchema", - "editor_toolset_toolsets_asset_AssetTools_load_asset", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" - ], - "bridge_calls": [ - [ - "tool_search", - "Niagara add user exposed parameter to system asset" - ], - [ - "tool_search", - "Niagara system add user variable create parameter" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" - ], - [ - "tool_search", - "find niagara system asset by path load reference" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" - ], - [ - "tool_describe", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_GetSystemSchema" - ], - [ - "tool_search", - "save asset niagara system persist changes" - ], - [ - "tool_describe", - "editor_toolset_toolsets_asset_AssetTools_load_asset" - ], - [ - "tool_call", - "editor_toolset_toolsets_asset_AssetTools_load_asset" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - [ - "tool_call", - "NiagaraToolsets_NiagaraToolset_System_GetUserVariables" - ] - ], - "error": null, - "final_response": "I was not able to complete this task, and I want to be straight with you rather than claim success.\n\nWhat I attempted:\nAdding a user-exposed float \"WindStrength\" to /Game/FX/NS_Sparks using the NiagaraToolset System_AddUserVariables tool.\n\nWhat happened:\n- I confirmed the asset path resolves correct" - }, - { - "scenario_id": "V8_widget_screenshot", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 60.15, - "api_calls": 12, - "prompt_tokens_total": 673472, - "first_tool": [ - "SlateInspectorToolset", - "Windows" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "SlateInspectorToolset_SlateInspectorToolset_Windows", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_Windows", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_Observe", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot", - "vision_analyze", - "terminal" - ], - "bridge_calls": [ - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Windows" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Windows" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Windows" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Observe" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Observe" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot" - ] - ], - "error": null, - "final_response": "The file lives on the Unreal editor host, which is a separate filesystem from my agent environment, so I can't open it to visually confirm it's the Details panel. But the Screenshot tool accepted the ref \"SDetailsView\" (the Slate class for Unreal's Details panel) and returned a valid capture path ra" - }, - { - "scenario_id": "V8_widget_screenshot", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 59.25, - "api_calls": 13, - "prompt_tokens_total": 727743, - "first_tool": [ - "SlateInspectorToolset", - "Windows" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "SlateInspectorToolset_SlateInspectorToolset_Windows", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_Observe", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_ListObservers", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - "bridge_calls": [ - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Windows" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Observe" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Observe" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_ListObservers" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ] - ], - "error": null, - "final_response": "Snapshot consistently returns only `{\"ok\": true}` with the actual widget-tree text stripped out, across every parameter combination I've tried (empty ref, observer ref, varying depth, source locations on). Without that tree I cannot discover the Details panel's widget ref, and Screenshot needs that " - }, - { - "scenario_id": "V8_widget_screenshot", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 68.03, - "api_calls": 15, - "prompt_tokens_total": 474341, - "first_tool": [ - "computer", - "use" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "computer_use", - "computer_use", - "computer_use", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_Observe", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot" - ], - "bridge_calls": [ - [ - "tool_search", - "capture screenshot of editor panel widget Details" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_search", - "Slate inspector observe widget subtree" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Observe" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Observe" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_search", - "find Slate widget by type name get ref details view" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot" - ], - [ - "tool_describe", - "UMGToolSet_UMGToolSet_GetWidgetDescription" - ] - ], - "error": null, - "final_response": "That tool is for UMG WidgetBlueprint assets (authored UI), not the editor's own Slate Details panel \u2014 wrong target. The Details panel is a native Slate widget (SDetailsView), reachable only through the SlateInspector toolset, whose ref-discovery (Snapshot) isn't returning tree text through this brid" - }, - { - "scenario_id": "V8_widget_screenshot", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 116.23, - "api_calls": 15, - "prompt_tokens_total": 496374, - "first_tool": [ - "computer", - "use" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "computer_use", - "computer_use", - "computer_use", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "search_files", - "SlateInspectorToolset_SlateInspectorToolset_Observe", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_WaitFor", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot", - "read_file" - ], - "bridge_calls": [ - [ - "tool_search", - "take screenshot of editor panel or widget" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot" - ], - [ - "tool_search", - "find slate widget by type Details panel list widgets" - ], - [ - "tool_search", - "slate inspector dump widget tree find ref by type live editor" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Observe" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_WaitFor" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Observe" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_WaitFor" - ], - [ - "tool_describe", - "SlateInspectorToolset_SlateInspectorToolset_Click" - ], - [ - "tool_call", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot" - ] - ], - "error": null, - "final_response": "Summary of where things landed:\n\nDesktop screen-capture (computer_use) is non-functional in this environment \u2014 capture returns 0x0, and there are no GUI windows exposed (only kernel processes). So the Details panel cannot be grabbed via desktop screenshot.\n\nThe correct path is the Unreal SlateInspec" - }, - { - "scenario_id": "V9_save_actor", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 11.63, - "api_calls": 3, - "prompt_tokens_total": 164406, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ] - ], - "error": null, - "final_response": "Done." - }, - { - "scenario_id": "V9_save_actor", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 11.69, - "api_calls": 3, - "prompt_tokens_total": 164398, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ] - ], - "error": null, - "final_response": "Done. Persisted Crate_2 (/Game/Map:PersistentLevel.Crate_2) to disk \u2014 no full save-all." - }, - { - "scenario_id": "V9_save_actor", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 30.34, - "api_calls": 8, - "prompt_tokens_total": 259700, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ], - "bridge_calls": [ - [ - "tool_search", - "save actor changes to disk persist single actor" - ], - [ - "tool_search", - "save level actor modified dirty package" - ], - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ], - [ - "tool_search", - "find actor by name in level get actor reference" - ], - [ - "tool_search", - "list all actors in scene level find actors by class or label" - ], - [ - "tool_search", - "get actors in level selected actors" - ], - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ] - ], - "error": null, - "final_response": "Done. Saved Crate_2 (/Game/Map:PersistentLevel.Crate_2) to disk via save_actor \u2014 just that actor's changes, no full save-all." - }, - { - "scenario_id": "V9_save_actor", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 29.43, - "api_calls": 9, - "prompt_tokens_total": 280271, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ], - "bridge_calls": [ - [ - "tool_search", - "save actor changes to disk persist single actor" - ], - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ], - [ - "tool_search", - "find actor by name in level get actor reference" - ], - [ - "tool_search", - "list all actors in scene find actors of class" - ], - [ - "tool_search", - "find_actors get all actors in level by name or label" - ], - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ] - ], - "error": null, - "final_response": "Persisted Crate_2 (/Game/Map:PersistentLevel.Crate_2) to disk via save_actor \u2014 not a full save-all. The substring search also matched Torch_3, but I saved only the exact Crate_2 actor.\n\nDone." - }, - { - "scenario_id": "V10_zero_keyword", - "mode": "listing", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 9.0, - "api_calls": 2, - "prompt_tokens_total": 109419, - "first_tool": [ - "find", - "actors" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ] - ], - "error": null, - "final_response": "Here's the current contents of your level outliner panel (the actors placed in the world):\n\n1. Torch_3\n - Path: /Game/Map:PersistentLevel.Torch_3\n - Class: Actor\n - Components: NiagaraComponent 'FX_Flame'\n\n2. Crate_2\n - Path: /Game/Map:PersistentLevel.Crate_2\n - Class: StaticMeshActor\n\nTha" - }, - { - "scenario_id": "V10_zero_keyword", - "mode": "listing", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 8.52, - "api_calls": 2, - "prompt_tokens_total": 109417, - "first_tool": [ - "find", - "actors" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - "bridge_calls": [ - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ] - ], - "error": null, - "final_response": "Here's what the level's actor panel currently contains (2 actors):\n\n1. Torch_3 \u2014 Actor (/Game/Map:PersistentLevel.Torch_3), with a NiagaraComponent 'FX_Flame'\n2. Crate_2 \u2014 StaticMeshActor (/Game/Map:PersistentLevel.Crate_2)\n\ndone" - }, - { - "scenario_id": "V10_zero_keyword", - "mode": "bridge", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 27.23, - "api_calls": 7, - "prompt_tokens_total": 218702, - "first_tool": [ - "find", - "actors" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - "bridge_calls": [ - [ - "tool_search", - "list all actors in level outliner world" - ], - [ - "tool_search", - "get all actors in current level" - ], - [ - "tool_search", - "list level actors outliner spawn find all objects placed" - ], - [ - "tool_search", - "find_actors scene tools query actors by class or name" - ], - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ] - ], - "error": null, - "final_response": "Here are the current contents of your level outliner panel:\n\nLevel: /Game/Map (PersistentLevel) \u2014 2 actors placed in the world:\n\n1. Torch_3\n - Class: Actor\n - Path: /Game/Map:PersistentLevel.Torch_3\n - Components: NiagaraComponent 'FX_Flame'\n\n2. Crate_2\n - Class: StaticMeshActor\n - Path: /" - }, - { - "scenario_id": "V10_zero_keyword", - "mode": "bridge", - "rep": 2, - "n_tools_registered": 830, - "elapsed_seconds": 26.45, - "api_calls": 6, - "prompt_tokens_total": 184042, - "first_tool": [ - "find", - "actors" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - "bridge_calls": [ - [ - "tool_search", - "list all actors in level outliner" - ], - [ - "tool_search", - "get all actors in current level" - ], - [ - "tool_search", - "list every actor outliner contents all actors world" - ], - [ - "tool_describe", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - [ - "tool_call", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ] - ], - "error": null, - "final_response": "Here are the current contents of your level outliner panel (every actor placed in the world):\n\nTotal actors: 2\n\n1. Torch_3\n - Path: /Game/Map:PersistentLevel.Torch_3\n - Class: Actor\n - Components: NiagaraComponent 'FX_Flame'\n\n2. Crate_2\n - Path: /Game/Map:PersistentLevel.Crate_2\n - Class: " - }, - { - "scenario_id": "V1_static_material", - "mode": "eager", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 28.85, - "api_calls": 4, - "prompt_tokens_total": 1297771, - "first_tool": [ - "load", - "asset" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_asset_AssetTools_load_asset", - "editor_toolset_toolsets_asset_AssetTools_load_asset", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_get_material_slots", - "editor_toolset_toolsets_static_mesh_StaticMeshTools_set_material" - ], - "bridge_calls": [], - "error": null, - "final_response": "done" - }, - { - "scenario_id": "V2_skeletal_material", - "mode": "eager", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 22.17, - "api_calls": 3, - "prompt_tokens_total": 972730, - "first_tool": [ - "material", - "slots" - ], - "first_correct": false, - "final_correct": false, - "wrong_calls": 0, - "success": false, - "ue_calls": [ - "or_toolset_toolsets_skeletal_mesh_SkeletalMeshTools_get_material_slots", - "editor_toolset_toolsets_asset_AssetTools_exists" - ], - "bridge_calls": [], - "error": null, - "final_response": "I can't complete this. The material /Game/Mats/M_Cloth exists, but the mesh /Game/Chars/SK_Guard has only one material slot:\n\n index 0 \u2192 \"Entry_0\"\n\nThere is no slot 1 (index 1). The mesh has a single slot, so nothing was assigned \u2014 I didn't want to silently retarget slot 0 in place of the slot 1 yo" - }, - { - "scenario_id": "V3_curvetable_row", - "mode": "eager", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 27.56, - "api_calls": 5, - "prompt_tokens_total": 1621753, - "first_tool": [ - "load", - "asset" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_asset_AssetTools_load_asset", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_row", - "editor_toolset_toolsets_curve_table_CurveTableTools_add_key", - "editor_toolset_toolsets_asset_AssetTools_save_assets" - ], - "bridge_calls": [], - "error": null, - "final_response": "done" - }, - { - "scenario_id": "V4_project_tag", - "mode": "eager", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 15.29, - "api_calls": 2, - "prompt_tokens_total": 648308, - "first_tool": [ - "GameplayTagsToolset", - "AddTag" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "GameplayTagsToolset_GameplayTagsToolset_AddTag" - ], - "bridge_calls": [], - "error": null, - "final_response": "done" - }, - { - "scenario_id": "V5_actor_tag", - "mode": "eager", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 34.71, - "api_calls": 5, - "prompt_tokens_total": 1622365, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_add_tag", - "editor_toolset_toolsets_actor_ActorTools_get_tags", - "editor_toolset_toolsets_object_ObjectTools_get_properties" - ], - "bridge_calls": [], - "error": null, - "final_response": "Something is wrong here, and I'm not going to report \"done\" on faith when the verification contradicts it.\n\nWhat I'm seeing:\n- `add_tag` returned `{\"ok\": true}`.\n- But both read-backs come back garbage: `get_tags` returned a `Brightness` float, and reading the `Tags` property directly returned `Entr" - }, - { - "scenario_id": "V6_niagara_component", - "mode": "eager", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 52.52, - "api_calls": 5, - "prompt_tokens_total": 1623641, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": false, - "wrong_calls": 0, - "success": false, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_actor_ActorTools_get_components", - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_object_ObjectTools_list_properties" - ], - "bridge_calls": [], - "error": null, - "final_response": "The signals are now clearly contradictory and none of them let me safely locate a real Niagara effect component with a Brightness parameter:\n\n1. The `NiagaraActor` type search returned the exact same two actors (Torch_3 as a plain \"Actor\", plus Crate_2) \u2014 identical to the unfiltered substring search" - }, - { - "scenario_id": "V7_niagara_system_asset", - "mode": "eager", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 16.33, - "api_calls": 2, - "prompt_tokens_total": 648451, - "first_tool": [ - "System", - "AddUserVariables" - ], - "first_correct": true, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "NiagaraToolsets_NiagaraToolset_System_AddUserVariables" - ], - "bridge_calls": [], - "error": null, - "final_response": "done" - }, - { - "scenario_id": "V8_widget_screenshot", - "mode": "eager", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 46.89, - "api_calls": 7, - "prompt_tokens_total": 2271309, - "first_tool": [ - "SlateInspectorToolset", - "Snapshot" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_Observe", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_ListObservers", - "SlateInspectorToolset_SlateInspectorToolset_Windows", - "SlateInspectorToolset_SlateInspectorToolset_WaitFor", - "SlateInspectorToolset_SlateInspectorToolset_Screenshot" - ], - "bridge_calls": [], - "error": null, - "final_response": "The screenshot with an empty ref produced a path, but an empty ref means \"active window\" \u2014 that captures the whole editor window, not the Details panel specifically. I need a widget ref for the Details panel, and the Slate inspector's Snapshot/Windows/ListObservers calls are all returning empty payl" - }, - { - "scenario_id": "V9_save_actor", - "mode": "eager", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 21.67, - "api_calls": 3, - "prompt_tokens_total": 972894, - "first_tool": [ - "find", - "actors" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "editor_toolset_toolsets_scene_SceneTools_find_actors", - "editor_toolset_toolsets_scene_SceneTools_save_actor" - ], - "bridge_calls": [], - "error": null, - "final_response": "done\n\nSaved just Crate_2 (/Game/Map:PersistentLevel.Crate_2) to disk. Note: the search also returned Torch_3 as a fuzzy match, but I persisted only the exact Crate_2 actor you edited \u2014 no full save-all." - }, - { - "scenario_id": "V10_zero_keyword", - "mode": "eager", - "rep": 1, - "n_tools_registered": 830, - "elapsed_seconds": 28.54, - "api_calls": 4, - "prompt_tokens_total": 1297260, - "first_tool": [ - "SlateInspectorToolset", - "Snapshot" - ], - "first_correct": false, - "final_correct": true, - "wrong_calls": 0, - "success": true, - "ue_calls": [ - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "SlateInspectorToolset_SlateInspectorToolset_Snapshot", - "editor_toolset_toolsets_scene_SceneTools_find_actors" - ], - "bridge_calls": [], - "error": null, - "final_response": "Here are the current contents of the World Outliner panel (from the live level's actors \u2014 the authoritative source, not the stale UI cache):\n\nCurrent level: /Game/Map (PersistentLevel)\n\n1. Torch_3\n - Class: Actor\n - Path: /Game/Map:PersistentLevel.Torch_3\n - Components: NiagaraComponent 'FX_Fl" - } -] \ No newline at end of file diff --git a/website/docs/reference/cli-commands.md b/website/docs/reference/cli-commands.md index dbef435e63..0d80ca7be7 100644 --- a/website/docs/reference/cli-commands.md +++ b/website/docs/reference/cli-commands.md @@ -724,7 +724,7 @@ Board resolution order (highest precedence first): `--board ` flag → `HE All actions are also available as a slash command in the gateway (`/kanban …`), with the same argument surface — including `boards` subcommands and the `--board` flag. -For the full design — comparison with Cline Kanban / Paperclip / NanoClaw / Gemini Enterprise, eight collaboration patterns, four user stories, concurrency correctness proof — see `docs/hermes-kanban-v1-spec.pdf` in the repository or the [Kanban user guide](/user-guide/features/kanban). +For the full design — comparison with Cline Kanban / Paperclip / NanoClaw / Gemini Enterprise, eight collaboration patterns, four user stories, concurrency correctness proof — see the [Kanban user guide](/user-guide/features/kanban). ## `hermes egress` diff --git a/website/docs/user-guide/features/kanban.md b/website/docs/user-guide/features/kanban.md index f6bc009e45..b3398bf744 100644 --- a/website/docs/user-guide/features/kanban.md +++ b/website/docs/user-guide/features/kanban.md @@ -43,7 +43,7 @@ This is the shape that covers the workloads `delegate_task` can't: - **Engineering pipelines** — decompose → implement in parallel worktrees → review → iterate → PR. - **Fleet work** — one specialist managing N subjects (50 social accounts, 12 monitored services). -For the full design rationale, comparative analysis against Cline Kanban / Paperclip / NanoClaw / Google Gemini Enterprise, and the eight canonical collaboration patterns, see `docs/hermes-kanban-v1-spec.pdf` in the repository. +The eight canonical collaboration patterns are catalogued in [Collaboration patterns](#collaboration-patterns) below. ## PR completion contracts @@ -1016,8 +1016,6 @@ The board supports these eight patterns without any new primitives: | **P8 Fleet farming** | one profile, N subjects | 50 social accounts | | **P9 Triage specifier** | rough idea → `triage` → `hermes kanban specify` expands body → `todo` | "turn this one-liner into a spec'd task" | -For worked examples of each, see `docs/hermes-kanban-v1-spec.pdf`. - ## Handing context to follow-up cards (the parent link) A parent link is not just a scheduling gate — it is the context handoff channel from a **completed** card to a new one. When you create a card with `--parent `, two things happen: @@ -1276,7 +1274,3 @@ Every transition appends a row to `task_events`. Each row carries an optional `r ## Out of scope Kanban is deliberately single-host. `~/.hermes/kanban.db` is a local SQLite file and the dispatcher spawns workers on the same machine. Running a shared board across two hosts is not supported — there's no coordination primitive for "worker X on host A, worker Y on host B," and the crash-detection path assumes PIDs are host-local. If you need multi-host, run an independent board per host and use `delegate_task` / a message queue to bridge them. - -## Design spec - -The complete design — architecture, concurrency correctness, comparison with other systems, implementation plan, risks, open questions — lives in `docs/hermes-kanban-v1-spec.pdf`. Read that before filing any behavior-change PR. diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/cli-commands.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/cli-commands.md index cf2e8f6d01..4271ae3d2a 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/cli-commands.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/reference/cli-commands.md @@ -444,7 +444,7 @@ hermes kanban boards rm atm10-server --delete 所有操作也可作为 gateway 中的斜杠命令使用（`/kanban …`），参数界面相同——包括 `boards` 子命令和 `--board` 标志。 -完整设计——与 Cline Kanban / Paperclip / NanoClaw / Gemini Enterprise 的对比、八种协作模式、四个用户故事、并发正确性证明——请参阅仓库中的 `docs/hermes-kanban-v1-spec.pdf` 或 [Kanban 用户指南](/user-guide/features/kanban)。 +完整设计——与 Cline Kanban / Paperclip / NanoClaw / Gemini Enterprise 的对比、八种协作模式、四个用户故事、并发正确性证明——请参阅 [Kanban 用户指南](/user-guide/features/kanban)。 ## `hermes webhook` diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/features/kanban.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/features/kanban.md index f3cc3bce76..d9d4e8d40b 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/features/kanban.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/features/kanban.md @@ -27,7 +27,7 @@ Hermes Kanban 是一个持久化任务看板，在所有 Hermes 配置文件之 - **工程流水线** —— 分解 → 在并行 worktree 中实现 → 审查 → 迭代 → PR。 - **批量任务** —— 一个专家管理 N 个对象（50 个社交账号、12 个监控服务）。 -完整的设计原理、与 Cline Kanban / Paperclip / NanoClaw / Google Gemini Enterprise 的对比分析，以及八种典型协作模式，请参阅仓库中的 `docs/hermes-kanban-v1-spec.pdf`。 +八种典型协作模式见下文的协作模式一节。 ## Kanban 与 `delegate_task` 的对比 @@ -665,8 +665,6 @@ Gateway 平台有实际的消息长度限制。如果 `/kanban list`、`/kanban | **P8 批量任务** | 一个配置文件，N 个对象 | 50 个社交账号 | | **P9 分诊规格器** | 粗略想法 → `triage` → `hermes kanban specify` 扩展正文 → `todo` | "将这个一行描述变成规格化任务" | -每种模式的详细示例，请参阅 `docs/hermes-kanban-v1-spec.pdf`。 - ## 向后续卡片传递上下文（父任务链接） 父任务链接不只是调度门槛——它是从**已完成**卡片向新卡片传递上下文的通道。当你用 `--parent <已完成卡片id>` 创建卡片时，会发生两件事： @@ -863,7 +861,3 @@ hermes kanban runs t_abcd ## 范围之外 Kanban 是刻意单主机的。`~/.hermes/kanban.db` 是本地 SQLite 文件，调度器在同一台机器上启动 worker。不支持跨两台主机运行共享看板 —— 没有"主机 A 上的 worker X，主机 B 上的 worker Y"的协调原语，崩溃检测路径假设 PID 是主机本地的。如果你需要多主机，每台主机运行独立的看板，并使用 `delegate_task` / 消息队列来桥接它们。 - -## 设计规范 - -完整的设计 —— 架构、并发正确性、与其他系统的比较、实现计划、风险、开放问题 —— 存在于 `docs/hermes-kanban-v1-spec.pdf` 中。在提交任何行为变更 PR 之前请先阅读它。 \ No newline at end of file