From b10179a1ac42957ba9d828ce94a151c43017fcc5 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:19:03 -0700 Subject: [PATCH 1/9] =?UTF-8?q?refactor(agent):=20moa=5Floop=20=E2=80=94?= =?UTF-8?q?=20extract=20fan-out/guidance/cache-plan=20phases,=20unify=20ca?= =?UTF-8?q?che-opt=20and=20placeholder=20helpers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/moa_loop.py | 875 +++++++++++++++++++++------------------------- 1 file changed, 402 insertions(+), 473 deletions(-) diff --git a/agent/moa_loop.py b/agent/moa_loop.py index 7b4944913d..631e726cf7 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -25,10 +25,10 @@ from agent.usage_pricing import CanonicalUsage logger = logging.getLogger(__name__) -# Privacy filter (moa.privacy_filter: '' | display | full, #59959). Secret shapes -# are handled by agent.redact; these two patterns add the PII classes it leaves -# alone (emails, formatted NA phones). The phone pattern requires explicit -# delimiters so line numbers, dates, times, SHAs, IPs and versions never match. +# Privacy filter (moa.privacy_filter: '' | display | full). Secret shapes are handled +# by agent.redact; these add the PII classes it leaves alone (emails, formatted NA +# phones). The phone pattern requires explicit delimiters so line numbers, dates, +# times, SHAs, IPs and versions never match. _MOA_EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b") _MOA_PHONE_RE = re.compile( r"(? Any: text = redact_sensitive_text(text, force=True, code_file=True) text = _MOA_EMAIL_RE.sub("[redacted email]", text) - text = _MOA_PHONE_RE.sub("[redacted phone]", text) - return text + return _MOA_PHONE_RE.sub("[redacted phone]", text) def _moa_privacy_mode(moa_raw: Any) -> str: - """Resolve the normalized privacy-filter mode from a raw ``moa`` config.""" + """Normalized privacy-filter mode from a raw ``moa`` config.""" from hermes_cli.moa_config import coerce_privacy_filter raw = moa_raw if isinstance(moa_raw, dict) else {} @@ -68,10 +67,7 @@ def _redact_reference_outputs( reference_outputs: list[tuple[str, str, Any]], ) -> list[tuple[str, str, Any]]: """Redact advisor text in reference-output tuples; accounting slot untouched.""" - return [ - (label, _redact_reference_text(text), acct) - for label, text, acct in reference_outputs - ] + return [(label, _redact_reference_text(text), acct) for label, text, acct in reference_outputs] def _redact_message_content(content: Any) -> Any: @@ -109,8 +105,8 @@ def _redact_trace_accounting(acct: Any) -> Any: ) -# Cold-start caches (#66793): the preset and each (provider, model) runtime are -# immutable for a turn, so avoid re-resolving them on every create() call. +# Cold-start caches: the preset and each (provider, model) runtime are immutable +# for a turn, so avoid re-resolving them on every create() call. _preset_cache_lock = threading.Lock() _preset_cache: dict[tuple, Any] = {} @@ -142,6 +138,7 @@ def _resolve_preset_cached(preset_name: str) -> tuple[dict[str, Any], Any]: _preset_cache[key] = preset return preset, moa_raw + _runtime_cache_lock = threading.Lock() _runtime_cache: dict[tuple[str, str], tuple[float, dict[str, Any]]] = {} @@ -157,9 +154,9 @@ class _RefAccounting: """Per-reference usage, cost and full trace (third slot of a reference-output tuple). Advisors may run on a different model than the aggregator, so cost is priced at - the advisor's OWN rate and summed in dollars. ``messages``/``output``/``model``/ - ``provider``/``temperature`` carry the full input/output for trace persistence - (only populated when tracing is on). + the advisor's OWN rate and summed in dollars. The trace fields (``messages``, + ``output``, ``model``, ``provider``, ``temperature``) are only populated when + tracing is on. """ usage: Any @@ -212,7 +209,6 @@ _REFERENCE_SYSTEM_PROMPT = ( ) - def _slot_label(slot: dict[str, Any]) -> str: label = f"{(slot.get('provider') or '').strip()}:{(slot.get('model') or '').strip()}" effort = str(slot.get("reasoning_effort") or "").strip() @@ -226,7 +222,7 @@ def _slot_reasoning_config(slot: dict[str, Any]) -> dict[str, Any] | None: from hermes_constants import parse_reasoning_effort return parse_reasoning_effort(effort) - except Exception: # pragma: no cover - defensive; bad config must not break MoA + except Exception: # pragma: no cover - bad config must not break MoA return None @@ -234,8 +230,8 @@ def _aggregator_reasoning_config(aggregator: dict[str, Any]) -> dict[str, Any] | """Resolve the aggregator's reasoning config: slot > per-model > global. The aggregator is the ACTING model, so it falls back through the shared - ``resolve_reasoning_config`` chokepoint (#64187). References deliberately do - not: inheriting a global ``xhigh`` into every advisor would multiply cost. + ``resolve_reasoning_config`` chokepoint. References deliberately do not: + inheriting a global ``xhigh`` into every advisor would multiply cost. """ cfg = _slot_reasoning_config(aggregator) if cfg is not None: @@ -244,10 +240,8 @@ def _aggregator_reasoning_config(aggregator: dict[str, Any]) -> dict[str, Any] | from hermes_cli.config import load_config from hermes_constants import resolve_reasoning_config - return resolve_reasoning_config( - load_config() or {}, str(aggregator.get("model") or "") - ) - except Exception: # pragma: no cover - defensive; bad config must not break MoA + return resolve_reasoning_config(load_config() or {}, str(aggregator.get("model") or "")) + except Exception: # pragma: no cover - bad config must not break MoA return None @@ -256,7 +250,8 @@ def _slot_runtime(slot: dict[str, Any]) -> dict[str, Any]: Gives the slot its provider's real api_mode/base_url/api_key instead of letting the auxiliary auto-detector guess. Falls back to bare provider/model on error - (uncached). Cached per (provider, model) with a short TTL (#66793). + (never cached: a transient error would pin bare kwargs for a TTL). Cached per + (provider, model) with a short TTL. """ provider = str(slot.get("provider") or "").strip() model = str(slot.get("model") or "").strip() @@ -264,10 +259,8 @@ def _slot_runtime(slot: dict[str, Any]) -> dict[str, Any]: now = time.monotonic() with _runtime_cache_lock: entry = _runtime_cache.get(cache_key) - if entry is not None: - stamped_at, cached = entry - if now - stamped_at < _RUNTIME_CACHE_TTL_SECONDS: - return cached + if entry is not None and now - entry[0] < _RUNTIME_CACHE_TTL_SECONDS: + return entry[1] out: dict[str, Any] = {"provider": provider, "model": model} try: from hermes_cli.runtime_provider import resolve_runtime_provider @@ -280,20 +273,15 @@ def _slot_runtime(slot: dict[str, Any]) -> dict[str, Any]: if isinstance(extra_body, dict) and extra_body: out["extra_body"] = dict(extra_body) except Exception as exc: # pragma: no cover - defensive - logger.debug("MoA slot runtime resolution failed for %s: %s", - _slot_label(slot), exc) - # Never cache a fallback result: a transient error would pin bare kwargs for a TTL. + logger.debug("MoA slot runtime resolution failed for %s: %s", _slot_label(slot), exc) return out with _runtime_cache_lock: _runtime_cache[cache_key] = (now, out) return out -def _merge_slot_extra_body( - slot_extra_body: Any, - caller_extra_body: Any, -) -> Any: - """Merge slot defaults with a caller override for ``call_llm``.""" +def _merge_slot_extra_body(slot_extra_body: Any, caller_extra_body: Any) -> Any: + """Merge slot defaults with a caller override (caller wins) for ``call_llm``.""" if isinstance(slot_extra_body, dict) and slot_extra_body: if isinstance(caller_extra_body, dict): return {**slot_extra_body, **caller_extra_body} @@ -303,6 +291,20 @@ def _merge_slot_extra_body( return caller_extra_body +def _agent_cache_opts(agent: Any) -> tuple[Any, Any]: + """The live agent's ``(_cache_disabled, _cache_ttl)``; ``(None, None)`` without an agent.""" + if agent is None: + return None, None + return getattr(agent, "_cache_disabled", None), getattr(agent, "_cache_ttl", None) + + +def _with_cache_disabled(runtime: dict[str, Any], cache_disabled: Any) -> dict[str, Any]: + """Pin the live agent's cache disable onto a runtime snapshot (None is a no-op).""" + if cache_disabled is None: + return runtime + return {**runtime, "_cache_disabled": cache_disabled} + + def _maybe_apply_moa_cache_control( messages: list[dict[str, Any]], runtime: dict[str, Any], @@ -314,9 +316,9 @@ def _maybe_apply_moa_cache_control( Same policy function and marker helper as the main loop; MoA has no static prefix so the legacy system-and-3 fallback is used. ``cache_disabled`` is - stamped onto the stub so ``cache_ttl: off`` is honored (#76085); ``cache_ttl`` - threads the agent's tier, clamped per destination (#84733). Returns the - messages unchanged on any error. + stamped onto the stub so ``cache_ttl: off`` is honored; ``cache_ttl`` threads + the agent's tier, clamped per destination. Returns the messages unchanged on + any error. """ try: from agent.agent_runtime_helpers import ( @@ -332,30 +334,26 @@ def _maybe_apply_moa_cache_control( # Explicit kwarg > runtime snapshot (threaded from the live agent) > config. if cache_disabled is None and "_cache_disabled" in runtime: cache_disabled = runtime.get("_cache_disabled") - - # blank_cache_policy_stub is the only sanctioned stub (carries _cache_disabled, #76085). - stub = blank_cache_policy_stub(cache_disabled) + provider = runtime.get("provider") or "" + model = runtime.get("model") or "" + # blank_cache_policy_stub is the only sanctioned stub (carries _cache_disabled). should_cache, native_layout = anthropic_prompt_cache_policy( - stub, - provider=runtime.get("provider") or "", + blank_cache_policy_stub(cache_disabled), + provider=provider, base_url=runtime.get("base_url") or "", api_mode=runtime.get("api_mode") or "", - model=runtime.get("model") or "", + model=model, ) if not should_cache: return messages return apply_anthropic_cache_control( messages, - cache_ttl=effective_cache_ttl( - # None → "5m"; cache-disabled routes already returned above. - cache_ttl, - provider=runtime.get("provider") or "", - model=runtime.get("model") or "", - ), + # None → "5m"; cache-disabled routes already returned above. + cache_ttl=effective_cache_ttl(cache_ttl, provider=provider, model=model), native_anthropic=native_layout, - # Envelope routes reject part-level markers in tool_result.content[] (#89886). + # Envelope routes reject part-level markers in tool_result.content[]. tool_part_markers=envelope_tool_part_cache_markers_supported( - runtime.get("provider") or "", runtime.get("base_url") or "" + provider, runtime.get("base_url") or "" ), ) except Exception as exc: # pragma: no cover - decoration must never break a call @@ -366,8 +364,8 @@ def _maybe_apply_moa_cache_control( def _price_reference_response( response: Any, slot: dict[str, Any], runtime: dict[str, Any] ) -> tuple[Any, Any, str | None, str | None]: - """Normalize a reference response's usage with the slot's OWN provider/api_mode and - price it at its own rate (hence fan-out cost is summed in dollars). Never raises.""" + """Normalize a reference's usage with the slot's OWN provider/api_mode and price it + at its own rate (hence fan-out cost is summed in dollars). Never raises.""" from agent.usage_pricing import estimate_usage_cost, normalize_usage usage = CanonicalUsage() @@ -406,9 +404,8 @@ def _run_reference( """Call one reference model; return ``(label, text, accounting)``. Never raises. The slot is resolved to its provider's real runtime and called through the - same ``call_llm`` path any model uses. Usage is normalized with the slot's OWN - provider/api_mode and priced at its own rate. A failed reference becomes a - labelled ``[failed: …]`` note. Runs inside a thread pool (call_llm is blocking). + same ``call_llm`` path any model uses. A failed reference becomes a labelled + ``[failed: …]`` note. Runs inside a thread pool (call_llm is blocking). """ label = _slot_label(slot) runtime = _slot_runtime(slot) @@ -417,12 +414,12 @@ def _run_reference( "provider": runtime.get("provider") or slot.get("provider"), "temperature": temperature, } + # The trimmed view already stripped the agent's system prompt; this is the only one. + messages = [{"role": "system", "content": _REFERENCE_SYSTEM_PROMPT}, *ref_messages] try: - # The trimmed view already stripped the agent's system prompt; this is the only one. - messages = [{"role": "system", "content": _REFERENCE_SYSTEM_PROMPT}, *ref_messages] - # Trim to THIS model's window (advisors may be smaller than the aggregator, #60345); + # Trim to THIS model's window (advisors may be smaller than the aggregator); # estimated after the system prompt is prepended so it counts too. - messages = _trim_messages_for_reference( + trimmed = _trim_messages_for_reference( messages, slot, runtime, @@ -431,52 +428,43 @@ def _run_reference( ) # Anthropic-style caching is opt-in per request; the advisory view is append-only # across iterations, so decorating lets iteration N+1 replay N's cached prefix. - # Pin the live agent disable onto the runtime (not a fresh config read, #76085). - cache_runtime = runtime - if cache_disabled is not None: - cache_runtime = {**runtime, "_cache_disabled": cache_disabled} - messages = _maybe_apply_moa_cache_control( - messages, cache_runtime, cache_ttl=cache_ttl + # The live agent disable is pinned onto the runtime (not a fresh config read). + trimmed = _maybe_apply_moa_cache_control( + trimmed, _with_cache_disabled(runtime, cache_disabled), cache_ttl=cache_ttl ) # Per-slot max_tokens beats the preset-level reference_max_tokens. - _slot_max_tokens: int | None = slot.get("max_tokens") - _effective_max_tokens = _slot_max_tokens if _slot_max_tokens is not None else max_tokens + slot_max_tokens = slot.get("max_tokens") extra_headers = None # Normalize provider aliases (github, github-copilot, ...) via the canonical table. from agent.auxiliary_client import _normalize_aux_provider - if _normalize_aux_provider(str(runtime.get("provider") or "")) in ( - "copilot", - "copilot-acp", - ): + if _normalize_aux_provider(str(runtime.get("provider") or "")) in ("copilot", "copilot-acp"): # Copilot gates premium models on request attribution; MoA fan-out serves the # user's current turn, so mirror the main agent's x-initiator header. extra_headers = {"x-initiator": "user"} response = call_llm( task="moa_reference", - messages=messages, + messages=trimmed, temperature=temperature, - max_tokens=_effective_max_tokens, + max_tokens=slot_max_tokens if slot_max_tokens is not None else max_tokens, timeout=reference_timeout, reasoning_config=_slot_reasoning_config(slot), extra_headers=extra_headers, **runtime, ) - _output_text = _extract_text(response) or "(empty response)" + output_text = _extract_text(response) or "(empty response)" acct = _RefAccounting( *_price_reference_response(response, slot, runtime), - messages=messages, - output=_output_text, + messages=trimmed, + output=output_text, **trace_fields, ) - return label, _output_text, acct + return label, output_text, acct except Exception as exc: logger.warning("MoA reference model %s failed: %s", label, exc) - return label, f"[failed: {exc}]", _RefAccounting( - CanonicalUsage(), - messages=[{"role": "system", "content": _REFERENCE_SYSTEM_PROMPT}, *ref_messages], - output=f"[failed: {exc}]", - **trace_fields, + note = f"[failed: {exc}]" + return label, note, _RefAccounting( + CanonicalUsage(), messages=messages, output=note, **trace_fields, ) @@ -495,7 +483,7 @@ def _trim_messages_for_reference( reserve_output_tokens: int | None = None, context_length_cache: Any = None, ) -> list[dict[str, Any]]: - """Trim an advisory request to fit a reference model's context window (#60345). + """Trim an advisory request to fit a reference model's context window. ``messages`` is the full request (advisory system prompt included). Budget = window minus ``reserve_output_tokens`` (or a default) minus a safety fraction. @@ -518,8 +506,8 @@ def _trim_messages_for_reference( return messages cache_key = (provider, model) - context_length: int | None = None - if isinstance(context_length_cache, dict) and cache_key in context_length_cache: + has_cache = isinstance(context_length_cache, dict) + if has_cache and cache_key in context_length_cache: context_length = context_length_cache[cache_key] else: try: @@ -530,12 +518,9 @@ def _trim_messages_for_reference( provider=provider, ) except Exception: - logger.debug( - "MoA reference context-length resolution failed for %s", - _slot_label(slot), - ) + logger.debug("MoA reference context-length resolution failed for %s", _slot_label(slot)) context_length = None - if isinstance(context_length_cache, dict): + if has_cache: # Cache failures too so a flaky metadata source is not re-probed per reference. context_length_cache[cache_key] = context_length @@ -555,7 +540,7 @@ def _trim_messages_for_reference( if estimated <= budget: return messages - has_system = bool(messages) and messages[0].get("role") == "system" + has_system = messages[0].get("role") == "system" head = [messages[0]] if has_system else [] body = list(messages[1:] if has_system else messages) @@ -575,12 +560,7 @@ def _trim_messages_for_reference( logger.info( "MoA reference %s: estimated %d tokens exceeds budget %d " "(window %d, output reserve %d); dropped %d oldest message(s).", - _slot_label(slot), - estimated, - budget, - context_length, - reserve, - dropped, + _slot_label(slot), estimated, budget, context_length, reserve, dropped, ) return trimmed @@ -591,6 +571,47 @@ _REFERENCE_POLL_INTERVAL_S = 5.0 _INTERRUPTED_REFERENCE_NOTE = "[skipped: interrupted by user]" +def _placeholder_output(slot: dict[str, Any], note: str) -> tuple[str, str, Any]: + """A reference-output tuple for a slot that was not (fully) run: zero accounting.""" + return _slot_label(slot), note, _RefAccounting(CanonicalUsage()) + + +def _settle_interrupted( + futures: dict[Any, int], + results: list, + reference_models: list[dict[str, Any]], + late_accounting_sink: Any, +) -> None: + """Fill every unfinished slot after a user interrupt. + + Never-dispatched futures are cancelled (nothing billed). Ones that finished + between the interrupt check and now keep their real output. Running ones cannot + be killed and WILL be billed on completion, so their eventual accounting is + handed to ``late_accounting_sink``. + """ + for future, idx in futures.items(): + if results[idx] is not None: + continue + slot = reference_models[idx] + if future.cancel(): + results[idx] = _placeholder_output(slot, _INTERRUPTED_REFERENCE_NOTE) + elif future.done(): + results[idx] = future.result() + else: + results[idx] = _placeholder_output(slot, _INTERRUPTED_REFERENCE_NOTE) + if late_accounting_sink is not None: + def _record_late(f: Any, _label: str = results[idx][0]) -> None: + try: + _lbl, _txt, _acct = f.result() + except Exception: # pragma: no cover - defensive + return + try: + late_accounting_sink(_label, _acct) + except Exception: # pragma: no cover - defensive + logger.debug("MoA: late accounting sink failed for %s", _label) + future.add_done_callback(_record_late) + + def _run_references_parallel( reference_models: list[dict[str, Any]], ref_messages: list[dict[str, Any]], @@ -617,29 +638,23 @@ def _run_references_parallel( results: list[tuple[str, str, Any] | None] = [None] * len(reference_models) futures: dict[Any, int] = {} - workers = min(_MAX_REFERENCE_WORKERS, len(reference_models)) # Executor threads start with an empty contextvars.Context; propagate the turn's # (approval callbacks + Nous Portal conversation tag) into each worker. from tools.thread_context import propagate_context_to_thread total = len(reference_models) completed = 0 - executor = ThreadPoolExecutor(max_workers=workers) + executor = ThreadPoolExecutor(max_workers=min(_MAX_REFERENCE_WORKERS, total)) interrupted = False # Shared per-fan-out context-length cache (dict get/set is GIL-atomic). - _ctx_len_cache: dict[tuple[str, str], int | None] = {} - cache_disabled = ( - getattr(agent, "_cache_disabled", None) if agent is not None else None - ) - # Agent's cache TTL for every advisor request (#84733); clamped in the decorator. - cache_ttl = getattr(agent, "_cache_ttl", None) if agent is not None else None + ctx_len_cache: dict[tuple[str, str], int | None] = {} + # Agent's cache disable + TTL for every advisor request; clamped in the decorator. + cache_disabled, cache_ttl = _agent_cache_opts(agent) try: for idx, slot in enumerate(reference_models): if slot.get("provider") == "moa": - results[idx] = ( - _slot_label(slot), - "[skipped: MoA presets cannot recursively reference MoA]", - _RefAccounting(CanonicalUsage()), + results[idx] = _placeholder_output( + slot, "[skipped: MoA presets cannot recursively reference MoA]" ) continue futures[ @@ -650,14 +665,13 @@ def _run_references_parallel( temperature=temperature, max_tokens=max_tokens, reference_timeout=reference_timeout, - context_length_cache=_ctx_len_cache, + context_length_cache=ctx_len_cache, cache_disabled=cache_disabled, cache_ttl=cache_ttl, ) ] = idx - # Collect every reference (no early exit except a user interrupt); progress - # callbacks fire per completion. + # Collect every reference (no early exit except a user interrupt). pending = set(futures) while pending: done, pending = _futures_wait(pending, timeout=_REFERENCE_POLL_INTERVAL_S) @@ -667,8 +681,7 @@ def _run_references_parallel( completed += 1 if progress_callback is not None: try: - label = _slot_label(reference_models[idx]) - progress_callback(completed, total, label) + progress_callback(completed, total, _slot_label(reference_models[idx])) except Exception as exc: # pragma: no cover - display must never break logger.debug("MoA progress_callback failed: %s", exc) if not pending: @@ -678,42 +691,7 @@ def _run_references_parallel( break if interrupted: - for future, idx in futures.items(): - if results[idx] is not None: - continue - if future.cancel(): - # Never dispatched — nothing was billed. - results[idx] = ( - _slot_label(reference_models[idx]), - _INTERRUPTED_REFERENCE_NOTE, - _RefAccounting(CanonicalUsage()), - ) - elif future.done(): - # Finished between the interrupt check and now: keep its real output/accounting. - results[idx] = future.result() - else: - # Already running — cannot be killed; the provider WILL bill when it - # completes, so hand its eventual accounting to the caller's sink. - label = _slot_label(reference_models[idx]) - results[idx] = ( - label, - _INTERRUPTED_REFERENCE_NOTE, - _RefAccounting(CanonicalUsage()), - ) - if late_accounting_sink is not None: - def _record_late(f: Any, _label: str = label) -> None: - try: - _lbl, _txt, _acct = f.result() - except Exception: # pragma: no cover - defensive - return - try: - late_accounting_sink(_label, _acct) - except Exception: # pragma: no cover - defensive - logger.debug( - "MoA: late accounting sink failed for %s", - _label, - ) - future.add_done_callback(_record_late) + _settle_interrupted(futures, results, reference_models, late_accounting_sink) finally: executor.shutdown(wait=not interrupted, cancel_futures=interrupted) @@ -788,8 +766,6 @@ def _reference_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: # yield a byte-identical view (stable advisory prefix for advisor caching). text = flatten_message_text(content) - if role == "system": - continue if role == "user": if not text.strip() and isinstance(content, list) and content: # Structured content with no text (e.g. image-only): an empty user @@ -802,9 +778,7 @@ def _reference_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: last_user_content = text rendered.append({"role": "user", "content": text}) elif role == "assistant": - parts: list[str] = [] - if text.strip(): - parts.append(text.strip()) + parts = [text.strip()] if text.strip() else [] calls_text = _render_tool_calls(msg.get("tool_calls")) if calls_text: parts.append(calls_text) @@ -813,14 +787,13 @@ def _reference_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: rendered.append({"role": "assistant", "content": "\n".join(parts)}) elif role == "tool": # Fold the tool result into the preceding assistant turn as text. - result_text = _truncate_tool_result(text) - block = f"[tool result: {result_text}]" + block = f"[tool result: {_truncate_tool_result(text)}]" if rendered and rendered[-1].get("role") == "assistant": rendered[-1]["content"] = rendered[-1]["content"] + "\n" + block else: # No assistant turn to attach to (e.g. a leading tool result). rendered.append({"role": "assistant", "content": block}) - # Any other role is ignored. + # system and any other role are ignored. # End on a user turn by appending a synthetic advisory request (Anthropic # rejects trailing assistant prefill); an existing trailing user turn is left as is. @@ -839,14 +812,13 @@ def _reference_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: return rendered - def _extract_text(response: Any) -> str: + """Assistant text of a completed response: transport-normalized, else ``choices[0]``.""" try: transport = get_transport("chat_completions") if transport is None: raise RuntimeError("chat_completions transport unavailable") - normalized = transport.normalize_response(response) - text = (normalized.content or "").strip() + text = (transport.normalize_response(response).content or "").strip() if text: return text except Exception: @@ -862,10 +834,7 @@ def _extract_text(response: Any) -> str: def _preset_temperature(preset: dict[str, Any], key: str) -> float | None: - """Read an optional preset temperature; None (absent/empty/null) = provider default. - - A hardcoded fallback previously forced 0.6/0.4 even when unset. - """ + """Read an optional preset temperature; None (absent/empty/null) = provider default.""" value = preset.get(key) if value is None or (isinstance(value, str) and not value.strip()): return None @@ -884,21 +853,16 @@ def _hash_messages(msgs: list[dict[str, Any]]) -> str: def _is_failed_reference(text: str) -> bool: """Whether a reference output is a ``[failed: …]`` / ``[skipped: …]`` sentinel.""" - sentinel = text.lstrip().lower() - return sentinel.startswith("[failed:") or sentinel.startswith("[skipped:") + return text.lstrip().lower().startswith(("[failed:", "[skipped:")) -def _successful_references( +def _split_references( reference_outputs: list[tuple[str, str, Any]], -) -> list[tuple[str, str, Any]]: - """Filter failed advice while preserving each accounting payload.""" - return [output for output in reference_outputs if not _is_failed_reference(output[1])] - - -def _failed_reference_labels( - reference_outputs: list[tuple[str, str, Any]], -) -> list[str]: - return [label for label, text, _accounting in reference_outputs if _is_failed_reference(text)] +) -> tuple[list[tuple[str, str, Any]], list[str]]: + """``(successful outputs, failed labels)``; accounting payloads are preserved.""" + successful = [o for o in reference_outputs if not _is_failed_reference(o[1])] + failed = [label for label, text, _acct in reference_outputs if _is_failed_reference(text)] + return successful, failed def _join_reference_outputs(outputs: list[tuple[str, str, Any]], degraded: str = "") -> str: @@ -931,6 +895,10 @@ def _degraded_notice(failed_labels: list[str], policy: str) -> str: return f"[Reference models unavailable: {', '.join(failed_labels)}]" +def _slot_labels(slots: list[dict[str, Any]]) -> str: + return ", ".join(_slot_label(slot) for slot in slots) + + def aggregate_moa_context( *, user_prompt: str, @@ -948,23 +916,19 @@ def aggregate_moa_context( Failures become model-specific notes instead of aborting the loop. ``reference_max_tokens`` caps ONLY the fan-out — capping the aggregator - truncated long syntheses (#53580). ``temperature`` / ``aggregator_temperature`` + truncated long syntheses. ``temperature`` / ``aggregator_temperature`` default to None (provider default). ``agent`` makes the fan-out interruptible. """ reference_models = [slot for slot in reference_models if slot.get("enabled", True)] - reference_outputs: list[tuple[str, str, Any]] = [] - ref_messages = _reference_messages(api_messages) reference_outputs = _run_references_parallel( reference_models, - ref_messages, + _reference_messages(api_messages), temperature=temperature, max_tokens=reference_max_tokens, reference_timeout=reference_timeout, agent=agent, ) - - successful_outputs = _successful_references(reference_outputs) - failed_labels = _failed_reference_labels(reference_outputs) + successful_outputs, failed_labels = _split_references(reference_outputs) # 'full' privacy mode also redacts advisor text before it reaches this synthesizer # ('display' has no surface here). Failed refs are already filtered out. @@ -986,12 +950,11 @@ def aggregate_moa_context( "MoA: all %d reference(s) failed — skipping aggregator synthesis", len(reference_outputs), ) - notice = degraded or "[Reference models unavailable]" return ( "[Mixture of Agents context — all reference models failed. " "Proceeding without aggregated guidance.]\n" - f"References: {', '.join(_slot_label(slot) for slot in reference_models)}\n\n" - f"{notice}" + f"References: {_slot_labels(reference_models)}\n\n" + f"{degraded or '[Reference models unavailable]'}" ) synth_prompt = ( @@ -1006,25 +969,14 @@ def aggregate_moa_context( agg_label = _slot_label(aggregator) agg_runtime = _slot_runtime(aggregator) - # Pin the live agent disable onto synthesis decoration (#76085); None is a no-op. - agg_cache_runtime = agg_runtime - _agg_cache_disabled = ( - getattr(agent, "_cache_disabled", None) if agent is not None else None - ) - if _agg_cache_disabled is not None: - agg_cache_runtime = { - **agg_runtime, - "_cache_disabled": _agg_cache_disabled, - } - # Agent's cache TTL for the one-shot synthesis path (#84733). - _agg_cache_ttl = getattr(agent, "_cache_ttl", None) if agent is not None else None + cache_disabled, cache_ttl = _agent_cache_opts(agent) try: # Same cache_control decoration as the advisor calls; this synthesis call is # a third independent MoA call path that otherwise re-bills its full input. agg_messages = _maybe_apply_moa_cache_control( [{"role": "user", "content": synth_prompt}], - agg_cache_runtime, - cache_ttl=_agg_cache_ttl, + _with_cache_disabled(agg_runtime, cache_disabled), + cache_ttl=cache_ttl, ) response = call_llm( task="moa_aggregator", @@ -1038,16 +990,13 @@ def aggregate_moa_context( logger.warning("MoA aggregator model %s failed: %s", agg_label, exc) synthesis = "" - if not synthesis: - synthesis = joined - return ( "[Mixture of Agents context — use this as private guidance for the " "normal Hermes agent loop. You may call tools, continue reasoning, or " "finish normally.]\n" f"Aggregator: {agg_label}\n" - f"References: {', '.join(_slot_label(slot) for slot in reference_models)}\n\n" - f"{synthesis.strip()}" + f"References: {_slot_labels(reference_models)}\n\n" + f"{(synthesis or joined).strip()}" ) @@ -1056,25 +1005,24 @@ def _completed_response_as_stream_chunk(response: Any) -> Any: Done at the MoA facade boundary so transports stay untouched. """ - choices = getattr(response, "choices", None) first_choice = choices[0] if isinstance(choices, (list, tuple)) and choices else None message = getattr(first_choice, "message", None) raw_tool_calls = getattr(message, "tool_calls", None) tool_call_deltas = None if isinstance(raw_tool_calls, (list, tuple)) and raw_tool_calls: - tool_call_deltas = [] - for index, tc in enumerate(raw_tool_calls): - function = getattr(tc, "function", None) - tool_call_deltas.append(SimpleNamespace( + tool_call_deltas = [ + SimpleNamespace( index=getattr(tc, "index", index), id=getattr(tc, "id", None), type=getattr(tc, "type", None) or "function", function=SimpleNamespace( - name=getattr(function, "name", None), - arguments=getattr(function, "arguments", None), + name=getattr(getattr(tc, "function", None), "name", None), + arguments=getattr(getattr(tc, "function", None), "arguments", None), ), - )) + ) + for index, tc in enumerate(raw_tool_calls) + ] delta = SimpleNamespace( content=getattr(message, "content", None), tool_calls=tool_call_deltas, @@ -1124,7 +1072,7 @@ def peel_reference_guidance( """Exact inverse of ``_attach_reference_guidance`` (the three attach shapes). Used by the failover redecoration chokepoint so a cache breakpoint never lands - on the turn-varying guidance (#72626). Returns a new list; inputs are not mutated. + on the turn-varying guidance. Returns a new list; inputs are not mutated. """ if not guidance or not messages: return messages @@ -1139,42 +1087,37 @@ def peel_reference_guidance( suffix = "\n\n" + guidance_text if isinstance(content, str) and content.endswith(suffix): # Attach shape (a): merged into a trailing string user turn. - peeled = dict(last) - peeled["content"] = content[: -len(suffix)] - return [*messages[:-1], peeled] + return [*messages[:-1], {**last, "content": content[: -len(suffix)]}] if isinstance(content, list) and content: last_part = content[-1] if isinstance(last_part, dict) and last_part.get("type", "text") == "text": text = last_part.get("text") or "" if text == suffix or text == guidance_text: - # Attach shape (b): guidance rode as its own trailing part. - peeled = dict(last) - peeled["content"] = list(content[:-1]) - if not peeled["content"]: - # Guidance was the only content: drop the whole message (mirrors shape c). + # Attach shape (b): guidance rode as its own trailing part. Guidance as + # the only content drops the whole message (mirrors shape c). + if len(content) == 1: return list(messages[:-1]) - return [*messages[:-1], peeled] + return [*messages[:-1], {**last, "content": list(content[:-1])}] if text.endswith(suffix): - new_part = dict(last_part) - new_part["text"] = text[: -len(suffix)] - peeled = dict(last) - peeled["content"] = [*content[:-1], new_part] - return [*messages[:-1], peeled] + new_part = {**last_part, "text": text[: -len(suffix)]} + return [*messages[:-1], {**last, "content": [*content[:-1], new_part]}] return messages class MoAChatCompletions: - """OpenAI-chat-compatible facade where the aggregator is the acting model.""" + """OpenAI-chat-compatible facade where the aggregator is the acting model. + + ``reference_callback(event, **kwargs)`` is an optional best-effort display hook: + "moa.reference" index, count, label, text + "moa.progress" refs_done, refs_total, label (per reference completion) + "moa.phase" phase, refs_done, refs_total, aggregator + "moa.aggregating" aggregator (label), ref_count + ``agent`` is the owning AIAgent; it lets the fan-out check ``_interrupt_requested``. + """ def __init__(self, preset_name: str, reference_callback: Any = None, agent: Any = None): self.preset_name = preset_name or "default" - # Optional best-effort display hook, called as ``reference_callback(event, **kwargs)``: - # "moa.reference" index, count, label, text - # "moa.progress" refs_done, refs_total, label (per reference completion) - # "moa.phase" phase, refs_done, refs_total, aggregator - # "moa.aggregating" aggregator (label), ref_count self.reference_callback = reference_callback - # Owning AIAgent (optional); lets the fan-out check agent._interrupt_requested. self._agent = agent # State-scoped reference cache keyed on the advisory-view signature: a new # user/tool message is a MISS (references re-run), a redundant create() with @@ -1183,9 +1126,9 @@ class MoAChatCompletions: self._ref_cache_outputs: list[tuple[str, str, Any]] = [] # Fan-out usage/cost from the latest cache-MISS create(), awaiting # consume_reference_usage (zero deposited on a HIT so spend counts once). + # The lock guards them against late-accounting callbacks on worker threads. self._pending_reference_usage: Any = CanonicalUsage() self._pending_reference_cost: Any = None - # Guards pending usage/cost against late-accounting callbacks on worker threads. self._accounting_lock = threading.Lock() # Resolved aggregator slot from the latest create(); cost accounting prices the # acting turn at its real model instead of the virtual preset name. @@ -1227,16 +1170,12 @@ class MoAChatCompletions: if not isinstance(accounting, _RefAccounting): return self._fold_pending_accounting(*_sum_reference_accounting([(label, "", accounting)])) - logger.debug( - "MoA: recorded late accounting for interrupted reference %s", label - ) + logger.debug("MoA: recorded late accounting for interrupted reference %s", label) def _fold_pending_accounting(self, usage: Any, cost: Any) -> None: """Add (never overwrite) fan-out spend so late interrupted-reference deposits survive.""" with self._accounting_lock: - self._pending_reference_usage = ( - self._pending_reference_usage or CanonicalUsage() - ) + usage + self._pending_reference_usage = (self._pending_reference_usage or CanonicalUsage()) + usage if cost is not None: self._pending_reference_cost = (self._pending_reference_cost or 0) + cost @@ -1303,41 +1242,30 @@ class MoAChatCompletions: _attach_reference_guidance(agg_messages, str(guidance)) return {**prepared, "messages": agg_messages} - def _call_prepared_aggregator( - self, prepared: dict[str, Any], api_kwargs: dict[str, Any] - ) -> Any: - """Send an already prepared MoA aggregator request exactly once.""" - agg_messages = prepared["messages"] - aggregator = prepared["aggregator"] - aggregator_temperature = prepared["aggregator_temperature"] - if aggregator.get("provider") == "moa": - raise RuntimeError("MoA aggregator cannot be another MoA preset") - agg_kwargs = dict(api_kwargs) - max_tokens: Any = agg_kwargs.get("max_tokens") - tools: Any = agg_kwargs.get("tools") - extra_body: Any = agg_kwargs.get("extra_body") - agg_runtime = _slot_runtime(aggregator) - try: - from agent.agent_runtime_helpers import ( - plan_cache_sections_for_destination, - ) + def _plan_aggregator_cache( + self, + agg_messages: list[dict[str, Any]], + tools: Any, + guidance: Any, + agg_runtime: dict[str, Any], + ) -> tuple[list[dict[str, Any]], Any]: + """Decorate the aggregator request with cache breakpoints for its destination. + + The guidance is peeled before planning and re-attached after so a breakpoint + never lands on the turn-varying block. On any error the undecorated request + is returned (warning, not debug: this is the aggregator's ONLY decoration path). + """ + try: + from agent.agent_runtime_helpers import plan_cache_sections_for_destination - guidance = prepared.get("guidance") planning_messages = agg_messages if guidance: - planning_messages = peel_reference_guidance( - agg_messages, - str(guidance), - ) + planning_messages = peel_reference_guidance(agg_messages, str(guidance)) # plan_cache_sections_for_destination returns request-local copies. # Tri-state cache_disabled: facades built via __new__ have no _agent; forcing - # False would suppress the planner's config fallback (#76085). + # False would suppress the planner's config fallback. _agent = getattr(self, "_agent", None) - _cache_disabled = ( - getattr(_agent, "_cache_disabled", None) - if _agent is not None - else None - ) + cache_disabled, cache_ttl = _agent_cache_opts(_agent) agg_messages, tools = plan_cache_sections_for_destination( planning_messages, tools, @@ -1345,21 +1273,31 @@ class MoAChatCompletions: base_url=agg_runtime.get("base_url") or "", api_mode=agg_runtime.get("api_mode") or "", model=agg_runtime.get("model") or "", - cache_disabled=_cache_disabled, - # Agent TTL + stable system prefix so MoA stops regressing 1h → 5m (#84733). - cache_ttl=getattr(_agent, "_cache_ttl", None), - static_system_prefix=getattr( - _agent, "_cached_system_prompt_static", None - ), + cache_disabled=cache_disabled, + # Agent TTL + stable system prefix so MoA does not regress 1h → 5m. + cache_ttl=cache_ttl, + static_system_prefix=getattr(_agent, "_cached_system_prompt_static", None), ) if guidance: _attach_reference_guidance(agg_messages, str(guidance)) except Exception as exc: # pragma: no cover - cache planning must not block MoA - # Warning, not debug: this is the aggregator's ONLY decoration path. logger.warning( "MoA aggregator cache plan failed — sending undecorated " "request (cache misses expected): %s", exc, ) + return agg_messages, tools + + def _call_prepared_aggregator( + self, prepared: dict[str, Any], api_kwargs: dict[str, Any] + ) -> Any: + """Send an already prepared MoA aggregator request exactly once.""" + aggregator = prepared["aggregator"] + if aggregator.get("provider") == "moa": + raise RuntimeError("MoA aggregator cannot be another MoA preset") + agg_runtime = _slot_runtime(aggregator) + agg_messages, tools = self._plan_aggregator_cache( + prepared["messages"], api_kwargs.get("tools"), prepared.get("guidance"), agg_runtime + ) # Record the exact aggregator INPUT into the pending trace; the persisted COPY # is redacted under any privacy mode while the live input stays raw. if self._pending_trace is not None: @@ -1377,26 +1315,23 @@ class MoAChatCompletions: stream_kwargs: dict[str, Any] = {} if stream: stream_kwargs["stream"] = True - stream_kwargs["stream_options"] = ( - api_kwargs.get("stream_options") or {"include_usage": True} - ) + stream_kwargs["stream_options"] = api_kwargs.get("stream_options") or {"include_usage": True} # The consumer's stream-read timeout must govern the aggregator stream. if api_kwargs.get("timeout") is not None: stream_kwargs["timeout"] = api_kwargs["timeout"] # Pop the runtime's extra_body and merge with the caller's (caller wins) so the # explicit kwarg never collides with **agg_runtime. agg_extra_body = _merge_slot_extra_body( - agg_runtime.pop("extra_body", None), - extra_body, + agg_runtime.pop("extra_body", None), api_kwargs.get("extra_body"), ) - _agg_response = call_llm( + agg_response = call_llm( task="moa_aggregator", messages=agg_messages, - temperature=aggregator_temperature, - max_tokens=max_tokens, + temperature=prepared["aggregator_temperature"], + max_tokens=api_kwargs.get("max_tokens"), tools=tools, extra_body=agg_extra_body, - # Same reasoning policy as the direct create() path (#64187). + # Same reasoning policy as the direct create() path. reasoning_config=_aggregator_reasoning_config(aggregator), **stream_kwargs, **agg_runtime, @@ -1404,20 +1339,19 @@ class MoAChatCompletions: # Non-streaming: capture the aggregator output inline. Streaming: the output # lands as the turn's assistant message; the trace marks it streamed. if self._pending_trace is not None: - if stream: - self._pending_trace["aggregator_streamed"] = True - self._pending_trace["aggregator_output"] = None - else: - self._pending_trace["aggregator_streamed"] = False + self._pending_trace["aggregator_streamed"] = stream + output = None + if not stream: try: - self._pending_trace["aggregator_output"] = _extract_text(_agg_response) + output = _extract_text(agg_response) except Exception: # pragma: no cover - defensive - self._pending_trace["aggregator_output"] = None - if stream and hasattr(_agg_response, "choices"): + output = None + self._pending_trace["aggregator_output"] = output + if stream and hasattr(agg_response, "choices"): # Some adapters (openai-codex Responses) return a completed response even - # when streaming was requested; hand the loop a one-chunk iterator (#55933). - return iter((_completed_response_as_stream_chunk(_agg_response),)) - return _agg_response + # when streaming was requested; hand the loop a one-chunk iterator. + return iter((_completed_response_as_stream_chunk(agg_response),)) + return agg_response def _fanout_cache_key( self, @@ -1427,12 +1361,12 @@ class MoAChatCompletions: ) -> tuple: """Compute the turn-scoped reference cache key per the preset's fan-out cadence. - Cadence (#67199, #63393). "user_turn" (default): advisors run once per user - turn — the signature hashes only the prefix up to the LAST USER message, so later - tool iterations are cache HITs. "per_iteration": re-run whenever the advisory - view changes. "every_n:": iteration 1 of a turn, then every Nth; in-between - iterations reuse the last on-cadence guidance (key pinned to that run so the - lookup is a HIT: no advisor calls, no double accounting, no re-emit). + "user_turn" (default): advisors run once per user turn — the signature hashes + only the prefix up to the LAST USER message, so later tool iterations are + cache HITs. "per_iteration": re-run whenever the advisory view changes. + "every_n:": iteration 1 of a turn, then every Nth; in-between iterations + reuse the last on-cadence guidance (key pinned to that run so the lookup is a + HIT: no advisor calls, no double accounting, no re-emit). """ fanout_mode = str(preset.get("fanout") or "user_turn").strip().lower() every_n = 0 @@ -1449,10 +1383,10 @@ class MoAChatCompletions: if fanout_mode == "user_turn" or every_n >= 2: # Last REAL user message: the synthetic _ADVISORY_INSTRUCTION marker must not # count or the prefix would grow (and re-sign) every iteration. - for _i in range(len(ref_messages) - 1, -1, -1): - _m = ref_messages[_i] - if _m.get("role") == "user" and _m.get("content") != _ADVISORY_INSTRUCTION: - turn_prefix = ref_messages[: _i + 1] + for i in range(len(ref_messages) - 1, -1, -1): + m = ref_messages[i] + if m.get("role") == "user" and m.get("content") != _ADVISORY_INSTRUCTION: + turn_prefix = ref_messages[: i + 1] break if fanout_mode == "user_turn": sig_messages = turn_prefix @@ -1460,18 +1394,18 @@ class MoAChatCompletions: # every_n bookkeeping: advance the counter only when the advisory STATE changed # (a streaming retry must not consume a cadence slot); reset on a new turn prefix. if every_n >= 2: - _turn_sig = _hash_messages(turn_prefix) - if _turn_sig != self._fanout_turn_sig: - self._fanout_turn_sig = _turn_sig + turn_sig = _hash_messages(turn_prefix) + if turn_sig != self._fanout_turn_sig: + self._fanout_turn_sig = turn_sig self._fanout_iteration_count = 0 self._fanout_last_state_sig = None - _state_sig = _hash_messages(ref_messages) - if _state_sig != self._fanout_last_state_sig: - self._fanout_last_state_sig = _state_sig + state_sig = _hash_messages(ref_messages) + if state_sig != self._fanout_last_state_sig: + self._fanout_last_state_sig = state_sig self._fanout_iteration_count += 1 # Iteration 1 is on-cadence; then every Nth iteration after it. - _on_cadence = (self._fanout_iteration_count - 1) % every_n == 0 - if not _on_cadence and self._ref_cache_outputs: + on_cadence = (self._fanout_iteration_count - 1) % every_n == 0 + if not on_cadence and self._ref_cache_outputs: return self._ref_cache_key return ( @@ -1480,154 +1414,109 @@ class MoAChatCompletions: tuple(_slot_label(s) for s in reference_models), ) - def create(self, **api_kwargs: Any) -> Any: - prepared_request = api_kwargs.pop("_moa_prepared_request", None) - if prepared_request is not None: - if not isinstance(prepared_request, dict): - raise TypeError("_moa_prepared_request must be a dict") - return self._call_prepared_aggregator(prepared_request, api_kwargs) - - preset, _moa_raw = _resolve_preset_cached(self.preset_name) - # Remembered on self so _call_prepared_aggregator redacts the trace consistently. - privacy_mode = _moa_privacy_mode(_moa_raw) - self._privacy_mode = privacy_mode - messages = list(api_kwargs.get("messages") or []) - reference_models = [ - slot for slot in (preset.get("reference_models") or []) - if slot.get("enabled", True) - ] - aggregator = preset.get("aggregator") or {} - # The MoA path's virtual model/provider have no pricing entry; expose the real slot. - self.last_aggregator_slot = dict(aggregator) if aggregator else None + def _run_fanout( + self, + preset: dict[str, Any], + ref_messages: list[dict[str, Any]], + reference_models: list[dict[str, Any]], + aggregator: dict[str, Any], + aggregator_temperature: Any, + cache_key: tuple, + ) -> list[tuple[str, str, Any]]: + """Cache-MISS path of ``create``: run the advisors, account, trace and emit.""" # No output caps by default (None → call_llm omits max_tokens). A preset MAY cap # ADVISOR output (dominant MoA latency); the acting aggregator is never capped. - reference_max_tokens = preset.get("reference_max_tokens") - # None = provider default (see _preset_temperature). - temperature = _preset_temperature(preset, "reference_temperature") - aggregator_temperature = _preset_temperature(preset, "aggregator_temperature") - # None = inherit auxiliary.moa_reference.timeout via call_llm. + # None reference_timeout = inherit auxiliary.moa_reference.timeout via call_llm. raw_reference_timeout = preset.get("reference_timeout") - reference_timeout = ( - float(raw_reference_timeout) if raw_reference_timeout else None + + def _progress(done: int, total: int, label: str) -> None: + self._emit("moa.progress", refs_done=done, refs_total=total, label=label) + + reference_outputs = _run_references_parallel( + reference_models, + ref_messages, + temperature=_preset_temperature(preset, "reference_temperature"), + max_tokens=preset.get("reference_max_tokens"), + progress_callback=_progress, + reference_timeout=float(raw_reference_timeout) if raw_reference_timeout else None, + agent=self._agent, + late_accounting_sink=self._record_late_reference_accounting, ) - degraded_reference_policy = str( - preset.get("degraded_reference_policy") or "loud" - ) - if aggregator_temperature is None and api_kwargs.get("temperature") is not None: - # The acting agent's own temperature applies to the aggregator (the acting model). - aggregator_temperature = api_kwargs.get("temperature") - - # A disabled preset = "use the aggregator directly". - if not preset.get("enabled", True): - reference_models = [] - - reference_outputs: list[tuple[str, str, Any]] = [] - ref_messages = _reference_messages(messages) - _cache_key = self._fanout_cache_key(preset, ref_messages, reference_models) - _refs_from_cache = _cache_key == self._ref_cache_key and bool(self._ref_cache_outputs) - - if _refs_from_cache: - reference_outputs = list(self._ref_cache_outputs) - # Cache HIT: references already ran and were accounted this turn. Deposit - # nothing, but do NOT zero pending totals (a late interrupted reference may - # have deposited real spend). No trace either — a repeat iteration is not a turn. - self._pending_trace = None + if any(text == _INTERRUPTED_REFERENCE_NOTE for _lbl, text, _acct in reference_outputs): + # An interrupted fan-out is a partial snapshot: never cache it (a HIT + # would replay placeholder notes every iteration). + self._ref_cache_key = None + self._ref_cache_outputs = [] else: - # Per-reference progress (``moa.progress``) through the same display hook. - def _progress(done: int, total: int, label: str) -> None: - self._emit( - "moa.progress", - refs_done=done, - refs_total=total, - label=label, - ) + self._ref_cache_key = cache_key + self._ref_cache_outputs = list(reference_outputs) + # Fold advisor spend into accounting exactly once per turn. + self._fold_pending_accounting(*_sum_reference_accounting(reference_outputs)) + # Stash the fan-out for trace persistence (aggregator input/label filled in + # later; output stitched in by consume_and_save_trace). Traces are persisted, + # so ANY active privacy mode redacts advisor text and per-advisor input/output. + privacy_mode = self._privacy_mode + if privacy_mode: + trace_refs = [ + (label, _redact_reference_text(text), _redact_trace_accounting(acct)) + for label, text, acct in reference_outputs + ] + else: + trace_refs = list(reference_outputs) + self._pending_trace = { + "preset": self.preset_name, + "reference_outputs": trace_refs, + "aggregator_slot": aggregator, + "aggregator_temperature": aggregator_temperature, + } + # Derived from the privacy-redacted trace_refs. + try: + from agent.moa_trace import slot_metrics - reference_outputs = _run_references_parallel( - reference_models, - ref_messages, - temperature=temperature, - max_tokens=reference_max_tokens, - progress_callback=_progress, - reference_timeout=reference_timeout, - agent=self._agent, - late_accounting_sink=self._record_late_reference_accounting, + self._last_reference_metrics = [ + slot_metrics(acct, label, output=text) for label, text, acct in trace_refs + ] + except Exception as exc: # pragma: no cover - never break a turn + logger.debug("MoA reference metrics render failed: %s", exc) + self._last_reference_metrics = None + + # Surface each reference's answer BEFORE the aggregator acts (once per turn). + # The cache keeps RAW text; redaction happens at each consuming surface. + ref_count = len(reference_outputs) + for idx, (label, text, _accounting) in enumerate(reference_outputs, start=1): + self._emit( + "moa.reference", + index=idx, + count=ref_count, + label=label, + text=_redact_reference_text(text) if privacy_mode else text, ) - interrupted_any = any( - text == _INTERRUPTED_REFERENCE_NOTE - for _lbl, text, _acct in reference_outputs + if ref_count: + # Phase transition: fan-out complete, aggregator about to act. + agg_label = _slot_label(aggregator) + self._emit( + "moa.phase", + phase="aggregator", + refs_done=ref_count, + refs_total=ref_count, + aggregator=agg_label, ) - if interrupted_any: - # An interrupted fan-out is a partial snapshot: never cache it (a HIT - # would replay placeholder notes every iteration). - self._ref_cache_key = None - self._ref_cache_outputs = [] - else: - self._ref_cache_key = _cache_key - self._ref_cache_outputs = list(reference_outputs) - # Fold advisor spend into accounting exactly once per turn. - self._fold_pending_accounting(*_sum_reference_accounting(reference_outputs)) - # Stash the fan-out for trace persistence (aggregator input/label filled in - # later; output stitched in by consume_and_save_trace). Traces are persisted, - # so ANY active privacy mode redacts advisor text and per-advisor input/output. - if privacy_mode: - _trace_refs = [ - (label, _redact_reference_text(text), _redact_trace_accounting(acct)) - for label, text, acct in reference_outputs - ] - else: - _trace_refs = list(reference_outputs) - self._pending_trace = { - "preset": self.preset_name, - "reference_outputs": _trace_refs, - "aggregator_slot": aggregator, - "aggregator_temperature": aggregator_temperature, - } - # Derived from the privacy-redacted _trace_refs. - try: - from agent.moa_trace import slot_metrics + self._emit("moa.aggregating", aggregator=agg_label, ref_count=ref_count) + return reference_outputs - self._last_reference_metrics = [ - slot_metrics(acct, label, output=text) - for label, text, acct in _trace_refs - ] - except Exception as exc: # pragma: no cover - never break a turn - logger.debug("MoA reference metrics render failed: %s", exc) - self._last_reference_metrics = None - - # Surface each reference's answer BEFORE the aggregator acts (once per turn). - # The cache keeps RAW text; redaction happens at each consuming surface. - _ref_count = len(reference_outputs) - for _idx, (_label, _text, _accounting) in enumerate(reference_outputs, start=1): - self._emit( - "moa.reference", - index=_idx, - count=_ref_count, - label=_label, - text=_redact_reference_text(_text) if privacy_mode else _text, - ) - if _ref_count: - # Phase transition: fan-out complete, aggregator about to act. - self._emit( - "moa.phase", - phase="aggregator", - refs_done=_ref_count, - refs_total=_ref_count, - aggregator=_slot_label(aggregator), - ) - self._emit( - "moa.aggregating", - aggregator=_slot_label(aggregator), - ref_count=_ref_count, - ) - - agg_messages = [dict(m) for m in messages] - successful_outputs = _successful_references(reference_outputs) - failed_labels = _failed_reference_labels(reference_outputs) - # 'full' privacy mode redacts advisor text reaching the AGGREGATOR too (#59959); - # 'display' leaves it raw. Applied to a per-call copy — the cache holds raw text. - _agg_refs = ( + def _build_guidance( + self, + reference_outputs: list[tuple[str, str, Any]], + aggregator: dict[str, Any], + degraded_reference_policy: str, + ) -> str | None: + """Render the reference block attached to the aggregator prompt (None = nothing).""" + successful_outputs, failed_labels = _split_references(reference_outputs) + # 'full' privacy mode redacts advisor text reaching the AGGREGATOR too; 'display' + # leaves it raw. Applied to a per-call copy — the cache holds raw text. + agg_refs = ( _redact_reference_outputs(successful_outputs) - if privacy_mode == "full" + if self._privacy_mode == "full" else successful_outputs ) degraded = _degraded_notice(failed_labels, degraded_reference_policy) @@ -1636,7 +1525,6 @@ class MoAChatCompletions: f"Preset: {self.preset_name}\n" f"Aggregator/acting model: {_slot_label(aggregator)}\n" ) - guidance: str | None = None if reference_outputs and not successful_outputs: # Every reference failed: the aggregator acts alone. Under the loud policy it # still gets the sanitized unavailability notice; under silent, nothing. @@ -1646,20 +1534,65 @@ class MoAChatCompletions: len(reference_outputs), ) if degraded: - guidance = ( + return ( f"{header}\n" "All reference models failed this turn — no advisory " "guidance is available. Act on your own judgment.\n\n" f"{degraded}" ) - elif _agg_refs or degraded: - guidance = ( + return None + if agg_refs or degraded: + return ( f"{header}" - f"References: {', '.join(label for label, _, _ in _agg_refs)}\n\n" + f"References: {', '.join(label for label, _, _ in agg_refs)}\n\n" "Use the reference responses below as private context. You are the aggregator and acting model: " "answer the user directly or call tools as needed.\n\n" - f"{_join_reference_outputs(_agg_refs, degraded)}" + f"{_join_reference_outputs(agg_refs, degraded)}" ) + return None + + def create(self, **api_kwargs: Any) -> Any: + prepared_request = api_kwargs.pop("_moa_prepared_request", None) + if prepared_request is not None: + if not isinstance(prepared_request, dict): + raise TypeError("_moa_prepared_request must be a dict") + return self._call_prepared_aggregator(prepared_request, api_kwargs) + + preset, moa_raw = _resolve_preset_cached(self.preset_name) + # Remembered on self so _call_prepared_aggregator redacts the trace consistently. + self._privacy_mode = _moa_privacy_mode(moa_raw) + messages = list(api_kwargs.get("messages") or []) + # A disabled preset = "use the aggregator directly". + reference_models = [ + slot for slot in (preset.get("reference_models") or []) + if slot.get("enabled", True) + ] if preset.get("enabled", True) else [] + aggregator = preset.get("aggregator") or {} + # The MoA path's virtual model/provider have no pricing entry; expose the real slot. + self.last_aggregator_slot = dict(aggregator) if aggregator else None + # None = provider default (see _preset_temperature); the acting agent's own + # temperature applies to the aggregator (the acting model). + aggregator_temperature = _preset_temperature(preset, "aggregator_temperature") + if aggregator_temperature is None and api_kwargs.get("temperature") is not None: + aggregator_temperature = api_kwargs.get("temperature") + + ref_messages = _reference_messages(messages) + cache_key = self._fanout_cache_key(preset, ref_messages, reference_models) + if cache_key == self._ref_cache_key and self._ref_cache_outputs: + # Cache HIT: references already ran and were accounted this turn. Deposit + # nothing, but do NOT zero pending totals (a late interrupted reference may + # have deposited real spend). No trace either — a repeat iteration is not a turn. + reference_outputs = list(self._ref_cache_outputs) + self._pending_trace = None + else: + reference_outputs = self._run_fanout( + preset, ref_messages, reference_models, aggregator, aggregator_temperature, cache_key + ) + + agg_messages = [dict(m) for m in messages] + guidance = self._build_guidance( + reference_outputs, aggregator, str(preset.get("degraded_reference_policy") or "loud") + ) if guidance: _attach_reference_guidance(agg_messages, guidance) @@ -1724,8 +1657,8 @@ def build_moa_facade(agent, preset_name: Any = None) -> MoAClient: Single construction point for ``MoAClient`` (agent_init, fallback restore, transport recovery, switch_model): a bare ``MoAClient(preset)`` would drop the - ``reference_callback`` relay and silence display events for the session - (#53802). The relay reads ``agent.tool_progress_callback`` at emit time. + ``reference_callback`` relay and silence display events for the session. + The relay reads ``agent.tool_progress_callback`` at emit time. """ def _moa_reference_relay(event: str, **kwargs: Any) -> None: cb = getattr(agent, "tool_progress_callback", None) @@ -1760,9 +1693,5 @@ def build_moa_facade(agent, preset_name: Any = None) -> MoAClient: except Exception: resolved_preset = "default" - return MoAClient( - resolved_preset, - reference_callback=_moa_reference_relay, - # Lets the fan-out wait be aborted on a user interrupt. - agent=agent, - ) + # ``agent`` lets the fan-out wait be aborted on a user interrupt. + return MoAClient(resolved_preset, reference_callback=_moa_reference_relay, agent=agent) From 23ae37b4bcbb02aa630e0d1f87c2f93de60c9a56 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:33:06 -0700 Subject: [PATCH 2/9] =?UTF-8?q?refactor(agent):=20plugin=5Fllm/nous=5Frate?= =?UTF-8?q?=5Fguard/moonshot=5Fschema/provider=5F*=20=E2=80=94=20unify=20a?= =?UTF-8?q?tomic=20write,=20compact=20docstrings,=20collapse=20defensive?= =?UTF-8?q?=20layers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/moa_loop.py | 323 +++++++++++++---------------------- agent/moa_trace.py | 17 +- agent/moonshot_schema.py | 76 ++++----- agent/nous_rate_guard.py | 52 ++---- agent/plugin_llm.py | 232 +++++++++---------------- agent/provider_base.py | 35 ++-- agent/provider_media.py | 35 ++-- agent/provider_projection.py | 30 +--- agent/provider_registry.py | 60 +++---- 9 files changed, 303 insertions(+), 557 deletions(-) diff --git a/agent/moa_loop.py b/agent/moa_loop.py index 631e726cf7..4a980fb604 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -25,10 +25,9 @@ from agent.usage_pricing import CanonicalUsage logger = logging.getLogger(__name__) -# Privacy filter (moa.privacy_filter: '' | display | full). Secret shapes are handled -# by agent.redact; these add the PII classes it leaves alone (emails, formatted NA -# phones). The phone pattern requires explicit delimiters so line numbers, dates, -# times, SHAs, IPs and versions never match. +# Privacy filter (moa.privacy_filter: '' | display | full): PII classes agent.redact +# leaves alone. The phone pattern requires explicit delimiters so line numbers, +# dates, times, SHAs, IPs and versions never match. _MOA_EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b") _MOA_PHONE_RE = re.compile( r"(? Any: """Redact secrets (central redactor) then MoA PII patterns. force=True: the privacy filter is its own opt-in, independent of the global - log-redaction toggle. code_file=True: advisory text is prose/code, so the - ENV/JSON assignment heuristics that mangle source snippets stay off. + log-redaction toggle. code_file=True: keeps the ENV/JSON assignment heuristics + (which mangle source snippets) off advisory prose/code. """ if not isinstance(text, str) or not text: return text @@ -105,18 +104,14 @@ def _redact_trace_accounting(acct: Any) -> Any: ) -# Cold-start caches: the preset and each (provider, model) runtime are immutable -# for a turn, so avoid re-resolving them on every create() call. +# Cold-start caches: preset and per-(provider, model) runtime are immutable for a turn. _preset_cache_lock = threading.Lock() _preset_cache: dict[tuple, Any] = {} def _resolve_preset_cached(preset_name: str) -> tuple[dict[str, Any], Any]: - """Return ``(preset, raw moa config)``, caching the resolved preset per config mtime. - - load_config() is (mtime_ns, size)-cached upstream; the saving here is skipping - resolve_moa_preset's full validation of the moa block on every create(). - """ + """``(preset, raw moa config)``; the resolved preset is cached per config mtime + (skips resolve_moa_preset's full validation of the moa block on every create()).""" from hermes_cli.config import get_config_path, load_config from hermes_cli.moa_config import resolve_moa_preset @@ -153,9 +148,8 @@ _MAX_REFERENCE_WORKERS = 8 class _RefAccounting: """Per-reference usage, cost and full trace (third slot of a reference-output tuple). - Advisors may run on a different model than the aggregator, so cost is priced at - the advisor's OWN rate and summed in dollars. The trace fields (``messages``, - ``output``, ``model``, ``provider``, ``temperature``) are only populated when + Cost is priced at the advisor's OWN rate and summed in dollars (advisors may run + on a different model than the aggregator). Trace fields are only populated when tracing is on. """ @@ -227,11 +221,10 @@ def _slot_reasoning_config(slot: dict[str, Any]) -> dict[str, Any] | None: def _aggregator_reasoning_config(aggregator: dict[str, Any]) -> dict[str, Any] | None: - """Resolve the aggregator's reasoning config: slot > per-model > global. + """Aggregator reasoning config: slot > per-model > global (shared chokepoint). - The aggregator is the ACTING model, so it falls back through the shared - ``resolve_reasoning_config`` chokepoint. References deliberately do not: - inheriting a global ``xhigh`` into every advisor would multiply cost. + References deliberately do NOT fall back: inheriting a global ``xhigh`` into + every advisor would multiply cost. """ cfg = _slot_reasoning_config(aggregator) if cfg is not None: @@ -246,12 +239,10 @@ def _aggregator_reasoning_config(aggregator: dict[str, Any]) -> dict[str, Any] | def _slot_runtime(slot: dict[str, Any]) -> dict[str, Any]: - """Resolve a slot to ``call_llm`` kwargs via ``resolve_runtime_provider``. + """Slot → ``call_llm`` kwargs with the provider's real api_mode/base_url/api_key. - Gives the slot its provider's real api_mode/base_url/api_key instead of letting - the auxiliary auto-detector guess. Falls back to bare provider/model on error - (never cached: a transient error would pin bare kwargs for a TTL). Cached per - (provider, model) with a short TTL. + Cached per (provider, model) with a short TTL. Falls back to bare provider/model + on error — never cached, or a transient error would pin bare kwargs for a TTL. """ provider = str(slot.get("provider") or "").strip() model = str(slot.get("model") or "").strip() @@ -314,11 +305,10 @@ def _maybe_apply_moa_cache_control( ) -> list[dict[str, Any]]: """Apply cache_control to an advisor/aggregator request when its route honors it. - Same policy function and marker helper as the main loop; MoA has no static - prefix so the legacy system-and-3 fallback is used. ``cache_disabled`` is - stamped onto the stub so ``cache_ttl: off`` is honored; ``cache_ttl`` threads - the agent's tier, clamped per destination. Returns the messages unchanged on - any error. + Same policy/marker helpers as the main loop; MoA has no static prefix so the + legacy system-and-3 fallback is used. ``cache_disabled`` is stamped onto the + stub so ``cache_ttl: off`` is honored; ``cache_ttl`` is clamped per destination. + Returns the messages unchanged on any error. """ try: from agent.agent_runtime_helpers import ( @@ -401,12 +391,8 @@ def _run_reference( cache_disabled: bool | None = None, cache_ttl: str | None = None, ) -> tuple[str, str, Any]: - """Call one reference model; return ``(label, text, accounting)``. Never raises. - - The slot is resolved to its provider's real runtime and called through the - same ``call_llm`` path any model uses. A failed reference becomes a labelled - ``[failed: …]`` note. Runs inside a thread pool (call_llm is blocking). - """ + """Call one reference model; return ``(label, text, accounting)``. Never raises: + a failed reference becomes a labelled ``[failed: …]`` note. Runs in a thread pool.""" label = _slot_label(slot) runtime = _slot_runtime(slot) trace_fields = { @@ -414,11 +400,10 @@ def _run_reference( "provider": runtime.get("provider") or slot.get("provider"), "temperature": temperature, } - # The trimmed view already stripped the agent's system prompt; this is the only one. + # The advisory view already stripped the agent's system prompt; this is the only one. messages = [{"role": "system", "content": _REFERENCE_SYSTEM_PROMPT}, *ref_messages] try: - # Trim to THIS model's window (advisors may be smaller than the aggregator); - # estimated after the system prompt is prepended so it counts too. + # Trim to THIS model's window (advisors may be smaller than the aggregator). trimmed = _trim_messages_for_reference( messages, slot, @@ -426,9 +411,8 @@ def _run_reference( reserve_output_tokens=max_tokens, context_length_cache=context_length_cache, ) - # Anthropic-style caching is opt-in per request; the advisory view is append-only - # across iterations, so decorating lets iteration N+1 replay N's cached prefix. - # The live agent disable is pinned onto the runtime (not a fresh config read). + # The advisory view is append-only across iterations, so cache_control lets + # iteration N+1 replay N's cached prefix. trimmed = _maybe_apply_moa_cache_control( trimmed, _with_cache_disabled(runtime, cache_disabled), cache_ttl=cache_ttl ) @@ -485,12 +469,11 @@ def _trim_messages_for_reference( ) -> list[dict[str, Any]]: """Trim an advisory request to fit a reference model's context window. - ``messages`` is the full request (advisory system prompt included). Budget = - window minus ``reserve_output_tokens`` (or a default) minus a safety fraction. - Drops the OLDEST frames after the system prompt while keeping: the system - prompt, a user-first body, and the trailing user turn plus one preceding turn - (even if still over budget). ``context_length_cache`` memoizes the window per - (provider, model) for the fan-out; unresolvable windows leave messages unchanged. + Budget = window − ``reserve_output_tokens`` (or a default) − a safety fraction. + Drops the OLDEST frames after the system prompt, always keeping a user-first + body and the trailing user turn plus one preceding turn (even if still over + budget). ``context_length_cache`` memoizes the window per (provider, model); + unresolvable windows leave messages unchanged. """ if not messages: return messages @@ -582,13 +565,9 @@ def _settle_interrupted( reference_models: list[dict[str, Any]], late_accounting_sink: Any, ) -> None: - """Fill every unfinished slot after a user interrupt. - - Never-dispatched futures are cancelled (nothing billed). Ones that finished - between the interrupt check and now keep their real output. Running ones cannot - be killed and WILL be billed on completion, so their eventual accounting is - handed to ``late_accounting_sink``. - """ + """Fill every unfinished slot after a user interrupt: cancel never-dispatched + futures (nothing billed), keep real output of ones that just finished, and hand + running ones (cannot be killed, WILL bill) to ``late_accounting_sink``.""" for future, idx in futures.items(): if results[idx] is not None: continue @@ -623,23 +602,20 @@ def _run_references_parallel( agent: Any = None, late_accounting_sink: Any = None, ) -> list[tuple[str, str, Any]]: - """Fan out all reference models in parallel; outputs are in ``reference_models`` order. + """Fan out all reference models in parallel; ``(label, text, _RefAccounting)`` per + slot in ``reference_models`` order. - Slots with ``provider == "moa"`` are skipped with a labelled note (recursion - guard). ``progress_callback(refs_done, refs_total, label)`` fires per completion - (best-effort). Each element is ``(label, text, _RefAccounting)``. - - With *agent*, the wait polls every ``_REFERENCE_POLL_INTERVAL_S`` so a user - interrupt can abort it (like tool_executor's batch). In-flight calls cannot be - killed; their eventual accounting goes to ``late_accounting_sink``. + ``provider == "moa"`` slots are skipped with a note (recursion guard). + ``progress_callback(refs_done, refs_total, label)`` fires per completion. With + *agent*, the wait polls every ``_REFERENCE_POLL_INTERVAL_S`` so a user interrupt + can abort it; in-flight calls cannot be killed and bill via ``late_accounting_sink``. """ if not reference_models: return [] results: list[tuple[str, str, Any] | None] = [None] * len(reference_models) futures: dict[Any, int] = {} - # Executor threads start with an empty contextvars.Context; propagate the turn's - # (approval callbacks + Nous Portal conversation tag) into each worker. + # Propagate the turn's contextvars (approval callbacks, Nous conversation tag). from tools.thread_context import propagate_context_to_thread total = len(reference_models) @@ -748,32 +724,28 @@ _ADVISORY_INSTRUCTION = ( def _reference_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: """Build the advisory (reference-model) view of the conversation. - Flattens the transcript to plain user/assistant TEXT turns: the system prompt - is dropped, tool_calls are rendered inline, and tool results are folded into - the preceding assistant turn as ``[tool result: ...]`` previews. Zero tool-role - messages / tool_calls arrays are emitted, so strict providers do not 400. - The view always ends on a ``user`` turn (Anthropic treats a trailing assistant - turn as prefill): a synthetic advisory request is APPENDED rather than - deleting context. The aggregator always receives the full transcript. + Plain user/assistant TEXT turns only: system prompt dropped, tool_calls rendered + inline, tool results folded into the preceding assistant turn as previews (no + tool-role messages / tool_calls arrays, so strict providers do not 400). Always + ends on a ``user`` turn (Anthropic treats a trailing assistant turn as prefill) + by APPENDING a synthetic request. The aggregator always gets the full transcript. """ rendered: list[dict[str, Any]] = [] last_user_content: str | None = None for msg in messages: role = msg.get("role") content = msg.get("content") - # Content may be a list (cache_control-decorated text parts, multimodal turns); - # flatten_message_text extracts text so decorated and undecorated transcripts - # yield a byte-identical view (stable advisory prefix for advisor caching). + # Decorated (cache_control parts) and undecorated transcripts must yield a + # byte-identical view so the advisory prefix stays cache-stable. text = flatten_message_text(content) if role == "user": if not text.strip() and isinstance(content, list) and content: - # Structured content with no text (e.g. image-only): an empty user - # message is rejected by strict providers and skipping breaks alternation. + # Image-only turn: empty user messages are rejected by strict providers + # and skipping would break alternation. text = "[user sent non-text content (e.g. an image attachment)]" if not text.strip(): - # Genuinely empty user turn: strict providers (Kimi, ZAI) 400 on it; dropping - # is safe because the advisory view is not strictly alternating anyway. + # Genuinely empty user turn: strict providers 400 on it; safe to drop. continue last_user_content = text rendered.append({"role": "user", "content": text}) @@ -795,8 +767,7 @@ def _reference_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: rendered.append({"role": "assistant", "content": block}) # system and any other role are ignored. - # End on a user turn by appending a synthetic advisory request (Anthropic - # rejects trailing assistant prefill); an existing trailing user turn is left as is. + # Anthropic rejects trailing assistant prefill: end on a synthetic user request. if rendered and rendered[-1].get("role") == "assistant": rendered.append({"role": "user", "content": _ADVISORY_INSTRUCTION}) @@ -915,9 +886,8 @@ def aggregate_moa_context( """Run configured reference models and synthesize their advice (one-shot /moa). Failures become model-specific notes instead of aborting the loop. - ``reference_max_tokens`` caps ONLY the fan-out — capping the aggregator - truncated long syntheses. ``temperature`` / ``aggregator_temperature`` - default to None (provider default). ``agent`` makes the fan-out interruptible. + ``reference_max_tokens`` caps ONLY the fan-out (capping the aggregator truncated + long syntheses). ``agent`` makes the fan-out interruptible. """ reference_models = [slot for slot in reference_models if slot.get("enabled", True)] reference_outputs = _run_references_parallel( @@ -930,8 +900,7 @@ def aggregate_moa_context( ) successful_outputs, failed_labels = _split_references(reference_outputs) - # 'full' privacy mode also redacts advisor text before it reaches this synthesizer - # ('display' has no surface here). Failed refs are already filtered out. + # 'full' privacy mode also redacts advisor text before it reaches the synthesizer. try: from hermes_cli.config import load_config as _load_config @@ -1001,10 +970,7 @@ def aggregate_moa_context( def _completed_response_as_stream_chunk(response: Any) -> Any: - """Adapt a completed response (``choices[0].message``) into one delta stream chunk. - - Done at the MoA facade boundary so transports stay untouched. - """ + """Adapt a completed response into one delta stream chunk (facade boundary only).""" choices = getattr(response, "choices", None) first_choice = choices[0] if isinstance(choices, (list, tuple)) and choices else None message = getattr(first_choice, "message", None) @@ -1046,12 +1012,10 @@ def _completed_response_as_stream_chunk(response: Any) -> Any: def _attach_reference_guidance(agg_messages: list[dict[str, Any]], guidance: str) -> None: """Attach the per-turn reference block at the END of the aggregator prompt. - The block varies per iteration; merging it into the (early) original user - message would diverge the prompt prefix and re-prefill the whole conversation - each step. Appending keeps ``[system][task][tool-history]`` cache-stable. A - trailing user turn is merged in place (string or content-part list — a new text - part rides AFTER the cache_control-marked part); otherwise a user message is - appended (two consecutive user turns would be rejected by strict providers). + The block varies per iteration; appending keeps ``[system][task][tool-history]`` + cache-stable. A trailing user turn is merged in place (string, or a new text part + AFTER the cache_control-marked part); otherwise a user message is appended (two + consecutive user turns would be rejected by strict providers). """ last = agg_messages[-1] if agg_messages else None if last is not None and last.get("role") == "user": @@ -1069,11 +1033,8 @@ def peel_reference_guidance( messages: list[dict[str, Any]], guidance: Any, ) -> list[dict[str, Any]]: - """Exact inverse of ``_attach_reference_guidance`` (the three attach shapes). - - Used by the failover redecoration chokepoint so a cache breakpoint never lands - on the turn-varying guidance. Returns a new list; inputs are not mutated. - """ + """Exact inverse of ``_attach_reference_guidance`` (the three attach shapes), so a + cache breakpoint never lands on the turn-varying guidance. Inputs are not mutated.""" if not guidance or not messages: return messages guidance_text = str(guidance) @@ -1107,39 +1068,33 @@ def peel_reference_guidance( class MoAChatCompletions: """OpenAI-chat-compatible facade where the aggregator is the acting model. - ``reference_callback(event, **kwargs)`` is an optional best-effort display hook: - "moa.reference" index, count, label, text - "moa.progress" refs_done, refs_total, label (per reference completion) - "moa.phase" phase, refs_done, refs_total, aggregator - "moa.aggregating" aggregator (label), ref_count - ``agent`` is the owning AIAgent; it lets the fan-out check ``_interrupt_requested``. + ``reference_callback(event, **kwargs)`` is an optional best-effort display hook + (events: ``moa.reference``, ``moa.progress``, ``moa.phase``, ``moa.aggregating``; + kwargs per ``_RELAY_EVENTS``). ``agent`` is the owning AIAgent; it lets the + fan-out check ``_interrupt_requested``. """ def __init__(self, preset_name: str, reference_callback: Any = None, agent: Any = None): self.preset_name = preset_name or "default" self.reference_callback = reference_callback self._agent = agent - # State-scoped reference cache keyed on the advisory-view signature: a new - # user/tool message is a MISS (references re-run), a redundant create() with - # identical state is a HIT (no re-run, no re-emit). + # Reference cache keyed on the advisory-view signature: new state = MISS + # (references re-run), identical state = HIT (no re-run, no re-emit). self._ref_cache_key: tuple | None = None self._ref_cache_outputs: list[tuple[str, str, Any]] = [] - # Fan-out usage/cost from the latest cache-MISS create(), awaiting - # consume_reference_usage (zero deposited on a HIT so spend counts once). - # The lock guards them against late-accounting callbacks on worker threads. + # Fan-out spend awaiting consume_reference_usage (nothing deposited on a HIT so + # spend counts once); the lock guards late-accounting callbacks on worker threads. self._pending_reference_usage: Any = CanonicalUsage() self._pending_reference_cost: Any = None self._accounting_lock = threading.Lock() - # Resolved aggregator slot from the latest create(); cost accounting prices the - # acting turn at its real model instead of the virtual preset name. + # Real aggregator slot so cost accounting prices the acting turn at its model. self.last_aggregator_slot: Any = None # Full-turn trace parts from a cache-MISS create(), flushed by consume_and_save_trace. self._pending_trace: Any = None # Per-advisor metrics for observability hooks; NOT consumed (post_api_request # fires on a different branch than consume_and_save_trace). self._last_reference_metrics: Any = None - # every_n cadence state, scoped to a single USER TURN (resets on a new user - # message) so iteration 1 of every turn is on-cadence. + # every_n cadence state, scoped to one USER TURN so iteration 1 is on-cadence. self._fanout_iteration_count = 0 self._fanout_turn_sig: str | None = None self._fanout_last_state_sig: str | None = None @@ -1147,10 +1102,8 @@ class MoAChatCompletions: self._privacy_mode: str = "" def consume_reference_usage(self) -> tuple[Any, Any]: - """Pop pending fan-out ``(CanonicalUsage, cost_usd_or_None)`` and reset both. - - Clearing prevents a streaming retry re-entering accounting from double-counting. - """ + """Pop pending fan-out ``(CanonicalUsage, cost_usd_or_None)`` and reset both + (so a streaming retry re-entering accounting cannot double-count).""" with self._accounting_lock: usage = self._pending_reference_usage or CanonicalUsage() cost = self._pending_reference_cost @@ -1163,10 +1116,7 @@ class MoAChatCompletions: return self._last_reference_metrics def _record_late_reference_accounting(self, label: str, accounting: Any) -> None: - """Fold a late-completing interrupted reference's real spend into pending totals. - - Registered as a done-callback on abandoned futures (they still bill). Thread-safe. - """ + """Done-callback for abandoned (still billing) futures: fold their real spend in.""" if not isinstance(accounting, _RefAccounting): return self._fold_pending_accounting(*_sum_reference_accounting([(label, "", accounting)])) @@ -1184,9 +1134,8 @@ class MoAChatCompletions: ) -> None: """Flush the pending full-turn trace to disk (no-op when nothing is pending). - ``aggregator_output_fallback`` is the caller's resolved acting text: on the - streaming path the output could not be captured at ``create()`` time, so it - is folded in here. Clears the pending trace; never raises. + ``aggregator_output_fallback`` is the caller's resolved acting text for the + streaming path (not capturable at ``create()`` time). Never raises. """ pending = self._pending_trace self._pending_trace = None @@ -1225,11 +1174,8 @@ class MoAChatCompletions: logger.debug("MoA reference_callback failed for %s: %s", event, exc) def prepare(self, messages: list[dict[str, Any]]) -> dict[str, Any]: - """Run the advisor fan-out and return the exact aggregator request. - - The loop measures this augmented prompt before its compression gate, then - hands the object back to ``create()`` so the fan-out is not repeated. - """ + """Run the advisor fan-out and return the exact aggregator request, which the + loop measures before its compression gate and hands back to ``create()``.""" return self.create(messages=messages, _moa_prepare_only=True) def rebase_prepared_request( @@ -1249,11 +1195,11 @@ class MoAChatCompletions: guidance: Any, agg_runtime: dict[str, Any], ) -> tuple[list[dict[str, Any]], Any]: - """Decorate the aggregator request with cache breakpoints for its destination. + """Cache-breakpoint the aggregator request for its destination. - The guidance is peeled before planning and re-attached after so a breakpoint - never lands on the turn-varying block. On any error the undecorated request - is returned (warning, not debug: this is the aggregator's ONLY decoration path). + Guidance is peeled before planning and re-attached after so a breakpoint never + lands on the turn-varying block. Any error → undecorated request (warning, not + debug: this is the aggregator's ONLY decoration path). """ try: from agent.agent_runtime_helpers import plan_cache_sections_for_destination @@ -1261,7 +1207,6 @@ class MoAChatCompletions: planning_messages = agg_messages if guidance: planning_messages = peel_reference_guidance(agg_messages, str(guidance)) - # plan_cache_sections_for_destination returns request-local copies. # Tri-state cache_disabled: facades built via __new__ have no _agent; forcing # False would suppress the planner's config fallback. _agent = getattr(self, "_agent", None) @@ -1298,8 +1243,7 @@ class MoAChatCompletions: agg_messages, tools = self._plan_aggregator_cache( prepared["messages"], api_kwargs.get("tools"), prepared.get("guidance"), agg_runtime ) - # Record the exact aggregator INPUT into the pending trace; the persisted COPY - # is redacted under any privacy mode while the live input stays raw. + # Trace the exact aggregator INPUT (persisted copy redacted; live input raw). if self._pending_trace is not None: self._pending_trace["aggregator_input_messages"] = ( _redact_trace_messages([dict(m) for m in agg_messages]) @@ -1307,8 +1251,6 @@ class MoAChatCompletions: else agg_messages ) self._pending_trace["aggregator_label"] = _slot_label(aggregator) - # The aggregator is the acting model: call it through the same request path - # any model uses, with max_tokens passed through (None → model maximum). # stream=True returns the RAW token stream (consumer reassembles + retries); # the non-streaming path forwards no stream/stream_options/timeout. stream = bool(api_kwargs.get("stream")) @@ -1319,8 +1261,7 @@ class MoAChatCompletions: # The consumer's stream-read timeout must govern the aggregator stream. if api_kwargs.get("timeout") is not None: stream_kwargs["timeout"] = api_kwargs["timeout"] - # Pop the runtime's extra_body and merge with the caller's (caller wins) so the - # explicit kwarg never collides with **agg_runtime. + # Pop the runtime's extra_body so the explicit kwarg never collides with **agg_runtime. agg_extra_body = _merge_slot_extra_body( agg_runtime.pop("extra_body", None), api_kwargs.get("extra_body"), ) @@ -1336,8 +1277,7 @@ class MoAChatCompletions: **stream_kwargs, **agg_runtime, ) - # Non-streaming: capture the aggregator output inline. Streaming: the output - # lands as the turn's assistant message; the trace marks it streamed. + # Streaming output lands as the turn's assistant message; the trace marks it. if self._pending_trace is not None: self._pending_trace["aggregator_streamed"] = stream output = None @@ -1359,14 +1299,12 @@ class MoAChatCompletions: ref_messages: list[dict[str, Any]], reference_models: list[dict[str, Any]], ) -> tuple: - """Compute the turn-scoped reference cache key per the preset's fan-out cadence. + """Turn-scoped reference cache key per the preset's fan-out cadence. - "user_turn" (default): advisors run once per user turn — the signature hashes - only the prefix up to the LAST USER message, so later tool iterations are - cache HITs. "per_iteration": re-run whenever the advisory view changes. - "every_n:": iteration 1 of a turn, then every Nth; in-between iterations - reuse the last on-cadence guidance (key pinned to that run so the lookup is a - HIT: no advisor calls, no double accounting, no re-emit). + "user_turn" (default) hashes only the prefix up to the LAST USER message, so + later tool iterations are HITs. "per_iteration" re-runs whenever the advisory + view changes. "every_n:": iteration 1 of a turn, then every Nth; in-between + iterations return the pinned last on-cadence key (HIT: no calls, no re-emit). """ fanout_mode = str(preset.get("fanout") or "user_turn").strip().lower() every_n = 0 @@ -1423,10 +1361,11 @@ class MoAChatCompletions: aggregator_temperature: Any, cache_key: tuple, ) -> list[tuple[str, str, Any]]: - """Cache-MISS path of ``create``: run the advisors, account, trace and emit.""" - # No output caps by default (None → call_llm omits max_tokens). A preset MAY cap - # ADVISOR output (dominant MoA latency); the acting aggregator is never capped. - # None reference_timeout = inherit auxiliary.moa_reference.timeout via call_llm. + """Cache-MISS path of ``create``: run the advisors, account, trace and emit. + + A preset MAY cap ADVISOR output (dominant MoA latency); the acting aggregator + is never capped. None timeout = inherit auxiliary.moa_reference.timeout. + """ raw_reference_timeout = preset.get("reference_timeout") def _progress(done: int, total: int, label: str) -> None: @@ -1452,9 +1391,8 @@ class MoAChatCompletions: self._ref_cache_outputs = list(reference_outputs) # Fold advisor spend into accounting exactly once per turn. self._fold_pending_accounting(*_sum_reference_accounting(reference_outputs)) - # Stash the fan-out for trace persistence (aggregator input/label filled in - # later; output stitched in by consume_and_save_trace). Traces are persisted, - # so ANY active privacy mode redacts advisor text and per-advisor input/output. + # Stash the fan-out for trace persistence (aggregator parts filled in later). + # Traces are persisted, so ANY active privacy mode redacts them. privacy_mode = self._privacy_mode if privacy_mode: trace_refs = [ @@ -1480,8 +1418,7 @@ class MoAChatCompletions: logger.debug("MoA reference metrics render failed: %s", exc) self._last_reference_metrics = None - # Surface each reference's answer BEFORE the aggregator acts (once per turn). - # The cache keeps RAW text; redaction happens at each consuming surface. + # Surface each answer BEFORE the aggregator acts; the cache keeps RAW text. ref_count = len(reference_outputs) for idx, (label, text, _accounting) in enumerate(reference_outputs, start=1): self._emit( @@ -1512,8 +1449,7 @@ class MoAChatCompletions: ) -> str | None: """Render the reference block attached to the aggregator prompt (None = nothing).""" successful_outputs, failed_labels = _split_references(reference_outputs) - # 'full' privacy mode redacts advisor text reaching the AGGREGATOR too; 'display' - # leaves it raw. Applied to a per-call copy — the cache holds raw text. + # 'full' privacy mode redacts advisor text reaching the AGGREGATOR too. agg_refs = ( _redact_reference_outputs(successful_outputs) if self._privacy_mode == "full" @@ -1526,8 +1462,7 @@ class MoAChatCompletions: f"Aggregator/acting model: {_slot_label(aggregator)}\n" ) if reference_outputs and not successful_outputs: - # Every reference failed: the aggregator acts alone. Under the loud policy it - # still gets the sanitized unavailability notice; under silent, nothing. + # Every reference failed: the aggregator acts alone (loud policy → notice). logger.warning( "MoA: all %d reference(s) failed — acting aggregator-alone " "without reference guidance", @@ -1579,9 +1514,8 @@ class MoAChatCompletions: ref_messages = _reference_messages(messages) cache_key = self._fanout_cache_key(preset, ref_messages, reference_models) if cache_key == self._ref_cache_key and self._ref_cache_outputs: - # Cache HIT: references already ran and were accounted this turn. Deposit - # nothing, but do NOT zero pending totals (a late interrupted reference may - # have deposited real spend). No trace either — a repeat iteration is not a turn. + # HIT: already ran and accounted. Do NOT zero pending totals (a late + # interrupted reference may have deposited) and no trace (not a new turn). reference_outputs = list(self._ref_cache_outputs) self._pending_trace = None else: @@ -1608,38 +1542,31 @@ class MoAChatCompletions: class MoAClient: + """OpenAI-client-shaped wrapper: ``client.chat.completions`` is a ``MoAChatCompletions``. + + The accounting/trace surface (``consume_reference_usage``, ``last_aggregator_slot``, + ``consume_and_save_trace``, ``last_reference_metrics``) is delegated to the facade. + """ + + _DELEGATED = ( + "consume_reference_usage", "last_aggregator_slot", + "consume_and_save_trace", "last_reference_metrics", + ) + def __init__(self, preset_name: str, reference_callback: Any = None, agent: Any = None): self.chat = type("_MoAChat", (), {})() self.chat.completions = MoAChatCompletions( preset_name, reference_callback=reference_callback, agent=agent, ) - def consume_reference_usage(self) -> Any: - """Pop pending reference-fan-out usage + cost from the completions facade.""" - return self.chat.completions.consume_reference_usage() - - @property - def last_aggregator_slot(self) -> Any: - """Resolved aggregator slot from the most recent create(), or None.""" - return getattr(self.chat.completions, "last_aggregator_slot", None) - - def consume_and_save_trace( - self, session_id: Any = None, aggregator_output_fallback: Any = None - ) -> None: - """Flush the pending full-turn MoA trace via the completions facade.""" - return self.chat.completions.consume_and_save_trace( - session_id, aggregator_output_fallback=aggregator_output_fallback - ) - - def last_reference_metrics(self) -> Any: - """Per-advisor metrics from the most recent fan-out, or None (read-only).""" - return self.chat.completions.last_reference_metrics() + def __getattr__(self, name: str) -> Any: + if name in MoAClient._DELEGATED: + return getattr(self.chat.completions, name) + raise AttributeError(name) -# Display-event relay table for build_moa_facade: event -> (primary kwarg, secondary -# kwarg or None, {tool_progress_callback kwarg: emit kwarg}). The callback signature is -# ``cb(event, label, text, None, **moa_*)``; "moa.progress" is rendered by frontends as -# a status-bar ``MOA: N/M refs done``. +# Relay table: event -> (primary kwarg, secondary kwarg or None, {cb kwarg: emit kwarg}). +# The callback signature is ``cb(event, label, text, None, **moa_*)``. _RELAY_EVENTS: dict[str, tuple[str, str | None, dict[str, str]]] = { "moa.reference": ("label", "text", {"moa_index": "index", "moa_count": "count"}), "moa.progress": ("label", None, {"moa_refs_done": "refs_done", "moa_refs_total": "refs_total"}), @@ -1653,13 +1580,9 @@ _RELAY_EVENTS: dict[str, tuple[str, str | None, dict[str, str]]] = { def build_moa_facade(agent, preset_name: Any = None) -> MoAClient: - """Build the MoA facade client for ``agent``, wiring the reference relay. - - Single construction point for ``MoAClient`` (agent_init, fallback restore, - transport recovery, switch_model): a bare ``MoAClient(preset)`` would drop the - ``reference_callback`` relay and silence display events for the session. - The relay reads ``agent.tool_progress_callback`` at emit time. - """ + """Single construction point for ``MoAClient``: a bare ``MoAClient(preset)`` would + drop the ``reference_callback`` relay and silence display events for the session. + The relay reads ``agent.tool_progress_callback`` at emit time.""" def _moa_reference_relay(event: str, **kwargs: Any) -> None: cb = getattr(agent, "tool_progress_callback", None) spec = _RELAY_EVENTS.get(event) diff --git a/agent/moa_trace.py b/agent/moa_trace.py index 964d72de41..b4ccade104 100644 --- a/agent/moa_trace.py +++ b/agent/moa_trace.py @@ -1,12 +1,9 @@ """Full MoA turn trace persistence (opt-in via config ``moa.save_traces``). -When enabled, every Mixture-of-Agents turn that runs the reference fan-out (a -cache MISS in ``MoAChatCompletions.create``) appends one JSON line to -``/moa-traces/.jsonl``: the exact messages each -reference received, each reference's full output, and the exact aggregator input -plus its output when available — what every model saw, said, and cost. - -Side-channel only: never enters the ``messages`` table, history or replay +Every MoA turn that runs the reference fan-out (a cache MISS in +``MoAChatCompletions.create``) appends one JSON line to +``/moa-traces/.jsonl``: what every model saw, said, and +cost. Side-channel only: never enters the ``messages`` table, history or replay (references are advisory side-calls whose rows would corrupt role alternation). Off by default; when off the only overhead is the config read. """ @@ -32,7 +29,7 @@ def _traces_enabled_and_dir() -> Optional[Path]: from hermes_cli.config import load_config moa_cfg = (load_config() or {}).get("moa") or {} - except Exception: # pragma: no cover - defensive: never break a turn over tracing + except Exception: # pragma: no cover - never break a turn over tracing return None if not moa_cfg.get("save_traces"): return None @@ -54,8 +51,8 @@ _COST_FIELDS = ("cost_usd", "cost_status", "cost_source") def _slot_trace(acct: Any, label: str) -> dict[str, Any]: - """Render one reference's _RefAccounting into a full trace dict, including - the FULL input messages and output (not the truncated display preview).""" + """One reference's _RefAccounting as a full trace dict, including the FULL + input messages and output (not the truncated display preview).""" usage = getattr(acct, "usage", None) return { "label": label, diff --git a/agent/moonshot_schema.py b/agent/moonshot_schema.py index ab7319f2cc..079280cfbb 100644 --- a/agent/moonshot_schema.py +++ b/agent/moonshot_schema.py @@ -1,16 +1,11 @@ -"""Helpers for translating OpenAI-style tool schemas to Moonshot's schema subset. +"""Translate OpenAI-style tool schemas to Moonshot's (Kimi) stricter JSON Schema subset. -Moonshot (Kimi) accepts a stricter subset of JSON Schema than OpenAI tool -calling; violations fail with HTTP 400 "tools.function.parameters is not a -valid moonshot flavored json schema". Rules applied here: - -1. Every property schema must carry a ``type`` (JSON Schema allows omitting it). -2. With ``anyOf``, ``type`` belongs on the children, never the parent. -3. Enum arrays under scalar types may not contain null / empty string. -4. Every object schema must carry a ``required`` array, even an empty one. - -The ``#/definitions/...`` → ``#/$defs/...`` rewrite for draft-07 refs lives in -``tools/mcp_tool._normalize_mcp_input_schema`` so it applies to all providers. +Violations fail with HTTP 400 "tools.function.parameters is not a valid moonshot +flavored json schema". Rules: (1) every property schema carries a ``type``; +(2) with ``anyOf``, ``type`` belongs on the children, never the parent; (3) enum +arrays under scalar types may not contain null / empty string; (4) every object +schema carries a ``required`` array, even an empty one. The ``#/definitions/`` → +``#/$defs/`` rewrite lives in ``tools/mcp_tool`` so it applies to all providers. """ from __future__ import annotations @@ -26,6 +21,10 @@ _SCHEMA_LIST_KEYS = frozenset({"anyOf", "oneOf", "allOf", "prefixItems"}) # Values are a single nested schema (additionalProperties may also be a bool). _SCHEMA_NODE_KEYS = frozenset({"items", "contains", "not", "additionalProperties", "propertyNames"}) +_SCALAR_TYPES = frozenset({"string", "integer", "number", "boolean"}) +# bool before int: bool is an int subclass. +_ENUM_SAMPLE_TYPES = ((bool, "boolean"), (int, "integer"), (float, "number")) + def _empty_object_schema() -> Dict[str, Any]: return {"type": "object", "properties": {}, "required": []} @@ -42,9 +41,9 @@ def _repair_schema(node: Any) -> Any: for key, value in node.items(): if key in _SCHEMA_MAP_KEYS and isinstance(value, dict): repaired[key] = {sub_key: _repair_schema(sub_val) for sub_key, sub_val in value.items()} - elif key in _SCHEMA_LIST_KEYS and isinstance(value, list): - repaired[key] = [_repair_schema(v) for v in value] - elif key in _SCHEMA_NODE_KEYS and isinstance(value, dict): + elif (key in _SCHEMA_LIST_KEYS and isinstance(value, list)) or ( + key in _SCHEMA_NODE_KEYS and isinstance(value, dict) + ): repaired[key] = _repair_schema(value) else: repaired[key] = value @@ -60,9 +59,7 @@ def _repair_schema(node: Any) -> Any: if len(non_null) > 1: repaired["anyOf"] = non_null return repaired - merge = {k: v for k, v in repaired.items() if k != "anyOf"} - merge.update(non_null[0]) - repaired = merge + repaired = {**{k: v for k, v in repaired.items() if k != "anyOf"}, **non_null[0]} # Moonshot also rejects the non-standard ``nullable`` keyword. repaired.pop("nullable", None) @@ -73,7 +70,7 @@ def _repair_schema(node: Any) -> Any: repaired = _fill_missing_type(repaired) # Rule 3: drop null/"" enum values under scalar types; drop an emptied enum. - if isinstance(repaired.get("enum"), list) and repaired.get("type") in {"string", "integer", "number", "boolean"}: + if isinstance(repaired.get("enum"), list) and repaired.get("type") in _SCALAR_TYPES: cleaned = [v for v in repaired["enum"] if v is not None and v != ""] if cleaned: repaired["enum"] = cleaned @@ -88,9 +85,8 @@ def _repair_schema(node: Any) -> Any: def _ensure_required_array(node: Dict[str, Any]) -> Dict[str, Any]: - """Guarantee an object schema carries a ``required`` list (Moonshot 400s - otherwise), pruning names that don't exist in ``properties`` — Moonshot - also rejects dangling names. Mutates and returns ``node``.""" + """Guarantee an object schema carries a ``required`` list, pruning names not in + ``properties`` (Moonshot also rejects dangling names). Mutates and returns ``node``.""" props = node.get("properties") req = node.get("required") if isinstance(req, list): @@ -102,7 +98,7 @@ def _ensure_required_array(node: Dict[str, Any]) -> Dict[str, Any]: def _fill_missing_type(node: Dict[str, Any]) -> Dict[str, Any]: - """Infer a reasonable ``type`` if this schema node has none. + """Infer a ``type`` if this schema node has none. A type list collapses to its first concrete member; otherwise ``properties``/``required``/``additionalProperties`` → object, @@ -111,10 +107,7 @@ def _fill_missing_type(node: Dict[str, Any]) -> Dict[str, Any]: """ node_type = node.get("type") if isinstance(node_type, list): - concrete = next( - (t for t in node_type if isinstance(t, str) and t not in {"", "null"}), - "string", - ) + concrete = next((t for t in node_type if isinstance(t, str) and t not in {"", "null"}), "string") return {**node, "type": concrete} if "type" in node and node_type not in {None, ""}: return node @@ -124,24 +117,20 @@ def _fill_missing_type(node: Dict[str, Any]) -> Dict[str, Any]: elif "items" in node or "prefixItems" in node: inferred = "array" elif isinstance(node.get("enum"), list) and node["enum"]: - sample = node["enum"][0] # bool before int: bool is an int subclass - scalar_types = ((bool, "boolean"), (int, "integer"), (float, "number")) - inferred = next((t for cls, t in scalar_types if isinstance(sample, cls)), "string") + sample = node["enum"][0] + inferred = next((t for cls, t in _ENUM_SAMPLE_TYPES if isinstance(sample, cls)), "string") else: inferred = "string" - return {**node, "type": inferred} def sanitize_moonshot_tool_parameters(parameters: Any) -> Dict[str, Any]: - """Return a deep-copied, Moonshot-compatible object schema; input is not mutated.""" + """Deep-copied, Moonshot-compatible object schema; input is not mutated.""" if not isinstance(parameters, dict): return _empty_object_schema() - repaired = _repair_schema(copy.deepcopy(parameters)) if not isinstance(repaired, dict): return _empty_object_schema() - # Top-level must be an object schema. repaired["type"] = "object" repaired.setdefault("properties", {}) @@ -155,22 +144,17 @@ def sanitize_moonshot_tools(tools: List[Dict[str, Any]]) -> List[Dict[str, Any]] """ if not tools: return tools - sanitized: List[Dict[str, Any]] = [] any_change = False for tool in tools: fn = tool.get("function") if isinstance(tool, dict) else None - if not isinstance(fn, dict): - sanitized.append(tool) - continue - params = fn.get("parameters") - repaired = sanitize_moonshot_tool_parameters(params) - if repaired is not params: - any_change = True - sanitized.append({**tool, "function": {**fn, "parameters": repaired}}) - else: - sanitized.append(tool) - + if isinstance(fn, dict): + params = fn.get("parameters") + repaired = sanitize_moonshot_tool_parameters(params) + if repaired is not params: + any_change = True + tool = {**tool, "function": {**fn, "parameters": repaired}} + sanitized.append(tool) return sanitized if any_change else tools diff --git a/agent/nous_rate_guard.py b/agent/nous_rate_guard.py index b5acb048b3..2e3e95f214 100644 --- a/agent/nous_rate_guard.py +++ b/agent/nous_rate_guard.py @@ -1,9 +1,9 @@ """Cross-session rate limit guard for Nous Portal. -Writes rate limit state to a shared file so all sessions (CLI, gateway, -cron, auxiliary) can check whether Nous Portal is currently rate-limited -before making requests. Without it each 429 fans out into up to 9 calls per -turn (3 SDK retries x 3 Hermes retries), all counted against RPH. +Writes rate limit state to a shared file so all sessions (CLI, gateway, cron, +auxiliary) can check whether Nous Portal is currently rate-limited before making +requests. Without it each 429 fans out into up to 9 calls per turn (3 SDK +retries x 3 Hermes retries), all counted against RPH. """ from __future__ import annotations @@ -12,10 +12,9 @@ import contextlib import json import logging import os -import tempfile import time from typing import Any, Mapping, Optional -from utils import atomic_replace +from utils import atomic_write_text from agent.rate_limit_tracker import ( _BUCKET_TAGS, _fmt_seconds, @@ -35,7 +34,7 @@ format_remaining = _fmt_seconds def _state_path() -> str: - """Return the path to the Nous rate limit state file.""" + """Path to the Nous rate limit state file.""" try: from hermes_constants import get_hermes_home base = get_hermes_home() @@ -47,11 +46,7 @@ def _state_path() -> str: def _parse_reset_seconds(headers: Optional[Mapping[str, str]]) -> Optional[float]: """Best reset estimate (seconds from now) from hourly, per-minute, then retry-after headers.""" lowered = lower_headers(headers) - for key in ( - "x-ratelimit-reset-requests-1h", - "x-ratelimit-reset-requests", - "retry-after", - ): + for key in ("x-ratelimit-reset-requests-1h", "x-ratelimit-reset-requests", "retry-after"): val = _safe_float(lowered.get(key), 0.0) if val > 0: return val @@ -71,44 +66,20 @@ def record_nous_rate_limit( """ now = time.time() reset_at = None - header_seconds = _parse_reset_seconds(headers) if header_seconds is not None: reset_at = now + header_seconds - if reset_at is None and isinstance(error_context, dict): ctx_reset = error_context.get("reset_at") if isinstance(ctx_reset, (int, float)) and ctx_reset > now: reset_at = float(ctx_reset) - if reset_at is None: reset_at = now + default_cooldown - path = _state_path() + state = {"reset_at": reset_at, "recorded_at": now, "reset_seconds": reset_at - now} try: - state_dir = os.path.dirname(path) - os.makedirs(state_dir, exist_ok=True) - - state = { - "reset_at": reset_at, - "recorded_at": now, - "reset_seconds": reset_at - now, - } - - fd, tmp_path = tempfile.mkstemp(dir=state_dir, suffix=".tmp") - try: - with os.fdopen(fd, "w", encoding="utf-8") as f: - json.dump(state, f) - atomic_replace(tmp_path, path) - except Exception: - with contextlib.suppress(OSError): - os.unlink(tmp_path) - raise - - logger.info( - "Nous rate limit recorded: resets in %.0fs (at %.0f)", - reset_at - now, reset_at, - ) + atomic_write_text(_state_path(), json.dumps(state)) + logger.info("Nous rate limit recorded: resets in %.0fs (at %.0f)", reset_at - now, reset_at) except Exception as exc: logger.debug("Failed to write Nous rate limit state: %s", exc) @@ -170,11 +141,10 @@ def is_genuine_nous_rate_limit( def _parse_buckets_from_headers( headers: Optional[Mapping[str, str]], ) -> dict[str, tuple[Optional[int], Optional[float]]]: - """Extract (remaining, reset_seconds) per bucket from x-ratelimit-* headers ({} if none).""" + """(remaining, reset_seconds) per bucket from x-ratelimit-* headers ({} if none).""" lowered = lower_headers(headers) if not has_rate_limit_headers(lowered): return {} - result: dict[str, tuple[Optional[int], Optional[float]]] = {} for _attr, tag in _BUCKET_TAGS: remaining = _safe_int(lowered.get(f"x-ratelimit-remaining-{tag}"), None) diff --git a/agent/plugin_llm.py b/agent/plugin_llm.py index c0ea2d8528..14ab45b15b 100644 --- a/agent/plugin_llm.py +++ b/agent/plugin_llm.py @@ -1,39 +1,12 @@ -""" -Plugin LLM facade — host-owned LLM access for trusted plugins. -============================================================== +"""Plugin LLM facade — host-owned LLM access for trusted plugins (``ctx.llm``). -Plugins that need their own out-of-band model call (rewrite a tool error, -translate inbound text, summarise a paste, score a scheduled job) get -``ctx.llm`` on :class:`~hermes_cli.plugins.PluginContext`: ``complete`` / -``complete_structured`` (text + image inputs, JSON schema validation) and their -async siblings ``acomplete`` / ``acomplete_structured``. - -Provider/model/agent_id/profile are explicit keyword arguments mirroring the -host config shape (``model.provider`` + ``model.model``) — no embedded slugs. -The host owns routing, auth, timeouts, and fallback; the plugin never sees raw -tokens or keys. Every override knob is gated by per-plugin trust flags:: - - plugins: - entries: - my-plugin: - llm: - allow_provider_override: true - allow_model_override: true - allowed_providers: [openrouter, anthropic] # optional - allowed_models: [openai/gpt-4o-mini] # optional - allow_agent_id_override: false - allow_profile_override: false - allow_task_override: false # borrow the host's built-in aux tasks - -The gate is fail-closed: a missing config block means "no overrides". - -``task=`` routes a call through a plugin-registered auxiliary model slot -(``ctx.register_auxiliary_task``). A plugin may always name a slot it -registered itself; ``allow_task_override`` additionally lets it use the host's -*built-in* auxiliary tasks. A foreign or unknown key is rejected loudly -(error + logged warning), never silently downgraded to the main model. - -Backed by :func:`agent.auxiliary_client.call_llm`. +``complete`` / ``complete_structured`` (text + image inputs, JSON schema validation) +and their async siblings. Provider/model/agent_id/profile are explicit keyword +arguments mirroring the host config shape; the host owns routing, auth, timeouts +and fallback, so the plugin never sees raw tokens or keys. Every override knob is +gated by the per-plugin ``plugins.entries..llm.allow_*_override`` trust flags +(fail-closed: a missing block means "no overrides"). Backed by +:func:`agent.auxiliary_client.call_llm`. """ from __future__ import annotations @@ -48,9 +21,7 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Uni logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# Public dataclasses -# --------------------------------------------------------------------------- +# -- public dataclasses ------------------------------------------------------- @dataclass @@ -63,7 +34,7 @@ class PluginLlmTextInput: @dataclass class PluginLlmImageInput: - """Image block. Provide ``data`` (raw bytes) or ``url`` (http(s)/data: URL). + """Image block: ``data`` (raw bytes) or ``url`` (http(s)/data: URL). ``mime_type`` is required for non-PNG bytes to render across providers.""" data: Optional[bytes] = None @@ -74,16 +45,12 @@ class PluginLlmImageInput: PluginLlmInput = Union[PluginLlmTextInput, PluginLlmImageInput, Dict[str, Any]] -"""A single structured input block: one of the dataclasses above or a plain dict -of the same shape (``{"type": "text", "text": ...}`` / -``{"type": "image", "data": , "mime_type": ..., "file_name": ...}`` / -``{"type": "image", "url": ...}``).""" +"""One structured input block: a dataclass above or a plain dict of the same shape.""" @dataclass class PluginLlmUsage: - """Token + cost usage. All fields optional — providers differ on what they - return. ``cost_usd`` is the host's best estimate.""" + """Token + cost usage; every field optional. ``cost_usd`` is the host's best estimate.""" input_tokens: int = 0 output_tokens: int = 0 @@ -109,9 +76,8 @@ class PluginLlmCompleteResult: class PluginLlmStructuredResult: """Result of :meth:`PluginLlm.complete_structured`. - ``parsed`` is set only when JSON output was requested (``json_mode`` or - ``json_schema``) AND the response was valid JSON; ``content_type`` is then - ``"json"``, otherwise ``"text"``.""" + ``parsed`` is set only when JSON output was requested AND the response was + valid JSON; ``content_type`` is then ``"json"``, otherwise ``"text"``.""" text: str provider: str @@ -123,9 +89,7 @@ class PluginLlmStructuredResult: audit: Dict[str, Any] = field(default_factory=dict) -# --------------------------------------------------------------------------- -# Trust gate -# --------------------------------------------------------------------------- +# -- trust gate --------------------------------------------------------------- @dataclass(frozen=True) @@ -157,7 +121,6 @@ _OVERRIDE_FLAGS = ( def _normalize_ref(raw: str) -> str: - """Lower-case + strip whitespace. Used for allowlist matching.""" return (raw or "").strip().lower() @@ -167,16 +130,12 @@ def _coerce_allowlist(raw: Any) -> tuple[Optional[frozenset], bool]: if not isinstance(raw, list): return None, False normalized = [_normalize_ref(item) for item in raw if isinstance(item, str)] - allow_any = "*" in normalized - cleaned = {item for item in normalized if item and item != "*"} - return frozenset(cleaned), allow_any + return frozenset(item for item in normalized if item and item != "*"), "*" in normalized def _resolve_trust_policy(plugin_id: str) -> _TrustPolicy: - """Read ``plugins.entries..llm`` from config.yaml. - - Missing config → fully restrictive policy. Resolved per call (not cached) - so config edits take effect without restarting the agent.""" + """Read ``plugins.entries..llm`` from config.yaml (missing → fully + restrictive). Resolved per call so config edits apply without a restart.""" if not plugin_id: return _TrustPolicy(plugin_id="") @@ -209,7 +168,6 @@ class PluginLlmTrustError(PermissionError): def _denied(plugin_id: str, what: str, flag: str) -> PluginLlmTrustError: - """Uniform "flag not set" trust error.""" return PluginLlmTrustError( f"Plugin {plugin_id!r} cannot {what} " f"(set plugins.entries.{plugin_id}.llm.{flag} to true to allow)." @@ -217,8 +175,7 @@ def _denied(plugin_id: str, what: str, flag: str) -> PluginLlmTrustError: def _gate_ref_override(policy: _TrustPolicy, kind: str, requested: str) -> str: - """Gate a ``provider`` / ``model`` override: trust flag, then optional - allowlist. Returns the stripped value or raises.""" + """Gate a ``provider`` / ``model`` override: trust flag, then optional allowlist.""" if not getattr(policy, f"allow_{kind}_override"): raise _denied(policy.plugin_id, f"override the {kind}", f"allow_{kind}_override") allowed = getattr(policy, f"allowed_{kind}s") @@ -242,9 +199,8 @@ def _check_overrides( policy: _TrustPolicy, *, requested_provider: Optional[str], requested_model: Optional[str], requested_agent_id: Optional[str], requested_profile: Optional[str], ) -> tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: - """Apply the trust gate; each override is gated independently, in the order - provider, model, agent_id, profile. Returns ``(provider, model, agent_id, - profile)`` (agent_id unstripped) or raises :class:`PluginLlmTrustError`.""" + """Gate each override independently, in the order provider, model, agent_id, + profile. Returns ``(provider, model, agent_id, profile)`` (agent_id unstripped).""" final_provider = _gate_ref_override(policy, "provider", requested_provider) if requested_provider else None final_model = _gate_ref_override(policy, "model", requested_model) if requested_model else None for kind, requested in (("agent_id", requested_agent_id), ("profile", requested_profile)): @@ -255,12 +211,12 @@ def _check_overrides( def _resolve_task_ownership(plugin_id: str) -> tuple[frozenset, frozenset]: - """Return ``(owned_keys, builtin_keys)`` for the task trust gate. + """``(owned_keys, builtin_keys)`` for the task trust gate. - Imports are lazy (circular import at plugin discovery). An unreadable - registry yields empty sets, failing the gate closed. Ownership matches on - the canonical id ``ctx.llm`` is bound to (``manifest.key or manifest.name``), - which is what ``register_auxiliary_task`` stores as the entry's ``plugin``.""" + Imports are lazy (circular import at plugin discovery); an unreadable registry + yields empty sets, failing the gate closed. Ownership matches on the canonical id + ``ctx.llm`` is bound to (``manifest.key or manifest.name``), which is what + ``register_auxiliary_task`` stores as the entry's ``plugin``.""" owned: set = set() builtin: set = set() try: @@ -289,15 +245,12 @@ def _check_task( ) -> Optional[str]: """Validate a plugin's requested auxiliary ``task`` key. - * unset / ``""`` / ``"auto"`` → ``None`` (main-model path). - * a key the plugin registered itself → allowed. - * a built-in key → allowed only with ``allow_task_override``. - * anything else → raises + logs a warning. Never silently downgraded to - ``auto``: that would mask the misconfiguration and could route to a main - model the user steered elsewhere on purpose.""" - if not requested_task: - return None - task = requested_task.strip() + unset / ``""`` / ``"auto"`` → ``None`` (main-model path); a key the plugin + registered itself → allowed; a built-in key → only with ``allow_task_override``; + anything else raises + logs. Never silently downgraded to ``auto``: that would + mask the misconfiguration and could route to a main model the user steered + elsewhere on purpose.""" + task = (requested_task or "").strip() if not task or task.lower() == "auto": return None @@ -323,9 +276,7 @@ def _check_task( ) -# --------------------------------------------------------------------------- -# Input normalization -# --------------------------------------------------------------------------- +# -- input normalization ------------------------------------------------------ def _normalize_input_block(block: PluginLlmInput) -> Dict[str, Any]: @@ -378,10 +329,9 @@ def _build_structured_messages( schema_name: Optional[str], system_prompt: Optional[str], ) -> List[Dict[str, Any]]: - """Build OpenAI-style messages for a structured call: optional system - message (prompt + JSON-only directive), then a user message whose first - text part is the instructions (+ schema name / JSON schema) followed by - the input blocks.""" + """OpenAI-style messages for a structured call: optional system message (prompt + + JSON-only directive), then a user message whose first text part is the + instructions (+ schema name / JSON schema) followed by the input blocks.""" messages: List[Dict[str, Any]] = [] sys_parts: List[str] = [system_prompt.strip()] if system_prompt else [] if json_mode or json_schema is not None: @@ -402,7 +352,6 @@ def _build_structured_messages( schema_text = str(json_schema) header = f"{header}\n\nJSON schema:\n{schema_text}" user_parts: List[Dict[str, Any]] = [{"type": "text", "text": header}] - for block in inputs: norm = _normalize_input_block(block) # always "text" or "image" user_parts.append({"type": "text", "text": norm["text"]} if norm["type"] == "text" else _image_part(norm)) @@ -410,16 +359,14 @@ def _build_structured_messages( return messages -# --------------------------------------------------------------------------- -# JSON parsing -# --------------------------------------------------------------------------- +# -- JSON parsing / response extraction -------------------------------------- _FENCE_RE = re.compile(r"```(?:json)?\s*(.+?)```", re.DOTALL | re.IGNORECASE) def _strip_code_fences(text: str) -> str: - """Return the first fenced code block's body, or the stripped text when unfenced.""" + """The first fenced code block's body, or the stripped text when unfenced.""" match = _FENCE_RE.search(text) return match.group(1).strip() if match else text.strip() @@ -427,10 +374,9 @@ def _strip_code_fences(text: str) -> str: def _parse_structured_text( *, text: str, json_mode: bool, json_schema: Optional[Any] ) -> tuple[Optional[Any], str]: - """Return ``(parsed, content_type)``: ``"json"`` when parsing (and schema - validation, if a schema was given) succeeded, ``"text"`` otherwise. - Schema violations raise ``ValueError``; a missing ``jsonschema`` package - skips validation with a debug log.""" + """``(parsed, content_type)``: ``"json"`` when parsing (and schema validation, if + given) succeeded, ``"text"`` otherwise. Schema violations raise ``ValueError``; + a missing ``jsonschema`` package skips validation with a debug log.""" if not (json_mode or json_schema is not None) or not text: return None, "text" @@ -453,15 +399,10 @@ def _parse_structured_text( return parsed, "json" -# --------------------------------------------------------------------------- -# Response extraction -# --------------------------------------------------------------------------- - - def _extract_usage(response: Any) -> PluginLlmUsage: - """Pull token usage out of an OpenAI-shaped response, tolerating provider - naming differences (Anthropic via the aux adapter: ``prompt_tokens`` / - ``completion_tokens``; direct OpenAI adds ``cache_read_input_tokens``).""" + """Token usage from an OpenAI-shaped response, tolerating provider naming + (``prompt_tokens``/``completion_tokens`` vs ``input_tokens``/``output_tokens``, + ``cache_read_input_tokens`` vs ``cache_read_tokens``).""" usage = PluginLlmUsage() raw = getattr(response, "usage", None) if raw is None: @@ -485,7 +426,7 @@ def _extract_usage(response: Any) -> PluginLlmUsage: def _extract_text(response: Any) -> str: - """Pull the assistant text out of an OpenAI-shaped response object.""" + """Assistant text of an OpenAI-shaped response (string or text-part list content).""" try: content = getattr(response.choices[0].message, "content", None) if isinstance(content, str): @@ -518,12 +459,11 @@ def _resolve_attribution( response: Any, route_info: Optional[Dict[str, str]] = None, ) -> tuple[str, str]: - """Decide what to record as ``result.provider`` / ``result.model``. + """``(provider, model)`` to record on the result. - Provider: route selected by ``auxiliary_client`` > explicit override > - current main provider > ``"auto"``. Model: ``response.model`` (providers - return the canonical id that actually ran, e.g. ``gpt-4o-2024-08-06``) > - route > override > current main model > ``"default"``.""" + Provider: route selected by ``auxiliary_client`` > explicit override > current + main provider > ``"auto"``. Model: ``response.model`` (the canonical id that + actually ran) > route > override > current main model > ``"default"``.""" route_info = route_info or {} provider = route_info.get("provider") or provider_override or _main_config_value("_read_main_provider", "auto") response_model = getattr(response, "model", None) @@ -532,14 +472,12 @@ def _resolve_attribution( return provider, route_info.get("model") or model_override or _main_config_value("_read_main_model", "default") -# --------------------------------------------------------------------------- -# PluginLlm facade -# --------------------------------------------------------------------------- +# -- PluginLlm facade --------------------------------------------------------- def _json_response_format(*, json_mode: bool, json_schema: Optional[Any]) -> Optional[Dict[str, Any]]: - """``extra_body.response_format`` for the request; falls back to - ``json_object`` without a schema so schema-blind providers still get a hint.""" + """``extra_body.response_format``; falls back to ``json_object`` without a + schema so schema-blind providers still get a hint.""" if json_schema is not None: schema = {"name": "plugin_structured_output", "schema": json_schema, "strict": False} return {"response_format": {"type": "json_schema", "json_schema": schema}} @@ -552,8 +490,7 @@ def _structured_spec( name: str, instructions: str, input: Sequence[PluginLlmInput], system_prompt: Optional[str], json_mode: bool, json_schema: Optional[Any], schema_name: Optional[str], ) -> Dict[str, Any]: - """Argument check for the structured methods (runs before the trust gate); - returns the spec ``_gate`` / ``_finish`` consume.""" + """Argument check for the structured methods (runs before the trust gate).""" if not instructions or not instructions.strip(): raise ValueError(f"{name} requires non-empty instructions") if not input: @@ -568,13 +505,10 @@ class PluginLlm: """Host-owned LLM access for one trusted plugin. Constructed by :class:`hermes_cli.plugins.PluginContext` and exposed as - ``ctx.llm``; the constructor binds plugin identity for trust enforcement, - so plugins should not instantiate it directly. - - Every public method is ``_gate`` (trust checks → call kwargs) → - ``_invoke_*`` (host ``call_llm`` or injected caller) → ``_finish`` - (result + audit log); the sync/async and plain/structured variants differ - only in which pieces they pass through.""" + ``ctx.llm``; the constructor binds plugin identity for trust enforcement, so + plugins should not instantiate it directly. Every public method is ``_gate`` + (trust checks → call kwargs) → ``_invoke_*`` (host ``call_llm`` or injected + caller) → ``_finish`` (result + audit log).""" def __init__( self, @@ -589,7 +523,7 @@ class PluginLlm: self._sync_caller = sync_caller self._async_caller = async_caller - # -- public sync API ---------------------------------------------------- + # -- public API ----------------------------------------------------------- def complete( self, @@ -607,10 +541,9 @@ class PluginLlm: ) -> PluginLlmCompleteResult: """Run a host-owned chat completion against the user's active model. - ``messages`` is the standard OpenAI shape. ``provider``/``model``/ - ``agent_id``/``profile`` are each gated by - ``plugins.entries..llm.allow_*_override``. ``task`` routes through - a plugin-registered auxiliary slot (see :func:`_check_task`).""" + ``provider``/``model``/``agent_id``/``profile`` are each gated by + ``plugins.entries..llm.allow_*_override``. ``task`` routes through a + plugin-registered auxiliary slot (see :func:`_check_task`).""" agent, kw = self._gate(provider, model, agent_id, profile, task, messages, temperature, max_tokens, timeout) return self._finish("complete", agent, kw, self._invoke_sync(kw), purpose) @@ -637,14 +570,11 @@ class PluginLlm: ``input`` accepts text and image blocks. With ``json_mode=True`` or a ``json_schema`` the response is parsed (and validated when the optional - ``jsonschema`` package is installed) into ``result.parsed``. - ``task`` routes as in :meth:`complete`.""" + ``jsonschema`` package is installed) into ``result.parsed``.""" spec = _structured_spec("complete_structured", instructions, input, system_prompt, json_mode, json_schema, schema_name) agent, kw = self._gate(provider, model, agent_id, profile, task, None, temperature, max_tokens, timeout, spec) return self._finish("complete_structured", agent, kw, self._invoke_sync(kw), purpose, spec) - # -- public async API --------------------------------------------------- - async def acomplete( self, messages: List[Dict[str, Any]], @@ -687,7 +617,7 @@ class PluginLlm: agent, kw = self._gate(provider, model, agent_id, profile, task, None, temperature, max_tokens, timeout, spec) return self._finish("acomplete_structured", agent, kw, await self._invoke_async(kw), purpose, spec) - # -- shared core -------------------------------------------------------- + # -- shared core ---------------------------------------------------------- def _gate( self, @@ -702,13 +632,11 @@ class PluginLlm: timeout: Optional[float], spec: Optional[Dict[str, Any]] = None, ) -> tuple[Optional[str], Dict[str, Any]]: - """Run the trust gate (task first, then overrides), then — for a - structured ``spec`` — build messages/response_format (input-shape errors - surface only after trust passes). Returns the effective agent id - (result-only) and the call kwargs handed to ``_invoke_*`` / an injected - caller, in the documented order: messages, provider_override, - model_override, profile_override, temperature, max_tokens, timeout, - extra_body, task.""" + """Trust gate (task first, then overrides), then — for a structured ``spec`` — + build messages/response_format (input-shape errors surface only after trust + passes). Returns the effective agent id and the call kwargs, in the documented + order: messages, provider_override, model_override, profile_override, + temperature, max_tokens, timeout, extra_body, task.""" policy = self._policy_loader(self._plugin_id) eff_task = _check_task(policy, plugin_id=self._plugin_id, requested_task=task) eff_provider, eff_model, eff_agent, eff_profile = _check_overrides( @@ -761,13 +689,13 @@ class PluginLlm: logger.info(fmt + "tokens=%d", *log_args, usage.total_tokens) return cls(**fields, audit=audit) - # -- host invocation --------------------------------------------------- + # -- host invocation ------------------------------------------------------ @staticmethod def _host_kwargs(kw: Dict[str, Any]) -> tuple[Dict[str, Any], Optional[Dict[str, str]]]: - """Translate call kwargs into ``call_llm`` kwargs. The auth profile - rides in ``extra_body.metadata.auth_profile``; ``route_info`` is only - requested when routing through a task slot.""" + """Call kwargs → ``call_llm`` kwargs. The auth profile rides in + ``extra_body.metadata.auth_profile``; ``route_info`` is only requested when + routing through a task slot.""" merged_extra = dict(kw["extra_body"] or {}) if kw["profile_override"]: merged_extra.setdefault("metadata", {})["auth_profile"] = kw["profile_override"] @@ -793,10 +721,9 @@ class PluginLlm: return provider, model, response def _invoke_sync(self, kw: Dict[str, Any]) -> tuple[str, str, Any]: - """Invoke the host's ``call_llm`` (lazy import: circular deps at plugin - discovery) and return ``(provider, model, response)``. ``task`` is - already trust-checked; ``None`` keeps the main model. An injected - ``sync_caller`` replaces the whole path and receives the call kwargs.""" + """Host ``call_llm`` (lazy import: circular deps at plugin discovery) → + ``(provider, model, response)``. An injected ``sync_caller`` replaces the + whole path and receives the call kwargs.""" if self._sync_caller is not None: return self._sync_caller(**kw) from agent.auxiliary_client import call_llm @@ -812,11 +739,6 @@ class PluginLlm: return self._attributed(kw, await async_call_llm(**call_kw), route_info) -# --------------------------------------------------------------------------- -# Test helpers -# --------------------------------------------------------------------------- - - def make_plugin_llm_for_test( *, plugin_id: str, @@ -824,8 +746,8 @@ def make_plugin_llm_for_test( sync_caller: Optional[Callable[..., Any]] = None, async_caller: Optional[Callable[..., Awaitable[Any]]] = None, ) -> PluginLlm: - """:class:`PluginLlm` with an injected policy and caller (no config.yaml, - no provider). Not part of the public plugin API.""" + """:class:`PluginLlm` with an injected policy and caller (no config.yaml, no + provider). Not part of the public plugin API.""" return PluginLlm(plugin_id=plugin_id, policy_loader=lambda _pid: policy, sync_caller=sync_caller, async_caller=async_caller) diff --git a/agent/provider_base.py b/agent/provider_base.py index 470b1c0a6c..68ab819e28 100644 --- a/agent/provider_base.py +++ b/agent/provider_base.py @@ -1,12 +1,10 @@ """Shared base classes for the pluggable-backend provider ABCs. -Every tool-provider ABC (browser, TTS, image/video gen, transcription, web -search, terminal env) shares the same identity + ``hermes tools`` picker -surface; the previous per-ABC copies of these defaults were byte-identical. -Concrete ABCs subclass :class:`ProviderBase` (or :class:`CatalogProviderBase` -when the backend also exposes a model catalog and is available by default) and -add only their domain methods. Plugins keep subclassing the concrete ABC, so -``isinstance`` checks and abstract-method sets are unchanged. +Every tool-provider ABC shares the same identity + ``hermes tools`` picker surface. +Concrete ABCs subclass :class:`ProviderBase` (or :class:`CatalogProviderBase` when +the backend also exposes a model catalog and is available by default) and add only +their domain methods. Plugins keep subclassing the concrete ABC, so ``isinstance`` +checks and abstract-method sets are unchanged. """ from __future__ import annotations @@ -36,16 +34,10 @@ class ProviderBase(abc.ABC): """Provider row for the ``hermes tools`` picker. Shape: ``{"name", "badge", "tag", "env_vars": [{"key", "prompt", "url"}, ...]}`` - (browser providers may add ``"post_setup"``). Default: a minimal entry - derived from ``display_name`` with no env vars — override to expose API - key prompts and badges. + (browser providers may add ``"post_setup"``). Override to expose API key + prompts and badges. """ - return { - "name": self.display_name, - "badge": "", - "tag": "", - "env_vars": [], - } + return {"name": self.display_name, "badge": "", "tag": "", "env_vars": []} class CatalogProviderBase(ProviderBase): @@ -59,19 +51,16 @@ class CatalogProviderBase(ProviderBase): def is_available(self) -> bool: """True when this provider can service calls (API key present, SDK importable). - Default True. Must NOT raise and must NOT make network calls — the picker - and ``hermes setup`` call it on every paint. + Must NOT raise and must NOT make network calls — the picker and + ``hermes setup`` call it on every paint. """ return True def list_models(self) -> List[Dict[str, Any]]: - """Model catalog entries (``{"id": ..., "display": ...}`` plus optional - provider-specific keys). Default: empty (no user-selectable models).""" + """Model catalog entries (``{"id": ..., "display": ...}`` + provider-specific keys).""" return [] def default_model(self) -> Optional[str]: """Id of the first catalog entry, or None when the catalog is empty.""" models = self.list_models() - if models: - return models[0].get("id") - return None + return models[0].get("id") if models else None diff --git a/agent/provider_media.py b/agent/provider_media.py index ba69a003fd..d3dfffe553 100644 --- a/agent/provider_media.py +++ b/agent/provider_media.py @@ -1,10 +1,10 @@ -"""Shared ``$HERMES_HOME/cache//`` materialisation helpers for the -image/video generation provider ABCs. +"""``$HERMES_HOME/cache//`` materialisation helpers for the image/video +generation provider ABCs. -Several backends (xAI, OpenAI, DeepInfra, FAL) return *ephemeral* delivery URLs -that expire before a downstream consumer (Telegram ``send_photo``, browser -fetch) can resolve them, so providers materialise the bytes locally at -tool-completion time. Filenames are ``__.``. +Several backends return *ephemeral* delivery URLs that expire before a downstream +consumer (Telegram ``send_photo``, browser fetch) can resolve them, so providers +materialise the bytes locally at tool-completion time. Filenames are +``__.``. """ from __future__ import annotations @@ -59,12 +59,11 @@ def save_url( ) -> Path: """Stream-download *url* into the cache with a size cap. - The extension comes from the response ``Content-Type`` (a small explicit - table — never inherit a type that points at HTML/JSON from a degenerate - response), then the URL suffix (some CDNs return - ``application/octet-stream``), then *default_extension*. Raises on any - network / HTTP / oversize / empty error so callers can fall back to the bare - URL; a partial file is never left behind. + The extension comes from the response ``Content-Type`` (an explicit table — + never inherit a type pointing at HTML/JSON from a degenerate response), then + the URL suffix (some CDNs return ``application/octet-stream``), then + *default_extension*. Raises on any network / HTTP / oversize / empty error so + callers can fall back to the bare URL; a partial file is never left behind. """ import requests @@ -75,15 +74,11 @@ def save_url( extension = content_types.get(content_type) if extension is None: url_path = url.split("?", 1)[0].lower() - for ext in url_extensions: - if url_path.endswith(f".{ext}"): - extension = "jpg" if ext == "jpeg" else ext - break - if extension is None: - extension = default_extension - + extension = next( + ("jpg" if ext == "jpeg" else ext for ext in url_extensions if url_path.endswith(f".{ext}")), + default_extension, + ) path = cache_path(kind, prefix, extension) - bytes_written = 0 with path.open("wb") as fh: for chunk in response.iter_content(chunk_size=chunk_size): diff --git a/agent/provider_projection.py b/agent/provider_projection.py index b3788941e5..dc56b98530 100644 --- a/agent/provider_projection.py +++ b/agent/provider_projection.py @@ -1,23 +1,13 @@ """Fold an agent-as-provider's own activity back into Hermes' turn state. -Some providers are *agents* (an ACP CLI behind a client shim; the codex -app-server takes an analogous path in ``agent/codex_runtime.py``): they run -their own tools inside their own session, so by the time Hermes sees the -response that work is done. Those calls must never come back as pending -``tool_calls`` (Hermes would re-run finished work), but two subsystems go blind -if they are merely summarised into ``reasoning``: - -* the **self-improvement loop**, which replays ``messages`` to distil memories - and skills; -* the **skill-review nudge**, whose ``_iters_since_skill`` counter only moves on - Hermes tool iterations. - -So the client hands both back on the completion object — -``hermes_projected_messages`` (completed ``assistant(tool_calls=[…])`` + -``tool(result)`` rows) and ``hermes_provider_tool_iterations`` — and this helper -applies them. Ordinary OpenAI-compatible clients set neither and are unaffected. -The splice is append-only through ``append_message`` so rows carry a timestamp -and persist like any other live-transcript append. +Agent providers (ACP CLI shims, the codex app-server) run their own tools, so that +work must never come back as pending ``tool_calls`` (Hermes would re-run it) — but +the self-improvement loop (replays ``messages``) and the skill-review nudge +(``_iters_since_skill`` counter) go blind if it is merely summarised into +``reasoning``. The client hands back ``hermes_projected_messages`` (completed +assistant/tool rows) and ``hermes_provider_tool_iterations`` on the completion +object; this helper applies them append-only via ``append_message`` (timestamped, +persisted). Ordinary OpenAI-compatible clients set neither and are unaffected. """ from __future__ import annotations @@ -46,9 +36,7 @@ def splice_provider_projection( append_message(messages, row) if rows: logger.debug( - "spliced %d provider-projected transcript row(s) from %s", - len(rows), - getattr(agent, "provider", "?"), + "spliced %d provider-projected transcript row(s) from %s", len(rows), getattr(agent, "provider", "?"), ) try: diff --git a/agent/provider_registry.py b/agent/provider_registry.py index f8c00ce0a4..27b859d820 100644 --- a/agent/provider_registry.py +++ b/agent/provider_registry.py @@ -1,14 +1,12 @@ """Shared engine behind the ``agent.*_registry`` provider registries. -Every pluggable-backend registry (browser, TTS, image/video gen, transcription, -web search, terminal env) has the same shape: a global name->provider map plus -per-profile *scoped* maps (multiplexed gateways), a lock, registration with -re-registration logging, and the snapshot/restore pair that -:mod:`hermes_cli.plugins` uses to unwind a plugin's registrations. Each -``*_registry`` module instantiates one :class:`ProviderRegistry` and re-exports -its bound methods under the historical module-level names via -:meth:`ProviderRegistry.export`, so call sites, ``patch("agent.x_registry.get_provider")`` -targets, and the ``_providers`` / ``_scoped_providers`` / ``_lock`` test hooks are unchanged. +Every pluggable-backend registry has the same shape: a global name->provider map +plus per-profile *scoped* maps (multiplexed gateways), a lock, registration with +re-registration logging, and the snapshot/restore pair :mod:`hermes_cli.plugins` +uses to unwind a plugin. Each ``*_registry`` module instantiates one +:class:`ProviderRegistry` and re-exports its bound methods under the historical +module-level names via :meth:`ProviderRegistry.export`, so ``patch("agent.x_registry.get_provider")`` +targets and the ``_providers`` / ``_scoped_providers`` / ``_lock`` test hooks are unchanged. """ from __future__ import annotations @@ -33,14 +31,11 @@ def lower_key(name: str) -> str: class ProviderRegistry(Generic[P]): """Global + per-scope provider map with plugin snapshot/restore support. - Args: - label: Human label used in log/error strings (``"Browser"``, ``"TTS"``). - provider_cls: ABC every registered instance must satisfy (TypeError otherwise). - logger: The owning module's logger, so record names stay per-registry. - normalize: Key normalizer — ``strip_key`` or ``lower_key`` (case-insensitive - registries mirror how their dispatcher normalizes the configured name). - builtin_names: Reserved names owned by in-tree implementations; a collision - calls ``on_builtin_collision(key)`` and, if that returns, skips registration. + ``normalize`` is ``strip_key`` or ``lower_key`` (case-insensitive registries mirror + how their dispatcher normalizes the configured name). ``builtin_names`` are reserved + for in-tree implementations; a collision calls ``on_builtin_collision(key)`` and, if + that returns, skips registration. ``logger`` is the owning module's so record names + stay per-registry. """ def __init__( @@ -67,8 +62,6 @@ class ProviderRegistry(Generic[P]): # "TTS provider" but "Registered browser provider": acronyms keep their case. self._log_label = label if label.isupper() else label[0].lower() + label[1:] - # -- internal helpers (caller holds the lock) --------------------------- - def _target(self, scope: Optional[str], *, create: bool) -> Dict[str, P]: if scope is None: return self._providers @@ -82,8 +75,6 @@ class ProviderRegistry(Generic[P]): else: self._scoped_generations[scope] = self._scoped_generations.get(scope, 0) + 1 - # -- registration ------------------------------------------------------- - def register(self, provider: P, *, scope: Optional[str] = None) -> None: """Register a provider; same-name re-registration overwrites (hot reload).""" if not isinstance(provider, self.provider_cls): @@ -107,17 +98,13 @@ class ProviderRegistry(Generic[P]): self._bump(scope) if existing is not None: self.logger.debug( - f"{self.label} provider '%s' re-registered (was %r)", - key, type(existing).__name__, + f"{self.label} provider '%s' re-registered (was %r)", key, type(existing).__name__, ) else: self.logger.debug( - f"Registered {self._log_label} provider '%s' (%s)", - key, type(provider).__name__, + f"Registered {self._log_label} provider '%s' (%s)", key, type(provider).__name__, ) - # -- lookup --------------------------------------------------------------- - def merged(self, scope: Optional[str] = None) -> Dict[str, P]: """Global map overlaid with the active profile's scoped map (a copy).""" with self._lock: @@ -146,8 +133,6 @@ class ProviderRegistry(Generic[P]): with self._lock: return self._generation, self._scoped_generations.get(active_scope, 0) - # -- plugin unload support (hermes_cli.plugins) ----------------------------- - def snapshot_registration(self, name: str, *, scope: Optional[str] = None) -> Optional[P]: """Exact-slot lookup (no global fallback) used to detect plugin ownership.""" with self._lock: @@ -180,14 +165,8 @@ class ProviderRegistry(Generic[P]): self._generation += 1 def export(self, namespace: Dict[str, Any]) -> None: - """Bind the historical module-level API into a ``*_registry`` module. - - Installs ``register_provider``/``list_providers``/``get_provider``/ - ``snapshot_registration``/``restore_registration``/``registry_generation``/ - ``_reset_for_tests`` plus the ``_providers``/``_scoped_providers``/``_lock`` - test hooks, so ``patch("agent.x_registry.get_provider")`` and direct - ``_providers`` manipulation in tests keep working unchanged. - """ + """Bind the historical module-level API (+ ``_providers``/``_scoped_providers``/ + ``_lock`` test hooks) into a ``*_registry`` module namespace.""" namespace.update( _providers=self._providers, _scoped_providers=self._scoped_providers, @@ -227,10 +206,9 @@ def configured_provider_name(section: str, logger: logging.Logger) -> Optional[s cfg = load_config_readonly() block = cfg.get(section) if isinstance(cfg, dict) else None - if isinstance(block, dict): - raw = block.get("provider") - if isinstance(raw, str) and raw.strip(): - configured = raw.strip() + raw = block.get("provider") if isinstance(block, dict) else None + if isinstance(raw, str) and raw.strip(): + configured = raw.strip() except Exception as exc: logger.debug("Could not read %s.provider from config: %s", section, exc) if configured: From 28a974aef5c9be08dcaeda3a0510bd621573fd77 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:44:36 -0700 Subject: [PATCH 3/9] refactor(agent): pack multi-line calls/signatures in r3-06-C files (AST-identical) --- agent/moa_loop.py | 207 ++++++++++------------------------- agent/moa_trace.py | 31 ++---- agent/nous_rate_guard.py | 12 +- agent/plugin_llm.py | 163 ++++++++------------------- agent/provider_media.py | 15 +-- agent/provider_projection.py | 4 +- agent/provider_registry.py | 28 ++--- 7 files changed, 124 insertions(+), 336 deletions(-) diff --git a/agent/moa_loop.py b/agent/moa_loop.py index 4a980fb604..0f02b39f40 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -98,8 +98,7 @@ def _redact_trace_accounting(acct: Any) -> Any: if not isinstance(acct, _RefAccounting): return acct return replace( - acct, - messages=_redact_trace_messages(acct.messages), + acct, messages=_redact_trace_messages(acct.messages), output=_redact_reference_text(acct.output), ) @@ -297,10 +296,7 @@ def _with_cache_disabled(runtime: dict[str, Any], cache_disabled: Any) -> dict[s def _maybe_apply_moa_cache_control( - messages: list[dict[str, Any]], - runtime: dict[str, Any], - *, - cache_disabled: bool | None = None, + messages: list[dict[str, Any]], runtime: dict[str, Any], *, cache_disabled: bool | None = None, cache_ttl: str | None = None, ) -> list[dict[str, Any]]: """Apply cache_control to an advisor/aggregator request when its route honors it. @@ -328,10 +324,8 @@ def _maybe_apply_moa_cache_control( model = runtime.get("model") or "" # blank_cache_policy_stub is the only sanctioned stub (carries _cache_disabled). should_cache, native_layout = anthropic_prompt_cache_policy( - blank_cache_policy_stub(cache_disabled), - provider=provider, - base_url=runtime.get("base_url") or "", - api_mode=runtime.get("api_mode") or "", + blank_cache_policy_stub(cache_disabled), provider=provider, + base_url=runtime.get("base_url") or "", api_mode=runtime.get("api_mode") or "", model=model, ) if not should_cache: @@ -369,11 +363,8 @@ def _price_reference_response( usage = CanonicalUsage() try: cost = estimate_usage_cost( - slot.get("model") or "", - usage, - provider=runtime.get("provider"), - base_url=runtime.get("base_url"), - api_key=runtime.get("api_key"), + slot.get("model") or "", usage, provider=runtime.get("provider"), + base_url=runtime.get("base_url"), api_key=runtime.get("api_key"), ) return usage, cost.amount_usd, cost.status, cost.source except Exception: # pragma: no cover - defensive @@ -381,14 +372,9 @@ def _price_reference_response( def _run_reference( - slot: dict[str, Any], - ref_messages: list[dict[str, Any]], - *, - temperature: float | None = None, - max_tokens: int | None = None, - reference_timeout: float | None = None, - context_length_cache: Any = None, - cache_disabled: bool | None = None, + slot: dict[str, Any], ref_messages: list[dict[str, Any]], *, temperature: float | None = None, + max_tokens: int | None = None, reference_timeout: float | None = None, + context_length_cache: Any = None, cache_disabled: bool | None = None, cache_ttl: str | None = None, ) -> tuple[str, str, Any]: """Call one reference model; return ``(label, text, accounting)``. Never raises: @@ -396,8 +382,7 @@ def _run_reference( label = _slot_label(slot) runtime = _slot_runtime(slot) trace_fields = { - "model": slot.get("model"), - "provider": runtime.get("provider") or slot.get("provider"), + "model": slot.get("model"), "provider": runtime.get("provider") or slot.get("provider"), "temperature": temperature, } # The advisory view already stripped the agent's system prompt; this is the only one. @@ -405,10 +390,7 @@ def _run_reference( try: # Trim to THIS model's window (advisors may be smaller than the aggregator). trimmed = _trim_messages_for_reference( - messages, - slot, - runtime, - reserve_output_tokens=max_tokens, + messages, slot, runtime, reserve_output_tokens=max_tokens, context_length_cache=context_length_cache, ) # The advisory view is append-only across iterations, so cache_control lets @@ -427,21 +409,15 @@ def _run_reference( # user's current turn, so mirror the main agent's x-initiator header. extra_headers = {"x-initiator": "user"} response = call_llm( - task="moa_reference", - messages=trimmed, - temperature=temperature, + task="moa_reference", messages=trimmed, temperature=temperature, max_tokens=slot_max_tokens if slot_max_tokens is not None else max_tokens, - timeout=reference_timeout, - reasoning_config=_slot_reasoning_config(slot), - extra_headers=extra_headers, - **runtime, + timeout=reference_timeout, reasoning_config=_slot_reasoning_config(slot), + extra_headers=extra_headers, **runtime, ) output_text = _extract_text(response) or "(empty response)" acct = _RefAccounting( - *_price_reference_response(response, slot, runtime), - messages=trimmed, - output=output_text, - **trace_fields, + *_price_reference_response(response, slot, runtime), messages=trimmed, + output=output_text, **trace_fields, ) return label, output_text, acct except Exception as exc: @@ -460,12 +436,8 @@ _REFERENCE_TRIM_SAFETY_FRACTION = 0.10 def _trim_messages_for_reference( - messages: list[dict[str, Any]], - slot: dict[str, str], - runtime: dict[str, Any], - *, - reserve_output_tokens: int | None = None, - context_length_cache: Any = None, + messages: list[dict[str, Any]], slot: dict[str, str], runtime: dict[str, Any], *, + reserve_output_tokens: int | None = None, context_length_cache: Any = None, ) -> list[dict[str, Any]]: """Trim an advisory request to fit a reference model's context window. @@ -495,10 +467,8 @@ def _trim_messages_for_reference( else: try: context_length = get_model_context_length( - model=model, - base_url=str(runtime.get("base_url") or ""), - api_key=str(runtime.get("api_key") or ""), - provider=provider, + model=model, base_url=str(runtime.get("base_url") or ""), + api_key=str(runtime.get("api_key") or ""), provider=provider, ) except Exception: logger.debug("MoA reference context-length resolution failed for %s", _slot_label(slot)) @@ -560,9 +530,7 @@ def _placeholder_output(slot: dict[str, Any], note: str) -> tuple[str, str, Any] def _settle_interrupted( - futures: dict[Any, int], - results: list, - reference_models: list[dict[str, Any]], + futures: dict[Any, int], results: list, reference_models: list[dict[str, Any]], late_accounting_sink: Any, ) -> None: """Fill every unfinished slot after a user interrupt: cancel never-dispatched @@ -592,15 +560,9 @@ def _settle_interrupted( def _run_references_parallel( - reference_models: list[dict[str, Any]], - ref_messages: list[dict[str, Any]], - *, - temperature: float | None = None, - max_tokens: int | None = None, - progress_callback: Any = None, - reference_timeout: float | None = None, - agent: Any = None, - late_accounting_sink: Any = None, + reference_models: list[dict[str, Any]], ref_messages: list[dict[str, Any]], *, + temperature: float | None = None, max_tokens: int | None = None, progress_callback: Any = None, + reference_timeout: float | None = None, agent: Any = None, late_accounting_sink: Any = None, ) -> list[tuple[str, str, Any]]: """Fan out all reference models in parallel; ``(label, text, _RefAccounting)`` per slot in ``reference_models`` order. @@ -635,15 +597,10 @@ def _run_references_parallel( continue futures[ executor.submit( - propagate_context_to_thread(_run_reference), - slot, - ref_messages, - temperature=temperature, - max_tokens=max_tokens, - reference_timeout=reference_timeout, - context_length_cache=ctx_len_cache, - cache_disabled=cache_disabled, - cache_ttl=cache_ttl, + propagate_context_to_thread(_run_reference), slot, ref_messages, + temperature=temperature, max_tokens=max_tokens, + reference_timeout=reference_timeout, context_length_cache=ctx_len_cache, + cache_disabled=cache_disabled, cache_ttl=cache_ttl, ) ] = idx @@ -871,16 +828,10 @@ def _slot_labels(slots: list[dict[str, Any]]) -> str: def aggregate_moa_context( - *, - user_prompt: str, - api_messages: list[dict[str, Any]], - reference_models: list[dict[str, Any]], - aggregator: dict[str, Any], - temperature: float | None = None, - aggregator_temperature: float | None = None, - reference_max_tokens: int | None = None, - reference_timeout: float | None = None, - degraded_reference_policy: str = "loud", + *, user_prompt: str, api_messages: list[dict[str, Any]], reference_models: list[dict[str, Any]], + aggregator: dict[str, Any], temperature: float | None = None, + aggregator_temperature: float | None = None, reference_max_tokens: int | None = None, + reference_timeout: float | None = None, degraded_reference_policy: str = "loud", agent: Any = None, ) -> str: """Run configured reference models and synthesize their advice (one-shot /moa). @@ -891,12 +842,8 @@ def aggregate_moa_context( """ reference_models = [slot for slot in reference_models if slot.get("enabled", True)] reference_outputs = _run_references_parallel( - reference_models, - _reference_messages(api_messages), - temperature=temperature, - max_tokens=reference_max_tokens, - reference_timeout=reference_timeout, - agent=agent, + reference_models, _reference_messages(api_messages), temperature=temperature, + max_tokens=reference_max_tokens, reference_timeout=reference_timeout, agent=agent, ) successful_outputs, failed_labels = _split_references(reference_outputs) @@ -944,15 +891,11 @@ def aggregate_moa_context( # a third independent MoA call path that otherwise re-bills its full input. agg_messages = _maybe_apply_moa_cache_control( [{"role": "user", "content": synth_prompt}], - _with_cache_disabled(agg_runtime, cache_disabled), - cache_ttl=cache_ttl, + _with_cache_disabled(agg_runtime, cache_disabled), cache_ttl=cache_ttl, ) response = call_llm( - task="moa_aggregator", - messages=agg_messages, - temperature=aggregator_temperature, - reasoning_config=_aggregator_reasoning_config(aggregator), - **agg_runtime, + task="moa_aggregator", messages=agg_messages, temperature=aggregator_temperature, + reasoning_config=_aggregator_reasoning_config(aggregator), **agg_runtime, ) synthesis = _extract_text(response) except Exception as exc: @@ -990,21 +933,17 @@ def _completed_response_as_stream_chunk(response: Any) -> Any: for index, tc in enumerate(raw_tool_calls) ] delta = SimpleNamespace( - content=getattr(message, "content", None), - tool_calls=tool_call_deltas, + content=getattr(message, "content", None), tool_calls=tool_call_deltas, reasoning_content=getattr(message, "reasoning_content", None), reasoning=getattr(message, "reasoning", None), reasoning_details=getattr(message, "reasoning_details", None), ) choice = SimpleNamespace( - index=getattr(first_choice, "index", 0), - delta=delta, + index=getattr(first_choice, "index", 0), delta=delta, finish_reason=getattr(first_choice, "finish_reason", None) or "stop", ) return SimpleNamespace( - id=getattr(response, "id", None), - model=getattr(response, "model", None), - choices=[choice], + id=getattr(response, "id", None), model=getattr(response, "model", None), choices=[choice], usage=getattr(response, "usage", None), ) @@ -1029,10 +968,7 @@ def _attach_reference_guidance(agg_messages: list[dict[str, Any]], guidance: str agg_messages.append({"role": "user", "content": guidance}) -def peel_reference_guidance( - messages: list[dict[str, Any]], - guidance: Any, -) -> list[dict[str, Any]]: +def peel_reference_guidance(messages: list[dict[str, Any]], guidance: Any) -> list[dict[str, Any]]: """Exact inverse of ``_attach_reference_guidance`` (the three attach shapes), so a cache breakpoint never lands on the turn-varying guidance. Inputs are not mutated.""" if not guidance or not messages: @@ -1150,8 +1086,7 @@ class MoAChatCompletions: if agg_output is None and aggregator_output_fallback: agg_output = aggregator_output_fallback save_moa_turn( - session_id=session_id, - preset_name=pending.get("preset", ""), + session_id=session_id, preset_name=pending.get("preset", ""), reference_outputs=pending.get("reference_outputs", []), aggregator_label=pending.get("aggregator_label", ""), aggregator_model=agg_slot.get("model"), @@ -1189,10 +1124,7 @@ class MoAChatCompletions: return {**prepared, "messages": agg_messages} def _plan_aggregator_cache( - self, - agg_messages: list[dict[str, Any]], - tools: Any, - guidance: Any, + self, agg_messages: list[dict[str, Any]], tools: Any, guidance: Any, agg_runtime: dict[str, Any], ) -> tuple[list[dict[str, Any]], Any]: """Cache-breakpoint the aggregator request for its destination. @@ -1294,9 +1226,7 @@ class MoAChatCompletions: return agg_response def _fanout_cache_key( - self, - preset: dict[str, Any], - ref_messages: list[dict[str, Any]], + self, preset: dict[str, Any], ref_messages: list[dict[str, Any]], reference_models: list[dict[str, Any]], ) -> tuple: """Turn-scoped reference cache key per the preset's fan-out cadence. @@ -1353,13 +1283,9 @@ class MoAChatCompletions: ) def _run_fanout( - self, - preset: dict[str, Any], - ref_messages: list[dict[str, Any]], - reference_models: list[dict[str, Any]], - aggregator: dict[str, Any], - aggregator_temperature: Any, - cache_key: tuple, + self, preset: dict[str, Any], ref_messages: list[dict[str, Any]], + reference_models: list[dict[str, Any]], aggregator: dict[str, Any], + aggregator_temperature: Any, cache_key: tuple, ) -> list[tuple[str, str, Any]]: """Cache-MISS path of ``create``: run the advisors, account, trace and emit. @@ -1372,14 +1298,11 @@ class MoAChatCompletions: self._emit("moa.progress", refs_done=done, refs_total=total, label=label) reference_outputs = _run_references_parallel( - reference_models, - ref_messages, + reference_models, ref_messages, temperature=_preset_temperature(preset, "reference_temperature"), - max_tokens=preset.get("reference_max_tokens"), - progress_callback=_progress, + max_tokens=preset.get("reference_max_tokens"), progress_callback=_progress, reference_timeout=float(raw_reference_timeout) if raw_reference_timeout else None, - agent=self._agent, - late_accounting_sink=self._record_late_reference_accounting, + agent=self._agent, late_accounting_sink=self._record_late_reference_accounting, ) if any(text == _INTERRUPTED_REFERENCE_NOTE for _lbl, text, _acct in reference_outputs): # An interrupted fan-out is a partial snapshot: never cache it (a HIT @@ -1402,10 +1325,8 @@ class MoAChatCompletions: else: trace_refs = list(reference_outputs) self._pending_trace = { - "preset": self.preset_name, - "reference_outputs": trace_refs, - "aggregator_slot": aggregator, - "aggregator_temperature": aggregator_temperature, + "preset": self.preset_name, "reference_outputs": trace_refs, + "aggregator_slot": aggregator, "aggregator_temperature": aggregator_temperature, } # Derived from the privacy-redacted trace_refs. try: @@ -1422,29 +1343,21 @@ class MoAChatCompletions: ref_count = len(reference_outputs) for idx, (label, text, _accounting) in enumerate(reference_outputs, start=1): self._emit( - "moa.reference", - index=idx, - count=ref_count, - label=label, + "moa.reference", index=idx, count=ref_count, label=label, text=_redact_reference_text(text) if privacy_mode else text, ) if ref_count: # Phase transition: fan-out complete, aggregator about to act. agg_label = _slot_label(aggregator) self._emit( - "moa.phase", - phase="aggregator", - refs_done=ref_count, - refs_total=ref_count, + "moa.phase", phase="aggregator", refs_done=ref_count, refs_total=ref_count, aggregator=agg_label, ) self._emit("moa.aggregating", aggregator=agg_label, ref_count=ref_count) return reference_outputs def _build_guidance( - self, - reference_outputs: list[tuple[str, str, Any]], - aggregator: dict[str, Any], + self, reference_outputs: list[tuple[str, str, Any]], aggregator: dict[str, Any], degraded_reference_policy: str, ) -> str | None: """Render the reference block attached to the aggregator prompt (None = nothing).""" @@ -1531,9 +1444,7 @@ class MoAChatCompletions: _attach_reference_guidance(agg_messages, guidance) prepared_request = { - "messages": agg_messages, - "guidance": guidance, - "aggregator": aggregator, + "messages": agg_messages, "guidance": guidance, "aggregator": aggregator, "aggregator_temperature": aggregator_temperature, } if api_kwargs.pop("_moa_prepare_only", False): @@ -1591,10 +1502,8 @@ def build_moa_facade(agent, preset_name: Any = None) -> MoAClient: primary, secondary, extra_map = spec try: cb( - event, - str(kwargs.get(primary) or ""), - str(kwargs.get(secondary) or "") if secondary else None, - None, + event, str(kwargs.get(primary) or ""), + str(kwargs.get(secondary) or "") if secondary else None, None, **{out: kwargs.get(src) for out, src in extra_map.items()}, ) except Exception: diff --git a/agent/moa_trace.py b/agent/moa_trace.py index b4ccade104..329b28ab7a 100644 --- a/agent/moa_trace.py +++ b/agent/moa_trace.py @@ -55,10 +55,8 @@ def _slot_trace(acct: Any, label: str) -> dict[str, Any]: input messages and output (not the truncated display preview).""" usage = getattr(acct, "usage", None) return { - "label": label, - **{f: getattr(acct, f, None) for f in _ACCT_FIELDS}, - "input_messages": getattr(acct, "messages", None), - "output": getattr(acct, "output", None), + "label": label, **{f: getattr(acct, f, None) for f in _ACCT_FIELDS}, + "input_messages": getattr(acct, "messages", None), "output": getattr(acct, "output", None), "usage": {f: getattr(usage, f, 0) for f in _USAGE_FIELDS} if usage is not None else {}, **{f: getattr(acct, f, None) for f in _COST_FIELDS}, } @@ -76,16 +74,9 @@ def slot_metrics(acct: Any, label: str, output: Any = None) -> dict[str, Any]: def save_moa_turn( - *, - session_id: Optional[str], - preset_name: str, - reference_outputs: list[tuple[str, str, Any]], - aggregator_label: str, - aggregator_model: Optional[str], - aggregator_provider: Optional[str], - aggregator_temperature: Any, - aggregator_input_messages: Any, - aggregator_output: Optional[str], + *, session_id: Optional[str], preset_name: str, reference_outputs: list[tuple[str, str, Any]], + aggregator_label: str, aggregator_model: Optional[str], aggregator_provider: Optional[str], + aggregator_temperature: Any, aggregator_input_messages: Any, aggregator_output: Optional[str], aggregator_streamed: bool, ) -> None: """Append one full MoA turn record to the session's trace JSONL, if enabled. @@ -113,14 +104,10 @@ def save_moa_turn( "preset": preset_name, "references": [_slot_trace(acct, label) for label, _text, acct in reference_outputs], "aggregator": { - "label": aggregator_label, - "model": aggregator_model, - "provider": aggregator_provider, - "temperature": aggregator_temperature, - "input_messages": aggregator_input_messages, - "output": aggregator_output, - "streamed": aggregator_streamed, - "output_location": output_location, + "label": aggregator_label, "model": aggregator_model, + "provider": aggregator_provider, "temperature": aggregator_temperature, + "input_messages": aggregator_input_messages, "output": aggregator_output, + "streamed": aggregator_streamed, "output_location": output_location, }, } with path.open("a", encoding="utf-8") as f: diff --git a/agent/nous_rate_guard.py b/agent/nous_rate_guard.py index 2e3e95f214..15654598a4 100644 --- a/agent/nous_rate_guard.py +++ b/agent/nous_rate_guard.py @@ -54,9 +54,7 @@ def _parse_reset_seconds(headers: Optional[Mapping[str, str]]) -> Optional[float def record_nous_rate_limit( - *, - headers: Optional[Mapping[str, str]] = None, - error_context: Optional[dict[str, Any]] = None, + *, headers: Optional[Mapping[str, str]] = None, error_context: Optional[dict[str, Any]] = None, default_cooldown: float = 300.0, ) -> None: """Record that Nous Portal is rate-limited in the shared state file. @@ -121,9 +119,7 @@ def _is_exhausted(remaining: Optional[int], reset: Optional[float]) -> bool: def is_genuine_nous_rate_limit( - *, - headers: Optional[Mapping[str, str]] = None, - last_known_state: Optional[Any] = None, + *, headers: Optional[Mapping[str, str]] = None, last_known_state: Optional[Any] = None, ) -> bool: """Decide whether a 429 from Nous Portal is a real account rate limit. @@ -154,9 +150,7 @@ def _parse_buckets_from_headers( return result -def _has_exhausted_bucket( - buckets: Mapping[str, tuple[Optional[int], Optional[float]]], -) -> bool: +def _has_exhausted_bucket(buckets: Mapping[str, tuple[Optional[int], Optional[float]]]) -> bool: return any(_is_exhausted(remaining, reset) for remaining, reset in buckets.values()) diff --git a/agent/plugin_llm.py b/agent/plugin_llm.py index 14ab45b15b..5acc3ae3d4 100644 --- a/agent/plugin_llm.py +++ b/agent/plugin_llm.py @@ -154,10 +154,8 @@ def _resolve_trust_policy(plugin_id: str) -> _TrustPolicy: allowed_models, allow_any_model = _coerce_allowlist(llm_cfg.get("allowed_models")) allowed_providers, allow_any_provider = _coerce_allowlist(llm_cfg.get("allowed_providers")) return _TrustPolicy( - plugin_id=plugin_id, - allowed_providers=allowed_providers, - allow_any_provider=allow_any_provider, - allowed_models=allowed_models, + plugin_id=plugin_id, allowed_providers=allowed_providers, + allow_any_provider=allow_any_provider, allowed_models=allowed_models, allow_any_model=allow_any_model, **{name: bool(llm_cfg.get(name, False)) for name in _OVERRIDE_FLAGS}, ) @@ -238,10 +236,7 @@ def _resolve_task_ownership(plugin_id: str) -> tuple[frozenset, frozenset]: def _check_task( - policy: _TrustPolicy, - *, - plugin_id: str, - requested_task: Optional[str], + policy: _TrustPolicy, *, plugin_id: str, requested_task: Optional[str], ) -> Optional[str]: """Validate a plugin's requested auxiliary ``task`` key. @@ -321,13 +316,8 @@ def _image_part(norm: Dict[str, Any]) -> Dict[str, Any]: def _build_structured_messages( - *, - instructions: str, - inputs: Sequence[PluginLlmInput], - json_mode: bool, - json_schema: Optional[Any], - schema_name: Optional[str], - system_prompt: Optional[str], + *, instructions: str, inputs: Sequence[PluginLlmInput], json_mode: bool, + json_schema: Optional[Any], schema_name: Optional[str], system_prompt: Optional[str], ) -> List[Dict[str, Any]]: """OpenAI-style messages for a structured call: optional system message (prompt + JSON-only directive), then a user message whose first text part is the @@ -453,10 +443,7 @@ def _main_config_value(reader: str, default: str) -> str: def _resolve_attribution( - *, - provider_override: Optional[str], - model_override: Optional[str], - response: Any, + *, provider_override: Optional[str], model_override: Optional[str], response: Any, route_info: Optional[Dict[str, str]] = None, ) -> tuple[str, str]: """``(provider, model)`` to record on the result. @@ -511,10 +498,7 @@ class PluginLlm: caller) → ``_finish`` (result + audit log).""" def __init__( - self, - *, - plugin_id: str, - policy_loader: Optional[Callable[[str], _TrustPolicy]] = None, + self, *, plugin_id: str, policy_loader: Optional[Callable[[str], _TrustPolicy]] = None, sync_caller: Optional[Callable[..., Any]] = None, async_caller: Optional[Callable[..., Awaitable[Any]]] = None, ) -> None: @@ -526,18 +510,11 @@ class PluginLlm: # -- public API ----------------------------------------------------------- def complete( - self, - messages: List[Dict[str, Any]], - *, - provider: Optional[str] = None, - model: Optional[str] = None, - temperature: Optional[float] = None, - max_tokens: Optional[int] = None, - timeout: Optional[float] = None, - agent_id: Optional[str] = None, - profile: Optional[str] = None, - purpose: Optional[str] = None, - task: Optional[str] = None, + self, messages: List[Dict[str, Any]], *, provider: Optional[str] = None, + model: Optional[str] = None, temperature: Optional[float] = None, + max_tokens: Optional[int] = None, timeout: Optional[float] = None, + agent_id: Optional[str] = None, profile: Optional[str] = None, + purpose: Optional[str] = None, task: Optional[str] = None, ) -> PluginLlmCompleteResult: """Run a host-owned chat completion against the user's active model. @@ -548,23 +525,13 @@ class PluginLlm: return self._finish("complete", agent, kw, self._invoke_sync(kw), purpose) def complete_structured( - self, - *, - instructions: str, - input: Sequence[PluginLlmInput], - json_schema: Optional[Any] = None, - json_mode: bool = False, - schema_name: Optional[str] = None, - system_prompt: Optional[str] = None, - provider: Optional[str] = None, - model: Optional[str] = None, - temperature: Optional[float] = None, - max_tokens: Optional[int] = None, - timeout: Optional[float] = None, - agent_id: Optional[str] = None, - profile: Optional[str] = None, - purpose: Optional[str] = None, - task: Optional[str] = None, + self, *, instructions: str, input: Sequence[PluginLlmInput], + json_schema: Optional[Any] = None, json_mode: bool = False, + schema_name: Optional[str] = None, system_prompt: Optional[str] = None, + provider: Optional[str] = None, model: Optional[str] = None, + temperature: Optional[float] = None, max_tokens: Optional[int] = None, + timeout: Optional[float] = None, agent_id: Optional[str] = None, + profile: Optional[str] = None, purpose: Optional[str] = None, task: Optional[str] = None, ) -> PluginLlmStructuredResult: """Run a bounded host-owned structured completion. @@ -576,41 +543,24 @@ class PluginLlm: return self._finish("complete_structured", agent, kw, self._invoke_sync(kw), purpose, spec) async def acomplete( - self, - messages: List[Dict[str, Any]], - *, - provider: Optional[str] = None, - model: Optional[str] = None, - temperature: Optional[float] = None, - max_tokens: Optional[int] = None, - timeout: Optional[float] = None, - agent_id: Optional[str] = None, - profile: Optional[str] = None, - purpose: Optional[str] = None, - task: Optional[str] = None, + self, messages: List[Dict[str, Any]], *, provider: Optional[str] = None, + model: Optional[str] = None, temperature: Optional[float] = None, + max_tokens: Optional[int] = None, timeout: Optional[float] = None, + agent_id: Optional[str] = None, profile: Optional[str] = None, + purpose: Optional[str] = None, task: Optional[str] = None, ) -> PluginLlmCompleteResult: """Async sibling of :meth:`complete`.""" agent, kw = self._gate(provider, model, agent_id, profile, task, messages, temperature, max_tokens, timeout) return self._finish("acomplete", agent, kw, await self._invoke_async(kw), purpose) async def acomplete_structured( - self, - *, - instructions: str, - input: Sequence[PluginLlmInput], - json_schema: Optional[Any] = None, - json_mode: bool = False, - schema_name: Optional[str] = None, - system_prompt: Optional[str] = None, - provider: Optional[str] = None, - model: Optional[str] = None, - temperature: Optional[float] = None, - max_tokens: Optional[int] = None, - timeout: Optional[float] = None, - agent_id: Optional[str] = None, - profile: Optional[str] = None, - purpose: Optional[str] = None, - task: Optional[str] = None, + self, *, instructions: str, input: Sequence[PluginLlmInput], + json_schema: Optional[Any] = None, json_mode: bool = False, + schema_name: Optional[str] = None, system_prompt: Optional[str] = None, + provider: Optional[str] = None, model: Optional[str] = None, + temperature: Optional[float] = None, max_tokens: Optional[int] = None, + timeout: Optional[float] = None, agent_id: Optional[str] = None, + profile: Optional[str] = None, purpose: Optional[str] = None, task: Optional[str] = None, ) -> PluginLlmStructuredResult: """Async sibling of :meth:`complete_structured`.""" spec = _structured_spec("acomplete_structured", instructions, input, system_prompt, json_mode, json_schema, schema_name) @@ -620,16 +570,9 @@ class PluginLlm: # -- shared core ---------------------------------------------------------- def _gate( - self, - provider: Optional[str], - model: Optional[str], - agent_id: Optional[str], - profile: Optional[str], - task: Optional[str], - messages: Optional[List[Dict[str, Any]]], - temperature: Optional[float], - max_tokens: Optional[int], - timeout: Optional[float], + self, provider: Optional[str], model: Optional[str], agent_id: Optional[str], + profile: Optional[str], task: Optional[str], messages: Optional[List[Dict[str, Any]]], + temperature: Optional[float], max_tokens: Optional[int], timeout: Optional[float], spec: Optional[Dict[str, Any]] = None, ) -> tuple[Optional[str], Dict[str, Any]]: """Trust gate (task first, then overrides), then — for a structured ``spec`` — @@ -648,25 +591,14 @@ class PluginLlm: messages = _build_structured_messages(**spec) extra_body = _json_response_format(json_mode=spec["json_mode"], json_schema=spec["json_schema"]) return eff_agent, dict( - messages=messages, - provider_override=eff_provider, - model_override=eff_model, - profile_override=eff_profile, - temperature=temperature, - max_tokens=max_tokens, - timeout=timeout, - extra_body=extra_body, - task=eff_task, + messages=messages, provider_override=eff_provider, model_override=eff_model, + profile_override=eff_profile, temperature=temperature, max_tokens=max_tokens, + timeout=timeout, extra_body=extra_body, task=eff_task, ) def _finish( - self, - name: str, - agent_id: Optional[str], - kw: Dict[str, Any], - invoked: tuple[str, str, Any], - purpose: Optional[str], - spec: Optional[Dict[str, Any]] = None, + self, name: str, agent_id: Optional[str], kw: Dict[str, Any], invoked: tuple[str, str, Any], + purpose: Optional[str], spec: Optional[Dict[str, Any]] = None, ) -> Any: """Build the result object + audit dict and emit the INFO audit line.""" real_provider, real_model, response = invoked @@ -701,15 +633,9 @@ class PluginLlm: merged_extra.setdefault("metadata", {})["auth_profile"] = kw["profile_override"] route_info: Optional[Dict[str, str]] = {} if kw["task"] else None return dict( - task=kw["task"], - provider=kw["provider_override"], - model=kw["model_override"], - messages=kw["messages"], - temperature=kw["temperature"], - max_tokens=kw["max_tokens"], - timeout=kw["timeout"], - extra_body=merged_extra or None, - route_info=route_info, + task=kw["task"], provider=kw["provider_override"], model=kw["model_override"], + messages=kw["messages"], temperature=kw["temperature"], max_tokens=kw["max_tokens"], + timeout=kw["timeout"], extra_body=merged_extra or None, route_info=route_info, ), route_info @staticmethod @@ -740,10 +666,7 @@ class PluginLlm: def make_plugin_llm_for_test( - *, - plugin_id: str, - policy: _TrustPolicy, - sync_caller: Optional[Callable[..., Any]] = None, + *, plugin_id: str, policy: _TrustPolicy, sync_caller: Optional[Callable[..., Any]] = None, async_caller: Optional[Callable[..., Awaitable[Any]]] = None, ) -> PluginLlm: """:class:`PluginLlm` with an injected policy and caller (no config.yaml, no diff --git a/agent/provider_media.py b/agent/provider_media.py index d3dfffe553..eed99fc5d2 100644 --- a/agent/provider_media.py +++ b/agent/provider_media.py @@ -44,18 +44,9 @@ def save_b64(kind: str, b64_data: str, *, prefix: str, extension: str) -> Path: def save_url( - kind: str, - url: str, - *, - prefix: str, - timeout: float, - max_bytes: int, - chunk_size: int, - content_types: Dict[str, str], - url_extensions: Tuple[str, ...], - default_extension: str, - label: str, - empty_error: str, + kind: str, url: str, *, prefix: str, timeout: float, max_bytes: int, chunk_size: int, + content_types: Dict[str, str], url_extensions: Tuple[str, ...], default_extension: str, + label: str, empty_error: str, ) -> Path: """Stream-download *url* into the cache with a size cap. diff --git a/agent/provider_projection.py b/agent/provider_projection.py index dc56b98530..36c3765266 100644 --- a/agent/provider_projection.py +++ b/agent/provider_projection.py @@ -22,9 +22,7 @@ logger = logging.getLogger(__name__) __all__ = ["splice_provider_projection"] -def splice_provider_projection( - agent: Any, response: Any, messages: list[dict[str, Any]] -) -> int: +def splice_provider_projection(agent: Any, response: Any, messages: list[dict[str, Any]]) -> int: """Append the provider's projected history rows and tick the nudge counter. Returns the number of rows spliced. Tolerates absent/garbage attributes so a diff --git a/agent/provider_registry.py b/agent/provider_registry.py index 27b859d820..549670ce95 100644 --- a/agent/provider_registry.py +++ b/agent/provider_registry.py @@ -39,13 +39,8 @@ class ProviderRegistry(Generic[P]): """ def __init__( - self, - *, - label: str, - provider_cls: type, - logger: logging.Logger, - normalize: Callable[[str], str] = strip_key, - builtin_names: FrozenSet[str] = frozenset(), + self, *, label: str, provider_cls: type, logger: logging.Logger, + normalize: Callable[[str], str] = strip_key, builtin_names: FrozenSet[str] = frozenset(), on_builtin_collision: Optional[Callable[[str], None]] = None, ) -> None: self.label = label @@ -168,25 +163,16 @@ class ProviderRegistry(Generic[P]): """Bind the historical module-level API (+ ``_providers``/``_scoped_providers``/ ``_lock`` test hooks) into a ``*_registry`` module namespace.""" namespace.update( - _providers=self._providers, - _scoped_providers=self._scoped_providers, - _lock=self._lock, - register_provider=self.register, - list_providers=self.list_providers, - get_provider=self.get_provider, - snapshot_registration=self.snapshot_registration, + _providers=self._providers, _scoped_providers=self._scoped_providers, _lock=self._lock, + register_provider=self.register, list_providers=self.list_providers, + get_provider=self.get_provider, snapshot_registration=self.snapshot_registration, restore_registration=self.restore_registration, - registry_generation=self.registry_generation, - _reset_for_tests=self.reset_for_tests, + registry_generation=self.registry_generation, _reset_for_tests=self.reset_for_tests, ) def is_available_safe( - provider: Any, - logger: logging.Logger, - fmt: str, - *, - level: int = logging.DEBUG, + provider: Any, logger: logging.Logger, fmt: str, *, level: int = logging.DEBUG, exc_info: bool = False, ) -> bool: """``bool(provider.is_available())`` that treats a raising provider as unavailable.""" From 1d7632bfb7298146c8e19b2d73b296e6ebc581a3 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:56:13 -0700 Subject: [PATCH 4/9] =?UTF-8?q?refactor(agent):=20moa=5Floop/plugin=5Fllm?= =?UTF-8?q?=20=E2=80=94=20collapse=20arg=20ladders,=20contextlib.suppress,?= =?UTF-8?q?=20usage=20extractor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/moa_loop.py | 68 +++++++++++++++++---------------------------- agent/plugin_llm.py | 34 ++++++++++++----------- 2 files changed, 44 insertions(+), 58 deletions(-) diff --git a/agent/moa_loop.py b/agent/moa_loop.py index 0f02b39f40..c398bfda88 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -7,6 +7,7 @@ context before each model iteration. from __future__ import annotations +import contextlib import hashlib import json import logging @@ -480,11 +481,9 @@ def _trim_messages_for_reference( if not isinstance(context_length, int) or context_length <= 0: return messages - reserve = ( - int(reserve_output_tokens) - if isinstance(reserve_output_tokens, int) and reserve_output_tokens > 0 - else _REFERENCE_DEFAULT_OUTPUT_RESERVE - ) + if not (isinstance(reserve_output_tokens, int) and reserve_output_tokens > 0): + reserve_output_tokens = _REFERENCE_DEFAULT_OUTPUT_RESERVE + reserve = int(reserve_output_tokens) budget = int(context_length * (1.0 - _REFERENCE_TRIM_SAFETY_FRACTION)) - reserve if budget <= 0: return messages @@ -647,6 +646,13 @@ def _field(obj: Any, name: str) -> Any: return obj.get(name) if isinstance(obj, dict) else getattr(obj, name, None) +def _dumps_or_str(value: Any) -> str: + try: + return json.dumps(value, ensure_ascii=False) + except Exception: + return str(value) + + def _render_tool_calls(tool_calls: Any) -> str: """Render an assistant turn's tool_calls as ``[called tool: name(args)]`` lines. @@ -657,15 +663,7 @@ def _render_tool_calls(tool_calls: Any) -> str: fn = _field(tc, "function") name = _field(fn, "name") or _field(tc, "name") or "tool" fn_args = _field(fn, "arguments") - if isinstance(fn_args, str): - args_text = fn_args - elif fn_args is not None: - try: - args_text = json.dumps(fn_args, ensure_ascii=False) - except Exception: - args_text = str(fn_args) - else: - args_text = "" + args_text = fn_args if isinstance(fn_args, str) else "" if fn_args is None else _dumps_or_str(fn_args) lines.append(f"[called tool: {name}({args_text})]" if args_text else f"[called tool: {name}]") return "\n".join(lines) @@ -733,10 +731,9 @@ def _reference_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: if last_user_content is not None: return [{"role": "user", "content": last_user_content}] for msg in reversed(messages): - if msg.get("role") == "user": - fallback_text = flatten_message_text(msg.get("content")) - if fallback_text.strip(): - return [{"role": "user", "content": fallback_text}] + fallback_text = flatten_message_text(msg.get("content")) if msg.get("role") == "user" else "" + if fallback_text.strip(): + return [{"role": "user", "content": fallback_text}] return rendered @@ -934,9 +931,7 @@ def _completed_response_as_stream_chunk(response: Any) -> Any: ] delta = SimpleNamespace( content=getattr(message, "content", None), tool_calls=tool_call_deltas, - reasoning_content=getattr(message, "reasoning_content", None), - reasoning=getattr(message, "reasoning", None), - reasoning_details=getattr(message, "reasoning_details", None), + **{k: getattr(message, k, None) for k in ("reasoning_content", "reasoning", "reasoning_details")}, ) choice = SimpleNamespace( index=getattr(first_choice, "index", 0), delta=delta, @@ -989,7 +984,7 @@ def peel_reference_guidance(messages: list[dict[str, Any]], guidance: Any) -> li last_part = content[-1] if isinstance(last_part, dict) and last_part.get("type", "text") == "text": text = last_part.get("text") or "" - if text == suffix or text == guidance_text: + if text in (suffix, guidance_text): # Attach shape (b): guidance rode as its own trailing part. Guidance as # the only content drops the whole message (mirrors shape c). if len(content) == 1: @@ -1014,8 +1009,7 @@ class MoAChatCompletions: self.preset_name = preset_name or "default" self.reference_callback = reference_callback self._agent = agent - # Reference cache keyed on the advisory-view signature: new state = MISS - # (references re-run), identical state = HIT (no re-run, no re-emit). + # Reference cache keyed on the advisory-view signature (HIT = no re-run, no re-emit). self._ref_cache_key: tuple | None = None self._ref_cache_outputs: list[tuple[str, str, Any]] = [] # Fan-out spend awaiting consume_reference_usage (nothing deposited on a HIT so @@ -1034,8 +1028,7 @@ class MoAChatCompletions: self._fanout_iteration_count = 0 self._fanout_turn_sig: str | None = None self._fanout_last_state_sig: str | None = None - # Normalized moa.privacy_filter mode ('' | 'display' | 'full'), refreshed per create(). - self._privacy_mode: str = "" + self._privacy_mode: str = "" # normalized moa.privacy_filter, refreshed per create() def consume_reference_usage(self) -> tuple[Any, Any]: """Pop pending fan-out ``(CanonicalUsage, cost_usd_or_None)`` and reset both @@ -1214,10 +1207,8 @@ class MoAChatCompletions: self._pending_trace["aggregator_streamed"] = stream output = None if not stream: - try: + with contextlib.suppress(Exception): # pragma: no cover - defensive output = _extract_text(agg_response) - except Exception: # pragma: no cover - defensive - output = None self._pending_trace["aggregator_output"] = output if stream and hasattr(agg_response, "choices"): # Some adapters (openai-codex Responses) return a completed response even @@ -1239,10 +1230,8 @@ class MoAChatCompletions: fanout_mode = str(preset.get("fanout") or "user_turn").strip().lower() every_n = 0 if fanout_mode.startswith("every_n:"): - try: + with contextlib.suppress(TypeError, ValueError): every_n = int(fanout_mode.split(":", 1)[1]) - except (TypeError, ValueError): - every_n = 0 if every_n < 2: # every_n:1 IS per-iteration (mirrors _coerce_fanout). fanout_mode = "per_iteration" @@ -1317,13 +1306,10 @@ class MoAChatCompletions: # Stash the fan-out for trace persistence (aggregator parts filled in later). # Traces are persisted, so ANY active privacy mode redacts them. privacy_mode = self._privacy_mode - if privacy_mode: - trace_refs = [ - (label, _redact_reference_text(text), _redact_trace_accounting(acct)) - for label, text, acct in reference_outputs - ] - else: - trace_refs = list(reference_outputs) + trace_refs = [ + (label, _redact_reference_text(text), _redact_trace_accounting(acct)) + for label, text, acct in reference_outputs + ] if privacy_mode else list(reference_outputs) self._pending_trace = { "preset": self.preset_name, "reference_outputs": trace_refs, "aggregator_slot": aggregator, "aggregator_temperature": aggregator_temperature, @@ -1500,14 +1486,12 @@ def build_moa_facade(agent, preset_name: Any = None) -> MoAClient: if cb is None or spec is None: return primary, secondary, extra_map = spec - try: + with contextlib.suppress(Exception): cb( event, str(kwargs.get(primary) or ""), str(kwargs.get(secondary) or "") if secondary else None, None, **{out: kwargs.get(src) for out, src in extra_map.items()}, ) - except Exception: - pass resolved_preset = preset_name if resolved_preset is None and getattr(agent, "provider", None) == "moa": diff --git a/agent/plugin_llm.py b/agent/plugin_llm.py index 5acc3ae3d4..d628418c20 100644 --- a/agent/plugin_llm.py +++ b/agent/plugin_llm.py @@ -393,26 +393,28 @@ def _extract_usage(response: Any) -> PluginLlmUsage: """Token usage from an OpenAI-shaped response, tolerating provider naming (``prompt_tokens``/``completion_tokens`` vs ``input_tokens``/``output_tokens``, ``cache_read_input_tokens`` vs ``cache_read_tokens``).""" - usage = PluginLlmUsage() raw = getattr(response, "usage", None) if raw is None: - return usage + return PluginLlmUsage() - def _g(name: str) -> int: - v = getattr(raw, name, None) - if v is None and isinstance(raw, dict): - v = raw.get(name) - try: - return int(v) if v is not None else 0 - except (TypeError, ValueError): - return 0 + def _g(*names: str) -> int: + for name in names: + v = getattr(raw, name, None) + if v is None and isinstance(raw, dict): + v = raw.get(name) + try: + if v is not None and int(v): + return int(v) + except (TypeError, ValueError): + pass + return 0 - usage.input_tokens = _g("prompt_tokens") or _g("input_tokens") - usage.output_tokens = _g("completion_tokens") or _g("output_tokens") - usage.total_tokens = _g("total_tokens") or (usage.input_tokens + usage.output_tokens) - usage.cache_read_tokens = _g("cache_read_input_tokens") or _g("cache_read_tokens") - usage.cache_write_tokens = _g("cache_creation_input_tokens") or _g("cache_write_tokens") - return usage + inp, out = _g("prompt_tokens", "input_tokens"), _g("completion_tokens", "output_tokens") + return PluginLlmUsage( + input_tokens=inp, output_tokens=out, total_tokens=_g("total_tokens") or (inp + out), + cache_read_tokens=_g("cache_read_input_tokens", "cache_read_tokens"), + cache_write_tokens=_g("cache_creation_input_tokens", "cache_write_tokens"), + ) def _extract_text(response: Any) -> str: From e34096ca19ae4007d681365055e14d08ea28624d Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:59:41 -0700 Subject: [PATCH 5/9] refactor(agent): pack remaining multi-line signatures (AST-identical) --- agent/moa_loop.py | 59 +++++++++++++------------------------- agent/plugin_llm.py | 57 ++++++++++++++---------------------- agent/provider_registry.py | 3 +- 3 files changed, 43 insertions(+), 76 deletions(-) diff --git a/agent/moa_loop.py b/agent/moa_loop.py index c398bfda88..2c8a598df1 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -63,9 +63,7 @@ def _moa_privacy_mode(moa_raw: Any) -> str: return coerce_privacy_filter(raw.get("privacy_filter")) -def _redact_reference_outputs( - reference_outputs: list[tuple[str, str, Any]], -) -> list[tuple[str, str, Any]]: +def _redact_reference_outputs(reference_outputs: list[tuple[str, str, Any]]) -> list[tuple[str, str, Any]]: """Redact advisor text in reference-output tuples; accounting slot untouched.""" return [(label, _redact_reference_text(text), acct) for label, text, acct in reference_outputs] @@ -99,8 +97,7 @@ def _redact_trace_accounting(acct: Any) -> Any: if not isinstance(acct, _RefAccounting): return acct return replace( - acct, messages=_redact_trace_messages(acct.messages), - output=_redact_reference_text(acct.output), + acct, messages=_redact_trace_messages(acct.messages), output=_redact_reference_text(acct.output), ) @@ -326,8 +323,7 @@ def _maybe_apply_moa_cache_control( # blank_cache_policy_stub is the only sanctioned stub (carries _cache_disabled). should_cache, native_layout = anthropic_prompt_cache_policy( blank_cache_policy_stub(cache_disabled), provider=provider, - base_url=runtime.get("base_url") or "", api_mode=runtime.get("api_mode") or "", - model=model, + base_url=runtime.get("base_url") or "", api_mode=runtime.get("api_mode") or "", model=model, ) if not should_cache: return messages @@ -374,9 +370,8 @@ def _price_reference_response( def _run_reference( slot: dict[str, Any], ref_messages: list[dict[str, Any]], *, temperature: float | None = None, - max_tokens: int | None = None, reference_timeout: float | None = None, - context_length_cache: Any = None, cache_disabled: bool | None = None, - cache_ttl: str | None = None, + max_tokens: int | None = None, reference_timeout: float | None = None, context_length_cache: Any = None, + cache_disabled: bool | None = None, cache_ttl: str | None = None, ) -> tuple[str, str, Any]: """Call one reference model; return ``(label, text, accounting)``. Never raises: a failed reference becomes a labelled ``[failed: …]`` note. Runs in a thread pool.""" @@ -424,9 +419,7 @@ def _run_reference( except Exception as exc: logger.warning("MoA reference model %s failed: %s", label, exc) note = f"[failed: {exc}]" - return label, note, _RefAccounting( - CanonicalUsage(), messages=messages, output=note, **trace_fields, - ) + return label, note, _RefAccounting(CanonicalUsage(), messages=messages, output=note, **trace_fields) # Output headroom reserved in the reference window when reference_max_tokens is unset. @@ -529,8 +522,7 @@ def _placeholder_output(slot: dict[str, Any], note: str) -> tuple[str, str, Any] def _settle_interrupted( - futures: dict[Any, int], results: list, reference_models: list[dict[str, Any]], - late_accounting_sink: Any, + futures: dict[Any, int], results: list, reference_models: list[dict[str, Any]], late_accounting_sink: Any, ) -> None: """Fill every unfinished slot after a user interrupt: cancel never-dispatched futures (nothing billed), keep real output of ones that just finished, and hand @@ -596,10 +588,9 @@ def _run_references_parallel( continue futures[ executor.submit( - propagate_context_to_thread(_run_reference), slot, ref_messages, - temperature=temperature, max_tokens=max_tokens, - reference_timeout=reference_timeout, context_length_cache=ctx_len_cache, - cache_disabled=cache_disabled, cache_ttl=cache_ttl, + propagate_context_to_thread(_run_reference), slot, ref_messages, temperature=temperature, + max_tokens=max_tokens, reference_timeout=reference_timeout, + context_length_cache=ctx_len_cache, cache_disabled=cache_disabled, cache_ttl=cache_ttl, ) ] = idx @@ -826,10 +817,9 @@ def _slot_labels(slots: list[dict[str, Any]]) -> str: def aggregate_moa_context( *, user_prompt: str, api_messages: list[dict[str, Any]], reference_models: list[dict[str, Any]], - aggregator: dict[str, Any], temperature: float | None = None, - aggregator_temperature: float | None = None, reference_max_tokens: int | None = None, - reference_timeout: float | None = None, degraded_reference_policy: str = "loud", - agent: Any = None, + aggregator: dict[str, Any], temperature: float | None = None, aggregator_temperature: float | None = None, + reference_max_tokens: int | None = None, reference_timeout: float | None = None, + degraded_reference_policy: str = "loud", agent: Any = None, ) -> str: """Run configured reference models and synthesize their advice (one-shot /moa). @@ -860,8 +850,7 @@ def aggregate_moa_context( # for the full provider timeout) and return only the sanitized notice. if reference_outputs and not successful_outputs: logger.warning( - "MoA: all %d reference(s) failed — skipping aggregator synthesis", - len(reference_outputs), + "MoA: all %d reference(s) failed — skipping aggregator synthesis", len(reference_outputs), ) return ( "[Mixture of Agents context — all reference models failed. " @@ -1058,9 +1047,7 @@ class MoAChatCompletions: if cost is not None: self._pending_reference_cost = (self._pending_reference_cost or 0) + cost - def consume_and_save_trace( - self, session_id: Any = None, aggregator_output_fallback: Any = None - ) -> None: + def consume_and_save_trace(self, session_id: Any = None, aggregator_output_fallback: Any = None) -> None: """Flush the pending full-turn trace to disk (no-op when nothing is pending). ``aggregator_output_fallback`` is the caller's resolved acting text for the @@ -1081,13 +1068,11 @@ class MoAChatCompletions: save_moa_turn( session_id=session_id, preset_name=pending.get("preset", ""), reference_outputs=pending.get("reference_outputs", []), - aggregator_label=pending.get("aggregator_label", ""), - aggregator_model=agg_slot.get("model"), + aggregator_label=pending.get("aggregator_label", ""), aggregator_model=agg_slot.get("model"), aggregator_provider=agg_slot.get("provider"), aggregator_temperature=pending.get("aggregator_temperature"), aggregator_input_messages=pending.get("aggregator_input_messages"), - aggregator_output=agg_output, - aggregator_streamed=bool(pending.get("aggregator_streamed")), + aggregator_output=agg_output, aggregator_streamed=bool(pending.get("aggregator_streamed")), ) except Exception as exc: # pragma: no cover - tracing must never break a turn logger.debug("MoA trace flush failed: %s", exc) @@ -1117,8 +1102,7 @@ class MoAChatCompletions: return {**prepared, "messages": agg_messages} def _plan_aggregator_cache( - self, agg_messages: list[dict[str, Any]], tools: Any, guidance: Any, - agg_runtime: dict[str, Any], + self, agg_messages: list[dict[str, Any]], tools: Any, guidance: Any, agg_runtime: dict[str, Any], ) -> tuple[list[dict[str, Any]], Any]: """Cache-breakpoint the aggregator request for its destination. @@ -1157,9 +1141,7 @@ class MoAChatCompletions: ) return agg_messages, tools - def _call_prepared_aggregator( - self, prepared: dict[str, Any], api_kwargs: dict[str, Any] - ) -> Any: + def _call_prepared_aggregator(self, prepared: dict[str, Any], api_kwargs: dict[str, Any]) -> Any: """Send an already prepared MoA aggregator request exactly once.""" aggregator = prepared["aggregator"] if aggregator.get("provider") == "moa": @@ -1287,8 +1269,7 @@ class MoAChatCompletions: self._emit("moa.progress", refs_done=done, refs_total=total, label=label) reference_outputs = _run_references_parallel( - reference_models, ref_messages, - temperature=_preset_temperature(preset, "reference_temperature"), + reference_models, ref_messages, temperature=_preset_temperature(preset, "reference_temperature"), max_tokens=preset.get("reference_max_tokens"), progress_callback=_progress, reference_timeout=float(raw_reference_timeout) if raw_reference_timeout else None, agent=self._agent, late_accounting_sink=self._record_late_reference_accounting, diff --git a/agent/plugin_llm.py b/agent/plugin_llm.py index d628418c20..0eeab9de89 100644 --- a/agent/plugin_llm.py +++ b/agent/plugin_llm.py @@ -154,9 +154,8 @@ def _resolve_trust_policy(plugin_id: str) -> _TrustPolicy: allowed_models, allow_any_model = _coerce_allowlist(llm_cfg.get("allowed_models")) allowed_providers, allow_any_provider = _coerce_allowlist(llm_cfg.get("allowed_providers")) return _TrustPolicy( - plugin_id=plugin_id, allowed_providers=allowed_providers, - allow_any_provider=allow_any_provider, allowed_models=allowed_models, - allow_any_model=allow_any_model, + plugin_id=plugin_id, allowed_providers=allowed_providers, allow_any_provider=allow_any_provider, + allowed_models=allowed_models, allow_any_model=allow_any_model, **{name: bool(llm_cfg.get(name, False)) for name in _OVERRIDE_FLAGS}, ) @@ -188,8 +187,7 @@ def _gate_ref_override(policy: _TrustPolicy, kind: str, requested: str) -> str: # Overrides gated by a bare trust flag (no allowlist): ``kind`` -> denial wording. _FLAG_ONLY_OVERRIDES = { - "agent_id": "run completions against a non-default agent id", - "profile": "override the auth profile", + "agent_id": "run completions against a non-default agent id", "profile": "override the auth profile", } @@ -235,9 +233,7 @@ def _resolve_task_ownership(plugin_id: str) -> tuple[frozenset, frozenset]: return frozenset(owned), frozenset(builtin) -def _check_task( - policy: _TrustPolicy, *, plugin_id: str, requested_task: Optional[str], -) -> Optional[str]: +def _check_task(policy: _TrustPolicy, *, plugin_id: str, requested_task: Optional[str]) -> Optional[str]: """Validate a plugin's requested auxiliary ``task`` key. unset / ``""`` / ``"auto"`` → ``None`` (main-model path); a key the plugin @@ -382,9 +378,7 @@ def _parse_structured_text( except ImportError: logger.debug("jsonschema unavailable; skipping schema validation") except jsonschema.ValidationError as exc: # type: ignore[attr-defined] - raise ValueError( - f"Plugin LLM structured output did not match schema: {exc.message}" - ) from exc + raise ValueError(f"Plugin LLM structured output did not match schema: {exc.message}") from exc return parsed, "json" @@ -512,10 +506,9 @@ class PluginLlm: # -- public API ----------------------------------------------------------- def complete( - self, messages: List[Dict[str, Any]], *, provider: Optional[str] = None, - model: Optional[str] = None, temperature: Optional[float] = None, - max_tokens: Optional[int] = None, timeout: Optional[float] = None, - agent_id: Optional[str] = None, profile: Optional[str] = None, + self, messages: List[Dict[str, Any]], *, provider: Optional[str] = None, model: Optional[str] = None, + temperature: Optional[float] = None, max_tokens: Optional[int] = None, + timeout: Optional[float] = None, agent_id: Optional[str] = None, profile: Optional[str] = None, purpose: Optional[str] = None, task: Optional[str] = None, ) -> PluginLlmCompleteResult: """Run a host-owned chat completion against the user's active model. @@ -527,12 +520,10 @@ class PluginLlm: return self._finish("complete", agent, kw, self._invoke_sync(kw), purpose) def complete_structured( - self, *, instructions: str, input: Sequence[PluginLlmInput], - json_schema: Optional[Any] = None, json_mode: bool = False, - schema_name: Optional[str] = None, system_prompt: Optional[str] = None, - provider: Optional[str] = None, model: Optional[str] = None, - temperature: Optional[float] = None, max_tokens: Optional[int] = None, - timeout: Optional[float] = None, agent_id: Optional[str] = None, + self, *, instructions: str, input: Sequence[PluginLlmInput], json_schema: Optional[Any] = None, + json_mode: bool = False, schema_name: Optional[str] = None, system_prompt: Optional[str] = None, + provider: Optional[str] = None, model: Optional[str] = None, temperature: Optional[float] = None, + max_tokens: Optional[int] = None, timeout: Optional[float] = None, agent_id: Optional[str] = None, profile: Optional[str] = None, purpose: Optional[str] = None, task: Optional[str] = None, ) -> PluginLlmStructuredResult: """Run a bounded host-owned structured completion. @@ -545,10 +536,9 @@ class PluginLlm: return self._finish("complete_structured", agent, kw, self._invoke_sync(kw), purpose, spec) async def acomplete( - self, messages: List[Dict[str, Any]], *, provider: Optional[str] = None, - model: Optional[str] = None, temperature: Optional[float] = None, - max_tokens: Optional[int] = None, timeout: Optional[float] = None, - agent_id: Optional[str] = None, profile: Optional[str] = None, + self, messages: List[Dict[str, Any]], *, provider: Optional[str] = None, model: Optional[str] = None, + temperature: Optional[float] = None, max_tokens: Optional[int] = None, + timeout: Optional[float] = None, agent_id: Optional[str] = None, profile: Optional[str] = None, purpose: Optional[str] = None, task: Optional[str] = None, ) -> PluginLlmCompleteResult: """Async sibling of :meth:`complete`.""" @@ -556,12 +546,10 @@ class PluginLlm: return self._finish("acomplete", agent, kw, await self._invoke_async(kw), purpose) async def acomplete_structured( - self, *, instructions: str, input: Sequence[PluginLlmInput], - json_schema: Optional[Any] = None, json_mode: bool = False, - schema_name: Optional[str] = None, system_prompt: Optional[str] = None, - provider: Optional[str] = None, model: Optional[str] = None, - temperature: Optional[float] = None, max_tokens: Optional[int] = None, - timeout: Optional[float] = None, agent_id: Optional[str] = None, + self, *, instructions: str, input: Sequence[PluginLlmInput], json_schema: Optional[Any] = None, + json_mode: bool = False, schema_name: Optional[str] = None, system_prompt: Optional[str] = None, + provider: Optional[str] = None, model: Optional[str] = None, temperature: Optional[float] = None, + max_tokens: Optional[int] = None, timeout: Optional[float] = None, agent_id: Optional[str] = None, profile: Optional[str] = None, purpose: Optional[str] = None, task: Optional[str] = None, ) -> PluginLlmStructuredResult: """Async sibling of :meth:`complete_structured`.""" @@ -572,10 +560,9 @@ class PluginLlm: # -- shared core ---------------------------------------------------------- def _gate( - self, provider: Optional[str], model: Optional[str], agent_id: Optional[str], - profile: Optional[str], task: Optional[str], messages: Optional[List[Dict[str, Any]]], - temperature: Optional[float], max_tokens: Optional[int], timeout: Optional[float], - spec: Optional[Dict[str, Any]] = None, + self, provider: Optional[str], model: Optional[str], agent_id: Optional[str], profile: Optional[str], + task: Optional[str], messages: Optional[List[Dict[str, Any]]], temperature: Optional[float], + max_tokens: Optional[int], timeout: Optional[float], spec: Optional[Dict[str, Any]] = None, ) -> tuple[Optional[str], Dict[str, Any]]: """Trust gate (task first, then overrides), then — for a structured ``spec`` — build messages/response_format (input-shape errors surface only after trust diff --git a/agent/provider_registry.py b/agent/provider_registry.py index 549670ce95..21d4b8e483 100644 --- a/agent/provider_registry.py +++ b/agent/provider_registry.py @@ -172,8 +172,7 @@ class ProviderRegistry(Generic[P]): def is_available_safe( - provider: Any, logger: logging.Logger, fmt: str, *, level: int = logging.DEBUG, - exc_info: bool = False, + provider: Any, logger: logging.Logger, fmt: str, *, level: int = logging.DEBUG, exc_info: bool = False, ) -> bool: """``bool(provider.is_available())`` that treats a raising provider as unavailable.""" try: From 7c76e5f219d8726548360b20a56dfd94c8710aed Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:10:21 -0700 Subject: [PATCH 6/9] refactor(agent): moa_loop guidance-input helper, explicit MoAClient delegation, blank squeeze --- agent/moa_loop.py | 94 +++++++++++++++--------------------- agent/moa_trace.py | 1 - agent/plugin_llm.py | 20 ++------ agent/plugin_stream_hooks.py | 2 - agent/provider_media.py | 2 - agent/provider_registry.py | 10 +--- 6 files changed, 45 insertions(+), 84 deletions(-) diff --git a/agent/moa_loop.py b/agent/moa_loop.py index 2c8a598df1..6fc0af6ac3 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -49,7 +49,6 @@ def _redact_reference_text(text: Any) -> Any: if not isinstance(text, str) or not text: return text from agent.redact import redact_sensitive_text - text = redact_sensitive_text(text, force=True, code_file=True) text = _MOA_EMAIL_RE.sub("[redacted email]", text) return _MOA_PHONE_RE.sub("[redacted phone]", text) @@ -58,7 +57,6 @@ def _redact_reference_text(text: Any) -> Any: def _moa_privacy_mode(moa_raw: Any) -> str: """Normalized privacy-filter mode from a raw ``moa`` config.""" from hermes_cli.moa_config import coerce_privacy_filter - raw = moa_raw if isinstance(moa_raw, dict) else {} return coerce_privacy_filter(raw.get("privacy_filter")) @@ -111,7 +109,6 @@ def _resolve_preset_cached(preset_name: str) -> tuple[dict[str, Any], Any]: (skips resolve_moa_preset's full validation of the moa block on every create()).""" from hermes_cli.config import get_config_path, load_config from hermes_cli.moa_config import resolve_moa_preset - try: cfg_stamp = get_config_path().stat().st_mtime_ns except OSError: @@ -211,7 +208,6 @@ def _slot_reasoning_config(slot: dict[str, Any]) -> dict[str, Any] | None: effort = slot.get("reasoning_effort") try: from hermes_constants import parse_reasoning_effort - return parse_reasoning_effort(effort) except Exception: # pragma: no cover - bad config must not break MoA return None @@ -229,7 +225,6 @@ def _aggregator_reasoning_config(aggregator: dict[str, Any]) -> dict[str, Any] | try: from hermes_cli.config import load_config from hermes_constants import resolve_reasoning_config - return resolve_reasoning_config(load_config() or {}, str(aggregator.get("model") or "")) except Exception: # pragma: no cover - bad config must not break MoA return None @@ -252,7 +247,6 @@ def _slot_runtime(slot: dict[str, Any]) -> dict[str, Any]: out: dict[str, Any] = {"provider": provider, "model": model} try: from hermes_cli.runtime_provider import resolve_runtime_provider - rt = resolve_runtime_provider(requested=provider, target_model=model) out.update({k: rt[k] for k in ("base_url", "api_key", "api_mode") if rt.get(k)}) request_overrides = rt.get("request_overrides") @@ -348,7 +342,6 @@ def _price_reference_response( """Normalize a reference's usage with the slot's OWN provider/api_mode and price it at its own rate (hence fan-out cost is summed in dollars). Never raises.""" from agent.usage_pricing import estimate_usage_cost, normalize_usage - usage = CanonicalUsage() raw_usage = getattr(response, "usage", None) if raw_usage: @@ -399,7 +392,6 @@ def _run_reference( extra_headers = None # Normalize provider aliases (github, github-copilot, ...) via the canonical table. from agent.auxiliary_client import _normalize_aux_provider - if _normalize_aux_provider(str(runtime.get("provider") or "")) in ("copilot", "copilot-acp"): # Copilot gates premium models on request attribution; MoA fan-out serves the # user's current turn, so mirror the main agent's x-initiator header. @@ -570,7 +562,6 @@ def _run_references_parallel( futures: dict[Any, int] = {} # Propagate the turn's contextvars (approval callbacks, Nous conversation tag). from tools.thread_context import propagate_context_to_thread - total = len(reference_models) completed = 0 executor = ThreadPoolExecutor(max_workers=min(_MAX_REFERENCE_WORKERS, total)) @@ -815,6 +806,19 @@ def _slot_labels(slots: list[dict[str, Any]]) -> str: return ", ".join(_slot_label(slot) for slot in slots) +def _guidance_inputs( + reference_outputs: list[tuple[str, str, Any]], privacy_full: bool, policy: str, +) -> tuple[list[tuple[str, str, Any]], str, bool]: + """``(advisor outputs for the aggregator, degraded notice, all_failed)``. + + 'full' privacy mode redacts advisor text reaching the aggregator (applied to a + per-call copy — caches hold raw text). Failed refs are filtered out first. + """ + successful, failed_labels = _split_references(reference_outputs) + agg_refs = _redact_reference_outputs(successful) if privacy_full else successful + return agg_refs, _degraded_notice(failed_labels, policy), bool(reference_outputs) and not successful + + def aggregate_moa_context( *, user_prompt: str, api_messages: list[dict[str, Any]], reference_models: list[dict[str, Any]], aggregator: dict[str, Any], temperature: float | None = None, aggregator_temperature: float | None = None, @@ -832,26 +836,19 @@ def aggregate_moa_context( reference_models, _reference_messages(api_messages), temperature=temperature, max_tokens=reference_max_tokens, reference_timeout=reference_timeout, agent=agent, ) - successful_outputs, failed_labels = _split_references(reference_outputs) - - # 'full' privacy mode also redacts advisor text before it reaches the synthesizer. + privacy_full = False try: from hermes_cli.config import load_config as _load_config - - if _moa_privacy_mode((_load_config() or {}).get("moa")) == "full": - successful_outputs = _redact_reference_outputs(successful_outputs) + privacy_full = _moa_privacy_mode((_load_config() or {}).get("moa")) == "full" except Exception: # pragma: no cover - privacy filter must never break a turn logger.debug("MoA privacy filter check failed", exc_info=True) - - degraded = _degraded_notice(failed_labels, degraded_reference_policy) - joined = _join_reference_outputs(successful_outputs, degraded) + agg_refs, degraded, all_failed = _guidance_inputs(reference_outputs, privacy_full, degraded_reference_policy) + joined = _join_reference_outputs(agg_refs, degraded) # Every reference failed: skip the aggregator (synthesizing over nothing can block # for the full provider timeout) and return only the sanitized notice. - if reference_outputs and not successful_outputs: - logger.warning( - "MoA: all %d reference(s) failed — skipping aggregator synthesis", len(reference_outputs), - ) + if all_failed: + logger.warning("MoA: all %d reference(s) failed — skipping aggregator synthesis", len(reference_outputs)) return ( "[Mixture of Agents context — all reference models failed. " "Proceeding without aggregated guidance.]\n" @@ -1059,7 +1056,6 @@ class MoAChatCompletions: return try: from agent.moa_trace import save_moa_turn - agg_slot = pending.get("aggregator_slot") or {} # Inline capture (non-streaming) beats the caller's streamed text. agg_output = pending.get("aggregator_output") @@ -1112,7 +1108,6 @@ class MoAChatCompletions: """ try: from agent.agent_runtime_helpers import plan_cache_sections_for_destination - planning_messages = agg_messages if guidance: planning_messages = peel_reference_guidance(agg_messages, str(guidance)) @@ -1298,7 +1293,6 @@ class MoAChatCompletions: # Derived from the privacy-redacted trace_refs. try: from agent.moa_trace import slot_metrics - self._last_reference_metrics = [ slot_metrics(acct, label, output=text) for label, text, acct in trace_refs ] @@ -1328,20 +1322,15 @@ class MoAChatCompletions: degraded_reference_policy: str, ) -> str | None: """Render the reference block attached to the aggregator prompt (None = nothing).""" - successful_outputs, failed_labels = _split_references(reference_outputs) - # 'full' privacy mode redacts advisor text reaching the AGGREGATOR too. - agg_refs = ( - _redact_reference_outputs(successful_outputs) - if self._privacy_mode == "full" - else successful_outputs + agg_refs, degraded, all_failed = _guidance_inputs( + reference_outputs, self._privacy_mode == "full", degraded_reference_policy ) - degraded = _degraded_notice(failed_labels, degraded_reference_policy) header = ( "[Mixture of Agents reference context]\n" f"Preset: {self.preset_name}\n" f"Aggregator/acting model: {_slot_label(aggregator)}\n" ) - if reference_outputs and not successful_outputs: + if all_failed: # Every reference failed: the aggregator acts alone (loud policy → notice). logger.warning( "MoA: all %d reference(s) failed — acting aggregator-alone " @@ -1420,27 +1409,27 @@ class MoAChatCompletions: class MoAClient: - """OpenAI-client-shaped wrapper: ``client.chat.completions`` is a ``MoAChatCompletions``. - - The accounting/trace surface (``consume_reference_usage``, ``last_aggregator_slot``, - ``consume_and_save_trace``, ``last_reference_metrics``) is delegated to the facade. - """ - - _DELEGATED = ( - "consume_reference_usage", "last_aggregator_slot", - "consume_and_save_trace", "last_reference_metrics", - ) + """OpenAI-client-shaped wrapper: ``client.chat.completions`` is a ``MoAChatCompletions``; + the accounting/trace surface below is delegated to that facade.""" def __init__(self, preset_name: str, reference_callback: Any = None, agent: Any = None): self.chat = type("_MoAChat", (), {})() - self.chat.completions = MoAChatCompletions( - preset_name, reference_callback=reference_callback, agent=agent, + self.chat.completions = MoAChatCompletions(preset_name, reference_callback=reference_callback, agent=agent) + + def consume_reference_usage(self) -> Any: + return self.chat.completions.consume_reference_usage() + + @property + def last_aggregator_slot(self) -> Any: + return getattr(self.chat.completions, "last_aggregator_slot", None) + + def consume_and_save_trace(self, session_id: Any = None, aggregator_output_fallback: Any = None) -> None: + return self.chat.completions.consume_and_save_trace( + session_id, aggregator_output_fallback=aggregator_output_fallback ) - def __getattr__(self, name: str) -> Any: - if name in MoAClient._DELEGATED: - return getattr(self.chat.completions, name) - raise AttributeError(name) + def last_reference_metrics(self) -> Any: + return self.chat.completions.last_reference_metrics() # Relay table: event -> (primary kwarg, secondary kwarg or None, {cb kwarg: emit kwarg}). @@ -1448,11 +1437,7 @@ class MoAClient: _RELAY_EVENTS: dict[str, tuple[str, str | None, dict[str, str]]] = { "moa.reference": ("label", "text", {"moa_index": "index", "moa_count": "count"}), "moa.progress": ("label", None, {"moa_refs_done": "refs_done", "moa_refs_total": "refs_total"}), - "moa.phase": ( - "aggregator", - None, - {"moa_phase": "phase", "moa_refs_done": "refs_done", "moa_refs_total": "refs_total"}, - ), + "moa.phase": ("aggregator", None, {"moa_phase": "phase", "moa_refs_done": "refs_done", "moa_refs_total": "refs_total"}), "moa.aggregating": ("aggregator", None, {"moa_ref_count": "ref_count"}), } @@ -1482,7 +1467,6 @@ def build_moa_facade(agent, preset_name: Any = None) -> MoAClient: try: from hermes_cli.config import load_config from hermes_cli.moa_config import normalize_moa_config - moa_cfg = normalize_moa_config(load_config().get("moa") or {}) presets = moa_cfg.get("presets") or {} if resolved_preset not in presets: diff --git a/agent/moa_trace.py b/agent/moa_trace.py index 329b28ab7a..06eb962dce 100644 --- a/agent/moa_trace.py +++ b/agent/moa_trace.py @@ -27,7 +27,6 @@ def _traces_enabled_and_dir() -> Optional[Path]: call (once per cache-MISS turn); ``moa.trace_dir`` overrides the default.""" try: from hermes_cli.config import load_config - moa_cfg = (load_config() or {}).get("moa") or {} except Exception: # pragma: no cover - never break a turn over tracing return None diff --git a/agent/plugin_llm.py b/agent/plugin_llm.py index 0eeab9de89..46fd27145b 100644 --- a/agent/plugin_llm.py +++ b/agent/plugin_llm.py @@ -112,11 +112,8 @@ class _TrustPolicy: # The ``allow_*_override`` config keys; each is a same-named ``_TrustPolicy`` field. _OVERRIDE_FLAGS = ( - "allow_provider_override", - "allow_model_override", - "allow_agent_id_override", - "allow_profile_override", - "allow_task_override", + "allow_provider_override", "allow_model_override", "allow_agent_id_override", + "allow_profile_override", "allow_task_override", ) @@ -217,7 +214,6 @@ def _resolve_task_ownership(plugin_id: str) -> tuple[frozenset, frozenset]: builtin: set = set() try: from hermes_cli.plugins import get_plugin_auxiliary_tasks - owned = { e.get("key") for e in get_plugin_auxiliary_tasks() if e.get("plugin") == plugin_id and isinstance(e.get("key"), str) and e.get("key") @@ -226,7 +222,6 @@ def _resolve_task_ownership(plugin_id: str) -> tuple[frozenset, frozenset]: pass try: from hermes_cli.main import _AUX_TASKS - builtin = {k for k, _name, _desc in _AUX_TASKS} except Exception: # pragma: no cover — main import failure pass @@ -664,13 +659,6 @@ def make_plugin_llm_for_test( __all__ = [ - "PluginLlm", - "PluginLlmTextInput", - "PluginLlmImageInput", - "PluginLlmInput", - "PluginLlmUsage", - "PluginLlmCompleteResult", - "PluginLlmStructuredResult", - "PluginLlmTrustError", - "make_plugin_llm_for_test", + "PluginLlm", "PluginLlmTextInput", "PluginLlmImageInput", "PluginLlmInput", "PluginLlmUsage", + "PluginLlmCompleteResult", "PluginLlmStructuredResult", "PluginLlmTrustError", "make_plugin_llm_for_test", ] diff --git a/agent/plugin_stream_hooks.py b/agent/plugin_stream_hooks.py index 7910fc7491..47bfa3ad5b 100644 --- a/agent/plugin_stream_hooks.py +++ b/agent/plugin_stream_hooks.py @@ -77,7 +77,6 @@ def _worker(dispatcher: _ConsumerDispatcher) -> None: def _registered_callbacks(hook_name: str) -> tuple[Callable[..., Any], ...]: try: from hermes_cli import plugins - return plugins.iter_hook_callbacks(hook_name) except Exception: logger.debug("plugin stream hook callback lookup failed: %s", hook_name, exc_info=True) @@ -149,7 +148,6 @@ def stream_reasoning_deltas_enabled() -> bool: """Return True only when the user opted plugins into reasoning deltas.""" try: from hermes_cli import config as config_mod - config = config_mod.load_config() return bool(config_mod.cfg_get(config, "plugins", "stream_reasoning_deltas", default=False)) except Exception: diff --git a/agent/provider_media.py b/agent/provider_media.py index eed99fc5d2..72b77c03e2 100644 --- a/agent/provider_media.py +++ b/agent/provider_media.py @@ -19,7 +19,6 @@ from typing import Dict, Tuple def cache_dir(kind: str) -> Path: """Return ``$HERMES_HOME/cache//``, creating parents as needed.""" from hermes_constants import get_hermes_home - path = get_hermes_home() / "cache" / kind path.mkdir(parents=True, exist_ok=True) return path @@ -57,7 +56,6 @@ def save_url( callers can fall back to the bare URL; a partial file is never left behind. """ import requests - response = requests.get(url, timeout=timeout, stream=True) response.raise_for_status() diff --git a/agent/provider_registry.py b/agent/provider_registry.py index 21d4b8e483..0c563e5c24 100644 --- a/agent/provider_registry.py +++ b/agent/provider_registry.py @@ -20,10 +20,6 @@ from hermes_constants import hermes_home_key P = TypeVar("P") -def strip_key(name: str) -> str: - return name.strip() - - def lower_key(name: str) -> str: return name.strip().lower() @@ -31,7 +27,7 @@ def lower_key(name: str) -> str: class ProviderRegistry(Generic[P]): """Global + per-scope provider map with plugin snapshot/restore support. - ``normalize`` is ``strip_key`` or ``lower_key`` (case-insensitive registries mirror + ``normalize`` is ``str.strip`` or ``lower_key`` (case-insensitive registries mirror how their dispatcher normalizes the configured name). ``builtin_names`` are reserved for in-tree implementations; a collision calls ``on_builtin_collision(key)`` and, if that returns, skips registration. ``logger`` is the owning module's so record names @@ -40,7 +36,7 @@ class ProviderRegistry(Generic[P]): def __init__( self, *, label: str, provider_cls: type, logger: logging.Logger, - normalize: Callable[[str], str] = strip_key, builtin_names: FrozenSet[str] = frozenset(), + normalize: Callable[[str], str] = str.strip, builtin_names: FrozenSet[str] = frozenset(), on_builtin_collision: Optional[Callable[[str], None]] = None, ) -> None: self.label = label @@ -188,7 +184,6 @@ def configured_provider_name(section: str, logger: logging.Logger) -> Optional[s configured: Optional[str] = None try: from hermes_cli.config import load_config_readonly - cfg = load_config_readonly() block = cfg.get(section) if isinstance(cfg, dict) else None raw = block.get("provider") if isinstance(block, dict) else None @@ -199,7 +194,6 @@ def configured_provider_name(section: str, logger: logging.Logger) -> Optional[s if configured: try: from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER - if configured.lower() == NOUS_MANAGED_PROVIDER: configured = "fal" except Exception: # pragma: no cover — helpers are in-repo From 21274c852367a6bfbdc915cba28b337cb5169b0f Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:12:17 -0700 Subject: [PATCH 7/9] refactor(agent): pack import lists (AST-identical) --- agent/moa_loop.py | 14 +++----------- agent/nous_rate_guard.py | 7 +------ 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/agent/moa_loop.py b/agent/moa_loop.py index 6fc0af6ac3..15109857d6 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -299,14 +299,9 @@ def _maybe_apply_moa_cache_control( Returns the messages unchanged on any error. """ try: - from agent.agent_runtime_helpers import ( - anthropic_prompt_cache_policy, - blank_cache_policy_stub, - ) + from agent.agent_runtime_helpers import anthropic_prompt_cache_policy, blank_cache_policy_stub from agent.prompt_caching import ( - apply_anthropic_cache_control, - effective_cache_ttl, - envelope_tool_part_cache_markers_supported, + apply_anthropic_cache_control, effective_cache_ttl, envelope_tool_part_cache_markers_supported, ) # Explicit kwarg > runtime snapshot (threaded from the live agent) > config. @@ -436,10 +431,7 @@ def _trim_messages_for_reference( if not messages: return messages - from agent.model_metadata import ( - estimate_messages_tokens_rough, - get_model_context_length, - ) + from agent.model_metadata import estimate_messages_tokens_rough, get_model_context_length model = str(slot.get("model") or "") provider = str(runtime.get("provider") or slot.get("provider") or "") diff --git a/agent/nous_rate_guard.py b/agent/nous_rate_guard.py index 15654598a4..3b87c8677a 100644 --- a/agent/nous_rate_guard.py +++ b/agent/nous_rate_guard.py @@ -16,12 +16,7 @@ import time from typing import Any, Mapping, Optional from utils import atomic_write_text from agent.rate_limit_tracker import ( - _BUCKET_TAGS, - _fmt_seconds, - _safe_float, - _safe_int, - has_rate_limit_headers, - lower_headers, + _BUCKET_TAGS, _fmt_seconds, _safe_float, _safe_int, has_rate_limit_headers, lower_headers, ) logger = logging.getLogger(__name__) From 749649c19e07e24686c25708b3447dd31add58d2 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:20:03 -0700 Subject: [PATCH 8/9] =?UTF-8?q?refactor(agent):=20moa=5Floop=20=E2=80=94?= =?UTF-8?q?=20extract=20reference=20window=20resolver,=20module-level=20la?= =?UTF-8?q?te-accounting=20callback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/moa_loop.py | 103 +++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 52 deletions(-) diff --git a/agent/moa_loop.py b/agent/moa_loop.py index 15109857d6..a407949d17 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -8,6 +8,7 @@ context before each model iteration. from __future__ import annotations import contextlib +import functools import hashlib import json import logging @@ -366,8 +367,7 @@ def _run_reference( label = _slot_label(slot) runtime = _slot_runtime(slot) trace_fields = { - "model": slot.get("model"), "provider": runtime.get("provider") or slot.get("provider"), - "temperature": temperature, + "model": slot.get("model"), "provider": runtime.get("provider") or slot.get("provider"), "temperature": temperature, } # The advisory view already stripped the agent's system prompt; this is the only one. messages = [{"role": "system", "content": _REFERENCE_SYSTEM_PROMPT}, *ref_messages] @@ -384,18 +384,15 @@ def _run_reference( ) # Per-slot max_tokens beats the preset-level reference_max_tokens. slot_max_tokens = slot.get("max_tokens") - extra_headers = None - # Normalize provider aliases (github, github-copilot, ...) via the canonical table. + # Copilot gates premium models on request attribution; MoA fan-out serves the + # user's current turn, so mirror the main agent's x-initiator header. from agent.auxiliary_client import _normalize_aux_provider - if _normalize_aux_provider(str(runtime.get("provider") or "")) in ("copilot", "copilot-acp"): - # Copilot gates premium models on request attribution; MoA fan-out serves the - # user's current turn, so mirror the main agent's x-initiator header. - extra_headers = {"x-initiator": "user"} + is_copilot = _normalize_aux_provider(str(runtime.get("provider") or "")) in ("copilot", "copilot-acp") response = call_llm( task="moa_reference", messages=trimmed, temperature=temperature, max_tokens=slot_max_tokens if slot_max_tokens is not None else max_tokens, timeout=reference_timeout, reasoning_config=_slot_reasoning_config(slot), - extra_headers=extra_headers, **runtime, + extra_headers={"x-initiator": "user"} if is_copilot else None, **runtime, ) output_text = _extract_text(response) or "(empty response)" acct = _RefAccounting( @@ -416,6 +413,32 @@ _REFERENCE_DEFAULT_OUTPUT_RESERVE = 8192 _REFERENCE_TRIM_SAFETY_FRACTION = 0.10 +def _reference_context_length(slot: dict[str, Any], runtime: dict[str, Any], cache: Any) -> int | None: + """Context window for a slot, memoized in ``cache`` per (provider, model) when given. + + Failures are cached too so a flaky metadata source is not re-probed per reference. + """ + from agent.model_metadata import get_model_context_length + + model = str(slot.get("model") or "") + provider = str(runtime.get("provider") or slot.get("provider") or "") + key = (provider, model) + has_cache = isinstance(cache, dict) + if has_cache and key in cache: + return cache[key] + try: + context_length = get_model_context_length( + model=model, base_url=str(runtime.get("base_url") or ""), + api_key=str(runtime.get("api_key") or ""), provider=provider, + ) + except Exception: + logger.debug("MoA reference context-length resolution failed for %s", _slot_label(slot)) + context_length = None + if has_cache: + cache[key] = context_length + return context_length + + def _trim_messages_for_reference( messages: list[dict[str, Any]], slot: dict[str, str], runtime: dict[str, Any], *, reserve_output_tokens: int | None = None, context_length_cache: Any = None, @@ -428,45 +451,19 @@ def _trim_messages_for_reference( budget). ``context_length_cache`` memoizes the window per (provider, model); unresolvable windows leave messages unchanged. """ - if not messages: + if not messages or not slot.get("model"): return messages + from agent.model_metadata import estimate_messages_tokens_rough - from agent.model_metadata import estimate_messages_tokens_rough, get_model_context_length - - model = str(slot.get("model") or "") - provider = str(runtime.get("provider") or slot.get("provider") or "") - if not model: - return messages - - cache_key = (provider, model) - has_cache = isinstance(context_length_cache, dict) - if has_cache and cache_key in context_length_cache: - context_length = context_length_cache[cache_key] - else: - try: - context_length = get_model_context_length( - model=model, base_url=str(runtime.get("base_url") or ""), - api_key=str(runtime.get("api_key") or ""), provider=provider, - ) - except Exception: - logger.debug("MoA reference context-length resolution failed for %s", _slot_label(slot)) - context_length = None - if has_cache: - # Cache failures too so a flaky metadata source is not re-probed per reference. - context_length_cache[cache_key] = context_length - + context_length = _reference_context_length(slot, runtime, context_length_cache) if not isinstance(context_length, int) or context_length <= 0: return messages - if not (isinstance(reserve_output_tokens, int) and reserve_output_tokens > 0): reserve_output_tokens = _REFERENCE_DEFAULT_OUTPUT_RESERVE reserve = int(reserve_output_tokens) budget = int(context_length * (1.0 - _REFERENCE_TRIM_SAFETY_FRACTION)) - reserve - if budget <= 0: - return messages - estimated = estimate_messages_tokens_rough(messages) - if estimated <= budget: + if budget <= 0 or estimated <= budget: return messages has_system = messages[0].get("role") == "system" @@ -522,16 +519,19 @@ def _settle_interrupted( else: results[idx] = _placeholder_output(slot, _INTERRUPTED_REFERENCE_NOTE) if late_accounting_sink is not None: - def _record_late(f: Any, _label: str = results[idx][0]) -> None: - try: - _lbl, _txt, _acct = f.result() - except Exception: # pragma: no cover - defensive - return - try: - late_accounting_sink(_label, _acct) - except Exception: # pragma: no cover - defensive - logger.debug("MoA: late accounting sink failed for %s", _label) - future.add_done_callback(_record_late) + future.add_done_callback(functools.partial(_record_late, late_accounting_sink, results[idx][0])) + + +def _record_late(sink: Any, label: str, future: Any) -> None: + """Done-callback for an abandoned reference future: forward its real accounting.""" + try: + _lbl, _txt, acct = future.result() + except Exception: # pragma: no cover - defensive + return + try: + sink(label, acct) + except Exception: # pragma: no cover - defensive + logger.debug("MoA: late accounting sink failed for %s", label) def _run_references_parallel( @@ -550,17 +550,16 @@ def _run_references_parallel( if not reference_models: return [] - results: list[tuple[str, str, Any] | None] = [None] * len(reference_models) + total = len(reference_models) + results: list[tuple[str, str, Any] | None] = [None] * total futures: dict[Any, int] = {} # Propagate the turn's contextvars (approval callbacks, Nous conversation tag). from tools.thread_context import propagate_context_to_thread - total = len(reference_models) completed = 0 executor = ThreadPoolExecutor(max_workers=min(_MAX_REFERENCE_WORKERS, total)) interrupted = False # Shared per-fan-out context-length cache (dict get/set is GIL-atomic). ctx_len_cache: dict[tuple[str, str], int | None] = {} - # Agent's cache disable + TTL for every advisor request; clamped in the decorator. cache_disabled, cache_ttl = _agent_cache_opts(agent) try: for idx, slot in enumerate(reference_models): From 7c08619130a9a8436eef106de63a8f0109a719ae Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:21:50 -0700 Subject: [PATCH 9/9] refactor(agent): moa_loop cadence key/interrupt bookkeeping as expressions --- agent/moa_loop.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/agent/moa_loop.py b/agent/moa_loop.py index a407949d17..f224950503 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -1208,11 +1208,13 @@ class MoAChatCompletions: if fanout_mode == "user_turn" or every_n >= 2: # Last REAL user message: the synthetic _ADVISORY_INSTRUCTION marker must not # count or the prefix would grow (and re-sign) every iteration. - for i in range(len(ref_messages) - 1, -1, -1): - m = ref_messages[i] - if m.get("role") == "user" and m.get("content") != _ADVISORY_INSTRUCTION: - turn_prefix = ref_messages[: i + 1] - break + last_user = next( + (i for i in range(len(ref_messages) - 1, -1, -1) + if ref_messages[i].get("role") == "user" and ref_messages[i].get("content") != _ADVISORY_INSTRUCTION), + None, + ) + if last_user is not None: + turn_prefix = ref_messages[: last_user + 1] if fanout_mode == "user_turn": sig_messages = turn_prefix @@ -1260,14 +1262,11 @@ class MoAChatCompletions: reference_timeout=float(raw_reference_timeout) if raw_reference_timeout else None, agent=self._agent, late_accounting_sink=self._record_late_reference_accounting, ) - if any(text == _INTERRUPTED_REFERENCE_NOTE for _lbl, text, _acct in reference_outputs): - # An interrupted fan-out is a partial snapshot: never cache it (a HIT - # would replay placeholder notes every iteration). - self._ref_cache_key = None - self._ref_cache_outputs = [] - else: - self._ref_cache_key = cache_key - self._ref_cache_outputs = list(reference_outputs) + # An interrupted fan-out is a partial snapshot: never cache it (a HIT would + # replay placeholder notes every iteration). + interrupted = any(text == _INTERRUPTED_REFERENCE_NOTE for _lbl, text, _acct in reference_outputs) + self._ref_cache_key = None if interrupted else cache_key + self._ref_cache_outputs = [] if interrupted else list(reference_outputs) # Fold advisor spend into accounting exactly once per turn. self._fold_pending_accounting(*_sum_reference_accounting(reference_outputs)) # Stash the fan-out for trace persistence (aggregator parts filled in later).