diff --git a/agent/client_lifecycle.py b/agent/client_lifecycle.py index 2fc520fa06..9f2202c37f 100644 --- a/agent/client_lifecycle.py +++ b/agent/client_lifecycle.py @@ -606,8 +606,9 @@ class ClientLifecycleMixin: api_key, base_url = creds.get("api_key"), creds.get("base_url") if not _valid_credential_pair(api_key, base_url): return False - if str(api_key).strip() == str(self.api_key or "").strip(): - return False # store holds the same key: nothing to adopt, no client rebuild + if (str(api_key).strip() == str(self.api_key or "").strip() + and str(base_url).strip().rstrip("/") == str(self.base_url or "").strip().rstrip("/")): + return False # store holds the same key on the same route: nothing to adopt, no client rebuild if require_account is not None: try: from hermes_cli.auth_constants import _decode_jwt_claims diff --git a/agent/error_classifier.py b/agent/error_classifier.py index 1b071843a2..d35dfca763 100644 --- a/agent/error_classifier.py +++ b/agent/error_classifier.py @@ -509,6 +509,7 @@ class _Ctx: approx_tokens: int context_length: int num_messages: int + base_url: str = "" # the route the call went to; "" when the caller did not say def __post_init__(self) -> None: self.error_type = type(self.error).__name__ @@ -568,7 +569,7 @@ def _nous_welcome_tier(c: _Ctx) -> Optional[Verdict]: if refusal["retry_after"] > 0: ctx["reset_at"] = time.time() + refusal["retry_after"] return _v(_R.rate_limit, should_fallback=True, error_context=ctx) - kind = welcome_route_refusal(status, c.msg) + kind = welcome_route_refusal(status, c.msg, c.base_url if c.provider == "nous" else None) if kind is None: return None ctx = {"welcome_route": kind} @@ -704,8 +705,12 @@ _STAGES: Sequence[Callable[[_Ctx], Optional[Verdict]]] = ( def classify_api_error( error: Exception, *, provider: str = "", model: str = "", approx_tokens: int = 0, context_length: int = 200000, num_messages: int = 0, + base_url: str = "", ) -> ClassifiedError: - """Classify an API error into a structured recovery recommendation (see ``_STAGES``).""" + """Classify an API error into a structured recovery recommendation (see ``_STAGES``). + + ``base_url`` (optional) is the route the call went to; the Nous welcome tier keys its + dark-tier 403 on it because that refusal carries no distinguishing message.""" status_code = _extract_status_code(error) # Copilot/GitHub Models RateLimitError may not set .status_code; force 429. if status_code is None and type(error).__name__ == "RateLimitError": @@ -713,7 +718,7 @@ def classify_api_error( body = _extract_error_body(error) c = _Ctx( error, status_code, body, _build_error_msg(error, body), provider, model, - approx_tokens, context_length, num_messages, + approx_tokens, context_length, num_messages, str(base_url or ""), ) verdict = next((v for v in (stage(c) for stage in _STAGES) if v is not None), _V_UNKNOWN) base = {"status_code": status_code, "provider": provider, "model": model, "message": _extract_message(error, body)} diff --git a/agent/nous_rate_guard.py b/agent/nous_rate_guard.py index 6d24c9c3a3..834027cec9 100644 --- a/agent/nous_rate_guard.py +++ b/agent/nous_rate_guard.py @@ -25,6 +25,11 @@ logger = logging.getLogger(__name__) # Reset windows shorter than this are transient upstream jitter, not a quota # exhaustion worth a cross-session breaker trip. _MIN_RESET_FOR_BREAKER_SECONDS = 60.0 +# The welcome tier's structured ``rate_limited`` refusal: a reset at or above this is an exhausted +# allowance (stop, tell the user when it refreshes and that signing in lifts it); below it the +# turn simply waits it out. The gateway's fairshare bucket names honest resets from a few seconds +# up to a minute, so the split sits where a quiet wait stops being quiet. +WELCOME_LONG_WAIT_SECONDS = 20.0 format_remaining = _fmt_seconds @@ -131,6 +136,19 @@ def is_genuine_nous_rate_limit( return last_known_state is not None and _has_exhausted_bucket_in_object(last_known_state) +def is_long_welcome_rate_limit(error_context: Any) -> bool: + """True for a Nous welcome-tier ``rate_limited`` refusal whose reset is long enough to be an + exhausted allowance (``WELCOME_LONG_WAIT_SECONDS``), as parsed into ``error_context`` + (``welcome_refusal`` from ``hermes_cli.anon_auth.parse_welcome_refusal``). Capacity refusals + (``at_capacity`` / ``admission_closed``) are never this: they are retried in place.""" + if not isinstance(error_context, dict): + return False + refusal = error_context.get("welcome_refusal") + if not isinstance(refusal, dict) or refusal.get("reason") != "rate_limited": + return False + return _safe_float(refusal.get("retry_after"), 0.0) >= WELCOME_LONG_WAIT_SECONDS + + def _parse_buckets_from_headers( headers: Optional[Mapping[str, str]], ) -> dict[str, tuple[Optional[int], Optional[float]]]: diff --git a/agent/turn_api_call.py b/agent/turn_api_call.py index 4649ccfff2..bf3ecec973 100644 --- a/agent/turn_api_call.py +++ b/agent/turn_api_call.py @@ -236,8 +236,10 @@ def nous_rate_limit_guard( if _nous_remaining is not None and _nous_remaining > 0: from hermes_cli import anon_auth reset = _fmt_nous_remaining(_nous_remaining) - if anon_auth.route_is_welcome_host(getattr(agent, "base_url", "")): - _nous_msg = anon_auth.FREE_TIER_RATE_LIMIT_CHAT.format(reset=reset) + _welcome = anon_auth.route_is_welcome_host(getattr(agent, "base_url", "")) + if _welcome: + _nous_msg = anon_auth.FREE_TIER_RATE_LIMIT_CHAT.format( + reset=anon_auth.friendly_wait(_nous_remaining)) else: _nous_msg = f"Your Nous account has hit its rate limit; it resets in {reset}." agent._buffer_vprint(f"⏳ {_nous_msg} Trying fallback...") @@ -251,8 +253,11 @@ def nous_rate_limit_guard( # No fallback — surface the buffered rate-limit context that led here. agent._flush_status_buffer() agent._persist_session(messages, conversation_history) + # The free tier's sentence already says what to do (wait, or sign in); the + # fallback-provider advice is for an install that runs its own providers. return _verdict("return", stamp_failure({ - "final_response": f"⏳ {_nous_msg}\n\n{site_copy('nous_rate_limit')}", + "final_response": (f"⏳ {_nous_msg}" if _welcome + else f"⏳ {_nous_msg}\n\n{site_copy('nous_rate_limit')}"), "messages": messages, "api_calls": api_call_count, "completed": False, diff --git a/agent/turn_api_error.py b/agent/turn_api_error.py index 27b383f6f9..9905d6774f 100644 --- a/agent/turn_api_error.py +++ b/agent/turn_api_error.py @@ -110,6 +110,7 @@ def handle_api_error( api_error, provider=getattr(agent, "provider", "") or "", model=getattr(agent, "model", "") or "", approx_tokens=approx_tokens, context_length=_ctx_len, num_messages=len(api_messages) if api_messages else 0, + base_url=str(getattr(agent, "base_url", "") or ""), ) logger.debug( "Error classified: reason=%s status=%s retryable=%s compress=%s rotate=%s fallback=%s", diff --git a/agent/turn_recovery.py b/agent/turn_recovery.py index a889465646..e0ac112910 100644 --- a/agent/turn_recovery.py +++ b/agent/turn_recovery.py @@ -267,6 +267,16 @@ def _print_nous_401_diagnostics(agent: Any, api_error: Exception) -> None: _plines(agent, "🔐 Nous 401 — Portal authentication failed.") if _body_text: _plines(agent, f" Response: {_body_text}") + try: + from hermes_cli.anon_auth import route_is_welcome_host + if route_is_welcome_host(getattr(agent, "base_url", "")): + # The free tier has no credits, no agent key and no auth.json to inspect: its session + # ended and could not be replaced. The two doors are a sign-in or another provider. + _plines(agent, " Your session ended and Hermes couldn't start a new one.", + " Sign in with a Nous account (it's free), or switch providers with /model.") + return + except Exception: + pass if not _print_nous_entitlement_guidance(agent, "Nous model access"): _plines(agent, " Most likely: Portal OAuth expired, account out of credits, or agent key revoked.") _plines( @@ -470,6 +480,42 @@ def _recover_format_errors( return False +def _recover_welcome_tier(agent: Any, classified: Any, _retry: TurnRetryState, error_context: Any) -> bool: + """Two one-shot repairs for the Nous free tier, both silent on the wire and named once in chat. + + ``model_not_free``: the session asked the welcome host for a model it does not serve; move + to the first alternate the gateway named (its own model) and retry, instead of failing the + turn. ``anon_on_paid_host``: this process is pointed at the paid host with a free-tier + identity (a stale route); re-read the credentials, which heals the URL, and retry.""" + ctx = error_context if isinstance(error_context, dict) else (getattr(classified, "error_context", None) or {}) + refusal = ctx.get("welcome_refusal") if isinstance(ctx, dict) else None + if isinstance(refusal, dict) and refusal.get("reason") == "model_not_free" and not _retry.welcome_model_switch_attempted: + _retry.welcome_model_switch_attempted = True + alternates = [a for a in (refusal.get("alternates") or []) if isinstance(a, str) and a] + requested = str(getattr(agent, "model", "") or "") + target = alternates[0] if alternates else None + if target and target != requested: + try: + agent.model = target + agent._nous_model_switch = (requested, target) + except Exception: + return False + _vlines(agent, f"↪️ {requested} isn't available without signing in; using {target} for now. Retrying...") + logger.info("%sNous free tier: moved %s -> %s after model_not_free", agent.log_prefix, requested, target) + return True + route = ctx.get("welcome_route") if isinstance(ctx, dict) else None + if route == "anon_on_paid_host" and not _retry.welcome_route_heal_attempted: + _retry.welcome_route_heal_attempted = True + try: + healed = bool(agent._try_refresh_nous_client_credentials(force=True)) + except Exception: + healed = False + if healed: + _vlines(agent, "🔐 Reconnected to the free model's own route. Retrying request...") + return True + return False + + def recover_after_classification( agent: Any, api_error: Exception, classified: Any, _retry: TurnRetryState, *, status_code: Optional[int], error_context: Any, messages: List[Dict[str, Any]], @@ -483,6 +529,9 @@ def recover_after_classification( Returns ``(retry_now, recovered_with_pool)``; the latter feeds the Nous rate-limit guard.""" from agent.conversation_loop import _is_nous_inference_route + if _recover_welcome_tier(agent, classified, _retry, error_context): + return True, False + if ( classified.reason == FailoverReason.billing and _is_nous_inference_route( @@ -666,6 +715,22 @@ def _welcome_tier_guidance(classified: Any, *, model: Any, in_chat: bool) -> str return welcome_route_refusal_copy(str(route), in_chat=in_chat) +def _welcome_outage_copy(base_url: Any, classified: Any) -> str: + """On the Nous free tier, a transport / server failure that outlived every retry reads as one + plain sentence (the free model is having trouble) rather than the technical summary. Empty + for every other route and for rate limits / billing, which have their own copy.""" + try: + from hermes_cli.anon_auth import FREE_TIER_OUTAGE_COPY, route_is_welcome_host + if not route_is_welcome_host(base_url): + return "" + if classified.reason in (FailoverReason.timeout, FailoverReason.overloaded, + FailoverReason.server_error, FailoverReason.unknown): + return FREE_TIER_OUTAGE_COPY + except Exception: + pass + return "" + + # Terminal status label per non-retryable reason (default names the HTTP status). _NONRETRYABLE_LABELS = { FailoverReason.content_policy_blocked: "The provider's safety filter refused this request", @@ -773,7 +838,9 @@ def nonretryable_client_error_result( api_call_count=api_call_count, provider=provider, base_url=base_url, model=model, ) if _welcome_hint: - _final_response = f"{_nonretryable_summary}\n\n{_welcome_tier_guidance(classified, model=model, in_chat=True)}" + # A free-tier refusal is fully explained by its own sentence; the raw provider summary + # (status codes, JSON) is for the log, not for a first-time user's chat. + _final_response = _welcome_tier_guidance(classified, model=model, in_chat=True) else: # Every surface reads final_response; the CLI hint lines above never reach chat. _final_response = nonretryable_copy( @@ -889,7 +956,9 @@ def max_retries_exhausted_result( summary=_final_summary, ) if _welcome_hint: - _final_response += f"\n\n{_welcome_tier_guidance(classified, model=model, in_chat=True)}" + _final_response = _welcome_tier_guidance(classified, model=model, in_chat=True) + else: + _final_response = _welcome_outage_copy(base_url, classified) or _final_response if _is_thinking_timeout: # Thinking-timeout guidance overrides stream-drop guidance, which would wrongly # suggest splitting large file writes. @@ -1297,10 +1366,16 @@ def _is_genuine_nous_rate_limit(agent: Any, api_error: Exception, error_context: capacity 429s (no exhausted bucket in headers or last-known state) are left alone.""" _genuine = False try: - from agent.nous_rate_guard import is_genuine_nous_rate_limit, record_nous_rate_limit + from agent.nous_rate_guard import ( + is_genuine_nous_rate_limit, is_long_welcome_rate_limit, record_nous_rate_limit) _err_resp = getattr(api_error, "response", None) _err_hdrs = getattr(_err_resp, "headers", None) if _err_resp else None - _genuine = is_genuine_nous_rate_limit(headers=_err_hdrs, last_known_state=agent._rate_limit_state) + # The welcome tier's structured ``rate_limited`` refusal names its own reset; a long one + # is an exhausted allowance whatever the headers say, and the one place the user is told + # that signing in lifts it. + _genuine = ( + is_long_welcome_rate_limit(error_context) + or is_genuine_nous_rate_limit(headers=_err_hdrs, last_known_state=agent._rate_limit_state)) if _genuine: record_nous_rate_limit(headers=_err_hdrs, error_context=error_context) else: diff --git a/agent/turn_retry_state.py b/agent/turn_retry_state.py index 24a28bffbe..243b8a2c75 100644 --- a/agent/turn_retry_state.py +++ b/agent/turn_retry_state.py @@ -19,6 +19,10 @@ class TurnRetryState: anthropic_auth_retry_attempted: bool = False nous_auth_retry_attempted: bool = False nous_paid_entitlement_refresh_attempted: bool = False + # Nous free tier: one model move onto the tier's own model after a ``model_not_free`` + # refusal, and one route re-read after a wrong-host refusal (``anon_on_paid_host``). + welcome_model_switch_attempted: bool = False + welcome_route_heal_attempted: bool = False copilot_auth_retry_attempted: bool = False # Copilot surfaces a stale credential as a 400 ``model_not_available_for_integrator`` # / ``model_not_supported``, not a 401 — separate guard from the 401 one. diff --git a/apps/desktop/AGENTS.md b/apps/desktop/AGENTS.md index 73ce47a2d4..4951e96d12 100644 --- a/apps/desktop/AGENTS.md +++ b/apps/desktop/AGENTS.md @@ -204,6 +204,13 @@ From `apps/desktop`, use a fresh temporary directory for each rehearsal and run `HERMES_PORTAL_BASE_URL=http://127.0.0.1:8765 HERMES_ANON_API_SECRET=test-secret HERMES_SHARED_AUTH_DIR=/.hermes/shared` before `npm run dev`. Stop Electron and its dev server after the run. +To rehearse the free tier's failure handling (the account service refusing or +unreachable at boot, the welcome host rate-limiting or refusing mid-chat), run +`python scripts/free_tier_fault_server.py` from the repo root and start the +desktop with the environment it prints; switch scenarios from the desktop's +JavaScript console. The walkthrough is in +`website/docs/developer-guide/free-tier-fault-rehearsal.md`. + ## The taste test before you hand off - Does every piece of state live with its authority, at the narrowest scope? diff --git a/apps/desktop/src/components/free-tier/sign-in-dialog.tsx b/apps/desktop/src/components/free-tier/sign-in-dialog.tsx index 1983739b8a..1e0d97a574 100644 --- a/apps/desktop/src/components/free-tier/sign-in-dialog.tsx +++ b/apps/desktop/src/components/free-tier/sign-in-dialog.tsx @@ -18,7 +18,7 @@ import { import { getGlobalModelOptions } from '@/hermes' import { type Translations, useI18n } from '@/i18n' import { CheckCircle2, Loader2 } from '@/lib/icons' -import { FREE_TIER_MODEL, NOUS_PROVIDER_ID, refreshFreeTierStatus } from '@/store/free-tier' +import { FREE_TIER_MODEL, friendlyWait, NOUS_PROVIDER_ID, refreshFreeTierStatus } from '@/store/free-tier' import { $freeTierSignIn, beginFreeTierSignIn, @@ -188,14 +188,21 @@ export function FreeTierSignInDialog({ onSelectModel }: FreeTierSignInDialogProp )} {state.status === 'failed' && ( - + - + {/* A terminal refusal (this version, a locked session, a proof-of-work + request) has nothing to retry: the backend's sentence names the way out. */} + {state.kind === 'unavailable' ? null : ( + + )} )} @@ -207,11 +214,25 @@ export function FreeTierSignInDialog({ onSelectModel }: FreeTierSignInDialogProp type FreeTierCopy = Translations['freeTier'] function failureHeading(kind: FreeTierSignInFailure, copy: FreeTierCopy): string { - return kind === 'timed_out' ? copy.timedOutHeading : copy.didNotComplete + switch (kind) { + case 'busy': + return copy.busyHeading + + case 'timed_out': + return copy.timedOutHeading + + default: + return copy.didNotComplete + } } -function failureBody(kind: FreeTierSignInFailure, message: null | string, copy: FreeTierCopy): string { +function failureBody(kind: FreeTierSignInFailure, message: null | string, retryAfter: number, copy: FreeTierCopy): string { switch (kind) { + case 'busy': + // The backend's sentence already names the wait it was given; ours fills + // in when an older backend sent none. + return message ?? copy.busyBody(friendlyWait(retryAfter || 60)) + case 'rejected': return copy.rejectedBody @@ -224,9 +245,12 @@ function failureBody(kind: FreeTierSignInFailure, message: null | string, copy: case 'timed_out': return copy.timedOutBody + case 'unreachable': + return message ?? copy.unreachableBody + default: // The backend's own wording when it sent one — it names the specific - // refusal (a busy account, a transport failure) better than we can. + // refusal (this version, a locked session) better than we can. return message ?? copy.errorBody } } diff --git a/apps/desktop/src/components/onboarding/free-tier-setup-notice.test.tsx b/apps/desktop/src/components/onboarding/free-tier-setup-notice.test.tsx new file mode 100644 index 0000000000..57a5ec2167 --- /dev/null +++ b/apps/desktop/src/components/onboarding/free-tier-setup-notice.test.tsx @@ -0,0 +1,136 @@ +import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { en } from '@/i18n/en' +import { $freeTierStatus, freeTierSetupFailure } from '@/store/free-tier' +import type { OnboardingContext } from '@/store/onboarding' +import type { FreeTierStatus } from '@/types/hermes' + +import { FreeTierSetupNotice, setupFailureCopy } from './free-tier-setup-notice' + +const NO_IDENTITY: FreeTierStatus = { + available: false, + enabled: true, + has_guest: false, + label: 'Nous · free tier', + model: 'nous/welcome', + notice_pending: false +} + +function ctxReturning(status: FreeTierStatus, provisioned?: FreeTierStatus): OnboardingContext & { calls: string[] } { + const calls: string[] = [] + let current = status + + return { + calls, + requestGateway: async (method: string): Promise => { + calls.push(method) + + if (method === 'free_tier.provision' && provisioned) { + current = provisioned + } + + if (method === 'free_tier.status') { + return current as T + } + + return { provider_configured: current.has_guest, ok: current.has_guest } as T + } + } +} + +afterEach(() => { + cleanup() + $freeTierStatus.set(null) + vi.restoreAllMocks() +}) + +describe('setupFailureCopy', () => { + const copy = en.freeTier.setupFailed + + it.each([ + ['anon_gate_closed', copy.gateClosed], + ['anon_gate_paused', copy.paused], + ['anon_unreachable', copy.unreachable], + ['anon_server_error', copy.serverError], + ['anon_pow_required', copy.powRequired], + ['anon_account_locked', copy.locked] + ])('%s has its own sentence', (code, expected) => { + const failure = freeTierSetupFailure({ ...NO_IDENTITY, error_code: code }) + + expect(failure && setupFailureCopy(failure, copy)).toBe(expected) + }) + + it('speaks the wait for a rate limit', () => { + const failure = freeTierSetupFailure({ ...NO_IDENTITY, error_code: 'anon_rate_limited', retry_after: 300 }) + + expect(failure && setupFailureCopy(failure, copy)).toContain('about 5 minutes') + }) + + it('falls back to the backend sentence for a code this build does not know', () => { + const failure = freeTierSetupFailure({ ...NO_IDENTITY, error: 'Something new.', error_code: 'anon_newer' }) + + expect(failure && setupFailureCopy(failure, copy)).toBe('Something new.') + }) + + it('never says the free service or the free model is off', () => { + for (const code of Object.keys(copy)) { + const failure = freeTierSetupFailure({ ...NO_IDENTITY, error_code: `anon_${code}` }) + const text = failure ? setupFailureCopy(failure, copy).toLowerCase() : '' + + expect(text).not.toMatch(/free (service|model|tier) is (off|switched off|unavailable|down)/) + expect(text).not.toMatch(/anonymous|guest|credential|token|rate limit/) + } + }) +}) + +describe('FreeTierSetupNotice', () => { + it('renders nothing when the backend reported no failure', async () => { + const ctx = ctxReturning(NO_IDENTITY) + render() + + await waitFor(() => expect(ctx.calls).toContain('free_tier.status')) + expect(screen.queryByTestId('free-tier-setup-notice')).toBeNull() + }) + + it('shows the sentence, the sign-in door, and a retry for a retryable refusal', async () => { + const ctx = ctxReturning({ ...NO_IDENTITY, error_code: 'anon_gate_paused', retryable: true, retry_after: 60 }) + render() + + await screen.findByTestId('free-tier-setup-notice') + expect(screen.getByText(en.freeTier.setupFailed.paused)).toBeTruthy() + expect(screen.getByText(en.freeTier.setupFailed.signInBelow)).toBeTruthy() + expect(screen.getByRole('button', { name: en.freeTier.setupFailed.tryAgain })).toBeTruthy() + }) + + it('offers no sign-in door and no retry when the service is unreachable and the code is terminal', async () => { + const ctx = ctxReturning({ ...NO_IDENTITY, error_code: 'anon_gate_closed', retryable: false }) + render() + + await screen.findByTestId('free-tier-setup-notice') + expect(screen.queryByRole('button')).toBeNull() + + cleanup() + $freeTierStatus.set(null) + const unreachable = ctxReturning({ ...NO_IDENTITY, error_code: 'anon_unreachable', retryable: true, retry_after: 15 }) + render() + + await screen.findByTestId('free-tier-setup-notice') + expect(screen.queryByText(en.freeTier.setupFailed.signInBelow)).toBeNull() + expect(screen.getByRole('button', { name: en.freeTier.setupFailed.tryAgain })).toBeTruthy() + }) + + it('the retry asks the backend once and the notice leaves when an identity appears', async () => { + const ctx = ctxReturning( + { ...NO_IDENTITY, error_code: 'anon_unreachable', retryable: true, retry_after: 15 }, + { ...NO_IDENTITY, available: true, has_guest: true } + ) + + render() + + fireEvent.click(await screen.findByRole('button', { name: en.freeTier.setupFailed.tryAgain })) + + await waitFor(() => expect(screen.queryByTestId('free-tier-setup-notice')).toBeNull()) + expect(ctx.calls.filter(method => method === 'free_tier.provision')).toHaveLength(1) + }) +}) diff --git a/apps/desktop/src/components/onboarding/free-tier-setup-notice.tsx b/apps/desktop/src/components/onboarding/free-tier-setup-notice.tsx new file mode 100644 index 0000000000..446156bfd8 --- /dev/null +++ b/apps/desktop/src/components/onboarding/free-tier-setup-notice.tsx @@ -0,0 +1,113 @@ +import { useStore } from '@nanostores/react' +import { useEffect, useState } from 'react' + +import { Button } from '@/components/ui/button' +import { type Translations, useI18n } from '@/i18n' +import { + $freeTierStatus, + type FreeTierSetupFailure, + freeTierSetupFailure, + friendlyWait, + provisionFreeTier, + refreshFreeTierStatus +} from '@/store/free-tier' +import { type OnboardingContext, refreshOnboarding } from '@/store/onboarding' + +type SetupFailedCopy = Translations['freeTier']['setupFailed'] + +/** One sentence per backend code. The backend's own sentence is the fallback for + * a code this build does not know, so a newer backend still reads as words. */ +export function setupFailureCopy(failure: FreeTierSetupFailure, copy: SetupFailedCopy): string { + switch (failure.code) { + case 'anon_gate_closed': + return copy.gateClosed + + case 'anon_gate_paused': + return copy.paused + + case 'anon_rate_limited': + return copy.rateLimited(friendlyWait(failure.retryAfter || 60)) + + case 'anon_unreachable': + return copy.unreachable + + case 'anon_server_error': + return copy.serverError + + case 'anon_pow_required': + return copy.powRequired + + case 'anon_account_locked': + return copy.locked + + default: + return failure.message || copy.generic + } +} + +/** + * The first-launch notice for a free tier that could not be set up: the boot + * bootstrap tried, the account service refused or could not be reached, and + * the user is looking at the provider picker with no idea why. Says what + * happened in one sentence, offers the user's own retry when a later attempt + * can succeed, and points at the Nous row below when signing in can help + * (never when the same service is the one that is unreachable). + * + * Renders nothing unless the backend reported a failure, so an older backend + * or a healthy boot leaves the picker exactly as it was. + */ +export function FreeTierSetupNotice({ ctx }: { ctx: OnboardingContext }) { + const { t } = useI18n() + const status = useStore($freeTierStatus) + const failure = freeTierSetupFailure(status) + const [retrying, setRetrying] = useState(false) + const copy = t.freeTier.setupFailed + + // The verdict is a local, zero-network read; make sure it is fresh for the + // picker even before the ambient status round has run. + useEffect(() => { + void refreshFreeTierStatus(ctx.requestGateway) + }, [ctx]) + + if (!failure) { + return null + } + + const retry = async () => { + if (retrying) { + return + } + + setRetrying(true) + + try { + const next = await provisionFreeTier(ctx.requestGateway) + + if (next?.has_guest) { + // The identity exists now: re-run the readiness round so the picker + // gives way to the ready screen. + await refreshOnboarding(ctx) + } + } finally { + setRetrying(false) + } + } + + return ( +
+

{setupFailureCopy(failure, copy)}

+ {failure.door === 'sign_in' ?

{copy.signInBelow}

: null} + {failure.retryable ? ( +
+ +
+ ) : null} +
+ ) +} diff --git a/apps/desktop/src/components/onboarding/index.tsx b/apps/desktop/src/components/onboarding/index.tsx index b40b504ef5..c64f385a37 100644 --- a/apps/desktop/src/components/onboarding/index.tsx +++ b/apps/desktop/src/components/onboarding/index.tsx @@ -14,9 +14,10 @@ import { isSubmitEnter } from '@/lib/ime' import { isProviderSetupErrorMessage } from '@/lib/provider-setup-errors' import { cn } from '@/lib/utils' import { $desktopBoot, type DesktopBootState } from '@/store/boot' -import { FREE_TIER_MODEL } from '@/store/free-tier' +import { $freeTierStatus, FREE_TIER_MODEL, freeTierSetupFailure } from '@/store/free-tier' import { openFreeTierSignIn } from '@/store/free-tier-sign-in' import { $introReveal, shouldPlayFirstRunIntro } from '@/store/intro-reveal' +import { $setupReadyTick } from '@/store/live-sync' import { $localModelsEnabled } from '@/store/local-models-flag' import { $desktopOnboarding, @@ -40,6 +41,7 @@ import { $onboardingSurfaces, onboardingSurfaceActive } from '@/store/onboarding import type { OAuthProvider } from '@/types/hermes' import { DocsLink, FlowPanel, Status } from './flow' +import { FreeTierSetupNotice } from './free-tier-setup-notice' import { DecodedLabel } from './glyph' import { FeaturedProviderRow, @@ -282,6 +284,25 @@ export function DesktopOnboardingOverlay({ } }, [ctx, enabled, onboarding.requested]) + // The boot bootstrap re-announces `setup.ready` when a background retry of + // the free-tier set-up succeeds after a failed first attempt. A picker that + // is up only because that set-up failed re-checks readiness and gives way + // on its own; a manual open, or a picker the user is mid-flow in, is left + // alone. + useEffect( + () => + $setupReadyTick.listen(() => { + const current = $desktopOnboarding.get() + + if (!current.manual && current.configured === false && current.flow.status === 'idle') { + void refreshOnboarding(ctx) + } + }), + [ctx] + ) + const freeTierStatus = useStore($freeTierStatus) + const setupFailure = !onboarding.manual ? freeTierSetupFailure(freeTierStatus) : null + // When the Providers settings page asked to connect a specific provider, the // store stashed its id. Once the provider list has loaded and we're back at // an idle picker, launch that exact OAuth flow so the user lands directly in @@ -341,8 +362,12 @@ export function DesktopOnboardingOverlay({ // (those are surfaced by FlowPanel, not as a banner). const rawReason = onboarding.reason?.trim() || null + // When the free tier itself failed to set up, its own notice explains the + // picker; the runtime check's technical reason ("No usable credentials + // found for nous.") would only restate it in the wrong words. const reason = rawReason && + !setupFailure && !isProviderSetupErrorMessage(rawReason) && rawReason !== DEFAULT_ONBOARDING_REASON && rawReason !== DEFAULT_MANUAL_ONBOARDING_REASON @@ -403,6 +428,7 @@ export function DesktopOnboardingOverlay({ ) : null}
{reason ? : null} + {ready && showPicker && !freeTierIntro && !onboarding.manual ? : null} {ready ? ( freeTierIntro ? ( diff --git a/apps/desktop/src/i18n/en.ts b/apps/desktop/src/i18n/en.ts index 49f196207a..70c0fe92e5 100644 --- a/apps/desktop/src/i18n/en.ts +++ b/apps/desktop/src/i18n/en.ts @@ -3477,15 +3477,39 @@ export const en: Translations = { notNow: 'Not now', tryAgain: 'Try again', startAgain: 'Start again', - didNotComplete: 'Sign-in did not complete', - rejectedBody: 'Sign-in was rejected in the browser. You are still on the free tier.', - supersededBody: 'A newer sign-in code replaced this one.', - timedOutHeading: 'Sign-in timed out', - timedOutBody: 'The code was not used in time. You are still on the free tier.', - retiredBody: 'This free-tier identity was already used or expired; a new one is set up on the next start.', - errorBody: 'Sign-in did not complete; run it again.', + didNotComplete: "Sign-in didn't finish", + rejectedBody: "No problem, you're still on the free Nous service. Sign in whenever you're ready.", + supersededBody: 'A newer sign-in code replaced this one. Use the newest one, or start again.', + timedOutHeading: 'That sign-in link has expired', + timedOutBody: "Start again whenever you're ready. You're still on the free Nous service.", + retiredBody: + 'Your session ended before the sign-in finished. Hermes will start a new one; then sign in again whenever you\'re ready.', + errorBody: "Sign-in didn't finish. Try again whenever you're ready.", + busyHeading: 'Almost there', + busyBody: wait => + `Hermes couldn't finish signing you in because the Nous service is busy. Try again in ${wait}. Your session is still here in the meantime.`, + unreachableBody: + "Hermes couldn't reach the Nous service to finish signing you in. Check your internet connection and try again. Your session is still here.", alreadySignedInHeading: 'Already signed in.', - alreadySignedInBody: 'This Hermes is already signed in to a Nous account.' + alreadySignedInBody: 'This Hermes is already signed in to a Nous account.', + setupFailed: { + gateClosed: + "This version of Hermes can't start without a Nous account. Sign in or create one, it's free and only takes a minute.", + paused: + 'Using Hermes without signing in is paused for a moment. Hermes will keep checking. Signing in is free and gets you going right now.', + rateLimited: wait => + `Lots of people are getting started right now, so Hermes will try again in ${wait}. Signing in is free and skips the wait.`, + unreachable: + "Hermes couldn't reach the Nous service. Check your internet connection, then tap Try again. Or connect another provider for now.", + serverError: 'The Nous service had a hiccup. Tap Try again in a moment, or connect another provider for now.', + powRequired: + "The Nous server asked for a proof of work, but that isn't implemented in your Agent yet. Sign in or create a free Nous account to continue.", + locked: "This session can't continue without signing in. Sign in or create a free Nous account to keep going.", + generic: "Hermes couldn't set up free access without signing in. Signing in is free, or connect another provider.", + signInBelow: 'Signing in is free and keeps the free model. Pick Nous below.', + tryAgain: 'Try again', + retrying: 'Trying again…' + } }, modelPicker: { diff --git a/apps/desktop/src/i18n/types.ts b/apps/desktop/src/i18n/types.ts index c70f0ad0dc..baf1bf3263 100644 --- a/apps/desktop/src/i18n/types.ts +++ b/apps/desktop/src/i18n/types.ts @@ -3015,8 +3015,30 @@ export interface Translations { timedOutBody: string retiredBody: string errorBody: string + /** The account service asked for a short wait mid sign-in (a busy account, a rate limit, the ops pause). */ + busyHeading: string + busyBody: (wait: string) => string + /** The account service could not be reached or errored mid sign-in. */ + unreachableBody: string alreadySignedInHeading: string alreadySignedInBody: string + // First-launch set-up failure notice: the free tier could not be created at boot. + // One sentence per backend code (`hermes_cli/anon_auth.py::ANON_*`); the copy never says + // the free MODEL is off — what is unavailable is using Hermes without signing in. + setupFailed: { + gateClosed: string + paused: string + rateLimited: (wait: string) => string + unreachable: string + serverError: string + powRequired: string + locked: string + generic: string + /** The sign-in door, when the account service is reachable: the Nous row sits right below. */ + signInBelow: string + tryAgain: string + retrying: string + } } modelPicker: { diff --git a/apps/desktop/src/lib/runtime-readiness.ts b/apps/desktop/src/lib/runtime-readiness.ts index cf8d9665ea..23b12ccce4 100644 --- a/apps/desktop/src/lib/runtime-readiness.ts +++ b/apps/desktop/src/lib/runtime-readiness.ts @@ -7,6 +7,13 @@ export interface SetupStatusSnapshot { free_tier?: boolean other_providers?: boolean inference_provider?: string + /** Present only when the boot bootstrap could not create the free-tier + * identity: the failure code, its sentence, and whether / when a retry can + * succeed. Same shape as `free_tier.status`. */ + error?: string + error_code?: string + retryable?: boolean + retry_after?: number } export interface RuntimeCheckSnapshot { diff --git a/apps/desktop/src/store/free-tier-setup-failure.test.ts b/apps/desktop/src/store/free-tier-setup-failure.test.ts new file mode 100644 index 0000000000..b30ec8b5af --- /dev/null +++ b/apps/desktop/src/store/free-tier-setup-failure.test.ts @@ -0,0 +1,115 @@ +import { afterEach, describe, expect, it } from 'vitest' + +import { + $freeTierStatus, + type FreeTierRequester, + freeTierSetupFailure, + friendlyWait, + provisionFreeTier +} from '@/store/free-tier' +import type { FreeTierStatus } from '@/types/hermes' + +const NO_IDENTITY: FreeTierStatus = { + available: false, + enabled: true, + has_guest: false, + label: 'Nous · free tier', + model: 'nous/welcome', + notice_pending: false +} + +afterEach(() => { + $freeTierStatus.set(null) +}) + +describe('freeTierSetupFailure', () => { + it('is nothing when the backend reported no failure, the tier is off, or an identity exists', () => { + expect(freeTierSetupFailure(null)).toBeNull() + expect(freeTierSetupFailure(NO_IDENTITY)).toBeNull() + expect(freeTierSetupFailure({ ...NO_IDENTITY, enabled: false, error_code: 'anon_unreachable' })).toBeNull() + expect(freeTierSetupFailure({ ...NO_IDENTITY, has_guest: true, error_code: 'anon_unreachable' })).toBeNull() + }) + + it('carries the code, the sentence, and the wait', () => { + expect( + freeTierSetupFailure({ + ...NO_IDENTITY, + error: 'Lots of people are getting started right now.', + error_code: 'anon_rate_limited', + retry_after: 41.6, + retryable: true + }) + ).toEqual({ + code: 'anon_rate_limited', + door: 'sign_in', + message: 'Lots of people are getting started right now.', + retryAfter: 42, + retryable: true + }) + }) + + it.each([ + ['anon_unreachable', 'retry'], + ['anon_server_error', 'retry'], + ['anon_gate_closed', 'sign_in'], + ['anon_gate_paused', 'sign_in'], + ['anon_rate_limited', 'sign_in'], + ['anon_pow_required', 'sign_in'], + ['anon_account_locked', 'sign_in'], + ['something_newer', 'sign_in'] + ])('%s opens the %s door', (code, door) => { + // A sign-in goes through the same service that just refused: only offer + // it when that service answered at all. + expect(freeTierSetupFailure({ ...NO_IDENTITY, error_code: code })?.door).toBe(door) + }) + + it('treats a missing retryable flag as not retryable', () => { + expect(freeTierSetupFailure({ ...NO_IDENTITY, error_code: 'anon_gate_closed' })?.retryable).toBe(false) + }) +}) + +describe('friendlyWait', () => { + it.each([ + [0, 'a few seconds'], + [15, 'a few seconds'], + [16, 'about a minute'], + [89, 'about a minute'], + [300, 'about 5 minutes'], + [3600, 'about an hour'], + [7200, 'about 2 hours'], + [Number.NaN, 'a few seconds'] + ])('%s seconds reads as "%s"', (seconds, expected) => { + expect(friendlyWait(seconds)).toBe(expected) + }) +}) + +describe('provisionFreeTier', () => { + it('asks the backend to try again, then re-reads the verdict', async () => { + const calls: string[] = [] + const after: FreeTierStatus = { ...NO_IDENTITY, available: true, has_guest: true } + + const requestGateway = (async (method: string): Promise => { + calls.push(method) + + return (method === 'free_tier.status' ? after : { has_guest: true, enabled: true }) as T + }) satisfies FreeTierRequester + + expect(await provisionFreeTier(requestGateway)).toEqual(after) + expect(calls).toEqual(['free_tier.provision', 'free_tier.status']) + expect($freeTierStatus.get()).toEqual(after) + }) + + it('still reports what the backend knows when the retry call itself fails', async () => { + const failed: FreeTierStatus = { ...NO_IDENTITY, error_code: 'anon_unreachable', retryable: true, retry_after: 15 } + + const requestGateway = (async (method: string): Promise => { + if (method === 'free_tier.provision') { + throw new Error('gateway away') + } + + return failed as T + }) satisfies FreeTierRequester + + expect(await provisionFreeTier(requestGateway)).toEqual(failed) + }) +}) diff --git a/apps/desktop/src/store/free-tier-sign-in.test.ts b/apps/desktop/src/store/free-tier-sign-in.test.ts index fd40ab94d3..8a3bd20687 100644 --- a/apps/desktop/src/store/free-tier-sign-in.test.ts +++ b/apps/desktop/src/store/free-tier-sign-in.test.ts @@ -66,3 +66,44 @@ describe('free-tier sign-in attempts', () => { expect(cancelOAuthSession).toHaveBeenCalledWith('session-a') }) }) + +describe('free-tier sign-in failure screens', () => { + it('maps the account service verdicts onto ruled screens', async () => { + const { signInFailureKind } = await import('./free-tier-sign-in') + + expect(signInFailureKind('account_busy')).toBe('busy') + expect(signInFailureKind('anon_rate_limited')).toBe('busy') + expect(signInFailureKind('anon_gate_paused')).toBe('busy') + expect(signInFailureKind('anon_unreachable')).toBe('unreachable') + expect(signInFailureKind('anon_server_error')).toBe('unreachable') + expect(signInFailureKind('anon_gate_closed')).toBe('unavailable') + expect(signInFailureKind('anon_pow_required')).toBe('unavailable') + expect(signInFailureKind('anon_account_locked')).toBe('unavailable') + expect(signInFailureKind('user_declined')).toBe('rejected') + expect(signInFailureKind('wat')).toBe('error') + expect(signInFailureKind(null)).toBe('error') + }) + + it('a busy verdict keeps the wait the service named', async () => { + const { $freeTierSignIn, beginFreeTierSignIn } = await import('./free-tier-sign-in') + startOAuthLogin.mockResolvedValueOnce(start('session-busy')) + pollOAuthSession.mockResolvedValue({ + error_message: 'Signing in could not finish because the service is busy.', + reason: 'anon_rate_limited', + retry_after: 45, + retryable: true, + session_id: 'session-busy', + status: 'error' + }) + + await beginFreeTierSignIn(requestGateway) + await vi.advanceTimersByTimeAsync(2000) + + expect($freeTierSignIn.get()).toEqual({ + kind: 'busy', + message: 'Signing in could not finish because the service is busy.', + retryAfter: 45, + status: 'failed' + }) + }) +}) diff --git a/apps/desktop/src/store/free-tier-sign-in.ts b/apps/desktop/src/store/free-tier-sign-in.ts index 953466147e..f25001ec45 100644 --- a/apps/desktop/src/store/free-tier-sign-in.ts +++ b/apps/desktop/src/store/free-tier-sign-in.ts @@ -7,10 +7,22 @@ import { type FreeTierRequester, NOUS_PROVIDER_ID, refreshFreeTierStatus } from const POLL_MS = 2000 const COPY_FLASH_MS = 1500 -/** Why a sign-in ended without tokens. Each maps to one ruled screen; anything - * the backend does not name (transport failure, `account_busy`) lands on - * `error`, which carries the backend's own message when there is one. */ -export type FreeTierSignInFailure = 'error' | 'rejected' | 'retired' | 'superseded' | 'timed_out' +/** Why a sign-in ended without tokens. Each maps to one ruled screen: + * `busy` (the account service asked for a short wait — a busy account, a + * rate limit, the ops pause), `unreachable` (the service could not be + * reached or errored; a sign-in cannot help until it is back), `unavailable` + * (a terminal refusal: this version, a proof-of-work request, a locked + * session), and the ruled outcomes of the transfer itself. Anything the + * backend does not name lands on `error`, which carries its own message. */ +export type FreeTierSignInFailure = + | 'busy' + | 'error' + | 'rejected' + | 'retired' + | 'superseded' + | 'timed_out' + | 'unavailable' + | 'unreachable' export type FreeTierSignInState = | { status: 'already_signed_in' } @@ -20,7 +32,9 @@ export type FreeTierSignInState = // requester of its own. The mounted host picks it up and drives the flow. | { status: 'requested' } | { email: null | string; model: null | string; status: 'completed' } - | { kind: FreeTierSignInFailure; message: null | string; status: 'failed' } + // `retryAfter`: the seconds the backend asked us to wait before trying + // again (0 when it named none); only `busy` screens read it. + | { kind: FreeTierSignInFailure; message: null | string; retryAfter: number; status: 'failed' } // `minting` is true only when the backend still has to create the free-tier // identity (the first `start` does it), which is the one case where the user // waits on something worth naming. @@ -78,9 +92,9 @@ function clearTimers() { const set = (state: FreeTierSignInState) => $freeTierSignIn.set(state) -const fail = (kind: FreeTierSignInFailure, message: null | string = null) => { +const fail = (kind: FreeTierSignInFailure, message: null | string = null, retryAfter = 0) => { clearTimers() - set({ kind, message: message?.trim() || null, status: 'failed' }) + set({ kind, message: message?.trim() || null, retryAfter: Math.max(0, Math.round(retryAfter) || 0), status: 'failed' }) } /** Every entry point calls this — Settings › Billing, the statusbar chip, the @@ -108,17 +122,32 @@ export function closeFreeTierSignIn() { set({ status: 'closed' }) } -// The reasons the backend names on a non-approved terminal poll. Anything else -// (including a bare `account_busy`) falls through to the generic error screen, -// which shows the backend's own message. +// The reasons the backend names on a non-approved terminal poll: the transfer's +// own outcomes, and the account service's `anon_*` verdicts when it was busy, +// unreachable or refused mid sign-in (`hermes_cli/anon_sign_in.py`). Anything +// else falls through to the generic error screen, which shows the backend's +// own message. const FAILURE_BY_REASON: Record = { + account_busy: 'busy', account_not_anonymous: 'retired', account_retired: 'retired', + anon_account_locked: 'unavailable', + anon_gate_closed: 'unavailable', + anon_gate_paused: 'busy', + anon_pow_required: 'unavailable', + anon_rate_limited: 'busy', + anon_server_error: 'unreachable', + anon_unreachable: 'unreachable', superseded: 'superseded', timeout: 'timed_out', user_declined: 'rejected' } +/** The screen a failed poll maps to, exported for the dialog's tests. */ +export function signInFailureKind(reason: null | string | undefined): FreeTierSignInFailure { + return FAILURE_BY_REASON[reason ?? ''] ?? 'error' +} + // Open a sign-in URL through the desktop bridge, falling back to window.open // when the bridge isn't there (dev preview, tests) so the flow never strands in // a waiting state. Same contract as the onboarding store's opener. @@ -245,8 +274,7 @@ async function pollOnce(sessionId: string, requestGateway: FreeTierRequester, mi clearTimers() if (result.status !== 'approved') { - const kind = FAILURE_BY_REASON[result.reason ?? ''] ?? 'error' - fail(kind, result.error_message ?? null) + fail(signInFailureKind(result.reason), result.error_message ?? null, Number(result.retry_after) || 0) return } diff --git a/apps/desktop/src/store/free-tier.ts b/apps/desktop/src/store/free-tier.ts index 061f55ff77..b545e13689 100644 --- a/apps/desktop/src/store/free-tier.ts +++ b/apps/desktop/src/store/free-tier.ts @@ -1,7 +1,7 @@ import { atom } from 'nanostores' import { onboardingSurfaceActive } from '@/store/onboarding-presence' -import type { FreeTierStatus } from '@/types/hermes' +import type { FreeTierErrorCode, FreeTierStatus } from '@/types/hermes' /** The model the free-tier route runs on. Used to recognise a session that is * still homed on the free tier after a sign-in. */ @@ -51,6 +51,87 @@ export async function refreshFreeTierStatus(requestGateway: FreeTierRequester): } } +/** + * Why the free tier is not set up, when the backend says it tried and could + * not. `null` when there is an identity, when the tier is off, or when the + * backend never reported a failure (an older backend, or no boot yet). + * + * `door` is which way forward the copy may honestly offer. The account service + * that refused is the same one a sign-in goes through: when it is unreachable + * or erroring, offering sign-in walks the user into a second failure, so those + * codes get "try again / another provider" only. + */ +export interface FreeTierSetupFailure { + code: FreeTierErrorCode | string + door: 'retry' | 'sign_in' + message: string + retryAfter: number + retryable: boolean +} + +const UNREACHABLE_CODES = new Set(['anon_server_error', 'anon_unreachable']) + +export function freeTierSetupFailure(status: FreeTierStatus | null): FreeTierSetupFailure | null { + if (!status || !status.enabled || status.has_guest) { + return null + } + + const code = typeof status.error_code === 'string' ? status.error_code.trim() : '' + + if (!code) { + return null + } + + return { + code, + door: UNREACHABLE_CODES.has(code) ? 'retry' : 'sign_in', + message: typeof status.error === 'string' ? status.error : '', + retryAfter: Math.max(0, Math.round(Number(status.retry_after) || 0)), + retryable: status.retryable === true + } +} + +/** + * A rounded, spoken duration for copy — "a few seconds", "about a minute", + * "about 5 minutes", "about an hour" — mirroring the backend's `friendly_wait` + * so a wait reads the same whichever side phrased it. Never a raw second count. + */ +export function friendlyWait(seconds: number): string { + const s = Math.max(0, Number.isFinite(seconds) ? seconds : 0) + + if (s <= 15) { + return 'a few seconds' + } + + if (s < 90) { + return 'about a minute' + } + + if (s < 3600) { + return `about ${Math.round(s / 60)} minutes` + } + + const hours = Math.round(s / 3600) + + return hours <= 1 ? 'about an hour' : `about ${hours} hours` +} + +/** + * The user's own retry of the free-tier set-up (`free_tier.provision`): the one + * attempt that may run inside the backend's cooldown. Re-reads the status + * afterwards so every surface keyed on it moves together. Returns the fresh + * status, or the last known one when the call itself failed. + */ +export async function provisionFreeTier(requestGateway: FreeTierRequester): Promise { + try { + await requestGateway('free_tier.provision') + } catch { + // The status read below still reports what the backend knows. + } + + return refreshFreeTierStatus(requestGateway) +} + /** Persist the one-time notice acknowledgement, then re-read so every surface * keyed on `notice_pending` drops away together. */ export async function ackFreeTierNotice(requestGateway: FreeTierRequester): Promise { diff --git a/apps/desktop/src/types/hermes.ts b/apps/desktop/src/types/hermes.ts index c4a52f5c44..8440e9d327 100644 --- a/apps/desktop/src/types/hermes.ts +++ b/apps/desktop/src/types/hermes.ts @@ -130,6 +130,11 @@ export interface OAuthPollResponse { * `account_not_anonymous` / `account_busy` / `timeout` (status `error`). * `error_message` carries the matching user-facing text. */ reason?: null | string + /** Failed sign-ins over a free-tier identity: the seconds the account service + * asked the client to wait before trying again (0 or absent when none). */ + retry_after?: null | number + /** Failed sign-ins over a free-tier identity: whether a later attempt can succeed. */ + retryable?: boolean | null session_id: string status: 'approved' | 'denied' | 'error' | 'expired' | 'pending' } @@ -149,8 +154,27 @@ export interface FreeTierStatus { model: string /** True until the one-time introduction has been acknowledged. */ notice_pending: boolean + /** Present only while `enabled` and no identity exists: why the last attempt + * to create one failed. `error_code` is one of the backend's `anon_*` codes + * (`hermes_cli/anon_auth.py`), `error` its sentence, `retryable` whether a + * later attempt can succeed, `retry_after` the seconds still to wait. */ + error?: string + error_code?: string + retryable?: boolean + retry_after?: number } +/** The backend's free-tier failure codes (`hermes_cli/anon_auth.py::ANON_*`). */ +export type FreeTierErrorCode = + | 'anon_account_locked' + | 'anon_credential_dead' + | 'anon_gate_closed' + | 'anon_gate_paused' + | 'anon_pow_required' + | 'anon_rate_limited' + | 'anon_server_error' + | 'anon_unreachable' + export interface MemoryProviderOAuthStatus { auth: 'apikey' | 'oauth' | null connected: boolean diff --git a/hermes_cli/anon_auth.py b/hermes_cli/anon_auth.py index 6cb01493f9..780fc376ed 100644 --- a/hermes_cli/anon_auth.py +++ b/hermes_cli/anon_auth.py @@ -26,8 +26,10 @@ holds, else mint under the shared-store lock. It is the only minter; nothing els from __future__ import annotations import logging +import math import os import time +from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Any, Callable, Dict, Optional @@ -66,8 +68,79 @@ class AnonCredentialDead(AuthError): """ -def _anon_err(message: str, code: str) -> AuthError: - return AuthError(message, code=code) +def _anon_err( + message: str, code: str, *, retry_after: Optional[float] = None, retryable: bool = True, +) -> AuthError: + return AuthError(message, code=code, retry_after=retry_after, retryable=retryable) + + +# --- Free-tier failure codes ------------------------------------------------------------------------ +# +# Every way the account service (NAS) or the wire can refuse the free tier, as one ``AuthError.code`` +# each. Surfaces key their copy on the code; the message on the error is the surface-agnostic +# fallback (no ``/login``, no ``hermes`` verb, no guest / anonymous / credential). ``retryable`` +# says whether a later attempt can succeed at all; ``retry_after`` is the wait the server named. +# +# What NAS actually sends (nous-account-service ``api/anonymous/gate.ts`` and the routes behind it): +# 404 ``not_found`` the surface is not enabled on this deployment (terminal) +# 503 ``temporarily_disabled`` the ops breaker is tripped (transient, no hint) +# 429 ``temporarily_unavailable`` + Retry-After per-address / per-credential limits +# 428 ``pow_required`` / ``pow_invalid`` / ``pow_replayed`` proof-of-work enforced (not implemented here) +# 404 ``unknown_token`` the credential was reaped or claimed (re-mint) +# 403 ``account_locked`` the account is locked (dead; never re-mint from it) +# 401 an outstanding JWT whose account is gone (re-mint) +ANON_GATE_CLOSED = "anon_gate_closed" # not enabled here: sign in, or another provider +ANON_GATE_PAUSED = "anon_gate_paused" # ops breaker: keep checking in the background +ANON_RATE_LIMITED = "anon_rate_limited" # too many sign-ups / exchanges: wait Retry-After +ANON_POW_REQUIRED = "anon_pow_required" # proof of work requested: deferred, sign in instead +ANON_ACCOUNT_LOCKED = "anon_account_locked" # dead, and no replacement is minted from it +ANON_CREDENTIAL_DEAD = "anon_credential_dead" # reaped or claimed: replaced silently, once +ANON_UNREACHABLE = "anon_unreachable" # timeout, DNS, refused connection +ANON_SERVER_ERROR = "anon_server_error" # 5xx, non-JSON, malformed success body +# Codes a later attempt cannot fix (for this process / this version). +ANON_TERMINAL_CODES = frozenset({ANON_GATE_CLOSED, ANON_POW_REQUIRED, ANON_ACCOUNT_LOCKED}) +# Codes that mean the account service itself is not answering: a sign-in (which goes through the +# same service) cannot help either, so surfaces offer "try again" / "another provider" only. +ANON_UNREACHABLE_CODES = frozenset({ANON_UNREACHABLE, ANON_SERVER_ERROR}) + +# Copy per code: what happened, then the one honest way forward. The free MODEL is never "off": +# what is unavailable is using Hermes without signing in, and signing in is free. +_SIGNIN_IS_FREE = "Signing in is free and keeps the free model." +ANON_FAILURE_COPY = { + ANON_GATE_CLOSED: f"This version can't be used without a Nous account. {_SIGNIN_IS_FREE}", + ANON_GATE_PAUSED: f"Using Hermes without signing in is paused for a moment. {_SIGNIN_IS_FREE}", + ANON_RATE_LIMITED: "Lots of people are getting started right now. Try again in {wait}. " + "Signing in is free and skips the wait.", + ANON_POW_REQUIRED: "The Nous server asked for a proof of work, but that isn't implemented in your " + "Agent yet. Sign in with a Nous account to continue.", + ANON_ACCOUNT_LOCKED: f"This session can't continue without signing in. {_SIGNIN_IS_FREE}", + ANON_CREDENTIAL_DEAD: "Your session ended. A new one starts on its own.", + ANON_UNREACHABLE: "The Nous service couldn't be reached. Check your internet connection and try again.", + ANON_SERVER_ERROR: "The Nous service had a hiccup. Try again in a moment.", +} + + +def friendly_wait(seconds: Any) -> str: + """A rounded, spoken duration for user copy: "a few seconds", "about a minute", "about 5 minutes", + "about an hour". Never a raw second count.""" + try: + s = max(0.0, float(seconds or 0)) + except (TypeError, ValueError): + s = 0.0 + if s <= 15: + return "a few seconds" + if s < 90: + return "about a minute" + if s < 3600: + return f"about {int(round(s / 60))} minutes" + hours = int(round(s / 3600)) + return "about an hour" if hours <= 1 else f"about {hours} hours" + + +def anon_failure_copy(code: str, *, retry_after: Any = None) -> str: + """The surface-agnostic sentence for a free-tier failure *code* (``ANON_FAILURE_COPY``).""" + template = ANON_FAILURE_COPY.get(code) or ANON_FAILURE_COPY[ANON_SERVER_ERROR] + return template.format(wait=friendly_wait(retry_after if retry_after else 60)) def guest_enabled() -> bool: @@ -115,6 +188,18 @@ def guest_carries_inference() -> bool: WELCOME_HOSTS = frozenset({"welcome-api.nousresearch.com"}) +# Dev-only: extra hostnames that count as the welcome host, comma-separated (for example +# ``127.0.0.1`` while ``NOUS_INFERENCE_BASE_URL`` points at ``scripts/free_tier_fault_server.py``). +# Read from the environment, which the user controls, so it sits at the same trust level as the +# URL override itself; it never widens the NETWORK-side allowlist in ``auth_nous``. +EXTRA_WELCOME_HOSTS_ENV = "HERMES_EXTRA_WELCOME_HOSTS" + + +def welcome_hosts() -> frozenset[str]: + """``WELCOME_HOSTS`` plus any ``HERMES_EXTRA_WELCOME_HOSTS`` entries (lowercased hostnames).""" + raw = os.environ.get(EXTRA_WELCOME_HOSTS_ENV) or "" + extra = {part.strip().lower() for part in raw.split(",") if part.strip()} + return WELCOME_HOSTS | frozenset(extra) if extra else WELCOME_HOSTS def pin_model_for_route(provider: Any, base_url: Any, model: Any) -> Any: @@ -150,7 +235,7 @@ def route_is_welcome_host(base_url: Any) -> bool: host = (urlparse(str(base_url or "")).hostname or "").lower() except ValueError: return False - return host in WELCOME_HOSTS + return host in welcome_hosts() def anon_secret() -> str: @@ -172,21 +257,34 @@ def _raise_for_anon_status(response: httpx.Response, *, action: str) -> Dict[str if not isinstance(payload, dict): payload = {} error = str(payload.get("error") or "") - if response.status_code in (200, 201): + status = response.status_code + if status in (200, 201): return payload - if response.status_code == 404 and error == "unknown_token": - raise AnonCredentialDead("Nous free-tier credential is no longer valid.", code="anon_credential_dead") - if response.status_code == 401 and error == "invalid_shared_secret": - raise _anon_err("Nous free tier is not open on this portal.", "anon_gate_closed") - if response.status_code == 401: - raise AnonCredentialDead("Nous free-tier credential was revoked.", code="anon_credential_dead") - if response.status_code == 429: - raise _anon_err("Nous free tier is rate limited; try again shortly.", "anon_rate_limited") - if response.status_code == 403 and error in {"anonymous_accounts_disabled", "circuit_open"}: - raise _anon_err("Nous free tier is currently disabled.", "anon_gate_closed") - raise _anon_err( - f"Nous free tier {action} failed ({response.status_code}{': ' + error if error else ''}).", - "anon_server_error") + if status == 404 and error == "unknown_token": + raise AnonCredentialDead(ANON_FAILURE_COPY[ANON_CREDENTIAL_DEAD], code=ANON_CREDENTIAL_DEAD, retryable=False) + if status == 404: + # The gate's "surface not enabled" answer is uniform with a nonexistent route on purpose. + raise _anon_err(ANON_FAILURE_COPY[ANON_GATE_CLOSED], ANON_GATE_CLOSED, retryable=False) + if status == 401 and error == "invalid_shared_secret": # pre-launch NAS builds only + raise _anon_err(ANON_FAILURE_COPY[ANON_GATE_CLOSED], ANON_GATE_CLOSED, retryable=False) + if status == 401: + raise AnonCredentialDead(ANON_FAILURE_COPY[ANON_CREDENTIAL_DEAD], code=ANON_CREDENTIAL_DEAD, retryable=False) + if status == 403 and error == "account_locked": + raise AnonCredentialDead(ANON_FAILURE_COPY[ANON_ACCOUNT_LOCKED], code=ANON_ACCOUNT_LOCKED, retryable=False) + if status == 403 and error in {"anonymous_accounts_disabled", "circuit_open"}: # pre-launch names + raise _anon_err(ANON_FAILURE_COPY[ANON_GATE_PAUSED], ANON_GATE_PAUSED) + if status == 429: + retry_after = parse_retry_after_seconds(response.headers) + raise _anon_err(anon_failure_copy(ANON_RATE_LIMITED, retry_after=retry_after), ANON_RATE_LIMITED, + retry_after=retry_after) + if status == 428 or error.startswith("pow_"): + raise _anon_err(ANON_FAILURE_COPY[ANON_POW_REQUIRED], ANON_POW_REQUIRED, retryable=False) + if status == 503 and error == "temporarily_disabled": + raise _anon_err(ANON_FAILURE_COPY[ANON_GATE_PAUSED], ANON_GATE_PAUSED, + retry_after=parse_retry_after_seconds(response.headers)) + logger.info("Nous free tier %s failed (%s%s)", action, status, f": {error}" if error else "") + raise _anon_err(ANON_FAILURE_COPY[ANON_SERVER_ERROR], ANON_SERVER_ERROR, + retry_after=parse_retry_after_seconds(response.headers)) def mint_guest(client: httpx.Client, portal_base_url: str) -> Dict[str, Any]: @@ -195,7 +293,8 @@ def mint_guest(client: httpx.Client, portal_base_url: str) -> Dict[str, Any]: payload = _raise_for_anon_status(response, action="sign-up") token = payload.get("token") if not isinstance(token, str) or not token.startswith("anon_"): - raise _anon_err("Nous free tier sign-up returned no credential.", "anon_server_error") + logger.info("Nous free tier sign-up returned no credential") + raise _anon_err(ANON_FAILURE_COPY[ANON_SERVER_ERROR], ANON_SERVER_ERROR) return payload @@ -208,7 +307,8 @@ def exchange_anon_jwt(client: httpx.Client, portal_base_url: str, anon_token: st f"{portal_base_url.rstrip('/')}/api/anonymous/token", headers=_anon_headers(), json={"token": anon_token}) payload = _raise_for_anon_status(response, action="token exchange") if not isinstance(payload.get("access_token"), str) or not payload["access_token"]: - raise _anon_err("Nous free tier token exchange returned no token.", "anon_server_error") + logger.info("Nous free tier token exchange returned no token") + raise _anon_err(ANON_FAILURE_COPY[ANON_SERVER_ERROR], ANON_SERVER_ERROR) return payload @@ -283,30 +383,101 @@ def _mint_locked( return state -# Per-process memo: one failed mint is enough for a process (a 429 or a closed gate must not be hit -# twice); ``clear_dead_guest`` resets it because a retired credential is a reason to mint again. -# The bool is the unscoped (launch profile) slot; routed multiplex profiles each get their own entry -# in the set — profile A's 429 must not stop profile B from ever getting an identity. -_mint_failed = False -_mint_failed_homes: set[str] = set() +# Per-process mint memo: a failed mint is not retried until its cooldown has passed (a closed gate +# is never retried; a 429 waits out ``Retry-After``; an unreachable portal climbs a short ladder), +# so a boot-time blip cannot hammer the portal AND cannot disable the free tier for the whole +# process the way a plain "tried once" flag did. ``clear_dead_guest`` resets it because a retired +# credential is a reason to mint again; an explicit user retry (``force=True``) bypasses it. +# Keyed per profile home ("" for the unscoped launch profile): profile A's 429 must not stop +# profile B from ever getting an identity. +_MINT_RETRY_LADDER = (15.0, 60.0, 300.0) # unreachable / server error, attempt 1, 2, 3+ +_MINT_PAUSED_MIN_WAIT = 60.0 # ops breaker: never poll it faster than this -def _mint_failed_for_profile() -> bool: +@dataclass +class MintFailure: + """Why the last mint for a profile failed and when the next one may run.""" + + code: str + message: str + retryable: bool + retry_after: float # the wait this failure asked for, in seconds (0 for a terminal code) + not_before: float # ``time.monotonic()`` before which ``ensure_portal_identity`` stays quiet + attempts: int = 1 + + def remaining(self) -> float: + return 0.0 if not self.retryable else max(0.0, self.not_before - time.monotonic()) + + def as_payload(self) -> Dict[str, Any]: + """The wire shape every status RPC carries: ``{error_code, error, retryable, retry_after}`` + with ``retry_after`` the seconds still to wait (whole, rounded up).""" + return {"error_code": self.code, "error": self.message, "retryable": self.retryable, + "retry_after": int(math.ceil(self.remaining())) if self.retryable else 0} + + +_mint_failures: Dict[str, MintFailure] = {} + + +def _mint_memo_key() -> str: from hermes_constants import get_hermes_home_override, hermes_home_key - if get_hermes_home_override() is None: - return _mint_failed - return hermes_home_key() in _mint_failed_homes + return "" if get_hermes_home_override() is None else hermes_home_key() -def _set_mint_failed(failed: bool) -> None: - global _mint_failed - from hermes_constants import get_hermes_home_override, hermes_home_key - if get_hermes_home_override() is None: - _mint_failed = failed - elif failed: - _mint_failed_homes.add(hermes_home_key()) +def _mint_failure_for_profile() -> Optional[MintFailure]: + return _mint_failures.get(_mint_memo_key()) + + +def _clear_mint_failure() -> None: + _mint_failures.pop(_mint_memo_key(), None) + + +def reset_mint_memo_for_tests() -> None: + _mint_failures.clear() + + +def last_mint_failure() -> Optional[Dict[str, Any]]: + """The most recent mint failure for this profile as a wire payload, or None (never failed, or + cleared by a later success / retirement).""" + failure = _mint_failure_for_profile() + return failure.as_payload() if failure else None + + +def _classify_mint_exception(exc: BaseException) -> AuthError: + """Every mint failure as one ``AuthError`` with a code: the portal's own refusals already are; + the wire's (timeout, DNS, refused connection) and anything else get a code here.""" + if isinstance(exc, AuthError): + if exc.code is None: + exc.code = ANON_SERVER_ERROR + return exc + transport = (TimeoutError, ConnectionError, OSError) + try: + transport = transport + (httpx.TimeoutException, httpx.TransportError) + except Exception: # httpx unavailable (lazy proxy): the stdlib set stands + pass + code = ANON_UNREACHABLE if isinstance(exc, transport) else ANON_SERVER_ERROR + wrapped = _anon_err(ANON_FAILURE_COPY[code], code) + wrapped.__cause__ = exc + return wrapped + + +def _note_mint_failure(err: AuthError) -> MintFailure: + code = str(err.code or ANON_SERVER_ERROR) + previous = _mint_failure_for_profile() + attempts = previous.attempts + 1 if previous and previous.code == code else 1 + retryable = code not in ANON_TERMINAL_CODES and err.retryable is not False + if not retryable: + wait, not_before = 0.0, float("inf") else: - _mint_failed_homes.discard(hermes_home_key()) + hinted = err.retry_after + wait = float(hinted) if hinted else _MINT_RETRY_LADDER[min(attempts, len(_MINT_RETRY_LADDER)) - 1] + if code == ANON_GATE_PAUSED: + wait = max(wait, _MINT_PAUSED_MIN_WAIT) + wait = max(1.0, wait) + not_before = time.monotonic() + wait + failure = MintFailure(code=code, message=str(err), retryable=retryable, retry_after=wait, + not_before=not_before, attempts=attempts) + _mint_failures[_mint_memo_key()] = failure + return failure def _reconcile_and_provision(*, timeout_seconds: float, carries_inference: bool = True) -> Optional[Dict[str, Any]]: @@ -350,7 +521,7 @@ def _reconcile_and_provision(*, timeout_seconds: float, carries_inference: bool def ensure_portal_identity( *, explicit: bool, timeout_seconds: float = GUEST_MINT_TIMEOUT_SECONDS, - carries_inference: bool = True, + carries_inference: bool = True, force: bool = False, ) -> Optional[Dict[str, Any]]: """Make sure this profile has a Nous identity (guest or account); mint a guest only if the shared store has none. Returns the ``providers.nous`` state, or None (disabled / failed once already). @@ -365,19 +536,33 @@ def ensure_portal_identity( first, then shared, matching every other Nous path. ``carries_inference=False`` leaves ``active_provider`` alone (the identity is for connectors; another provider does inference). Blocking, bounded by ``timeout_seconds``; the bootstrap puts it on its own thread. + + A failed mint is memoised with a cooldown (``MintFailure``): until it passes, and for a + terminal code forever, this returns None without touching the portal. ``force=True`` is the + user's own retry (the desktop's ``free_tier.provision`` button): it makes exactly one attempt + regardless of the cooldown. Every failure raises an ``AuthError`` whose ``code`` is one of the + ``ANON_*`` codes and whose ``retry_after`` / ``retryable`` say what a caller may do next. """ if not explicit: raise ValueError("ensure_portal_identity: only explicit creators may call this (explicit=True)") if not guest_enabled(): return None - if _mint_failed_for_profile() and not current_nous_state(): - return None # this profile already tried and failed in this process; do not hammer the portal + failure = _mint_failure_for_profile() + if failure and not force and not current_nous_state() and time.monotonic() < failure.not_before: + return None # in cooldown (or terminal) for this profile; do not hammer the portal try: - return _reconcile_and_provision( + state = _reconcile_and_provision( timeout_seconds=timeout_seconds, carries_inference=carries_inference) - except Exception: - _set_mint_failed(True) - raise + except Exception as exc: + err = _classify_mint_exception(exc) + noted = _note_mint_failure(err) + logger.info("Nous free tier not set up (%s, attempt %d%s)", noted.code, noted.attempts, + f", next try in {noted.retry_after:.0f}s" if noted.retryable else ", not retried") + if err is exc: + raise + raise err from exc + _clear_mint_failure() + return state def refresh_guest_state(state: Dict[str, Any], client: httpx.Client) -> None: @@ -390,7 +575,7 @@ def refresh_guest_state(state: Dict[str, Any], client: httpx.Client) -> None: """ anon_token = state.get("anon_token") if not isinstance(anon_token, str) or not anon_token: - raise AnonCredentialDead("Nous free-tier credential is missing.", code="anon_credential_dead") + raise AnonCredentialDead(ANON_FAILURE_COPY[ANON_CREDENTIAL_DEAD], code=ANON_CREDENTIAL_DEAD, retryable=False) from hermes_cli.auth import _nous_portal_base_url apply_exchange_to_state(state, exchange_anon_jwt(client, _nous_portal_base_url(state), anon_token)) @@ -422,7 +607,7 @@ def clear_dead_guest(reason: str, *, dead_token: Optional[str] = None) -> None: shared = _read_shared_nous_state() if token and is_guest_state(shared) and shared.get("anon_token") == token: _clear_shared_nous_state(reason) - _set_mint_failed(False) + _clear_mint_failure() logger.info("Nous free-tier identity retired (%s); a new one is set up on next use", reason) @@ -448,15 +633,22 @@ _WELCOME_ROUTE_REFUSALS = ( ("anonymous accounts are not accepted", "tier_disabled"), ) _WELCOME_ROUTE_COPY = { - "anon_on_paid_host": "The Nous free tier must use its own inference host ({host}); " - "Hermes is pointed at the paid one. Restart Hermes to re-read the route, " - "or unset NOUS_INFERENCE_BASE_URL if you set it.", - "named_on_welcome_host": "This Nous account must use the Nous Portal inference host, " - "not the free tier's. Run /model and pick the Nous row again.", - "tier_disabled": "The Nous free tier is switched off right now. {signin}", + # Only reachable when the route heal (``turn_recovery._recover_welcome_tier``) could not move + # the session: the one cause left is a user-set NOUS_INFERENCE_BASE_URL naming the paid host. + "anon_on_paid_host": "This install is set to use a different Nous server (NOUS_INFERENCE_BASE_URL). " + "Unset it to use the free model, or sign in. {signin}", + "named_on_welcome_host": "This Nous account needs to reconnect. {model_hint}", + "tier_disabled": "Using Hermes without signing in is switched off right now. " + "Sign in to keep chatting, it's free and keeps the free model. {signin}", } -_SIGNIN_CHAT = "Sign in with a Nous account for the full catalog: /login." -_SIGNIN_TERMINAL = "Sign in with a Nous account for the full catalog: `hermes auth upgrade`." +# The sign-in door, phrased for a chat surface (slash command) and for a terminal. +_SIGNIN_CHAT = "To sign in: /login." +_SIGNIN_TERMINAL = "To sign in: `hermes auth upgrade`." +_MODEL_HINT_CHAT = "Run /model and pick the Nous row again." +_MODEL_HINT_TERMINAL = "Run `hermes model` and pick the Nous row again." +# Terminal copy for a free-model outage once the retries are spent (5xx, transport failure). +FREE_TIER_OUTAGE_COPY = ("The free model is having trouble responding right now. " + "Try sending your message again in a minute.") def parse_welcome_refusal(body: Any) -> Optional[Dict[str, Any]]: @@ -491,37 +683,50 @@ def welcome_refusal_copy(refusal: Dict[str, Any], *, model: str = "", in_chat: b alternates = refusal.get("alternates") or [] serves = alternates[0] if alternates else GUEST_MODEL retry = int(refusal.get("retry_after") or 0) - wait = f"Retrying in {retry}s." if retry > 0 else "Try again shortly." + wait = friendly_wait(retry) if retry > 0 else "a little while" if reason == "model_not_free": - what = f"{model} isn't on the Nous free tier" if model else "That model isn't on the Nous free tier" - return f"{what}; it serves {serves} only. {signin}" + what = f"{model} isn't" if model else "That model isn't" + return (f"{what} available without signing in, so Hermes uses {serves} for now. " + f"Sign in for more models. {signin}") if reason == "feature_not_free": - return f"This feature isn't on the Nous free tier. {signin}" + return f"That isn't available without signing in. Sign in to use it, it's free. {signin}" if reason == "at_capacity": - return f"The Nous free tier is at capacity and briefly paused. {wait} {signin}" + return ("Chatting without signing in is really busy right now. Sign in to skip the queue, " + f"it's free, or try again in {wait}. {signin}") if reason == "admission_closed": - return f"The Nous free tier isn't admitting new sessions right now. {wait} {signin}" + return ("Chatting without signing in is full right now. Sign in to keep going, " + f"it's free, or try again in {wait}. {signin}") if reason == "rate_limited": - return f"Nous free tier rate limit active \u2014 resets in {retry}s. {signin}" - return f"The Nous free tier refused this request ({reason}). {signin}" + return (f"You've used up the allowance for chatting without signing in. It refreshes in {wait}. " + f"Sign in for a bigger allowance, it's free. {signin}") + return f"Hermes couldn't send that without signing in. Signing in is free. {signin}" -def welcome_route_refusal(status: Any, message: Any) -> Optional[str]: - """Which host cross-refusal a gateway 400/403 is, by its message; None for any other error. +def welcome_route_refusal(status: Any, message: Any, base_url: Any = None) -> Optional[str]: + """Which host cross-refusal a gateway 400/403 is; None for any other error. ``"anon_on_paid_host"``: a free-tier JWT reached the paid host. ``"named_on_welcome_host"``: an account or API key reached the free tier's host. ``"tier_disabled"``: the tier is dark - (``WELCOME_MODE=off``). Each is deterministic for the request: retrying cannot help.""" + (``WELCOME_MODE=off``). Each is deterministic for the request: retrying cannot help. + + The dark-tier 403 is keyed on the ROUTE, not the message: the gateway's permission error + carries only its generic sentence (the detail stays in its logs), so any 403 answered by the + welcome host means the tier refused this install. The message needles remain for gateways + that do spell it out, and for the two wrong-host 400s.""" if status not in (400, 403): return None text = str(message or "").lower() - return next((kind for needle, kind in _WELCOME_ROUTE_REFUSALS if needle in text), None) + kind = next((kind for needle, kind in _WELCOME_ROUTE_REFUSALS if needle in text), None) + if kind is None and status == 403 and route_is_welcome_host(base_url): + return "tier_disabled" + return kind def welcome_route_refusal_copy(kind: str, *, in_chat: bool = True) -> str: - template = _WELCOME_ROUTE_COPY.get(kind) or "The Nous inference gateway refused this route." + template = _WELCOME_ROUTE_COPY.get(kind) or "Hermes couldn't reach the free model on this route." return template.format( - host=DEFAULT_NOUS_WELCOME_URL, signin=_SIGNIN_CHAT if in_chat else _SIGNIN_TERMINAL) + host=DEFAULT_NOUS_WELCOME_URL, signin=_SIGNIN_CHAT if in_chat else _SIGNIN_TERMINAL, + model_hint=_MODEL_HINT_CHAT if in_chat else _MODEL_HINT_TERMINAL) def note_model_switch(agent: Any, headers: Any) -> Optional[str]: @@ -657,7 +862,8 @@ def register_promotion_intent( json={"token": anon_token, "user_code": user_code, "device_code": device_code}) payload = _raise_for_anon_status(response, action="sign-in") if not isinstance(payload.get("claim_code"), str) or not payload["claim_code"]: - raise _anon_err("Nous free tier sign-in returned no transfer code.", "anon_server_error") + logger.info("Nous free tier sign-in returned no transfer code") + raise _anon_err(ANON_FAILURE_COPY[ANON_SERVER_ERROR], ANON_SERVER_ERROR) return payload diff --git a/hermes_cli/anon_sign_in.py b/hermes_cli/anon_sign_in.py index c1d677e217..4bf9ef3431 100644 --- a/hermes_cli/anon_sign_in.py +++ b/hermes_cli/anon_sign_in.py @@ -14,17 +14,27 @@ from hermes_cli.auth_constants import httpx UPGRADE_START = "Sign in with a Nous account to unlock more models and tools." UPGRADE_ALREADY_SIGNED_IN = "Already signed in." UPGRADE_DO_NOT_SHARE = "Do not share this code." -UPGRADE_TIMED_OUT = "Sign-in timed out; run the command again." -UPGRADE_NOT_COMPLETED = "Sign-in did not complete; run the command again." +UPGRADE_TIMED_OUT = "That sign-in link has expired. Start again whenever you're ready." +UPGRADE_NOT_COMPLETED = "Sign-in didn't finish. Try again whenever you're ready." UPGRADE_UNAVAILABLE = "The free tier is not available right now; run `hermes auth add nous` to sign in." UPGRADE_REASON_COPY = { - "user_declined": "Sign-in was rejected in the browser.", - "superseded": "A newer sign-in code replaced this one.", - "account_retired": "This free-tier identity was already used or expired; a new one is set up on the next start.", - "account_not_anonymous": "This free-tier identity was already used or expired; a new one is set up on the next start.", - "account_busy": "The transfer could not run; run the command again.", + "user_declined": "No problem, you're still on the free Nous service. Sign in whenever you're ready.", + "superseded": "A newer sign-in code replaced this one. Use the newest one, or start again.", + "account_retired": "Your session ended before the sign-in finished. A new one starts on its own; " + "sign in again whenever you're ready.", + "account_not_anonymous": "Your session ended before the sign-in finished. A new one starts on its own; " + "sign in again whenever you're ready.", + "account_busy": "Something's still finishing up on your account. Give it a few seconds, then try signing in again.", } _RETIRED_REASONS = frozenset({"account_retired", "account_not_anonymous"}) +# Reasons a later attempt can succeed at: the desktop offers "try again" after the named wait. +RETRYABLE_SIGN_IN_REASONS = frozenset({"account_busy"}) +# The account service was busy or unreachable mid sign-in (an ``anon_*`` code from +# ``anon_auth``): the identity is untouched, so the copy reassures before the way forward. +UPGRADE_SERVICE_BUSY = ("Signing in couldn't finish because the Nous service is busy. " + "Try again in {wait}. Your session is still here in the meantime.") +UPGRADE_SERVICE_UNREACHABLE = ("The Nous service couldn't be reached to finish signing you in. " + "Check your internet connection and try again. Your session is still here.") UPGRADE_NO_DEFAULT_TERMINAL = "No default model is set yet; run `hermes model` to pick one." UPGRADE_NO_DEFAULT_CHAT = "No default model is set yet; run /model to pick one." @@ -38,8 +48,8 @@ LOGIN_DM_ONLY = "Sign in from a direct message with Hermes." LOGIN_BUSY_ELSEWHERE = "Another sign-in is already running on this Hermes. Try again in a few minutes." LOGIN_NOT_ALLOWED = "Only an operator of this Hermes can sign it in." FREE_TIER_RATE_LIMIT_CHAT = ( - "Nous free tier rate limit active \u2014 resets in {reset}. " - "Sign in with a Nous account for higher limits: /login.") + "You've used up the allowance for chatting without signing in. It refreshes in {reset}. " + "Sign in for a bigger allowance, it's free: /login.") def format_wait_line(expires_in: int) -> str: @@ -174,19 +184,41 @@ class Retired(SignInState): @dataclass(frozen=True) class Failed(SignInState): + """``reason``: a promotion outcome reason (``account_busy``...), an ``anon_*`` code from the + account service (busy, paused, unreachable...), or "" for anything unnamed. ``retry_after``: + the wait the service asked for, in seconds (0 when it named none). ``retryable``: whether a + later attempt can succeed, so a renderer knows to offer "try again" and when.""" + reason: str = "" detail: str = "" + retry_after: float = 0.0 kind: ClassVar[str] = "failed" @property - def copy(self) -> str: - return UPGRADE_REASON_COPY.get(self.reason, UPGRADE_NOT_COMPLETED) + def retryable(self) -> bool: + from hermes_cli import anon_auth as _core + if self.reason in RETRYABLE_SIGN_IN_REASONS: + return True + return bool(self.reason.startswith("anon_") and self.reason not in _core.ANON_TERMINAL_CODES) @property - def copy_terminal(self) -> str: + def copy(self) -> str: + from hermes_cli import anon_auth as _core ruled = UPGRADE_REASON_COPY.get(self.reason) if ruled: return ruled + if self.reason in _core.ANON_UNREACHABLE_CODES: + return UPGRADE_SERVICE_UNREACHABLE + if self.reason in (_core.ANON_RATE_LIMITED, _core.ANON_GATE_PAUSED): + return UPGRADE_SERVICE_BUSY.format(wait=_core.friendly_wait(self.retry_after or 60)) + if self.reason in _core.ANON_FAILURE_COPY: + return _core.anon_failure_copy(self.reason, retry_after=self.retry_after) + return UPGRADE_NOT_COMPLETED + + @property + def copy_terminal(self) -> str: + if self.reason and self.copy != UPGRADE_NOT_COMPLETED: + return self.copy return f"Sign-in failed: {self.detail}" if self.detail else UPGRADE_NOT_COMPLETED @@ -216,6 +248,20 @@ class Unavailable(SignInState): return f"{UPGRADE_UNAVAILABLE} ({self.detail})" if self.detail else UPGRADE_UNAVAILABLE +def _failed_from_exception(exc: BaseException) -> Failed: + """A ``Failed`` that keeps the account service's own verdict: the ``anon_*`` code and wait hint + an ``AuthError`` carries, or the wire's shape for a transport error. The raw detail never + reaches a chat; ``copy_terminal`` may show it when nothing better is known.""" + from hermes_cli import anon_auth as _core + err = _core._classify_mint_exception(exc) + reason = str(err.code or "") + if reason == _core.ANON_SERVER_ERROR and not isinstance(exc, _core.AuthError): + # An unnamed local failure (a bad CA bundle, a lock timeout): keep today's generic copy + # and its terminal detail rather than blaming the service. + return Failed(reason="", detail=str(exc)) + return Failed(reason=reason, detail=str(exc), retry_after=float(err.retry_after or 0.0)) + + def _outcome_state(outcome: Dict[str, Any], anon_token: str) -> SignInState: """The one reason -> state mapping in the tree, for a promotion that did not complete. @@ -381,7 +427,7 @@ def run_sign_in( yield TimedOut(detail=str(exc)) return except Exception as exc: - yield Failed(reason="", detail=str(exc)) + yield _failed_from_exception(exc) return try: diff --git a/hermes_cli/auth_constants.py b/hermes_cli/auth_constants.py index 740940d2ae..c6042ad21b 100644 --- a/hermes_cli/auth_constants.py +++ b/hermes_cli/auth_constants.py @@ -140,11 +140,17 @@ class AuthError(RuntimeError): def __init__( self, message: str, *, provider: str = "", code: Optional[str] = None, relogin_required: bool = False, + retry_after: Optional[float] = None, retryable: Optional[bool] = None, ) -> None: super().__init__(message) self.provider = provider self.code = code self.relogin_required = relogin_required + # Optional wait hint in seconds (a server ``Retry-After`` or a client cooldown) and whether a + # later attempt can succeed at all. None = the raiser did not say; callers treat None as + # "retryable, no hint" for transport-shaped errors and as terminal for auth refusals. + self.retry_after = retry_after + self.retryable = retryable def _provider_error_factory(provider: str) -> Callable[..., AuthError]: diff --git a/hermes_cli/auth_nous.py b/hermes_cli/auth_nous.py index 8d91230270..b5c5381964 100644 --- a/hermes_cli/auth_nous.py +++ b/hermes_cli/auth_nous.py @@ -1013,10 +1013,14 @@ def resolve_nous_runtime_credentials( return _resolve_nous_runtime_credentials( timeout_seconds=timeout_seconds, insecure=insecure, ca_bundle=ca_bundle, force_refresh=force_refresh, stale_access_token=stale_access_token) - except AnonCredentialDead: + except AnonCredentialDead as dead_exc: from hermes_cli.auth import get_provider_auth_state + from hermes_cli.anon_auth import ANON_ACCOUNT_LOCKED dead = get_provider_auth_state("nous") or {} - clear_dead_guest("anon_credential_dead", dead_token=dead.get("anon_token")) + clear_dead_guest(str(dead_exc.code or "anon_credential_dead"), dead_token=dead.get("anon_token")) + # A locked account is retired but never silently replaced: the way forward is a sign-in. + if dead_exc.code == ANON_ACCOUNT_LOCKED: + raise if ensure_portal_identity(explicit=True, timeout_seconds=timeout_seconds) is None: raise return _resolve_nous_runtime_credentials( diff --git a/hermes_cli/free_tier_bootstrap.py b/hermes_cli/free_tier_bootstrap.py index eca44e5b2f..030b104970 100644 --- a/hermes_cli/free_tier_bootstrap.py +++ b/hermes_cli/free_tier_bootstrap.py @@ -38,11 +38,25 @@ class SetupRecord: has_identity: bool # a Nous identity (free tier or account) is on disk other_providers: bool # the inventory found something usable BESIDES the free tier error: str = "" # why the mint did not happen, when it did not; "" otherwise + # The structured form of ``error`` (``anon_auth.ANON_*`` codes): what a renderer keys its copy + # and its doors on. ``retryable`` says whether a later attempt can succeed; ``retry_after`` is + # the seconds still to wait before one may (0 when none is owed, or the code is terminal). + error_code: str = "" + retryable: bool = False + retry_after: int = 0 finished_at: float = field(default_factory=time.time) def as_payload(self) -> Dict[str, Any]: return asdict(self) + def failure_fields(self) -> Dict[str, Any]: + """The additive ``{error, error_code, retryable, retry_after}`` block every status RPC + carries when the mint did not happen; ``{}`` otherwise.""" + if not self.error_code: + return {} + return {"error": self.error, "error_code": self.error_code, "retryable": self.retryable, + "retry_after": self.retry_after} + _lock = threading.Lock() _record: Optional[SetupRecord] = None @@ -96,6 +110,40 @@ def _resolve_inference() -> str: return "" +def _build_record(*, other: bool, force: bool) -> SetupRecord: + """One inventory-then-mint pass into a record. ``force`` is the user's own retry: it makes one + attempt even inside the mint memo's cooldown (``anon_auth.ensure_portal_identity``).""" + from hermes_cli import anon_auth + + error = "" + failure: Dict[str, Any] = {} + state: Optional[Dict[str, Any]] = anon_auth.current_nous_state() + if anon_auth.guest_enabled(): + try: + # ``other`` decides whether the mint may also claim ``active_provider`` (NS-845 Q1.3). + state = anon_auth.ensure_portal_identity(explicit=True, carries_inference=not other, force=force) + except Exception as exc: + error = str(exc) + logger.info("Nous free tier not set up at boot: %s", exc) + if state is None: + # Either this attempt failed (the memo now holds why) or an earlier one did and its + # cooldown still runs: the record carries that verdict either way. + failure = anon_auth.last_mint_failure() or {} + error = error or str(failure.get("error") or "") + free_tier = bool(state) and anon_auth.is_guest_state(state) and anon_auth.guest_enabled() + return SetupRecord( + provider_configured=other or free_tier or (bool(state) and not anon_auth.is_guest_state(state)), + inference_provider=_resolve_inference(), + free_tier=free_tier, + has_identity=bool(state), + other_providers=other, + error=error, + error_code=str(failure.get("error_code") or ""), + retryable=bool(failure.get("retryable", False)), + retry_after=int(failure.get("retry_after") or 0), + ) + + def run_bootstrap(*, announce: bool = True) -> SetupRecord: """Inventory -> ensure identity (gate permitting) -> resolve inference -> record -> broadcast. @@ -113,27 +161,7 @@ def run_bootstrap(*, announce: bool = True) -> SetupRecord: return _record _started = True - from hermes_cli import anon_auth - - other = _inventory_other_providers() - error = "" - state: Optional[Dict[str, Any]] = anon_auth.current_nous_state() - if anon_auth.guest_enabled(): - try: - # ``other`` decides whether the mint may also claim ``active_provider`` (NS-845 Q1.3). - state = anon_auth.ensure_portal_identity(explicit=True, carries_inference=not other) - except Exception as exc: - error = str(exc) - logger.info("Nous free tier not set up at boot: %s", exc) - free_tier = bool(state) and anon_auth.is_guest_state(state) and anon_auth.guest_enabled() - record = SetupRecord( - provider_configured=other or free_tier or (bool(state) and not anon_auth.is_guest_state(state)), - inference_provider=_resolve_inference(), - free_tier=free_tier, - has_identity=bool(state), - other_providers=other, - error=error, - ) + record = _build_record(other=_inventory_other_providers(), force=False) with _lock: _record = record _done.set() @@ -142,6 +170,56 @@ def run_bootstrap(*, announce: bool = True) -> SetupRecord: return record +# Background retries after a boot-time mint failure: the memo's cooldown decides WHEN (a server +# ``Retry-After``, the ops-breaker floor, or the unreachable ladder), this decides HOW MANY before +# the process stops trying on its own (the user's retry button, ``free_tier.provision``, is not +# counted). A terminal code (gate closed, proof of work, locked) is never retried. +BOOTSTRAP_RETRY_ATTEMPTS = 3 +_sleep = time.sleep # seam for tests + + +def retry_bootstrap_mint(*, force: bool = False, announce: bool = True) -> SetupRecord: + """Re-run the mint once (``force`` bypasses the cooldown), replace the record and announce it. + + The desktop's ``free_tier.provision`` calls this with ``force=True``; the background loop calls + it as each cooldown passes. Returns the existing record untouched when no bootstrap ran yet + (nothing to replace) or when an identity already exists.""" + global _record + current = _record + if current is None: + return run_bootstrap(announce=announce) + if current.has_identity: + return current + record = _build_record(other=current.other_providers, force=force) + with _lock: + _record = record + if announce: + _broadcast(record) + return record + + +def _retry_until_settled() -> None: + """The background loop behind ``start_background_bootstrap``: wait out each cooldown and try + again, up to ``BOOTSTRAP_RETRY_ATTEMPTS``, while the record says a later attempt can succeed.""" + for _ in range(BOOTSTRAP_RETRY_ATTEMPTS): + record = _record + if record is None or record.has_identity or not record.error_code or not record.retryable: + return + _sleep(max(1, int(record.retry_after or 0))) + record = retry_bootstrap_mint(force=False) + if record.has_identity: + logger.info("Nous free tier set up after a boot-time retry") + return + + +def _bootstrap_then_retry() -> None: + run_bootstrap() + try: + _retry_until_settled() + except Exception as exc: # the loop is best effort; the record already says what happened + logger.debug("free tier bootstrap retry loop stopped: %s", exc) + + def _broadcast(record: SetupRecord) -> None: try: from tui_gateway.server import _broadcast_global_event @@ -152,6 +230,6 @@ def _broadcast(record: SetupRecord) -> None: def start_background_bootstrap() -> threading.Thread: """``hermes serve`` entry: run on a daemon thread so a slow portal never delays the socket.""" - thread = threading.Thread(target=run_bootstrap, daemon=True, name="free-tier-bootstrap") + thread = threading.Thread(target=_bootstrap_then_retry, daemon=True, name="free-tier-bootstrap") thread.start() return thread diff --git a/hermes_cli/web_routers/oauth.py b/hermes_cli/web_routers/oauth.py index 72a44f369e..c804336dc7 100644 --- a/hermes_cli/web_routers/oauth.py +++ b/hermes_cli/web_routers/oauth.py @@ -720,6 +720,8 @@ async def poll_oauth_session(provider_id: str, session_id: str, profile: Optiona # Nous over a free-tier identity: why a transfer ended, who signed in, and the default model # the completion settled on (None when the config was on the user's own model). "reason": sess.get("reason"), "account_email": sess.get("account_email"), "model": sess.get("model"), + # Failed sign-ins over a free-tier identity: can a later attempt succeed, and after how long. + "retryable": sess.get("retryable"), "retry_after": sess.get("retry_after"), } diff --git a/hermes_cli/web_server_oauth.py b/hermes_cli/web_server_oauth.py index ebabee383e..78510ff777 100644 --- a/hermes_cli/web_server_oauth.py +++ b/hermes_cli/web_server_oauth.py @@ -260,6 +260,9 @@ def _record_sign_in_state(sess: Dict[str, Any], state: Any) -> None: sess["status"] = "error" sess["reason"] = state.reason or "error" sess["error_message"] = state.copy # the chat form: no raw exception reaches the UI + # Whether a later attempt can succeed, and the wait the service named (seconds). + sess["retryable"] = bool(getattr(state, "retryable", False)) + sess["retry_after"] = int(getattr(state, "retry_after", 0) or 0) return # already_signed_in / unavailable: the start route refuses these, so this is unreachable # through the dashboard; record rather than crash. diff --git a/scripts/free_tier_fault_server.py b/scripts/free_tier_fault_server.py new file mode 100644 index 0000000000..468cbca512 --- /dev/null +++ b/scripts/free_tier_fault_server.py @@ -0,0 +1,513 @@ +#!/usr/bin/env python3 +"""Fault-injecting stand-in for the two services behind the Nous free tier. + +One process plays both the account service (NAS, ``/api/anonymous/*`` and the device-code +endpoints a sign-in touches) and the welcome inference host (``/v1/chat/completions``), answering +with the exact status codes, bodies and headers the real services send, so Hermes' failure +handling can be rehearsed end to end without touching production. Which failure it serves is a +live switch: change it from the desktop's JavaScript console, ``curl``, or a browser, and watch +the desktop react. + +Run:: + + python scripts/free_tier_fault_server.py # 127.0.0.1:8765 + python scripts/free_tier_fault_server.py --port 9000 --inference rate_limited + +then start Hermes Desktop pointed at it (the script prints the exact environment lines). See +``website/docs/developer-guide/free-tier-fault-rehearsal.md`` for the walkthrough. + +Control surface (all CORS-open, so a renderer can call them):: + + GET /__scenarios the catalogue: every scenario, per service, with what it does + GET /__scenario the switches as they stand + POST /__scenario {"nas": ..., "inference": ..., "once": bool, "retry_after": N} + POST /__signin {"status": "completed" | "voided", "reason": "..."} settle a pending sign-in + GET /__log the last requests the server answered + POST /__reset back to the happy path, log cleared + +Stdlib only: no dependencies, nothing to install. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import secrets +import threading +import time +from collections import deque +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, Optional +from urllib.parse import parse_qs, urlparse + +WELCOME_MODEL = "nous/welcome" +REAL_WELCOME_HOST = "welcome-api.nousresearch.com" +REAL_PAID_URL = "https://inference-api.nousresearch.com" +GENERIC_403 = "You tried to access something that you don't have permissions for." +UPGRADE_URL = "https://portal.nousresearch.com/signup" + +# --- Scenario catalogue -------------------------------------------------------------------------- +# +# Names are what the control endpoint accepts. Each entry says what the real service does in that +# situation, so a rehearsal reads as the situation rather than the status code. + +NAS_SCENARIOS: Dict[str, str] = { + "ok": "The account service answers normally: sign-ups mint, exchanges return a JWT.", + "not_enabled": "404 not_found on every /api/anonymous route: the surface is not enabled on this deployment.", + "paused": "503 temporarily_disabled: the ops breaker is tripped (transient, no wait hinted).", + "rate_limited": "429 temporarily_unavailable with Retry-After: too many sign-ups from this address.", + "server_error": "500 with a non-JSON body on every /api/anonymous route.", + "timeout": "Every /api/anonymous request hangs past the client's 5 s budget.", + "pow": "Token exchange answers 428 pow_required: proof of work enforced (sign-ups still mint).", + "locked": "Token exchange answers 403 account_locked: the account is locked.", + "dead_once": "The NEXT token exchange answers 404 unknown_token (reaped / claimed); the replacement works.", + "signin_busy": "Starting a sign-in (promotion-intent) answers 429 with Retry-After.", + "signin_paused": "Starting a sign-in answers 503 temporarily_disabled.", + "signin_server_error": "Starting a sign-in answers 500.", +} + +INFERENCE_SCENARIOS: Dict[str, str] = { + "ok": "Chat completions answer with a short canned reply (streaming or not).", + "rate_limited": "429 reason=rate_limited, retry_after 600: the allowance is used up (long wait -> stop and say so).", + "rate_limited_short": "429 reason=rate_limited, retry_after 5: a short wait the turn rides out quietly.", + "at_capacity": "429 reason=at_capacity, retry_after 30: the tier is busy (retried in place, bounded).", + "model_not_free": "429 reason=model_not_free, alternates=[nous/welcome]: the session asked for another model.", + "tier_disabled": "403 with only the generic permission message: WELCOME_MODE=off on the gateway.", + "wrong_host": "400 'Anonymous accounts must use https://welcome-api...': the route points at the paid host.", + "bare_429": "429 with x-ratelimit-* headers showing an exhausted bucket and no reason field.", + "upstream_503": "503 'The requested model is currently unavailable.': the upstream provider is down.", + "upstream_500": "500 generic: an upstream 5xx surfaced by the gateway.", + "invalid_token": "401 invalid_token / anonymous_credential_revoked: the session behind the JWT is gone.", + "timeout": "The request hangs for two minutes.", +} + + +def _jwt(**claims: Any) -> str: + def seg(obj: Any) -> str: + return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode() + payload = {"sub": "nas_user:rehearsal", "client_id": "nas-anonymous", "account_tier": "anonymous", + "scope": "inference:invoke tool:invoke", "iss": "free-tier-fault-server", + "iat": int(time.time()), "exp": int(time.time()) + 900, **claims} + return f"{seg({'alg': 'RS256', 'typ': 'JWT'})}.{seg(payload)}.{seg({'sig': 'rehearsal'})}" + + +class State: + """The switches, guarded by one lock (the server is threaded).""" + + def __init__(self, *, nas: str = "ok", inference: str = "ok", welcome_url: str = "") -> None: + self.lock = threading.Lock() + self.nas = nas + self.inference = inference + self.once = False + self.retry_after: Optional[int] = None + self.welcome_url = welcome_url + self.minted = 0 + self.dead_tokens: set[str] = set() + self.signin: Dict[str, Any] = {"status": "pending"} + self.claim_codes: Dict[str, str] = {} + self.log: deque[Dict[str, Any]] = deque(maxlen=100) + + def snapshot(self) -> Dict[str, Any]: + with self.lock: + return {"nas": self.nas, "inference": self.inference, "once": self.once, + "retry_after": self.retry_after, "minted": self.minted, "signin": dict(self.signin), + "welcome_url": self.welcome_url} + + def set(self, payload: Dict[str, Any]) -> Dict[str, Any]: + with self.lock: + if "nas" in payload: + name = str(payload["nas"] or "ok") + if name not in NAS_SCENARIOS: + raise ValueError(f"unknown nas scenario {name!r}; one of {sorted(NAS_SCENARIOS)}") + self.nas = name + if "inference" in payload: + name = str(payload["inference"] or "ok") + if name not in INFERENCE_SCENARIOS: + raise ValueError(f"unknown inference scenario {name!r}; one of {sorted(INFERENCE_SCENARIOS)}") + self.inference = name + if "once" in payload: + self.once = bool(payload["once"]) + if "retry_after" in payload: + value = payload["retry_after"] + self.retry_after = None if value in (None, "") else max(0, int(value)) + return self.snapshot() + + def consume_once(self, service: str) -> None: + """After a one-shot failure was served, drop that service back to the happy path.""" + with self.lock: + if self.once: + setattr(self, service, "ok") + + def reset(self) -> None: + with self.lock: + self.nas = self.inference = "ok" + self.once = False + self.retry_after = None + self.dead_tokens.clear() + self.signin = {"status": "pending"} + self.claim_codes.clear() + self.log.clear() + + +STATE = State() + + +class Handler(BaseHTTPRequestHandler): + server_version = "free-tier-fault-server/1" + + # --- plumbing -------------------------------------------------------------------------------- + + def log_message(self, fmt: str, *args: Any) -> None: # quieter than the default + pass + + def _cors(self) -> None: + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + self.send_header("Access-Control-Allow-Headers", "content-type, authorization") + + def _send(self, status: int, body: Any = None, *, headers: Optional[Dict[str, str]] = None, + raw: Optional[bytes] = None, content_type: str = "application/json") -> None: + data = raw if raw is not None else json.dumps(body if body is not None else {}).encode() + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(data))) + for key, value in (headers or {}).items(): + self.send_header(key, value) + self._cors() + self.end_headers() + self.wfile.write(data) + with STATE.lock: + STATE.log.append({"at": time.time(), "method": self.command, "path": self.path, "status": status, + "scenario": {"nas": STATE.nas, "inference": STATE.inference}}) + + def _read_json(self) -> Dict[str, Any]: + length = int(self.headers.get("Content-Length") or 0) + raw = self.rfile.read(length) if length else b"" + if not raw: + return {} + try: + parsed = json.loads(raw) + return parsed if isinstance(parsed, dict) else {} + except ValueError: + # Form-encoded (the device-code endpoints post forms). + return {k: v[0] for k, v in parse_qs(raw.decode(errors="replace")).items()} + + def do_OPTIONS(self) -> None: # noqa: N802 + self.send_response(204) + self._cors() + self.end_headers() + + def do_GET(self) -> None: # noqa: N802 + path = urlparse(self.path).path + if path == "/__scenarios": + self._send(200, {"nas": NAS_SCENARIOS, "inference": INFERENCE_SCENARIOS}) + elif path == "/__scenario": + query = {k: v[0] for k, v in parse_qs(urlparse(self.path).query).items()} + if query: + self._control_set(query) + else: + self._send(200, STATE.snapshot()) + elif path == "/__log": + with STATE.lock: + self._send(200, {"requests": list(STATE.log)}) + elif path == "/v1/models": + self._send(200, {"object": "list", "data": [{"id": WELCOME_MODEL, "object": "model", "owned_by": "nous"}]}) + elif path.startswith("/__claim"): + self._send(200, raw=( + b"

Free tier fault server

This stands in for the sign-in page. " + b"Settle the sign-in with POST /__signin.

"), + content_type="text/html") + elif path in ("/", "/healthcheck"): + self._send(200, {"ok": True, "service": "free-tier-fault-server"}) + else: + self._send(404, {"status": 404, "message": "Couldn't find that, sorry."}) + + def do_POST(self) -> None: # noqa: N802 + path = urlparse(self.path).path + body = self._read_json() + if path == "/__scenario": + self._control_set(body) + elif path == "/__signin": + with STATE.lock: + STATE.signin = {"status": str(body.get("status") or "pending"), **{ + k: v for k, v in body.items() if k != "status"}} + self._send(200, STATE.snapshot()) + elif path == "/__reset": + STATE.reset() + self._send(200, STATE.snapshot()) + elif path.startswith("/api/anonymous/"): + self._nas(path, body) + elif path == "/api/oauth/device/code": + self._device_code(body) + elif path == "/api/oauth/token": + self._oauth_token(body) + elif path in ("/v1/chat/completions", "/chat/completions"): + self._inference(body) + else: + self._send(404, {"status": 404, "message": "Couldn't find that, sorry."}) + + def _control_set(self, payload: Dict[str, Any]) -> None: + try: + self._send(200, STATE.set(payload)) + except ValueError as exc: + self._send(400, {"error": str(exc)}) + + # --- the account service (NAS) --------------------------------------------------------------- + + def _retry_after(self, default: int) -> int: + with STATE.lock: + return default if STATE.retry_after is None else STATE.retry_after + + def _nas_gate(self, scenario: str) -> bool: + """The shared gate every /api/anonymous route runs first. True when a refusal was sent.""" + if scenario == "not_enabled": + self._send(404, {"error": "not_found"}) + elif scenario == "paused": + self._send(503, {"error": "temporarily_disabled", + "error_description": "The anonymous-account surface is currently switched off."}) + elif scenario == "server_error": + self._send(500, raw=b"Internal Server Error", content_type="text/html") + elif scenario == "timeout": + time.sleep(12) + return False + else: + return False + STATE.consume_once("nas") + return True + + def _nas(self, path: str, body: Dict[str, Any]) -> None: + with STATE.lock: + scenario = STATE.nas + if self._nas_gate(scenario): + return + if path == "/api/anonymous/create": + if scenario == "rate_limited": + wait = self._retry_after(30) + STATE.consume_once("nas") + self._send(429, {"error": "temporarily_unavailable", + "error_description": "Too many anonymous account creations from this address. Try again later."}, + headers={"Retry-After": str(wait)}) + return + with STATE.lock: + STATE.minted += 1 + n = STATE.minted + token = f"anon_rehearsal_{n:04d}_{secrets.token_hex(4)}" + if scenario == "dead_once": + pass # the credential itself is fine; its first exchange is what dies + self._send(201, {"user_id": f"nas_user:rehearsal-{n}", "org_id": f"nas_org:rehearsal-{n}", + "token": token, "idle_ttl_days": 14}) + return + if path == "/api/anonymous/token": + token = str(body.get("token") or "") + if not token.startswith("anon_"): + self._send(400, {"error": "invalid_request", "error_description": 'Body must be { token: "anon_…" }.'}) + return + with STATE.lock: + dead = token in STATE.dead_tokens + if scenario == "dead_once" and not dead: + # One-shot by definition: this credential is gone for good, the next one works. + STATE.dead_tokens.add(token) + STATE.nas = "ok" + dead = True + if dead: + self._send(404, {"error": "unknown_token", "error_description": "No anonymous account matches this token."}) + return + if scenario == "rate_limited": + STATE.consume_once("nas") + self._send(429, {"error": "temporarily_unavailable", + "error_description": "Too many token exchanges. Try again later."}, + headers={"Retry-After": str(self._retry_after(30))}) + return + if scenario == "pow": + STATE.consume_once("nas") + self._send(428, {"error": "pow_required", "pow": { + "challenge": secrets.token_hex(16), "alg": "blake2s-hashcash-v1", "bits": 31, "count": 16, + "expires_in": 1800}}) + return + if scenario == "locked": + STATE.consume_once("nas") + self._send(403, {"error": "account_locked"}) + return + with STATE.lock: + welcome = STATE.welcome_url + payload: Dict[str, Any] = { + "access_token": _jwt(sid=token[-8:]), "token_type": "Bearer", "expires_in": 900, + "user_id": "nas_user:rehearsal", "org_id": "nas_org:rehearsal"} + if welcome: + payload["inference_base_url"] = welcome + self._send(200, payload) + return + if path == "/api/anonymous/promotion-intent": + if scenario == "signin_busy": + STATE.consume_once("nas") + self._send(429, {"error": "temporarily_unavailable", + "error_description": "Too many promotion requests. Try again later."}, + headers={"Retry-After": str(self._retry_after(45))}) + return + if scenario == "signin_paused": + STATE.consume_once("nas") + self._send(503, {"error": "temporarily_disabled", + "error_description": "The anonymous-account surface is currently switched off."}) + return + if scenario == "signin_server_error": + STATE.consume_once("nas") + self._send(500, {"error": "internal_error"}) + return + code = f"{secrets.token_hex(2).upper()}-{secrets.token_hex(2).upper()}" + with STATE.lock: + STATE.claim_codes[code] = str(body.get("token") or "") + STATE.signin = {"status": "pending"} + host = self.headers.get("Host") or "127.0.0.1" + self._send(200, {"claim_code": code, "claim_url": f"http://{host}/__claim?code={code}", + "expires_in": 900, "interval": 2}) + return + if path == "/api/anonymous/promotion-status": + with STATE.lock: + outcome = dict(STATE.signin) + self._send(200, outcome if outcome.get("status") != "pending" else {"status": "pending"}) + return + if path == "/api/anonymous/claim": + self._send(401, {"error": "unauthorized"}) + return + self._send(404, {"error": "not_found"}) + + def _device_code(self, body: Dict[str, Any]) -> None: + host = self.headers.get("Host") or "127.0.0.1" + code = secrets.token_hex(8) + self._send(200, {"device_code": f"dc_{code}", "user_code": f"{code[:4].upper()}-{code[4:8].upper()}", + "verification_uri": f"http://{host}/__claim", + "verification_uri_complete": f"http://{host}/__claim?device={code}", + "expires_in": 900, "interval": 2}) + + def _oauth_token(self, body: Dict[str, Any]) -> None: + # A completed sign-in is not something this stand-in can finish honestly (the real + # portal issues signed tokens the agent then verifies), so the token poll stays pending. + self._send(400, {"error": "authorization_pending", + "error_description": "The free tier fault server never completes a sign-in; rehearse the failure paths here and the happy path against staging."}) + + # --- the welcome inference host -------------------------------------------------------------- + + def _refusal(self, status: int, message: str, *, reason: str, retry_after: int, + alternates: Optional[list] = None, extra_headers: Optional[Dict[str, str]] = None) -> None: + headers = {"Retry-After": str(retry_after)} + if reason in ("rate_limited", "at_capacity"): + headers["RateLimit-Policy"] = '"fairshare";q=0;qu="tokens";w=60' + headers["RateLimit"] = f'"fairshare";r=0;t={retry_after}' + headers.update(extra_headers or {}) + STATE.consume_once("inference") + self._send(status, {"status": status, "message": message, "reason": reason, "retry_after": retry_after, + "alternates": alternates or [], "upgrade_url": UPGRADE_URL}, headers=headers) + + def _inference(self, body: Dict[str, Any]) -> None: + with STATE.lock: + scenario = STATE.inference + model = str(body.get("model") or WELCOME_MODEL) + if scenario == "rate_limited": + self._refusal(429, "You've reached this model's current fair-share rate limit. It adapts to demand — " + "retry after the indicated delay, or try an alternate model.", + reason="rate_limited", retry_after=self._retry_after(600)) + elif scenario == "rate_limited_short": + self._refusal(429, "You've reached this model's current fair-share rate limit. It adapts to demand — " + "retry after the indicated delay, or try an alternate model.", + reason="rate_limited", retry_after=self._retry_after(5)) + elif scenario == "at_capacity": + self._refusal(429, "The free tier is at capacity and briefly paused. It reopens automatically — " + "retry after the indicated delay.", reason="at_capacity", retry_after=self._retry_after(30)) + elif scenario == "model_not_free": + self._refusal(429, "This model isn't available on the free tier.", reason="model_not_free", + retry_after=0, alternates=[WELCOME_MODEL]) + elif scenario == "tier_disabled": + STATE.consume_once("inference") + self._send(403, {"status": 403, "message": GENERIC_403}) + elif scenario == "wrong_host": + STATE.consume_once("inference") + self._send(400, {"status": 400, "message": "This request is not valid. Check the model name and other " + f"parameters. Additional info: Anonymous accounts must use https://{REAL_WELCOME_HOST} for inference."}) + elif scenario == "bare_429": + STATE.consume_once("inference") + self._send(429, {"status": 429, "message": "Hold up for a bit, you've exceeded the rate limit on your API key."}, + headers={"x-ratelimit-limit-requests": "30", "x-ratelimit-remaining-requests": "0", + "x-ratelimit-reset-requests": str(self._retry_after(600)), + "Retry-After": str(self._retry_after(600))}) + elif scenario == "upstream_503": + STATE.consume_once("inference") + self._send(503, {"status": 503, "message": "The requested model is currently unavailable."}) + elif scenario == "upstream_500": + STATE.consume_once("inference") + self._send(500, {"status": 500, "message": "Something unexpected happened while processing your request. " + "Please try again in a moment, or contact us if the issue persists."}) + elif scenario == "invalid_token": + STATE.consume_once("inference") + self._send(401, {"status": 401, "error": "invalid_token", "subcause": "anonymous_credential_revoked", + "message": "The anonymous account behind this token is gone"}) + elif scenario == "timeout": + STATE.consume_once("inference") + time.sleep(120) + self._send(504, {"status": 504, "message": "timed out"}) + else: + self._reply(model, stream=bool(body.get("stream"))) + + def _reply(self, model: str, *, stream: bool) -> None: + text = ("Hello from the free tier fault server. Everything is working; switch a scenario on " + "POST /__scenario to rehearse a failure.") + now = int(time.time()) + ident = f"chatcmpl-rehearsal-{secrets.token_hex(4)}" + if not stream: + self._send(200, {"id": ident, "object": "chat.completion", "created": now, "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, + "finish_reason": "stop"}], + "usage": {"prompt_tokens": 12, "completion_tokens": 24, "total_tokens": 36}}) + return + chunks = [] + for i, piece in enumerate(text.split(" ")): + delta = {"content": (" " if i else "") + piece} + if i == 0: + delta["role"] = "assistant" + chunks.append({"id": ident, "object": "chat.completion.chunk", "created": now, "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": None}]}) + chunks.append({"id": ident, "object": "chat.completion.chunk", "created": now, "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 12, "completion_tokens": 24, "total_tokens": 36}}) + raw = "".join(f"data: {json.dumps(c)}\n\n" for c in chunks) + "data: [DONE]\n\n" + self._send(200, raw=raw.encode(), content_type="text/event-stream") + + +def make_server(host: str = "127.0.0.1", port: int = 8765, *, nas: str = "ok", inference: str = "ok", + welcome_url: Optional[str] = None) -> ThreadingHTTPServer: + """Build (not start) the server; tests bind port 0 and read ``server.server_address``.""" + global STATE + STATE = State(nas=nas, inference=inference, welcome_url=welcome_url or "") + server = ThreadingHTTPServer((host, port), Handler) + server.daemon_threads = True + if not welcome_url: + STATE.welcome_url = f"http://{server.server_address[0]}:{server.server_address[1]}/v1" + return server + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8765) + parser.add_argument("--nas", default="ok", choices=sorted(NAS_SCENARIOS)) + parser.add_argument("--inference", default="ok", choices=sorted(INFERENCE_SCENARIOS)) + args = parser.parse_args() + server = make_server(args.host, args.port, nas=args.nas, inference=args.inference) + base = f"http://{args.host}:{args.port}" + print(f"free tier fault server on {base} (nas={args.nas}, inference={args.inference})") + print("Point Hermes at it with:") + print(f" export HERMES_GUEST_ONBOARDING=1 HERMES_PORTAL_BASE_URL={base} " + f"NOUS_INFERENCE_BASE_URL={base}/v1 HERMES_EXTRA_WELCOME_HOSTS={args.host}") + print("Switch a scenario with:") + print(f" curl -s -X POST {base}/__scenario -d '{{\"inference\": \"rate_limited\"}}'") + print(f" fetch('{base}/__scenario', {{method: 'POST', body: JSON.stringify({{nas: 'paused'}})}})") + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() + + +if __name__ == "__main__": + main() diff --git a/tests/agent/test_nous_rate_guard.py b/tests/agent/test_nous_rate_guard.py index 91e75bab29..949ee3f9cc 100644 --- a/tests/agent/test_nous_rate_guard.py +++ b/tests/agent/test_nous_rate_guard.py @@ -301,7 +301,7 @@ class TestWelcomeRouteCopy: "https://welcome-api.nousresearch.com/v1", monkeypatch ) - expected = anon_auth.FREE_TIER_RATE_LIMIT_CHAT.format(reset="10m") + expected = anon_auth.FREE_TIER_RATE_LIMIT_CHAT.format(reset=anon_auth.friendly_wait(600)) assert verdict.action == "return" assert statuses == [f"⏳ {expected}"] assert expected in verdict.result["final_response"] diff --git a/tests/agent/test_nous_welcome_client_contract.py b/tests/agent/test_nous_welcome_client_contract.py index 356ea223bd..6ff7017cfc 100644 --- a/tests/agent/test_nous_welcome_client_contract.py +++ b/tests/agent/test_nous_welcome_client_contract.py @@ -131,13 +131,13 @@ class TestRefusalCopy: def test_copy_names_the_served_model_and_the_sign_in(self): refusal = anon_auth.parse_welcome_refusal({"reason": "model_not_free", "alternates": ["nous/welcome"]}) chat = anon_auth.welcome_refusal_copy(refusal, model="gpt-5", in_chat=True) - assert chat == "gpt-5 isn't on the Nous free tier; it serves nous/welcome only. Sign in with a Nous account for the full catalog: /login." + assert chat == "gpt-5 isn't available without signing in, so Hermes uses nous/welcome for now. Sign in for more models. To sign in: /login." terminal = anon_auth.welcome_refusal_copy(refusal, model="gpt-5", in_chat=False) assert "`hermes auth upgrade`" in terminal and "/login" not in terminal def test_capacity_copy_carries_the_retry(self): refusal = anon_auth.parse_welcome_refusal({"reason": "at_capacity", "retry_after": 30}) - assert "Retrying in 30s." in anon_auth.welcome_refusal_copy(refusal) + assert "try again in about a minute" in anon_auth.welcome_refusal_copy(refusal) @pytest.mark.parametrize("copy_fn, args", [ (anon_auth.welcome_refusal_copy, ({"reason": r},)) for r in sorted(anon_auth.WELCOME_REFUSAL_REASONS) diff --git a/tests/agent/test_turn_retry_state.py b/tests/agent/test_turn_retry_state.py index 2b7cb6b56b..367981ebeb 100644 --- a/tests/agent/test_turn_retry_state.py +++ b/tests/agent/test_turn_retry_state.py @@ -18,6 +18,8 @@ EXPECTED_FIELDS = { "anthropic_auth_retry_attempted", "nous_auth_retry_attempted", "nous_paid_entitlement_refresh_attempted", + "welcome_model_switch_attempted", + "welcome_route_heal_attempted", "copilot_auth_retry_attempted", "copilot_stale_cred_retry_attempted", "vertex_auth_retry_attempted", diff --git a/tests/agent/test_welcome_tier_recovery.py b/tests/agent/test_welcome_tier_recovery.py new file mode 100644 index 0000000000..dc00a99a6a --- /dev/null +++ b/tests/agent/test_welcome_tier_recovery.py @@ -0,0 +1,124 @@ +"""Nous free tier, inference side: the dark-tier 403 keyed on the route, the one-shot model move +after ``model_not_free``, the wrong-host heal, the long-wait rule for structured ``rate_limited`` +refusals, and the plain outage sentence once retries are spent.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from agent.error_classifier import FailoverReason, classify_api_error +from agent.turn_retry_state import TurnRetryState + +WELCOME = "https://welcome-api.nousresearch.com/v1" +PAID = "https://inference-api.nousresearch.com/v1" + + +class MockAPIError(Exception): + def __init__(self, message, *, status_code=None, body=None): + super().__init__(message) + self.message = message + self.status_code = status_code + self.body = body + + +def _generic_403(): + body = {"status": 403, "message": "You tried to access something that you don't have permissions for."} + return MockAPIError(f"Error code: 403 - {body}", status_code=403, body=body) + + +class TestDarkTier403: + def test_a_generic_403_from_the_welcome_host_is_the_tier_refusing(self): + result = classify_api_error(_generic_403(), provider="nous", model="nous/welcome", base_url=WELCOME) + assert result.reason == FailoverReason.auth_permanent + assert result.retryable is False and result.should_fallback is True + assert result.error_context["welcome_route"] == "tier_disabled" + + def test_the_same_403_from_the_paid_host_stays_an_ordinary_403(self): + result = classify_api_error(_generic_403(), provider="nous", model="nous/welcome", base_url=PAID) + assert "welcome_route" not in result.error_context + + def test_a_403_from_another_provider_on_any_host_is_untouched(self): + result = classify_api_error(_generic_403(), provider="openrouter", base_url=WELCOME) + assert "welcome_route" not in result.error_context + + +def _agent(**overrides): + lines = [] + agent = SimpleNamespace( + provider="nous", model="gpt-5", base_url=WELCOME, log_prefix="", + _vprint=lambda text, force=False: lines.append(text), + _try_refresh_nous_client_credentials=lambda **kw: True, + ) + for k, v in overrides.items(): + setattr(agent, k, v) + agent.lines = lines + return agent + + +class TestOneShotRecoveries: + def test_model_not_free_moves_the_session_onto_the_alternate_and_retries_once(self): + from agent.turn_recovery import _recover_welcome_tier + agent = _agent() + ctx = {"welcome_refusal": {"reason": "model_not_free", "retry_after": 0, + "alternates": ["nous/welcome"], "upgrade_url": ""}} + retry = TurnRetryState() + assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), retry, ctx) is True + assert agent.model == "nous/welcome" + assert agent._nous_model_switch == ("gpt-5", "nous/welcome") + assert "without signing in" in agent.lines[0] + # Once: a second refusal in the same attempt falls through to the terminal path. + assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), retry, ctx) is False + + def test_model_not_free_without_an_alternate_does_nothing(self): + from agent.turn_recovery import _recover_welcome_tier + agent = _agent() + ctx = {"welcome_refusal": {"reason": "model_not_free", "retry_after": 0, "alternates": [], "upgrade_url": ""}} + assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), TurnRetryState(), ctx) is False + assert agent.model == "gpt-5" + + def test_a_wrong_host_refusal_re_reads_the_route_once(self): + from agent.turn_recovery import _recover_welcome_tier + calls = [] + agent = _agent(_try_refresh_nous_client_credentials=lambda **kw: calls.append(kw) or True) + ctx = {"welcome_route": "anon_on_paid_host"} + retry = TurnRetryState() + assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), retry, ctx) is True + assert calls == [{"force": True}] + assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), retry, ctx) is False + + def test_a_wrong_host_refusal_whose_heal_fails_falls_through(self): + from agent.turn_recovery import _recover_welcome_tier + agent = _agent(_try_refresh_nous_client_credentials=lambda **kw: False) + ctx = {"welcome_route": "anon_on_paid_host"} + assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), TurnRetryState(), ctx) is False + + +class TestLongWaitRule: + @pytest.mark.parametrize("reason,retry_after,expected", [ + ("rate_limited", 20, True), ("rate_limited", 600, True), ("rate_limited", 19, False), + ("rate_limited", 0, False), ("at_capacity", 30, False), ("admission_closed", 30, False), + ]) + def test_only_a_long_rate_limited_refusal_is_an_exhausted_allowance(self, reason, retry_after, expected): + from agent.nous_rate_guard import is_long_welcome_rate_limit + ctx = {"welcome_refusal": {"reason": reason, "retry_after": retry_after, "alternates": [], "upgrade_url": ""}} + assert is_long_welcome_rate_limit(ctx) is expected + + def test_no_refusal_is_not_long(self): + from agent.nous_rate_guard import is_long_welcome_rate_limit + assert is_long_welcome_rate_limit({}) is False and is_long_welcome_rate_limit(None) is False + + +class TestOutageCopy: + @pytest.mark.parametrize("reason", [FailoverReason.timeout, FailoverReason.overloaded, + FailoverReason.server_error, FailoverReason.unknown]) + def test_a_spent_transport_failure_on_the_welcome_host_reads_as_one_sentence(self, reason): + from agent.turn_recovery import _welcome_outage_copy + from hermes_cli.anon_auth import FREE_TIER_OUTAGE_COPY + assert _welcome_outage_copy(WELCOME, SimpleNamespace(reason=reason)) == FREE_TIER_OUTAGE_COPY + + def test_other_routes_and_other_reasons_keep_the_technical_summary(self): + from agent.turn_recovery import _welcome_outage_copy + assert _welcome_outage_copy(PAID, SimpleNamespace(reason=FailoverReason.timeout)) == "" + assert _welcome_outage_copy(WELCOME, SimpleNamespace(reason=FailoverReason.rate_limit)) == "" diff --git a/tests/hermes_cli/test_anon_auth_core.py b/tests/hermes_cli/test_anon_auth_core.py index 9c5b8f2f77..a04649898b 100644 --- a/tests/hermes_cli/test_anon_auth_core.py +++ b/tests/hermes_cli/test_anon_auth_core.py @@ -88,7 +88,7 @@ def portal(monkeypatch, tmp_path): kw["transport"] = httpx.MockTransport(fake.handler) super().__init__(*a, **kw) monkeypatch.setattr(httpx, "Client", _RoutedClient) - anon_auth._mint_failed = False + anon_auth.reset_mint_memo_for_tests() from hermes_cli import free_tier_bootstrap as _fb _fb.reset_for_tests() # resolve_nous_access_token memoises the last token for 5 s per profile home (dict); a token minted diff --git a/tests/hermes_cli/test_anon_failure_modes.py b/tests/hermes_cli/test_anon_failure_modes.py new file mode 100644 index 0000000000..00fcd79c1a --- /dev/null +++ b/tests/hermes_cli/test_anon_failure_modes.py @@ -0,0 +1,348 @@ +"""Nous free tier: every way the account service or the wire can refuse the free tier, and what +Hermes does with each (the failure-mode contract behind the desktop's onboarding copy). + +Driven through a fake NAS whose responses are the ones the real service sends (see the code table +in ``hermes_cli.anon_auth``), never through mocked-away client code. +""" + +from __future__ import annotations + +import base64 +import json +import time + +import httpx +import pytest + +from hermes_cli import anon_auth, anon_sign_in, free_tier_bootstrap +from hermes_cli.auth import _load_auth_store + +PORTAL = "https://portal.example.test" +WELCOME = "https://welcome-api.nousresearch.com/v1" + + +def _jwt(**claims) -> str: + def seg(obj): + return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode() + payload = {"sub": "nas_user:1", "client_id": "nas-anonymous", "account_tier": "anonymous", + "scope": "inference:invoke tool:invoke", "exp": int(time.time()) + 900, **claims} + return f"{seg({'alg': 'RS256'})}.{seg(payload)}.sig" + + +class FakeNas: + """The anonymous surface as NAS ships it. ``create_response`` / ``token_response`` override the + happy path with one canned refusal; ``raise_transport`` simulates the wire failing.""" + + def __init__(self): + self.calls: list[tuple[str, str]] = [] + self.minted = 0 + self.create_response: httpx.Response | None = None + self.token_response: httpx.Response | None = None + self.raise_transport: Exception | None = None + + def handler(self, request: httpx.Request) -> httpx.Response: + path = request.url.path + self.calls.append((request.method, path)) + if self.raise_transport is not None: + raise self.raise_transport + if path == "/api/anonymous/create": + if self.create_response is not None: + return self.create_response + self.minted += 1 + return httpx.Response(201, json={"user_id": f"nas_user:{self.minted}", "org_id": "nas_org:1", + "token": f"anon_{self.minted:04d}", "idle_ttl_days": 14}) + if path == "/api/anonymous/token": + if self.token_response is not None: + return self.token_response + return httpx.Response(200, json={"access_token": _jwt(), "token_type": "Bearer", "expires_in": 900, + "user_id": "nas_user:1", "org_id": "nas_org:1", + "inference_base_url": WELCOME}) + return httpx.Response(500, json={"error": f"unexpected {path}"}) + + def creates(self) -> int: + return [p for _, p in self.calls].count("/api/anonymous/create") + + +@pytest.fixture +def nas(monkeypatch, tmp_path): + fake = FakeNas() + monkeypatch.setenv("HERMES_PORTAL_BASE_URL", PORTAL) + monkeypatch.setenv("HERMES_SHARED_AUTH_DIR", str(tmp_path / "shared-store")) + monkeypatch.setenv("HERMES_GUEST_ONBOARDING", "1") + for var in ("OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "NOUS_API_KEY"): + monkeypatch.delenv(var, raising=False) + from hermes_cli import auth_nous + + def _client(timeout_seconds, verify): + return httpx.Client(transport=httpx.MockTransport(fake.handler), base_url=PORTAL) + monkeypatch.setattr(auth_nous, "_nous_http_client", _client) + # The runtime resolver builds its own client; route it through the fake too. + real_client = httpx.Client + + class _RoutedClient(real_client): + def __init__(self, *a, **kw): + kw.pop("verify", None) + kw["transport"] = httpx.MockTransport(fake.handler) + super().__init__(*a, **kw) + monkeypatch.setattr(httpx, "Client", _RoutedClient) + monkeypatch.setattr("agent.bedrock_adapter.has_aws_credentials", lambda: False) + anon_auth.reset_mint_memo_for_tests() + free_tier_bootstrap.reset_for_tests() + from hermes_cli import auth as auth_mod + monkeypatch.setattr(auth_mod, "_RESOLVE_TOKEN_CACHE", {}) + return fake + + +def _mint_error(nas: FakeNas) -> anon_auth.AuthError: + with pytest.raises(anon_auth.AuthError) as exc: + anon_auth.ensure_portal_identity(explicit=True) + return exc.value + + +def _exchange_error(nas: FakeNas) -> anon_auth.AuthError: + """Mint (the credential is persisted before any exchange), then exchange it at first use.""" + from hermes_cli.auth_nous import resolve_nous_runtime_credentials + assert anon_auth.is_guest_state(anon_auth.ensure_portal_identity(explicit=True)) + with pytest.raises(anon_auth.AuthError) as exc: + resolve_nous_runtime_credentials() + return exc.value + + +# --- What NAS sends -> which code ------------------------------------------------------------------ + + +class TestNasRefusalCodes: + def test_surface_not_enabled_is_a_terminal_gate(self, nas): + nas.create_response = httpx.Response(404, json={"error": "not_found"}) + err = _mint_error(nas) + assert err.code == anon_auth.ANON_GATE_CLOSED and err.retryable is False + assert "Nous account" in str(err) and "free" in str(err) + # Terminal: no later attempt this process, whatever the clock says. + assert anon_auth.ensure_portal_identity(explicit=True) is None + assert nas.creates() == 1 + assert anon_auth.last_mint_failure() == { + "error_code": anon_auth.ANON_GATE_CLOSED, "error": str(err), "retryable": False, "retry_after": 0} + + def test_ops_breaker_is_a_retryable_pause_with_a_floor(self, nas): + nas.create_response = httpx.Response( + 503, json={"error": "temporarily_disabled", "error_description": "switched off"}) + err = _mint_error(nas) + assert err.code == anon_auth.ANON_GATE_PAUSED + failure = anon_auth.last_mint_failure() + assert failure["retryable"] is True + assert failure["retry_after"] >= 59 # never polled faster than the paused floor + + def test_too_many_sign_ups_honours_retry_after(self, nas): + nas.create_response = httpx.Response( + 429, json={"error": "temporarily_unavailable"}, headers={"Retry-After": "42"}) + err = _mint_error(nas) + assert err.code == anon_auth.ANON_RATE_LIMITED and err.retry_after == 42 + assert "about a minute" in str(err) + assert anon_auth.last_mint_failure()["retry_after"] == 42 + + def test_proof_of_work_is_deferred_with_the_agreed_sentence(self, nas): + nas.token_response = httpx.Response(428, json={"error": "pow_required", "pow": {"bits": 31}}) + err = _exchange_error(nas) + assert err.code == anon_auth.ANON_POW_REQUIRED and err.retryable is False + assert str(err).startswith("The Nous server asked for a proof of work, but that isn't implemented") + # The credential from ``create`` is kept, so a later NAS without PoW exchanges it instead + # of minting again. + assert anon_auth.is_guest_state(_load_auth_store()["providers"]["nous"]) + assert nas.creates() == 1 + + def test_locked_account_is_dead_and_never_replaced(self, nas): + nas.token_response = httpx.Response(403, json={"error": "account_locked"}) + err = _exchange_error(nas) + assert isinstance(err, anon_auth.AnonCredentialDead) + assert err.code == anon_auth.ANON_ACCOUNT_LOCKED + # Retired, and NOT replaced by a fresh identity: the way forward is a sign-in. + assert "nous" not in _load_auth_store().get("providers", {}) + assert nas.creates() == 1 + + def test_unknown_token_is_replaced_once_at_first_use(self, nas): + from hermes_cli.auth_nous import resolve_nous_runtime_credentials + first = anon_auth.ensure_portal_identity(explicit=True) + original = nas.handler + + def _first_only(request): + # Only the FIRST credential is dead; its replacement exchanges normally. + if request.url.path == "/api/anonymous/token" and json.loads(request.content)["token"] == first["anon_token"]: + nas.calls.append((request.method, request.url.path)) + return httpx.Response(404, json={"error": "unknown_token"}) + return original(request) + nas.handler = _first_only # type: ignore[method-assign] + try: + creds = resolve_nous_runtime_credentials() + finally: + nas.handler = original # type: ignore[method-assign] + assert creds["base_url"].startswith(WELCOME) + assert nas.creates() == 2 + assert _load_auth_store()["providers"]["nous"]["anon_token"] != first["anon_token"] + + def test_a_5xx_or_non_json_body_is_a_retryable_server_error(self, nas): + nas.create_response = httpx.Response(500, text="oops") + err = _mint_error(nas) + assert err.code == anon_auth.ANON_SERVER_ERROR and err.retryable is not False + assert "hiccup" in str(err) + + def test_the_wire_failing_is_unreachable(self, nas): + nas.raise_transport = httpx.ConnectTimeout("no route") + err = _mint_error(nas) + assert err.code == anon_auth.ANON_UNREACHABLE + assert isinstance(err.__cause__, httpx.ConnectTimeout) + assert "internet connection" in str(err) + + +# --- The cooldown memo --------------------------------------------------------------------------- + + +class TestMintCooldown: + def test_a_transient_failure_is_retried_after_its_cooldown_not_never(self, nas, monkeypatch): + nas.raise_transport = httpx.ConnectTimeout("no route") + _mint_error(nas) + assert anon_auth.ensure_portal_identity(explicit=True) is None # inside the cooldown + assert nas.creates() == 1 + failure = anon_auth._mint_failure_for_profile() + assert failure.retry_after == anon_auth._MINT_RETRY_LADDER[0] + monkeypatch.setattr(anon_auth.time, "monotonic", lambda: failure.not_before + 1) + nas.raise_transport = None + state = anon_auth.ensure_portal_identity(explicit=True) # the cooldown passed + assert anon_auth.is_guest_state(state) + assert nas.creates() == 2 + assert anon_auth.last_mint_failure() is None # success clears the memo + + def test_repeated_transient_failures_climb_the_ladder(self, nas, monkeypatch): + nas.raise_transport = httpx.ConnectTimeout("no route") + waits = [] + for _ in range(4): + _mint_error(nas) + failure = anon_auth._mint_failure_for_profile() + waits.append(failure.retry_after) + monkeypatch.setattr(anon_auth.time, "monotonic", lambda f=failure: f.not_before + 1) + assert waits == [15.0, 60.0, 300.0, 300.0] + + def test_the_users_own_retry_bypasses_the_cooldown_once(self, nas): + nas.raise_transport = httpx.ConnectTimeout("no route") + _mint_error(nas) + nas.raise_transport = None + assert anon_auth.ensure_portal_identity(explicit=True) is None + assert anon_auth.is_guest_state(anon_auth.ensure_portal_identity(explicit=True, force=True)) + assert nas.creates() == 2 + + def test_a_terminal_code_ignores_even_a_forced_retry_by_making_one_attempt_only(self, nas): + nas.create_response = httpx.Response(404, json={"error": "not_found"}) + _mint_error(nas) + with pytest.raises(anon_auth.AuthError): + anon_auth.ensure_portal_identity(explicit=True, force=True) # the click = one attempt + assert nas.creates() == 2 + assert anon_auth.ensure_portal_identity(explicit=True) is None # and still terminal + + +# --- The boot record and its retries ------------------------------------------------------------- + + +class TestBootstrapRecord: + def test_the_record_carries_the_code_and_the_wait(self, nas): + nas.create_response = httpx.Response( + 429, json={"error": "temporarily_unavailable"}, headers={"Retry-After": "30"}) + record = free_tier_bootstrap.run_bootstrap(announce=False) + assert record.has_identity is False and record.free_tier is False + assert record.error_code == anon_auth.ANON_RATE_LIMITED + assert record.retryable is True and 28 <= record.retry_after <= 30 + assert record.failure_fields() == { + "error": record.error, "error_code": anon_auth.ANON_RATE_LIMITED, "retryable": True, + "retry_after": record.retry_after} + + def test_a_clean_boot_carries_no_failure_block(self, nas): + record = free_tier_bootstrap.run_bootstrap(announce=False) + assert record.free_tier is True and record.failure_fields() == {} + + def test_the_background_loop_retries_a_transient_failure_until_it_settles(self, nas, monkeypatch): + nas.raise_transport = httpx.ConnectTimeout("no route") + slept = [] + + def _sleep(seconds): + slept.append(seconds) + # The cooldown passes while we "slept". + failure = anon_auth._mint_failure_for_profile() + monkeypatch.setattr(anon_auth.time, "monotonic", lambda f=failure: f.not_before + 1) + if len(slept) == 2: + nas.raise_transport = None # the network comes back on the second wait + monkeypatch.setattr(free_tier_bootstrap, "_sleep", _sleep) + free_tier_bootstrap._bootstrap_then_retry() + record = free_tier_bootstrap.current_record() + assert record.has_identity is True and record.free_tier is True + assert slept == [15, 60] + assert nas.creates() == 3 + + def test_the_background_loop_never_retries_a_terminal_code(self, nas, monkeypatch): + nas.create_response = httpx.Response(404, json={"error": "not_found"}) + monkeypatch.setattr(free_tier_bootstrap, "_sleep", lambda s: pytest.fail("must not sleep")) + free_tier_bootstrap._bootstrap_then_retry() + assert free_tier_bootstrap.current_record().error_code == anon_auth.ANON_GATE_CLOSED + assert nas.creates() == 1 + + def test_the_background_loop_is_bounded(self, nas, monkeypatch): + nas.raise_transport = httpx.ConnectTimeout("no route") + + def _sleep(seconds): + failure = anon_auth._mint_failure_for_profile() + monkeypatch.setattr(anon_auth.time, "monotonic", lambda f=failure: f.not_before + 1) + monkeypatch.setattr(free_tier_bootstrap, "_sleep", _sleep) + free_tier_bootstrap._bootstrap_then_retry() + assert nas.creates() == 1 + free_tier_bootstrap.BOOTSTRAP_RETRY_ATTEMPTS + assert free_tier_bootstrap.current_record().error_code == anon_auth.ANON_UNREACHABLE + + def test_the_desktop_retry_refreshes_the_boot_record(self, nas): + nas.raise_transport = httpx.ConnectTimeout("no route") + free_tier_bootstrap.run_bootstrap(announce=False) + nas.raise_transport = None + record = free_tier_bootstrap.retry_bootstrap_mint(force=True, announce=False) + assert record.free_tier is True and record.failure_fields() == {} + assert free_tier_bootstrap.current_record() is record + + +# --- Signing in while the service misbehaves ------------------------------------------------------ + + +class TestSignInFailures: + @pytest.mark.parametrize("code,retry_after,retryable,needle", [ + (anon_auth.ANON_RATE_LIMITED, 45, True, "busy"), + (anon_auth.ANON_GATE_PAUSED, 0, True, "busy"), + (anon_auth.ANON_UNREACHABLE, 0, True, "internet connection"), + (anon_auth.ANON_GATE_CLOSED, 0, False, "Nous account"), + (anon_auth.ANON_POW_REQUIRED, 0, False, "proof of work"), + ]) + def test_a_service_verdict_keeps_its_code_wait_and_copy(self, code, retry_after, retryable, needle): + state = anon_sign_in._failed_from_exception( + anon_auth._anon_err("x", code, retry_after=retry_after or None, retryable=retryable)) + assert state.kind == "failed" and state.reason == code + assert state.retryable is retryable + assert state.retry_after == retry_after + assert needle in state.copy + assert "Hermes " not in state.copy and "http" not in state.copy + + def test_the_wire_failing_reads_as_unreachable(self): + state = anon_sign_in._failed_from_exception(httpx.ReadTimeout("slow")) + assert state.reason == anon_auth.ANON_UNREACHABLE and state.retryable is True + + def test_an_unnamed_local_failure_keeps_the_generic_copy_and_its_terminal_detail(self): + state = anon_sign_in._failed_from_exception(RuntimeError("bad CA bundle")) + assert state.reason == "" and state.copy == anon_sign_in.UPGRADE_NOT_COMPLETED + assert state.copy_terminal == "Sign-in failed: bad CA bundle" + + def test_account_busy_is_retryable(self): + state = anon_sign_in.Failed(reason="account_busy") + assert state.retryable is True and "few seconds" in state.copy + + +# --- The spoken wait ----------------------------------------------------------------------------- + + +@pytest.mark.parametrize("seconds,expected", [ + (0, "a few seconds"), (5, "a few seconds"), (30, "about a minute"), (89, "about a minute"), + (300, "about 5 minutes"), (3599, "about 60 minutes"), (3600, "about an hour"), (7200, "about 2 hours"), + ("nope", "a few seconds"), +]) +def test_friendly_wait_never_prints_raw_seconds(seconds, expected): + assert anon_auth.friendly_wait(seconds) == expected diff --git a/tests/hermes_cli/test_anon_surfaces.py b/tests/hermes_cli/test_anon_surfaces.py index 19cf5d140f..d29919881b 100644 --- a/tests/hermes_cli/test_anon_surfaces.py +++ b/tests/hermes_cli/test_anon_surfaces.py @@ -192,6 +192,7 @@ def test_no_chat_copy_of_any_sign_in_state_leaks_a_terminal_verb_or_a_forbidden_ "model_changed": True, "reason": "unknown", "detail": "private detail", + "retry_after": 0.0, } forbidden = re.compile(r"claim|nous portal|anonymous|guest", re.IGNORECASE) terminal_or_url = re.compile(r"hermes |https?://", re.IGNORECASE) diff --git a/tests/hermes_cli/test_anon_upgrade.py b/tests/hermes_cli/test_anon_upgrade.py index 15ffa1df59..4e5657daa1 100644 --- a/tests/hermes_cli/test_anon_upgrade.py +++ b/tests/hermes_cli/test_anon_upgrade.py @@ -115,7 +115,7 @@ def portal(monkeypatch, tmp_path): kw["transport"] = httpx.MockTransport(fake.handler) super().__init__(*a, **kw) monkeypatch.setattr(httpx, "Client", _RoutedClient) - anon_auth._mint_failed = False + anon_auth.reset_mint_memo_for_tests() return fake @@ -149,7 +149,7 @@ class TestUpgrade: code = anon_auth.upgrade_guest(_args()) out = capsys.readouterr().out assert code == 1 - assert "Sign-in was rejected in the browser." in out + assert anon_auth.UPGRADE_REASON_COPY["user_declined"] in out assert portal.token_grants == 0 assert _auth_file_path().read_bytes() == before assert _shared_store(tmp_path) == shared_before diff --git a/tests/hermes_cli/test_multiplex_cli_cache_scope.py b/tests/hermes_cli/test_multiplex_cli_cache_scope.py index dc5a358e8e..1d815a168c 100644 --- a/tests/hermes_cli/test_multiplex_cli_cache_scope.py +++ b/tests/hermes_cli/test_multiplex_cli_cache_scope.py @@ -211,8 +211,7 @@ def test_failed_guest_mint_only_suppresses_that_profile(homes, monkeypatch, tmp_ import hermes_cli.anon_auth as anon import hermes_cli.auth_nous as auth_nous - monkeypatch.setattr(anon, "_mint_failed", False) - monkeypatch.setattr(anon, "_mint_failed_homes", set(), raising=False) + anon.reset_mint_memo_for_tests() status = {"code": 429} attempts: list[str] = [] diff --git a/tests/scripts/test_free_tier_fault_server.py b/tests/scripts/test_free_tier_fault_server.py new file mode 100644 index 0000000000..4130185b71 --- /dev/null +++ b/tests/scripts/test_free_tier_fault_server.py @@ -0,0 +1,189 @@ +"""``scripts/free_tier_fault_server.py`` speaks the real free-tier wire contract: Hermes' own client +code, pointed at it, reaches the same ``anon_*`` codes and welcome refusals it reaches against +production. That is what makes a rehearsal against it worth anything.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +import threading +from pathlib import Path + +import httpx +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "scripts" / "free_tier_fault_server.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("free_tier_fault_server", SCRIPT) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def server(): + module = _load() + srv = module.make_server("127.0.0.1", 0) + thread = threading.Thread(target=srv.serve_forever, daemon=True) + thread.start() + host, port = srv.server_address + base = f"http://{host}:{port}" + try: + yield module, base + finally: + srv.shutdown() + srv.server_close() + + +def _post(base: str, path: str, body=None, **kw) -> httpx.Response: + return httpx.post(f"{base}{path}", json=body or {}, timeout=5.0, **kw) + + +class TestControlSurface: + def test_catalogue_switches_and_reset(self, server): + module, base = server + catalogue = httpx.get(f"{base}/__scenarios", timeout=5.0).json() + assert set(catalogue["nas"]) == set(module.NAS_SCENARIOS) + assert set(catalogue["inference"]) == set(module.INFERENCE_SCENARIOS) + + state = _post(base, "/__scenario", {"inference": "rate_limited", "retry_after": 42}).json() + assert state["inference"] == "rate_limited" and state["retry_after"] == 42 + assert httpx.get(f"{base}/__scenario?nas=paused", timeout=5.0).json()["nas"] == "paused" + assert _post(base, "/__scenario", {"nas": "nope"}).status_code == 400 + + assert _post(base, "/__reset").json() == {**_post(base, "/__reset").json(), "nas": "ok", "inference": "ok"} + + def test_the_control_surface_is_cors_open_for_a_renderer(self, server): + _module, base = server + preflight = httpx.options(f"{base}/__scenario", timeout=5.0) + assert preflight.status_code == 204 + assert preflight.headers["access-control-allow-origin"] == "*" + assert "POST" in preflight.headers["access-control-allow-methods"] + + def test_once_drops_back_to_the_happy_path_after_one_failure(self, server): + _module, base = server + _post(base, "/__scenario", {"inference": "upstream_503", "once": True}) + assert _post(base, "/v1/chat/completions", {"model": "nous/welcome", "messages": []}).status_code == 503 + assert _post(base, "/v1/chat/completions", {"model": "nous/welcome", "messages": []}).status_code == 200 + + +class TestNasContract: + def test_the_happy_path_mints_and_exchanges(self, server): + _module, base = server + minted = _post(base, "/api/anonymous/create").json() + assert minted["token"].startswith("anon_") and minted["idle_ttl_days"] == 14 + exchanged = _post(base, "/api/anonymous/token", {"token": minted["token"]}).json() + assert exchanged["access_token"].count(".") == 2 and exchanged["expires_in"] == 900 + assert exchanged["inference_base_url"] == f"{base}/v1" + + @pytest.mark.parametrize("scenario,status,error", [ + ("not_enabled", 404, "not_found"), + ("paused", 503, "temporarily_disabled"), + ("rate_limited", 429, "temporarily_unavailable"), + ]) + def test_gate_and_limit_refusals_match_the_service(self, server, scenario, status, error): + _module, base = server + _post(base, "/__scenario", {"nas": scenario}) + response = _post(base, "/api/anonymous/create") + assert response.status_code == status and response.json()["error"] == error + if status == 429: + assert int(response.headers["Retry-After"]) > 0 + + def test_exchange_refusals_match_the_service(self, server): + _module, base = server + token = _post(base, "/api/anonymous/create").json()["token"] + for scenario, status, error in (("pow", 428, "pow_required"), ("locked", 403, "account_locked")): + _post(base, "/__scenario", {"nas": scenario}) + response = _post(base, "/api/anonymous/token", {"token": token}) + assert response.status_code == status and response.json()["error"] == error + _post(base, "/__scenario", {"nas": "dead_once"}) + assert _post(base, "/api/anonymous/token", {"token": token}).json()["error"] == "unknown_token" + fresh = _post(base, "/api/anonymous/create").json()["token"] + assert _post(base, "/api/anonymous/token", {"token": fresh}).status_code == 200 + + def test_hermes_own_client_reaches_the_same_codes_against_it(self, server, monkeypatch, tmp_path): + """The point of the stand-in: ``ensure_portal_identity`` classifies its answers exactly as + it classifies production's.""" + from hermes_cli import anon_auth, free_tier_bootstrap + _module, base = server + monkeypatch.setenv("HERMES_PORTAL_BASE_URL", base) + monkeypatch.setenv("HERMES_SHARED_AUTH_DIR", str(tmp_path / "shared-store")) + monkeypatch.setenv("HERMES_GUEST_ONBOARDING", "1") + anon_auth.reset_mint_memo_for_tests() + free_tier_bootstrap.reset_for_tests() + + _post(base, "/__scenario", {"nas": "paused"}) + with pytest.raises(anon_auth.AuthError) as exc: + anon_auth.ensure_portal_identity(explicit=True) + assert exc.value.code == anon_auth.ANON_GATE_PAUSED + + _post(base, "/__scenario", {"nas": "ok"}) + state = anon_auth.ensure_portal_identity(explicit=True, force=True) + assert anon_auth.is_guest_state(state) + + +class TestInferenceContract: + def test_the_happy_path_answers_streaming_and_plain(self, server): + _module, base = server + plain = _post(base, "/v1/chat/completions", {"model": "nous/welcome", "messages": []}).json() + assert plain["choices"][0]["message"]["content"] + streamed = _post(base, "/v1/chat/completions", {"model": "nous/welcome", "messages": [], "stream": True}) + assert streamed.headers["content-type"].startswith("text/event-stream") + assert streamed.text.rstrip().endswith("data: [DONE]") + + @pytest.mark.parametrize("scenario,reason,retry_after", [ + ("rate_limited", "rate_limited", 600), ("rate_limited_short", "rate_limited", 5), + ("at_capacity", "at_capacity", 30), ("model_not_free", "model_not_free", 0), + ]) + def test_structured_refusals_parse_as_welcome_refusals(self, server, scenario, reason, retry_after): + from hermes_cli.anon_auth import parse_welcome_refusal + _module, base = server + _post(base, "/__scenario", {"inference": scenario}) + response = _post(base, "/v1/chat/completions", {"model": "gpt-5", "messages": []}) + assert response.status_code == 429 + refusal = parse_welcome_refusal(response.json()) + assert refusal["reason"] == reason and refusal["retry_after"] == retry_after + if reason == "model_not_free": + assert refusal["alternates"] == ["nous/welcome"] + + def test_route_refusals_and_outages_match_the_gateway(self, server): + from hermes_cli.anon_auth import welcome_route_refusal + _module, base = server + _post(base, "/__scenario", {"inference": "wrong_host"}) + response = _post(base, "/v1/chat/completions", {"model": "nous/welcome", "messages": []}) + assert welcome_route_refusal(response.status_code, response.json()["message"]) == "anon_on_paid_host" + _post(base, "/__scenario", {"inference": "tier_disabled"}) + response = _post(base, "/v1/chat/completions", {"model": "nous/welcome", "messages": []}) + assert response.status_code == 403 and "reason" not in response.json() + assert welcome_route_refusal(403, response.json()["message"], f"{base}/v1") is None # not a welcome host... + _post(base, "/__scenario", {"inference": "bare_429"}) + response = _post(base, "/v1/chat/completions", {"model": "nous/welcome", "messages": []}) + assert response.headers["x-ratelimit-remaining-requests"] == "0" + for scenario, status in (("upstream_503", 503), ("upstream_500", 500), ("invalid_token", 401)): + _post(base, "/__scenario", {"inference": scenario}) + assert _post(base, "/v1/chat/completions", {"model": "nous/welcome", "messages": []}).status_code == status + + def test_the_dev_host_override_makes_the_stand_in_the_welcome_host(self, server, monkeypatch): + from hermes_cli.anon_auth import route_is_welcome_host, welcome_route_refusal + _module, base = server + assert route_is_welcome_host(f"{base}/v1") is False + monkeypatch.setenv("HERMES_EXTRA_WELCOME_HOSTS", "127.0.0.1, localhost") + assert route_is_welcome_host(f"{base}/v1") is True + assert route_is_welcome_host("http://localhost:9/v1") is True + # ...and with it, the generic 403 reads as the tier refusing, exactly as in production. + assert welcome_route_refusal(403, "You tried to access something", f"{base}/v1") == "tier_disabled" + assert route_is_welcome_host("https://welcome-api.nousresearch.com/v1") is True + assert route_is_welcome_host("https://inference-api.nousresearch.com/v1") is False + + +def test_the_script_runs_as_a_program(): + import subprocess + result = subprocess.run([sys.executable, str(SCRIPT), "--help"], capture_output=True, text=True, timeout=30) + assert result.returncode == 0 + assert "--inference" in result.stdout and "rate_limited" in result.stdout + assert json.dumps(sorted(_load().INFERENCE_SCENARIOS)) # the catalogue is JSON-serialisable diff --git a/tests/tui_gateway/test_free_tier_rpc.py b/tests/tui_gateway/test_free_tier_rpc.py index bbe94cfbb5..c1c6606de9 100644 --- a/tests/tui_gateway/test_free_tier_rpc.py +++ b/tests/tui_gateway/test_free_tier_rpc.py @@ -105,7 +105,7 @@ def test_provision_sets_the_free_tier_up_through_the_lifecycle_primitive(tmp_pat monkeypatch.setattr(anon_auth, "ensure_portal_identity", fake_provision) assert _call("free_tier.provision") == {"has_guest": True, "enabled": True} - assert calls == [{"explicit": True}] + assert calls == [{"explicit": True, "force": True}] assert _call("free_tier.provision") == {"has_guest": True, "enabled": True} assert len(calls) == 1 # idempotent: an identity exists, nothing is minted diff --git a/tui_gateway/methods_config.py b/tui_gateway/methods_config.py index 424fb48210..fe00bca2be 100644 --- a/tui_gateway/methods_config.py +++ b/tui_gateway/methods_config.py @@ -282,9 +282,11 @@ def _(rid, params: dict) -> dict: if record is None: return {"provider_configured": bool(_has_any_provider_configured(strict_profile_scope=bool(profile))), **scoped} + # ``failure_fields`` rides along only when the free-tier mint did not happen: the code, + # the sentence, and whether / when a retry can succeed (``free_tier.provision``). return {"provider_configured": record.provider_configured, "ready": True, "free_tier": record.free_tier, "other_providers": record.other_providers, - "inference_provider": record.inference_provider, **scoped} + "inference_provider": record.inference_provider, **record.failure_fields(), **scoped} return _readiness_check(rid, params, probe) except Exception as e: return _err(rid, 5016, str(e)) diff --git a/tui_gateway/methods_free_tier.py b/tui_gateway/methods_free_tier.py index c71590ce5d..8ceacf63d1 100644 --- a/tui_gateway/methods_free_tier.py +++ b/tui_gateway/methods_free_tier.py @@ -31,10 +31,15 @@ def _(rid, params: dict) -> dict: from hermes_cli import anon_auth has_guest = anon_auth.has_guest() enabled = anon_auth.guest_enabled() - return _ok(rid, { + payload = { "has_guest": has_guest, "enabled": enabled, "available": has_guest and enabled, "notice_pending": bool(has_guest and enabled and anon_auth.guest_notice_pending()), - "model": anon_auth.GUEST_MODEL, "label": anon_auth.FREE_TIER_LABEL}) + "model": anon_auth.GUEST_MODEL, "label": anon_auth.FREE_TIER_LABEL} + if enabled and not has_guest: + # Why there is no identity, when the last attempt to make one failed: + # ``{error, error_code, retryable, retry_after}`` (the mint memo's verdict). + payload.update(anon_auth.last_mint_failure() or {}) + return _ok(rid, payload) except Exception as e: return _err(rid, 5090, str(e)) @@ -45,21 +50,32 @@ def _(rid, params: dict) -> dict: """Explicit retry of the free-tier set-up for the focused profile: adopt the shared store's identity, else mint one (blocking, short timeout). The boot bootstrap normally did this already; the desktop calls this when the record says the identity is missing (portal down at boot, gate - turned on later) and the user asks again. ``{has_guest, enabled}``; ``error`` when the portal - refused.""" + turned on later) and the user asks again. The user's click is the one attempt that may run + inside the mint memo's cooldown. ``{has_guest, enabled}``, plus + ``{error, error_code, retryable, retry_after}`` when the portal refused.""" try: from hermes_cli import anon_auth + from hermes_cli import free_tier_bootstrap enabled = anon_auth.guest_enabled() - error = None + failure = None if enabled and not anon_auth.has_guest(): - try: - anon_auth.ensure_portal_identity(explicit=True) - except Exception as exc: - logger.info("free tier provisioning failed: %s", exc) - error = str(exc) - payload = {"has_guest": anon_auth.has_guest(), "enabled": enabled} - if error: - payload["error"] = error + if free_tier_bootstrap.current_record() is not None and not params.get("profile"): + # The launch profile: refresh the boot record too, so ``setup.status`` and the + # ``setup.ready`` listeners move with the outcome. + failure = free_tier_bootstrap.retry_bootstrap_mint(force=True).failure_fields() + else: + try: + anon_auth.ensure_portal_identity(explicit=True, force=True) + except Exception as exc: + logger.info("free tier provisioning failed: %s", exc) + err = anon_auth._classify_mint_exception(exc) + failure = {"error": str(err), "error_code": str(err.code or ""), + "retryable": err.code not in anon_auth.ANON_TERMINAL_CODES, + "retry_after": int(err.retry_after or 0)} + has_guest = anon_auth.has_guest() + payload = {"has_guest": has_guest, "enabled": enabled} + if enabled and not has_guest: + payload.update(anon_auth.last_mint_failure() or failure or {}) return _ok(rid, payload) except Exception as e: return _err(rid, 5092, str(e)) diff --git a/website/docs/developer-guide/free-tier-fault-rehearsal.md b/website/docs/developer-guide/free-tier-fault-rehearsal.md new file mode 100644 index 0000000000..3b9859770d --- /dev/null +++ b/website/docs/developer-guide/free-tier-fault-rehearsal.md @@ -0,0 +1,152 @@ +--- +sidebar_position: 6 +title: "Rehearsing Free-Tier Failures" +description: "Drive every way the Nous free tier can refuse or fail, from the desktop's JavaScript console, against a local stand-in for the account service and the welcome host" +--- + +# Rehearsing Free-Tier Failures + +The Nous free tier depends on two services: the account service (NAS), which creates the +free-tier identity and exchanges it for short-lived tokens, and the welcome inference host, which +serves the free model under its own rate limits and capacity caps. Hermes has a ruled behaviour for +every way either one can refuse or fail (see `hermes_cli/anon_auth.py`, the `ANON_*` codes and the +welcome refusal parser). This page is how you watch those behaviours happen on a real desktop +without touching production. + +`scripts/free_tier_fault_server.py` is a single stdlib-only process that plays **both** services +and answers with the exact status codes, bodies and headers the real ones send. Which failure it +serves is a live switch you flip from the desktop's JavaScript console, `curl`, or a browser tab. + +## 1. Start the stand-in + +From the repo root: + +```bash +python scripts/free_tier_fault_server.py +``` + +It listens on `127.0.0.1:8765` and prints the environment Hermes needs. You can also start it +already failing: + +```bash +python scripts/free_tier_fault_server.py --nas paused --inference rate_limited +``` + +## 2. Start Hermes Desktop against it + +Use a fresh temporary home so the rehearsal starts from a first launch, and add the four +variables the server printed to the usual guided-onboarding command from `apps/desktop`: + +```bash +D=$(mktemp -d) +env -u NODE_ENV \ + HERMES_GUEST_ONBOARDING=1 \ + HERMES_HOME=$D/.hermes HERMES_DESKTOP_USER_DATA_DIR=$D/electron-user-data \ + HERMES_SHARED_AUTH_DIR=$D/.hermes/shared \ + HERMES_PORTAL_BASE_URL=http://127.0.0.1:8765 \ + NOUS_INFERENCE_BASE_URL=http://127.0.0.1:8765/v1 \ + HERMES_EXTRA_WELCOME_HOSTS=127.0.0.1 \ + npm run dev +``` + +What each one does: + +| Variable | Why | +|---|---| +| `HERMES_GUEST_ONBOARDING=1` | The free tier's launch gate; nothing is created without it. | +| `HERMES_PORTAL_BASE_URL` | Sends every account-service call (sign-up, token exchange, sign-in) to the stand-in. | +| `NOUS_INFERENCE_BASE_URL` | Sends chat completions to the stand-in instead of the welcome host. This override is trusted as the user's own setting and bypasses the network-side host allowlist. | +| `HERMES_EXTRA_WELCOME_HOSTS` | Dev-only. Makes Hermes treat `127.0.0.1` as the welcome host, so the free-tier route rules (model pinning, the dark-tier 403, the "sign in for a bigger allowance" copy) apply exactly as they do in production. Comma-separated hostnames; ports are ignored. It never widens the allowlist for URLs the network hands back. | + +On a clean start the desktop boots, mints an identity against the stand-in, and lands on the +free-tier ready screen. Send a message and the stand-in replies with a canned line. + +## 3. Flip a scenario + +From the desktop's JavaScript console (View → Toggle Developer Tools): + +```js +await (await fetch('http://127.0.0.1:8765/__scenario', { + method: 'POST', + body: JSON.stringify({ inference: 'rate_limited' }) +})).json() +``` + +Or from a shell: + +```bash +curl -s -X POST http://127.0.0.1:8765/__scenario -d '{"nas": "paused"}' +``` + +Then do the thing the scenario is about (send a message, restart the app, click *Try again*, +start a sign-in) and watch what the desktop says. The full catalogue, with what each scenario +means, is at `GET /__scenarios`; the current switches at `GET /__scenario`; the requests the +server has answered, with the scenario that was live for each, at `GET /__log`. + +Control fields: + +| Field | Meaning | +|---|---| +| `nas` | The account-service scenario (sign-up, token exchange, sign-in). | +| `inference` | The welcome-host scenario (chat completions). | +| `once` | `true` makes the next failure a one-off: after serving it, that service drops back to `ok`. Use it for "rate limited once, then fine". | +| `retry_after` | Overrides the wait the server names, in seconds, for any scenario that names one. | + +`POST /__reset` puts everything back to the happy path and clears the log. + +## 4. What to rehearse, and what you should see + +**At boot (the account service).** Set the `nas` scenario, then quit and relaunch the desktop +with a fresh `HERMES_HOME`, or click *Try again* on the notice. + +| Scenario | The desktop should | +|---|---| +| `not_enabled` | Show the provider picker with the "can't start without a Nous account" notice, no retry button, and the Nous row as the way in. | +| `paused` | Show the "paused for a moment" notice with *Try again*; the backend retries on its own with a one-minute floor, and the picker gives way by itself when the stand-in is set back to `ok`. | +| `rate_limited` | Show the "lots of people are getting started" notice naming the wait; the backend retries after `Retry-After`. | +| `server_error`, `timeout` | Show the "couldn't reach / had a hiccup" notice with *Try again* and **no** sign-in pointer, since the same service would refuse the sign-in. | +| `pow` | Show the proof-of-work sentence and point at sign-in; no retry. | +| `locked` | Show the "can't continue without signing in" notice. | +| `dead_once` | Nothing visible: the next token exchange fails as "unknown token", Hermes retires the identity and mints a replacement silently. Check `GET /__log` to see the two exchanges. | + +**Mid-chat (the welcome host).** Set the `inference` scenario, then send a message. + +| Scenario | The desktop should | +|---|---| +| `rate_limited` | Stop and say the allowance is used up, name the reset ("about 10 minutes"), and offer sign-in. The next send is refused locally until the reset. | +| `rate_limited_short` | Wait the few seconds quietly, then the reply arrives (set `once: true` first). | +| `at_capacity` | Retry with the named delay, then, once the retries are spent, say the free service is busy and offer both doors. | +| `model_not_free` | Only reachable when the session asks for another model; Hermes moves back to the free model once and retries. | +| `tier_disabled` | Say that using Hermes without signing in is switched off, stop retrying, and offer both doors. | +| `wrong_host` | Heal silently: Hermes re-reads its route and retries. The "different Nous server" sentence appears only if `NOUS_INFERENCE_BASE_URL` is the thing pointing at the wrong place. | +| `bare_429` | Same as `rate_limited`, driven by the `x-ratelimit-*` headers instead of the body. | +| `upstream_503`, `upstream_500` | Retry with backoff, then say the free model is having trouble responding. | +| `invalid_token` | Silent: one credential refresh, then a re-mint against the account service. | +| `timeout` | The turn's own transport timeout and retries. | + +**Signing in.** With an identity in place, open the sign-in dialog from the status-bar chip, with +`nas` set to `signin_busy`, `signin_paused` or `signin_server_error`. The dialog should show +the "busy, try again in about a minute" or "couldn't reach" screen with *Try again*, and the +free-tier session should still be there afterwards. The stand-in can start a sign-in and report +`pending`, and you can settle it as declined or busy: + +```bash +curl -s -X POST http://127.0.0.1:8765/__signin -d '{"status": "voided", "reason": "user_declined"}' +``` + +A **completed** sign-in is deliberately not something the stand-in can finish: the real portal +issues signed tokens the agent then verifies. Rehearse the happy path of a sign-in against +staging. + +## 5. Notes + +- The stand-in is for rehearsal, not for tests of Hermes' own logic. Those live in + `tests/hermes_cli/test_anon_failure_modes.py` and `tests/agent/test_welcome_tier_recovery.py` + and drive the same client code through in-process fakes. `tests/scripts/test_free_tier_fault_server.py` + checks that the stand-in speaks the real contract, which is what makes a rehearsal against it + meaningful. +- Because `HERMES_EXTRA_WELCOME_HOSTS` changes routing rules, never set it outside a rehearsal + environment. It is read from the environment only, is never written to config, and is not + shown in setup. +- The renderer talks to the stand-in's control surface directly (it is CORS-open). If a future + content-security policy blocks that, the `curl` forms above do the same thing. diff --git a/website/sidebars.ts b/website/sidebars.ts index f837b14ada..65fd401311 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -777,6 +777,7 @@ const sidebars: SidebarsConfig = { items: [ 'developer-guide/contributing', 'developer-guide/worktree-ui-dev', + 'developer-guide/free-tier-fault-rehearsal', { type: 'category', label: 'Architecture',