From e6f5f109f9250e7edfaaab1ef64c076c92e4e6cc Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:12:37 -0700 Subject: [PATCH 01/29] =?UTF-8?q?refactor(hermes=5Fcli):=20cron.py=20?= =?UTF-8?q?=E2=80=94=20fold=20subcommand=20wrappers=20into=20dispatch=20ta?= =?UTF-8?q?ble,=20dedupe=20unverified-target=20rendering,=20compact=20narr?= =?UTF-8?q?ative=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/cron.py | 376 +++++++++++++++++---------------------------- 1 file changed, 145 insertions(+), 231 deletions(-) diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index 104f80b470..f30de0d93c 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -11,12 +11,10 @@ sys.path.insert(0, str(PROJECT_ROOT)) from hermes_cli.colors import Colors, color -# Gateway-lifecycle command detection lives in ``cron.lifecycle_guard`` so it -# can be shared across every job-creation path (CLI + the agent's ``cronjob`` -# model tool via ``cron.jobs.create_job``) without a circular import. Re-export -# ``_contains_gateway_lifecycle_command`` here for back-compat: ``tools/ -# terminal_tool.py`` imports it from this module to hard-block the same -# commands at execution time when ``_HERMES_GATEWAY=1``. +# Gateway-lifecycle command detection lives in ``cron.lifecycle_guard`` (shared by every +# job-creation path without a circular import). Re-exported here because +# ``tools/terminal_tool.py`` imports it from this module to hard-block the same commands +# at execution time when ``_HERMES_GATEWAY=1``. from cron.lifecycle_guard import ( # noqa: F401 (re-exported for terminal_tool) contains_gateway_lifecycle_command as _contains_gateway_lifecycle_command, ) @@ -45,11 +43,10 @@ def _cron_api(**kwargs): def _active_cron_provider_name() -> str: - """Name of the resolved cron scheduler provider ('builtin', 'chronos', …). + """Resolved cron scheduler provider name ('builtin', 'chronos', …); 'builtin' on any failure. - Best-effort + offline (``resolve_cron_scheduler`` reads config and the provider's - ``is_available()`` contract forbids network). Returns 'builtin' on any failure so callers fall - back to the historical ticker-based checks. + Best-effort + offline (``resolve_cron_scheduler`` reads config; ``is_available()`` forbids + network), so callers fall back to the historical ticker-based checks. """ try: from cron.scheduler_provider import resolve_cron_scheduler @@ -60,39 +57,33 @@ def _active_cron_provider_name() -> str: def _builtin_gateway_liveness() -> Optional[bool]: - """Tri-state liveness of the builtin cron scheduler's trigger. + """Tri-state liveness of the builtin cron scheduler's trigger (None = unknown). - Single source of truth shared by the CLI (``_warn_if_gateway_not_running``) and the ``cronjob`` - model tool (#87033): the builtin ticker only runs inside the gateway process, so a scheduled job - with no live gateway can never fire. Non-builtin providers (e.g. + Shared by the CLI and the ``cronjob`` model tool: the builtin ticker only runs inside the + gateway process, so a scheduled job with no live gateway can never fire. Non-builtin + providers fire jobs without the gateway. """ try: if _active_cron_provider_name() != "builtin": return True # external provider fires jobs without the gateway - # The gateway runtime lock is held for exactly the gateway's lifetime, so it - # is a more reliable "is the ticker's process alive" signal than PID scanning - # — and inside the gateway process it short-circuits to True, so the in-gateway - # cron tool never emits a false "gateway not running" (find_gateway_pids can - # transiently miss the gateway just after a restart). + # The gateway runtime lock is held for exactly the gateway's lifetime — a more + # reliable "ticker's process is alive" signal than PID scanning, and inside the + # gateway process it short-circuits to True so the in-gateway cron tool never + # emits a false "gateway not running" (find_gateway_pids can transiently miss the + # gateway just after a restart). try: from gateway.status import is_gateway_runtime_lock_active if is_gateway_runtime_lock_active(): return True except Exception: - # A crashing lock probe is "unknown", not "dead" — let the pid - # scan below still decide instead of collapsing the whole - # tri-state to None. - pass - from hermes_cli.gateway import ( - find_gateway_pids, - named_profile_served_by_running_multiplexer, - ) + pass # a crashing lock probe is "unknown", not "dead" — let the pid scan decide + from hermes_cli.gateway import find_gateway_pids, named_profile_served_by_running_multiplexer if find_gateway_pids(): return True - # Satellite profile: no local gateway.pid, but the default multiplexer - # ticks this profile's cron store (#97120). + # Satellite profile: no local gateway.pid, but the default multiplexer ticks this + # profile's cron store. return named_profile_served_by_running_multiplexer() except Exception: return None @@ -101,12 +92,11 @@ def _builtin_gateway_liveness() -> Optional[bool]: def _warn_if_gateway_not_running() -> None: """Warn that scheduled jobs won't fire unless the gateway is running. - The cron ticker only runs inside the gateway (``_start_cron_ticker`` in gateway/run.py); there - is no standalone cron daemon. Without a running gateway, ``next_run_at`` passes but jobs never - fire and ``last_run_at`` stays null — the most common cron support report (#51038). + The cron ticker only runs inside the gateway (no standalone daemon): without one, + ``next_run_at`` passes but jobs never fire and ``last_run_at`` stays null — the most + common cron support report. """ - # _builtin_gateway_liveness never raises (it maps probe failures to None), - # so no guard is needed here — False is the only warn-worthy state. + # _builtin_gateway_liveness never raises; False is the only warn-worthy state. if _builtin_gateway_liveness() is not False: return @@ -138,10 +128,10 @@ def _format_lateness(seconds: float) -> str: def _dispatch_display(dispatch: dict) -> Optional[str]: - """One-line scheduled-vs-actual dispatch summary for a job (#99879). + """One-line scheduled-vs-actual dispatch summary; None when the stamp is malformed. - None when the stamp is malformed. On-time dispatches render dim; late/catch-up dispatches render - loudly so a run fired long after gateway downtime doesn't look like an ordinary success. + On-time dispatches render dim; late/catch-up dispatches render loudly so a run fired long + after gateway downtime doesn't look like an ordinary success. """ if not isinstance(dispatch, dict): return None @@ -170,6 +160,10 @@ def _print_banner(title: str) -> None: print() +def _unverified_targets(unverified) -> str: + return ", ".join(str(t) for t in unverified) if isinstance(unverified, list) else str(unverified) + + def cron_list(show_all: bool = False): """List all scheduled jobs.""" from cron.jobs import effective_job_state, list_jobs @@ -187,23 +181,18 @@ def cron_list(show_all: bool = False): job_id = job.get("id", "?") name = job.get("name", "(unnamed)") schedule = job.get("schedule_display", job.get("schedule", {}).get("value", "?")) - # Derive from the scheduler-honoured flag — never show [paused] when - # enabled=true (half-paused contradiction must not look frozen). + # Scheduler-honoured flag — never show [paused] when enabled=true. state = effective_job_state(job) next_run = job.get("next_run_at", "?") - # `repeat` may be present-but-null in the job record (e.g. a one-shot - # job persisted with "repeat": null), so coalesce to {} rather than - # relying on the dict-default, which only applies to a missing key. + # `repeat` / `deliver` may be present-but-null in the record (a one-shot persisted + # with "repeat": null), so coalesce rather than rely on the dict-default, which only + # applies to a missing key — a null deliver would crash `", ".join(None)`. repeat_info = job.get("repeat") or {} repeat_times = repeat_info.get("times") repeat_completed = repeat_info.get("completed", 0) repeat_str = f"{repeat_completed}/{repeat_times}" if repeat_times else "∞" - # `deliver` may be present-but-null in the job record (same pitfall as - # `repeat` above), so coalesce to the default rather than relying on the - # dict-default, which only applies to a missing key. A null value would - # otherwise reach `", ".join(None)` and crash the whole listing (#32896). deliver = job.get("deliver") or ["local"] if isinstance(deliver, str): deliver = [deliver] @@ -242,16 +231,14 @@ def cron_list(show_all: bool = False): if workdir: print(f" Workdir: {workdir}") - # Execution history last_status = job.get("last_status") if last_status: last_run = job.get("last_run_at", "?") if last_status == "ok": status_display = color("ok", Colors.GREEN) elif last_status == "delivery_failed": - # The agent succeeded but the result never reached the user — - # not green, and the detail lives in last_delivery_error - # (last_error is None for these runs). + # Agent succeeded but the result never reached the user — not green; the + # detail lives in last_delivery_error (last_error is None for these runs). detail = job.get("last_delivery_error") or "?" status_display = color(f"delivery_failed: {detail}", Colors.YELLOW) else: @@ -276,15 +263,13 @@ def cron_list(show_all: bool = False): if delivery_err: print(f" {color('⚠ Delivery failed:', Colors.YELLOW)} {delivery_err}") - # A live adapter acked the last send but returned no message_id / - # raw_response (Slack/Matrix/Mattermost shape): accepted as delivered, - # but say so here rather than only in a WARNING log line. + # A live adapter acked the last send but returned no message_id / raw_response + # (Slack/Matrix/Mattermost shape): accepted as delivered, but say so here. unverified = job.get("last_delivery_unverified") if unverified: - targets = ", ".join(str(t) for t in unverified) if isinstance(unverified, list) else str(unverified) print( f" {color('⚠ Delivery UNVERIFIED:', Colors.YELLOW)} " - f"adapter acked {targets} without message_id/raw_response" + f"adapter acked {_unverified_targets(unverified)} without message_id/raw_response" ) fire_err = job.get("last_fire_error") @@ -305,9 +290,8 @@ def cron_tick(): try: tick(verbose=True) except CronTickYielded as exc: - # Not expected on this surface (a one-shot CLI process has no boot - # fingerprint, so the yield gate is inert) — but if a future caller - # records one, report cleanly instead of a traceback. + # Not expected here (a one-shot CLI process has no boot fingerprint, so the yield + # gate is inert) — report cleanly instead of a traceback if a future caller records one. print(color(f"✗ {exc}", Colors.YELLOW)) print( " A fresher gateway process owns the runtime lock and will fire " @@ -315,10 +299,8 @@ def cron_tick(): ) return 1 except OSError as exc: - # tick() now propagates real lock-acquisition failures (EMFILE, - # EACCES on open, ...) instead of swallowing them as contention - # (#87644). For the one-shot CLI surface, report cleanly instead of - # dumping a traceback; the gateway ticker loop handles its own retry. + # tick() propagates real lock-acquisition failures (EMFILE, EACCES on open, ...) + # instead of swallowing them as contention; the gateway ticker loop handles its own retry. print(color(f"✗ Cron tick failed: {exc}", Colors.RED)) print(" Check `hermes cron status` and the gateway log for details.") return 1 @@ -343,19 +325,14 @@ def cron_runs(job_id: Optional[str] = None, limit: int = 20): print(f" {record['error']}") -_INCIDENT_STATE_COLORS = { - "detected": Colors.RED, - "alerted": Colors.YELLOW, - "closed": Colors.GREEN, -} +_INCIDENT_STATE_COLORS = {"detected": Colors.RED, "alerted": Colors.YELLOW, "closed": Colors.GREEN} def cron_incidents(args) -> int: - """List or acknowledge durable cron failure incidents. + """List (``[--state ]``) or ``ack `` durable cron failure incidents. - ``hermes cron incidents [--state ]`` lists incidents (the stored error is redacted and - truncated at write time, safe for terminal display); ``hermes cron incidents ack `` closes - one so its failure ping stays silent until the error signature changes. + The stored error is redacted and truncated at write time, safe for terminal display; + acking closes an incident so its failure ping stays silent until the error signature changes. """ from cron.incidents import ack_incident, list_incidents @@ -394,13 +371,11 @@ def cron_incidents(args) -> int: if inc.get("output_file"): print(f" Output: {inc['output_file']}") print() - print( - color( - f" {len(incidents)} incident(s) | ack one with: " - "hermes cron incidents ack ", - Colors.DIM, - ) - ) + print(color( + f" {len(incidents)} incident(s) | ack one with: " + "hermes cron incidents ack ", + Colors.DIM, + )) return 0 @@ -421,9 +396,8 @@ _FD_EXHAUSTION_HINT = ( def _print_ticker_health(pids: list) -> None: """Report builtin-ticker liveness for a gateway process known to be alive. - The gateway PROCESS is alive — but the cron ticker THREAD inside it can die silently, or stay - alive while every tick fails. Check both the liveness heartbeat and the last-successful-tick - marker so we don't report "will fire" when the ticker is dead or failing (#32612, #32895). + The ticker THREAD can die silently or stay alive while every tick fails, so check both + the liveness heartbeat and the last-successful-tick marker before saying "will fire". """ from cron.jobs import ( get_ticker_heartbeat_age, @@ -433,10 +407,9 @@ def _print_ticker_health(pids: list) -> None: ) from cron.scheduler import _is_fd_exhaustion_text as _cron_is_fd_exhaustion_text - # Allow ~3 missed ticker iterations (+ a little slack) before declaring - # trouble. Derived from the shared interval constant so this threshold - # tracks the ticker cadence instead of assuming a hardcoded 60s. - STALE_AFTER = TICKER_INTERVAL_SECONDS * 3 + 20 # = 200s at the 60s default + # ~3 missed ticker iterations (+ slack) before declaring trouble; derived from the shared + # interval so the threshold tracks the ticker cadence (= 200s at the 60s default). + STALE_AFTER = TICKER_INTERVAL_SECONDS * 3 + 20 hb_age = get_ticker_heartbeat_age() ok_age = get_ticker_success_age() pid_line = f" PID: {', '.join(map(str, pids))}" if pids else None @@ -447,9 +420,8 @@ def _print_ticker_health(pids: list) -> None: print(pid_line) if hb_age is None: - # No heartbeat file means the ticker thread has never started: gateway - # not in a cron-enabled profile, started moments ago (heartbeat is - # written after startup), or a config issue blocking the ticker. + # No heartbeat file: ticker never started (non-cron profile, gateway started moments + # ago, or a config issue blocking the ticker). _warn("⚠ Gateway is running but the cron ticker has not reported a heartbeat.") print(" Cron jobs will NOT fire until the ticker writes its first heartbeat.") print(" If the gateway just started, wait ~60s and re-run `hermes cron status`.") @@ -462,18 +434,15 @@ def _print_ticker_health(pids: list) -> None: ) print(" Cron jobs may NOT be firing. Restart: hermes gateway restart") elif ok_age is not None and ok_age > STALE_AFTER: - # Loop is alive (fresh heartbeat) but no tick has SUCCEEDED in a - # long time → ticks are failing every iteration. + # Loop alive (fresh heartbeat) but no tick SUCCEEDED in a long time → failing every iteration. _warn( "⚠ Gateway and cron ticker are running, but no tick has " f"succeeded in {int(ok_age)}s — ticks may be failing." ) last_error = get_ticker_last_error() if last_error: - # Show WHY ticks fail — e.g. a root-rewritten jobs.json - # (PermissionError) that silently locked out the ticker's uid for - # ~14h in the field (#68483), or fd exhaustion (EMFILE) that used - # to stall the scheduler invisibly (#87644). + # Show WHY ticks fail — e.g. a root-rewritten jobs.json (PermissionError) that + # silently locked out the ticker's uid, or fd exhaustion (EMFILE). print(color(f" Last tick error: {last_error}", Colors.RED)) if "Permission denied" in last_error: print(color(_PERMISSION_HINT, Colors.YELLOW)) @@ -497,13 +466,9 @@ def cron_status(): provider = _active_cron_provider_name() if provider != "builtin": - # An external provider (e.g. Chronos) does NOT run the in-process 60s - # ticker — it arms one external one-shot per job and is fired by a - # NAS-mediated webhook, so between fires there is intentionally NO - # ticker thread and NO heartbeat file. Reporting the ticker-heartbeat - # staleness here would always say "stalled / not firing" on a perfectly - # healthy Chronos instance. Report the provider instead and skip the - # ticker-liveness heuristics entirely. + # An external provider (e.g. Chronos) arms one external one-shot per job, fired by a + # NAS-mediated webhook: between fires there is intentionally NO ticker thread and NO + # heartbeat file, so the ticker-liveness heuristics would always say "stalled". print(color( f"✓ Cron provider: {provider} — jobs fire via the managed scheduler, " "not the in-process ticker.", @@ -518,11 +483,9 @@ def cron_status(): pids = find_gateway_pids() gateway_alive_via_lock = False if not pids: - # Same false-alarm class the cronjob tool fixed (#95947): the pid scan - # can transiently miss a live gateway (just after a restart) while the - # runtime lock — held for exactly the gateway's lifetime — proves the - # ticker's process is alive. Only declare "not running" when both the - # scan AND the lock say so. + # The pid scan can transiently miss a live gateway (just after a restart) while + # the runtime lock — held for exactly the gateway's lifetime — proves the ticker's + # process is alive. Only declare "not running" when both agree. try: from gateway.status import get_running_pid, is_gateway_runtime_lock_active @@ -548,50 +511,40 @@ def cron_status(): print() - def _print_active_jobs_summary(jobs) -> None: - """Print the ' active job(s)' + next-run line shared by every status - path (built-in ticker AND external provider).""" - if jobs: - next_runs = [j.get("next_run_at") for j in jobs if j.get("next_run_at")] - print(f" {len(jobs)} active job(s)") - if next_runs: - print(f" Next run: {min(next_runs)}") - # Missed-run visibility (#99879): call out jobs whose LAST dispatch - # was late or a catch-up so post-downtime late fires are visible at - # status level, not just buried per-job in `hermes cron list`. - late = [ - j for j in jobs - if isinstance(j.get("last_dispatch"), dict) - and j["last_dispatch"].get("kind") in ("late", "catch_up") - ] - if late: - print() - print(color( - f" ⚠ {len(late)} job(s) last fired late (missed-fire catch-up):", - Colors.YELLOW, - )) - for j in late: - d = j["last_dispatch"] - print( - f" {j.get('id', '?')} {j.get('name', '(unnamed)')}: " - f"scheduled {d.get('scheduled_at', '?')}, " - f"ran {d.get('dispatched_at', '?')} " - + color( - f"({_format_lateness(d.get('lateness_seconds', 0))} late)", - Colors.YELLOW, - ) - ) - else: + """Print the ' active job(s)' + next-run line shared by every status path.""" + if not jobs: print(" No active jobs") + return + next_runs = [j.get("next_run_at") for j in jobs if j.get("next_run_at")] + print(f" {len(jobs)} active job(s)") + if next_runs: + print(f" Next run: {min(next_runs)}") + # Missed-run visibility: call out jobs whose LAST dispatch was late or a catch-up so + # post-downtime late fires show at status level, not just per-job in `cron list`. + late = [ + j for j in jobs + if isinstance(j.get("last_dispatch"), dict) + and j["last_dispatch"].get("kind") in ("late", "catch_up") + ] + if late: + print() + print(color(f" ⚠ {len(late)} job(s) last fired late (missed-fire catch-up):", Colors.YELLOW)) + for j in late: + d = j["last_dispatch"] + print( + f" {j.get('id', '?')} {j.get('name', '(unnamed)')}: " + f"scheduled {d.get('scheduled_at', '?')}, " + f"ran {d.get('dispatched_at', '?')} " + + color(f"({_format_lateness(d.get('lateness_seconds', 0))} late)", Colors.YELLOW) + ) def _scripts_dir_for_cron() -> Path: - """Return the scripts directory used by cron jobs. + """Scripts directory used by cron jobs. - Prefer ``cron.jobs.CRON_DIR.parent`` over a fresh ``get_hermes_home()`` call so tests and - profile-aware callers that monkeypatch cron storage inspect the same Hermes home the jobs were - loaded from. + ``cron.jobs.CRON_DIR.parent`` rather than a fresh ``get_hermes_home()`` so tests and + profile-aware callers that monkeypatch cron storage inspect the same Hermes home. """ from cron.jobs import CRON_DIR @@ -599,7 +552,7 @@ def _scripts_dir_for_cron() -> Path: def _script_health_issue(script: str) -> Optional[str]: - """Return a human-readable script issue, or ``None`` when the path is OK.""" + """Human-readable script issue, or ``None`` when the path is OK.""" scripts_dir = _scripts_dir_for_cron().resolve() raw = Path(script).expanduser() path = raw.resolve() if raw.is_absolute() else (scripts_dir / raw).resolve() @@ -616,15 +569,14 @@ def _script_health_issue(script: str) -> Optional[str]: return None -# Grace period before an overdue ``next_run_at`` is reported. The ticker runs -# once a minute and a busy tick can push dispatch a few minutes late; only a -# next_run_at parked well in the past means the job is silently not firing -# (ticker dead, gateway down, or a wedged fire-claim). +# Grace before an overdue ``next_run_at`` is reported: the ticker runs once a minute and a +# busy tick can push dispatch a few minutes late; only a next_run_at parked well in the past +# means the job is silently not firing (ticker dead, gateway down, wedged fire-claim). _OVERDUE_GRACE_SECONDS = 15 * 60 def _next_run_overdue_issue(next_run: str) -> Optional[str]: - """Return an issue string when ``next_run_at`` is parked in the past.""" + """Issue string when ``next_run_at`` is parked in the past.""" from datetime import datetime, timezone try: @@ -646,9 +598,8 @@ def _cron_doctor_issues_for_job(job: Dict[str, Any]) -> List[str]: issues: List[str] = [] last_status = str(job.get("last_status") or "").strip().lower() - # "delivery_failed" means the agent run itself succeeded, so it is not a - # failed last run — the dedicated delivery issue below reports it (and - # last_error is None, which would render as "unknown error" here). + # "delivery_failed" means the agent run itself succeeded: the dedicated delivery issue + # below reports it (and last_error is None, which would render as "unknown error" here). if last_status and last_status not in {"ok", "delivery_failed"}: err = str(job.get("last_error") or "unknown error").strip() issues.append(f"last run failed: {err}") @@ -659,8 +610,9 @@ def _cron_doctor_issues_for_job(job: Dict[str, Any]) -> List[str]: unverified = job.get("last_delivery_unverified") if unverified: - targets = ", ".join(str(t) for t in unverified) if isinstance(unverified, list) else str(unverified) - issues.append(f"last delivery unverified (adapter acked without evidence): {targets}") + issues.append( + f"last delivery unverified (adapter acked without evidence): {_unverified_targets(unverified)}" + ) if job.get("enabled", True) and job.get("state") not in {"paused", "completed"}: next_run = str(job.get("next_run_at") or "").strip() @@ -720,17 +672,10 @@ def cron_doctor() -> int: _JOB_ARG_FIELDS = ( - ("name", "name"), - ("deliver", "deliver"), - ("failure_deliver", "failure_deliver"), - ("repeat", "repeat"), - ("script", "script"), - ("workdir", "workdir"), - ("model", "model"), - ("provider", "model_provider"), - ("monitor_script", "monitor_script"), - ("monitor_url", "monitor_url"), - ("continuity", "continuity"), + ("name", "name"), ("deliver", "deliver"), ("failure_deliver", "failure_deliver"), + ("repeat", "repeat"), ("script", "script"), ("workdir", "workdir"), ("model", "model"), + ("provider", "model_provider"), ("monitor_script", "monitor_script"), + ("monitor_url", "monitor_url"), ("continuity", "continuity"), ("reasoning_effort", "reasoning_effort"), ) @@ -757,12 +702,9 @@ def _print_job_details(job_data: Dict[str, Any]) -> None: def cron_create(args): - # The gateway-lifecycle guard lives in cron.jobs.create_job so it fires on - # every job-creation path (this CLI subcommand AND the agent's `cronjob` - # model tool, which calls create_job directly). When it blocks, create_job - # raises GatewayLifecycleBlocked, the `cronjob` tool wrapper catches it and - # returns it as result["error"], and the `if not result.get("success")` - # branch below prints it in red and exits 1 — same UX as before. + # The gateway-lifecycle guard lives in cron.jobs.create_job so it fires on every + # job-creation path (CLI AND the agent's `cronjob` tool); a block surfaces as + # result["error"] and is printed in red below. result = _cron_api( action="create", schedule=args.schedule, @@ -844,16 +786,13 @@ def cron_edit(args): def _job_action(action: str, job_id: str, success_verb: str) -> int: _stateless_token = None if action == "run": - # One-shot CLI: this process exits as soon as the command returns, so - # a background-dispatched run (daemon thread of THIS process) would be - # orphaned mid-LLM-call — the delegation dies 'unknown' and the job's - # execution row is stuck 'claimed', blocking future runs (#86721). - # The background path in ``_try_dispatch_background_run`` triggers when - # the CLI inherits a gateway/desktop session env (HERMES_SESSION_KEY); - # declare the channel stateless so ``async_delivery_supported()`` gates - # it off and the run executes synchronously to completion instead. - # The declaration is scoped to this call (token reset in ``finally``) - # so in-process callers (tests, embedding apps) are not tainted. + # One-shot CLI: the process exits as soon as the command returns, so a + # background-dispatched run (daemon thread of THIS process — triggered when the CLI + # inherits a gateway/desktop session env, HERMES_SESSION_KEY) would be orphaned + # mid-LLM-call, leaving the execution row stuck 'claimed'. Declare the channel + # stateless so ``async_delivery_supported()`` gates it off and the run executes + # synchronously. Scoped to this call (token reset in ``finally``) so in-process + # callers (tests, embedding apps) are not tainted. try: from gateway.session_context import _SESSION_ASYNC_DELIVERY @@ -874,14 +813,9 @@ def _job_action(action: str, job_id: str, success_verb: str) -> int: print(f" Next run: {result['job']['next_run_at']}") if action == "run": job = result.get("job", {}) - # A manual run can be dispatched to the gateway daemon's background - # delegation worker instead of executing inline (e.g. when the CLI - # process inherits a gateway/desktop session env and the run - # resolves a session key). Such responses carry - # execution_mode="background" and/or a delegation_id, and the job - # keeps running AFTER this CLI process exits — a terminal - # success/failure verdict would be a lie (#83340). Report the - # background dispatch instead of claiming the run failed. + # A manual run may be dispatched to the gateway daemon's background delegation worker + # (execution_mode="background" and/or a delegation_id) and keeps running AFTER this + # CLI exits — a terminal success/failure verdict would be a lie, so report the dispatch. delegation_id = job.get("delegation_id") if job.get("execution_mode") == "background" or delegation_id: if delegation_id: @@ -927,9 +861,9 @@ def cron_resume(args) -> int: def cron_notepad(args) -> int: """Handle ``hermes cron notepad [get|set|delete|list]``. - This CLI is the write path for the per-job durable KV scratchpad (``cron/notepad.py``): a - running cron agent updates its own notepad via its terminal tool, and the scheduler injects - non-empty notepads into the job prompt on each run. + Write path for the per-job durable KV scratchpad (``cron/notepad.py``): a running cron + agent updates its own notepad via its terminal tool, and the scheduler injects non-empty + notepads into the job prompt on each run. """ from cron import notepad @@ -977,46 +911,26 @@ def cron_notepad(args) -> int: return 1 -def _cron_list_cmd(args) -> int: - cron_list(getattr(args, "all", False)) - return 0 - - -def _cron_status_cmd(args) -> int: - cron_status() - return 0 - - -def _cron_runs_cmd(args) -> int: - cron_runs(getattr(args, "job_id", None), getattr(args, "limit", 20)) - return 0 - - -def _cron_remove_cmd(args) -> int: - return _job_action("remove", args.job_id, "Removed") - - -# Subcommand -> handler. Late-bound lambdas so module-level monkeypatching of -# the underlying functions keeps working. +# Subcommand -> handler. Late-bound lambdas so module-level monkeypatching of the underlying +# functions keeps working. cron_list/cron_status/cron_runs always return None -> exit 0. _CRON_SUBCOMMANDS = { - "list": _cron_list_cmd, - "status": _cron_status_cmd, - "doctor": lambda args: cron_doctor(), - "tick": lambda args: cron_tick(), - "runs": _cron_runs_cmd, - "history": _cron_runs_cmd, - "incidents": lambda args: cron_incidents(args), - "notepad": lambda args: cron_notepad(args), - "create": lambda args: cron_create(args), - "add": lambda args: cron_create(args), - "edit": lambda args: cron_edit(args), - "pause": lambda args: _job_action("pause", args.job_id, "Paused"), - "resume": lambda args: cron_resume(args), - "run": lambda args: _job_action("run", args.job_id, "Triggered"), - "remove": _cron_remove_cmd, - "rm": _cron_remove_cmd, - "delete": _cron_remove_cmd, + "list": lambda a: cron_list(getattr(a, "all", False)) or 0, + "status": lambda a: cron_status() or 0, + "doctor": lambda a: cron_doctor(), + "tick": lambda a: cron_tick(), + "runs": lambda a: cron_runs(getattr(a, "job_id", None), getattr(a, "limit", 20)) or 0, + "incidents": lambda a: cron_incidents(a), + "notepad": lambda a: cron_notepad(a), + "create": lambda a: cron_create(a), + "edit": lambda a: cron_edit(a), + "pause": lambda a: _job_action("pause", a.job_id, "Paused"), + "resume": lambda a: cron_resume(a), + "run": lambda a: _job_action("run", a.job_id, "Triggered"), + "remove": lambda a: _job_action("remove", a.job_id, "Removed"), } +_CRON_SUBCOMMANDS["history"] = _CRON_SUBCOMMANDS["runs"] +_CRON_SUBCOMMANDS["add"] = _CRON_SUBCOMMANDS["create"] +_CRON_SUBCOMMANDS["rm"] = _CRON_SUBCOMMANDS["delete"] = _CRON_SUBCOMMANDS["remove"] def cron_command(args): From 1a94479d176d90585f0ab067ba1d4b065be46fa3 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:16:16 -0700 Subject: [PATCH 02/29] =?UTF-8?q?refactor(hermes=5Fcli):=20debug.py=20?= =?UTF-8?q?=E2=80=94=20unify=20paste=20upload=20fallback=20loop,=20tail-bu?= =?UTF-8?q?dget=20helper,=20shared=20User-Agent,=20compact=20docstrings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/debug.py | 342 +++++++++++++------------------------------- 1 file changed, 98 insertions(+), 244 deletions(-) diff --git a/hermes_cli/debug.py b/hermes_cli/debug.py index 834fb41f12..7fe588bb35 100644 --- a/hermes_cli/debug.py +++ b/hermes_cli/debug.py @@ -18,9 +18,8 @@ from utils import atomic_replace logger = logging.getLogger(__name__) -# Banner prepended to upload-bound log content when redaction is enabled. -# Visible in the public paste so reviewers know the content was sanitized. -# Kept short; the trailing newline guarantees the banner sits on its own line. +# Prepended to upload-bound content when redaction is enabled so reviewers of the public +# paste know it was sanitized; trailing newline keeps it on its own line. _REDACTION_BANNER = ( "[hermes debug share: log content redacted at upload time. " "run with --no-redact to disable]\n" @@ -32,33 +31,25 @@ _EMAIL_ADDRESS_RE = re.compile( r"(?![A-Za-z0-9._%+-])" ) - -# --------------------------------------------------------------------------- -# Paste services — try paste.rs first, dpaste.com as fallback. -# --------------------------------------------------------------------------- - +# Paste services — paste.rs first, dpaste.com as fallback. _PASTE_RS_URL = "https://paste.rs/" _DPASTE_COM_URL = "https://dpaste.com/api/" +_USER_AGENT = "hermes-agent/debug-share" -# Maximum bytes to read from a single log file for upload. -# paste.rs caps at ~1 MB; we stay under that with headroom. +# Max bytes read from one log file for upload (paste.rs caps at ~1 MB; keep headroom). _MAX_LOG_BYTES = 512_000 -# Auto-delete pastes after this many seconds (6 hours). +# Auto-delete pastes after 6 hours. _AUTO_DELETE_SECONDS = 21600 -# --------------------------------------------------------------------------- -# Pending-deletion tracking (replaces the old fork-and-sleep subprocess). -# --------------------------------------------------------------------------- +# ── Pending-deletion tracking ────────────────────────────────────────────── +# Deletion is driven by the gateway's cron ticker (``gateway/run.py::_start_cron_ticker`` +# calls ``_sweep_expired_pastes`` hourly); ``hermes debug`` also sweeps opportunistically on +# entry for CLI-only users who never start the gateway. Replaces the old fork-and-sleep +# subprocess that leaked ~20 MB of resident interpreter per share. def _pending_file() -> Path: - """Path to ``~/.hermes/pastes/pending.json``. - - Deletion is now driven by the gateway's cron ticker (``gateway/run.py::_start_cron_ticker``) - which calls ``_sweep_expired_pastes`` once per hour. ``hermes debug share`` also runs an - opportunistic sweep on entry as a fallback for CLI-only users who never start the gateway. - """ return get_hermes_home() / "pastes" / "pending.json" @@ -72,7 +63,6 @@ def _load_pending() -> list[dict]: return [] if not isinstance(data, list): return [] - # Filter to well-formed entries only return [e for e in data if isinstance(e, dict) and "url" in e and "expire_at" in e] @@ -84,16 +74,14 @@ def _save_pending(entries: list[dict]) -> None: tmp.write_text(json.dumps(entries, indent=2), encoding="utf-8") atomic_replace(tmp, path) except OSError: - # Non-fatal — worst case the user has to run ``hermes debug delete`` - # manually. - pass + pass # non-fatal — worst case the user runs ``hermes debug delete`` manually def _sweep_expired_pastes(now: Optional[float] = None) -> tuple[int, int]: - """Synchronously DELETE any pending pastes whose ``expire_at`` has passed. + """Synchronously DELETE pending pastes whose ``expire_at`` has passed → (deleted, remaining). - Returns ``(deleted, remaining)``. Best-effort: failed deletes stay in the pending file for - the next sweep. Silent, since it runs on every ``hermes debug`` invocation. + Best-effort and silent: failed deletes stay pending for the next sweep, up to 24h past + expiration (paste.rs GCs them eventually). """ entries = _load_pending() if not entries: @@ -112,21 +100,17 @@ def _sweep_expired_pastes(now: Optional[float] = None) -> tuple[int, int]: remaining.append(entry) continue - url = entry.get("url", "") try: - if delete_paste(url): + if delete_paste(entry.get("url", "")): deleted += 1 continue except Exception: - # Network hiccup, 404 (already gone), etc. — drop the entry - # after a grace period; don't retry forever. - pass + pass # network hiccup, 404 (already gone), ... - # Retain failed deletes for up to 24h past expiration, then give up. if expire_at + 86400 > current: remaining.append(entry) else: - deleted += 1 # count as reaped (paste.rs will GC eventually) + deleted += 1 # count as reaped if deleted: _save_pending(remaining) @@ -135,16 +119,14 @@ def _sweep_expired_pastes(now: Optional[float] = None) -> tuple[int, int]: def _best_effort_sweep_expired_pastes() -> None: - """Attempt pending-paste cleanup without letting /debug fail offline.""" + """Pending-paste cleanup that never lets /debug fail offline.""" try: _sweep_expired_pastes() except Exception: pass -# --------------------------------------------------------------------------- -# Privacy / delete helpers -# --------------------------------------------------------------------------- +# ── Privacy / delete helpers ─────────────────────────────────────────────── _PRIVACY_NOTICE = """\ ⚠️ This will upload system info + logs to a PUBLIC paste service. @@ -172,7 +154,7 @@ _GATEWAY_PRIVACY_NOTICE = ( def _extract_paste_id(url: str) -> Optional[str]: - """Extract the paste ID from a paste.rs or dpaste.com URL.""" + """Paste ID from a paste.rs URL (dpaste.com pastes have no deletable ID).""" url = url.strip().rstrip("/") for prefix in ("https://paste.rs/", "http://paste.rs/"): if url.startswith(prefix): @@ -181,40 +163,25 @@ def _extract_paste_id(url: str) -> Optional[str]: def delete_paste(url: str) -> bool: - """Delete a paste from paste.rs. Returns True on success. - - Only paste.rs supports unauthenticated DELETE. dpaste.com pastes expire automatically but cannot - be deleted via API. - """ + """Delete a paste.rs paste (the only service with unauthenticated DELETE). True on success.""" paste_id = _extract_paste_id(url) if not paste_id: - raise ValueError( - f"Cannot delete: only paste.rs URLs are supported. Got: {url}" - ) + raise ValueError(f"Cannot delete: only paste.rs URLs are supported. Got: {url}") - target = f"{_PASTE_RS_URL}{paste_id}" req = urllib.request.Request( - target, method="DELETE", - headers={"User-Agent": "hermes-agent/debug-share"}, + f"{_PASTE_RS_URL}{paste_id}", method="DELETE", headers={"User-Agent": _USER_AGENT}, ) with urllib.request.urlopen(req, timeout=30) as resp: return 200 <= resp.status < 300 def _schedule_auto_delete(urls: list[str], delay_seconds: int = _AUTO_DELETE_SECONDS): - """Record *urls* for deletion ``delay_seconds`` from now. - - Only paste.rs URLs are recorded (dpaste.com auto-expires); entries are merged into any existing - pending.json. The old fork-and-sleep subprocesses leaked ~20 MB of resident interpreter per - ``hermes debug share``. This replacement is stateless: the gateway's cron ticker sweeps expired - entries once per hour, ``hermes debug share`` runs an opportunistic sweep as a fallback for - CLI-only users, and paste.rs's own retention policy is the last resort. - """ + """Record paste.rs *urls* for deletion ``delay_seconds`` from now (merged into pending.json).""" paste_rs_urls = [u for u in urls if _extract_paste_id(u)] if not paste_rs_urls: return - # Dedupe by URL: keep the later expire_at if same URL appears twice + # Dedupe by URL, keeping the later expire_at. by_url: dict[str, float] = {e["url"]: float(e["expire_at"]) for e in _load_pending()} expire_at = time.time() + delay_seconds for u in paste_rs_urls: @@ -226,7 +193,7 @@ def _post_paste(service: str, endpoint: str, body: bytes, content_type: str) -> """POST *body* to a paste service and return the paste URL it echoes back.""" req = urllib.request.Request( endpoint, data=body, method="POST", - headers={"Content-Type": content_type, "User-Agent": "hermes-agent/debug-share"}, + headers={"Content-Type": content_type, "User-Agent": _USER_AGENT}, ) with urllib.request.urlopen(req, timeout=30) as resp: url = resp.read().decode("utf-8").strip() @@ -236,21 +203,14 @@ def _post_paste(service: str, endpoint: str, body: bytes, content_type: str) -> def _upload_paste_rs(content: str) -> str: - """Upload to paste.rs. Returns the paste URL.""" return _post_paste("paste.rs", _PASTE_RS_URL, content.encode("utf-8"), "text/plain; charset=utf-8") def _upload_dpaste_com(content: str, expiry_days: int = 7) -> str: - """Upload to dpaste.com. Returns the paste URL.""" boundary = "----HermesDebugBoundary9f3c" def _field(name: str, value: str) -> str: - return ( - f"--{boundary}\r\n" - f'Content-Disposition: form-data; name="{name}"\r\n' - f"\r\n" - f"{value}\r\n" - ) + return f'--{boundary}\r\nContent-Disposition: form-data; name="{name}"\r\n\r\n{value}\r\n' body = ( _field("content", content) @@ -264,28 +224,18 @@ def _upload_dpaste_com(content: str, expiry_days: int = 7) -> str: def upload_to_pastebin(content: str, expiry_days: int = 7) -> str: """Upload *content* to a paste service, trying paste.rs then dpaste.com.""" errors: list[str] = [] - - # Try paste.rs first (simple, fast) - try: - return _upload_paste_rs(content) - except Exception as exc: - errors.append(f"paste.rs: {exc}") - - # Fallback: dpaste.com (supports expiry) - try: - return _upload_dpaste_com(content, expiry_days=expiry_days) - except Exception as exc: - errors.append(f"dpaste.com: {exc}") - - raise RuntimeError( - "Failed to upload to any paste service:\n " + "\n ".join(errors) - ) + for service, upload in ( + ("paste.rs", lambda: _upload_paste_rs(content)), + ("dpaste.com", lambda: _upload_dpaste_com(content, expiry_days=expiry_days)), + ): + try: + return upload() + except Exception as exc: + errors.append(f"{service}: {exc}") + raise RuntimeError("Failed to upload to any paste service:\n " + "\n ".join(errors)) -# --------------------------------------------------------------------------- -# Log file reading -# --------------------------------------------------------------------------- - +# ── Log file reading ─────────────────────────────────────────────────────── @dataclass class LogSnapshot: @@ -304,11 +254,9 @@ def _primary_log_path(log_name: str) -> Optional[Path]: return (get_hermes_home() / "logs" / filename) if filename else None -# Logs written by a client process rather than by this backend. When the -# desktop app talks to a remote/docker/SSH backend, `hermes debug share` runs -# on the *backend* and can never see them — a bare "(file not found)" then -# reads as "the app logged nothing" and sends triage down a dead end, which is -# exactly the wrong answer when the client is the thing being debugged. +# Logs written by a client process rather than this backend. When the desktop app talks to a +# remote/docker/SSH backend, `hermes debug share` runs on the *backend* and can never see +# them — a bare "(file not found)" would read as "the app logged nothing" and misdirect triage. _CLIENT_SIDE_LOGS = { "desktop": ( "written by Hermes Desktop on the machine running the app, not by this " @@ -330,32 +278,21 @@ def _missing_log_note(log_name: str) -> str: def _resolve_log_path(log_name: str) -> Optional[Path]: - """Find the log file for *log_name*, falling back to the .1 rotation. - - Returns the first non-empty candidate (primary, then .1), or None. Callers distinguish - 'empty primary' from 'truly missing' via :func:`_primary_log_path`. - """ + """First non-empty candidate for *log_name* (primary, then the .1 rotation), or None.""" primary = _primary_log_path(log_name) if primary is None: return None - - if primary.exists() and primary.stat().st_size > 0: - return primary - - rotated = primary.parent / f"{primary.name}.1" - if rotated.exists() and rotated.stat().st_size > 0: - return rotated - + for candidate in (primary, primary.parent / f"{primary.name}.1"): + if candidate.exists() and candidate.stat().st_size > 0: + return candidate return None def _redact_log_text(text: str) -> str: - """Run ``redact_sensitive_text`` with ``force=True`` over upload-bound text. + """Force-mode ``redact_sensitive_text`` (+ email scrub) over upload-bound text. - Uses ``force=True`` so redaction fires regardless of the operator's ``security.redact_secrets`` - setting. The local on-disk log file is not modified; only the in-memory copy headed for the - public paste service is sanitized. Returns the redacted text (or the original when empty / non- - string). + ``force=True`` so redaction fires regardless of the operator's ``security.redact_secrets`` + setting; only the in-memory copy headed for the paste service is sanitized. """ if not text: return text @@ -366,22 +303,13 @@ def _redact_log_text(text: str) -> str: def _capture_log_snapshot( - log_name: str, - *, - tail_lines: int, - max_bytes: int = _MAX_LOG_BYTES, - redact: bool = True, + log_name: str, *, tail_lines: int, max_bytes: int = _MAX_LOG_BYTES, redact: bool = True, ) -> LogSnapshot: - """Capture a log once and derive summary/full-log views from it. + """Capture a log once and derive the summary tail and full-log views from it. - The report tail and standalone log upload must come from the same file snapshot. Otherwise a - rotation/truncate between reads can make the report look newer than the uploaded ``agent.log`` - paste. - - When ``redact`` is True (the default), both ``tail_text`` and ``full_text`` are run through - ``_redact_log_text`` so the snapshot returned is upload-safe. The on-disk log file is never - modified. Pass ``redact=False`` to capture original log content (used by ``hermes debug share - --no-redact``). + Both views must come from the same read: a rotation/truncate between reads would make the + report look newer than the uploaded ``agent.log`` paste. With ``redact`` both texts are + upload-safe; the on-disk file is never modified. """ log_path = _resolve_log_path(log_name) if log_path is None: @@ -400,9 +328,8 @@ def _capture_log_snapshot( raw = f.read() truncated = False else: - # Read from the end until we have enough bytes for the - # standalone upload and enough newline context to render the - # summary tail from the same snapshot. + # Read from the end until we have enough bytes for the standalone upload and + # enough newline context to render the summary tail from the same snapshot. chunk_size = 8192 pos = size chunks: list[bytes] = [] @@ -425,10 +352,8 @@ def _capture_log_snapshot( full_raw = raw if truncated and len(full_raw) > max_bytes: cut = len(full_raw) - max_bytes - # Check whether the cut lands exactly on a line boundary. If the - # byte just before the cut position is a newline the first retained - # byte starts a complete line and we should keep it. Only drop a - # partial first line when we're genuinely mid-line. + # Only drop a partial first line when the cut lands genuinely mid-line (the byte + # before the cut is not a newline). on_boundary = cut > 0 and full_raw[cut - 1 : cut] == b"\n" full_raw = full_raw[cut:] if not on_boundary and b"\n" in full_raw: @@ -456,22 +381,19 @@ _REPORT_LOGS = ("agent", "errors", "gateway", "gui", "desktop") _FULL_LOGS = ("agent", "gateway", "gui", "desktop") -def _capture_default_log_snapshots( - log_lines: int, *, redact: bool = True -) -> dict[str, LogSnapshot]: +def _tail_budget(name: str, log_lines: int) -> int: + return log_lines if name == "agent" else min(log_lines, 100) + + +def _capture_default_log_snapshots(log_lines: int, *, redact: bool = True) -> dict[str, LogSnapshot]: """Capture all logs used by debug-share exactly once.""" - errors_lines = min(log_lines, 100) return { - name: _capture_log_snapshot( - name, tail_lines=log_lines if name == "agent" else errors_lines, redact=redact - ) + name: _capture_log_snapshot(name, tail_lines=_tail_budget(name, log_lines), redact=redact) for name in _REPORT_LOGS } -# --------------------------------------------------------------------------- -# Debug report collection -# --------------------------------------------------------------------------- +# ── Debug report collection ──────────────────────────────────────────────── def _capture_dump() -> str: """Run ``hermes dump`` and return its stdout as a string.""" @@ -493,15 +415,11 @@ def _capture_dump() -> str: def collect_debug_report( - *, - log_lines: int = 200, - dump_text: str = "", - log_snapshots: Optional[dict[str, LogSnapshot]] = None, + *, log_lines: int = 200, dump_text: str = "", log_snapshots: Optional[dict[str, LogSnapshot]] = None, ) -> str: - """Build the summary debug report: system dump + log tails. + """Build the summary debug report (system dump + log tails) as upload-ready text. ``dump_text`` is pre-captured dump output; when empty, ``hermes dump`` is run internally. - Returns plain text ready for upload. """ buf = io.StringIO() @@ -512,11 +430,9 @@ def collect_debug_report( if log_snapshots is None: log_snapshots = _capture_default_log_snapshots(log_lines) - # ── Sanitiser heal counters (#96870) ───────────────────────────────── - # In-process, in-memory counters: populated when this report is built - # inside a process that ran agent turns (gateway /debug share); empty - # from a fresh CLI process, where the errors.log tail below carries the - # same escalation lines instead. + # Sanitiser heal counters: in-process, in-memory — populated when this report is built + # inside a process that ran agent turns (gateway /debug share); empty from a fresh CLI + # process, where the errors.log tail below carries the same escalation lines instead. try: from agent.agent_runtime_helpers import get_sanitizer_heal_stats @@ -532,48 +448,33 @@ def collect_debug_report( except Exception: pass - # ── Recent log tails (summary only) ────────────────────────────────── - errors_lines = min(log_lines, 100) buf.write("\n") for name in _REPORT_LOGS: - lines = log_lines if name == "agent" else errors_lines - buf.write(f"\n--- {name}.log (last {lines} lines) ---\n") + buf.write(f"\n--- {name}.log (last {_tail_budget(name, log_lines)} lines) ---\n") buf.write(log_snapshots[name].tail_text) buf.write("\n") return buf.getvalue() -# --------------------------------------------------------------------------- -# Shared bundle collection (used by both the paste.rs and Nous-S3 paths) -# --------------------------------------------------------------------------- +# ── Shared bundle collection (paste.rs and Nous-S3 paths) ────────────────── -# Bundle format identifier embedded in the Nous-S3 JSON envelope. The -# discord-support viewer keys off this string to parse the bundle. +# Bundle format identifier in the Nous-S3 JSON envelope; the discord-support viewer keys off it. _NOUS_BUNDLE_FORMAT = "hermes-debug-share/1" -def collect_share_bundle( - log_lines: int = 200, - redact: bool = True, -) -> dict[str, str]: +def collect_share_bundle(log_lines: int = 200, redact: bool = True) -> dict[str, str]: """Collect the debug report + full logs as a label→text mapping. - The dump header is prepended to each full log (mirroring the historical paste behaviour) so - every file is self-contained, and the redaction banner is prepended when ``redact`` is True. + The dump header is prepended to each full log so every file is self-contained, and the + redaction banner is prepended when ``redact`` is True. """ dump_text = _capture_dump() log_snapshots = _capture_default_log_snapshots(log_lines, redact=redact) - report = collect_debug_report( - log_lines=log_lines, - dump_text=dump_text, - log_snapshots=log_snapshots, - ) - # Visible banner so reviewers know redaction was applied at upload time. + report = collect_debug_report(log_lines=log_lines, dump_text=dump_text, log_snapshots=log_snapshots) banner = _REDACTION_BANNER if redact else "" bundle: dict[str, str] = {"report": banner + report} - # Prepend dump header to each full log so every file is self-contained. for name in _FULL_LOGS: full = log_snapshots[name].full_text if full: @@ -582,32 +483,25 @@ def collect_share_bundle( def build_nous_bundle(bundle: dict[str, str], redact: bool = True) -> bytes: - """Gzip-compress a :func:`collect_share_bundle` mapping into the Nous envelope. + """Gzip a :func:`collect_share_bundle` mapping into the Nous envelope. - The JSON shape (``format: hermes-debug-share/1``, ``redacted``, ``created``, ``files``) is - what the discord-support viewer parses — keep it stable. + The JSON shape (``format``, ``redacted``, ``created``, ``files``) is what the + discord-support viewer parses — keep it stable. """ - created = datetime.datetime.now(datetime.timezone.utc).isoformat() envelope = { "format": _NOUS_BUNDLE_FORMAT, "redacted": bool(redact), - "created": created, + "created": datetime.datetime.now(datetime.timezone.utc).isoformat(), "files": bundle, } return gzip.compress(json.dumps(envelope).encode("utf-8")) -# --------------------------------------------------------------------------- -# CLI entry points -# --------------------------------------------------------------------------- +# ── CLI entry points ─────────────────────────────────────────────────────── @dataclass class DebugShareResult: - """Structured outcome of a ``debug share`` upload. - - Returned by :func:`build_debug_share` so non-CLI callers (dashboard web server, gateway) can - render the uploaded paste URLs as real links instead of scraping printed text. - """ + """Outcome of a ``debug share`` upload, so non-CLI callers can render real links.""" urls: dict # label -> paste URL (e.g. {"Report": "...", "agent.log": "..."}) failures: list # human-readable "label: error" strings for optional uploads @@ -616,40 +510,28 @@ class DebugShareResult: report: str = "" # the summary report text (kept for local fallback) -def build_debug_share( - *, - log_lines: int = 200, - expiry: int = 7, - redact: bool = True, -) -> DebugShareResult: +def build_debug_share(*, log_lines: int = 200, expiry: int = 7, redact: bool = True) -> DebugShareResult: """Collect the debug report + full logs, upload each, return the URLs. - This is the shared core behind ``hermes debug share`` (CLI) and the dashboard ``POST - /api/ops/debug-share`` endpoint. It performs blocking network I/O (paste uploads) — callers - inside an event loop must run it in a worker thread. + Shared core behind ``hermes debug share`` and the dashboard ``POST /api/ops/debug-share``. + Blocking network I/O — callers inside an event loop must run it in a worker thread. """ _best_effort_sweep_expired_pastes() - # Collect the report + full logs (force-redacted when redact=True) via the - # shared collector so the paste.rs and Nous-S3 paths build identical, - # identically-redacted bundles. The dump header + redaction banner are - # applied inside collect_share_bundle. bundle = collect_share_bundle(log_lines=log_lines, redact=redact) if redact: - logger.info( - "hermes debug share: applied force-mode redaction to log snapshots before upload" - ) + logger.info("hermes debug share: applied force-mode redaction to log snapshots before upload") report = bundle["report"] urls: dict[str, str] = {} failures: list[str] = [] - # 1. Summary report (required — raises on failure so callers can fall back) + # Summary report is required — raises on failure so callers can fall back. urls["Report"] = upload_to_pastebin(report, expiry_days=expiry) - # 2-5. Full logs (optional — failures are collected, not raised) + # Full logs are optional — failures are collected, not raised. for name in _FULL_LOGS: label = f"{name}.log" content = bundle.get(label) @@ -660,24 +542,16 @@ def build_debug_share( except Exception as exc: failures.append(f"{label}: {exc}") - # Schedule auto-deletion after 6 hours. _schedule_auto_delete(list(urls.values())) return DebugShareResult( - urls=urls, - failures=failures, - redacted=redact, - auto_delete_seconds=_AUTO_DELETE_SECONDS, - report=report, + urls=urls, failures=failures, redacted=redact, + auto_delete_seconds=_AUTO_DELETE_SECONDS, report=report, ) def _confirm_upload(args) -> bool: - """Require explicit consent before any debug-share upload. - - The privacy notice is printed by the caller. This gates the actual upload: with ``--yes`` (or - ``-y``) we proceed unprompted; otherwise we ask an interactive ``[y/N]`` question. - """ + """Gate the actual upload: ``--yes`` proceeds unprompted, else ask an interactive [y/N].""" if bool(getattr(args, "yes", False)): return True if not sys.stdin.isatty(): @@ -702,14 +576,11 @@ def run_debug_share(args): """Collect debug report + full logs, upload each, print URLs.""" log_lines = getattr(args, "lines", 200) expiry = getattr(args, "expire", 7) - local_only = getattr(args, "local", False) - nous = getattr(args, "nous", False) redact = not getattr(args, "no_redact", False) - if local_only: - # Local-only path never uploads — render the report to stdout and bail - # before any network I/O. Reuses the shared collector so the rendered - # output matches exactly what would be uploaded. + if getattr(args, "local", False): + # Never uploads — render via the shared collector so the output matches exactly what + # would be uploaded, and bail before any network I/O. _best_effort_sweep_expired_pastes() print("Collecting debug report...") bundle = collect_share_bundle(log_lines=log_lines, redact=redact) @@ -721,7 +592,7 @@ def run_debug_share(args): print(body) return - if nous: + if getattr(args, "nous", False): _run_debug_share_nous(args, log_lines=log_lines, redact=redact) return @@ -732,17 +603,12 @@ def run_debug_share(args): print("Uploading...") try: - result = build_debug_share( - log_lines=log_lines, - expiry=expiry, - redact=redact, - ) + result = build_debug_share(log_lines=log_lines, expiry=expiry, redact=redact) except RuntimeError as exc: print(f"\nUpload failed: {exc}", file=sys.stderr) print("\nRun `hermes debug share --local` to print the report instead.\n") sys.exit(1) - # Print results label_width = max(len(k) for k in result.urls) print("\nDebug report uploaded:") for label, url in result.urls.items(): @@ -753,10 +619,7 @@ def run_debug_share(args): hours = result.auto_delete_seconds // 3600 print(f"\n⏱ Pastes will auto-delete in {hours} hours.") - - # Manual delete fallback print("To delete now: hermes debug delete ") - print("\nShare these links with the Hermes team for support.") @@ -776,12 +639,7 @@ _NOUS_PRIVACY_NOTICE = """\ def _run_debug_share_nous(args, *, log_lines: int, redact: bool) -> None: - """Handle ``hermes debug share --nous``: upload the bundle to Nous-S3. - - Collects the same force-redacted bundle as the paste path, gzips it into the Nous envelope, - requests a signed URL from NAS, uploads, and prints the private viewer link. On any failure - falls back to a clear error that suggests ``--local``. - """ + """``hermes debug share --nous``: gzip the same bundle into the Nous envelope, upload to Nous-S3.""" from hermes_cli.diagnostics_upload import share_to_nous print(_NOUS_PRIVACY_NOTICE) @@ -797,9 +655,7 @@ def _run_debug_share_nous(args, *, log_lines: int, redact: bool) -> None: bundle = collect_share_bundle(log_lines=log_lines, redact=redact) if redact: - logger.info( - "hermes debug share --nous: applied force-mode redaction before upload" - ) + logger.info("hermes debug share --nous: applied force-mode redaction before upload") blob = build_nous_bundle(bundle, redact=redact) print("Uploading to Nous diagnostics storage...") @@ -864,8 +720,6 @@ def run_debug_delete(args): def run_debug(args): """Route debug subcommands.""" # Opportunistic sweep of expired pastes on every ``hermes debug`` call. - # Replaces the old per-paste sleeping subprocess that used to leak as - # one orphaned Python interpreter per scheduled deletion. _best_effort_sweep_expired_pastes() subcmd = getattr(args, "debug_command", None) From 08881f5997fed68aa95c6f142c553e83f954df2d Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:20:12 -0700 Subject: [PATCH 03/29] =?UTF-8?q?refactor(hermes=5Fcli):=20container=5Fboo?= =?UTF-8?q?t.py=20=E2=80=94=20dedupe=20/proc=20argv=20decoding,=20slot-act?= =?UTF-8?q?ion=20and=20exec-script=20helpers,=20compact=20rationale=20comm?= =?UTF-8?q?ents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/container_boot.py | 392 +++++++++++++---------------------- 1 file changed, 143 insertions(+), 249 deletions(-) diff --git a/hermes_cli/container_boot.py b/hermes_cli/container_boot.py index 442310dfe6..e823c0b972 100644 --- a/hermes_cli/container_boot.py +++ b/hermes_cli/container_boot.py @@ -1,8 +1,8 @@ """Container-boot reconciliation of per-profile gateway s6 services. -Wired into the image as /etc/cont-init.d/02-reconcile-profiles by the Dockerfile (Phase 4 Task 4.0). -Runs as root after 01-hermes-setup (the stage2 hook) has chowned the volume and seeded $HERMES_HOME, -but before s6-rc starts user services. +Wired into the image as /etc/cont-init.d/02-reconcile-profiles. Runs as root after +01-hermes-setup (the stage2 hook) has chowned the volume and seeded $HERMES_HOME, but +before s6-rc starts user services. """ from __future__ import annotations @@ -15,50 +15,32 @@ from typing import Literal, Sequence log = logging.getLogger(__name__) -# Only this desired state triggers automatic restart. Everything else -# (startup_failed, starting, stopped, missing) registers the slot in -# the down state and waits for explicit user action — this avoids the -# crash-loop where a broken gateway keeps being restarted across -# `docker restart` cycles. Older installs only have gateway_state; -# newer lifecycle commands persist desired_state separately so a transient -# runtime state (draining/startup_failed) does not erase the operator's -# durable start/stop intent across pod/container recreation. +# Only this desired state triggers automatic restart. Everything else (startup_failed, +# starting, stopped, missing) registers the slot down and waits for explicit user action — +# avoiding the crash-loop where a broken gateway keeps restarting across `docker restart`. +# Older installs only have gateway_state; newer lifecycle commands persist desired_state +# separately so a transient runtime state does not erase the operator's durable intent. _AUTOSTART_STATES = frozenset({"running"}) -# Transient runtime sub-states of a RUNNING gateway. A gateway only ever -# reaches these while it is up and serving, so they are NOT an operator stop -# and NOT a failed boot: -# - `draining` — written by the drain watcher / scale-to-zero go-dormant -# path when an in-flight quiesce begins (gateway/run.py). -# - `degraded` — written when the gateway comes up with some platforms -# queued for retry, then "falls through to the normal -# running state" (gateway/run.py #5196): the process is up, -# serving cron + whatever platforms connected, and the -# reconnect watcher takes the rest from there. -# -# When a gateway is hard-killed *while in one of these states* (a container/VM -# recreate SIGTERMs it before `_stop_impl` reaches its terminal-state persist), -# the last value left in gateway_state.json is the transient sub-state. With no -# explicit `desired_state` to fall back to, treating that literal value as the -# autostart intent would leave the gateway DOWN on every subsequent boot — the -# gateway never comes back, the dashboard is up but messaging stays dark -# (observed on a relay-opted-in staging instance stranded at `draining`, -# 2026-06; `degraded` is the same wedge class). Map these transient sub-states -# to `running` so a stranded marker reads as the run-intent it actually -# represents. This mirrors gateway/run.py's #42675 handling, which persists -# `running` (not the mid-shutdown `draining`) when an unexpected signal tears -# the gateway down — extended here to the case where the gateway died before it -# could persist anything at all. -# -# `starting` / `startup_failed` are deliberately NOT included: those mean the -# gateway died mid-boot or failed to come up, so auto-restarting them would -# reintroduce the crash-loop the down-marker guard exists to prevent. +# Transient runtime sub-states of a RUNNING gateway — reached only while up and serving, so +# NOT an operator stop and NOT a failed boot: +# - `draining` — drain watcher / scale-to-zero go-dormant path, in-flight quiesce begun. +# - `degraded` — came up with some platforms queued for retry, then fell through to the +# normal running state; the reconnect watcher takes it from there. +# When a gateway is hard-killed *in one of these states* (container/VM recreate SIGTERMs it +# before `_stop_impl` persists a terminal state), the marker left in gateway_state.json is +# the transient sub-state. With no `desired_state` to fall back to, treating it literally +# would leave the gateway DOWN on every subsequent boot (observed: a relay-opted-in staging +# instance stranded at `draining`; `degraded` is the same wedge class). Map them to `running` +# — mirrors gateway/run.py persisting `running` (not mid-shutdown `draining`) on unexpected +# signal, extended to the case where the gateway died before persisting anything. +# `starting` / `startup_failed` are deliberately NOT included: those mean the gateway died +# mid-boot or failed to come up, and auto-restarting would reintroduce the crash-loop. _TRANSIENT_RUNNING_STATES = frozenset({"draining", "degraded"}) -# Stale runtime files we sweep before recreating service slots. These -# all hold container-namespaced state (PIDs, process tables) that's -# garbage post-restart — a numerically-equal PID in the new container -# is a different process. See the Risk Register in the plan. +# Stale runtime files swept before recreating service slots: they hold container-namespaced +# state (PIDs, process tables) that's garbage post-restart — a numerically-equal PID in the +# new container is a different process. _STALE_RUNTIME_FILES = ("gateway.pid", "processes.json") ReconcileActionLabel = Literal["started", "registered", "skipped"] @@ -70,15 +52,24 @@ class ReconcileAction: profile: str prior_state: str | None action: ReconcileActionLabel - # How the profile's previous gateway life ended: "clean" (exit path ran), - # "unclean" (sentinel still says running — SIGKILL/OOM/VM death), or - # "unknown" (no sentinel / never ran). See gateway.lifecycle_ledger - # (NS-608): at container boot this is the one place that can stamp - # "the previous container life ended violently" into a durable, - # volume-persisted log line. + # How the previous gateway life ended: "clean" (exit path ran), "unclean" (sentinel still + # says running — SIGKILL/OOM/VM death), or "unknown" (no sentinel / never ran). Container + # boot is the one place that can stamp "the previous life ended violently" into a + # durable, volume-persisted log line (gateway.lifecycle_ledger). prior_exit: str = "unknown" +def _slot_action( + profile: str, profile_dir: Path, prior_state: str | None, start: bool, +) -> ReconcileAction: + return ReconcileAction( + profile=profile, + prior_state=prior_state, + action="started" if start else "registered", + prior_exit=_read_prior_exit_label(profile_dir), + ) + + def reconcile_profile_gateways( *, hermes_home: Path, @@ -88,19 +79,17 @@ def reconcile_profile_gateways( ) -> list[ReconcileAction]: """Recreate s6 service registrations for every persistent profile. - Always registers a ``gateway-default`` slot for the root profile (the implicit profile that - lives at the top of ``$HERMES_HOME``, not under ``profiles/``). The dispatcher in - ``hermes_cli.gateway`` maps an empty profile suffix to ``gateway-default``, so this slot is what + Always registers a ``gateway-default`` slot for the root profile (the implicit profile at + the top of ``$HERMES_HOME``, not under ``profiles/``): the dispatcher in + ``hermes_cli.gateway`` maps an empty profile suffix to it, so it is what ``hermes gateway start`` (no ``-p``) targets. """ actions: list[ReconcileAction] = [] - # A multiplexing root/default gateway owns inbound platform connections - # for every profile. Named slots must still be registered (so explicit - # lifecycle management remains available), but booting them from their - # persisted run intent would create additional multiplex owners. - # Keep the boot reconciler aligned with the gateway that will own these - # slots. The runtime resolver gives a recognized environment override + # A multiplexing root/default gateway owns inbound platform connections for every + # profile. Named slots must still be registered (explicit lifecycle management stays + # available), but booting them from their persisted run intent would create additional + # multiplex owners. The runtime resolver gives a recognized environment override # precedence over config.yaml and otherwise preserves the configured value. from gateway.config import load_gateway_config from utils import is_truthy_value @@ -113,51 +102,32 @@ def reconcile_profile_gateways( "using the GATEWAY_MULTIPLEX_PROFILES override if set.", exc_info=True, ) - multiplex_profiles = is_truthy_value( - os.environ.get("GATEWAY_MULTIPLEX_PROFILES"), - ) + multiplex_profiles = is_truthy_value(os.environ.get("GATEWAY_MULTIPLEX_PROFILES")) - # Default profile — always register, even if nothing has ever - # populated the root profile dir. The slot exists so - # ``hermes gateway start`` (no ``-p``) has somewhere to land; - # auto-up only when the prior state was "running" (same rule as - # named profiles). If the container was launched with the legacy - # `gateway run` command and no state exists yet, seed that intent - # as `running` so the s6 reconciler preserves the pre-s6 behavior. + # Default profile — always register, even if nothing has ever populated the root profile + # dir; auto-up only when the prior state was "running" (same rule as named profiles). A + # legacy `gateway run` container with no state yet seeds that intent as `running` so the + # s6 reconciler preserves the pre-s6 behavior. legacy_default_state = _maybe_migrate_legacy_gateway_run_state( - hermes_home, - container_argv=container_argv, - dry_run=dry_run, + hermes_home, container_argv=container_argv, dry_run=dry_run, ) default_prior_state = legacy_default_state or _read_desired_state(hermes_home) default_should_start = default_prior_state in _AUTOSTART_STATES if not dry_run: _cleanup_stale_runtime_files(hermes_home) _register_service(scandir, "default", start=default_should_start) - actions.append(ReconcileAction( - profile="default", - prior_state=default_prior_state, - action="started" if default_should_start else "registered", - prior_exit=_read_prior_exit_label(hermes_home), - )) + actions.append(_slot_action("default", hermes_home, default_prior_state, default_should_start)) profiles_root = hermes_home / "profiles" if profiles_root.is_dir(): for entry in sorted(profiles_root.iterdir()): - if not entry.is_dir(): + # SOUL.md is always seeded by `hermes profile create` (config.yaml is not — that + # comes later via `hermes setup`): the "real profile" marker so stray dirs + # (backups, manual mkdir) aren't picked up. + if not entry.is_dir() or not (entry / "SOUL.md").exists(): continue - # SOUL.md is always seeded by `hermes profile create` (config.yaml - # is not — that comes later via `hermes setup`). Use it as the - # "real profile" marker so stray dirs (backups, manual mkdir) - # aren't picked up. - if not (entry / "SOUL.md").exists(): - continue - # The "default" service name is reserved for the root - # profile (above) — if a user has somehow created a - # ``profiles/default/`` directory, skip it to avoid the - # slot collision. Their gateway would still be reachable - # via ``hermes -p default-named gateway start`` if they - # rename the directory; we don't try to disambiguate here. + # "default" is reserved for the root profile (above); skip a stray + # ``profiles/default/`` rather than collide on the slot. if entry.name == "default": log.warning( "profiles/default/ exists — skipping to avoid colliding " @@ -166,20 +136,13 @@ def reconcile_profile_gateways( continue prior_state = _read_desired_state(entry) - should_start = ( - not multiplex_profiles and prior_state in _AUTOSTART_STATES - ) + should_start = not multiplex_profiles and prior_state in _AUTOSTART_STATES if not dry_run: _cleanup_stale_runtime_files(entry) _register_service(scandir, entry.name, start=should_start) - actions.append(ReconcileAction( - profile=entry.name, - prior_state=prior_state, - action="started" if should_start else "registered", - prior_exit=_read_prior_exit_label(entry), - )) + actions.append(_slot_action(entry.name, entry, prior_state, should_start)) if not dry_run: _write_reconcile_log(hermes_home, actions) @@ -194,10 +157,9 @@ def _maybe_migrate_legacy_gateway_run_state( ) -> str | None: """Seed root gateway_state for pre-s6 `gateway run` containers. - The tini image let Docker users run the gateway as the container command (`docker run ... - gateway run`). After the s6 migration, profile gateways are restored from persisted - gateway_state.json; a legacy container with no state file would therefore register the default - service down and never start. + The tini image let Docker users run the gateway as the container command. After the s6 + migration profile gateways are restored from persisted gateway_state.json, so a legacy + container with no state file would register the default service down and never start. """ state_file = hermes_home / "gateway_state.json" if state_file.exists(): @@ -221,40 +183,33 @@ def _maybe_migrate_legacy_gateway_run_state( return "running" +def _cmdline_argv(cmdline: Path) -> tuple[str, ...]: + raw = cmdline.read_bytes() + return tuple(part.decode("utf-8", "replace") for part in raw.split(b"\0") if part) + + def _read_container_argv() -> tuple[str, ...]: """Best-effort read of the container's main program argv. s6-overlay v2: PID 1 is ``/init`` and its argv holds ``main-wrapper.sh``. v3: PID 1 is - ``s6-svscan`` and the real command lives on another PID, so after the PID 1 fast path we scan - ``/proc/*/cmdline`` for a process whose argv contains ``main-wrapper.sh``. + ``s6-svscan`` and the real command lives on another PID, so after the PID 1 fast path we + scan ``/proc/*/cmdline`` for a process whose argv contains ``main-wrapper.sh``. """ - # Fast path: PID 1 is the command itself (s6-overlay v2 / tini). try: - raw = Path("/proc/1/cmdline").read_bytes() - argv = tuple( - part.decode("utf-8", "replace") for part in raw.split(b"\0") if part - ) + argv = _cmdline_argv(Path("/proc/1/cmdline")) if any("main-wrapper.sh" in part for part in argv): return argv except OSError: pass - # Slow path: s6-overlay v3 — PID 1 is s6-svscan; find the - # rc.init-launched process whose argv contains main-wrapper.sh. try: - proc_dir = Path("/proc") - for entry in proc_dir.iterdir(): + for entry in Path("/proc").iterdir(): if not entry.name.isdigit(): continue try: - raw = (entry / "cmdline").read_bytes() + argv = _cmdline_argv(entry / "cmdline") except OSError: continue - argv = tuple( - part.decode("utf-8", "replace") - for part in raw.split(b"\0") - if part - ) if any("main-wrapper.sh" in part for part in argv): return argv except OSError: @@ -266,20 +221,13 @@ def _read_container_argv() -> tuple[str, ...]: def _strip_container_argv_prefix(argv: Sequence[str]) -> list[str]: """Strip the s6/wrapper prefix off the container argv, leaving the hermes args. - Rather than peel each leading token positionally (which silently breaks the moment s6 changes - its launcher shape again — exactly what happened in the v2→v3 bump), drop everything up to and - including the ``main-wrapper.sh`` token: that wrapper path is the stable boundary the image - owns, and the subcommand always follows it. + Drop everything through the ``main-wrapper.sh`` token rather than peel leading tokens + positionally (which broke on the s6 v2→v3 launcher-shape change): the wrapper path is the + stable boundary the image owns, and the subcommand always follows it. """ args = list(argv) - # Preferred boundary: everything through main-wrapper.sh is launcher - # prefix. Covers s6-overlay v2 (`/init …main-wrapper.sh …`) and v3 - # (`/bin/sh -e …rc.init top …main-wrapper.sh …`) with one rule. - wrapper_idx = next( - (i for i, a in enumerate(args) if a.endswith("main-wrapper.sh")), - None, - ) + wrapper_idx = next((i for i, a in enumerate(args) if a.endswith("main-wrapper.sh")), None) if wrapper_idx is not None: args = args[wrapper_idx + 1 :] elif args and Path(args[0]).name == "init": @@ -297,7 +245,7 @@ def _strip_container_argv_prefix(argv: Sequence[str]) -> list[str]: def _is_legacy_gateway_run_request(argv: Sequence[str]) -> bool: - """Return True for Docker commands equivalent to `gateway run`.""" + """True for Docker commands equivalent to `gateway run`.""" args = _strip_container_argv_prefix(argv) if "--no-supervise" in args: return False @@ -305,58 +253,44 @@ def _is_legacy_gateway_run_request(argv: Sequence[str]) -> bool: def _is_dashboard_container(argv: Sequence[str]) -> bool: - """Return True when the container's command is the dashboard. - - A dashboard-only container (``hermes dashboard ...``) never spawns or supervises per-profile - gateways — that is the gateway container's job. - """ + """True when the container's command is the dashboard (which never supervises gateways).""" args = _strip_container_argv_prefix(argv) return bool(args) and args[0] == "dashboard" def _read_desired_state(profile_dir: Path) -> str | None: - """Read the persisted gateway desired state for reconciliation. + """Persisted gateway desired state for reconciliation. - Newer state files carry ``desired_state``: operator intent written by s6 lifecycle commands. - Older files only carry ``gateway_state``; keep that as a compatibility fallback so existing - running/stopped profiles preserve their behavior until the next explicit start/stop. - - Missing or unparseable files count as "no desired state" so we don't bork the whole - reconciliation on a corrupt file. + Newer state files carry ``desired_state`` (operator intent from s6 lifecycle commands); + older ones only ``gateway_state``, kept as a fallback so existing profiles preserve their + behavior until the next explicit start/stop. Missing/unparseable files count as "no + desired state" so a corrupt file can't bork the whole reconciliation. """ state_file = profile_dir / "gateway_state.json" if not state_file.exists(): return None try: data = json.loads(state_file.read_text(encoding="utf-8")) - desired_state = data.get("desired_state") - if desired_state is not None: - return desired_state - gateway_state = data.get("gateway_state") - if gateway_state in _TRANSIENT_RUNNING_STATES: - return "running" - return gateway_state except (OSError, json.JSONDecodeError): - log.warning( - "could not read %s; treating as no prior state", state_file, - ) + log.warning("could not read %s; treating as no prior state", state_file) return None + desired_state = data.get("desired_state") + if desired_state is not None: + return desired_state + gateway_state = data.get("gateway_state") + if gateway_state in _TRANSIENT_RUNNING_STATES: + return "running" + return gateway_state def _cleanup_stale_runtime_files(profile_dir: Path) -> None: - """Remove gateway.pid and processes.json — they reference PIDs in the dead container's process - namespace and would otherwise confuse the newly-started gateway's process-mismatch checks. - """ + """Remove PID-namespace-bound runtime files that would confuse the new gateway's checks.""" for name in _STALE_RUNTIME_FILES: (profile_dir / name).unlink(missing_ok=True) def _read_prior_exit_label(profile_dir: Path) -> str: - """How the profile's previous gateway life ended (clean/unclean/unknown). - - Thin, exception-free wrapper over :func:`gateway.lifecycle_ledger.read_prior_exit_label` — cont- - init runs in a minimal environment and forensics must never block reconciliation (NS-608). - """ + """Exception-free ``lifecycle_ledger.read_prior_exit_label``: forensics never block cont-init.""" try: from gateway.lifecycle_ledger import read_prior_exit_label return read_prior_exit_label(profile_dir) @@ -364,36 +298,34 @@ def _read_prior_exit_label(profile_dir: Path) -> str: return "unknown" +def _write_exec(path: Path, content: str) -> None: + path.write_text(content, encoding="utf-8") + path.chmod(0o755) + + def _register_service(scandir: Path, profile: str, *, start: bool) -> None: """Recreate the s6 service slot for one profile. - Mirrors ``S6ServiceManager.register_profile_gateway`` but sets start state via the ``down`` - marker directly: cont-init.d runs as root before s6-svscan scans the dynamic scandir, so the - manager's ``s6-svscanctl -a`` would fail with no control socket. Built in a sibling temp dir - and ``Path.replace``d into place so an interrupted write never leaves a half-populated dir. + Mirrors ``S6ServiceManager.register_profile_gateway`` but sets start state via the + ``down`` marker directly: cont-init.d runs as root before s6-svscan scans the dynamic + scandir, so the manager's ``s6-svscanctl -a`` would fail with no control socket. Built in + a sibling temp dir and ``Path.replace``d into place so an interrupted write never leaves + a half-populated dir. """ import shutil from hermes_cli.service_manager import ( - S6ServiceManager, - _seed_supervise_skeleton, - validate_profile_name, + S6ServiceManager, _seed_supervise_skeleton, validate_profile_name, ) validate_profile_name(profile) service_dir = scandir / f"gateway-{profile}" - # Dot-prefix the staging dir so s6-svscan skips it while half-built - # (s6-svscan ignores scandir entries whose name starts with "."). - # A non-dotted ``.tmp`` staging name is supervised AS ROOT by any - # concurrent ``s6-svscanctl -a`` rescan the moment it has a valid - # ``type``/``run``, creating a root-owned ``supervise/`` that makes - # ``_seed_supervise_skeleton`` EACCES — see the matching comment in - # ``S6ServiceManager.register_profile_gateway``. The atomic - # ``tmp_dir.replace(service_dir)`` below renames to the dotless live - # name, so the published slot is unchanged. + # Dot-prefix the staging dir so s6-svscan skips it while half-built. A non-dotted staging + # name is supervised AS ROOT by any concurrent ``s6-svscanctl -a`` rescan the moment it + # has a valid ``type``/``run``, creating a root-owned ``supervise/`` that makes + # ``_seed_supervise_skeleton`` EACCES (see ``S6ServiceManager.register_profile_gateway``). tmp_dir = service_dir.with_name("." + service_dir.name + ".tmp") - # Wipe any leftover tmp from a previous interrupted run. if tmp_dir.exists(): shutil.rmtree(tmp_dir, ignore_errors=True) tmp_dir.mkdir(parents=True) @@ -401,49 +333,28 @@ def _register_service(scandir: Path, profile: str, *, start: bool) -> None: try: (tmp_dir / "type").write_text("longrun\n", encoding="utf-8") - # Reuse the manager's run-script rendering — single source of - # truth so register_profile_gateway and reconcile_profile_gateways - # stay consistent. extra_env is empty here; users who need - # per-profile env can set it via the profile's config.yaml - # (which the gateway itself loads). - run = tmp_dir / "run" - run.write_text(S6ServiceManager._render_run_script(profile, extra_env={}), encoding="utf-8") - run.chmod(0o755) + # Reuse the manager's script rendering — single source of truth so both registration + # paths stay consistent. extra_env is empty here; per-profile env comes from the + # profile's config.yaml (which the gateway itself loads). + _write_exec(tmp_dir / "run", S6ServiceManager._render_run_script(profile, extra_env={})) + _write_exec(tmp_dir / "finish", S6ServiceManager._render_finish_script()) - finish = tmp_dir / "finish" - finish.write_text(S6ServiceManager._render_finish_script(), encoding="utf-8") - finish.chmod(0o755) + # Persistent log rotation. + (tmp_dir / "log").mkdir() + _write_exec(tmp_dir / "log" / "run", S6ServiceManager._render_log_run(profile)) - # Persistent log rotation (OQ8-C). - log_subdir = tmp_dir / "log" - log_subdir.mkdir() - log_run = log_subdir / "run" - log_run.write_text(S6ServiceManager._render_log_run(profile), encoding="utf-8") - log_run.chmod(0o755) - - # The presence of a `down` file tells s6-supervise to NOT - # start the service when s6-svscan picks it up. User brings - # it up explicitly with `hermes -p gateway start` - # (which routes through the Phase 4 - # _dispatch_via_service_manager_if_s6 helper to `s6-svc -u`). + # A `down` file tells s6-supervise NOT to start the service when s6-svscan picks it + # up; the user brings it up with `hermes -p gateway start` (→ `s6-svc -u`). if not start: (tmp_dir / "down").touch() - # Pre-create the supervise/ skeleton with hermes ownership - # BEFORE we publish the slot. Mirrors the same pre-creation - # step in S6ServiceManager.register_profile_gateway — when - # s6-svscan picks the published slot up, the s6-supervise it - # spawns will EEXIST our dirs/FIFOs and inherit hermes - # ownership, so runtime s6-svc / s6-svstat / s6-svwait calls - # (all dispatched as the hermes user) won't hit EACCES. See - # ``_seed_supervise_skeleton`` in service_manager.py for the - # full rationale. + # Pre-create supervise/ with hermes ownership BEFORE publishing: the s6-supervise + # spawned on pickup will EEXIST our dirs/FIFOs and inherit that ownership, so runtime + # s6-svc / s6-svstat / s6-svwait calls (dispatched as the hermes user) won't EACCES. _seed_supervise_skeleton(tmp_dir) - # Publish atomically. Path.replace handles the existing-target - # case the same way os.rename does on POSIX: the target is - # silently replaced, so a previous reconcile pass's slot is - # cleanly overwritten in one operation. + # Publish atomically: Path.replace silently replaces an existing target on POSIX, so a + # previous reconcile pass's slot is overwritten in one operation. if service_dir.exists(): shutil.rmtree(service_dir) tmp_dir.replace(service_dir) @@ -452,32 +363,29 @@ def _register_service(scandir: Path, profile: str, *, start: bool) -> None: raise -def _write_reconcile_log( - hermes_home: Path, actions: list[ReconcileAction], -) -> None: +# 256 KiB soft cap on container-boot.log, rotated to .1 when crossed (~3000 lines at ~80 B +# each — about a year of daily reboots on a 5-profile container). Tuned for grep-ability +# more than space (the persistent volume has GB). +_LOG_ROTATE_BYTES = 256 * 1024 + + +def _write_reconcile_log(hermes_home: Path, actions: list[ReconcileAction]) -> None: """Append one line per profile to $HERMES_HOME/logs/container-boot.log. - Operators inspect this to debug "why didn't my profile come back up". Keeping a separate log - file (vs. mixing into agent.log) lets troubleshooters grep for "profile=foo" without wading - through unrelated activity. - - Size-bounded: when the file exceeds ``_LOG_ROTATE_BYTES`` (defaults to 256 KiB ≈ 3000 reconcile - lines), the current file is renamed to ``container-boot.log.1`` (replacing any previous - rotation) before the new entries are appended. + A separate file (vs. agent.log) lets operators debugging "why didn't my profile come back + up" grep for "profile=foo" without unrelated activity. Rotated to ``.1`` (replacing any + previous rotation) before appending once it exceeds ``_LOG_ROTATE_BYTES``. """ import time log_dir = hermes_home / "logs" log_dir.mkdir(parents=True, exist_ok=True) log_path = log_dir / "container-boot.log" - # Rotate before opening to append, so the new entries always land - # in a fresh file when we crossed the threshold last time. try: if log_path.exists() and log_path.stat().st_size >= _LOG_ROTATE_BYTES: log_path.replace(log_dir / "container-boot.log.1") except OSError as exc: - # Rotation failure is non-fatal — keep appending to the - # existing file rather than losing the entry entirely. + # Non-fatal — keep appending rather than lose the entry entirely. log.warning("could not rotate %s: %s", log_path, exc) ts = time.strftime("%Y-%m-%dT%H:%M:%S%z") @@ -489,25 +397,13 @@ def _write_reconcile_log( ) -# 256 KiB soft cap on container-boot.log; rotated to .1 when crossed. -# At ~80 B per reconcile-action line this is ~3000 lines, or about a -# year of daily reboots on a 5-profile container. Two files = ~512 KiB -# worst case. Tuned for visibility (small enough to grep / cat without -# scrolling forever) more than space (the persistent volume has GB). -_LOG_ROTATE_BYTES = 256 * 1024 - - def main() -> int: """Entry point invoked from /etc/cont-init.d/02-reconcile-profiles.""" - # A dashboard-only container never spawns or supervises per-profile - # gateways, so reconciling their s6 slots here is pure waste — and - # actively harmful: when the gateway and dashboard containers share a - # bind-mounted HERMES_HOME, both race to flock() the same s6-log lock - # files under logs/gateways//lock, producing "Resource busy" - # failures and a restart storm. Detect the role from PID 1 argv and - # skip reconciliation in the dashboard container. No operator flag: - # the role is a fact about the container's command, and a flag can be - # forgotten in a hand-written manifest, reintroducing the storm. + # A dashboard-only container never supervises per-profile gateways, and reconciling here + # is actively harmful: when gateway and dashboard containers share a bind-mounted + # HERMES_HOME, both race to flock() the same s6-log lock files under + # logs/gateways//lock → "Resource busy" restart storm. The role is detected from + # PID 1 argv, not an operator flag — a flag can be forgotten in a hand-written manifest. if _is_dashboard_container(_read_container_argv()): print( "reconcile: skipping (dashboard container — does not need " @@ -517,9 +413,7 @@ def main() -> int: hermes_home = Path(os.environ.get("HERMES_HOME", "/opt/data")) scandir = Path(os.environ.get("S6_PROFILE_GATEWAY_SCANDIR", "/run/service")) - actions = reconcile_profile_gateways( - hermes_home=hermes_home, scandir=scandir, - ) + actions = reconcile_profile_gateways(hermes_home=hermes_home, scandir=scandir) for a in actions: print( f"reconcile: profile={a.profile} " From ecbb843cadd57c51450825417072a9ff0bf78396 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:26:45 -0700 Subject: [PATCH 04/29] =?UTF-8?q?refactor(hermes=5Fcli):=20copilot=5Fauth.?= =?UTF-8?q?py=20=E2=80=94=20hoist=20stdlib=20imports,=20lift=20=5Fpost=5Ff?= =?UTF-8?q?orm=20and=20cache-freshness=20helper,=20compact=20rationale=20c?= =?UTF-8?q?omments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/copilot_auth.py | 323 ++++++++++++++----------------------- 1 file changed, 124 insertions(+), 199 deletions(-) diff --git a/hermes_cli/copilot_auth.py b/hermes_cli/copilot_auth.py index aed84419bf..dae90f7358 100644 --- a/hermes_cli/copilot_auth.py +++ b/hermes_cli/copilot_auth.py @@ -6,13 +6,17 @@ GH_TOKEN env var 3. GITHUB_TOKEN env var 4. gh auth token CLI fallback from __future__ import annotations +import hashlib import json import logging import os +import re import shutil import subprocess import threading import time +import urllib.parse +import urllib.request from pathlib import Path from typing import Optional @@ -20,13 +24,10 @@ from hermes_cli._subprocess_compat import IS_WINDOWS, windows_hide_flags logger = logging.getLogger(__name__) -# OAuth device code flow constants — VS Code's GitHub App client ID. -# The previous opencode OAuth App ID (Ov23li8tweQw6odWQebz) produces gho_* -# tokens that cannot be exchanged for Copilot API JWTs (404 on -# /copilot_internal/v2/token). VS Code's App ID produces ghu_* tokens -# that support exchange, which is required to access internal-only models -# (e.g. claude-opus-4.6-1m) and enterprise endpoints. -# Tested on Individual and Enterprise accounts. +# OAuth device code flow — VS Code's GitHub App client ID. The opencode OAuth App ID +# (Ov23li8tweQw6odWQebz) produces gho_* tokens that cannot be exchanged for Copilot API JWTs +# (404 on /copilot_internal/v2/token); VS Code's produces ghu_* tokens that support exchange, +# required for internal-only models and enterprise endpoints. Tested on Individual + Enterprise. COPILOT_OAUTH_CLIENT_ID = "Iv1.b507a08c87ecfe98" # ghp_ classic PATs are rejected by the Copilot API (gho_ / github_pat_ / ghu_ work). _CLASSIC_PAT_PREFIX = "ghp_" @@ -58,12 +59,10 @@ def validate_copilot_token(token: str) -> tuple[bool, str]: def resolve_copilot_token() -> tuple[str, str]: - """Resolve a GitHub token suitable for Copilot API use. + """Resolve a GitHub token suitable for Copilot API use → (token, source); ("", "") if none. - Returns (token, source) where source describes where the token came from. Raises ValueError if - only a classic PAT is available. + Raises ValueError if only a classic PAT is available. """ - # 1. Check env vars in priority order any_env_var_set = False for env_var in COPILOT_ENV_VARS: val = os.getenv(env_var, "").strip() @@ -71,21 +70,14 @@ def resolve_copilot_token() -> tuple[str, str]: any_env_var_set = True valid, msg = validate_copilot_token(val) if not valid: - logger.warning( - "Token from %s is not supported: %s", env_var, msg - ) + logger.warning("Token from %s is not supported: %s", env_var, msg) continue return val, env_var - # 2. Fall back to gh auth token — but ONLY when no Copilot env var was - # explicitly set. When the user exported GITHUB_TOKEN (even an - # unsupported classic PAT), their intent is to use *that* token, not - # to silently substitute one from the gh CLI credential store. - # Skipping the subprocess here also avoids a slow `gh auth token` - # call (up to 5s timeout on Windows) on every cold start that scans - # Copilot auth state — a measurable contributor to the ~14s - # cold-start stall (#60800). The user can run `copilot login` or - # set a supported token (gho_*/github_pat_*/ghu_) explicitly. + # Fall back to gh auth token ONLY when no Copilot env var was explicitly set: an exported + # GITHUB_TOKEN (even an unsupported classic PAT) means the user intends *that* token, not + # one silently substituted from the gh credential store. Skipping the subprocess also + # avoids a slow `gh auth token` call (up to 5s on Windows) on every cold start. if any_env_var_set: logger.debug( "Copilot env var(s) set but none held a supported token; " @@ -98,37 +90,32 @@ def resolve_copilot_token() -> tuple[str, str]: if token: valid, msg = validate_copilot_token(token) if not valid: - raise ValueError( - f"Token from `gh auth token` is a classic PAT (ghp_*). {msg}" - ) + raise ValueError(f"Token from `gh auth token` is a classic PAT (ghp_*). {msg}") return token, "gh auth token" return "", "" def _gh_cli_candidates() -> list[str]: - """Return candidate ``gh`` binary paths, including common Homebrew installs.""" + """Candidate ``gh`` binary paths, including common Homebrew installs.""" candidates: list[str] = [c for c in (shutil.which("gh"),) if c] for candidate in ( "/opt/homebrew/bin/gh", "/usr/local/bin/gh", str(Path.home() / ".local" / "bin" / "gh"), ): - if candidate not in candidates and os.path.isfile(candidate) and os.access(candidate, os.X_OK): + if (candidate not in candidates and os.path.isfile(candidate) + and os.access(candidate, os.X_OK)): candidates.append(candidate) return candidates -# ``gh auth token`` result cache. The probe shells out to the gh CLI, and when -# gh has no credential store for this HOME (fresh profile, desktop-spawned -# backend, CI) it can block for its full 5s subprocess timeout — on keyring / -# D-Bus prompts rather than returning immediately. Provider inventory builds -# (``/api/model/options``, ``hermes tools``) probe Copilot auth several times -# per request, so an uncached miss turns one settings-page load into a 4×5s -# stall that exceeds the Desktop renderer's 15s IPC budget and paints an error -# (observed Aug 2026: Models/Providers settings pages timing out on every -# open). Successes and failures are both cached; a short TTL keeps a freshly -# run ``gh auth login`` discoverable without restarting the backend. +# ``gh auth token`` result cache. When gh has no credential store for this HOME (fresh +# profile, desktop-spawned backend, CI) the probe can block for its full 5s timeout on +# keyring / D-Bus prompts. Provider inventory builds (``/api/model/options``, ``hermes tools``) +# probe Copilot auth several times per request, so an uncached miss turned one settings-page +# load into a 4×5s stall exceeding the Desktop renderer's 15s IPC budget. Successes and +# failures are both cached; a short TTL keeps a fresh ``gh auth login`` discoverable. _GH_CLI_TOKEN_CACHE_TTL_SECONDS = 300.0 _gh_cli_token_cache: tuple[float, Optional[str]] | None = None @@ -140,17 +127,13 @@ def _invalidate_gh_cli_token_cache() -> None: def _try_gh_cli_token() -> Optional[str]: - """Return a token from ``gh auth token`` when the GitHub CLI is available. - - The result (including a miss) is cached for a short TTL — see the cache comment above. Callers - that just re-authenticated can call ``_invalidate_gh_cli_token_cache()`` to re-probe - immediately. - """ + """Token from ``gh auth token`` when available; the result (incl. a miss) is cached per TTL.""" global _gh_cli_token_cache now = time.monotonic() - if _gh_cli_token_cache is not None and now - _gh_cli_token_cache[0] < _GH_CLI_TOKEN_CACHE_TTL_SECONDS: - return _gh_cli_token_cache[1] + cache = _gh_cli_token_cache + if cache is not None and now - cache[0] < _GH_CLI_TOKEN_CACHE_TTL_SECONDS: + return cache[1] token = _probe_gh_cli_token() _gh_cli_token_cache = (now, token) @@ -161,10 +144,9 @@ def _probe_gh_cli_token() -> Optional[str]: """Uncached ``gh auth token`` subprocess probe (see ``_try_gh_cli_token``).""" hostname = os.getenv("COPILOT_GH_HOST", "").strip() - # Build a clean env so gh doesn't short-circuit on GITHUB_TOKEN / GH_TOKEN - clean_env = {k: v for k, v in os.environ.items() - if k not in {"GITHUB_TOKEN", "GH_TOKEN"}} - # Never let gh open an interactive prompt from a backend process. + # Clean env so gh doesn't short-circuit on GITHUB_TOKEN / GH_TOKEN, and never let it open + # an interactive prompt from a backend process. + clean_env = {k: v for k, v in os.environ.items() if k not in {"GITHUB_TOKEN", "GH_TOKEN"}} clean_env.setdefault("GH_PROMPT_DISABLED", "1") clean_env.setdefault("GH_NO_UPDATE_NOTIFIER", "1") @@ -175,13 +157,8 @@ def _probe_gh_cli_token() -> Optional[str]: cmd += ["--hostname", hostname] try: result = subprocess.run( - cmd, - capture_output=True, - text=True, encoding='utf-8', errors='replace', - timeout=5, - env=clean_env, - stdin=subprocess.DEVNULL, - **_popen_kwargs, + cmd, capture_output=True, text=True, encoding='utf-8', errors='replace', + timeout=5, env=clean_env, stdin=subprocess.DEVNULL, **_popen_kwargs, ) except (FileNotFoundError, subprocess.TimeoutExpired) as exc: logger.debug("gh CLI token lookup failed (%s): %s", gh_path, exc) @@ -198,33 +175,29 @@ _DEVICE_CODE_TERMINAL_ERRORS = { "access_denied": " ✗ Authorization was denied.", } + +def _post_form(url: str, fields: dict, timeout: float) -> dict: + req = urllib.request.Request( + url, + data=urllib.parse.urlencode(fields).encode(), + headers={ + "Accept": "application/json", + "Content-Type": "application/x-www-form-urlencoded", + "User-Agent": "HermesAgent/1.0", + }, + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode()) + + def copilot_device_code_login( - *, - host: str = "github.com", - timeout_seconds: float = 300, + *, host: str = "github.com", timeout_seconds: float = 300, ) -> Optional[str]: """Run the GitHub OAuth device code flow for Copilot.""" - import urllib.request - import urllib.parse - domain = host.rstrip("/") device_code_url = f"https://{domain}/login/device/code" access_token_url = f"https://{domain}/login/oauth/access_token" - def _post_form(url: str, fields: dict, timeout: float) -> dict: - req = urllib.request.Request( - url, - data=urllib.parse.urlencode(fields).encode(), - headers={ - "Accept": "application/json", - "Content-Type": "application/x-www-form-urlencoded", - "User-Agent": "HermesAgent/1.0", - }, - ) - with urllib.request.urlopen(req, timeout=timeout) as resp: - return json.loads(resp.read().decode()) - - # Step 1: Request device code try: device_data = _post_form( device_code_url, {"client_id": COPILOT_OAUTH_CLIENT_ID, "scope": "read:user"}, 15 @@ -243,14 +216,12 @@ def copilot_device_code_login( print(" ✗ GitHub did not return a device code.") return None - # Step 2: Show instructions print() print(f" Open this URL in your browser: {verification_uri}") print(f" Enter this code: {user_code}") print() print(" Waiting for authorization...", end="", flush=True) - # Step 3: Poll for completion deadline = time.monotonic() + timeout_seconds while time.monotonic() < deadline: @@ -297,8 +268,8 @@ def copilot_device_code_login( # ─── Copilot Token Exchange ──────────────────────────────────────────────── -# Module-level cache for exchanged Copilot API tokens. -# Maps raw_token_fingerprint -> (api_token, expires_at_epoch, base_url). +# In-process cache of exchanged Copilot API tokens: +# raw_token_fingerprint -> (api_token, expires_at_epoch, base_url). _jwt_cache: dict[str, tuple[str, float, Optional[str]]] = {} _JWT_REFRESH_MARGIN_SECONDS = 120 # refresh 2 min before expiry @@ -307,32 +278,27 @@ _TOKEN_EXCHANGE_URL = "https://api.github.com/copilot_internal/v2/token" _EDITOR_VERSION = "vscode/1.104.1" _EXCHANGE_USER_AGENT = "GitHubCopilotChat/0.26.7" -# Transient-failure hardening for the token exchange. Gateway startup often -# races network readiness (launchd relaunch, DHCP/VPN settling); a single-shot -# exchange that fails there silently degrades to the RAW GitHub token, which the -# Copilot server routes to the "copilot-language-server" integrator whose model -# allowlist omits enterprise-only models (e.g. claude-opus-4.8) → HTTP 400 on -# every turn until the next restart. Retry a few times, and persist the last -# good exchanged JWT to disk so a restart during a blip reuses the still-valid -# ~30-min token instead of degrading. +# Transient-failure hardening. Gateway startup often races network readiness (launchd +# relaunch, DHCP/VPN settling); a single-shot exchange that fails there silently degrades to +# the RAW GitHub token, which the Copilot server routes to the "copilot-language-server" +# integrator whose model allowlist omits enterprise-only models → HTTP 400 on every turn +# until the next restart. Retry a few times, and persist the last good exchanged JWT so a +# restart during a blip reuses the still-valid ~30-min token instead of degrading. _EXCHANGE_MAX_ATTEMPTS = 3 _EXCHANGE_BACKOFF_BASE_SECONDS = 1.5 # sleeps ~1.5s, ~3.0s between attempts _JWT_DISK_FILENAME = ".copilot_jwt.json" _JWT_DISK_MAX_BYTES = 1_048_576 # 1 MiB cap on the persisted JWT store read -# Negative cache for failed exchanges. Without it, every load_pool("copilot") -# call re-runs the full exchange — and on a permanently-rejected token -# (HTTP 403: account not Copilot-entitled, expired grant, org policy) the -# retry backoff burned ~4.5s of time.sleep() on EVERY provider-discovery -# pass. The /model picker, delegation child spawns, and the web dashboard -# all walk that path, so a single bad Copilot token made all of them crawl. -# Maps raw-token fingerprint -> epoch until which exchange attempts are -# skipped (raise immediately). Success clears the entry. +# Negative cache for failed exchanges: raw-token fingerprint -> epoch until which attempts +# raise immediately (success clears the entry). Without it every load_pool("copilot") re-ran +# the full exchange, and on a permanently-rejected token (403: not Copilot-entitled, expired +# grant, org policy) the retry backoff burned ~4.5s of sleep on EVERY provider-discovery pass +# (/model picker, delegation child spawns, web dashboard). _exchange_failure_cache: dict[str, float] = {} -# Single-flight guard per token fingerprint: concurrent callers (the dashboard -# polls /api/credentials/pool every few seconds, each poll off-loop) wait on -# the ONE in-flight exchange and then hit the positive/negative cache, instead -# of each spawning their own hung resolver thread during a DNS outage. +# Single-flight guard per token fingerprint: concurrent callers (the dashboard polls +# /api/credentials/pool every few seconds, each poll off-loop) wait on the ONE in-flight +# exchange and then hit the positive/negative cache, instead of each spawning their own hung +# resolver thread during a DNS outage. _exchange_locks: dict[str, threading.Lock] = {} _exchange_locks_guard = threading.Lock() @@ -343,26 +309,26 @@ def _exchange_lock_for(fp: str) -> threading.Lock: if lock is None: lock = _exchange_locks[fp] = threading.Lock() return lock + + _EXCHANGE_FAILURE_TTL_TRANSIENT_SECONDS = 60.0 # network blips: retry soon _EXCHANGE_FAILURE_TTL_PERMANENT_SECONDS = 1800.0 # 401/403/404: won't heal -# HTTP statuses that indicate the token itself is rejected — retrying with -# backoff is pointless (the retry loop exists for startup network races, -# not for auth rejections) and sleeping on them just blocks the caller. +# HTTP statuses meaning the token itself is rejected — retrying with backoff is pointless +# (the retry loop exists for startup network races) and sleeping on them just blocks the caller. _EXCHANGE_PERMANENT_HTTP_STATUSES = frozenset({401, 403, 404}) def _token_fingerprint(raw_token: str) -> str: """Short fingerprint of a raw token for cache keying (avoids storing full token).""" - import hashlib return hashlib.sha256(raw_token.encode()).hexdigest()[:16] def _read_jwt_store(path: Path) -> Optional[dict]: """Bounded read of the on-disk JWT store → dict, or None if missing/unusable. - Single chokepoint for every read of the persisted store (load, eviction, save-merge). A well- - formed store is a few KB; a file over the 1 MiB cap or with non-dict content is treated as - unusable so a corrupt/oversized file can't balloon memory or get rewritten back out. + Single chokepoint for every read (load, eviction, save-merge). A well-formed store is a + few KB; a file over the 1 MiB cap or with non-dict content is unusable so a corrupt or + oversized file can't balloon memory or get rewritten back out. """ if not path.exists(): return None @@ -393,17 +359,15 @@ def _write_jwt_store(path: Path, store: dict) -> None: def evict_cached_exchanged_token(raw_token: str) -> None: """Drop any cached exchanged JWT for ``raw_token`` (in-process + on-disk). - Used by the runtime stale-credential recovery path: when a live request starts failing with a - Copilot ``model_not_available_for_integrator`` / ``model_not_supported`` 400, the cached - exchanged token (or a degraded raw fallback that was cached in its place) is stale. + Used by the runtime stale-credential recovery path when a live request starts failing with + a Copilot ``model_not_available_for_integrator`` / ``model_not_supported`` 400. """ if not raw_token: return fp = _token_fingerprint(raw_token) _jwt_cache.pop(fp, None) - # Also clear any negative-cache entry: eviction is an explicit "force a - # fresh exchange" signal from the stale-credential recovery path, so the - # next exchange_copilot_token() must be allowed to hit the network. + # Eviction is an explicit "force a fresh exchange" signal, so the negative-cache entry + # must go too — the next exchange_copilot_token() must be allowed to hit the network. _exchange_failure_cache.pop(fp, None) path = _jwt_disk_path() if not path: @@ -427,11 +391,7 @@ def _jwt_disk_path() -> Optional[Path]: def _load_jwt_from_disk(fp: str) -> Optional[tuple[str, float, Optional[str]]]: - """Load a persisted exchanged JWT for ``fp`` → (api_token, expires_at, base_url). - - An oversized/corrupt store is treated as unusable — the caller re-exchanges (bound shared with - eviction/save via _read_jwt_store). - """ + """Persisted exchanged JWT for ``fp`` → (api_token, expires_at, base_url), or None.""" path = _jwt_disk_path() if not path: return None @@ -448,9 +408,7 @@ def _load_jwt_from_disk(fp: str) -> Optional[tuple[str, float, Optional[str]]]: return None -def _save_jwt_to_disk( - fp: str, api_token: str, expires_at: float, base_url: Optional[str] -) -> None: +def _save_jwt_to_disk(fp: str, api_token: str, expires_at: float, base_url: Optional[str]) -> None: """Persist an exchanged JWT (0o600), pruning expired entries.""" path = _jwt_disk_path() if not path: @@ -462,33 +420,25 @@ def _save_jwt_to_disk( for k, v in (_read_jwt_store(path) or {}).items() if isinstance(v, dict) and float(v.get("expires_at", 0) or 0) > now } - store[fp] = { - "api_token": api_token, - "expires_at": expires_at, - "base_url": base_url, - } + store[fp] = {"api_token": api_token, "expires_at": expires_at, "base_url": base_url} _write_jwt_store(path, store) except Exception as exc: logger.debug("Failed to persist Copilot JWT: %s", exc) -# Hard wall-clock cap for the token-exchange HTTP call. urllib's ``timeout`` -# only bounds socket operations AFTER DNS resolution succeeds; getaddrinfo -# blocks in C and ignores it entirely, so on a networkless Windows host the -# resolver can hang for many minutes (observed: a 17-minute event-loop stall -# on 2026-08-22 that took the whole backend down with it). +# Hard wall-clock cap for the token-exchange HTTP call. urllib's ``timeout`` only bounds +# socket operations AFTER DNS resolution succeeds; getaddrinfo blocks in C and ignores it, +# so on a networkless Windows host the resolver can hang for many minutes (observed: a +# 17-minute event-loop stall that took the whole backend down). _DNS_GRACE_SECONDS = 5.0 def _urlopen_bounded(req, timeout: float): """urlopen() with a hard wall-clock cap of timeout + _DNS_GRACE_SECONDS. - Runs the call on a daemon thread and abandons it if the cap fires, so a - DNS/getaddrinfo hang cannot block the caller indefinitely. Raises the - worker's exception, or TimeoutError when the cap fires. + Runs the call on a daemon thread and abandons it if the cap fires, so a DNS/getaddrinfo + hang cannot block the caller. Raises the worker's exception, or TimeoutError on the cap. """ - import urllib.request - box: dict = {} abandoned = threading.Event() @@ -499,8 +449,7 @@ def _urlopen_bounded(req, timeout: float): box["exc"] = exc return if abandoned.is_set(): - # The caller already timed out; nobody will read this response, - # so release its socket instead of leaking it with the thread. + # Caller already timed out; nobody will read this response — release its socket. try: resp.close() except Exception: @@ -508,9 +457,7 @@ def _urlopen_bounded(req, timeout: float): return box["resp"] = resp - t = threading.Thread( - target=_worker, name="copilot-token-exchange", daemon=True - ) + t = threading.Thread(target=_worker, name="copilot-token-exchange", daemon=True) t.start() t.join(timeout + _DNS_GRACE_SECONDS) if t.is_alive(): @@ -526,19 +473,25 @@ def _urlopen_bounded(req, timeout: float): return box["resp"] -def exchange_copilot_token(raw_token: str, *, timeout: float = 10.0) -> tuple[str, float, Optional[str]]: - """Exchange a raw GitHub token for a short-lived Copilot API token. +def _cache_entry_fresh(cached) -> bool: + return bool(cached) and time.time() < cached[1] - _JWT_REFRESH_MARGIN_SECONDS - The token is a semicolon-separated string (not a JWT) used as a Bearer token. ``base_url`` is - the account-specific host: the exchange's ``endpoints.api`` (enterprise/proxied accounts), else - derived from the token's ``proxy-ep``; individual accounts have neither, so it is None. Cached - in-process until close to expiry. Raises ``ValueError`` on failure. + +def exchange_copilot_token( + raw_token: str, *, timeout: float = 10.0, +) -> tuple[str, float, Optional[str]]: + """Exchange a raw GitHub token for a Copilot API token → (token, expires_at, base_url). + + The token is a semicolon-separated string (not a JWT) used as a Bearer token. ``base_url`` + is the account-specific host: the exchange's ``endpoints.api`` (enterprise/proxied + accounts), else derived from the token's ``proxy-ep``; individual accounts have neither, + so it is None. Cached in-process until close to expiry. Raises ``ValueError`` on failure. """ fp = _token_fingerprint(raw_token) # Fast path outside the lock: a valid in-process JWT needs no exchange. cached = _jwt_cache.get(fp) - if cached and time.time() < cached[1] - _JWT_REFRESH_MARGIN_SECONDS: + if _cache_entry_fresh(cached): return cached with _exchange_lock_for(fp): @@ -546,26 +499,20 @@ def exchange_copilot_token(raw_token: str, *, timeout: float = 10.0) -> tuple[st def _exchange_copilot_token_locked( - raw_token: str, fp: str, *, timeout: float + raw_token: str, fp: str, *, timeout: float, ) -> tuple[str, float, Optional[str]]: - import urllib.request - - # Re-check the in-process cache under the lock (a concurrent caller may have - # just completed the exchange we were queued behind), then the on-disk - # cache: a fresh process (e.g. gateway restart) has an empty in-process - # cache but may have a still-valid persisted JWT. Reusing it avoids a - # network round-trip at startup — precisely when the network is most likely - # to be flaky and the single-shot exchange would degrade to the raw token. + # Re-check the in-process cache under the lock (a concurrent caller may have just completed + # the exchange we were queued behind), then the on-disk cache: a fresh process (gateway + # restart) has an empty in-process cache but may hold a still-valid persisted JWT, and + # reusing it avoids a network round-trip precisely when the network is most likely flaky. for lookup in (_jwt_cache.get, _load_jwt_from_disk): cached = lookup(fp) - if cached and time.time() < cached[1] - _JWT_REFRESH_MARGIN_SECONDS: + if _cache_entry_fresh(cached): _jwt_cache[fp] = cached return cached - # Negative cache: a recent exchange failure for this token means the - # network round-trip (and its retry backoff) would just repeat. Fail - # fast so provider discovery / picker opens don't block on a token we - # already know is rejected or unreachable. + # Negative cache: fail fast so provider discovery / picker opens don't block on a token + # we already know is rejected or unreachable. _fail_until = _exchange_failure_cache.get(fp, 0.0) if time.time() < _fail_until: raise ValueError( @@ -584,13 +531,9 @@ def _exchange_copilot_token_locked( }, ) - # Retry with backoff. Startup network races (launchd relaunch, VPN/DHCP - # settling) make the first attempt flaky; without this the sole failure - # silently degrades to the raw token for the whole process lifetime. - # Permanent HTTP rejections (401/403/404 — token not Copilot-entitled, - # revoked, or org-blocked) skip the retry loop entirely: backoff exists - # for transient network races, and sleeping on an auth rejection just - # blocks the caller for ~4.5s with an identical outcome. + # Retry with backoff for startup network races; permanent HTTP rejections (401/403/404) + # skip the loop entirely — sleeping on an auth rejection blocks the caller for ~4.5s with + # an identical outcome. data = None last_exc: Optional[Exception] = None permanent_failure = False @@ -631,12 +574,10 @@ def _exchange_copilot_token_locked( expires_at = data.get("expires_at", 0) expires_at = float(expires_at) if expires_at else time.time() + 1800 - # Resolve the account-specific API base URL. GitHub advertises the - # authoritative endpoint under ``endpoints.api`` in the exchange response - # (it differs for Copilot Enterprise / proxied accounts). When the - # response omits it, fall back to deriving the host from the ``proxy-ep`` - # field embedded in the exchanged token. Individual accounts have neither, - # so ``base_url`` stays None and callers use the registry default. + # Account-specific API base URL: GitHub advertises the authoritative endpoint under + # ``endpoints.api`` (differs for Copilot Enterprise / proxied accounts); when omitted, + # derive the host from the ``proxy-ep`` field embedded in the exchanged token. Individual + # accounts have neither, so ``base_url`` stays None and callers use the registry default. endpoints = data.get("endpoints") base_url: Optional[str] = ( str(endpoints.get("api") or "").strip().rstrip("/") if isinstance(endpoints, dict) else "" @@ -644,44 +585,30 @@ def _exchange_copilot_token_locked( _jwt_cache[fp] = (api_token, expires_at, base_url) _save_jwt_to_disk(fp, api_token, expires_at, base_url) - logger.debug( - "Copilot token exchanged, expires_at=%s, base_url=%s", - expires_at, - base_url, - ) + logger.debug("Copilot token exchanged, expires_at=%s, base_url=%s", expires_at, base_url) return api_token, expires_at, base_url def _derive_base_url_from_proxy_ep(token: str) -> Optional[str]: - """Derive the Copilot API base URL from a proxy-ep field in the token. + """Copilot API base URL from the token's ``proxy-ep`` (``proxy.`` host → ``api.``), or None. - The token looks like ``tid=…;exp=…;proxy-ep=proxy.enterprise.githubcopilot.com;…``; the API host - is that value with the leading ``proxy.`` replaced by ``api.``. None if proxy-ep is absent. + The token looks like ``tid=…;exp=…;proxy-ep=proxy.enterprise.githubcopilot.com;…``. """ - import re m = re.search(r'(?:^|;)\s*proxy-ep=([^;\s]+)', token) if not m: return None - # Strip scheme if present proxy_ep = re.sub(r"^https?://", "", m.group(1), count=1).rstrip("/") - - # Replace leading "proxy." with "api." if proxy_ep.startswith("proxy."): proxy_ep = "api." + proxy_ep[len("proxy."):] return f"https://{proxy_ep}" def get_copilot_api_token(raw_token: str) -> tuple[str, Optional[str]]: - """Exchange a raw GitHub token for a Copilot API token, with fallback. + """``(api_token, base_url)`` from the exchange, or ``(raw_token, None)`` when it fails. - Convenience wrapper: returns ``(api_token, base_url)`` on success, or ``(raw_token, None)`` if - the exchange fails (e.g. network error, unsupported account type). This preserves existing - behaviour for accounts that don't need exchange while enabling access to internal-only models - for those that do. - - ``base_url`` is the account-specific API endpoint advertised by the exchange (``endpoints.api``, - with a ``proxy-ep`` fallback), or None for individual accounts. + The fallback preserves behaviour for accounts that don't need exchange (network error, + unsupported account type) while enabling internal-only models for those that do. """ if not raw_token: return raw_token, None @@ -696,9 +623,7 @@ def get_copilot_api_token(raw_token: str) -> tuple[str, Optional[str]]: # ─── Copilot API Headers ─────────────────────────────────────────────────── def copilot_request_headers( - *, - is_agent_turn: bool = True, - is_vision: bool = False, + *, is_agent_turn: bool = True, is_vision: bool = False, ) -> dict[str, str]: """Build the standard headers for Copilot API requests.""" headers: dict[str, str] = { From 268ccf8616fe388314ee697e7b2c820beff779cb Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:30:38 -0700 Subject: [PATCH 05/29] =?UTF-8?q?refactor(hermes=5Fcli):=20dashboard=5Fpro?= =?UTF-8?q?cs.py=20=E2=80=94=20shared=20ps=20run=20kwargs,=20platform=20ki?= =?UTF-8?q?ll=20dispatch,=20hoist=20=5Fpid=5Fexists=20import,=20compact=20?= =?UTF-8?q?docstrings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/dashboard_procs.py | 248 +++++++++++++++------------------- 1 file changed, 112 insertions(+), 136 deletions(-) diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py index f0bce19595..8f04b8a802 100644 --- a/hermes_cli/dashboard_procs.py +++ b/hermes_cli/dashboard_procs.py @@ -1,8 +1,7 @@ """Dashboard process-hygiene helpers — extracted from ``hermes_cli/main.py``. -Helpers that STAY in ``hermes_cli.main`` are reached through the lazy ``_m()`` -reference so monkeypatches on ``hermes_cli.main.`` keep working and -imports stay one-way (main.py imports this module, never the reverse). +Helpers that STAY in ``hermes_cli.main`` are reached through the lazy ``_m()`` reference so +monkeypatches on ``hermes_cli.main.`` keep working and imports stay one-way. """ import os @@ -10,14 +9,15 @@ import subprocess import sys from pathlib import Path -# Cmdline substrings identifying the long-lived server. ``hermes serve`` is the -# same server under the headless name the desktop app spawns; it is reaped on -# update for the same frontend/backend-mismatch reason as ``dashboard``. +# Cmdline substrings identifying the long-lived server. ``hermes serve`` is the same server +# under the headless name the desktop app spawns; it is reaped on update for the same +# frontend/backend-mismatch reason as ``dashboard``. _DASHBOARD_PATTERNS = tuple( f"{launcher} {cmd}" for cmd in ("dashboard", "serve") for launcher in ("hermes", "hermes_cli.main", "hermes_cli/main.py") ) +_PS_RUN_KWARGS = dict(capture_output=True, text=True, encoding="utf-8", errors="replace") def _m(): @@ -34,13 +34,11 @@ def _empty_result() -> dict[str, list]: def _scan_dashboard_processes(*, exclude_pids: set[int] | None = None) -> list[tuple[int, str]]: """Return matching ``dashboard``/``serve`` processes with their cmdlines. - ``hermes update`` swaps files on disk while a forgotten dashboard keeps the - old Python backend in memory against the new JS bundle — a silent mismatch - (new auth headers → every API call 401s). *exclude_pids* must never be - returned: Hermes Desktop sets ``HERMES_DESKTOP_CHILD_PID`` on the backend - it spawns so an auto-update never kills the backend it manages itself. - - Returns an empty list on any scan error (missing ps/wmic, timeout, etc.). + ``hermes update`` swaps files on disk while a forgotten dashboard keeps the old Python + backend in memory against the new JS bundle — a silent mismatch (new auth headers → every + API call 401s). *exclude_pids* must never be returned: Hermes Desktop sets + ``HERMES_DESKTOP_CHILD_PID`` on the backend it spawns so an auto-update never kills the + backend it manages itself. Empty list on any scan error (missing ps/wmic, timeout, ...). """ self_pid = os.getpid() found: list[tuple[int, str]] = [] @@ -51,11 +49,10 @@ def _scan_dashboard_processes(*, exclude_pids: set[int] | None = None) -> list[t try: if sys.platform == "win32": - # errors="ignore": wmic may emit the system code page; a decode - # error would leave stdout=None. bounded_probe_run (not run()): - # run()'s post-timeout cleanup joins pipe readers unbounded and a - # conhost descendant holding duplicated handles wedges it forever. - # It also passes CREATE_NO_WINDOW for the pythonw.exe backend. + # errors="ignore": wmic may emit the system code page; a decode error would leave + # stdout=None. bounded_probe_run (not run()): run()'s post-timeout cleanup joins + # pipe readers unbounded and a conhost descendant holding duplicated handles + # wedges it forever. It also passes CREATE_NO_WINDOW for the pythonw.exe backend. from hermes_cli._subprocess_compat import bounded_probe_run result = bounded_probe_run( @@ -77,12 +74,9 @@ def _scan_dashboard_processes(*, exclude_pids: set[int] | None = None) -> list[t pass else: # ps (not `pgrep -f "hermes.*dashboard"`) keeps us consistent with - # gateway._scan_gateway_pids and avoids a greedy regex matching - # unrelated cmdlines that merely contain both words. - result = subprocess.run( - ["ps", "-A", "-o", "pid=,command="], - capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10, - ) + # gateway._scan_gateway_pids and avoids a greedy regex matching unrelated + # cmdlines that merely contain both words. + result = subprocess.run(["ps", "-A", "-o", "pid=,command="], timeout=10, **_PS_RUN_KWARGS) if result.returncode == 0: for line in getattr(result, "stdout", "").split("\n"): parts = line.strip().split(None, 1) @@ -99,9 +93,9 @@ def _scan_dashboard_processes(*, exclude_pids: set[int] | None = None) -> list[t found = [proc for proc in found if proc[0] not in exclude_pids] # Spawn-ledger augmentation: substring patterns miss profiled launches - # (`hermes --profile p serve ...`). Every serve/dashboard registers itself - # in the spawn ledger with live-verified (pid, create_time) — positive - # identity. Add entries the scan missed, preferring the ledger's full argv. + # (`hermes --profile p serve ...`). Every serve/dashboard registers itself in the spawn + # ledger with live-verified (pid, create_time) — positive identity. Add entries the scan + # missed, preferring the ledger's full argv. try: from hermes_cli.process_identity import ledger_entries @@ -163,11 +157,10 @@ def _profile_flag_value(argv: list[str]) -> str | None: def _is_ephemeral_port_zero_backend(argv: list[str]) -> bool: """True for Desktop-style ``serve|dashboard --port 0`` backends. - Ephemeral-port backends are owned by Hermes Desktop (or are PPID-1 orphans - of a prior update respawn). Replaying them after ``hermes update`` - multiplies listening backends because ``--port 0`` always binds a fresh - port. Covers both ``serve`` and the legacy ``dashboard --no-open`` - fallback older Desktop runtimes use. + Ephemeral-port backends are owned by Hermes Desktop (or are PPID-1 orphans of a prior + update respawn). Replaying them after ``hermes update`` multiplies listening backends + because ``--port 0`` always binds a fresh port. Covers ``serve`` and the legacy + ``dashboard --no-open`` fallback older Desktop runtimes use. """ if _dashboard_subcommand_index(argv) is None: return False @@ -206,17 +199,16 @@ def _resolved_home(home: str) -> Path: def _normalized_home_for_compare(home: str) -> str: - """Install-identity key for *home*: symlinked / differently-spelled roots - compare equal (same normalization as ``home:`` respawn keys).""" + """Install-identity key for *home*: symlinked / differently-spelled roots compare equal.""" return os.path.normcase(str(_resolved_home(home))) def _profile_key_for_respawn(argv: list[str], hermes_home: str | None = None) -> str: """Stable owner key: ``HERMES_HOME`` when known, else ``--profile`` / ``-p``. - ``HERMES_HOME`` ending in ``profiles/`` → ``profile:`` so it - shares a cap with an explicit ``--profile``; other homes keep a resolved - ``home:`` key so unrelated installs never collapse together. + ``HERMES_HOME`` ending in ``profiles/`` → ``profile:`` so it shares a cap with + an explicit ``--profile``; other homes keep a resolved ``home:`` key so unrelated installs + never collapse together. """ if hermes_home: parts = _resolved_home(hermes_home).parts @@ -233,19 +225,18 @@ def _filter_dashboard_respawn_candidates( ) -> list[list[str]]: """Select which killed manual backends to respawn after ``hermes update``. - Candidates are ``(pid, argv, hermes_home)``; *own_home* (default - ``get_hermes_home()``) is a parameter so tests can pin it. Rules: - 1. Never resurrect Desktop ephemeral ``--port 0`` backends — Desktop owns - their lifecycle; they are the PPID-1 orphans that multiplied across updates. - 2. Never replay a backend from a **foreign** ``HERMES_HOME``: the respawn - is argv-only (no ``env=``), so it would come back on the *updating* - install's home and steal the foreign install's fixed port, leaving its - supervisor to crash-loop on ``EADDRINUSE``. Unreadable (``None``) stays eligible. + Candidates are ``(pid, argv, hermes_home)``; *own_home* (default ``get_hermes_home()``) + is a parameter so tests can pin it. Rules: + 1. Never resurrect Desktop ephemeral ``--port 0`` backends — Desktop owns their lifecycle. + 2. Never replay a backend from a **foreign** ``HERMES_HOME``: the respawn is argv-only + (no ``env=``), so it would come back on the *updating* install's home and steal the + foreign install's fixed port, leaving its supervisor to crash-loop on ``EADDRINUSE``. + Unreadable (``None``) stays eligible. 3. Dedupe by normalized cmdline. 4. At most one backend per profile / home. Does **not** blanket-skip PPID-1: a prior update respawn detaches with - ``start_new_session=True``, so fixed-port manual backends sit under init - and must stay eligible next update. + ``start_new_session=True``, so fixed-port manual backends sit under init and must stay + eligible next update. """ if own_home is None: try: @@ -277,11 +268,7 @@ def _filter_dashboard_respawn_candidates( def _exclude_pids_from_env() -> set[int]: - """PIDs Desktop marks as live backends (``HERMES_DESKTOP_CHILD_PID``). - - Desktop may manage several backends (one per active profile) and passes - them comma-separated; a lone int still parses for back-compat. - """ + """PIDs Desktop marks as live backends (``HERMES_DESKTOP_CHILD_PID``, comma-separated or a lone int).""" out: set[int] = set() for part in os.environ.get("HERMES_DESKTOP_CHILD_PID", "").split(","): part = part.strip() @@ -299,8 +286,8 @@ def _kill_pids_windows(pids: list[int], killed: list[int], failed: list[tuple[in from gateway.status import get_process_start_time from hermes_cli._subprocess_compat import pid_is_hermes, windows_hide_flags - # Capture identity immediately after discovery: a PID reused before the - # destructive action fails the start-time check. + # Capture identity immediately after discovery: a PID reused before the destructive + # action fails the start-time check. pid_start_times = {pid: get_process_start_time(pid) for pid in pids} for pid in pids: try: @@ -330,6 +317,8 @@ def _kill_pids_posix(pids: list[int], killed: list[int], failed: list[tuple[int, import signal as _signal import time as _time + from gateway.status import _pid_exists + def _send(pid: int, sig) -> None: try: os.kill(pid, sig) @@ -348,7 +337,6 @@ def _kill_pids_posix(pids: list[int], killed: list[int], failed: list[tuple[int, while pending and _time.monotonic() < deadline: _time.sleep(0.1) # os.kill(pid, 0) is NOT a no-op on Windows; use the portable check. - from gateway.status import _pid_exists alive = [p for p in pending if _pid_exists(p)] killed.extend(p for p in pending if p not in alive) pending = alive @@ -366,22 +354,20 @@ def _kill_stale_dashboard_processes( """Kill running ``hermes dashboard`` / ``hermes serve`` processes. Called at the end of ``hermes update`` (default ``reason``) and from - ``hermes dashboard --stop``; after an update the running process serves - stale Python against a fresh JS bundle. POSIX: SIGTERM, ~3s grace, SIGKILL - survivors. Windows: ``taskkill /F``. + ``hermes dashboard --stop``. POSIX: SIGTERM, ~3s grace, SIGKILL survivors. Windows: + ``taskkill /F``. - With ``restart_managed`` (update path only — ``--stop`` never restarts) a - detected ``hermes-dashboard.service`` is restarted through systemd, any - other killed PID owned by a systemd unit has that unit restarted after the - kill (systemd treats our SIGTERM as a clean stop, so ``Restart=on-failure`` - never fires), and manual PIDs are respawned from their captured argv. - *already_restarted_units* (no ``.service`` suffix) were restarted by the + With ``restart_managed`` (update path only — ``--stop`` never restarts) a detected + ``hermes-dashboard.service`` is restarted through systemd, any other killed PID owned by a + systemd unit has that unit restarted after the kill (systemd treats our SIGTERM as a clean + stop, so ``Restart=on-failure`` never fires), and manual PIDs are respawned from their + captured argv. *already_restarted_units* (no ``.service`` suffix) were restarted by the caller already; PIDs they own are left untouched, not killed twice. """ if restart_managed and _m()._restart_managed_dashboard_service(reason): - # The dashboard unit is handled but every OTHER backend is not (a host - # may also run hermes-serve.service hosting tui_gateway): record the - # unit as handled (the filter below drops PIDs it owns) and keep going. + # The dashboard unit is handled but every OTHER backend is not (a host may also run + # hermes-serve.service hosting tui_gateway): record the unit as handled (the filter + # below drops PIDs it owns) and keep going. _dash_unit = getattr(_m(), "_DASHBOARD_SYSTEMD_UNIT", "hermes-dashboard.service") already_restarted_units = set(already_restarted_units or ()) | { str(_dash_unit).removesuffix(".service") @@ -389,17 +375,17 @@ def _kill_stale_dashboard_processes( exclude = _exclude_pids_from_env() if restart_managed: - # An SSH-owned backend belongs to an attached Desktop client even when - # the updater runs from an unrelated shell; killing it strands that - # client's fixed SSH port-forward. Same ownership records as the reaper. + # An SSH-owned backend belongs to an attached Desktop client even when the updater + # runs from an unrelated shell; killing it strands that client's fixed SSH + # port-forward. Same ownership records as the reaper. exclude |= _lock_owned_serve_pids() pids = _m()._find_stale_dashboard_pids(exclude_pids=exclude or None) if not pids: return _empty_result() - # Snapshot systemd cgroup/unit and argv BEFORE killing (the cgroup - # disappears with the process). Linux + update path only. + # Snapshot systemd cgroup/unit and argv BEFORE killing (the cgroup disappears with the + # process). Linux + update path only. pid_cgroup: dict[int, str | None] = {} pid_service: dict[int, str | None] = {} pid_cmdline: dict[int, list[str]] = {} @@ -409,8 +395,8 @@ def _kill_stale_dashboard_processes( pid_cgroup[pid] = _m()._get_pid_cgroup_path(pid) pid_service[pid] = _m()._get_systemd_service_for_pid(pid) if not pid_service[pid]: - # Manual process: keep exact argv + HERMES_HOME for the - # post-update respawn and its per-profile cap. + # Manual process: keep exact argv + HERMES_HOME for the post-update respawn + # and its per-profile cap. cmdline = _m()._dashboard_cmdline_for_pid(pid) if cmdline: pid_cmdline[pid] = cmdline @@ -428,10 +414,7 @@ def _kill_stale_dashboard_processes( killed: list[int] = [] failed: list[tuple[int, str]] = [] - if sys.platform == "win32": - _kill_pids_windows(pids, killed, failed) - else: - _kill_pids_posix(pids, killed, failed) + (_kill_pids_windows if sys.platform == "win32" else _kill_pids_posix)(pids, killed, failed) for pid in killed: print(f" ✓ stopped PID {pid}") @@ -452,11 +435,10 @@ def _restart_killed_backends( killed: list[int], pid_service: dict[int, str | None], pid_cgroup: dict[int, str | None], pid_cmdline: dict[int, list[str]], pid_home: dict[int, str | None], ) -> list[int]: - """Update path: restart systemd-owned units, respawn manual argv. + """Update path: restart systemd-owned units, respawn manual argv. Returns PIDs not brought back. - Respawns are detached, headless, logged to logs/dashboard-restart.log; - Desktop ``--port 0`` backends are filtered out and duplicates collapse to - one per profile. Returns the PIDs that were not brought back. + Respawns are detached, headless, logged to logs/dashboard-restart.log; Desktop ``--port 0`` + backends are filtered out and duplicates collapse to one per profile. """ unrecovered: list[int] = [] failed_restarts: list[tuple[str, str]] = [] @@ -505,17 +487,16 @@ def _detect_concurrent_hermes_instances( ) -> list[tuple[int, str]]: """Find other live processes whose .exe is one of our entry-point shims. - Windows blocks DELETE/REPLACE on a running .exe (and RENAME when opened - without ``FILE_SHARE_DELETE``); Desktop spawns ``hermes.EXE`` as a backend - child, so the update's quarantine rename fails with ``[WinError 32]``. + Windows blocks DELETE/REPLACE on a running .exe (and RENAME when opened without + ``FILE_SHARE_DELETE``); Desktop spawns ``hermes.EXE`` as a backend child, so the update's + quarantine rename fails with ``[WinError 32]``. - Returns ``(pid, process_name)`` for processes whose ``exe`` matches a venv - shim (``hermes.exe`` / ``hermes-gateway.exe``). Excludes our own PID and - every *shim* ancestor: the setuptools launcher is a separate native process - from the ``python.exe`` it loads, so otherwise every update reports its own - launcher. ``proc.parents()`` (whole chain at once) because a per-hop loop - bailed on the first AccessDenied. Only shim ancestors are excluded so a - second hermes.exe under a non-Hermes parent (Desktop child) is still flagged. + Returns ``(pid, process_name)`` for processes whose ``exe`` matches a venv shim. Excludes + our own PID and every *shim* ancestor: the setuptools launcher is a separate native + process from the ``python.exe`` it loads, so otherwise every update reports its own + launcher. ``proc.parents()`` (whole chain at once) because a per-hop loop bailed on the + first AccessDenied. Only shim ancestors are excluded so a second hermes.exe under a + non-Hermes parent (Desktop child) is still flagged. Empty off-Windows, without psutil, or with no other instances. Never raises. """ @@ -568,10 +549,11 @@ def _detect_concurrent_hermes_instances( def _is_desktop_local_serve_cmdline(command: str) -> bool: - """True for the Desktop-local serve shape ``hermes serve [--isolated] - --host 127.0.0.1 --port 0``. Long-lived headless serves (``--host - --port 9119``) must never match — those are operator-managed - remote backends that legitimately run with ppid 1 under launchd/nohup.""" + """True for the Desktop-local serve shape ``hermes serve [--isolated] --host 127.0.0.1 --port 0``. + + Long-lived headless serves (``--host --port 9119``) must never match — + those are operator-managed remote backends that legitimately run with ppid 1. + """ cmd = command.lower() if "serve" not in cmd or ("hermes" not in cmd and "hermes_cli" not in cmd): return False @@ -584,15 +566,11 @@ def _is_desktop_local_serve_cmdline(command: str) -> bool: def _process_ppid(pid: int) -> int | None: - """Best-effort parent pid lookup. None on failure (and always on Windows, - where orphan reap is handled by the desktop tree-kill).""" + """Best-effort parent pid; None on failure (and always on Windows, where the desktop tree-kill reaps orphans).""" try: if sys.platform == "win32": return None - result = subprocess.run( - ["ps", "-o", "ppid=", "-p", str(pid)], - capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=5, - ) + result = subprocess.run(["ps", "-o", "ppid=", "-p", str(pid)], timeout=5, **_PS_RUN_KWARGS) if result.returncode != 0 or not result.stdout: return None return int(result.stdout.strip().split()[0]) @@ -601,13 +579,12 @@ def _process_ppid(pid: int) -> int | None: # --- SSH remote-backend lock ownership ------------------------------------- -# ``backend.lock.json`` is written by the Desktop SSH runtime on the *remote* -# host for every ``hermes serve`` it spawns (apps/desktop/electron/ -# remote-lifecycle.ts). A backend another client/machine started is legitimate -# and lock-owned even with no parent here (sshd exited → ppid 1). The reap must -# NEVER kill a PID a valid lock claims — that once killed a production backend. -# Schema constants mirror the writer; a mismatched record is simply ignored -# (the reap only ever *spares*). +# ``backend.lock.json`` is written by the Desktop SSH runtime on the *remote* host for every +# ``hermes serve`` it spawns (apps/desktop/electron/remote-lifecycle.ts). A backend another +# client/machine started is legitimate and lock-owned even with no parent here (sshd exited → +# ppid 1). The reap must NEVER kill a PID a valid lock claims — that once killed a production +# backend. Schema constants mirror the writer; a mismatched record is simply ignored (the reap +# only ever *spares*). _LOCKFILE_SCHEMA_VERSION = 2 _PROTOCOL_VERSION = 1 _REMOTE_LOCK_SUBDIR = "desktop-ssh" @@ -627,8 +604,8 @@ def _is_hex(value: object, length: int) -> bool: def _valid_lockfile_payload(parsed: object, ownership_id: str) -> bool: """Validate a parsed ``backend.lock.json`` body, mirroring readLockfile(). - An invalid lock is "no ownership claim", which never causes a kill — the - reap only ever *adds* lock-owned PIDs to its spare-set. + An invalid lock is "no ownership claim", which never causes a kill — the reap only ever + *adds* lock-owned PIDs to its spare-set. """ if ( not isinstance(parsed, dict) @@ -650,16 +627,18 @@ def _valid_lockfile_payload(parsed: object, ownership_id: str) -> bool: value = parsed.get(field) if not isinstance(value, str) or len(value) > 1024: return False - # logPath is ``{lock_root}/{ownershipId}/{spawnNonce}.log``. Only the - # suffix is checked so a relocated HERMES_HOME doesn't falsely reject a - # legitimate remote-owned backend (a false reject re-introduces the kill). + # logPath is ``{lock_root}/{ownershipId}/{spawnNonce}.log``. Only the suffix is checked so + # a relocated HERMES_HOME doesn't falsely reject a legitimate remote-owned backend (a false + # reject re-introduces the kill). return parsed["logPath"].endswith(f"/{ownership_id}/{parsed['spawnNonce']}.log") def _lock_owned_serve_pids(base_dir: Path | None = None) -> set[int]: - """PIDs claimed by valid ``{hermes_home}/desktop-ssh//backend.lock.json`` - records — legitimately owned (incl. SSH backends other clients started) and - spared by the reap. Best-effort: a bad record contributes no PID; never raises.""" + """PIDs claimed by valid ``{hermes_home}/desktop-ssh//backend.lock.json`` records. + + Legitimately owned (incl. SSH backends other clients started) and spared by the reap. + Best-effort: a bad record contributes no PID; never raises. + """ import json root = base_dir if base_dir is not None else _hermes_home_dir() / _REMOTE_LOCK_SUBDIR @@ -695,8 +674,8 @@ def _lock_owned_serve_pids(base_dir: Path | None = None) -> set[int]: return owned -# Grace window before an orphaned-looking backend may be reaped. Covers the -# gap between process start and the Desktop client writing backend.lock.json. +# Grace window before an orphaned-looking backend may be reaped: covers the gap between +# process start and the Desktop client writing backend.lock.json. _REAP_MIN_AGE_SECONDS = 180.0 @@ -715,22 +694,19 @@ def _reap_orphaned_desktop_local_serves( ) -> dict[str, list]: """Kill leftover Desktop-local ``hermes serve`` backends with no parent. - When Electron dies uncleanly, ``serve --host 127.0.0.1 --port 0`` children - get reparented to pid 1 with their MCP trees alive; each Desktop boot then - stacks a fresh backend on the corpses until EMFILE. The parent-death - watchdog (HERMES_PARENT_PID) prevents *future* orphans; this clears - *already* orphaned ones when a new Desktop backend starts. + When Electron dies uncleanly, ``serve --host 127.0.0.1 --port 0`` children get reparented + to pid 1 with their MCP trees alive; each Desktop boot then stacks a fresh backend on the + corpses until EMFILE. The parent-death watchdog (HERMES_PARENT_PID) prevents *future* + orphans; this clears *already* orphaned ones when a new Desktop backend starts. - A candidate is reaped only if ALL hold: Desktop-local shape (never a - fixed-port remote serve); ppid 1 (or 0 on some supervisors); not self / - parent / a HERMES_DESKTOP_CHILD_PID; not claimed by a valid - ``backend.lock.json`` (SSH backends other clients started legitimately sit - at ppid 1 — killing them is an incident, not cleanup); older than - ``_REAP_MIN_AGE_SECONDS`` with a determinable age — Desktop writes the lock - only after HERMES_BACKEND_READY, so during concurrent multi-profile startup - a live sibling is briefly unowned and indistinguishable from a corpse - (mutual-reap storm); a real corpse just waits for a later scan. - Best-effort; failures never raise to the caller. + A candidate is reaped only if ALL hold: Desktop-local shape (never a fixed-port remote + serve); ppid 1 (or 0 on some supervisors); not self / parent / a HERMES_DESKTOP_CHILD_PID; + not claimed by a valid ``backend.lock.json`` (SSH backends other clients started + legitimately sit at ppid 1 — killing them is an incident, not cleanup); older than + ``_REAP_MIN_AGE_SECONDS`` with a determinable age — Desktop writes the lock only after + HERMES_BACKEND_READY, so during concurrent multi-profile startup a live sibling is briefly + unowned and indistinguishable from a corpse (mutual-reap storm); a real corpse just waits + for a later scan. Best-effort; failures never raise to the caller. """ import signal as _signal import time as _time @@ -795,8 +771,8 @@ def _reap_orphaned_desktop_local_serves( except OSError: failed.append(pid) - # Brief grace, then SIGKILL survivors. psutil.pid_exists rather than - # os.kill(pid, 0), which is a Windows footgun the linter blocks everywhere. + # Brief grace, then SIGKILL survivors. psutil.pid_exists rather than os.kill(pid, 0), + # which is a Windows footgun the linter blocks everywhere. sleep_fn(1.5) import psutil From be5654efb20d688f79768b085e4ae1fb3efa3988 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:33:47 -0700 Subject: [PATCH 06/29] =?UTF-8?q?refactor(hermes=5Fcli):=20group=20D=20?= =?UTF-8?q?=E2=80=94=20AST-neutral=20bracket=20hugging=20/=20arg=20packing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/container_boot.py | 24 +++++------------------- hermes_cli/copilot_auth.py | 4 +--- hermes_cli/cron.py | 21 ++++++--------------- hermes_cli/dashboard_procs.py | 13 ++++--------- hermes_cli/debug.py | 5 +---- 5 files changed, 17 insertions(+), 50 deletions(-) diff --git a/hermes_cli/container_boot.py b/hermes_cli/container_boot.py index e823c0b972..0a51cdcf76 100644 --- a/hermes_cli/container_boot.py +++ b/hermes_cli/container_boot.py @@ -63,18 +63,13 @@ def _slot_action( profile: str, profile_dir: Path, prior_state: str | None, start: bool, ) -> ReconcileAction: return ReconcileAction( - profile=profile, - prior_state=prior_state, - action="started" if start else "registered", + profile=profile, prior_state=prior_state, action="started" if start else "registered", prior_exit=_read_prior_exit_label(profile_dir), ) def reconcile_profile_gateways( - *, - hermes_home: Path, - scandir: Path, - dry_run: bool = False, + *, hermes_home: Path, scandir: Path, dry_run: bool = False, container_argv: Sequence[str] | None = None, ) -> list[ReconcileAction]: """Recreate s6 service registrations for every persistent profile. @@ -150,10 +145,7 @@ def reconcile_profile_gateways( def _maybe_migrate_legacy_gateway_run_state( - hermes_home: Path, - *, - container_argv: Sequence[str] | None, - dry_run: bool, + hermes_home: Path, *, container_argv: Sequence[str] | None, dry_run: bool ) -> str | None: """Seed root gateway_state for pre-s6 `gateway run` containers. @@ -405,20 +397,14 @@ def main() -> int: # logs/gateways//lock → "Resource busy" restart storm. The role is detected from # PID 1 argv, not an operator flag — a flag can be forgotten in a hand-written manifest. if _is_dashboard_container(_read_container_argv()): - print( - "reconcile: skipping (dashboard container — does not need " - "per-profile gateways)" - ) + print("reconcile: skipping (dashboard container — does not need per-profile gateways)") return 0 hermes_home = Path(os.environ.get("HERMES_HOME", "/opt/data")) scandir = Path(os.environ.get("S6_PROFILE_GATEWAY_SCANDIR", "/run/service")) actions = reconcile_profile_gateways(hermes_home=hermes_home, scandir=scandir) for a in actions: - print( - f"reconcile: profile={a.profile} " - f"prior_state={a.prior_state} action={a.action}" - ) + print(f"reconcile: profile={a.profile} prior_state={a.prior_state} action={a.action}") return 0 diff --git a/hermes_cli/copilot_auth.py b/hermes_cli/copilot_auth.py index dae90f7358..7c434bf754 100644 --- a/hermes_cli/copilot_auth.py +++ b/hermes_cli/copilot_auth.py @@ -100,9 +100,7 @@ def _gh_cli_candidates() -> list[str]: """Candidate ``gh`` binary paths, including common Homebrew installs.""" candidates: list[str] = [c for c in (shutil.which("gh"),) if c] for candidate in ( - "/opt/homebrew/bin/gh", - "/usr/local/bin/gh", - str(Path.home() / ".local" / "bin" / "gh"), + "/opt/homebrew/bin/gh", "/usr/local/bin/gh", str(Path.home() / ".local" / "bin" / "gh") ): if (candidate not in candidates and os.path.isfile(candidate) and os.access(candidate, os.X_OK)): diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index f30de0d93c..c8a028da6b 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -400,9 +400,7 @@ def _print_ticker_health(pids: list) -> None: the liveness heartbeat and the last-successful-tick marker before saying "will fire". """ from cron.jobs import ( - get_ticker_heartbeat_age, - get_ticker_last_error, - get_ticker_success_age, + get_ticker_heartbeat_age, get_ticker_last_error, get_ticker_success_age, TICKER_INTERVAL_SECONDS, ) from cron.scheduler import _is_fd_exhaustion_text as _cron_is_fd_exhaustion_text @@ -706,13 +704,10 @@ def cron_create(args): # job-creation path (CLI AND the agent's `cronjob` tool); a block surfaces as # result["error"] and is printed in red below. result = _cron_api( - action="create", - schedule=args.schedule, - prompt=args.prompt, + action="create", schedule=args.schedule, prompt=args.prompt, skill=getattr(args, "skill", None), skills=_normalize_skills(getattr(args, "skill", None), getattr(args, "skills", None)), - no_agent=getattr(args, "no_agent", False) or None, - **_job_api_kwargs(args), + no_agent=getattr(args, "no_agent", False) or None, **_job_api_kwargs(args), ) if not result.get("success"): print(color(f"Failed to create job: {result.get('error', 'unknown error')}", Colors.RED)) @@ -759,13 +754,9 @@ def cron_edit(args): final_skills.append(skill) result = _cron_api( - action="update", - job_id=args.job_id, - schedule=getattr(args, "schedule", None), - prompt=getattr(args, "prompt", None), - skills=final_skills, - no_agent=getattr(args, "no_agent", None), - **_job_api_kwargs(args), + action="update", job_id=args.job_id, schedule=getattr(args, "schedule", None), + prompt=getattr(args, "prompt", None), skills=final_skills, + no_agent=getattr(args, "no_agent", None), **_job_api_kwargs(args), ) if not result.get("success"): print(color(f"Failed to update job: {result.get('error', 'unknown error')}", Colors.RED)) diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py index 8f04b8a802..cdd8b217ed 100644 --- a/hermes_cli/dashboard_procs.py +++ b/hermes_cli/dashboard_procs.py @@ -56,8 +56,7 @@ def _scan_dashboard_processes(*, exclude_pids: set[int] | None = None) -> list[t from hermes_cli._subprocess_compat import bounded_probe_run result = bounded_probe_run( - ["wmic", "process", "get", "ProcessId,CommandLine", "/FORMAT:LIST"], - timeout=10, + ["wmic", "process", "get", "ProcessId,CommandLine", "/FORMAT:LIST"], timeout=10, errors="ignore", ) if result is None or result.returncode != 0 or result.stdout is None: @@ -219,9 +218,7 @@ def _profile_key_for_respawn(argv: list[str], hermes_home: str | None = None) -> def _filter_dashboard_respawn_candidates( - candidates: list[tuple[int, list[str], str | None]], - *, - own_home: str | None = None, + candidates: list[tuple[int, list[str], str | None]], *, own_home: str | None = None ) -> list[list[str]]: """Select which killed manual backends to respawn after ``hermes update``. @@ -346,10 +343,8 @@ def _kill_pids_posix(pids: list[int], killed: list[int], failed: list[tuple[int, def _kill_stale_dashboard_processes( - reason: str = "the running backend no longer matches the updated frontend", - *, - restart_managed: bool = False, - already_restarted_units: "set[str] | None" = None, + reason: str = "the running backend no longer matches the updated frontend", *, + restart_managed: bool = False, already_restarted_units: "set[str] | None" = None, ) -> dict[str, list]: """Kill running ``hermes dashboard`` / ``hermes serve`` processes. diff --git a/hermes_cli/debug.py b/hermes_cli/debug.py index 7fe588bb35..41627fcbcf 100644 --- a/hermes_cli/debug.py +++ b/hermes_cli/debug.py @@ -646,10 +646,7 @@ def _run_debug_share_nous(args, *, log_lines: int, redact: bool) -> None: if not _confirm_upload(args): return if not redact: - print( - "⚠️ --no-redact is set: secrets in your logs will NOT be redacted " - "before upload.\n" - ) + print("⚠️ --no-redact is set: secrets in your logs will NOT be redacted before upload.\n") print("Collecting debug report...") _best_effort_sweep_expired_pastes() From 7302a097b1a0fe136552b9805cfaf33dc184b3c4 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:42:29 -0700 Subject: [PATCH 07/29] =?UTF-8?q?refactor(hermes=5Fcli):=20dashboard=5Freg?= =?UTF-8?q?ister.py=20=E2=80=94=20hoist=20urlparse=20import,=20compact=20d?= =?UTF-8?q?ocstrings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/dashboard_register.py | 78 ++++++++++++-------------------- 1 file changed, 30 insertions(+), 48 deletions(-) diff --git a/hermes_cli/dashboard_register.py b/hermes_cli/dashboard_register.py index 93dcb1e872..7045fa98ce 100644 --- a/hermes_cli/dashboard_register.py +++ b/hermes_cli/dashboard_register.py @@ -1,11 +1,9 @@ """``hermes dashboard register`` — register a self-hosted dashboard OAuth client. -Automates the manual flow (Nous Portal ``/local-dashboards`` → "register" → -paste the ``agent:{id}`` client ID into ``~/.hermes/.env``): resolve a fresh -Nous access token from the stored login, POST -``{portal}/api/oauth/self-hosted-client`` (the portal creates a SELF_HOSTED -client in the caller's org; the ``agent:`` prefix is applied server-side), -write ``HERMES_DASHBOARD_OAUTH_CLIENT_ID`` (+ portal/public URL when +Automates the manual Nous Portal ``/local-dashboards`` flow: resolve a fresh Nous access +token from the stored login, POST ``{portal}/api/oauth/self-hosted-client`` (the portal +creates a SELF_HOSTED client in the caller's org; the ``agent:`` prefix is applied +server-side), write ``HERMES_DASHBOARD_OAUTH_CLIENT_ID`` (+ portal/public URL when warranted) into ``.env`` idempotently, then print the gate-engagement hint. """ @@ -18,12 +16,12 @@ import sys import urllib.error import urllib.request from typing import Optional +from urllib.parse import urlparse _DEFAULT_PORTAL = "https://portal.nousresearch.com" -# Docker-style adjective_noun names (underscore-joined so they drop into a -# label field). The portal has no uniqueness constraint (the row id is the -# key), so collisions are harmless and never retried. +# Docker-style adjective_noun names (underscore-joined so they drop into a label field). The +# portal has no uniqueness constraint (the row id is the key), so collisions are harmless. _NAME_ADJECTIVES = ( "amber", "bold", "brave", "bright", "calm", "clever", "cosmic", "crisp", "dreamy", "eager", "electric", "fancy", "gentle", "golden", "happy", @@ -48,12 +46,8 @@ def _generate_dashboard_name() -> str: def _resolve_portal_base_url(override: Optional[str] = None) -> str: - """Portal base URL for the registration request. - - Precedence: explicit *override* (``--portal-url`` / env — the token must - have been minted by that same portal), then the ``portal_base_url`` stored - on the Nous login (the issuer, so the correct default), then production. - """ + """Portal base URL: explicit *override* (the token must have been minted by that same + portal), then the ``portal_base_url`` stored on the Nous login (the issuer), then production.""" if isinstance(override, str) and override.strip(): return override.rstrip("/") try: @@ -72,14 +66,10 @@ def _register_self_hosted_client( ) -> dict: """POST to the portal's self-hosted-client endpoint and return the JSON body. - ``existing_client_id`` (persisted from a prior run) makes the portal update - that record in place instead of minting a duplicate — this is what makes - re-running idempotent. The portal falls back to creating a fresh client if - the id no longer resolves in the caller's org, so passing it is always safe. - - ``name`` is ``None`` on the update path without an explicit ``--name``; - omitting it tells the portal to keep the stored name. Required on create. - + ``existing_client_id`` (persisted from a prior run) makes the portal update that record in + place instead of minting a duplicate — what makes re-running idempotent; the portal falls + back to creating a fresh client if the id no longer resolves, so passing it is always safe. + ``name`` is ``None`` on the update path without ``--name`` (portal keeps the stored name). Raises RuntimeError with a user-facing message on non-2xx or transport failure. """ url = f"{portal_base_url.rstrip('/')}/api/oauth/self-hosted-client" @@ -111,9 +101,7 @@ def _register_self_hosted_client( raise RuntimeError( detail or "Your account is not permitted to register a self-hosted dashboard." ) from exc - raise RuntimeError( - f"Portal returned HTTP {exc.code}" + (f": {detail}" if detail else "") - ) from exc + raise RuntimeError(f"Portal returned HTTP {exc.code}" + (f": {detail}" if detail else "")) from exc except urllib.error.URLError as exc: raise RuntimeError(f"Could not reach Nous Portal at {portal_base_url}: {exc.reason}") from exc @@ -143,8 +131,6 @@ def _print_post_register_hint( if custom_redirect_uri: # Example host matches the one the user registered. try: - from urllib.parse import urlparse - host = urlparse(custom_redirect_uri).hostname or "your-host" except Exception: host = "your-host" @@ -176,7 +162,7 @@ def _env_value(key: str) -> Optional[str]: def _save_env_quietly(key: str, value: str) -> bool: - """Persist *key*; False on failure (non-fatal: client_id is load-bearing).""" + """Persist *key*; False on failure (non-fatal: only client_id is load-bearing).""" from hermes_cli.config import save_env_value try: @@ -190,12 +176,10 @@ def _public_url_from_redirect(redirect_uri: Optional[str]) -> str: """Origin (``scheme://host[:port]``) of *redirect_uri*, or ``""``. ``dashboard_auth/routes._redirect_uri`` rebuilds the callback as - ``HERMES_DASHBOARD_PUBLIC_URL + "/auth/callback"``, so the runtime consumes - the ORIGIN — persisting the raw redirect URI would double up the path. + ``HERMES_DASHBOARD_PUBLIC_URL + "/auth/callback"``, so the runtime consumes the ORIGIN — + persisting the raw redirect URI would double up the path. """ try: - from urllib.parse import urlparse - parsed = urlparse(redirect_uri or "") if parsed.scheme in ("http", "https") and parsed.netloc: return f"{parsed.scheme}://{parsed.netloc}" @@ -209,8 +193,8 @@ def cmd_dashboard_register(args) -> None: from hermes_cli.auth import AuthError, resolve_nous_access_token from hermes_cli.config import is_managed, save_env_value - # Managed (Docker/hosted) installs get HERMES_DASHBOARD_OAUTH_CLIENT_ID - # stamped in by the orchestrator; save_env_value refuses to write anyway. + # Managed (Docker/hosted) installs get HERMES_DASHBOARD_OAUTH_CLIENT_ID stamped in by the + # orchestrator; save_env_value refuses to write anyway. if is_managed(): print("✗ `hermes dashboard register` is not available in a managed/hosted install.\n" " The dashboard OAuth client is provisioned by the hosting platform.") @@ -227,21 +211,20 @@ def cmd_dashboard_register(args) -> None: print(f"✗ Could not resolve a Nous Portal access token: {exc}") sys.exit(1) - # An *explicitly supplied* portal (flag or env) is an intentional choice we - # persist (overwriting in place); a portal merely inferred from the stored - # login keeps the conservative write-only-if-absent behaviour so .env isn't - # cluttered for the common production case. + # An *explicitly supplied* portal (flag or env) is an intentional choice we persist + # (overwriting in place); a portal merely inferred from the stored login keeps the + # write-only-if-absent behaviour so .env isn't cluttered for the common production case. portal_override = getattr(args, "portal_url", None) or os.environ.get("HERMES_DASHBOARD_PORTAL_URL") custom_portal_supplied = bool(isinstance(portal_override, str) and portal_override.strip()) portal_base_url = _resolve_portal_base_url(portal_override) - # Idempotency: re-send a locally held client_id so the portal UPDATES that - # record instead of creating a duplicate (no id = new dashboard). + # Idempotency: re-send a locally held client_id so the portal UPDATES that record instead + # of creating a duplicate (no id = new dashboard). stored = _env_value("HERMES_DASHBOARD_OAUTH_CLIENT_ID") existing_client_id = (stored.strip() or None) if isinstance(stored, str) else None - # Auto-generate a name ONLY for a first registration; on a re-run without - # --name leave it unset so the portal preserves the stored name. + # Auto-generate a name ONLY for a first registration; on a re-run without --name leave it + # unset so the portal preserves the stored name. name = getattr(args, "name", None) or (None if existing_client_id else _generate_dashboard_name()) custom_redirect_uri = getattr(args, "redirect_uri", None) @@ -269,9 +252,8 @@ def cmd_dashboard_register(args) -> None: f" Set it manually: HERMES_DASHBOARD_OAUTH_CLIENT_ID={client_id}") sys.exit(1) - # Portal URL: explicit custom portal → always persist (even when it equals - # production; the user asked). Inferred portal → only when unset AND it - # differs from the production default. + # Portal URL: explicit custom portal → always persist (even when it equals production; the + # user asked). Inferred portal → only when unset AND it differs from the production default. existing_portal = _env_value("HERMES_DASHBOARD_PORTAL_URL") should_write_portal = ( existing_portal != portal_base_url @@ -280,8 +262,8 @@ def cmd_dashboard_register(args) -> None: ) wrote_portal_url = should_write_portal and _save_env_quietly("HERMES_DASHBOARD_PORTAL_URL", portal_base_url) - # Public URL derived from --redirect-uri: written in place when supplied, - # a no-op when it already matches, never on a localhost-only install. + # Public URL derived from --redirect-uri: written in place when supplied, a no-op when it + # already matches, never on a localhost-only install. public_url = _public_url_from_redirect(custom_redirect_uri) wrote_public_url = bool( public_url From 7a643acfc966d29c337ee04c119ad3f736a7ea93 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:51:45 -0700 Subject: [PATCH 08/29] =?UTF-8?q?refactor(hermes=5Fcli):=20group=20D=20?= =?UTF-8?q?=E2=80=94=20merge=20fragmented=20string=20literals,=20hug=20cal?= =?UTF-8?q?l=20layouts=20(AST-neutral)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/container_boot.py | 23 ++++------- hermes_cli/copilot_auth.py | 53 +++++++++--------------- hermes_cli/cron.py | 71 ++++++++++++-------------------- hermes_cli/dashboard_procs.py | 13 +++--- hermes_cli/dashboard_register.py | 6 +-- hermes_cli/debug.py | 26 ++++-------- 6 files changed, 70 insertions(+), 122 deletions(-) diff --git a/hermes_cli/container_boot.py b/hermes_cli/container_boot.py index 0a51cdcf76..5266f363f5 100644 --- a/hermes_cli/container_boot.py +++ b/hermes_cli/container_boot.py @@ -62,10 +62,9 @@ class ReconcileAction: def _slot_action( profile: str, profile_dir: Path, prior_state: str | None, start: bool, ) -> ReconcileAction: - return ReconcileAction( - profile=profile, prior_state=prior_state, action="started" if start else "registered", - prior_exit=_read_prior_exit_label(profile_dir), - ) + return ReconcileAction(profile=profile, prior_state=prior_state, + action="started" if start else "registered", + prior_exit=_read_prior_exit_label(profile_dir)) def reconcile_profile_gateways( @@ -92,11 +91,8 @@ def reconcile_profile_gateways( try: multiplex_profiles = load_gateway_config().multiplex_profiles except Exception: - log.warning( - "Unable to load gateway configuration during container boot; " - "using the GATEWAY_MULTIPLEX_PROFILES override if set.", - exc_info=True, - ) + log.warning("Unable to load gateway configuration during container boot; using the " + "GATEWAY_MULTIPLEX_PROFILES override if set.", exc_info=True) multiplex_profiles = is_truthy_value(os.environ.get("GATEWAY_MULTIPLEX_PROFILES")) # Default profile — always register, even if nothing has ever populated the root profile @@ -104,8 +100,7 @@ def reconcile_profile_gateways( # legacy `gateway run` container with no state yet seeds that intent as `running` so the # s6 reconciler preserves the pre-s6 behavior. legacy_default_state = _maybe_migrate_legacy_gateway_run_state( - hermes_home, container_argv=container_argv, dry_run=dry_run, - ) + hermes_home, container_argv=container_argv, dry_run=dry_run) default_prior_state = legacy_default_state or _read_desired_state(hermes_home) default_should_start = default_prior_state in _AUTOSTART_STATES if not dry_run: @@ -124,10 +119,8 @@ def reconcile_profile_gateways( # "default" is reserved for the root profile (above); skip a stray # ``profiles/default/`` rather than collide on the slot. if entry.name == "default": - log.warning( - "profiles/default/ exists — skipping to avoid colliding " - "with the reserved root-profile s6 slot", - ) + log.warning("profiles/default/ exists — skipping to avoid colliding with the " + "reserved root-profile s6 slot") continue prior_state = _read_desired_state(entry) diff --git a/hermes_cli/copilot_auth.py b/hermes_cli/copilot_auth.py index 7c434bf754..e8040871d1 100644 --- a/hermes_cli/copilot_auth.py +++ b/hermes_cli/copilot_auth.py @@ -79,11 +79,9 @@ def resolve_copilot_token() -> tuple[str, str]: # one silently substituted from the gh credential store. Skipping the subprocess also # avoids a slow `gh auth token` call (up to 5s on Windows) on every cold start. if any_env_var_set: - logger.debug( - "Copilot env var(s) set but none held a supported token; " - "skipping `gh auth token` fallback to honor explicit env-var " - "intent (and avoid the subprocess cost on cold start, #60800)." - ) + logger.debug("Copilot env var(s) set but none held a supported token; skipping `gh auth " + "token` fallback to honor explicit env-var intent (and avoid the subprocess " + "cost on cold start, #60800).") return "", "" token = _try_gh_cli_token() @@ -99,9 +97,8 @@ def resolve_copilot_token() -> tuple[str, str]: def _gh_cli_candidates() -> list[str]: """Candidate ``gh`` binary paths, including common Homebrew installs.""" candidates: list[str] = [c for c in (shutil.which("gh"),) if c] - for candidate in ( - "/opt/homebrew/bin/gh", "/usr/local/bin/gh", str(Path.home() / ".local" / "bin" / "gh") - ): + for candidate in ("/opt/homebrew/bin/gh", "/usr/local/bin/gh", + str(Path.home() / ".local" / "bin" / "gh")): if (candidate not in candidates and os.path.isfile(candidate) and os.access(candidate, os.X_OK)): candidates.append(candidate) @@ -154,10 +151,9 @@ def _probe_gh_cli_token() -> Optional[str]: if hostname: cmd += ["--hostname", hostname] try: - result = subprocess.run( - cmd, capture_output=True, text=True, encoding='utf-8', errors='replace', - timeout=5, env=clean_env, stdin=subprocess.DEVNULL, **_popen_kwargs, - ) + result = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8', + errors='replace', timeout=5, env=clean_env, + stdin=subprocess.DEVNULL, **_popen_kwargs) except (FileNotFoundError, subprocess.TimeoutExpired) as exc: logger.debug("gh CLI token lookup failed (%s): %s", gh_path, exc) continue @@ -168,10 +164,8 @@ def _probe_gh_cli_token() -> Optional[str]: # ─── OAuth Device Code Flow ──────────────────────────────────────────────── -_DEVICE_CODE_TERMINAL_ERRORS = { - "expired_token": " ✗ Device code expired. Please try again.", - "access_denied": " ✗ Authorization was denied.", -} +_DEVICE_CODE_TERMINAL_ERRORS = {"expired_token": " ✗ Device code expired. Please try again.", + "access_denied": " ✗ Authorization was denied."} def _post_form(url: str, fields: dict, timeout: float) -> dict: @@ -197,9 +191,8 @@ def copilot_device_code_login( access_token_url = f"https://{domain}/login/oauth/access_token" try: - device_data = _post_form( - device_code_url, {"client_id": COPILOT_OAUTH_CLIENT_ID, "scope": "read:user"}, 15 - ) + device_data = _post_form(device_code_url, + {"client_id": COPILOT_OAUTH_CLIENT_ID, "scope": "read:user"}, 15) except Exception as exc: logger.error("Failed to initiate device authorization: %s", exc) print(f" ✗ Failed to start device authorization: {exc}") @@ -332,9 +325,8 @@ def _read_jwt_store(path: Path) -> Optional[dict]: return None try: if path.stat().st_size > _JWT_DISK_MAX_BYTES: - logger.debug( - "Persisted Copilot JWT store exceeds %d bytes; ignoring", _JWT_DISK_MAX_BYTES - ) + logger.debug("Persisted Copilot JWT store exceeds %d bytes; ignoring", + _JWT_DISK_MAX_BYTES) return None loaded = json.loads(path.read_text(encoding="utf-8")) return loaded if isinstance(loaded, dict) else None @@ -549,10 +541,8 @@ def _exchange_copilot_token_locked( break if attempt < _EXCHANGE_MAX_ATTEMPTS: sleep_s = _EXCHANGE_BACKOFF_BASE_SECONDS * attempt - logger.debug( - "Copilot token exchange attempt %d/%d failed (%s); retrying in %.1fs", - attempt, _EXCHANGE_MAX_ATTEMPTS, exc, sleep_s, - ) + logger.debug("Copilot token exchange attempt %d/%d failed (%s); retrying in %.1fs", + attempt, _EXCHANGE_MAX_ATTEMPTS, exc, sleep_s) time.sleep(sleep_s) if data is None: ttl = ( @@ -624,13 +614,10 @@ def copilot_request_headers( *, is_agent_turn: bool = True, is_vision: bool = False, ) -> dict[str, str]: """Build the standard headers for Copilot API requests.""" - headers: dict[str, str] = { - "Editor-Version": _EDITOR_VERSION, - "User-Agent": "HermesAgent/1.0", - "Copilot-Integration-Id": "vscode-chat", - "Openai-Intent": "conversation-edits", - "x-initiator": "agent" if is_agent_turn else "user", - } + headers: dict[str, str] = {"Editor-Version": _EDITOR_VERSION, "User-Agent": "HermesAgent/1.0", + "Copilot-Integration-Id": "vscode-chat", + "Openai-Intent": "conversation-edits", + "x-initiator": "agent" if is_agent_turn else "user"} if is_vision: headers["Copilot-Vision-Request"] = "true" diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index c8a028da6b..65fc418eb5 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -254,10 +254,8 @@ def cron_list(show_all: bool = False): latest_execution = job.get("latest_execution") if latest_execution: - print( - f" Execution: {latest_execution.get('status', '?')} " - f"{latest_execution.get('id', '?')}" - ) + print(f" Execution: {latest_execution.get('status', '?')} " + f"{latest_execution.get('id', '?')}") delivery_err = job.get("last_delivery_error") if delivery_err: @@ -267,17 +265,13 @@ def cron_list(show_all: bool = False): # (Slack/Matrix/Mattermost shape): accepted as delivered, but say so here. unverified = job.get("last_delivery_unverified") if unverified: - print( - f" {color('⚠ Delivery UNVERIFIED:', Colors.YELLOW)} " - f"adapter acked {_unverified_targets(unverified)} without message_id/raw_response" - ) + print(f" {color('⚠ Delivery UNVERIFIED:', Colors.YELLOW)} adapter acked " + f"{_unverified_targets(unverified)} without message_id/raw_response") fire_err = job.get("last_fire_error") if isinstance(fire_err, dict) and fire_err.get("detail"): - print( - f" {color('⚠ Missed scheduled fire:', Colors.RED)} " - f"{fire_err.get('at', '?')} {fire_err['detail']}" - ) + print(f" {color('⚠ Missed scheduled fire:', Colors.RED)} " + f"{fire_err.get('at', '?')} {fire_err['detail']}") print() @@ -293,10 +287,8 @@ def cron_tick(): # Not expected here (a one-shot CLI process has no boot fingerprint, so the yield # gate is inert) — report cleanly instead of a traceback if a future caller records one. print(color(f"✗ {exc}", Colors.YELLOW)) - print( - " A fresher gateway process owns the runtime lock and will fire " - "due jobs; this stale process yielded its tick." - ) + print(" A fresher gateway process owns the runtime lock and will fire due jobs; this " + "stale process yielded its tick.") return 1 except OSError as exc: # tick() propagates real lock-acquisition failures (EMFILE, EACCES on open, ...) @@ -316,11 +308,9 @@ def cron_runs(job_id: Optional[str] = None, limit: int = 20): print("No cron execution attempts recorded.") return for record in records: - print( - f"{record.get('id', '?')} {record.get('status', '?'):<9} " - f"job={record.get('job_id', '?')} source={record.get('source', '?')} " - f"{record.get('claimed_at', '?')}" - ) + print(f"{record.get('id', '?')} {record.get('status', '?'):<9} " + f"job={record.get('job_id', '?')} source={record.get('source', '?')} " + f"{record.get('claimed_at', '?')}") if record.get("error"): print(f" {record['error']}") @@ -379,18 +369,12 @@ def cron_incidents(args) -> int: return 0 -_PERMISSION_HINT = ( - " Hint: jobs.json may be owned by another user " - "(e.g. rewritten by a root `docker exec hermes " - "hermes cron ...`). Fix ownership to match the " - "gateway user, and prefer `docker exec -u :`." -) -_FD_EXHAUSTION_HINT = ( - " Hint: the ticker hit file-descriptor exhaustion " - "(EMFILE). The scheduler now retries with backoff and " - "attempts fd reclamation, but if the leak persists, " - "restart the gateway to recover scheduling." -) +_PERMISSION_HINT = (" Hint: jobs.json may be owned by another user (e.g. rewritten by a root " + "`docker exec hermes hermes cron ...`). Fix ownership to match the gateway " + "user, and prefer `docker exec -u :`.") +_FD_EXHAUSTION_HINT = (" Hint: the ticker hit file-descriptor exhaustion (EMFILE). The scheduler " + "now retries with backoff and attempts fd reclamation, but if the leak " + "persists, restart the gateway to recover scheduling.") def _print_ticker_health(pids: list) -> None: @@ -669,13 +653,11 @@ def cron_doctor() -> int: return 1 -_JOB_ARG_FIELDS = ( - ("name", "name"), ("deliver", "deliver"), ("failure_deliver", "failure_deliver"), - ("repeat", "repeat"), ("script", "script"), ("workdir", "workdir"), ("model", "model"), - ("provider", "model_provider"), ("monitor_script", "monitor_script"), - ("monitor_url", "monitor_url"), ("continuity", "continuity"), - ("reasoning_effort", "reasoning_effort"), -) +_JOB_ARG_FIELDS = (("name", "name"), ("deliver", "deliver"), ("failure_deliver", "failure_deliver"), + ("repeat", "repeat"), ("script", "script"), ("workdir", "workdir"), + ("model", "model"), ("provider", "model_provider"), + ("monitor_script", "monitor_script"), ("monitor_url", "monitor_url"), + ("continuity", "continuity"), ("reasoning_effort", "reasoning_effort")) def _job_api_kwargs(args) -> Dict[str, Any]: @@ -753,11 +735,10 @@ def cron_edit(args): if skill not in final_skills: final_skills.append(skill) - result = _cron_api( - action="update", job_id=args.job_id, schedule=getattr(args, "schedule", None), - prompt=getattr(args, "prompt", None), skills=final_skills, - no_agent=getattr(args, "no_agent", None), **_job_api_kwargs(args), - ) + result = _cron_api(action="update", job_id=args.job_id, + schedule=getattr(args, "schedule", None), + prompt=getattr(args, "prompt", None), skills=final_skills, + no_agent=getattr(args, "no_agent", None), **_job_api_kwargs(args)) if not result.get("success"): print(color(f"Failed to update job: {result.get('error', 'unknown error')}", Colors.RED)) return 1 diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py index cdd8b217ed..552359d16d 100644 --- a/hermes_cli/dashboard_procs.py +++ b/hermes_cli/dashboard_procs.py @@ -295,12 +295,10 @@ def _kill_pids_windows(pids: list[int], killed: list[int], failed: list[tuple[in if not pid_is_hermes(pid, expected_start_time=expected_start_time): failed.append((pid, "not hermes-owned or process identity changed")) continue - result = subprocess.run( - ["taskkill", "/PID", str(pid), "/F"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL, - text=True, encoding="utf-8", errors="replace", timeout=10, - creationflags=windows_hide_flags(), - ) + result = subprocess.run(["taskkill", "/PID", str(pid), "/F"], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, stdin=subprocess.DEVNULL, text=True, + encoding="utf-8", errors="replace", timeout=10, + creationflags=windows_hide_flags()) if result.returncode == 0: killed.append(pid) else: @@ -365,8 +363,7 @@ def _kill_stale_dashboard_processes( # below drops PIDs it owns) and keep going. _dash_unit = getattr(_m(), "_DASHBOARD_SYSTEMD_UNIT", "hermes-dashboard.service") already_restarted_units = set(already_restarted_units or ()) | { - str(_dash_unit).removesuffix(".service") - } + str(_dash_unit).removesuffix(".service")} exclude = _exclude_pids_from_env() if restart_managed: diff --git a/hermes_cli/dashboard_register.py b/hermes_cli/dashboard_register.py index 7045fa98ce..0dca8de774 100644 --- a/hermes_cli/dashboard_register.py +++ b/hermes_cli/dashboard_register.py @@ -93,10 +93,8 @@ def _register_self_hosted_client( except Exception: pass if exc.code == 401: - raise RuntimeError( - "Nous Portal rejected the access token (401). " - "Try `hermes auth add nous` to re-authenticate." - ) from exc + raise RuntimeError("Nous Portal rejected the access token (401). Try `hermes auth add " + "nous` to re-authenticate.") from exc if exc.code == 403: raise RuntimeError( detail or "Your account is not permitted to register a self-hosted dashboard." diff --git a/hermes_cli/debug.py b/hermes_cli/debug.py index 41627fcbcf..a0d4e16161 100644 --- a/hermes_cli/debug.py +++ b/hermes_cli/debug.py @@ -168,9 +168,8 @@ def delete_paste(url: str) -> bool: if not paste_id: raise ValueError(f"Cannot delete: only paste.rs URLs are supported. Got: {url}") - req = urllib.request.Request( - f"{_PASTE_RS_URL}{paste_id}", method="DELETE", headers={"User-Agent": _USER_AGENT}, - ) + req = urllib.request.Request(f"{_PASTE_RS_URL}{paste_id}", method="DELETE", + headers={"User-Agent": _USER_AGENT}) with urllib.request.urlopen(req, timeout=30) as resp: return 200 <= resp.status < 300 @@ -191,10 +190,8 @@ def _schedule_auto_delete(urls: list[str], delay_seconds: int = _AUTO_DELETE_SEC def _post_paste(service: str, endpoint: str, body: bytes, content_type: str) -> str: """POST *body* to a paste service and return the paste URL it echoes back.""" - req = urllib.request.Request( - endpoint, data=body, method="POST", - headers={"Content-Type": content_type, "User-Agent": _USER_AGENT}, - ) + req = urllib.request.Request(endpoint, data=body, method="POST", + headers={"Content-Type": content_type, "User-Agent": _USER_AGENT}) with urllib.request.urlopen(req, timeout=30) as resp: url = resp.read().decode("utf-8").strip() if not url.startswith("http"): @@ -488,12 +485,9 @@ def build_nous_bundle(bundle: dict[str, str], redact: bool = True) -> bytes: The JSON shape (``format``, ``redacted``, ``created``, ``files``) is what the discord-support viewer parses — keep it stable. """ - envelope = { - "format": _NOUS_BUNDLE_FORMAT, - "redacted": bool(redact), - "created": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "files": bundle, - } + envelope = {"format": _NOUS_BUNDLE_FORMAT, "redacted": bool(redact), + "created": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "files": bundle} return gzip.compress(json.dumps(envelope).encode("utf-8")) @@ -544,10 +538,8 @@ def build_debug_share(*, log_lines: int = 200, expiry: int = 7, redact: bool = T _schedule_auto_delete(list(urls.values())) - return DebugShareResult( - urls=urls, failures=failures, redacted=redact, - auto_delete_seconds=_AUTO_DELETE_SECONDS, report=report, - ) + return DebugShareResult(urls=urls, failures=failures, redacted=redact, + auto_delete_seconds=_AUTO_DELETE_SECONDS, report=report) def _confirm_upload(args) -> bool: From c599b9ef9b161e36dcb6f0b51fb988f71f0b5b88 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:59:24 -0700 Subject: [PATCH 09/29] =?UTF-8?q?refactor(hermes=5Fcli):=20cron.py=20?= =?UTF-8?q?=E2=80=94=20table-driven=20job=20listing=20rows,=20state=20badg?= =?UTF-8?q?es=20and=20detail=20lines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/cron.py | 95 ++++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 50 deletions(-) diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index 65fc418eb5..210fa0252e 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -164,6 +164,9 @@ def _unverified_targets(unverified) -> str: return ", ".join(str(t) for t in unverified) if isinstance(unverified, list) else str(unverified) +_STATE_BADGES = {"paused": ("[paused]", Colors.YELLOW), "completed": ("[completed]", Colors.BLUE)} + + def cron_list(show_all: bool = False): """List all scheduled jobs.""" from cron.jobs import effective_job_state, list_jobs @@ -179,61 +182,47 @@ def cron_list(show_all: bool = False): for job in jobs: job_id = job.get("id", "?") - name = job.get("name", "(unnamed)") - schedule = job.get("schedule_display", job.get("schedule", {}).get("value", "?")) # Scheduler-honoured flag — never show [paused] when enabled=true. state = effective_job_state(job) - next_run = job.get("next_run_at", "?") + badge = _STATE_BADGES.get(state) or ( + ("[active]", Colors.GREEN) if job.get("enabled", True) else ("[disabled]", Colors.RED) + ) # `repeat` / `deliver` may be present-but-null in the record (a one-shot persisted # with "repeat": null), so coalesce rather than rely on the dict-default, which only # applies to a missing key — a null deliver would crash `", ".join(None)`. repeat_info = job.get("repeat") or {} repeat_times = repeat_info.get("times") - repeat_completed = repeat_info.get("completed", 0) - repeat_str = f"{repeat_completed}/{repeat_times}" if repeat_times else "∞" - + repeat_str = f"{repeat_info.get('completed', 0)}/{repeat_times}" if repeat_times else "∞" deliver = job.get("deliver") or ["local"] if isinstance(deliver, str): deliver = [deliver] - deliver_str = ", ".join(deliver) + rows = [ + ("Name", job.get("name", "(unnamed)")), + ("Schedule", job.get("schedule_display", job.get("schedule", {}).get("value", "?"))), + ("Repeat", repeat_str), + ("Next run", job.get("next_run_at", "?")), + ("Deliver", ", ".join(deliver)), + ] skills = job.get("skills") or ([job["skill"]] if job.get("skill") else []) - if state == "paused": - status = color("[paused]", Colors.YELLOW) - elif state == "completed": - status = color("[completed]", Colors.BLUE) - elif job.get("enabled", True): - status = color("[active]", Colors.GREEN) - else: - status = color("[disabled]", Colors.RED) - - print(f" {color(job_id, Colors.YELLOW)} {status}") - print(f" Name: {name}") - print(f" Schedule: {schedule}") - print(f" Repeat: {repeat_str}") - print(f" Next run: {next_run}") - print(f" Deliver: {deliver_str}") if skills: - print(f" Skills: {', '.join(skills)}") - script = job.get("script") - if script: - print(f" Script: {script}") + rows.append(("Skills", ", ".join(skills))) + if job.get("script"): + rows.append(("Script", job["script"])) monitor_source = job.get("monitor_script") or job.get("monitor_url") if monitor_source: - print(f" Monitor: {monitor_source} (agent runs only on output change)") + rows.append(("Monitor", f"{monitor_source} (agent runs only on output change)")) mon_state = job.get("monitor_state") or {} if mon_state.get("last_changed_at"): - print(f" Changed: {mon_state['last_changed_at']}") + rows.append(("Changed", mon_state["last_changed_at"])) if job.get("no_agent"): - print(f" Mode: {color('no-agent', Colors.DIM)} (script stdout delivered directly)") - workdir = job.get("workdir") - if workdir: - print(f" Workdir: {workdir}") + rows.append(("Mode", f"{color('no-agent', Colors.DIM)} (script stdout delivered directly)")) + if job.get("workdir"): + rows.append(("Workdir", job["workdir"])) last_status = job.get("last_status") if last_status: - last_run = job.get("last_run_at", "?") if last_status == "ok": status_display = color("ok", Colors.GREEN) elif last_status == "delivery_failed": @@ -246,16 +235,21 @@ def cron_list(show_all: bool = False): streak = int(job.get("failure_streak") or 0) if streak >= 2: status_display += color(f" ({streak} failures in a row)", Colors.RED) - print(f" Last run: {last_run} {status_display}") + rows.append(("Last run", f"{job.get('last_run_at', '?')} {status_display}")) dispatch_line = _dispatch_display(job.get("last_dispatch")) if dispatch_line: - print(f" Dispatch: {dispatch_line}") - + rows.append(("Dispatch", dispatch_line)) latest_execution = job.get("latest_execution") if latest_execution: - print(f" Execution: {latest_execution.get('status', '?')} " - f"{latest_execution.get('id', '?')}") + rows.append(( + "Execution", + f"{latest_execution.get('status', '?')} {latest_execution.get('id', '?')}", + )) + + print(f" {color(job_id, Colors.YELLOW)} {color(*badge)}") + for label, value in rows: + print(f" {label + ':':<11}{value}") delivery_err = job.get("last_delivery_error") if delivery_err: @@ -665,20 +659,21 @@ def _job_api_kwargs(args) -> Dict[str, Any]: return {api_key: getattr(args, attr, None) for api_key, attr in _JOB_ARG_FIELDS} +_JOB_DETAIL_LINES = ( + ("script", " Script: {}"), + ("monitor_script", " Monitor: {} (agent runs only on output change)"), + ("monitor_url", " Monitor: {} (agent runs only on output change)"), + ("no_agent", " Mode: no-agent (script stdout delivered directly)"), + ("continuity", " Continuity: on (each run sees the previous run's output)"), + ("workdir", " Workdir: {}"), +) + + def _print_job_details(job_data: Dict[str, Any]) -> None: """Print the optional Script/Monitor/Mode/Continuity/Workdir lines of a job record.""" - if job_data.get("script"): - print(f" Script: {job_data['script']}") - if job_data.get("monitor_script"): - print(f" Monitor: {job_data['monitor_script']} (agent runs only on output change)") - if job_data.get("monitor_url"): - print(f" Monitor: {job_data['monitor_url']} (agent runs only on output change)") - if job_data.get("no_agent"): - print(" Mode: no-agent (script stdout delivered directly)") - if job_data.get("continuity"): - print(" Continuity: on (each run sees the previous run's output)") - if job_data.get("workdir"): - print(f" Workdir: {job_data['workdir']}") + for key, template in _JOB_DETAIL_LINES: + if job_data.get(key): + print(template.format(job_data[key])) def cron_create(args): From 15dd4b47d49db057739de0586d1d3bd4556c99d4 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:01:51 -0700 Subject: [PATCH 10/29] =?UTF-8?q?refactor(hermes=5Fcli):=20dashboard=5Fpro?= =?UTF-8?q?cs.py=20=E2=80=94=20collapse=20reap=20SIGKILL=20branch,=20or-de?= =?UTF-8?q?fault=20injected=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/dashboard_procs.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py index 552359d16d..25b71536e5 100644 --- a/hermes_cli/dashboard_procs.py +++ b/hermes_cli/dashboard_procs.py @@ -705,9 +705,9 @@ def _reap_orphaned_desktop_local_serves( signal_term = _signal.SIGTERM if signal_term is None else signal_term signal_kill = getattr(_signal, "SIGKILL", _signal.SIGTERM) if signal_kill is None else signal_kill - sleep_fn = _time.sleep if sleep_fn is None else sleep_fn - lock_owned_pids_fn = _lock_owned_serve_pids if lock_owned_pids_fn is None else lock_owned_pids_fn - process_age_seconds_fn = _process_age_seconds if process_age_seconds_fn is None else process_age_seconds_fn + sleep_fn = sleep_fn or _time.sleep + lock_owned_pids_fn = lock_owned_pids_fn or _lock_owned_serve_pids + process_age_seconds_fn = process_age_seconds_fn or _process_age_seconds if sys.platform == "win32": # Windows desktop uses taskkill tree teardown; orphan scan is POSIX. @@ -771,11 +771,9 @@ def _reap_orphaned_desktop_local_serves( for pid, _cmd in targets: if pid in failed: continue - if not psutil.pid_exists(pid): - killed.append(pid) - continue try: - os.kill(pid, signal_kill) + if psutil.pid_exists(pid): + os.kill(pid, signal_kill) killed.append(pid) except ProcessLookupError: killed.append(pid) From 706043e18586a00dda00bf808e120da77dd0ec1c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:11:44 -0700 Subject: [PATCH 11/29] =?UTF-8?q?refactor(hermes=5Fcli):=20dashboard=5Fpro?= =?UTF-8?q?cs.py=20=E2=80=94=20extract=20process-table=20iteration=20from?= =?UTF-8?q?=20scanner,=20compact=20docstrings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/dashboard_procs.py | 203 +++++++++++++++------------------- 1 file changed, 87 insertions(+), 116 deletions(-) diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py index 25b71536e5..8e672c0f04 100644 --- a/hermes_cli/dashboard_procs.py +++ b/hermes_cli/dashboard_procs.py @@ -31,6 +31,48 @@ def _empty_result() -> dict[str, list]: return {"matched": [], "killed": [], "failed": []} +def _append_row(rows: list[tuple[int, str]], pid_text: str, command: str) -> None: + try: + rows.append((int(pid_text), command)) + except ValueError: + pass + + +def _iter_process_table() -> list[tuple[int, str]]: + """``(pid, cmdline)`` for every process, via wmic (Windows) or ps. Raises on scan failure.""" + rows: list[tuple[int, str]] = [] + if sys.platform == "win32": + # errors="ignore": wmic may emit the system code page; a decode error would leave + # stdout=None. bounded_probe_run (not run()): run()'s post-timeout cleanup joins pipe + # readers unbounded and a conhost descendant holding duplicated handles wedges it + # forever. It also passes CREATE_NO_WINDOW for the pythonw.exe backend. + from hermes_cli._subprocess_compat import bounded_probe_run + + result = bounded_probe_run( + ["wmic", "process", "get", "ProcessId,CommandLine", "/FORMAT:LIST"], + timeout=10, errors="ignore", + ) + if result is None or result.returncode != 0 or result.stdout is None: + return rows + current_cmd = "" + for line in result.stdout.split("\n"): + line = line.strip() + if line.startswith("CommandLine="): + current_cmd = line[len("CommandLine=") :] + elif line.startswith("ProcessId="): + _append_row(rows, line[len("ProcessId=") :], current_cmd) + return rows + # ps (not `pgrep -f "hermes.*dashboard"`) keeps us consistent with gateway._scan_gateway_pids + # and avoids a greedy regex matching unrelated cmdlines that merely contain both words. + result = subprocess.run(["ps", "-A", "-o", "pid=,command="], timeout=10, **_PS_RUN_KWARGS) + if result.returncode == 0: + for line in getattr(result, "stdout", "").split("\n"): + parts = line.strip().split(None, 1) + if len(parts) == 2 and "grep" not in line: + _append_row(rows, parts[0], parts[1]) + return rows + + def _scan_dashboard_processes(*, exclude_pids: set[int] | None = None) -> list[tuple[int, str]]: """Return matching ``dashboard``/``serve`` processes with their cmdlines. @@ -41,56 +83,15 @@ def _scan_dashboard_processes(*, exclude_pids: set[int] | None = None) -> list[t backend it manages itself. Empty list on any scan error (missing ps/wmic, timeout, ...). """ self_pid = os.getpid() - found: list[tuple[int, str]] = [] - - def _consider(pid: int, command: str) -> None: - if pid != self_pid and any(p in command for p in _DASHBOARD_PATTERNS): - found.append((pid, command)) - + skip = {self_pid, *(exclude_pids or ())} try: - if sys.platform == "win32": - # errors="ignore": wmic may emit the system code page; a decode error would leave - # stdout=None. bounded_probe_run (not run()): run()'s post-timeout cleanup joins - # pipe readers unbounded and a conhost descendant holding duplicated handles - # wedges it forever. It also passes CREATE_NO_WINDOW for the pythonw.exe backend. - from hermes_cli._subprocess_compat import bounded_probe_run - - result = bounded_probe_run( - ["wmic", "process", "get", "ProcessId,CommandLine", "/FORMAT:LIST"], timeout=10, - errors="ignore", - ) - if result is None or result.returncode != 0 or result.stdout is None: - return [] - current_cmd = "" - for line in result.stdout.split("\n"): - line = line.strip() - if line.startswith("CommandLine="): - current_cmd = line[len("CommandLine=") :] - elif line.startswith("ProcessId="): - try: - _consider(int(line[len("ProcessId=") :]), current_cmd) - except ValueError: - pass - else: - # ps (not `pgrep -f "hermes.*dashboard"`) keeps us consistent with - # gateway._scan_gateway_pids and avoids a greedy regex matching unrelated - # cmdlines that merely contain both words. - result = subprocess.run(["ps", "-A", "-o", "pid=,command="], timeout=10, **_PS_RUN_KWARGS) - if result.returncode == 0: - for line in getattr(result, "stdout", "").split("\n"): - parts = line.strip().split(None, 1) - if len(parts) != 2 or "grep" in line: - continue - try: - _consider(int(parts[0]), parts[1]) - except ValueError: - continue + found = [ + (pid, cmd) for pid, cmd in _iter_process_table() + if pid not in skip and any(p in cmd for p in _DASHBOARD_PATTERNS) + ] except (FileNotFoundError, subprocess.TimeoutExpired, OSError): return [] - if exclude_pids: - found = [proc for proc in found if proc[0] not in exclude_pids] - # Spawn-ledger augmentation: substring patterns miss profiled launches # (`hermes --profile p serve ...`). Every serve/dashboard registers itself in the spawn # ledger with live-verified (pid, create_time) — positive identity. Add entries the scan @@ -98,18 +99,11 @@ def _scan_dashboard_processes(*, exclude_pids: set[int] | None = None) -> list[t try: from hermes_cli.process_identity import ledger_entries - seen = {pid for pid, _ in found} + seen = {pid for pid, _ in found} | skip for entry in ledger_entries(): pid = entry.get("pid") - if ( - entry.get("purpose") not in ("serve", "dashboard") - or not isinstance(pid, int) - or pid == self_pid - or pid in seen - or (exclude_pids and pid in exclude_pids) - ): - continue - found.append((pid, str(entry.get("argv") or ""))) + if entry.get("purpose") in ("serve", "dashboard") and isinstance(pid, int) and pid not in seen: + found.append((pid, str(entry.get("argv") or ""))) except Exception: pass # ledger unavailable → scan-only behavior @@ -265,7 +259,7 @@ def _filter_dashboard_respawn_candidates( def _exclude_pids_from_env() -> set[int]: - """PIDs Desktop marks as live backends (``HERMES_DESKTOP_CHILD_PID``, comma-separated or a lone int).""" + """PIDs Desktop marks as live backends (``HERMES_DESKTOP_CHILD_PID``, comma-separated).""" out: set[int] = set() for part in os.environ.get("HERMES_DESKTOP_CHILD_PID", "").split(","): part = part.strip() @@ -344,18 +338,14 @@ def _kill_stale_dashboard_processes( reason: str = "the running backend no longer matches the updated frontend", *, restart_managed: bool = False, already_restarted_units: "set[str] | None" = None, ) -> dict[str, list]: - """Kill running ``hermes dashboard`` / ``hermes serve`` processes. + """Kill running ``hermes dashboard`` / ``hermes serve`` processes (update end, ``--stop``). - Called at the end of ``hermes update`` (default ``reason``) and from - ``hermes dashboard --stop``. POSIX: SIGTERM, ~3s grace, SIGKILL survivors. Windows: - ``taskkill /F``. - - With ``restart_managed`` (update path only — ``--stop`` never restarts) a detected - ``hermes-dashboard.service`` is restarted through systemd, any other killed PID owned by a - systemd unit has that unit restarted after the kill (systemd treats our SIGTERM as a clean - stop, so ``Restart=on-failure`` never fires), and manual PIDs are respawned from their - captured argv. *already_restarted_units* (no ``.service`` suffix) were restarted by the - caller already; PIDs they own are left untouched, not killed twice. + With ``restart_managed`` (update path only) a detected ``hermes-dashboard.service`` is + restarted through systemd, any other killed PID owned by a systemd unit has that unit + restarted after the kill (systemd treats our SIGTERM as a clean stop, so + ``Restart=on-failure`` never fires), and manual PIDs are respawned from their captured + argv. *already_restarted_units* (no ``.service`` suffix) were restarted by the caller + already; PIDs they own are left untouched, not killed twice. """ if restart_managed and _m()._restart_managed_dashboard_service(reason): # The dashboard unit is handled but every OTHER backend is not (a host may also run @@ -420,7 +410,8 @@ def _kill_stale_dashboard_processes( if killed: print(" Restart the dashboard when you're ready:\n hermes dashboard --port ") - return {"matched": list(pids), "killed": list(killed), "failed": list(failed), "unrecovered": list(unrecovered)} + return {"matched": list(pids), "killed": list(killed), "failed": list(failed), + "unrecovered": list(unrecovered)} def _restart_killed_backends( @@ -477,19 +468,14 @@ def _norm_exe(path) -> str: def _detect_concurrent_hermes_instances( scripts_dir: Path, *, exclude_pid: int | None = None ) -> list[tuple[int, str]]: - """Find other live processes whose .exe is one of our entry-point shims. - - Windows blocks DELETE/REPLACE on a running .exe (and RENAME when opened without - ``FILE_SHARE_DELETE``); Desktop spawns ``hermes.EXE`` as a backend child, so the update's - quarantine rename fails with ``[WinError 32]``. - - Returns ``(pid, process_name)`` for processes whose ``exe`` matches a venv shim. Excludes - our own PID and every *shim* ancestor: the setuptools launcher is a separate native - process from the ``python.exe`` it loads, so otherwise every update reports its own - launcher. ``proc.parents()`` (whole chain at once) because a per-hop loop bailed on the - first AccessDenied. Only shim ancestors are excluded so a second hermes.exe under a - non-Hermes parent (Desktop child) is still flagged. + """``(pid, name)`` of other live processes whose .exe is one of our entry-point shims. + Windows blocks DELETE/REPLACE on a running .exe; Desktop spawns ``hermes.EXE`` as a backend + child, so the update's quarantine rename fails with ``[WinError 32]``. Excludes our own + PID and every *shim* ancestor (the setuptools launcher is a separate native process from + the ``python.exe`` it loads — otherwise every update reports its own launcher); a second + hermes.exe under a non-Hermes parent (Desktop child) is still flagged. ``proc.parents()`` + (whole chain at once) because a per-hop loop bailed on the first AccessDenied. Empty off-Windows, without psutil, or with no other instances. Never raises. """ if not _m()._is_windows(): @@ -529,19 +515,15 @@ def _detect_concurrent_hermes_instances( info = proc.info except Exception: continue - pid = info.get("pid") - exe = info.get("exe") - if not exe or pid is None or pid in exclude_pids: - continue - if _norm_exe(exe) in shim_paths: - name = info.get("name") or Path(exe).name - matches.append((int(pid), str(name))) + pid, exe = info.get("pid"), info.get("exe") + if exe and pid is not None and pid not in exclude_pids and _norm_exe(exe) in shim_paths: + matches.append((int(pid), str(info.get("name") or Path(exe).name))) return matches def _is_desktop_local_serve_cmdline(command: str) -> bool: - """True for the Desktop-local serve shape ``hermes serve [--isolated] --host 127.0.0.1 --port 0``. + """True for the Desktop-local shape ``hermes serve [--isolated] --host 127.0.0.1 --port 0``. Long-lived headless serves (``--host --port 9119``) must never match — those are operator-managed remote backends that legitimately run with ppid 1. @@ -558,7 +540,7 @@ def _is_desktop_local_serve_cmdline(command: str) -> bool: def _process_ppid(pid: int) -> int | None: - """Best-effort parent pid; None on failure (and always on Windows, where the desktop tree-kill reaps orphans).""" + """Best-effort parent pid; None on failure (always on Windows: desktop tree-kill reaps there).""" try: if sys.platform == "win32": return None @@ -573,10 +555,9 @@ def _process_ppid(pid: int) -> int | None: # --- SSH remote-backend lock ownership ------------------------------------- # ``backend.lock.json`` is written by the Desktop SSH runtime on the *remote* host for every # ``hermes serve`` it spawns (apps/desktop/electron/remote-lifecycle.ts). A backend another -# client/machine started is legitimate and lock-owned even with no parent here (sshd exited → -# ppid 1). The reap must NEVER kill a PID a valid lock claims — that once killed a production -# backend. Schema constants mirror the writer; a mismatched record is simply ignored (the reap -# only ever *spares*). +# client started is legitimate and lock-owned even with no parent here (sshd exited → ppid 1); +# the reap must NEVER kill a PID a valid lock claims — that once killed a production backend. +# Schema constants mirror the writer; a mismatched record is ignored (the reap only *spares*). _LOCKFILE_SCHEMA_VERSION = 2 _PROTOCOL_VERSION = 1 _REMOTE_LOCK_SUBDIR = "desktop-ssh" @@ -594,11 +575,7 @@ def _is_hex(value: object, length: int) -> bool: def _valid_lockfile_payload(parsed: object, ownership_id: str) -> bool: - """Validate a parsed ``backend.lock.json`` body, mirroring readLockfile(). - - An invalid lock is "no ownership claim", which never causes a kill — the reap only ever - *adds* lock-owned PIDs to its spare-set. - """ + """Validate a parsed ``backend.lock.json`` body, mirroring readLockfile().""" if ( not isinstance(parsed, dict) or parsed.get("schemaVersion") != _LOCKFILE_SCHEMA_VERSION @@ -628,7 +605,6 @@ def _valid_lockfile_payload(parsed: object, ownership_id: str) -> bool: def _lock_owned_serve_pids(base_dir: Path | None = None) -> set[int]: """PIDs claimed by valid ``{hermes_home}/desktop-ssh//backend.lock.json`` records. - Legitimately owned (incl. SSH backends other clients started) and spared by the reap. Best-effort: a bad record contributes no PID; never raises. """ import json @@ -688,17 +664,14 @@ def _reap_orphaned_desktop_local_serves( When Electron dies uncleanly, ``serve --host 127.0.0.1 --port 0`` children get reparented to pid 1 with their MCP trees alive; each Desktop boot then stacks a fresh backend on the - corpses until EMFILE. The parent-death watchdog (HERMES_PARENT_PID) prevents *future* - orphans; this clears *already* orphaned ones when a new Desktop backend starts. - - A candidate is reaped only if ALL hold: Desktop-local shape (never a fixed-port remote - serve); ppid 1 (or 0 on some supervisors); not self / parent / a HERMES_DESKTOP_CHILD_PID; - not claimed by a valid ``backend.lock.json`` (SSH backends other clients started - legitimately sit at ppid 1 — killing them is an incident, not cleanup); older than - ``_REAP_MIN_AGE_SECONDS`` with a determinable age — Desktop writes the lock only after - HERMES_BACKEND_READY, so during concurrent multi-profile startup a live sibling is briefly - unowned and indistinguishable from a corpse (mutual-reap storm); a real corpse just waits - for a later scan. Best-effort; failures never raise to the caller. + corpses until EMFILE. The parent-death watchdog prevents *future* orphans; this clears + *already* orphaned ones. A candidate is reaped only if ALL hold: Desktop-local shape + (never a fixed-port remote serve); ppid 1 (or 0); not self / parent / + HERMES_DESKTOP_CHILD_PID; not claimed by a valid ``backend.lock.json`` (SSH backends other + clients started legitimately sit at ppid 1 — killing them is an incident); older than + ``_REAP_MIN_AGE_SECONDS`` with a determinable age (Desktop writes the lock only after + HERMES_BACKEND_READY, so a live sibling in concurrent multi-profile startup is briefly + unowned and indistinguishable from a corpse). Best-effort; never raises. """ import signal as _signal import time as _time @@ -719,13 +692,11 @@ def _reap_orphaned_desktop_local_serves( except Exception: return set() # never let lock scanning block or widen the reap - exclude = _exclude_pids_from_env() - exclude.add(os.getpid()) + exclude = _exclude_pids_from_env() | {os.getpid()} | _owned_pids() try: exclude.add(os.getppid()) # the desktop / sshd wrapper except Exception: pass - exclude |= _owned_pids() try: scanned = _scan_dashboard_processes(exclude_pids=exclude) @@ -755,7 +726,7 @@ def _reap_orphaned_desktop_local_serves( killed: list[int] = [] failed: list[int] = [] - for pid, _cmd in targets: + for pid in matched: try: os.kill(pid, signal_term) except ProcessLookupError: @@ -768,7 +739,7 @@ def _reap_orphaned_desktop_local_serves( sleep_fn(1.5) import psutil - for pid, _cmd in targets: + for pid in matched: if pid in failed: continue try: From 8f047aced9f0ec8f5d8ca52438437dfa65716692 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:13:57 -0700 Subject: [PATCH 12/29] =?UTF-8?q?refactor(hermes=5Fcli):=20debug.py=20?= =?UTF-8?q?=E2=80=94=20lift=20tail-reader=20out=20of=20=5Fcapture=5Flog=5F?= =?UTF-8?q?snapshot;=20cron=20status=20lock-probe=20collapse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/cron.py | 8 +++---- hermes_cli/debug.py | 52 +++++++++++++++++++++++---------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index 210fa0252e..7bcc5bdb31 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -465,11 +465,9 @@ def cron_status(): try: from gateway.status import get_running_pid, is_gateway_runtime_lock_active - if is_gateway_runtime_lock_active(): - gateway_alive_via_lock = True - lock_pid = get_running_pid() - if lock_pid: - pids = [lock_pid] + gateway_alive_via_lock = is_gateway_runtime_lock_active() + lock_pid = get_running_pid() if gateway_alive_via_lock else None + pids = [lock_pid] if lock_pid else pids except Exception: pass if pids or gateway_alive_via_lock: diff --git a/hermes_cli/debug.py b/hermes_cli/debug.py index a0d4e16161..3403f33c42 100644 --- a/hermes_cli/debug.py +++ b/hermes_cli/debug.py @@ -299,6 +299,32 @@ def _redact_log_text(text: str) -> str: return _EMAIL_ADDRESS_RE.sub("[REDACTED_EMAIL]", text) +def _read_tail_bytes(log_path: Path, size: int, max_bytes: int, tail_lines: int) -> tuple[bytes, bool]: + """Read the whole file, or enough of its tail for both views → (raw, truncated). + + For oversized files, read backwards until we have ``max_bytes`` for the standalone upload + AND enough newline context to render the summary tail from the same snapshot. + """ + with open(log_path, "rb") as f: + if size <= max_bytes: + return f.read(), False + chunk_size = 8192 + pos = size + chunks: list[bytes] = [] + total = 0 + newline_count = 0 + while pos > 0 and (total < max_bytes or newline_count <= tail_lines + 1) and total < max_bytes * 2: + read_size = min(chunk_size, pos) + pos -= read_size + f.seek(pos) + chunk = f.read(read_size) + chunks.insert(0, chunk) + total += len(chunk) + newline_count += chunk.count(b"\n") + chunk_size = min(chunk_size * 2, 65536) + return b"".join(chunks), pos > 0 + + def _capture_log_snapshot( log_name: str, *, tail_lines: int, max_bytes: int = _MAX_LOG_BYTES, redact: bool = True, ) -> LogSnapshot: @@ -320,31 +346,7 @@ def _capture_log_snapshot( # race: file was truncated between _resolve_log_path and stat return LogSnapshot(path=log_path, tail_text="(file empty)", full_text=None) - with open(log_path, "rb") as f: - if size <= max_bytes: - raw = f.read() - truncated = False - else: - # Read from the end until we have enough bytes for the standalone upload and - # enough newline context to render the summary tail from the same snapshot. - chunk_size = 8192 - pos = size - chunks: list[bytes] = [] - total = 0 - newline_count = 0 - - while pos > 0 and (total < max_bytes or newline_count <= tail_lines + 1) and total < max_bytes * 2: - read_size = min(chunk_size, pos) - pos -= read_size - f.seek(pos) - chunk = f.read(read_size) - chunks.insert(0, chunk) - total += len(chunk) - newline_count += chunk.count(b"\n") - chunk_size = min(chunk_size * 2, 65536) - - raw = b"".join(chunks) - truncated = pos > 0 + raw, truncated = _read_tail_bytes(log_path, size, max_bytes, tail_lines) full_raw = raw if truncated and len(full_raw) > max_bytes: From ffe8b15a3ed1e4fd6a0a1d85db685b90e225eff1 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:21:22 -0700 Subject: [PATCH 13/29] =?UTF-8?q?refactor(hermes=5Fcli):=20group=20D=20?= =?UTF-8?q?=E2=80=94=20collapse=20doctor/edit/notepad/sweep=20branches,=20?= =?UTF-8?q?debug=20subcommand=20dispatch,=20wrap=20new=20long=20lines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/container_boot.py | 2 +- hermes_cli/cron.py | 74 +++++++++++++------------------- hermes_cli/dashboard_procs.py | 6 ++- hermes_cli/dashboard_register.py | 3 +- hermes_cli/debug.py | 74 ++++++++++++++++---------------- 5 files changed, 73 insertions(+), 86 deletions(-) diff --git a/hermes_cli/container_boot.py b/hermes_cli/container_boot.py index 5266f363f5..262b0d838a 100644 --- a/hermes_cli/container_boot.py +++ b/hermes_cli/container_boot.py @@ -275,7 +275,7 @@ def _cleanup_stale_runtime_files(profile_dir: Path) -> None: def _read_prior_exit_label(profile_dir: Path) -> str: - """Exception-free ``lifecycle_ledger.read_prior_exit_label``: forensics never block cont-init.""" + """Exception-free ``lifecycle_ledger.read_prior_exit_label`` — forensics never block boot.""" try: from gateway.lifecycle_ledger import read_prior_exit_label return read_prior_exit_label(profile_dir) diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index 7bcc5bdb31..9d67b08b68 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -21,13 +21,10 @@ from cron.lifecycle_guard import ( # noqa: F401 (re-exported for terminal_tool def _normalize_skills(single_skill=None, skills: Optional[Iterable[str]] = None) -> Optional[List[str]]: - if skills is None: - if single_skill is None: - return None - raw_items = [single_skill] - else: - raw_items = list(skills) - + """Deduped, stripped skill names; None when neither argument was given.""" + if skills is None and single_skill is None: + return None + raw_items = list(skills) if skills is not None else [single_skill] normalized: List[str] = [] for item in raw_items: text = str(item or "").strip() @@ -78,7 +75,9 @@ def _builtin_gateway_liveness() -> Optional[bool]: return True except Exception: pass # a crashing lock probe is "unknown", not "dead" — let the pid scan decide - from hermes_cli.gateway import find_gateway_pids, named_profile_served_by_running_multiplexer + from hermes_cli.gateway import ( + find_gateway_pids, named_profile_served_by_running_multiplexer, + ) if find_gateway_pids(): return True @@ -161,7 +160,9 @@ def _print_banner(title: str) -> None: def _unverified_targets(unverified) -> str: - return ", ".join(str(t) for t in unverified) if isinstance(unverified, list) else str(unverified) + if isinstance(unverified, list): + return ", ".join(str(t) for t in unverified) + return str(unverified) _STATE_BADGES = {"paused": ("[paused]", Colors.YELLOW), "completed": ("[completed]", Colors.BLUE)} @@ -217,7 +218,8 @@ def cron_list(show_all: bool = False): if mon_state.get("last_changed_at"): rows.append(("Changed", mon_state["last_changed_at"])) if job.get("no_agent"): - rows.append(("Mode", f"{color('no-agent', Colors.DIM)} (script stdout delivered directly)")) + mode = color("no-agent", Colors.DIM) + " (script stdout delivered directly)" + rows.append(("Mode", mode)) if job.get("workdir"): rows.append(("Workdir", job["workdir"])) @@ -410,7 +412,7 @@ def _print_ticker_health(pids: list) -> None: ) print(" Cron jobs may NOT be firing. Restart: hermes gateway restart") elif ok_age is not None and ok_age > STALE_AFTER: - # Loop alive (fresh heartbeat) but no tick SUCCEEDED in a long time → failing every iteration. + # Loop alive (fresh heartbeat) but no tick SUCCEEDED in a long time → every tick fails. _warn( "⚠ Gateway and cron ticker are running, but no tick has " f"succeeded in {int(ok_age)}s — ticks may be failing." @@ -503,7 +505,8 @@ def _print_active_jobs_summary(jobs) -> None: ] if late: print() - print(color(f" ⚠ {len(late)} job(s) last fired late (missed-fire catch-up):", Colors.YELLOW)) + print(color(f" ⚠ {len(late)} job(s) last fired late (missed-fire catch-up):", + Colors.YELLOW)) for j in late: d = j["last_dispatch"] print( @@ -584,26 +587,20 @@ def _cron_doctor_issues_for_job(job: Dict[str, Any]) -> List[str]: unverified = job.get("last_delivery_unverified") if unverified: - issues.append( - f"last delivery unverified (adapter acked without evidence): {_unverified_targets(unverified)}" - ) + issues.append("last delivery unverified (adapter acked without evidence): " + + _unverified_targets(unverified)) if job.get("enabled", True) and job.get("state") not in {"paused", "completed"}: next_run = str(job.get("next_run_at") or "").strip() - if not next_run: - issues.append("active job has no next_run_at") - else: - overdue = _next_run_overdue_issue(next_run) - if overdue: - issues.append(overdue) + issue = _next_run_overdue_issue(next_run) if next_run else "active job has no next_run_at" + if issue: + issues.append(issue) script = str(job.get("script") or "").strip() if job.get("no_agent") and not script: issues.append("no-agent job has no script") - if script: - script_issue = _script_health_issue(script) - if script_issue: - issues.append(script_issue) + if script and (script_issue := _script_health_issue(script)): + issues.append(script_issue) workdir = str(job.get("workdir") or "").strip() if workdir and not Path(workdir).expanduser().exists(): @@ -617,11 +614,7 @@ def cron_doctor() -> int: from cron.jobs import list_jobs jobs = list_jobs(include_disabled=False) - findings: List[tuple[Dict[str, Any], List[str]]] = [] - for job in jobs: - issues = _cron_doctor_issues_for_job(job) - if issues: - findings.append((job, issues)) + findings = [(job, issues) for job in jobs if (issues := _cron_doctor_issues_for_job(job))] if not findings: print(color("✓ Cron doctor found no issues", Colors.GREEN)) @@ -635,9 +628,7 @@ def cron_doctor() -> int: print(color(f"Cron doctor found {issue_count} issue(s) across {len(findings)} job(s):", Colors.YELLOW)) print() for job, issues in findings: - job_id = job.get("id", "?") - name = job.get("name", "(unnamed)") - print(f" {color(job_id, Colors.YELLOW)} {name}") + print(f" {color(job.get('id', '?'), Colors.YELLOW)} {job.get('name', '(unnamed)')}") for issue in issues: print(f" - {issue}") print() @@ -724,9 +715,7 @@ def cron_edit(args): final_skills = replacement_skills elif add_skills or remove_skills: final_skills = [skill for skill in existing_skills if skill not in remove_skills] - for skill in add_skills: - if skill not in final_skills: - final_skills.append(skill) + final_skills += [skill for skill in add_skills if skill not in final_skills] result = _cron_api(action="update", job_id=args.job_id, schedule=getattr(args, "schedule", None), @@ -782,14 +771,12 @@ def _job_action(action: str, job_id: str, success_verb: str) -> int: # (execution_mode="background" and/or a delegation_id) and keeps running AFTER this # CLI exits — a terminal success/failure verdict would be a lie, so report the dispatch. delegation_id = job.get("delegation_id") - if job.get("execution_mode") == "background" or delegation_id: - if delegation_id: - print(f" Running in background (delegation {delegation_id}).") - else: - print(" Running in background.") + if delegation_id: + print(f" Running in background (delegation {delegation_id}).") + elif job.get("execution_mode") == "background": + print(" Running in background.") elif job.get("executed"): - outcome = "succeeded" if job.get("execution_success") else "failed" - print(f" Ran now: {outcome}.") + print(f" Ran now: {'succeeded' if job.get('execution_success') else 'failed'}.") elif job.get("execution_skipped"): print(f" {job['execution_skipped']}") else: @@ -862,7 +849,6 @@ def cron_notepad(args) -> int: print(color(f"No notepad key '{key}' for job {job_id}.", Colors.YELLOW)) return 1 - # list (default) notes = notepad.list_notes(job_id) if not notes: print(color(f"Notepad for job {job_id} is empty.", Colors.DIM)) diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py index 8e672c0f04..0171877f09 100644 --- a/hermes_cli/dashboard_procs.py +++ b/hermes_cli/dashboard_procs.py @@ -102,7 +102,9 @@ def _scan_dashboard_processes(*, exclude_pids: set[int] | None = None) -> list[t seen = {pid for pid, _ in found} | skip for entry in ledger_entries(): pid = entry.get("pid") - if entry.get("purpose") in ("serve", "dashboard") and isinstance(pid, int) and pid not in seen: + if entry.get("purpose") not in ("serve", "dashboard") or not isinstance(pid, int): + continue + if pid not in seen: found.append((pid, str(entry.get("argv") or ""))) except Exception: pass # ledger unavailable → scan-only behavior @@ -540,7 +542,7 @@ def _is_desktop_local_serve_cmdline(command: str) -> bool: def _process_ppid(pid: int) -> int | None: - """Best-effort parent pid; None on failure (always on Windows: desktop tree-kill reaps there).""" + """Best-effort parent pid; None on failure (always None on Windows: desktop tree-kill reaps).""" try: if sys.platform == "win32": return None diff --git a/hermes_cli/dashboard_register.py b/hermes_cli/dashboard_register.py index 0dca8de774..03751e916a 100644 --- a/hermes_cli/dashboard_register.py +++ b/hermes_cli/dashboard_register.py @@ -99,7 +99,8 @@ def _register_self_hosted_client( raise RuntimeError( detail or "Your account is not permitted to register a self-hosted dashboard." ) from exc - raise RuntimeError(f"Portal returned HTTP {exc.code}" + (f": {detail}" if detail else "")) from exc + suffix = f": {detail}" if detail else "" + raise RuntimeError(f"Portal returned HTTP {exc.code}{suffix}") from exc except urllib.error.URLError as exc: raise RuntimeError(f"Could not reach Nous Portal at {portal_base_url}: {exc.reason}") from exc diff --git a/hermes_cli/debug.py b/hermes_cli/debug.py index 3403f33c42..9cfc106507 100644 --- a/hermes_cli/debug.py +++ b/hermes_cli/debug.py @@ -63,7 +63,7 @@ def _load_pending() -> list[dict]: return [] if not isinstance(data, list): return [] - return [e for e in data if isinstance(e, dict) and "url" in e and "expire_at" in e] + return [e for e in data if isinstance(e, dict) and {"url", "expire_at"} <= e.keys()] def _save_pending(entries: list[dict]) -> None: @@ -99,18 +99,14 @@ def _sweep_expired_pastes(now: Optional[float] = None) -> tuple[int, int]: if expire_at > current: remaining.append(entry) continue - try: - if delete_paste(entry.get("url", "")): - deleted += 1 - continue + gone = delete_paste(entry.get("url", "")) except Exception: - pass # network hiccup, 404 (already gone), ... - - if expire_at + 86400 > current: - remaining.append(entry) + gone = False # network hiccup, 404 (already gone), ... + if gone or expire_at + 86400 <= current: + deleted += 1 # deleted, or given up on → count as reaped else: - deleted += 1 # count as reaped + remaining.append(entry) if deleted: _save_pending(remaining) @@ -299,7 +295,9 @@ def _redact_log_text(text: str) -> str: return _EMAIL_ADDRESS_RE.sub("[REDACTED_EMAIL]", text) -def _read_tail_bytes(log_path: Path, size: int, max_bytes: int, tail_lines: int) -> tuple[bytes, bool]: +def _read_tail_bytes( + log_path: Path, size: int, max_bytes: int, tail_lines: int, +) -> tuple[bytes, bool]: """Read the whole file, or enough of its tail for both views → (raw, truncated). For oversized files, read backwards until we have ``max_bytes`` for the standalone upload @@ -313,7 +311,8 @@ def _read_tail_bytes(log_path: Path, size: int, max_bytes: int, tail_lines: int) chunks: list[bytes] = [] total = 0 newline_count = 0 - while pos > 0 and (total < max_bytes or newline_count <= tail_lines + 1) and total < max_bytes * 2: + while (pos > 0 and total < max_bytes * 2 + and (total < max_bytes or newline_count <= tail_lines + 1)): read_size = min(chunk_size, pos) pos -= read_size f.seek(pos) @@ -384,7 +383,9 @@ def _tail_budget(name: str, log_lines: int) -> int: return log_lines if name == "agent" else min(log_lines, 100) -def _capture_default_log_snapshots(log_lines: int, *, redact: bool = True) -> dict[str, LogSnapshot]: +def _capture_default_log_snapshots( + log_lines: int, *, redact: bool = True, +) -> dict[str, LogSnapshot]: """Capture all logs used by debug-share exactly once.""" return { name: _capture_log_snapshot(name, tail_lines=_tail_budget(name, log_lines), redact=redact) @@ -414,7 +415,8 @@ def _capture_dump() -> str: def collect_debug_report( - *, log_lines: int = 200, dump_text: str = "", log_snapshots: Optional[dict[str, LogSnapshot]] = None, + *, log_lines: int = 200, dump_text: str = "", + log_snapshots: Optional[dict[str, LogSnapshot]] = None, ) -> str: """Build the summary debug report (system dump + log tails) as upload-ready text. @@ -471,7 +473,8 @@ def collect_share_bundle(log_lines: int = 200, redact: bool = True) -> dict[str, dump_text = _capture_dump() log_snapshots = _capture_default_log_snapshots(log_lines, redact=redact) - report = collect_debug_report(log_lines=log_lines, dump_text=dump_text, log_snapshots=log_snapshots) + report = collect_debug_report(log_lines=log_lines, dump_text=dump_text, + log_snapshots=log_snapshots) banner = _REDACTION_BANNER if redact else "" bundle: dict[str, str] = {"report": banner + report} for name in _FULL_LOGS: @@ -506,7 +509,9 @@ class DebugShareResult: report: str = "" # the summary report text (kept for local fallback) -def build_debug_share(*, log_lines: int = 200, expiry: int = 7, redact: bool = True) -> DebugShareResult: +def build_debug_share( + *, log_lines: int = 200, expiry: int = 7, redact: bool = True, +) -> DebugShareResult: """Collect the debug report + full logs, upload each, return the URLs. Shared core behind ``hermes debug share`` and the dashboard ``POST /api/ops/debug-share``. @@ -517,7 +522,9 @@ def build_debug_share(*, log_lines: int = 200, expiry: int = 7, redact: bool = T bundle = collect_share_bundle(log_lines=log_lines, redact=redact) if redact: - logger.info("hermes debug share: applied force-mode redaction to log snapshots before upload") + logger.info( + "hermes debug share: applied force-mode redaction to log snapshots before upload" + ) report = bundle["report"] @@ -607,12 +614,9 @@ def run_debug_share(args): print("\nDebug report uploaded:") for label, url in result.urls.items(): print(f" {label:<{label_width}} {url}") - if result.failures: print(f"\n (failed to upload: {', '.join(result.failures)})") - - hours = result.auto_delete_seconds // 3600 - print(f"\n⏱ Pastes will auto-delete in {hours} hours.") + print(f"\n⏱ Pastes will auto-delete in {result.auto_delete_seconds // 3600} hours.") print("To delete now: hermes debug delete ") print("\nShare these links with the Hermes team for support.") @@ -633,7 +637,7 @@ _NOUS_PRIVACY_NOTICE = """\ def _run_debug_share_nous(args, *, log_lines: int, redact: bool) -> None: - """``hermes debug share --nous``: gzip the same bundle into the Nous envelope, upload to Nous-S3.""" + """``hermes debug share --nous``: gzip the same bundle into the Nous envelope → Nous-S3.""" from hermes_cli.diagnostics_upload import share_to_nous print(_NOUS_PRIVACY_NOTICE) @@ -664,17 +668,12 @@ def _run_debug_share_nous(args, *, log_lines: int, redact: bool) -> None: sys.exit(1) view_url = res.get("viewUrl") or res.get("view_url") - print("\nDebug bundle uploaded to Nous (private):") - if view_url: - print(f" View URL {view_url}") - else: - print(f" (no view URL returned; upload id: {res.get('id', '?')})") - expires_at = res.get("expiresAt") or res.get("expires_at") - if expires_at: - print(f"\n⏱ Auto-deletes at {expires_at} (14-day retention).") - else: - print("\n⏱ Auto-deletes after 14 days.") + print("\nDebug bundle uploaded to Nous (private):") + print(f" View URL {view_url}" if view_url + else f" (no view URL returned; upload id: {res.get('id', '?')})") + print(f"\n⏱ Auto-deletes at {expires_at} (14-day retention)." if expires_at + else "\n⏱ Auto-deletes after 14 days.") print( "\nShare this private link with the Nous team — only Nous staff " @@ -713,13 +712,12 @@ def run_debug(args): # Opportunistic sweep of expired pastes on every ``hermes debug`` call. _best_effort_sweep_expired_pastes() - subcmd = getattr(args, "debug_command", None) - if subcmd == "share": - run_debug_share(args) - elif subcmd == "delete": - run_debug_delete(args) - else: + handlers = {"share": run_debug_share, "delete": run_debug_delete} + handler = handlers.get(getattr(args, "debug_command", None)) + if handler is None: print(_DEBUG_USAGE) + else: + handler(args) _DEBUG_USAGE = """\ From 87d30c47011347c93e7a492a4129e053c44701fb Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:30:58 -0700 Subject: [PATCH 14/29] =?UTF-8?q?refactor(hermes=5Fcli):=20cron/debug=20?= =?UTF-8?q?=E2=80=94=20merge=20consecutive=20prints=20into=20single=20lite?= =?UTF-8?q?rals=20(byte-identical=20output)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/cron.py | 102 ++++++++++++------------------- hermes_cli/dashboard_register.py | 3 +- hermes_cli/debug.py | 29 ++++----- 3 files changed, 53 insertions(+), 81 deletions(-) diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index 9d67b08b68..40fbb0aaf4 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -100,9 +100,9 @@ def _warn_if_gateway_not_running() -> None: return print(color(" ⚠ Gateway is not running — jobs won't fire automatically.", Colors.YELLOW)) - print(color(" Start it with: hermes gateway install", Colors.DIM)) - print(color(" sudo hermes gateway install --system # Linux servers", Colors.DIM)) - print(color(" Check status: hermes cron status", Colors.DIM)) + print(color(" Start it with: hermes gateway install\n" + " sudo hermes gateway install --system # Linux servers\n" + " Check status: hermes cron status", Colors.DIM)) def _format_lateness(seconds: float) -> str: @@ -143,19 +143,16 @@ def _dispatch_display(dispatch: dict) -> Optional[str]: if kind == "on_time": return color(f"on time (scheduled {scheduled})", Colors.DIM) label = "catch-up after missed fire" if kind == "catch_up" else "late" - return ( - color(f"⚠ {label}: ", Colors.YELLOW) - + f"scheduled {scheduled}, ran {actual} " - + color(f"({lateness} late)", Colors.YELLOW) - ) + return (color(f"⚠ {label}: ", Colors.YELLOW) + f"scheduled {scheduled}, ran {actual} " + + color(f"({lateness} late)", Colors.YELLOW)) def _print_banner(title: str) -> None: """Boxed cyan section header shared by ``cron list`` and ``cron incidents``.""" print() - print(color("┌" + "─" * 73 + "┐", Colors.CYAN)) - print(color("│" + " " * 25 + title.ljust(48) + "│", Colors.CYAN)) - print(color("└" + "─" * 73 + "┘", Colors.CYAN)) + rule = "─" * 73 + for line in (f"┌{rule}┐", "│" + " " * 25 + title.ljust(48) + "│", f"└{rule}┘"): + print(color(line, Colors.CYAN)) print() @@ -175,8 +172,8 @@ def cron_list(show_all: bool = False): jobs = list_jobs(include_disabled=show_all) if not jobs: - print(color("No scheduled jobs.", Colors.DIM)) - print(color("Create one with 'hermes cron create ...' or the /cron command in chat.", Colors.DIM)) + print(color("No scheduled jobs.\nCreate one with 'hermes cron create ...' " + "or the /cron command in chat.", Colors.DIM)) return _print_banner("Scheduled Jobs") @@ -357,11 +354,8 @@ def cron_incidents(args) -> int: if inc.get("output_file"): print(f" Output: {inc['output_file']}") print() - print(color( - f" {len(incidents)} incident(s) | ack one with: " - "hermes cron incidents ack ", - Colors.DIM, - )) + print(color(f" {len(incidents)} incident(s) | ack one with: hermes cron incidents ack ", + Colors.DIM)) return 0 @@ -401,22 +395,18 @@ def _print_ticker_health(pids: list) -> None: # No heartbeat file: ticker never started (non-cron profile, gateway started moments # ago, or a config issue blocking the ticker). _warn("⚠ Gateway is running but the cron ticker has not reported a heartbeat.") - print(" Cron jobs will NOT fire until the ticker writes its first heartbeat.") - print(" If the gateway just started, wait ~60s and re-run `hermes cron status`.") - print(" If heartbeat never appears, restart: hermes gateway restart") + print(" Cron jobs will NOT fire until the ticker writes its first heartbeat.\n" + " If the gateway just started, wait ~60s and re-run `hermes cron status`.\n" + " If heartbeat never appears, restart: hermes gateway restart") elif hb_age > STALE_AFTER: # No heartbeat at all → the ticker thread is gone. - _warn( - "⚠ Gateway is running but the cron ticker looks STALLED — " - f"no heartbeat for {int(hb_age)}s (expected every ~60s)." - ) + _warn("⚠ Gateway is running but the cron ticker looks STALLED — " + f"no heartbeat for {int(hb_age)}s (expected every ~60s).") print(" Cron jobs may NOT be firing. Restart: hermes gateway restart") elif ok_age is not None and ok_age > STALE_AFTER: # Loop alive (fresh heartbeat) but no tick SUCCEEDED in a long time → every tick fails. - _warn( - "⚠ Gateway and cron ticker are running, but no tick has " - f"succeeded in {int(ok_age)}s — ticks may be failing." - ) + _warn("⚠ Gateway and cron ticker are running, but no tick has " + f"succeeded in {int(ok_age)}s — ticks may be failing.") last_error = get_ticker_last_error() if last_error: # Show WHY ticks fail — e.g. a root-rewritten jobs.json (PermissionError) that @@ -447,16 +437,10 @@ def cron_status(): # An external provider (e.g. Chronos) arms one external one-shot per job, fired by a # NAS-mediated webhook: between fires there is intentionally NO ticker thread and NO # heartbeat file, so the ticker-liveness heuristics would always say "stalled". - print(color( - f"✓ Cron provider: {provider} — jobs fire via the managed scheduler, " - "not the in-process ticker.", - Colors.GREEN, - )) - print(color( - " (No ticker heartbeat is expected for an external provider; " - "due jobs are delivered by an authenticated webhook.)", - Colors.DIM, - )) + print(color(f"✓ Cron provider: {provider} — jobs fire via the managed scheduler, " + "not the in-process ticker.", Colors.GREEN)) + print(color(" (No ticker heartbeat is expected for an external provider; " + "due jobs are delivered by an authenticated webhook.)", Colors.DIM)) else: pids = find_gateway_pids() gateway_alive_via_lock = False @@ -476,11 +460,11 @@ def cron_status(): _print_ticker_health(pids) else: print(color("✗ Gateway is not running — cron jobs will NOT fire", Colors.RED)) - print() - print(" To enable automatic execution:") - print(" hermes gateway install # Install as a user service") - print(" sudo hermes gateway install --system # Linux servers: boot-time system service") - print(" hermes gateway # Or run in foreground") + print("\n To enable automatic execution:\n" + " hermes gateway install # Install as a user service\n" + " sudo hermes gateway install --system " + "# Linux servers: boot-time system service\n" + " hermes gateway # Or run in foreground") print() _print_active_jobs_summary(list_jobs(include_disabled=False)) @@ -509,12 +493,10 @@ def _print_active_jobs_summary(jobs) -> None: Colors.YELLOW)) for j in late: d = j["last_dispatch"] - print( - f" {j.get('id', '?')} {j.get('name', '(unnamed)')}: " - f"scheduled {d.get('scheduled_at', '?')}, " - f"ran {d.get('dispatched_at', '?')} " - + color(f"({_format_lateness(d.get('lateness_seconds', 0))} late)", Colors.YELLOW) - ) + late_by = _format_lateness(d.get("lateness_seconds", 0)) + print(f" {j.get('id', '?')} {j.get('name', '(unnamed)')}: " + f"scheduled {d.get('scheduled_at', '?')}, ran {d.get('dispatched_at', '?')} " + + color(f"({late_by} late)", Colors.YELLOW)) def _scripts_dir_for_cron() -> Path: @@ -679,8 +661,7 @@ def cron_create(args): print(color(f"Failed to create job: {result.get('error', 'unknown error')}", Colors.RED)) return 1 print(color(f"Created job: {result['job_id']}", Colors.GREEN)) - print(f" Name: {result['name']}") - print(f" Schedule: {result['schedule']}") + print(f" Name: {result['name']}\n Schedule: {result['schedule']}") if result.get("skills"): print(f" Skills: {', '.join(result['skills'])}") _print_job_details(result.get("job", {})) @@ -727,12 +708,9 @@ def cron_edit(args): updated = result["job"] print(color(f"Updated job: {updated['job_id']}", Colors.GREEN)) - print(f" Name: {updated['name']}") - print(f" Schedule: {updated['schedule']}") - if updated.get("skills"): - print(f" Skills: {', '.join(updated['skills'])}") - else: - print(" Skills: none") + print(f" Name: {updated['name']}\n Schedule: {updated['schedule']}") + print(f" Skills: {', '.join(updated['skills'])}" if updated.get("skills") else + " Skills: none") _print_job_details(updated) return 0 @@ -854,8 +832,8 @@ def cron_notepad(args) -> int: print(color(f"Notepad for job {job_id} is empty.", Colors.DIM)) return 0 for note in notes: - print(f" {color(note['key'], Colors.YELLOW)} = {note['value']}") - print(f" {color('updated: ' + str(note['updated_at']), Colors.DIM)}") + print(f" {color(note['key'], Colors.YELLOW)} = {note['value']}\n" + f" {color('updated: ' + str(note['updated_at']), Colors.DIM)}") return 0 except ValueError as exc: print(color(f"Notepad error: {exc}", Colors.RED)) @@ -891,6 +869,6 @@ def cron_command(args): if handler is not None: return handler(args) - print(f"Unknown cron command: {subcmd}") - print("Usage: hermes cron [list|create|edit|pause|resume|run|remove|status|runs|doctor|tick]") + print(f"Unknown cron command: {subcmd}\n" + "Usage: hermes cron [list|create|edit|pause|resume|run|remove|status|runs|doctor|tick]") sys.exit(1) diff --git a/hermes_cli/dashboard_register.py b/hermes_cli/dashboard_register.py index 03751e916a..5045ad17cf 100644 --- a/hermes_cli/dashboard_register.py +++ b/hermes_cli/dashboard_register.py @@ -116,8 +116,7 @@ def _print_post_register_hint( """Print the success summary + the gate-engagement caveat.""" from hermes_cli.config import get_env_path - print(f"\n Wrote to {get_env_path()}:") - print(" HERMES_DASHBOARD_OAUTH_CLIENT_ID=" + str(client_id)) + print(f"\n Wrote to {get_env_path()}:\n HERMES_DASHBOARD_OAUTH_CLIENT_ID={client_id}") if wrote_portal_url: print(" HERMES_DASHBOARD_PORTAL_URL=" + str(portal_base_url)) if public_url: diff --git a/hermes_cli/debug.py b/hermes_cli/debug.py index 9cfc106507..6da739e807 100644 --- a/hermes_cli/debug.py +++ b/hermes_cli/debug.py @@ -600,8 +600,7 @@ def run_debug_share(args): print(_PRIVACY_NOTICE) if not _confirm_upload(args): return - print("Collecting debug report...") - print("Uploading...") + print("Collecting debug report...\nUploading...") try: result = build_debug_share(log_lines=log_lines, expiry=expiry, redact=redact) @@ -616,9 +615,9 @@ def run_debug_share(args): print(f" {label:<{label_width}} {url}") if result.failures: print(f"\n (failed to upload: {', '.join(result.failures)})") - print(f"\n⏱ Pastes will auto-delete in {result.auto_delete_seconds // 3600} hours.") - print("To delete now: hermes debug delete ") - print("\nShare these links with the Hermes team for support.") + print(f"\n⏱ Pastes will auto-delete in {result.auto_delete_seconds // 3600} hours.\n" + "To delete now: hermes debug delete \n" + "\nShare these links with the Hermes team for support.") _NOUS_PRIVACY_NOTICE = """\ @@ -675,24 +674,20 @@ def _run_debug_share_nous(args, *, log_lines: int, redact: bool) -> None: print(f"\n⏱ Auto-deletes at {expires_at} (14-day retention)." if expires_at else "\n⏱ Auto-deletes after 14 days.") - print( - "\nShare this private link with the Nous team — only Nous staff " - "(via Google login) can open it." - ) - print( - "\nPick up the discussion in:\n" - " GitHub Issues https://github.com/NousResearch/hermes-agent/issues\n" - " Nous Portal Support https://portal.nousresearch.com/help\n" - " Discord https://discord.gg/NousResearch" - ) + print("\nShare this private link with the Nous team — only Nous staff " + "(via Google login) can open it.\n" + "\nPick up the discussion in:\n" + " GitHub Issues https://github.com/NousResearch/hermes-agent/issues\n" + " Nous Portal Support https://portal.nousresearch.com/help\n" + " Discord https://discord.gg/NousResearch") def run_debug_delete(args): """Delete one or more paste URLs uploaded by /debug.""" urls = getattr(args, "urls", []) if not urls: - print("Usage: hermes debug delete [ ...]") - print(" Deletes paste.rs pastes uploaded by 'hermes debug share'.") + print("Usage: hermes debug delete [ ...]\n" + " Deletes paste.rs pastes uploaded by 'hermes debug share'.") return for url in urls: From e4f8ab0e8ef44a5b618a5a3b064b0a23f4dd114a Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:48:27 -0700 Subject: [PATCH 15/29] =?UTF-8?q?refactor(hermes=5Fcli):=20group=20D=20?= =?UTF-8?q?=E2=80=94=20flatten=20notepad/doctor/lateness/lockfile=20branch?= =?UTF-8?q?es,=20single-raise=20HTTP=20error=20mapping?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/cron.py | 79 +++++++++++++------------------- hermes_cli/dashboard_procs.py | 17 ++----- hermes_cli/dashboard_register.py | 18 ++++---- 3 files changed, 46 insertions(+), 68 deletions(-) diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index 40fbb0aaf4..d44e229377 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -116,14 +116,8 @@ def _format_lateness(seconds: float) -> str: minutes, _ = divmod(seconds, 60) hours, minutes = divmod(minutes, 60) days, hours = divmod(hours, 24) - parts = [] - if days: - parts.append(f"{days}d") - if hours: - parts.append(f"{hours}h") - if minutes and not days: - parts.append(f"{minutes}m") - return " ".join(parts) or "0m" + parts = [(days, "d"), (hours, "h"), (minutes if not days else 0, "m")] + return " ".join(f"{n}{unit}" for n, unit in parts if n) or "0m" def _dispatch_display(dispatch: dict) -> Optional[str]: @@ -545,12 +539,10 @@ def _next_run_overdue_issue(next_run: str) -> Optional[str]: if dt.tzinfo is None: dt = dt.replace(tzinfo=timezone.utc) overdue_s = (datetime.now(timezone.utc) - dt).total_seconds() - if overdue_s > _OVERDUE_GRACE_SECONDS: - hours = overdue_s / 3600 - if hours >= 1: - return f"next_run_at is {hours:.1f}h overdue — job is not firing (is the scheduler running?)" - return f"next_run_at is {overdue_s / 60:.0f}m overdue — job is not firing (is the scheduler running?)" - return None + if overdue_s <= _OVERDUE_GRACE_SECONDS: + return None + amount = f"{overdue_s / 3600:.1f}h" if overdue_s >= 3600 else f"{overdue_s / 60:.0f}m" + return f"next_run_at is {amount} overdue — job is not firing (is the scheduler running?)" def _cron_doctor_issues_for_job(job: Dict[str, Any]) -> List[str]: @@ -600,10 +592,8 @@ def cron_doctor() -> int: if not findings: print(color("✓ Cron doctor found no issues", Colors.GREEN)) - if jobs: - print(color(f" Checked {len(jobs)} active job(s).", Colors.DIM)) - else: - print(color(" No active jobs configured.", Colors.DIM)) + print(color(f" Checked {len(jobs)} active job(s)." if jobs else " No active jobs configured.", + Colors.DIM)) return 0 issue_count = sum(len(issues) for _, issues in findings) @@ -705,7 +695,6 @@ def cron_edit(args): if not result.get("success"): print(color(f"Failed to update job: {result.get('error', 'unknown error')}", Colors.RED)) return 1 - updated = result["job"] print(color(f"Updated job: {updated['job_id']}", Colors.GREEN)) print(f" Name: {updated['name']}\n Schedule: {updated['schedule']}") @@ -807,34 +796,32 @@ def cron_notepad(args) -> int: return 1 try: - if action in ("set", "get", "delete"): - usage_args = "set " if action == "set" else f"{action} " - if key is None or (action == "set" and value is None): - print(color(f"Usage: hermes cron notepad {usage_args}", Colors.RED)) - return 1 - if action == "set": - notepad.set_note(job_id, key, value) - print(color(f"Set notepad key '{key}' for job {job_id}.", Colors.GREEN)) - return 0 - if action == "get": - stored = notepad.get_note(job_id, key) - if stored is not None: - print(stored) - return 0 - elif notepad.delete_note(job_id, key): - print(color(f"Deleted notepad key '{key}' for job {job_id}.", Colors.GREEN)) - return 0 - print(color(f"No notepad key '{key}' for job {job_id}.", Colors.YELLOW)) - return 1 - - notes = notepad.list_notes(job_id) - if not notes: - print(color(f"Notepad for job {job_id} is empty.", Colors.DIM)) + if action not in ("set", "get", "delete"): # list (default) + notes = notepad.list_notes(job_id) + if not notes: + print(color(f"Notepad for job {job_id} is empty.", Colors.DIM)) + for note in notes: + print(f" {color(note['key'], Colors.YELLOW)} = {note['value']}\n" + f" {color('updated: ' + str(note['updated_at']), Colors.DIM)}") return 0 - for note in notes: - print(f" {color(note['key'], Colors.YELLOW)} = {note['value']}\n" - f" {color('updated: ' + str(note['updated_at']), Colors.DIM)}") - return 0 + usage_args = "set " if action == "set" else f"{action} " + if key is None or (action == "set" and value is None): + print(color(f"Usage: hermes cron notepad {usage_args}", Colors.RED)) + return 1 + if action == "set": + notepad.set_note(job_id, key, value) + print(color(f"Set notepad key '{key}' for job {job_id}.", Colors.GREEN)) + return 0 + if action == "get": + stored = notepad.get_note(job_id, key) + if stored is not None: + print(stored) + return 0 + elif notepad.delete_note(job_id, key): + print(color(f"Deleted notepad key '{key}' for job {job_id}.", Colors.GREEN)) + return 0 + print(color(f"No notepad key '{key}' for job {job_id}.", Colors.YELLOW)) + return 1 except ValueError as exc: print(color(f"Notepad error: {exc}", Colors.RED)) return 1 diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py index 0171877f09..f37f2ad59c 100644 --- a/hermes_cli/dashboard_procs.py +++ b/hermes_cli/dashboard_procs.py @@ -626,21 +626,14 @@ def _lock_owned_serve_pids(base_dir: Path | None = None) -> set[int]: # Mirror validateOwnershipId(): exactly 32 lowercase hex chars. if not entry.is_dir() or not _is_hex(ownership_id, 32) or not lock_path.is_file(): continue - with open(lock_path, "rb") as handle: - data = handle.read() - except OSError: - continue - if len(data) > 65536: - continue - try: + data = lock_path.read_bytes() + if len(data) > 65536: + continue parsed = json.loads(data) - except (UnicodeDecodeError, ValueError): + except (OSError, UnicodeDecodeError, ValueError): continue if _valid_lockfile_payload(parsed, ownership_id): - try: - owned.add(int(parsed["pid"])) - except (TypeError, ValueError): - continue + owned.add(parsed["pid"]) # validated as int above return owned diff --git a/hermes_cli/dashboard_register.py b/hermes_cli/dashboard_register.py index 5045ad17cf..5f1f2cf716 100644 --- a/hermes_cli/dashboard_register.py +++ b/hermes_cli/dashboard_register.py @@ -86,21 +86,19 @@ def _register_self_hosted_client( payload = json.loads(resp.read().decode()) except urllib.error.HTTPError as exc: # Structured JSON errors: {error, error_description}. - detail = "" try: err_body = json.loads(exc.read().decode()) detail = err_body.get("error_description") or err_body.get("error") or "" except Exception: - pass + detail = "" if exc.code == 401: - raise RuntimeError("Nous Portal rejected the access token (401). Try `hermes auth add " - "nous` to re-authenticate.") from exc - if exc.code == 403: - raise RuntimeError( - detail or "Your account is not permitted to register a self-hosted dashboard." - ) from exc - suffix = f": {detail}" if detail else "" - raise RuntimeError(f"Portal returned HTTP {exc.code}{suffix}") from exc + message = ("Nous Portal rejected the access token (401). " + "Try `hermes auth add nous` to re-authenticate.") + elif exc.code == 403: + message = detail or "Your account is not permitted to register a self-hosted dashboard." + else: + message = f"Portal returned HTTP {exc.code}" + (f": {detail}" if detail else "") + raise RuntimeError(message) from exc except urllib.error.URLError as exc: raise RuntimeError(f"Could not reach Nous Portal at {portal_base_url}: {exc.reason}") from exc From 2295bcc15abd8ff343ab0f6c42052432ad3a3902 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:57:16 -0700 Subject: [PATCH 16/29] =?UTF-8?q?refactor(hermes=5Fcli):=20copilot=5Fauth.?= =?UTF-8?q?py=20=E2=80=94=20one=20guarded=20accessor=20for=20the=20three?= =?UTF-8?q?=20JWT-store=20operations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/copilot_auth.py | 72 +++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/hermes_cli/copilot_auth.py b/hermes_cli/copilot_auth.py index e8040871d1..e75f1f783a 100644 --- a/hermes_cli/copilot_auth.py +++ b/hermes_cli/copilot_auth.py @@ -346,6 +346,27 @@ def _write_jwt_store(path: Path, store: dict) -> None: os.replace(tmp, path) +def _jwt_disk_path() -> Optional[Path]: + """Path to the on-disk exchanged-JWT cache (profile-aware), or None.""" + try: + from hermes_constants import get_hermes_home + return Path(get_hermes_home()) / _JWT_DISK_FILENAME + except Exception: + return None + + +def _with_jwt_store(verb: str, op): + """Run ``op(path, store_or_None)`` against the disk store; any failure is logged, never raised.""" + path = _jwt_disk_path() + if not path: + return None + try: + return op(path, _read_jwt_store(path)) + except Exception as exc: + logger.debug("Failed to %s Copilot JWT: %s", verb, exc) + return None + + def evict_cached_exchanged_token(raw_token: str) -> None: """Drop any cached exchanged JWT for ``raw_token`` (in-process + on-disk). @@ -359,61 +380,40 @@ def evict_cached_exchanged_token(raw_token: str) -> None: # Eviction is an explicit "force a fresh exchange" signal, so the negative-cache entry # must go too — the next exchange_copilot_token() must be allowed to hit the network. _exchange_failure_cache.pop(fp, None) - path = _jwt_disk_path() - if not path: - return - try: - store = _read_jwt_store(path) + + def _evict(path, store): if store is not None and fp in store: del store[fp] _write_jwt_store(path, store) - except Exception as exc: - logger.debug("Failed to evict cached Copilot JWT: %s", exc) - -def _jwt_disk_path() -> Optional[Path]: - """Path to the on-disk exchanged-JWT cache (profile-aware), or None.""" - try: - from hermes_constants import get_hermes_home - return Path(get_hermes_home()) / _JWT_DISK_FILENAME - except Exception: - return None + _with_jwt_store("evict cached", _evict) def _load_jwt_from_disk(fp: str) -> Optional[tuple[str, float, Optional[str]]]: """Persisted exchanged JWT for ``fp`` → (api_token, expires_at, base_url), or None.""" - path = _jwt_disk_path() - if not path: - return None - try: - entry = (_read_jwt_store(path) or {}).get(fp) + def _load(path, store): + entry = (store or {}).get(fp) if not isinstance(entry, dict): return None api_token = entry.get("api_token", "") expires_at = float(entry.get("expires_at", 0) or 0) - if api_token and expires_at: - return api_token, expires_at, entry.get("base_url") - except Exception as exc: - logger.debug("Failed to load persisted Copilot JWT: %s", exc) - return None + return (api_token, expires_at, entry.get("base_url")) if api_token and expires_at else None + + return _with_jwt_store("load persisted", _load) def _save_jwt_to_disk(fp: str, api_token: str, expires_at: float, base_url: Optional[str]) -> None: """Persist an exchanged JWT (0o600), pruning expired entries.""" - path = _jwt_disk_path() - if not path: - return - try: + def _save(path, store): now = time.time() - store = { - k: v - for k, v in (_read_jwt_store(path) or {}).items() + kept = { + k: v for k, v in (store or {}).items() if isinstance(v, dict) and float(v.get("expires_at", 0) or 0) > now } - store[fp] = {"api_token": api_token, "expires_at": expires_at, "base_url": base_url} - _write_jwt_store(path, store) - except Exception as exc: - logger.debug("Failed to persist Copilot JWT: %s", exc) + kept[fp] = {"api_token": api_token, "expires_at": expires_at, "base_url": base_url} + _write_jwt_store(path, kept) + + _with_jwt_store("persist", _save) # Hard wall-clock cap for the token-exchange HTTP call. urllib's ``timeout`` only bounds From 2cac493fecb2a2000f97c95b1b2fed68c923f6fa Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:10:21 -0700 Subject: [PATCH 17/29] =?UTF-8?q?refactor(hermes=5Fcli):=20group=20D=20?= =?UTF-8?q?=E2=80=94=20drop=20blank=20line=20after=20local=20imports=20(AS?= =?UTF-8?q?T-neutral)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/container_boot.py | 1 - hermes_cli/cron.py | 16 ---------------- hermes_cli/dashboard_procs.py | 10 ---------- hermes_cli/dashboard_register.py | 5 ----- hermes_cli/debug.py | 4 ---- 5 files changed, 36 deletions(-) diff --git a/hermes_cli/container_boot.py b/hermes_cli/container_boot.py index 262b0d838a..9348aae643 100644 --- a/hermes_cli/container_boot.py +++ b/hermes_cli/container_boot.py @@ -87,7 +87,6 @@ def reconcile_profile_gateways( # precedence over config.yaml and otherwise preserves the configured value. from gateway.config import load_gateway_config from utils import is_truthy_value - try: multiplex_profiles = load_gateway_config().multiplex_profiles except Exception: diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index d44e229377..04912889dc 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -35,7 +35,6 @@ def _normalize_skills(single_skill=None, skills: Optional[Iterable[str]] = None) def _cron_api(**kwargs): from tools.cronjob_tools import cronjob as cronjob_tool - return json.loads(cronjob_tool(**kwargs)) @@ -47,7 +46,6 @@ def _active_cron_provider_name() -> str: """ try: from cron.scheduler_provider import resolve_cron_scheduler - return resolve_cron_scheduler().name or "builtin" except Exception: return "builtin" @@ -70,7 +68,6 @@ def _builtin_gateway_liveness() -> Optional[bool]: # gateway just after a restart). try: from gateway.status import is_gateway_runtime_lock_active - if is_gateway_runtime_lock_active(): return True except Exception: @@ -162,7 +159,6 @@ _STATE_BADGES = {"paused": ("[paused]", Colors.YELLOW), "completed": ("[complete def cron_list(show_all: bool = False): """List all scheduled jobs.""" from cron.jobs import effective_job_state, list_jobs - jobs = list_jobs(include_disabled=show_all) if not jobs: @@ -289,7 +285,6 @@ def cron_tick(): def cron_runs(job_id: Optional[str] = None, limit: int = 20): """Show indexed durable cron execution history.""" from cron.executions import list_executions - records = list_executions(job_id=job_id, limit=limit) if not records: print("No cron execution attempts recorded.") @@ -312,7 +307,6 @@ def cron_incidents(args) -> int: acking closes an incident so its failure ping stays silent until the error signature changes. """ from cron.incidents import ack_incident, list_incidents - action = getattr(args, "incident_action", "list") if action == "ack": incident_id = getattr(args, "incident_id", None) @@ -372,7 +366,6 @@ def _print_ticker_health(pids: list) -> None: TICKER_INTERVAL_SECONDS, ) from cron.scheduler import _is_fd_exhaustion_text as _cron_is_fd_exhaustion_text - # ~3 missed ticker iterations (+ slack) before declaring trouble; derived from the shared # interval so the threshold tracks the ticker cadence (= 200s at the 60s default). STALE_AFTER = TICKER_INTERVAL_SECONDS * 3 + 20 @@ -423,7 +416,6 @@ def cron_status(): """Show cron execution status.""" from cron.jobs import list_jobs from hermes_cli.gateway import find_gateway_pids - print() provider = _active_cron_provider_name() @@ -444,7 +436,6 @@ def cron_status(): # process is alive. Only declare "not running" when both agree. try: from gateway.status import get_running_pid, is_gateway_runtime_lock_active - gateway_alive_via_lock = is_gateway_runtime_lock_active() lock_pid = get_running_pid() if gateway_alive_via_lock else None pids = [lock_pid] if lock_pid else pids @@ -500,7 +491,6 @@ def _scripts_dir_for_cron() -> Path: profile-aware callers that monkeypatch cron storage inspect the same Hermes home. """ from cron.jobs import CRON_DIR - return CRON_DIR.parent / "scripts" @@ -531,7 +521,6 @@ _OVERDUE_GRACE_SECONDS = 15 * 60 def _next_run_overdue_issue(next_run: str) -> Optional[str]: """Issue string when ``next_run_at`` is parked in the past.""" from datetime import datetime, timezone - try: dt = datetime.fromisoformat(next_run.replace("Z", "+00:00")) except ValueError: @@ -586,7 +575,6 @@ def _cron_doctor_issues_for_job(job: Dict[str, Any]) -> List[str]: def cron_doctor() -> int: """Run read-only cron health checks and return a shell-friendly status.""" from cron.jobs import list_jobs - jobs = list_jobs(include_disabled=False) findings = [(job, issues) for job in jobs if (issues := _cron_doctor_issues_for_job(job))] @@ -662,7 +650,6 @@ def cron_create(args): def cron_edit(args): from cron.jobs import AmbiguousJobReference, resolve_job_ref - try: job = resolve_job_ref(args.job_id) except AmbiguousJobReference as exc: @@ -716,7 +703,6 @@ def _job_action(action: str, job_id: str, success_verb: str) -> int: # callers (tests, embedding apps) are not tainted. try: from gateway.session_context import _SESSION_ASYNC_DELIVERY - _stateless_token = _SESSION_ASYNC_DELIVERY.set(False) except Exception: _stateless_token = None @@ -761,7 +747,6 @@ def cron_resume(args) -> int: return 1 return _job_action("resume", args.job_id, "Resumed") from cron.jobs import AmbiguousJobReference, _hermes_now, rearm_oneshot - if run_now: run_at = _hermes_now().isoformat() try: @@ -785,7 +770,6 @@ def cron_notepad(args) -> int: notepads into the job prompt on each run. """ from cron import notepad - job_id = str(getattr(args, "job_id", "") or "") action = getattr(args, "notepad_action", None) or "list" key = getattr(args, "key", None) diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py index f37f2ad59c..5d3074267c 100644 --- a/hermes_cli/dashboard_procs.py +++ b/hermes_cli/dashboard_procs.py @@ -23,7 +23,6 @@ _PS_RUN_KWARGS = dict(capture_output=True, text=True, encoding="utf-8", errors=" def _m(): """Lazy ``hermes_cli.main`` reference (call-time; keeps patches working).""" from hermes_cli import main - return main @@ -47,7 +46,6 @@ def _iter_process_table() -> list[tuple[int, str]]: # readers unbounded and a conhost descendant holding duplicated handles wedges it # forever. It also passes CREATE_NO_WINDOW for the pythonw.exe backend. from hermes_cli._subprocess_compat import bounded_probe_run - result = bounded_probe_run( ["wmic", "process", "get", "ProcessId,CommandLine", "/FORMAT:LIST"], timeout=10, errors="ignore", @@ -98,7 +96,6 @@ def _scan_dashboard_processes(*, exclude_pids: set[int] | None = None) -> list[t # missed, preferring the ledger's full argv. try: from hermes_cli.process_identity import ledger_entries - seen = {pid for pid, _ in found} | skip for entry in ledger_entries(): pid = entry.get("pid") @@ -116,7 +113,6 @@ def _hermes_home_for_pid(pid: int) -> str | None: """Best-effort ``HERMES_HOME`` from *pid*'s environment (psutil, then /proc).""" try: import psutil - home = psutil.Process(pid).environ().get("HERMES_HOME") if home: return home @@ -234,7 +230,6 @@ def _filter_dashboard_respawn_candidates( if own_home is None: try: from hermes_constants import get_hermes_home - own_home = str(get_hermes_home()) except Exception: own_home = "" @@ -278,7 +273,6 @@ def _kill_pids_windows(pids: list[int], killed: list[int], failed: list[tuple[in """``taskkill /F`` each PID after re-verifying its identity.""" from gateway.status import get_process_start_time from hermes_cli._subprocess_compat import pid_is_hermes, windows_hide_flags - # Capture identity immediately after discovery: a PID reused before the destructive # action fails the start-time check. pid_start_times = {pid: get_process_start_time(pid) for pid in pids} @@ -610,7 +604,6 @@ def _lock_owned_serve_pids(base_dir: Path | None = None) -> set[int]: Best-effort: a bad record contributes no PID; never raises. """ import json - root = base_dir if base_dir is not None else _hermes_home_dir() / _REMOTE_LOCK_SUBDIR owned: set[int] = set() if not root.is_dir(): @@ -647,7 +640,6 @@ def _process_age_seconds(pid: int) -> float: import time as _time import psutil as _psutil - return max(0.0, _time.time() - _psutil.Process(pid).create_time()) @@ -670,7 +662,6 @@ def _reap_orphaned_desktop_local_serves( """ import signal as _signal import time as _time - signal_term = _signal.SIGTERM if signal_term is None else signal_term signal_kill = getattr(_signal, "SIGKILL", _signal.SIGTERM) if signal_kill is None else signal_kill sleep_fn = sleep_fn or _time.sleep @@ -733,7 +724,6 @@ def _reap_orphaned_desktop_local_serves( # which is a Windows footgun the linter blocks everywhere. sleep_fn(1.5) import psutil - for pid in matched: if pid in failed: continue diff --git a/hermes_cli/dashboard_register.py b/hermes_cli/dashboard_register.py index 5f1f2cf716..61419dbf52 100644 --- a/hermes_cli/dashboard_register.py +++ b/hermes_cli/dashboard_register.py @@ -52,7 +52,6 @@ def _resolve_portal_base_url(override: Optional[str] = None) -> str: return override.rstrip("/") try: from hermes_cli.auth import DEFAULT_NOUS_PORTAL_URL, get_provider_auth_state - base = (get_provider_auth_state("nous") or {}).get("portal_base_url") chosen = base if isinstance(base, str) and base.strip() else str(DEFAULT_NOUS_PORTAL_URL) return chosen.rstrip("/") @@ -113,7 +112,6 @@ def _print_post_register_hint( ) -> None: """Print the success summary + the gate-engagement caveat.""" from hermes_cli.config import get_env_path - print(f"\n Wrote to {get_env_path()}:\n HERMES_DASHBOARD_OAUTH_CLIENT_ID={client_id}") if wrote_portal_url: print(" HERMES_DASHBOARD_PORTAL_URL=" + str(portal_base_url)) @@ -150,7 +148,6 @@ def _print_post_register_hint( def _env_value(key: str) -> Optional[str]: """Stored ``.env`` value, or ``None`` on any read failure.""" from hermes_cli.config import get_env_value - try: return get_env_value(key) except Exception: @@ -160,7 +157,6 @@ def _env_value(key: str) -> Optional[str]: def _save_env_quietly(key: str, value: str) -> bool: """Persist *key*; False on failure (non-fatal: only client_id is load-bearing).""" from hermes_cli.config import save_env_value - try: save_env_value(key, value) return True @@ -188,7 +184,6 @@ def cmd_dashboard_register(args) -> None: """Register a self-hosted dashboard OAuth client with Nous Portal.""" from hermes_cli.auth import AuthError, resolve_nous_access_token from hermes_cli.config import is_managed, save_env_value - # Managed (Docker/hosted) installs get HERMES_DASHBOARD_OAUTH_CLIENT_ID stamped in by the # orchestrator; save_env_value refuses to write anyway. if is_managed(): diff --git a/hermes_cli/debug.py b/hermes_cli/debug.py index 6da739e807..64447bfbcb 100644 --- a/hermes_cli/debug.py +++ b/hermes_cli/debug.py @@ -242,7 +242,6 @@ class LogSnapshot: def _primary_log_path(log_name: str) -> Optional[Path]: """Where *log_name* would live if present. Doesn't check existence.""" from hermes_cli.logs import LOG_FILES - filename = LOG_FILES.get(log_name) return (get_hermes_home() / "logs" / filename) if filename else None @@ -290,7 +289,6 @@ def _redact_log_text(text: str) -> str: if not text: return text from agent.redact import redact_sensitive_text - text = redact_sensitive_text(text, force=True) return _EMAIL_ADDRESS_RE.sub("[REDACTED_EMAIL]", text) @@ -436,7 +434,6 @@ def collect_debug_report( # process, where the errors.log tail below carries the same escalation lines instead. try: from agent.agent_runtime_helpers import get_sanitizer_heal_stats - heal_stats = get_sanitizer_heal_stats() if heal_stats: buf.write("\n\n--- transcript sanitiser heal counters ---\n") @@ -638,7 +635,6 @@ _NOUS_PRIVACY_NOTICE = """\ def _run_debug_share_nous(args, *, log_lines: int, redact: bool) -> None: """``hermes debug share --nous``: gzip the same bundle into the Nous envelope → Nous-S3.""" from hermes_cli.diagnostics_upload import share_to_nous - print(_NOUS_PRIVACY_NOTICE) if not _confirm_upload(args): return From a4daa0b780410c91dc02bf99e0f5a1c6e72df3ac Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:14:53 -0700 Subject: [PATCH 18/29] refactor(hermes_cli): debug._capture_dump via contextlib; container_boot argv probe as one loop --- hermes_cli/container_boot.py | 32 ++++++++++++++------------------ hermes_cli/debug.py | 23 +++++++---------------- 2 files changed, 21 insertions(+), 34 deletions(-) diff --git a/hermes_cli/container_boot.py b/hermes_cli/container_boot.py index 9348aae643..038ff5bf88 100644 --- a/hermes_cli/container_boot.py +++ b/hermes_cli/container_boot.py @@ -179,26 +179,22 @@ def _read_container_argv() -> tuple[str, ...]: ``s6-svscan`` and the real command lives on another PID, so after the PID 1 fast path we scan ``/proc/*/cmdline`` for a process whose argv contains ``main-wrapper.sh``. """ - try: - argv = _cmdline_argv(Path("/proc/1/cmdline")) + def _cmdlines(): + yield Path("/proc/1/cmdline") + try: + for entry in Path("/proc").iterdir(): + if entry.name.isdigit(): + yield entry / "cmdline" + except OSError: + return + + for cmdline in _cmdlines(): + try: + argv = _cmdline_argv(cmdline) + except OSError: + continue if any("main-wrapper.sh" in part for part in argv): return argv - except OSError: - pass - - try: - for entry in Path("/proc").iterdir(): - if not entry.name.isdigit(): - continue - try: - argv = _cmdline_argv(entry / "cmdline") - except OSError: - continue - if any("main-wrapper.sh" in part for part in argv): - return argv - except OSError: - pass - return () diff --git a/hermes_cli/debug.py b/hermes_cli/debug.py index 64447bfbcb..420eb225dc 100644 --- a/hermes_cli/debug.py +++ b/hermes_cli/debug.py @@ -1,5 +1,6 @@ """``hermes debug`` debug tools for Hermes Agent.""" +import contextlib import datetime import gzip import io @@ -11,6 +12,7 @@ import time import urllib.request from dataclasses import dataclass from pathlib import Path +from types import SimpleNamespace from typing import Optional from hermes_constants import get_hermes_home @@ -396,19 +398,9 @@ def _capture_default_log_snapshots( def _capture_dump() -> str: """Run ``hermes dump`` and return its stdout as a string.""" from hermes_cli.dump import run_dump - - class _FakeArgs: - show_keys = False - - old_stdout = sys.stdout - sys.stdout = capture = io.StringIO() - try: - run_dump(_FakeArgs()) - except SystemExit: - pass - finally: - sys.stdout = old_stdout - + capture = io.StringIO() + with contextlib.redirect_stdout(capture), contextlib.suppress(SystemExit): + run_dump(SimpleNamespace(show_keys=False)) return capture.getvalue() @@ -448,9 +440,8 @@ def collect_debug_report( buf.write("\n") for name in _REPORT_LOGS: - buf.write(f"\n--- {name}.log (last {_tail_budget(name, log_lines)} lines) ---\n") - buf.write(log_snapshots[name].tail_text) - buf.write("\n") + buf.write(f"\n--- {name}.log (last {_tail_budget(name, log_lines)} lines) ---\n" + f"{log_snapshots[name].tail_text}\n") return buf.getvalue() From 3441011a4b71b059750f013aa1f6992dd651f421 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:23:38 -0700 Subject: [PATCH 19/29] =?UTF-8?q?refactor(hermes=5Fcli):=20dashboard=5Fpro?= =?UTF-8?q?cs.py=20=E2=80=94=20reap=20candidate=20filter=20as=20one=20comp?= =?UTF-8?q?rehension,=20contextlib.suppress=20for=20swallow-only=20guards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/dashboard_procs.py | 62 +++++++++++++---------------------- 1 file changed, 22 insertions(+), 40 deletions(-) diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py index 5d3074267c..5ec2e96868 100644 --- a/hermes_cli/dashboard_procs.py +++ b/hermes_cli/dashboard_procs.py @@ -4,6 +4,7 @@ Helpers that STAY in ``hermes_cli.main`` are reached through the lazy ``_m()`` r monkeypatches on ``hermes_cli.main.`` keep working and imports stay one-way. """ +import contextlib import os import subprocess import sys @@ -111,13 +112,10 @@ def _scan_dashboard_processes(*, exclude_pids: set[int] | None = None) -> list[t def _hermes_home_for_pid(pid: int) -> str | None: """Best-effort ``HERMES_HOME`` from *pid*'s environment (psutil, then /proc).""" - try: + with contextlib.suppress(Exception): import psutil - home = psutil.Process(pid).environ().get("HERMES_HOME") - if home: + if home := psutil.Process(pid).environ().get("HERMES_HOME"): return home - except Exception: - pass try: raw = Path(f"/proc/{pid}/environ").read_bytes() except OSError: @@ -372,14 +370,11 @@ def _kill_stale_dashboard_processes( for pid in pids: pid_cgroup[pid] = _m()._get_pid_cgroup_path(pid) pid_service[pid] = _m()._get_systemd_service_for_pid(pid) - if not pid_service[pid]: + if not pid_service[pid] and (cmdline := _m()._dashboard_cmdline_for_pid(pid)): # Manual process: keep exact argv + HERMES_HOME for the post-update respawn # and its per-profile cap. - cmdline = _m()._dashboard_cmdline_for_pid(pid) - if cmdline: - pid_cmdline[pid] = cmdline - pid_home[pid] = _hermes_home_for_pid(pid) - + pid_cmdline[pid] = cmdline + pid_home[pid] = _hermes_home_for_pid(pid) if already_restarted_units: pids = [ pid for pid in pids @@ -443,10 +438,9 @@ def _restart_killed_backends( print(f" ⚠ {svc}: {err}") respawn_cmds = _filter_dashboard_respawn_candidates(respawn_candidates) - if respawn_cmds: - failed_cmds = _m()._respawn_dashboard_processes(respawn_cmds) - if failed_cmds: - unrecovered.extend(p for p in killed if pid_cmdline.get(p) in failed_cmds) + failed_cmds = _m()._respawn_dashboard_processes(respawn_cmds) if respawn_cmds else None + if failed_cmds: + unrecovered.extend(p for p in killed if pid_cmdline.get(p) in failed_cmds) if failed_restarts or unrecovered: print(" Restart anything not auto-restarted when you're ready:\n hermes dashboard --port ") @@ -489,23 +483,18 @@ def _detect_concurrent_hermes_instances( seed = int(exclude_pid) if exclude_pid is not None else os.getpid() exclude_pids: set[int] = {seed} # Broad ``except Exception``: psutil may be partially stubbed in tests. - try: + with contextlib.suppress(Exception): for ancestor in psutil.Process(seed).parents(): - try: + with contextlib.suppress(Exception): anc_exe = ancestor.exe() if anc_exe and _norm_exe(anc_exe) in shim_paths: exclude_pids.add(int(ancestor.pid)) - except Exception: - continue - except Exception: - pass matches: list[tuple[int, str]] = [] try: proc_iter = psutil.process_iter(["pid", "exe", "name"]) except Exception: return [] - for proc in proc_iter: try: info = proc.info @@ -692,26 +681,22 @@ def _reap_orphaned_desktop_local_serves( # Re-read ownership: a lock may have been written between scan and now. owned_now = _owned_pids() - targets: list[tuple[int, str]] = [] - for pid, cmd in scanned: - if not _is_desktop_local_serve_cmdline(cmd) or pid in owned_now: - continue - if _process_ppid(pid) not in (0, 1): - continue - try: - if process_age_seconds_fn(pid) < _REAP_MIN_AGE_SECONDS: - continue + def _is_stale_orphan(pid: int) -> bool: + try: # never let a liveness probe failure widen the reap + return process_age_seconds_fn(pid) >= _REAP_MIN_AGE_SECONDS except Exception: - continue # never let a liveness probe failure widen the reap - targets.append((pid, cmd)) + return False - if not targets: + matched = [ + pid for pid, cmd in scanned + if _is_desktop_local_serve_cmdline(cmd) and pid not in owned_now + and _process_ppid(pid) in (0, 1) and _is_stale_orphan(pid) + ] + if not matched: return _empty_result() - matched = [pid for pid, _ in targets] killed: list[int] = [] failed: list[int] = [] - for pid in matched: try: os.kill(pid, signal_term) @@ -736,9 +721,6 @@ def _reap_orphaned_desktop_local_serves( except OSError: failed.append(pid) - try: + with contextlib.suppress(Exception): print(f"⟲ Reaped {len(killed)} orphaned desktop-local serve backend(s) ({reason}): {killed or matched}") - except Exception: - pass - return {"matched": matched, "killed": killed, "failed": failed} From f1526d59d5ccf7aa3d26dc5575cab10b576134b8 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:26:44 -0700 Subject: [PATCH 20/29] =?UTF-8?q?refactor(hermes=5Fcli):=20group=20D=20?= =?UTF-8?q?=E2=80=94=20contextlib.suppress=20for=20swallow-only=20try/exce?= =?UTF-8?q?pt=20blocks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/cron.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index 04912889dc..03d37ed8ca 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -1,5 +1,6 @@ """Cron subcommand for hermes CLI.""" +import contextlib import json import re import sys @@ -66,12 +67,10 @@ def _builtin_gateway_liveness() -> Optional[bool]: # gateway process it short-circuits to True so the in-gateway cron tool never # emits a false "gateway not running" (find_gateway_pids can transiently miss the # gateway just after a restart). - try: + with contextlib.suppress(Exception): # a crashing probe is "unknown" — let the pid scan decide from gateway.status import is_gateway_runtime_lock_active if is_gateway_runtime_lock_active(): return True - except Exception: - pass # a crashing lock probe is "unknown", not "dead" — let the pid scan decide from hermes_cli.gateway import ( find_gateway_pids, named_profile_served_by_running_multiplexer, ) @@ -434,13 +433,11 @@ def cron_status(): # The pid scan can transiently miss a live gateway (just after a restart) while # the runtime lock — held for exactly the gateway's lifetime — proves the ticker's # process is alive. Only declare "not running" when both agree. - try: + with contextlib.suppress(Exception): from gateway.status import get_running_pid, is_gateway_runtime_lock_active gateway_alive_via_lock = is_gateway_runtime_lock_active() lock_pid = get_running_pid() if gateway_alive_via_lock else None pids = [lock_pid] if lock_pid else pids - except Exception: - pass if pids or gateway_alive_via_lock: _print_ticker_health(pids) else: @@ -701,11 +698,9 @@ def _job_action(action: str, job_id: str, success_verb: str) -> int: # stateless so ``async_delivery_supported()`` gates it off and the run executes # synchronously. Scoped to this call (token reset in ``finally``) so in-process # callers (tests, embedding apps) are not tainted. - try: + with contextlib.suppress(Exception): from gateway.session_context import _SESSION_ASYNC_DELIVERY _stateless_token = _SESSION_ASYNC_DELIVERY.set(False) - except Exception: - _stateless_token = None try: result = _cron_api(action=action, job_id=job_id) finally: From a127441317fb3e33446e846741b48974ca890c23 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:30:42 -0700 Subject: [PATCH 21/29] =?UTF-8?q?refactor(hermes=5Fcli):=20copilot=5Fauth.?= =?UTF-8?q?py=20=E2=80=94=20lift=20exchange=20retry=20loop=20into=20=5Ffet?= =?UTF-8?q?ch=5Fexchange=5Fwith=5Fretry,=20tighten=20device-flow=20polling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/copilot_auth.py | 96 +++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 52 deletions(-) diff --git a/hermes_cli/copilot_auth.py b/hermes_cli/copilot_auth.py index e75f1f783a..cecc6451c5 100644 --- a/hermes_cli/copilot_auth.py +++ b/hermes_cli/copilot_auth.py @@ -207,10 +207,7 @@ def copilot_device_code_login( print(" ✗ GitHub did not return a device code.") return None - print() - print(f" Open this URL in your browser: {verification_uri}") - print(f" Enter this code: {user_code}") - print() + print(f"\n Open this URL in your browser: {verification_uri}\n Enter this code: {user_code}\n") print(" Waiting for authorization...", end="", flush=True) deadline = time.monotonic() + timeout_seconds @@ -238,22 +235,18 @@ def copilot_device_code_login( error = result.get("error", "") if error == "slow_down": - # RFC 8628: add 5 seconds to polling interval + # RFC 8628: add 5 seconds to polling interval (or honor a server-supplied one) server_interval = result.get("interval") - if isinstance(server_interval, (int, float)) and server_interval > 0: - interval = int(server_interval) - else: - interval += 5 + is_num = isinstance(server_interval, (int, float)) and server_interval > 0 + interval = int(server_interval) if is_num else interval + 5 if error in ("authorization_pending", "slow_down"): print(".", end="", flush=True) continue if error: - print() - print(_DEVICE_CODE_TERMINAL_ERRORS.get(error, f" ✗ Authorization failed: {error}")) + print("\n" + _DEVICE_CODE_TERMINAL_ERRORS.get(error, f" ✗ Authorization failed: {error}")) return None - print() - print(" ✗ Timed out waiting for authorization.") + print("\n ✗ Timed out waiting for authorization.") return None @@ -463,6 +456,41 @@ def _urlopen_bounded(req, timeout: float): return box["resp"] +def _fetch_exchange_with_retry(req, timeout: float, fp: str) -> dict: + """GET the exchange with backoff for startup network races; raises ValueError on failure. + + Permanent HTTP rejections (401/403/404) skip the retry loop entirely — sleeping on an auth + rejection blocks the caller for ~4.5s with an identical outcome. Failures populate the + negative cache (long TTL for permanent, short for transient); success clears it. + """ + last_exc: Optional[Exception] = None + permanent_failure = False + for attempt in range(1, _EXCHANGE_MAX_ATTEMPTS + 1): + try: + with _urlopen_bounded(req, timeout) as resp: + data = json.loads(resp.read().decode()) + _exchange_failure_cache.pop(fp, None) + return data + except Exception as exc: # noqa: BLE001 — retry all, re-raise below + last_exc = exc + status = getattr(exc, "code", None) or getattr(exc, "status", None) + permanent_failure = status in _EXCHANGE_PERMANENT_HTTP_STATUSES + if permanent_failure: + logger.debug("Copilot token exchange rejected (HTTP %s); not retrying", status) + break + if attempt < _EXCHANGE_MAX_ATTEMPTS: + sleep_s = _EXCHANGE_BACKOFF_BASE_SECONDS * attempt + logger.debug("Copilot token exchange attempt %d/%d failed (%s); retrying in %.1fs", + attempt, _EXCHANGE_MAX_ATTEMPTS, exc, sleep_s) + time.sleep(sleep_s) + ttl = (_EXCHANGE_FAILURE_TTL_PERMANENT_SECONDS if permanent_failure + else _EXCHANGE_FAILURE_TTL_TRANSIENT_SECONDS) + _exchange_failure_cache[fp] = time.time() + ttl + raise ValueError( + f"Copilot token exchange failed after {_EXCHANGE_MAX_ATTEMPTS} attempts: {last_exc}" + ) from last_exc + + def _cache_entry_fresh(cached) -> bool: return bool(cached) and time.time() < cached[1] - _JWT_REFRESH_MARGIN_SECONDS @@ -521,46 +549,12 @@ def _exchange_copilot_token_locked( }, ) - # Retry with backoff for startup network races; permanent HTTP rejections (401/403/404) - # skip the loop entirely — sleeping on an auth rejection blocks the caller for ~4.5s with - # an identical outcome. - data = None - last_exc: Optional[Exception] = None - permanent_failure = False - for attempt in range(1, _EXCHANGE_MAX_ATTEMPTS + 1): - try: - with _urlopen_bounded(req, timeout) as resp: - data = json.loads(resp.read().decode()) - break - except Exception as exc: # noqa: BLE001 — retry all, re-raise below - last_exc = exc - status = getattr(exc, "code", None) or getattr(exc, "status", None) - permanent_failure = status in _EXCHANGE_PERMANENT_HTTP_STATUSES - if permanent_failure: - logger.debug("Copilot token exchange rejected (HTTP %s); not retrying", status) - break - if attempt < _EXCHANGE_MAX_ATTEMPTS: - sleep_s = _EXCHANGE_BACKOFF_BASE_SECONDS * attempt - logger.debug("Copilot token exchange attempt %d/%d failed (%s); retrying in %.1fs", - attempt, _EXCHANGE_MAX_ATTEMPTS, exc, sleep_s) - time.sleep(sleep_s) - if data is None: - ttl = ( - _EXCHANGE_FAILURE_TTL_PERMANENT_SECONDS - if permanent_failure - else _EXCHANGE_FAILURE_TTL_TRANSIENT_SECONDS - ) - _exchange_failure_cache[fp] = time.time() + ttl - raise ValueError( - f"Copilot token exchange failed after {_EXCHANGE_MAX_ATTEMPTS} attempts: {last_exc}" - ) from last_exc - _exchange_failure_cache.pop(fp, None) + data = _fetch_exchange_with_retry(req, timeout, fp) api_token = data.get("token", "") if not api_token: raise ValueError("Copilot token exchange returned empty token") - expires_at = data.get("expires_at", 0) - expires_at = float(expires_at) if expires_at else time.time() + 1800 + expires_at = float(data.get("expires_at") or 0) or time.time() + 1800 # Account-specific API base URL: GitHub advertises the authoritative endpoint under # ``endpoints.api`` (differs for Copilot Enterprise / proxied accounts); when omitted, @@ -585,10 +579,8 @@ def _derive_base_url_from_proxy_ep(token: str) -> Optional[str]: m = re.search(r'(?:^|;)\s*proxy-ep=([^;\s]+)', token) if not m: return None - proxy_ep = re.sub(r"^https?://", "", m.group(1), count=1).rstrip("/") - if proxy_ep.startswith("proxy."): - proxy_ep = "api." + proxy_ep[len("proxy."):] + proxy_ep = re.sub(r"^proxy\.", "api.", proxy_ep, count=1) return f"https://{proxy_ep}" From 7ea400249590df006e410d0bd718c084bf4dfd9d Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:40:25 -0700 Subject: [PATCH 22/29] =?UTF-8?q?refactor(hermes=5Fcli):=20cron.py=20?= =?UTF-8?q?=E2=80=94=20lift=20manual-run=20verdict=20into=20=5Frun=5Foutco?= =?UTF-8?q?me,=20flatten=20cron=5Fresume=20mode=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/cron.py | 48 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index 03d37ed8ca..293b58c879 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -714,46 +714,46 @@ def _job_action(action: str, job_id: str, success_verb: str) -> int: if action in {"resume", "run"} and result.get("job", {}).get("next_run_at"): print(f" Next run: {result['job']['next_run_at']}") if action == "run": - job = result.get("job", {}) - # A manual run may be dispatched to the gateway daemon's background delegation worker - # (execution_mode="background" and/or a delegation_id) and keeps running AFTER this - # CLI exits — a terminal success/failure verdict would be a lie, so report the dispatch. - delegation_id = job.get("delegation_id") - if delegation_id: - print(f" Running in background (delegation {delegation_id}).") - elif job.get("execution_mode") == "background": - print(" Running in background.") - elif job.get("executed"): - print(f" Ran now: {'succeeded' if job.get('execution_success') else 'failed'}.") - elif job.get("execution_skipped"): - print(f" {job['execution_skipped']}") - else: - print(" It will run on the next scheduler tick.") + print(f" {_run_outcome(result.get('job', {}))}") return 0 +def _run_outcome(job: Dict[str, Any]) -> str: + """One-line verdict for a manual run. + + A run may be dispatched to the gateway daemon's background delegation worker + (execution_mode="background" and/or a delegation_id) and keeps running AFTER this CLI + exits — a terminal success/failure verdict would be a lie, so report the dispatch. + """ + if job.get("delegation_id"): + return f"Running in background (delegation {job['delegation_id']})." + if job.get("execution_mode") == "background": + return "Running in background." + if job.get("executed"): + return f"Ran now: {'succeeded' if job.get('execution_success') else 'failed'}." + return job.get("execution_skipped") or "It will run on the next scheduler tick." + + def cron_resume(args) -> int: """Resume a paused job or explicitly re-arm a completed one-shot.""" run_at = getattr(args, "run_at", None) run_now = getattr(args, "run_now", False) - if bool(run_at) == bool(run_now): - if run_at or run_now: - print(color("Use exactly one of --at or --run-now.", Colors.RED)) - return 1 + if run_at and run_now: + print(color("Use exactly one of --at or --run-now.", Colors.RED)) + return 1 + if not run_at and not run_now: return _job_action("resume", args.job_id, "Resumed") from cron.jobs import AmbiguousJobReference, _hermes_now, rearm_oneshot - if run_now: - run_at = _hermes_now().isoformat() try: - job = rearm_oneshot(args.job_id, run_at) + job = rearm_oneshot(args.job_id, _hermes_now().isoformat() if run_now else run_at) except (AmbiguousJobReference, ValueError) as exc: print(color(f"Failed to re-arm job: {exc}", Colors.RED)) return 1 if not job: print(color(f"Job not found: {args.job_id}", Colors.RED)) return 1 - print(color(f"Re-armed job: {job.get('name', args.job_id)} ({args.job_id})", Colors.GREEN)) - print(f" Next run: {job.get('next_run_at')}") + print(color(f"Re-armed job: {job.get('name', args.job_id)} ({args.job_id})", Colors.GREEN) + + f"\n Next run: {job.get('next_run_at')}") return 0 From 7ff29f4837f14b7811df978a1bcc955e94bca364 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:42:26 -0700 Subject: [PATCH 23/29] =?UTF-8?q?refactor(hermes=5Fcli):=20cron.py=20?= =?UTF-8?q?=E2=80=94=20split=20cron=5Flist=20into=20=5Fjob=5Frows/=5Fjob?= =?UTF-8?q?=5Fwarnings/=5Flast=5Frun=5Fdisplay?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/cron.py | 155 +++++++++++++++++++++------------------------ 1 file changed, 72 insertions(+), 83 deletions(-) diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index 293b58c879..60d7ddf3df 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -168,98 +168,87 @@ def cron_list(show_all: bool = False): _print_banner("Scheduled Jobs") for job in jobs: - job_id = job.get("id", "?") # Scheduler-honoured flag — never show [paused] when enabled=true. - state = effective_job_state(job) - badge = _STATE_BADGES.get(state) or ( + badge = _STATE_BADGES.get(effective_job_state(job)) or ( ("[active]", Colors.GREEN) if job.get("enabled", True) else ("[disabled]", Colors.RED) ) - - # `repeat` / `deliver` may be present-but-null in the record (a one-shot persisted - # with "repeat": null), so coalesce rather than rely on the dict-default, which only - # applies to a missing key — a null deliver would crash `", ".join(None)`. - repeat_info = job.get("repeat") or {} - repeat_times = repeat_info.get("times") - repeat_str = f"{repeat_info.get('completed', 0)}/{repeat_times}" if repeat_times else "∞" - deliver = job.get("deliver") or ["local"] - if isinstance(deliver, str): - deliver = [deliver] - - rows = [ - ("Name", job.get("name", "(unnamed)")), - ("Schedule", job.get("schedule_display", job.get("schedule", {}).get("value", "?"))), - ("Repeat", repeat_str), - ("Next run", job.get("next_run_at", "?")), - ("Deliver", ", ".join(deliver)), - ] - skills = job.get("skills") or ([job["skill"]] if job.get("skill") else []) - if skills: - rows.append(("Skills", ", ".join(skills))) - if job.get("script"): - rows.append(("Script", job["script"])) - monitor_source = job.get("monitor_script") or job.get("monitor_url") - if monitor_source: - rows.append(("Monitor", f"{monitor_source} (agent runs only on output change)")) - mon_state = job.get("monitor_state") or {} - if mon_state.get("last_changed_at"): - rows.append(("Changed", mon_state["last_changed_at"])) - if job.get("no_agent"): - mode = color("no-agent", Colors.DIM) + " (script stdout delivered directly)" - rows.append(("Mode", mode)) - if job.get("workdir"): - rows.append(("Workdir", job["workdir"])) - - last_status = job.get("last_status") - if last_status: - if last_status == "ok": - status_display = color("ok", Colors.GREEN) - elif last_status == "delivery_failed": - # Agent succeeded but the result never reached the user — not green; the - # detail lives in last_delivery_error (last_error is None for these runs). - detail = job.get("last_delivery_error") or "?" - status_display = color(f"delivery_failed: {detail}", Colors.YELLOW) - else: - status_display = color(f"{last_status}: {job.get('last_error', '?')}", Colors.RED) - streak = int(job.get("failure_streak") or 0) - if streak >= 2: - status_display += color(f" ({streak} failures in a row)", Colors.RED) - rows.append(("Last run", f"{job.get('last_run_at', '?')} {status_display}")) - - dispatch_line = _dispatch_display(job.get("last_dispatch")) - if dispatch_line: - rows.append(("Dispatch", dispatch_line)) - latest_execution = job.get("latest_execution") - if latest_execution: - rows.append(( - "Execution", - f"{latest_execution.get('status', '?')} {latest_execution.get('id', '?')}", - )) - - print(f" {color(job_id, Colors.YELLOW)} {color(*badge)}") - for label, value in rows: + print(f" {color(job.get('id', '?'), Colors.YELLOW)} {color(*badge)}") + for label, value in _job_rows(job): print(f" {label + ':':<11}{value}") - - delivery_err = job.get("last_delivery_error") - if delivery_err: - print(f" {color('⚠ Delivery failed:', Colors.YELLOW)} {delivery_err}") - - # A live adapter acked the last send but returned no message_id / raw_response - # (Slack/Matrix/Mattermost shape): accepted as delivered, but say so here. - unverified = job.get("last_delivery_unverified") - if unverified: - print(f" {color('⚠ Delivery UNVERIFIED:', Colors.YELLOW)} adapter acked " - f"{_unverified_targets(unverified)} without message_id/raw_response") - - fire_err = job.get("last_fire_error") - if isinstance(fire_err, dict) and fire_err.get("detail"): - print(f" {color('⚠ Missed scheduled fire:', Colors.RED)} " - f"{fire_err.get('at', '?')} {fire_err['detail']}") - + for line in _job_warnings(job): + print(f" {line}") print() _warn_if_gateway_not_running() +def _last_run_display(job: Dict[str, Any]) -> str: + last_status = job["last_status"] + if last_status == "ok": + return color("ok", Colors.GREEN) + if last_status == "delivery_failed": + # Agent succeeded but the result never reached the user — not green; the detail + # lives in last_delivery_error (last_error is None for these runs). + return color(f"delivery_failed: {job.get('last_delivery_error') or '?'}", Colors.YELLOW) + display = color(f"{last_status}: {job.get('last_error', '?')}", Colors.RED) + streak = int(job.get("failure_streak") or 0) + if streak >= 2: + display += color(f" ({streak} failures in a row)", Colors.RED) + return display + + +def _job_rows(job: Dict[str, Any]) -> List[tuple[str, str]]: + """``(label, value)`` detail rows for one job in ``cron list``.""" + # `repeat` / `deliver` may be present-but-null in the record (a one-shot persisted with + # "repeat": null), so coalesce rather than rely on the dict-default, which only applies + # to a missing key — a null deliver would crash `", ".join(None)`. + repeat_info = job.get("repeat") or {} + repeat_times = repeat_info.get("times") + deliver = job.get("deliver") or ["local"] + skills = job.get("skills") or ([job["skill"]] if job.get("skill") else []) + monitor_source = job.get("monitor_script") or job.get("monitor_url") + mon_state = job.get("monitor_state") or {} + latest_execution = job.get("latest_execution") or {} + optional = [ + ("Skills", ", ".join(skills) if skills else ""), + ("Script", job.get("script")), + ("Monitor", f"{monitor_source} (agent runs only on output change)" if monitor_source else ""), + ("Changed", mon_state.get("last_changed_at") if monitor_source else ""), + ("Mode", color("no-agent", Colors.DIM) + " (script stdout delivered directly)" + if job.get("no_agent") else ""), + ("Workdir", job.get("workdir")), + ("Last run", f"{job.get('last_run_at', '?')} {_last_run_display(job)}" + if job.get("last_status") else ""), + ("Dispatch", _dispatch_display(job.get("last_dispatch"))), + ("Execution", f"{latest_execution.get('status', '?')} {latest_execution.get('id', '?')}" + if latest_execution else ""), + ] + return [ + ("Name", job.get("name", "(unnamed)")), + ("Schedule", job.get("schedule_display", job.get("schedule", {}).get("value", "?"))), + ("Repeat", f"{repeat_info.get('completed', 0)}/{repeat_times}" if repeat_times else "∞"), + ("Next run", job.get("next_run_at", "?")), + ("Deliver", deliver if isinstance(deliver, str) else ", ".join(deliver)), + ] + [(label, value) for label, value in optional if value] + + +def _job_warnings(job: Dict[str, Any]) -> List[str]: + """Delivery / fire warning lines for one job in ``cron list``.""" + lines = [] + if job.get("last_delivery_error"): + lines.append(f"{color('⚠ Delivery failed:', Colors.YELLOW)} {job['last_delivery_error']}") + # A live adapter acked the last send but returned no message_id / raw_response + # (Slack/Matrix/Mattermost shape): accepted as delivered, but say so here. + if unverified := job.get("last_delivery_unverified"): + lines.append(f"{color('⚠ Delivery UNVERIFIED:', Colors.YELLOW)} adapter acked " + f"{_unverified_targets(unverified)} without message_id/raw_response") + fire_err = job.get("last_fire_error") + if isinstance(fire_err, dict) and fire_err.get("detail"): + lines.append(f"{color('⚠ Missed scheduled fire:', Colors.RED)} " + f"{fire_err.get('at', '?')} {fire_err['detail']}") + return lines + + def cron_tick(): """Run due jobs once and exit.""" from cron.scheduler import CronTickYielded, tick From 97293216ee761fc3bda886c79fcac14b647ac346 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:46:32 -0700 Subject: [PATCH 24/29] =?UTF-8?q?refactor(hermes=5Fcli):=20container=5Fboo?= =?UTF-8?q?t.py=20=E2=80=94=20second=20pass=20compacting=20rationale=20com?= =?UTF-8?q?ments=20(every=20WHY=20kept)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/container_boot.py | 146 ++++++++++++++--------------------- 1 file changed, 60 insertions(+), 86 deletions(-) diff --git a/hermes_cli/container_boot.py b/hermes_cli/container_boot.py index 038ff5bf88..d6f944799b 100644 --- a/hermes_cli/container_boot.py +++ b/hermes_cli/container_boot.py @@ -15,32 +15,25 @@ from typing import Literal, Sequence log = logging.getLogger(__name__) -# Only this desired state triggers automatic restart. Everything else (startup_failed, -# starting, stopped, missing) registers the slot down and waits for explicit user action — -# avoiding the crash-loop where a broken gateway keeps restarting across `docker restart`. -# Older installs only have gateway_state; newer lifecycle commands persist desired_state -# separately so a transient runtime state does not erase the operator's durable intent. +# Only this desired state auto-restarts; everything else (startup_failed, starting, stopped, +# missing) registers the slot down and waits for the user — avoiding the crash-loop where a +# broken gateway keeps restarting across `docker restart`. Older installs only have +# gateway_state; newer lifecycle commands persist desired_state separately so a transient +# runtime state does not erase the operator's durable intent. _AUTOSTART_STATES = frozenset({"running"}) -# Transient runtime sub-states of a RUNNING gateway — reached only while up and serving, so -# NOT an operator stop and NOT a failed boot: -# - `draining` — drain watcher / scale-to-zero go-dormant path, in-flight quiesce begun. -# - `degraded` — came up with some platforms queued for retry, then fell through to the -# normal running state; the reconnect watcher takes it from there. -# When a gateway is hard-killed *in one of these states* (container/VM recreate SIGTERMs it -# before `_stop_impl` persists a terminal state), the marker left in gateway_state.json is -# the transient sub-state. With no `desired_state` to fall back to, treating it literally -# would leave the gateway DOWN on every subsequent boot (observed: a relay-opted-in staging -# instance stranded at `draining`; `degraded` is the same wedge class). Map them to `running` -# — mirrors gateway/run.py persisting `running` (not mid-shutdown `draining`) on unexpected -# signal, extended to the case where the gateway died before persisting anything. -# `starting` / `startup_failed` are deliberately NOT included: those mean the gateway died -# mid-boot or failed to come up, and auto-restarting would reintroduce the crash-loop. +# Transient sub-states of a RUNNING gateway (`draining`: in-flight quiesce; `degraded`: up +# with some platforms queued for reconnect) — NOT an operator stop, NOT a failed boot. When a +# gateway is hard-killed in one of them (container recreate SIGTERMs it before `_stop_impl` +# persists a terminal state) and there is no `desired_state`, reading the marker literally +# would leave the gateway DOWN on every later boot (observed: staging instance stranded at +# `draining`). Map them to `running`, mirroring gateway/run.py's unexpected-signal handling. +# `starting` / `startup_failed` are deliberately excluded: auto-restarting a gateway that died +# mid-boot reintroduces the crash-loop. _TRANSIENT_RUNNING_STATES = frozenset({"draining", "degraded"}) -# Stale runtime files swept before recreating service slots: they hold container-namespaced -# state (PIDs, process tables) that's garbage post-restart — a numerically-equal PID in the -# new container is a different process. +# Swept before recreating slots: container-namespaced state (PIDs, process tables) is garbage +# post-restart — a numerically-equal PID in the new container is a different process. _STALE_RUNTIME_FILES = ("gateway.pid", "processes.json") ReconcileActionLabel = Literal["started", "registered", "skipped"] @@ -52,9 +45,8 @@ class ReconcileAction: profile: str prior_state: str | None action: ReconcileActionLabel - # How the previous gateway life ended: "clean" (exit path ran), "unclean" (sentinel still - # says running — SIGKILL/OOM/VM death), or "unknown" (no sentinel / never ran). Container - # boot is the one place that can stamp "the previous life ended violently" into a + # "clean" (exit path ran) / "unclean" (sentinel still says running — SIGKILL/OOM/VM death) + # / "unknown". Boot is the one place that can stamp a violent previous death into a # durable, volume-persisted log line (gateway.lifecycle_ledger). prior_exit: str = "unknown" @@ -74,17 +66,14 @@ def reconcile_profile_gateways( """Recreate s6 service registrations for every persistent profile. Always registers a ``gateway-default`` slot for the root profile (the implicit profile at - the top of ``$HERMES_HOME``, not under ``profiles/``): the dispatcher in - ``hermes_cli.gateway`` maps an empty profile suffix to it, so it is what - ``hermes gateway start`` (no ``-p``) targets. + the top of ``$HERMES_HOME``): ``hermes_cli.gateway`` maps an empty profile suffix to it, + so it is what ``hermes gateway start`` (no ``-p``) targets. """ actions: list[ReconcileAction] = [] - # A multiplexing root/default gateway owns inbound platform connections for every - # profile. Named slots must still be registered (explicit lifecycle management stays - # available), but booting them from their persisted run intent would create additional - # multiplex owners. The runtime resolver gives a recognized environment override - # precedence over config.yaml and otherwise preserves the configured value. + # A multiplexing root gateway owns inbound platform connections for every profile: named + # slots are still registered (lifecycle management stays available) but must not boot + # from their persisted run intent, or they would become additional multiplex owners. from gateway.config import load_gateway_config from utils import is_truthy_value try: @@ -94,10 +83,9 @@ def reconcile_profile_gateways( "GATEWAY_MULTIPLEX_PROFILES override if set.", exc_info=True) multiplex_profiles = is_truthy_value(os.environ.get("GATEWAY_MULTIPLEX_PROFILES")) - # Default profile — always register, even if nothing has ever populated the root profile - # dir; auto-up only when the prior state was "running" (same rule as named profiles). A - # legacy `gateway run` container with no state yet seeds that intent as `running` so the - # s6 reconciler preserves the pre-s6 behavior. + # Default profile — always registered; auto-up only when the prior state was "running" + # (same rule as named profiles). A legacy `gateway run` container with no state yet seeds + # that intent as `running` so the pre-s6 behavior is preserved. legacy_default_state = _maybe_migrate_legacy_gateway_run_state( hermes_home, container_argv=container_argv, dry_run=dry_run) default_prior_state = legacy_default_state or _read_desired_state(hermes_home) @@ -110,9 +98,8 @@ def reconcile_profile_gateways( profiles_root = hermes_home / "profiles" if profiles_root.is_dir(): for entry in sorted(profiles_root.iterdir()): - # SOUL.md is always seeded by `hermes profile create` (config.yaml is not — that - # comes later via `hermes setup`): the "real profile" marker so stray dirs - # (backups, manual mkdir) aren't picked up. + # SOUL.md is always seeded by `hermes profile create` (config.yaml comes later via + # `hermes setup`): the "real profile" marker that skips stray dirs (backups, mkdir). if not entry.is_dir() or not (entry / "SOUL.md").exists(): continue # "default" is reserved for the root profile (above); skip a stray @@ -141,9 +128,8 @@ def _maybe_migrate_legacy_gateway_run_state( ) -> str | None: """Seed root gateway_state for pre-s6 `gateway run` containers. - The tini image let Docker users run the gateway as the container command. After the s6 - migration profile gateways are restored from persisted gateway_state.json, so a legacy - container with no state file would register the default service down and never start. + The tini image let users run the gateway as the container command; post-s6, gateways are + restored from gateway_state.json, so such a container would register down and never start. """ state_file = hermes_home / "gateway_state.json" if state_file.exists(): @@ -173,11 +159,10 @@ def _cmdline_argv(cmdline: Path) -> tuple[str, ...]: def _read_container_argv() -> tuple[str, ...]: - """Best-effort read of the container's main program argv. + """Best-effort read of the container's main program argv (the one holding ``main-wrapper.sh``). - s6-overlay v2: PID 1 is ``/init`` and its argv holds ``main-wrapper.sh``. v3: PID 1 is - ``s6-svscan`` and the real command lives on another PID, so after the PID 1 fast path we - scan ``/proc/*/cmdline`` for a process whose argv contains ``main-wrapper.sh``. + s6-overlay v2: PID 1 is ``/init``. v3: PID 1 is ``s6-svscan`` and the real command lives on + another PID, so after the PID 1 fast path we scan ``/proc/*/cmdline``. """ def _cmdlines(): yield Path("/proc/1/cmdline") @@ -201,9 +186,8 @@ def _read_container_argv() -> tuple[str, ...]: def _strip_container_argv_prefix(argv: Sequence[str]) -> list[str]: """Strip the s6/wrapper prefix off the container argv, leaving the hermes args. - Drop everything through the ``main-wrapper.sh`` token rather than peel leading tokens - positionally (which broke on the s6 v2→v3 launcher-shape change): the wrapper path is the - stable boundary the image owns, and the subcommand always follows it. + Drops everything through the ``main-wrapper.sh`` token — the stable boundary the image + owns — rather than peeling tokens positionally (which broke on the s6 v2→v3 bump). """ args = list(argv) @@ -239,12 +223,10 @@ def _is_dashboard_container(argv: Sequence[str]) -> bool: def _read_desired_state(profile_dir: Path) -> str | None: - """Persisted gateway desired state for reconciliation. + """Persisted gateway desired state: ``desired_state`` (operator intent), else ``gateway_state``. - Newer state files carry ``desired_state`` (operator intent from s6 lifecycle commands); - older ones only ``gateway_state``, kept as a fallback so existing profiles preserve their - behavior until the next explicit start/stop. Missing/unparseable files count as "no - desired state" so a corrupt file can't bork the whole reconciliation. + The older key is a fallback so existing profiles keep their behavior until the next explicit + start/stop. Missing/unparseable files count as "no state" so a corrupt file can't bork boot. """ state_file = profile_dir / "gateway_state.json" if not state_file.exists(): @@ -286,11 +268,10 @@ def _write_exec(path: Path, content: str) -> None: def _register_service(scandir: Path, profile: str, *, start: bool) -> None: """Recreate the s6 service slot for one profile. - Mirrors ``S6ServiceManager.register_profile_gateway`` but sets start state via the - ``down`` marker directly: cont-init.d runs as root before s6-svscan scans the dynamic - scandir, so the manager's ``s6-svscanctl -a`` would fail with no control socket. Built in - a sibling temp dir and ``Path.replace``d into place so an interrupted write never leaves - a half-populated dir. + Mirrors ``S6ServiceManager.register_profile_gateway`` but sets start state via the ``down`` + marker: cont-init.d runs before s6-svscan scans the scandir, so ``s6-svscanctl -a`` has no + control socket yet. Built in a sibling temp dir and ``Path.replace``d into place so an + interrupted write never leaves a half-populated dir. """ import shutil @@ -300,10 +281,9 @@ def _register_service(scandir: Path, profile: str, *, start: bool) -> None: validate_profile_name(profile) service_dir = scandir / f"gateway-{profile}" - # Dot-prefix the staging dir so s6-svscan skips it while half-built. A non-dotted staging - # name is supervised AS ROOT by any concurrent ``s6-svscanctl -a`` rescan the moment it - # has a valid ``type``/``run``, creating a root-owned ``supervise/`` that makes - # ``_seed_supervise_skeleton`` EACCES (see ``S6ServiceManager.register_profile_gateway``). + # Dot-prefix the staging dir so s6-svscan skips it while half-built: a non-dotted name is + # supervised AS ROOT by any concurrent rescan the moment it has ``type``/``run``, creating + # a root-owned ``supervise/`` that makes ``_seed_supervise_skeleton`` EACCES. tmp_dir = service_dir.with_name("." + service_dir.name + ".tmp") if tmp_dir.exists(): @@ -313,9 +293,8 @@ def _register_service(scandir: Path, profile: str, *, start: bool) -> None: try: (tmp_dir / "type").write_text("longrun\n", encoding="utf-8") - # Reuse the manager's script rendering — single source of truth so both registration - # paths stay consistent. extra_env is empty here; per-profile env comes from the - # profile's config.yaml (which the gateway itself loads). + # Reuse the manager's script rendering so both registration paths stay consistent. + # extra_env is empty: per-profile env comes from the profile's config.yaml. _write_exec(tmp_dir / "run", S6ServiceManager._render_run_script(profile, extra_env={})) _write_exec(tmp_dir / "finish", S6ServiceManager._render_finish_script()) @@ -323,18 +302,16 @@ def _register_service(scandir: Path, profile: str, *, start: bool) -> None: (tmp_dir / "log").mkdir() _write_exec(tmp_dir / "log" / "run", S6ServiceManager._render_log_run(profile)) - # A `down` file tells s6-supervise NOT to start the service when s6-svscan picks it - # up; the user brings it up with `hermes -p gateway start` (→ `s6-svc -u`). + # `down` tells s6-supervise NOT to start on pickup; `hermes -p gateway start` + # brings it up (→ `s6-svc -u`). if not start: (tmp_dir / "down").touch() - # Pre-create supervise/ with hermes ownership BEFORE publishing: the s6-supervise - # spawned on pickup will EEXIST our dirs/FIFOs and inherit that ownership, so runtime - # s6-svc / s6-svstat / s6-svwait calls (dispatched as the hermes user) won't EACCES. + # Pre-create supervise/ with hermes ownership BEFORE publishing: s6-supervise will EEXIST + # our dirs/FIFOs and inherit it, so runtime s6-svc calls as the hermes user won't EACCES. _seed_supervise_skeleton(tmp_dir) - # Publish atomically: Path.replace silently replaces an existing target on POSIX, so a - # previous reconcile pass's slot is overwritten in one operation. + # Publish atomically (Path.replace overwrites a previous pass's slot in one operation). if service_dir.exists(): shutil.rmtree(service_dir) tmp_dir.replace(service_dir) @@ -343,18 +320,16 @@ def _register_service(scandir: Path, profile: str, *, start: bool) -> None: raise -# 256 KiB soft cap on container-boot.log, rotated to .1 when crossed (~3000 lines at ~80 B -# each — about a year of daily reboots on a 5-profile container). Tuned for grep-ability -# more than space (the persistent volume has GB). +# 256 KiB soft cap on container-boot.log (~3000 lines ≈ a year of daily reboots on a +# 5-profile container), rotated to .1 when crossed. Tuned for grep-ability, not space. _LOG_ROTATE_BYTES = 256 * 1024 def _write_reconcile_log(hermes_home: Path, actions: list[ReconcileAction]) -> None: - """Append one line per profile to $HERMES_HOME/logs/container-boot.log. + """Append one line per profile to $HERMES_HOME/logs/container-boot.log (rotated to ``.1``). - A separate file (vs. agent.log) lets operators debugging "why didn't my profile come back - up" grep for "profile=foo" without unrelated activity. Rotated to ``.1`` (replacing any - previous rotation) before appending once it exceeds ``_LOG_ROTATE_BYTES``. + A separate file (vs. agent.log) lets operators grep "profile=foo" when debugging "why + didn't my profile come back up". """ import time log_dir = hermes_home / "logs" @@ -379,11 +354,10 @@ def _write_reconcile_log(hermes_home: Path, actions: list[ReconcileAction]) -> N def main() -> int: """Entry point invoked from /etc/cont-init.d/02-reconcile-profiles.""" - # A dashboard-only container never supervises per-profile gateways, and reconciling here - # is actively harmful: when gateway and dashboard containers share a bind-mounted - # HERMES_HOME, both race to flock() the same s6-log lock files under - # logs/gateways//lock → "Resource busy" restart storm. The role is detected from - # PID 1 argv, not an operator flag — a flag can be forgotten in a hand-written manifest. + # A dashboard-only container never supervises gateways, and reconciling here is harmful: + # with a shared bind-mounted HERMES_HOME both containers race to flock() the same s6-log + # lock files → "Resource busy" restart storm. Detected from PID 1 argv, not an operator + # flag — a flag can be forgotten in a hand-written manifest. if _is_dashboard_container(_read_container_argv()): print("reconcile: skipping (dashboard container — does not need per-profile gateways)") return 0 From 5f392d511cbf6bcd04932d15b597c2a6d29bd237 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:51:03 -0700 Subject: [PATCH 25/29] =?UTF-8?q?refactor(hermes=5Fcli):=20debug.py=20?= =?UTF-8?q?=E2=80=94=20iterate=20bundle=20items=20for=20upload/local=20pri?= =?UTF-8?q?nt=20instead=20of=20re-deriving=20labels?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/debug.py | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/hermes_cli/debug.py b/hermes_cli/debug.py index 420eb225dc..fa28e5cbd0 100644 --- a/hermes_cli/debug.py +++ b/hermes_cli/debug.py @@ -460,14 +460,12 @@ def collect_share_bundle(log_lines: int = 200, redact: bool = True) -> dict[str, """ dump_text = _capture_dump() log_snapshots = _capture_default_log_snapshots(log_lines, redact=redact) - report = collect_debug_report(log_lines=log_lines, dump_text=dump_text, log_snapshots=log_snapshots) banner = _REDACTION_BANNER if redact else "" bundle: dict[str, str] = {"report": banner + report} for name in _FULL_LOGS: - full = log_snapshots[name].full_text - if full: + if full := log_snapshots[name].full_text: bundle[f"{name}.log"] = banner + dump_text + f"\n\n--- full {name}.log ---\n" + full return bundle @@ -506,35 +504,24 @@ def build_debug_share( Blocking network I/O — callers inside an event loop must run it in a worker thread. """ _best_effort_sweep_expired_pastes() - bundle = collect_share_bundle(log_lines=log_lines, redact=redact) - if redact: logger.info( "hermes debug share: applied force-mode redaction to log snapshots before upload" ) - report = bundle["report"] - - urls: dict[str, str] = {} failures: list[str] = [] - - # Summary report is required — raises on failure so callers can fall back. - urls["Report"] = upload_to_pastebin(report, expiry_days=expiry) - - # Full logs are optional — failures are collected, not raised. - for name in _FULL_LOGS: - label = f"{name}.log" - content = bundle.get(label) - if not content: + # Summary report is required — raises on failure so callers can fall back; full logs are + # optional — failures are collected, not raised. + urls = {"Report": upload_to_pastebin(report, expiry_days=expiry)} + for label, content in bundle.items(): + if label == "report": continue try: urls[label] = upload_to_pastebin(content, expiry_days=expiry) except Exception as exc: failures.append(f"{label}: {exc}") - _schedule_auto_delete(list(urls.values())) - return DebugShareResult(urls=urls, failures=failures, redacted=redact, auto_delete_seconds=_AUTO_DELETE_SECONDS, report=report) @@ -574,11 +561,9 @@ def run_debug_share(args): print("Collecting debug report...") bundle = collect_share_bundle(log_lines=log_lines, redact=redact) print(bundle["report"]) - for name in _FULL_LOGS: - body = bundle.get(f"{name}.log") - if body: - print(f"\n\n{'=' * 60}\nFULL {name}.log\n{'=' * 60}\n") - print(body) + for label, body in bundle.items(): + if label != "report": + print(f"\n\n{'=' * 60}\nFULL {label}\n{'=' * 60}\n\n{body}") return if getattr(args, "nous", False): From 02586dbed0e30c5ed43fe4b2ae4a2503c5b58ad8 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:58:35 -0700 Subject: [PATCH 26/29] refactor(hermes_cli): cron incidents rows table, debug share flow tightening, copilot env-var loop invert --- hermes_cli/copilot_auth.py | 20 +++++++++++--------- hermes_cli/cron.py | 26 ++++++++++++++------------ hermes_cli/debug.py | 23 ++++++++--------------- 3 files changed, 33 insertions(+), 36 deletions(-) diff --git a/hermes_cli/copilot_auth.py b/hermes_cli/copilot_auth.py index cecc6451c5..18d8f0ba37 100644 --- a/hermes_cli/copilot_auth.py +++ b/hermes_cli/copilot_auth.py @@ -66,13 +66,13 @@ def resolve_copilot_token() -> tuple[str, str]: any_env_var_set = False for env_var in COPILOT_ENV_VARS: val = os.getenv(env_var, "").strip() - if val: - any_env_var_set = True - valid, msg = validate_copilot_token(val) - if not valid: - logger.warning("Token from %s is not supported: %s", env_var, msg) - continue + if not val: + continue + any_env_var_set = True + valid, msg = validate_copilot_token(val) + if valid: return val, env_var + logger.warning("Token from %s is not supported: %s", env_var, msg) # Fall back to gh auth token ONLY when no Copilot env var was explicitly set: an exported # GITHUB_TOKEN (even an unsupported classic PAT) means the user intends *that* token, not @@ -207,7 +207,8 @@ def copilot_device_code_login( print(" ✗ GitHub did not return a device code.") return None - print(f"\n Open this URL in your browser: {verification_uri}\n Enter this code: {user_code}\n") + print(f"\n Open this URL in your browser: {verification_uri}\n" + f" Enter this code: {user_code}\n") print(" Waiting for authorization...", end="", flush=True) deadline = time.monotonic() + timeout_seconds @@ -243,7 +244,8 @@ def copilot_device_code_login( print(".", end="", flush=True) continue if error: - print("\n" + _DEVICE_CODE_TERMINAL_ERRORS.get(error, f" ✗ Authorization failed: {error}")) + print("\n" + _DEVICE_CODE_TERMINAL_ERRORS.get(error, + f" ✗ Authorization failed: {error}")) return None print("\n ✗ Timed out waiting for authorization.") @@ -349,7 +351,7 @@ def _jwt_disk_path() -> Optional[Path]: def _with_jwt_store(verb: str, op): - """Run ``op(path, store_or_None)`` against the disk store; any failure is logged, never raised.""" + """Run ``op(path, store_or_None)`` against the disk store; failures are logged, never raised.""" path = _jwt_disk_path() if not path: return None diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index 60d7ddf3df..760cabbce6 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -67,7 +67,8 @@ def _builtin_gateway_liveness() -> Optional[bool]: # gateway process it short-circuits to True so the in-gateway cron tool never # emits a false "gateway not running" (find_gateway_pids can transiently miss the # gateway just after a restart). - with contextlib.suppress(Exception): # a crashing probe is "unknown" — let the pid scan decide + # A crashing lock probe is "unknown", not "dead" — let the pid scan decide. + with contextlib.suppress(Exception): from gateway.status import is_gateway_runtime_lock_active if is_gateway_runtime_lock_active(): return True @@ -212,7 +213,8 @@ def _job_rows(job: Dict[str, Any]) -> List[tuple[str, str]]: optional = [ ("Skills", ", ".join(skills) if skills else ""), ("Script", job.get("script")), - ("Monitor", f"{monitor_source} (agent runs only on output change)" if monitor_source else ""), + ("Monitor", f"{monitor_source} (agent runs only on output change)" if monitor_source + else ""), ("Changed", mon_state.get("last_changed_at") if monitor_source else ""), ("Mode", color("no-agent", Colors.DIM) + " (script stdout delivered directly)" if job.get("no_agent") else ""), @@ -318,17 +320,17 @@ def cron_incidents(args) -> int: _print_banner("Cron Failure Incidents") for inc in incidents: state_display = color(inc["state"], _INCIDENT_STATE_COLORS.get(inc["state"], Colors.DIM)) - print(f" {color(inc['id'], Colors.YELLOW)} {state_display}") - print(f" Job: {inc['job_id']}") - print(f" Type: {inc.get('failure_type', 'unknown')}") - print(f" First seen: {inc.get('first_seen_at', '?')}") - print(f" Last seen: {inc.get('last_seen_at', '?')}") error_text = re.sub(r"\s+", " ", inc.get("error") or "").strip() if len(error_text) > 160: error_text = error_text[:157].rstrip() + "..." - print(f" Error: {error_text}") - if inc.get("output_file"): - print(f" Output: {inc['output_file']}") + rows = [("Job", inc["job_id"]), ("Type", inc.get("failure_type", "unknown")), + ("First seen", inc.get("first_seen_at", "?")), + ("Last seen", inc.get("last_seen_at", "?")), ("Error", error_text), + ("Output", inc.get("output_file"))] + print(f" {color(inc['id'], Colors.YELLOW)} {state_display}") + for label, value in rows: + if label != "Output" or value: + print(f" {label + ':':<12}{value}") print() print(color(f" {len(incidents)} incident(s) | ack one with: hermes cron incidents ack ", Colors.DIM)) @@ -566,8 +568,8 @@ def cron_doctor() -> int: if not findings: print(color("✓ Cron doctor found no issues", Colors.GREEN)) - print(color(f" Checked {len(jobs)} active job(s)." if jobs else " No active jobs configured.", - Colors.DIM)) + summary = f" Checked {len(jobs)} active job(s)." if jobs else " No active jobs configured." + print(color(summary, Colors.DIM)) return 0 issue_count = sum(len(issues) for _, issues in findings) diff --git a/hermes_cli/debug.py b/hermes_cli/debug.py index fa28e5cbd0..436e10614f 100644 --- a/hermes_cli/debug.py +++ b/hermes_cli/debug.py @@ -531,21 +531,18 @@ def _confirm_upload(args) -> bool: if bool(getattr(args, "yes", False)): return True if not sys.stdin.isatty(): - print( - "ERROR: Non-interactive mode requires --yes to confirm upload.\n" - " This prevents accidental exposure of personal data.\n" - " Use --local to view the report without uploading.", - file=sys.stderr, - ) + print("ERROR: Non-interactive mode requires --yes to confirm upload.\n" + " This prevents accidental exposure of personal data.\n" + " Use --local to view the report without uploading.", file=sys.stderr) sys.exit(1) try: answer = input("Upload debug report? [y/N] ").strip().lower() except (EOFError, KeyboardInterrupt): answer = "" - if answer not in ("y", "yes"): - print("Aborted.") - return False - return True + if answer in ("y", "yes"): + return True + print("Aborted.") + return False def run_debug_share(args): @@ -574,7 +571,6 @@ def run_debug_share(args): if not _confirm_upload(args): return print("Collecting debug report...\nUploading...") - try: result = build_debug_share(log_lines=log_lines, expiry=expiry, redact=redact) except RuntimeError as exc: @@ -618,15 +614,12 @@ def _run_debug_share_nous(args, *, log_lines: int, redact: bool) -> None: print("⚠️ --no-redact is set: secrets in your logs will NOT be redacted before upload.\n") print("Collecting debug report...") _best_effort_sweep_expired_pastes() - bundle = collect_share_bundle(log_lines=log_lines, redact=redact) if redact: logger.info("hermes debug share --nous: applied force-mode redaction before upload") - blob = build_nous_bundle(bundle, redact=redact) - print("Uploading to Nous diagnostics storage...") try: - res = share_to_nous(blob) + res = share_to_nous(build_nous_bundle(bundle, redact=redact)) except Exception as exc: print( f"\nNous upload failed: {exc}\n" From 9f854665e8e18a1e1f6dc86d632b51522841624e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:02:34 -0700 Subject: [PATCH 27/29] =?UTF-8?q?refactor(hermes=5Fcli):=20dashboard=5Fpro?= =?UTF-8?q?cs/copilot=5Fauth=20=E2=80=94=20compact=20rationale=20docstring?= =?UTF-8?q?s=20and=20comments=20(every=20WHY=20kept)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/copilot_auth.py | 52 ++++++++-------- hermes_cli/dashboard_procs.py | 108 ++++++++++++++-------------------- 2 files changed, 66 insertions(+), 94 deletions(-) diff --git a/hermes_cli/copilot_auth.py b/hermes_cli/copilot_auth.py index 18d8f0ba37..fca2f96b70 100644 --- a/hermes_cli/copilot_auth.py +++ b/hermes_cli/copilot_auth.py @@ -24,10 +24,9 @@ from hermes_cli._subprocess_compat import IS_WINDOWS, windows_hide_flags logger = logging.getLogger(__name__) -# OAuth device code flow — VS Code's GitHub App client ID. The opencode OAuth App ID -# (Ov23li8tweQw6odWQebz) produces gho_* tokens that cannot be exchanged for Copilot API JWTs -# (404 on /copilot_internal/v2/token); VS Code's produces ghu_* tokens that support exchange, -# required for internal-only models and enterprise endpoints. Tested on Individual + Enterprise. +# OAuth device code flow — VS Code's GitHub App client ID: it mints ghu_* tokens that can be +# exchanged for Copilot API JWTs (required for internal-only models and enterprise endpoints). +# The opencode App ID (Ov23li8tweQw6odWQebz) mints gho_* tokens that 404 on exchange. COPILOT_OAUTH_CLIENT_ID = "Iv1.b507a08c87ecfe98" # ghp_ classic PATs are rejected by the Copilot API (gho_ / github_pat_ / ghu_ work). _CLASSIC_PAT_PREFIX = "ghp_" @@ -105,12 +104,11 @@ def _gh_cli_candidates() -> list[str]: return candidates -# ``gh auth token`` result cache. When gh has no credential store for this HOME (fresh -# profile, desktop-spawned backend, CI) the probe can block for its full 5s timeout on -# keyring / D-Bus prompts. Provider inventory builds (``/api/model/options``, ``hermes tools``) -# probe Copilot auth several times per request, so an uncached miss turned one settings-page -# load into a 4×5s stall exceeding the Desktop renderer's 15s IPC budget. Successes and -# failures are both cached; a short TTL keeps a fresh ``gh auth login`` discoverable. +# ``gh auth token`` result cache. With no credential store for this HOME (fresh profile, CI) +# the probe blocks for its full 5s timeout on keyring / D-Bus prompts, and provider inventory +# builds probe Copilot auth several times per request — an uncached miss made one settings +# page a 4×5s stall past the Desktop renderer's 15s IPC budget. Misses are cached too; a short +# TTL keeps a fresh ``gh auth login`` discoverable. _GH_CLI_TOKEN_CACHE_TTL_SECONDS = 300.0 _gh_cli_token_cache: tuple[float, Optional[str]] | None = None @@ -264,27 +262,24 @@ _TOKEN_EXCHANGE_URL = "https://api.github.com/copilot_internal/v2/token" _EDITOR_VERSION = "vscode/1.104.1" _EXCHANGE_USER_AGENT = "GitHubCopilotChat/0.26.7" -# Transient-failure hardening. Gateway startup often races network readiness (launchd -# relaunch, DHCP/VPN settling); a single-shot exchange that fails there silently degrades to -# the RAW GitHub token, which the Copilot server routes to the "copilot-language-server" -# integrator whose model allowlist omits enterprise-only models → HTTP 400 on every turn -# until the next restart. Retry a few times, and persist the last good exchanged JWT so a -# restart during a blip reuses the still-valid ~30-min token instead of degrading. +# Transient-failure hardening. Gateway startup races network readiness (launchd relaunch, +# DHCP/VPN settling); a single-shot exchange failing there silently degrades to the RAW GitHub +# token, which Copilot routes to the "copilot-language-server" integrator whose allowlist omits +# enterprise-only models → HTTP 400 every turn until restart. Retry, and persist the last good +# JWT so a restart during a blip reuses the still-valid ~30-min token. _EXCHANGE_MAX_ATTEMPTS = 3 _EXCHANGE_BACKOFF_BASE_SECONDS = 1.5 # sleeps ~1.5s, ~3.0s between attempts _JWT_DISK_FILENAME = ".copilot_jwt.json" _JWT_DISK_MAX_BYTES = 1_048_576 # 1 MiB cap on the persisted JWT store read -# Negative cache for failed exchanges: raw-token fingerprint -> epoch until which attempts -# raise immediately (success clears the entry). Without it every load_pool("copilot") re-ran -# the full exchange, and on a permanently-rejected token (403: not Copilot-entitled, expired -# grant, org policy) the retry backoff burned ~4.5s of sleep on EVERY provider-discovery pass -# (/model picker, delegation child spawns, web dashboard). +# Negative cache: fingerprint -> epoch until which attempts raise immediately (success clears +# it). Without it a permanently-rejected token (403: not entitled, expired grant, org policy) +# burned ~4.5s of retry backoff on EVERY provider-discovery pass (/model picker, delegation +# spawns, dashboard). _exchange_failure_cache: dict[str, float] = {} -# Single-flight guard per token fingerprint: concurrent callers (the dashboard polls -# /api/credentials/pool every few seconds, each poll off-loop) wait on the ONE in-flight -# exchange and then hit the positive/negative cache, instead of each spawning their own hung -# resolver thread during a DNS outage. +# Single-flight guard per fingerprint: concurrent callers (dashboard polls /api/credentials/ +# pool every few seconds) wait on the ONE in-flight exchange and then hit the cache, instead +# of each spawning its own hung resolver thread during a DNS outage. _exchange_locks: dict[str, threading.Lock] = {} _exchange_locks_guard = threading.Lock() @@ -411,10 +406,9 @@ def _save_jwt_to_disk(fp: str, api_token: str, expires_at: float, base_url: Opti _with_jwt_store("persist", _save) -# Hard wall-clock cap for the token-exchange HTTP call. urllib's ``timeout`` only bounds -# socket operations AFTER DNS resolution succeeds; getaddrinfo blocks in C and ignores it, -# so on a networkless Windows host the resolver can hang for many minutes (observed: a -# 17-minute event-loop stall that took the whole backend down). +# Hard wall-clock cap for the exchange call: urllib's ``timeout`` only bounds socket ops AFTER +# DNS succeeds; getaddrinfo blocks in C and ignores it, so a networkless Windows host can hang +# for many minutes (observed: a 17-minute event-loop stall that took the backend down). _DNS_GRACE_SECONDS = 5.0 diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py index 5ec2e96868..afde2e30dc 100644 --- a/hermes_cli/dashboard_procs.py +++ b/hermes_cli/dashboard_procs.py @@ -10,9 +10,8 @@ import subprocess import sys from pathlib import Path -# Cmdline substrings identifying the long-lived server. ``hermes serve`` is the same server -# under the headless name the desktop app spawns; it is reaped on update for the same -# frontend/backend-mismatch reason as ``dashboard``. +# Cmdline substrings identifying the long-lived server; ``hermes serve`` is the same server +# under the headless name the desktop app spawns, reaped on update for the same reason. _DASHBOARD_PATTERNS = tuple( f"{launcher} {cmd}" for cmd in ("dashboard", "serve") @@ -42,10 +41,9 @@ def _iter_process_table() -> list[tuple[int, str]]: """``(pid, cmdline)`` for every process, via wmic (Windows) or ps. Raises on scan failure.""" rows: list[tuple[int, str]] = [] if sys.platform == "win32": - # errors="ignore": wmic may emit the system code page; a decode error would leave - # stdout=None. bounded_probe_run (not run()): run()'s post-timeout cleanup joins pipe - # readers unbounded and a conhost descendant holding duplicated handles wedges it - # forever. It also passes CREATE_NO_WINDOW for the pythonw.exe backend. + # errors="ignore": wmic may emit the system code page (a decode error leaves stdout + # None). bounded_probe_run, not run(): run()'s post-timeout cleanup joins pipe readers + # unbounded and a conhost descendant holding duplicated handles wedges it forever. from hermes_cli._subprocess_compat import bounded_probe_run result = bounded_probe_run( ["wmic", "process", "get", "ProcessId,CommandLine", "/FORMAT:LIST"], @@ -73,13 +71,11 @@ def _iter_process_table() -> list[tuple[int, str]]: def _scan_dashboard_processes(*, exclude_pids: set[int] | None = None) -> list[tuple[int, str]]: - """Return matching ``dashboard``/``serve`` processes with their cmdlines. + """``(pid, cmdline)`` of running ``dashboard``/``serve`` processes; empty on any scan error. - ``hermes update`` swaps files on disk while a forgotten dashboard keeps the old Python - backend in memory against the new JS bundle — a silent mismatch (new auth headers → every - API call 401s). *exclude_pids* must never be returned: Hermes Desktop sets - ``HERMES_DESKTOP_CHILD_PID`` on the backend it spawns so an auto-update never kills the - backend it manages itself. Empty list on any scan error (missing ps/wmic, timeout, ...). + A forgotten dashboard keeps the old Python backend in memory against the new JS bundle + after ``hermes update`` (new auth headers → every API call 401s). *exclude_pids* must never + be returned: Desktop marks the backend it manages via ``HERMES_DESKTOP_CHILD_PID``. """ self_pid = os.getpid() skip = {self_pid, *(exclude_pids or ())} @@ -91,10 +87,8 @@ def _scan_dashboard_processes(*, exclude_pids: set[int] | None = None) -> list[t except (FileNotFoundError, subprocess.TimeoutExpired, OSError): return [] - # Spawn-ledger augmentation: substring patterns miss profiled launches - # (`hermes --profile p serve ...`). Every serve/dashboard registers itself in the spawn - # ledger with live-verified (pid, create_time) — positive identity. Add entries the scan - # missed, preferring the ledger's full argv. + # Spawn-ledger augmentation: substring patterns miss profiled launches (`hermes --profile + # p serve`); the ledger holds live-verified (pid, create_time) — positive identity. try: from hermes_cli.process_identity import ledger_entries seen = {pid for pid, _ in found} | skip @@ -146,10 +140,8 @@ def _profile_flag_value(argv: list[str]) -> str | None: def _is_ephemeral_port_zero_backend(argv: list[str]) -> bool: """True for Desktop-style ``serve|dashboard --port 0`` backends. - Ephemeral-port backends are owned by Hermes Desktop (or are PPID-1 orphans of a prior - update respawn). Replaying them after ``hermes update`` multiplies listening backends - because ``--port 0`` always binds a fresh port. Covers ``serve`` and the legacy - ``dashboard --no-open`` fallback older Desktop runtimes use. + Owned by Hermes Desktop (or PPID-1 orphans of a prior respawn); replaying them after + ``hermes update`` multiplies listening backends because ``--port 0`` binds a fresh port. """ if _dashboard_subcommand_index(argv) is None: return False @@ -210,20 +202,14 @@ def _profile_key_for_respawn(argv: list[str], hermes_home: str | None = None) -> def _filter_dashboard_respawn_candidates( candidates: list[tuple[int, list[str], str | None]], *, own_home: str | None = None ) -> list[list[str]]: - """Select which killed manual backends to respawn after ``hermes update``. + """Select which killed manual backends ``(pid, argv, hermes_home)`` to respawn after update. - Candidates are ``(pid, argv, hermes_home)``; *own_home* (default ``get_hermes_home()``) - is a parameter so tests can pin it. Rules: - 1. Never resurrect Desktop ephemeral ``--port 0`` backends — Desktop owns their lifecycle. - 2. Never replay a backend from a **foreign** ``HERMES_HOME``: the respawn is argv-only - (no ``env=``), so it would come back on the *updating* install's home and steal the - foreign install's fixed port, leaving its supervisor to crash-loop on ``EADDRINUSE``. - Unreadable (``None``) stays eligible. - 3. Dedupe by normalized cmdline. 4. At most one backend per profile / home. - - Does **not** blanket-skip PPID-1: a prior update respawn detaches with - ``start_new_session=True``, so fixed-port manual backends sit under init and must stay - eligible next update. + Rules: never resurrect Desktop ``--port 0`` backends (Desktop owns them); never replay a + backend from a **foreign** ``HERMES_HOME`` (the respawn is argv-only, so it would come back + on the updating install's home and steal the foreign install's fixed port → its supervisor + crash-loops on ``EADDRINUSE``; unreadable ``None`` stays eligible); dedupe by normalized + cmdline; at most one backend per profile / home. PPID-1 is NOT skipped: a prior respawn + detaches with ``start_new_session=True``, so fixed-port manual backends sit under init. """ if own_home is None: try: @@ -334,12 +320,10 @@ def _kill_stale_dashboard_processes( ) -> dict[str, list]: """Kill running ``hermes dashboard`` / ``hermes serve`` processes (update end, ``--stop``). - With ``restart_managed`` (update path only) a detected ``hermes-dashboard.service`` is - restarted through systemd, any other killed PID owned by a systemd unit has that unit - restarted after the kill (systemd treats our SIGTERM as a clean stop, so - ``Restart=on-failure`` never fires), and manual PIDs are respawned from their captured - argv. *already_restarted_units* (no ``.service`` suffix) were restarted by the caller - already; PIDs they own are left untouched, not killed twice. + With ``restart_managed`` (update only) systemd-owned PIDs get their unit restarted after the + kill (systemd treats our SIGTERM as a clean stop, so ``Restart=on-failure`` never fires) and + manual PIDs are respawned from captured argv. PIDs owned by *already_restarted_units* (no + ``.service`` suffix) are left untouched, not killed twice. """ if restart_managed and _m()._restart_managed_dashboard_service(reason): # The dashboard unit is handled but every OTHER backend is not (a host may also run @@ -460,13 +444,11 @@ def _detect_concurrent_hermes_instances( ) -> list[tuple[int, str]]: """``(pid, name)`` of other live processes whose .exe is one of our entry-point shims. - Windows blocks DELETE/REPLACE on a running .exe; Desktop spawns ``hermes.EXE`` as a backend - child, so the update's quarantine rename fails with ``[WinError 32]``. Excludes our own - PID and every *shim* ancestor (the setuptools launcher is a separate native process from - the ``python.exe`` it loads — otherwise every update reports its own launcher); a second - hermes.exe under a non-Hermes parent (Desktop child) is still flagged. ``proc.parents()`` - (whole chain at once) because a per-hop loop bailed on the first AccessDenied. - Empty off-Windows, without psutil, or with no other instances. Never raises. + Windows blocks DELETE/REPLACE on a running .exe, so a Desktop-spawned ``hermes.EXE`` makes + the update's quarantine rename fail with ``[WinError 32]``. Excludes our PID and every + *shim* ancestor (the setuptools launcher is a separate native process from the + ``python.exe`` it loads); ``proc.parents()`` at once because a per-hop loop bailed on the + first AccessDenied. Empty off-Windows / without psutil. Never raises. """ if not _m()._is_windows(): return [] @@ -539,10 +521,10 @@ def _process_ppid(pid: int) -> int | None: # --- SSH remote-backend lock ownership ------------------------------------- # ``backend.lock.json`` is written by the Desktop SSH runtime on the *remote* host for every -# ``hermes serve`` it spawns (apps/desktop/electron/remote-lifecycle.ts). A backend another -# client started is legitimate and lock-owned even with no parent here (sshd exited → ppid 1); -# the reap must NEVER kill a PID a valid lock claims — that once killed a production backend. -# Schema constants mirror the writer; a mismatched record is ignored (the reap only *spares*). +# ``hermes serve`` it spawns (apps/desktop/electron/remote-lifecycle.ts). Such a backend is +# legitimate even with no parent here (sshd exited → ppid 1); the reap must NEVER kill a PID a +# valid lock claims — that once killed a production backend. Schema mirrors the writer; a +# mismatched record is ignored (the reap only *spares*). _LOCKFILE_SCHEMA_VERSION = 2 _PROTOCOL_VERSION = 1 _REMOTE_LOCK_SUBDIR = "desktop-ssh" @@ -581,9 +563,8 @@ def _valid_lockfile_payload(parsed: object, ownership_id: str) -> bool: value = parsed.get(field) if not isinstance(value, str) or len(value) > 1024: return False - # logPath is ``{lock_root}/{ownershipId}/{spawnNonce}.log``. Only the suffix is checked so - # a relocated HERMES_HOME doesn't falsely reject a legitimate remote-owned backend (a false - # reject re-introduces the kill). + # logPath is ``{lock_root}/{ownershipId}/{spawnNonce}.log``; only the suffix is checked so a + # relocated HERMES_HOME can't falsely reject a legitimate backend (= re-introduce the kill). return parsed["logPath"].endswith(f"/{ownership_id}/{parsed['spawnNonce']}.log") @@ -636,18 +617,15 @@ def _reap_orphaned_desktop_local_serves( *, reason: str = "orphaned desktop-local hermes serve", signal_term=None, signal_kill=None, sleep_fn=None, lock_owned_pids_fn=None, process_age_seconds_fn=None, ) -> dict[str, list]: - """Kill leftover Desktop-local ``hermes serve`` backends with no parent. + """Kill leftover Desktop-local ``hermes serve`` backends with no parent. Never raises. - When Electron dies uncleanly, ``serve --host 127.0.0.1 --port 0`` children get reparented - to pid 1 with their MCP trees alive; each Desktop boot then stacks a fresh backend on the - corpses until EMFILE. The parent-death watchdog prevents *future* orphans; this clears - *already* orphaned ones. A candidate is reaped only if ALL hold: Desktop-local shape - (never a fixed-port remote serve); ppid 1 (or 0); not self / parent / - HERMES_DESKTOP_CHILD_PID; not claimed by a valid ``backend.lock.json`` (SSH backends other - clients started legitimately sit at ppid 1 — killing them is an incident); older than - ``_REAP_MIN_AGE_SECONDS`` with a determinable age (Desktop writes the lock only after - HERMES_BACKEND_READY, so a live sibling in concurrent multi-profile startup is briefly - unowned and indistinguishable from a corpse). Best-effort; never raises. + When Electron dies uncleanly its ``serve --host 127.0.0.1 --port 0`` children are + reparented to pid 1 with their MCP trees alive; each Desktop boot then stacks a fresh + backend on the corpses until EMFILE. Reaped only if ALL hold: Desktop-local shape; ppid + 0/1; not self / parent / HERMES_DESKTOP_CHILD_PID; not claimed by a valid + ``backend.lock.json`` (SSH backends other clients started legitimately sit at ppid 1); + older than ``_REAP_MIN_AGE_SECONDS`` with a determinable age (Desktop writes the lock only + after HERMES_BACKEND_READY, so a live sibling mid-startup is briefly unowned). """ import signal as _signal import time as _time From b7c5e3a205dd9a4372c31cf5448c1a7ab2d44254 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:03:28 -0700 Subject: [PATCH 28/29] refactor(hermes_cli): cron doctor summary line width --- hermes_cli/cron.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py index 760cabbce6..239f1177a7 100644 --- a/hermes_cli/cron.py +++ b/hermes_cli/cron.py @@ -568,8 +568,8 @@ def cron_doctor() -> int: if not findings: print(color("✓ Cron doctor found no issues", Colors.GREEN)) - summary = f" Checked {len(jobs)} active job(s)." if jobs else " No active jobs configured." - print(color(summary, Colors.DIM)) + note = f" Checked {len(jobs)} active job(s)." if jobs else " No active jobs configured." + print(color(note, Colors.DIM)) return 0 issue_count = sum(len(issues) for _, issues in findings) From f1d0eaaa85e20022e867d1862a254c9b0d683069 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:07:30 -0700 Subject: [PATCH 29/29] =?UTF-8?q?refactor(hermes=5Fcli):=20group=20D=20?= =?UTF-8?q?=E2=80=94=20defensive-layer=20collapse=20(lockfile=20validation?= =?UTF-8?q?,=20pending=20loader,=20gh=20candidates,=20env=20pid=20parsing)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/copilot_auth.py | 9 ++++--- hermes_cli/dashboard_procs.py | 44 ++++++++++++++--------------------- hermes_cli/debug.py | 9 ++----- 3 files changed, 23 insertions(+), 39 deletions(-) diff --git a/hermes_cli/copilot_auth.py b/hermes_cli/copilot_auth.py index fca2f96b70..f7b0066389 100644 --- a/hermes_cli/copilot_auth.py +++ b/hermes_cli/copilot_auth.py @@ -96,11 +96,10 @@ def resolve_copilot_token() -> tuple[str, str]: def _gh_cli_candidates() -> list[str]: """Candidate ``gh`` binary paths, including common Homebrew installs.""" candidates: list[str] = [c for c in (shutil.which("gh"),) if c] - for candidate in ("/opt/homebrew/bin/gh", "/usr/local/bin/gh", - str(Path.home() / ".local" / "bin" / "gh")): - if (candidate not in candidates and os.path.isfile(candidate) - and os.access(candidate, os.X_OK)): - candidates.append(candidate) + candidates += [ + c for c in ("/opt/homebrew/bin/gh", "/usr/local/bin/gh", str(Path.home() / ".local/bin/gh")) + if c not in candidates and os.path.isfile(c) and os.access(c, os.X_OK) + ] return candidates diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py index afde2e30dc..c0f61950a3 100644 --- a/hermes_cli/dashboard_procs.py +++ b/hermes_cli/dashboard_procs.py @@ -243,13 +243,8 @@ def _exclude_pids_from_env() -> set[int]: """PIDs Desktop marks as live backends (``HERMES_DESKTOP_CHILD_PID``, comma-separated).""" out: set[int] = set() for part in os.environ.get("HERMES_DESKTOP_CHILD_PID", "").split(","): - part = part.strip() - if not part: - continue - try: + with contextlib.suppress(ValueError): out.add(int(part)) - except ValueError: - continue return out @@ -265,18 +260,17 @@ def _kill_pids_windows(pids: list[int], killed: list[int], failed: list[tuple[in expected_start_time = pid_start_times.get(pid) if expected_start_time is None: failed.append((pid, "could not verify process identity")) - continue - if not pid_is_hermes(pid, expected_start_time=expected_start_time): + elif not pid_is_hermes(pid, expected_start_time=expected_start_time): failed.append((pid, "not hermes-owned or process identity changed")) - continue - result = subprocess.run(["taskkill", "/PID", str(pid), "/F"], stdout=subprocess.PIPE, - stderr=subprocess.PIPE, stdin=subprocess.DEVNULL, text=True, - encoding="utf-8", errors="replace", timeout=10, - creationflags=windows_hide_flags()) - if result.returncode == 0: - killed.append(pid) else: - failed.append((pid, (result.stderr or result.stdout or "").strip())) + result = subprocess.run( + ["taskkill", "/PID", str(pid), "/F"], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, stdin=subprocess.DEVNULL, text=True, encoding="utf-8", + errors="replace", timeout=10, creationflags=windows_hide_flags()) + if result.returncode == 0: + killed.append(pid) + else: + failed.append((pid, (result.stderr or result.stdout or "").strip())) except (FileNotFoundError, subprocess.TimeoutExpired, OSError) as e: failed.append((pid, str(e))) @@ -552,17 +546,15 @@ def _valid_lockfile_payload(parsed: object, ownership_id: str) -> bool: or not _is_hex(parsed.get("tokenFingerprint"), 32) ): return False - pid = parsed.get("pid") - port = parsed.get("port") - if not isinstance(pid, int) or not 0 < pid <= 4194304: + pid, port = parsed.get("pid"), parsed.get("port") + if not (isinstance(pid, int) and 0 < pid <= 4194304): return False - if not isinstance(port, int) or not 0 <= port <= 65535: + if not (isinstance(port, int) and 0 <= port <= 65535): return False # String fields must be present and bounded (the writer enforces <=1024). - for field in ("profile", "hermesPath", "hermesHome", "logPath", "startedAt"): - value = parsed.get(field) - if not isinstance(value, str) or len(value) > 1024: - return False + if any(not isinstance(parsed.get(f), str) or len(parsed[f]) > 1024 + for f in ("profile", "hermesPath", "hermesHome", "logPath", "startedAt")): + return False # logPath is ``{lock_root}/{ownershipId}/{spawnNonce}.log``; only the suffix is checked so a # relocated HERMES_HOME can't falsely reject a legitimate backend (= re-introduce the kill). return parsed["logPath"].endswith(f"/{ownership_id}/{parsed['spawnNonce']}.log") @@ -576,10 +568,8 @@ def _lock_owned_serve_pids(base_dir: Path | None = None) -> set[int]: import json root = base_dir if base_dir is not None else _hermes_home_dir() / _REMOTE_LOCK_SUBDIR owned: set[int] = set() - if not root.is_dir(): - return owned try: - entries = list(root.iterdir()) + entries = list(root.iterdir()) if root.is_dir() else [] except OSError: return owned for entry in entries: diff --git a/hermes_cli/debug.py b/hermes_cli/debug.py index 436e10614f..f2edfb3003 100644 --- a/hermes_cli/debug.py +++ b/hermes_cli/debug.py @@ -56,11 +56,8 @@ def _pending_file() -> Path: def _load_pending() -> list[dict]: - path = _pending_file() - if not path.exists(): - return [] try: - data = json.loads(path.read_text(encoding="utf-8")) + data = json.loads(_pending_file().read_text(encoding="utf-8")) except (OSError, ValueError): return [] if not isinstance(data, list): @@ -265,10 +262,8 @@ def _missing_log_note(log_name: str) -> str: reason = _CLIENT_SIDE_LOGS.get(log_name) if reason is None: return "(file not found)" - primary = _primary_log_path(log_name) - where = f" — expected at {primary}" if primary else "" - return f"(not on this host: {reason}{where})" + return f"(not on this host: {reason}{f' — expected at {primary}' if primary else ''})" def _resolve_log_path(log_name: str) -> Optional[Path]: