fix(cli): self-heal cooked-mode termios drift that freezes CLI input

When a prompt_toolkit run_in_terminal cooked->raw restore is lost (cancelled
coroutine, racing chained cross-thread windows from background-review
summaries / process-notification prints), the tty stays in cooked mode while
the Application still expects raw. The kernel line-buffers keystrokes and the
CLI appears to stop taking input even though the event loop is healthy.

Observed live 2026-08-13: interactive session left in 'icanon echo' after a
background skill-review fork + notify_on_complete turn; only an external
stty rescue restored input.

Fix: _heal_cooked_mode_drift() re-applies prompt_toolkit's own raw-mode flag
surgery when stdin's lflag has drifted cooked, and process_loop's idle branch
runs a rate-limited _check_termios_drift() watchdog that skips legitimate
cooked windows (app._running_in_terminal), agent-running phases, non-tty
stdin, and Windows.
This commit is contained in:
Teknium
2026-08-13 13:49:51 -07:00
parent d002167390
commit 2ae96939f5
2 changed files with 212 additions and 0 deletions
+118
View File
@@ -2689,6 +2689,63 @@ def _query_osc11_background() -> str | None:
pass
def _heal_cooked_mode_drift(fd: int) -> bool:
"""Detect and heal cooked-mode termios drift on *fd* while prompt_toolkit
expects raw mode.
prompt_toolkit's ``run_in_terminal`` / ``in_terminal`` wraps every
"print above the prompt" in a ``cooked_mode()`` context: it flips the
tty back to cooked (ICANON/ECHO/ISIG), runs the function, then restores
raw mode. Hermes schedules those windows cross-thread constantly — the
background self-review's ``💾`` summary, background process notification
drains, curses pickers — and if a restore is ever lost (coroutine
cancelled mid-window, racing chains, an external writer touching the
shared tty), the terminal is left in cooked mode while the Application
still believes it owns raw mode. The kernel line-buffers every
keystroke and the CLI appears to "stop taking input" even though the
process is perfectly healthy (observed live: pts in ``icanon echo``
while the event loop idled normally in ``ep_poll``).
This helper is the last line of defense for that whole class: when the
lflag has drifted back to cooked, re-apply prompt_toolkit's own raw-mode
flag surgery (mirrors ``prompt_toolkit.input.vt100.raw_mode``) in place.
Returns True when drift was detected and healed, False when the tty was
already raw (or could not be inspected).
POSIX-only by construction — callers must not invoke this on Windows
(no termios; prompt_toolkit uses the win32 console API there instead).
"""
try:
import termios
attrs = termios.tcgetattr(fd)
except Exception:
return False
lflag = attrs[3]
if not (lflag & (termios.ICANON | termios.ECHO)):
return False # still raw — nothing to do
# Same surgery as prompt_toolkit.input.vt100.raw_mode._patch_lflag /
# _patch_iflag, applied to the *current* attrs so any user settings
# (speed, size-independent flags) are preserved.
attrs[3] = lflag & ~(
termios.ECHO | termios.ICANON | termios.IEXTEN | termios.ISIG
)
attrs[0] = attrs[0] & ~(
termios.IXON
| termios.IXOFF
| termios.ICRNL
| termios.INLCR
| termios.IGNCR
)
# VMIN=1 so reads return per-byte (Solaris-derived systems default to 4;
# prompt_toolkit sets this explicitly in raw_mode.__enter__).
attrs[6][termios.VMIN] = 1
try:
termios.tcsetattr(fd, termios.TCSANOW, attrs)
except Exception:
return False
return True
def _detect_light_mode() -> bool:
global _LIGHT_MODE_CACHE
if _LIGHT_MODE_CACHE is not None:
@@ -4421,6 +4478,8 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
self._pending_edit_snapshots = {}
self._last_input_mode_recovery = 0.0
self._input_mode_recovery_notice_shown = False
self._last_termios_drift_check = 0.0
self._termios_drift_notice_shown = False
# Configuration - priority: CLI args > env vars > config file
# Model comes from: CLI arg or config.yaml (single source of truth).
@@ -7981,6 +8040,57 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
f"If this repeats, run /new or restart this tab.{_RST}"
)
def _check_termios_drift(self) -> None:
"""Watchdog: heal the tty if it drifted back to cooked mode.
See ``_heal_cooked_mode_drift`` for the failure class (a lost
``run_in_terminal`` cooked→raw restore leaves the terminal
line-buffering keystrokes while the prompt_toolkit app believes it
owns raw mode — the CLI looks dead but the process is healthy).
Called from ``process_loop``'s idle branch, so a drifted terminal
self-heals within ~a second of the agent going idle instead of
requiring an external ``stty`` rescue. Skipped while a
``run_in_terminal`` window is legitimately holding cooked mode
(``app._running_in_terminal``), while the agent is running (approval
prompts and sudo prompts legitimately manipulate the tty), and on
Windows (no termios).
"""
if os.name == "nt":
return
app = getattr(self, "_app", None)
if app is None or not getattr(app, "_is_running", False):
return
# A run_in_terminal window is *supposed* to be cooked — don't fight it.
if getattr(app, "_running_in_terminal", False):
return
now = time.monotonic()
if now - self._last_termios_drift_check < 1.0:
return
self._last_termios_drift_check = now
try:
if not sys.stdin.isatty():
return
fd = sys.stdin.fileno()
except Exception:
return
if _heal_cooked_mode_drift(fd):
logger.warning(
"Healed cooked-mode termios drift on stdin — a "
"run_in_terminal cooked→raw restore was lost."
)
# Redraw so the prompt is visibly alive again.
try:
self._invalidate()
except Exception:
pass
if not self._termios_drift_notice_shown:
self._termios_drift_notice_shown = True
_cprint(
f" {_DIM}Recovered terminal from cooked-mode drift "
f"(input should respond normally again).{_RST}"
)
def _preprocess_images_with_vision(self, text: str, images: list, *, announce: bool = True) -> str:
@@ -17919,6 +18029,14 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
# Periodic config watcher — auto-reload MCP on mcp_servers change
if not self._agent_running:
self._check_config_mcp_changes()
# Heal cooked-mode termios drift (lost
# run_in_terminal restore) before draining
# notifications — a drifted tty makes the CLI
# look dead even though the loop is healthy.
try:
self._check_termios_drift()
except Exception:
pass
# Check for background process notifications (completions
# and watch pattern matches) while agent is idle.
try: