Jack Lau
49cc3708e5
`_spawn_gateway_restart` already reuses an in-flight `hermes gateway restart` child so a double-clicked button cannot start two racing restarts. That guard evaporates exactly when it is needed most: the child exits as soon as it has handed the restart to the supervisor (or to the running gateway), long before the gateway is actually back, so a stale cached dashboard frontend re-firing its own restart every few seconds cleared the guard on every attempt and started a fresh restart each time. #89034 measured the result on an s6-supervised container: 77 `gateway-restart started` entries, 17 of them inside one minute. Each one SIGHUPs a gateway that is still coming up, and killing it mid-FTS5-write corrupted `state.db` ("database disk image is malformed", 203x in agent.log) until the operator recreated the file by hand. Requests for the same profile within GATEWAY_RESTART_COOLDOWN_SECONDS of the last spawn are now coalesced onto that spawn and logged, so a storm produces one restart instead of one per request. The window is fixed rather than health-gated on purpose: a gateway that never comes back would leave a health-gated restart action permanently inert, which is a worse failure than the flood it prevents. The cooldown state is kept outside `_ACTION_PROCS` because completed action children are reaped out of that table, and a guard that disappears when the child exits is the bug being fixed. Only the *frontend-flood* half of #89034 is addressed here. The s6 `finish` death-cap the report also asks for is a separate change to `hermes_cli/service_manager.py` with a much larger blast radius, and is left for a maintainer decision.
68 lines
2.4 KiB
Python
68 lines
2.4 KiB
Python
"""Tests for _spawn_gateway_restart orphan-reap guard (#77276)."""
|
|
from __future__ import annotations
|
|
|
|
import subprocess
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def reset_restart_cooldown():
|
|
"""Clear the #89034 repeat-restart cooldown between cases.
|
|
|
|
``_spawn_gateway_restart`` now coalesces a second restart request that
|
|
arrives within ``GATEWAY_RESTART_COOLDOWN_SECONDS`` of the last spawn, so
|
|
without this the first case's spawn suppresses the second case's.
|
|
"""
|
|
import hermes_cli.web_server as web_server
|
|
|
|
web_server._LAST_GATEWAY_RESTART = None
|
|
yield
|
|
web_server._LAST_GATEWAY_RESTART = None
|
|
|
|
|
|
class TestSpawnGatewayRestartReapsOrphans:
|
|
"""_spawn_gateway_restart must reap orphaned gateways before spawning."""
|
|
|
|
@patch("hermes_cli.web_server._gateway_subcommand", return_value=["gateway", "restart"])
|
|
@patch("hermes_cli.web_server._spawn_hermes_action")
|
|
@patch("hermes_cli.web_server._ACTION_PROCS", {})
|
|
def test_reap_called_before_spawn(self, mock_spawn, mock_subcmd):
|
|
"""Orphan reap runs before the new gateway process is spawned."""
|
|
mock_proc = MagicMock(spec=subprocess.Popen)
|
|
mock_proc.poll.return_value = None
|
|
mock_spawn.return_value = mock_proc
|
|
|
|
from hermes_cli.web_server import _spawn_gateway_restart
|
|
|
|
with patch(
|
|
"hermes_cli.gateway._reap_unsupervised_gateway_orphans"
|
|
) as mock_reap:
|
|
proc, reused = _spawn_gateway_restart()
|
|
|
|
mock_reap.assert_called_once()
|
|
mock_spawn.assert_called_once()
|
|
assert proc is mock_proc
|
|
assert reused is False
|
|
|
|
@patch("hermes_cli.web_server._gateway_subcommand", return_value=["gateway", "restart"])
|
|
@patch("hermes_cli.web_server._spawn_hermes_action")
|
|
@patch("hermes_cli.web_server._ACTION_PROCS", {})
|
|
def test_reap_failure_does_not_block_spawn(self, mock_spawn, mock_subcmd):
|
|
"""If reap raises, the restart still proceeds."""
|
|
mock_proc = MagicMock(spec=subprocess.Popen)
|
|
mock_proc.poll.return_value = None
|
|
mock_spawn.return_value = mock_proc
|
|
|
|
from hermes_cli.web_server import _spawn_gateway_restart
|
|
|
|
with patch(
|
|
"hermes_cli.gateway._reap_unsupervised_gateway_orphans",
|
|
side_effect=OSError("permission denied"),
|
|
):
|
|
proc, reused = _spawn_gateway_restart()
|
|
|
|
mock_spawn.assert_called_once()
|
|
assert proc is mock_proc
|