From 4beca7a943bbef9039e70a5ced391a15b9d8fe6a Mon Sep 17 00:00:00 2001 From: Shannon Sands Date: Mon, 17 Aug 2026 13:36:52 +1000 Subject: [PATCH] feat(kanban): memory-aware dispatch guard + memory-derived default concurrency cap (OOF-30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two production incidents (OOF-77 "larrikin-lollies", OOF-30 "synclare-task-manager") followed the same shape: no kanban.max_in_progress configured, a busy board, and a 1 GiB hosted VM. The dispatcher fanned out 26-31 concurrent workers, the host went into swap-thrash/OOM, and the whole machine — dashboard included — became unreachable. NAS restart loops then masked the problem: each restart "recovered" briefly before the kanban dispatcher immediately respawned unbounded workers. Building on the cherry-picked max_in_progress-across-both-lanes fix (PR #28695, credit @Dusk1e), this adds two complementary safeguards to hermes_cli/kanban_db.py: 1. Memory-DERIVED default concurrency cap. When kanban.max_in_progress is unset, resolve_max_in_progress() derives a default of clamp(MemTotal / 512 MiB, 2, 8) — e.g. 2 workers on a 1 GiB VM, 8 on 4 GiB+. Explicit config always wins in either direction. On hosts where total memory can't be read (macOS/Windows dev machines), the default stays None (no cap — unchanged behaviour). Wired into both dispatch entry points (gateway/kanban_watchers.py and hermes kanban dispatch) so behaviour matches regardless of path. 2. Live memory-PRESSURE guard inside dispatch_once. A static cap can't see the host's actual memory state (other tenants, bloated long-lived workers). The dispatcher now samples system memory each tick via gateway.lifecycle_ledger.sample_memory() and classifies it with gateway.memory_status.classify_pressure() (same thresholds as the dashboard memory banner and OOM-suspicion heuristics from NS-608/NS-656): critical -> spawn nothing this tick; elevated -> at most one new worker; unknown -> no restriction (fail-open). Reclaim/promotion bookkeeping still runs under pressure, and deferred tasks stay queued — nothing is dropped. Restriction is surfaced on DispatchResult.memory_pressure and logged. Tests: tests/hermes_cli/test_kanban_memory_guard.py (14 tests) covers the derived cap (floor/ceiling/fail-open/explicit-config-wins), the pressure classifier, and dispatch behaviour under critical/elevated/ unknown pressure including defer-not-drop and bookkeeping-still-runs. An autouse fixture in tests/conftest.py pins the memory sample to "no data" suite-wide so existing dispatch tests don't depend on the CI runner's live memory state (opt-out marker: real_memory_guard). --- gateway/kanban_watchers.py | 14 + hermes_cli/config_defaults.py | 10 + hermes_cli/kanban.py | 4 + hermes_cli/kanban_db.py | 145 ++++++++++- tests/conftest.py | 28 ++ tests/hermes_cli/test_kanban_memory_guard.py | 253 +++++++++++++++++++ 6 files changed, 446 insertions(+), 8 deletions(-) create mode 100644 tests/hermes_cli/test_kanban_memory_guard.py diff --git a/gateway/kanban_watchers.py b/gateway/kanban_watchers.py index f5018e18c0..eb267c10ac 100644 --- a/gateway/kanban_watchers.py +++ b/gateway/kanban_watchers.py @@ -1293,6 +1293,20 @@ class GatewayKanbanWatchersMixin: max_in_progress = None else: logger.info("kanban dispatcher: max_in_progress=%s", max_in_progress) + # When the operator never set kanban.max_in_progress, fall back to a + # memory-derived default (OOF-30/OOF-77): unbounded fan-out on small + # hosted VMs has repeatedly swap-thrashed the whole machine. Explicit + # config always wins; None stays None on hosts where total memory + # can't be read (macOS/Windows dev machines). + effective_max_in_progress = _kb.resolve_max_in_progress(max_in_progress) + if max_in_progress is None and effective_max_in_progress is not None: + logger.info( + "kanban dispatcher: kanban.max_in_progress unset; using " + "memory-derived default max_in_progress=%d " + "(set kanban.max_in_progress in config.yaml to override)", + effective_max_in_progress, + ) + max_in_progress = effective_max_in_progress raw_failure_limit = kanban_cfg.get("failure_limit", _kb.DEFAULT_FAILURE_LIMIT) try: diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 4ea20ad87e..3281f41dea 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -2529,6 +2529,16 @@ DEFAULT_CONFIG = { # assignee to any installed profile. When unset, falls back to the # default profile. A task never ends up with assignee=None. "default_assignee": "", + # Global concurrency cap (#33488): when set to a positive int, the + # board never has more than N tasks in 'running' at once, across + # both the ready and review dispatch lanes. Unset (None) means + # "derive from system memory" (OOF-30/OOF-77): the dispatcher caps + # concurrency at roughly MemTotal / 512 MiB, clamped to [2, 8] — + # e.g. 2 workers on a 1 GiB VM. On hosts where total memory can't + # be read (macOS/Windows), unset falls back to no cap. Set an + # explicit value to override the derived default in either + # direction. + "max_in_progress": None, # Per-profile concurrency cap (#21582). When set to a positive int, # no single profile can have more than N workers running at once, # even if the global max_in_progress / max_spawn caps would allow diff --git a/hermes_cli/kanban.py b/hermes_cli/kanban.py index 0671c13359..79519831bc 100644 --- a/hermes_cli/kanban.py +++ b/hermes_cli/kanban.py @@ -2643,6 +2643,10 @@ def _cmd_dispatch(args: argparse.Namespace) -> int: _kanban_cfg.get("max_in_progress_per_profile") ) max_in_progress = _coerce_positive_int(_kanban_cfg.get("max_in_progress")) + # Memory-derived default when unset (OOF-30/OOF-77) — same + # fallback the gateway-embedded dispatcher applies, so behaviour + # matches regardless of which path runs the tick. + max_in_progress = kb.resolve_max_in_progress(max_in_progress) # CLI --max overrides config kanban.max_spawn when both are present; # CLI is the more explicit signal so it wins. cli_max = getattr(args, "max", None) diff --git a/hermes_cli/kanban_db.py b/hermes_cli/kanban_db.py index 1fe97f76b8..06f511d177 100644 --- a/hermes_cli/kanban_db.py +++ b/hermes_cli/kanban_db.py @@ -8075,6 +8075,13 @@ class DispatchResult: DB writes this tick — the lock holder is making progress on the same board. This is the steady-state signal that a single-writer guard is actively preventing two dispatchers from racing on ``kanban.db``.""" + memory_pressure: Optional[str] = None + """System memory pressure observed at spawn time when the memory guard + restricted this tick (OOF-30/OOF-77): ``"critical"`` — no new workers + were spawned this tick; ``"elevated"`` — at most one new worker was + spawned. ``None`` when memory was fine/unknown and the guard imposed + no restriction. Reclaim/promotion bookkeeping still ran either way; + deferred tasks stay queued for the next tick.""" # Bounded registry of recently-reaped worker child exits, populated by the @@ -9603,6 +9610,112 @@ def review_dispatch_enabled() -> bool: return True +# --------------------------------------------------------------------------- +# Memory-aware dispatch guard (OOF-30 / OOF-77) +# +# Two production incidents ("larrikin-lollies", "synclare-task-manager") +# followed the same shape: no ``kanban.max_in_progress`` configured, a busy +# board, and a 1 GiB VM — the dispatcher fanned out 26-31 concurrent workers, +# the host went into swap-thrash/OOM, and the dashboard (and everything else +# on the machine) became unreachable. Two complementary safeguards: +# +# 1. A memory-DERIVED default concurrency cap when the operator never set +# ``kanban.max_in_progress`` (``resolve_max_in_progress``) — sized from +# MemTotal so a 1 GiB VM defaults to 2 workers, not unlimited. +# 2. A live memory-PRESSURE guard inside the dispatch tick itself +# (``_memory_pressure_level``) — even a correctly-sized static cap can't +# see other tenants of the box, so under real observed pressure the +# dispatcher stops adding workers regardless of configured caps. +# +# Both fail open: on non-Linux hosts or any read error the sample is empty, +# the derived default is None (no cap — unchanged behaviour), and the +# pressure level is "unknown" (no spawn restriction). +# --------------------------------------------------------------------------- + +# Assumed per-worker memory footprint for the derived default cap. Hermes +# workers are full agent processes (Python + model client + tool subprocesses); +# ~512 MiB is a deliberately conservative planning number so the derived cap +# errs toward fewer workers on small VMs. +MEMORY_GUARD_MB_PER_WORKER = 512 +# Bounds for the derived default: never below 2 (a board must still make +# progress on the smallest hosted VM) and never above 8 (operators who want +# more fan-out on big iron should say so explicitly in config). +DERIVED_MAX_IN_PROGRESS_FLOOR = 2 +DERIVED_MAX_IN_PROGRESS_CEILING = 8 + + +def _system_memory_sample() -> dict: + """Best-effort system memory snapshot (KiB values), ``{}`` when unknown. + + Delegates to :func:`gateway.lifecycle_ledger.sample_memory` (pure /proc + reads, Linux-only, never raises). Local import keeps ``kanban_db`` + importable in stripped-down environments without the gateway package. + Module-level indirection is also the test seam — the shared conftest + patches this to ``{}`` so suite results don't depend on the CI runner's + live memory state. + """ + try: + from gateway.lifecycle_ledger import sample_memory + return sample_memory() or {} + except Exception: + return {} + + +def derive_default_max_in_progress(sample: Optional[Mapping[str, Any]] = None) -> Optional[int]: + """Memory-derived default for ``kanban.max_in_progress`` when unset. + + ``clamp(MemTotal / MEMORY_GUARD_MB_PER_WORKER, FLOOR, CEILING)`` — e.g. + a 1 GiB VM derives 2, a 4 GiB VM derives 8. Returns ``None`` (no cap, + pre-fix behaviour) when total memory can't be determined, so dev + machines on macOS/Windows are unaffected. + """ + if sample is None: + sample = _system_memory_sample() + total_kib = sample.get("mem_total_kib") + if isinstance(total_kib, bool) or not isinstance(total_kib, int) or total_kib <= 0: + return None + workers = (total_kib // 1024) // MEMORY_GUARD_MB_PER_WORKER + return max( + DERIVED_MAX_IN_PROGRESS_FLOOR, + min(workers, DERIVED_MAX_IN_PROGRESS_CEILING), + ) + + +def resolve_max_in_progress(configured: Optional[int]) -> Optional[int]: + """Return the effective global concurrency cap for a dispatch tick. + + An explicit operator-configured value always wins. When unset, fall back + to the memory-derived default (see :func:`derive_default_max_in_progress`). + Callers that parse config (gateway dispatcher, ``hermes kanban dispatch``) + should route through this so both paths agree. + """ + if configured is not None: + return configured + return derive_default_max_in_progress() + + +def _memory_pressure_level(sample: Optional[Mapping[str, Any]] = None) -> str: + """Classify current system memory pressure: ok/elevated/critical/unknown. + + Reuses :func:`gateway.memory_status.classify_pressure` so the dispatcher's + idea of "critical" matches the memory banner users see on the dashboard + and the lifecycle ledger's OOM-suspicion heuristics (NS-608/NS-656). + ``unknown`` (non-Linux, read failure) imposes no restriction — the guard + must never brick dispatch on hosts where /proc isn't available. + """ + if sample is None: + sample = _system_memory_sample() + if not sample: + return "unknown" + try: + from gateway.memory_status import classify_pressure + return classify_pressure( + sample.get("mem_available_kib"), sample.get("mem_total_kib") + ) + except Exception: + return "unknown" + + def dispatch_once( conn: sqlite3.Connection, *, @@ -9762,14 +9875,6 @@ def _dispatch_once_locked( result.timed_out = enforce_max_runtime(conn) result.promoted = recompute_ready(conn, failure_limit=failure_limit) - # Both knobs are total in-flight caps. Collapse them before either lane - # dispatches so ready and review workers consume the same budget without - # subtracting the already-running count twice. - if max_in_progress is not None and ( - max_spawn is None or max_in_progress < max_spawn - ): - max_spawn = max_in_progress - # Count tasks already running so max_spawn enforces concurrency rather # than a per-tick spawn budget. See the docstring above for the full # rationale; the short version is that a 60-second tick interval with a @@ -9805,6 +9910,30 @@ def _dispatch_once_locked( if spawn_budget is None or spawn_budget > remaining: spawn_budget = remaining + # Memory-pressure guard (OOF-30/OOF-77): even a well-chosen static cap + # can't see the host's actual memory state (other tenants, bloated + # long-lived workers, dashboard growth). Under observed pressure the + # dispatcher stops adding load: critical -> spawn nothing this tick; + # elevated -> at most one new worker. Reclaim/promotion above already + # ran, so board bookkeeping stays live either way, and deferred tasks + # simply wait for a later tick. "unknown" imposes no restriction. + pressure = _memory_pressure_level() + if pressure == "critical": + result.memory_pressure = pressure + _log.warning( + "kanban dispatch: system memory pressure is critical; " + "spawning no new workers this tick (deferred, not dropped)" + ) + return result + if pressure == "elevated": + result.memory_pressure = pressure + if spawn_budget is None or spawn_budget > 1: + _log.warning( + "kanban dispatch: system memory pressure is elevated; " + "limiting to at most 1 new worker this tick" + ) + spawn_budget = 1 + ready_rows = conn.execute( "SELECT id, assignee FROM tasks " "WHERE status = 'ready' AND claim_lock IS NULL " diff --git a/tests/conftest.py b/tests/conftest.py index 431d254202..a1d0a034c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -562,6 +562,28 @@ def _isolate_hermes_home(_hermetic_environment): return None +@pytest.fixture(autouse=True) +def _neutralize_kanban_memory_guard(request, monkeypatch): + """Pin the kanban dispatcher's memory guard to "no data" for every test. + + The dispatcher consults live system memory before spawning (OOF-30/ + OOF-77: memory-derived default cap + pressure-based spawn restriction). + Left un-patched, dispatch tests would pass or fail based on how loaded + the CI runner happens to be. Defaulting the sample to ``{}`` makes the + derived cap ``None`` and the pressure level ``"unknown"`` — i.e. the + pre-guard behaviour every existing test was written against. Tests that + exercise the guard itself opt out with + ``@pytest.mark.real_memory_guard`` or patch the seam directly. + """ + if request.node.get_closest_marker("real_memory_guard"): + return + try: + from hermes_cli import kanban_db as _kb_mod + except Exception: + return + monkeypatch.setattr(_kb_mod, "_system_memory_sample", lambda: {}, raising=False) + + @pytest.fixture(autouse=True) def _neutralize_webbrowser(monkeypatch): """Record browser-open attempts instead of opening real browser windows.""" @@ -1142,6 +1164,12 @@ def pytest_configure(config): # noqa: D401 — pytest hook "created in the current environment (needs admin/developer mode " "on Windows).", ) + config.addinivalue_line( + "markers", + "real_memory_guard: bypass the autouse fixture that pins the kanban " + "dispatcher's memory guard to 'no data' — only for tests that " + "exercise the guard itself with their own patched samples.", + ) # NOTE: linux_only / macos_only / windows_only are declared in # pyproject.toml's ``markers`` list, not here — they are part of the # project's public marker vocabulary (``pytest --markers``, and the CI diff --git a/tests/hermes_cli/test_kanban_memory_guard.py b/tests/hermes_cli/test_kanban_memory_guard.py new file mode 100644 index 0000000000..ae145e96b6 --- /dev/null +++ b/tests/hermes_cli/test_kanban_memory_guard.py @@ -0,0 +1,253 @@ +"""Memory-aware kanban dispatch guard (OOF-30 / OOF-77). + +Two production incidents shared the same failure shape: no +``kanban.max_in_progress`` configured, a busy board, and a small hosted VM — +the dispatcher fanned out 26-31 concurrent workers, the host swap-thrashed, +and the whole machine (dashboard included) became unreachable. + +Covers the two safeguards added in response: + +1. :func:`hermes_cli.kanban_db.derive_default_max_in_progress` / + :func:`hermes_cli.kanban_db.resolve_max_in_progress` — memory-derived + default global concurrency cap when the operator never set one. +2. The live memory-pressure guard inside ``dispatch_once`` — critical + pressure spawns nothing; elevated pressure spawns at most one; unknown + imposes no restriction (fail-open). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from hermes_cli import kanban_db as kb + + +@pytest.fixture +def kanban_home(tmp_path, monkeypatch): + """Isolated HERMES_HOME with an empty kanban DB.""" + home = tmp_path / ".hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + kb.init_db() + return home + + +GIB = 1024 * 1024 # KiB per GiB + + +# --------------------------------------------------------------------------- +# derive_default_max_in_progress / resolve_max_in_progress +# --------------------------------------------------------------------------- + + +def test_derived_cap_small_vm_floors_at_two(): + # 1 GiB VM (the incident shape): 1024 // 512 = 2 workers. + assert kb.derive_default_max_in_progress({"mem_total_kib": 1 * GIB}) == 2 + # Even tiny VMs keep a floor of 2 so boards still make progress. + assert kb.derive_default_max_in_progress({"mem_total_kib": GIB // 4}) == 2 + + +def test_derived_cap_scales_with_memory_and_ceilings(): + assert kb.derive_default_max_in_progress({"mem_total_kib": 2 * GIB}) == 4 + assert kb.derive_default_max_in_progress({"mem_total_kib": 4 * GIB}) == 8 + # Big iron clamps at the ceiling — explicit config is the escape hatch. + assert kb.derive_default_max_in_progress({"mem_total_kib": 64 * GIB}) == 8 + + +def test_derived_cap_fails_open_without_memtotal(): + assert kb.derive_default_max_in_progress({}) is None + assert kb.derive_default_max_in_progress({"mem_total_kib": 0}) is None + assert kb.derive_default_max_in_progress({"mem_total_kib": -5}) is None + assert kb.derive_default_max_in_progress({"mem_total_kib": True}) is None + assert kb.derive_default_max_in_progress({"mem_total_kib": "1048576"}) is None + + +def test_resolve_max_in_progress_explicit_config_wins(monkeypatch): + monkeypatch.setattr( + kb, "_system_memory_sample", lambda: {"mem_total_kib": 1 * GIB} + ) + # Operator said 6 on a 1 GiB box — their call, even above the derived 2. + assert kb.resolve_max_in_progress(6) == 6 + assert kb.resolve_max_in_progress(1) == 1 + + +def test_resolve_max_in_progress_derives_when_unset(monkeypatch): + monkeypatch.setattr( + kb, "_system_memory_sample", lambda: {"mem_total_kib": 1 * GIB} + ) + assert kb.resolve_max_in_progress(None) == 2 + + +def test_resolve_max_in_progress_unset_and_unknown_memory_is_uncapped(monkeypatch): + monkeypatch.setattr(kb, "_system_memory_sample", lambda: {}) + assert kb.resolve_max_in_progress(None) is None + + +# --------------------------------------------------------------------------- +# _memory_pressure_level +# --------------------------------------------------------------------------- + + +def test_pressure_level_unknown_on_empty_sample(monkeypatch): + monkeypatch.setattr(kb, "_system_memory_sample", lambda: {}) + assert kb._memory_pressure_level() == "unknown" + + +def test_pressure_level_classifies_via_gateway_thresholds(): + ok = {"mem_available_kib": GIB // 2, "mem_total_kib": 1 * GIB} + critical = {"mem_available_kib": 32 * 1024, "mem_total_kib": 1 * GIB} + elevated = {"mem_available_kib": 100 * 1024, "mem_total_kib": 1 * GIB} + assert kb._memory_pressure_level(ok) == "ok" + assert kb._memory_pressure_level(critical) == "critical" + assert kb._memory_pressure_level(elevated) == "elevated" + + +# --------------------------------------------------------------------------- +# dispatch_once under pressure +# --------------------------------------------------------------------------- + + +def _pressure_sample(level: str) -> dict: + total = 1 * GIB + if level == "critical": + return {"mem_available_kib": 32 * 1024, "mem_total_kib": total} + if level == "elevated": + return {"mem_available_kib": 100 * 1024, "mem_total_kib": total} + return {"mem_available_kib": total // 2, "mem_total_kib": total} + + +def test_dispatch_spawns_nothing_under_critical_pressure( + kanban_home, all_assignees_spawnable, monkeypatch, +): + monkeypatch.setattr( + kb, "_system_memory_sample", lambda: _pressure_sample("critical") + ) + spawns = [] + + def fake_spawn(task, workspace, board=None): + spawns.append(task.id) + return 42 + + with kb.connect() as conn: + for title in ("a", "b", "c"): + kb.create_task(conn, title=title, assignee="alice") + res = kb.dispatch_once(conn, spawn_fn=fake_spawn) + + assert not spawns + assert not res.spawned + assert res.memory_pressure == "critical" + + +def test_dispatch_critical_pressure_defers_not_drops( + kanban_home, all_assignees_spawnable, monkeypatch, +): + """Tasks skipped under pressure stay 'ready' and spawn once memory clears.""" + sample = {"value": _pressure_sample("critical")} + monkeypatch.setattr(kb, "_system_memory_sample", lambda: sample["value"]) + spawns = [] + + def fake_spawn(task, workspace, board=None): + spawns.append(task.id) + return 42 + + with kb.connect() as conn: + task = kb.create_task(conn, title="a", assignee="alice") + kb.dispatch_once(conn, spawn_fn=fake_spawn) + assert not spawns + row = kb.get_task(conn, task) + assert row is not None and row.status == "ready" + + sample["value"] = _pressure_sample("ok") + res = kb.dispatch_once(conn, spawn_fn=fake_spawn) + + assert spawns == [task] + assert res.memory_pressure is None + + +def test_dispatch_elevated_pressure_spawns_at_most_one( + kanban_home, all_assignees_spawnable, monkeypatch, +): + monkeypatch.setattr( + kb, "_system_memory_sample", lambda: _pressure_sample("elevated") + ) + spawns = [] + + def fake_spawn(task, workspace, board=None): + spawns.append(task.id) + return 42 + + with kb.connect() as conn: + for title in ("a", "b", "c"): + kb.create_task(conn, title=title, assignee="alice") + res = kb.dispatch_once(conn, spawn_fn=fake_spawn) + + assert len(spawns) == 1 + assert res.memory_pressure == "elevated" + + +def test_dispatch_elevated_pressure_does_not_widen_tighter_budget( + kanban_home, all_assignees_spawnable, monkeypatch, +): + """A caller cap already at 0 remaining must not be widened to 1.""" + monkeypatch.setattr( + kb, "_system_memory_sample", lambda: _pressure_sample("elevated") + ) + spawns = [] + + def fake_spawn(task, workspace, board=None): + spawns.append(task.id) + return 42 + + with kb.connect() as conn: + running = kb.create_task(conn, title="running", assignee="alice") + kb.claim_task(conn, running) + kb.create_task(conn, title="ready", assignee="bob") + res = kb.dispatch_once(conn, spawn_fn=fake_spawn, max_in_progress=1) + + assert not spawns + assert not res.spawned + + +def test_dispatch_unknown_pressure_imposes_no_restriction( + kanban_home, all_assignees_spawnable, monkeypatch, +): + monkeypatch.setattr(kb, "_system_memory_sample", lambda: {}) + spawns = [] + + def fake_spawn(task, workspace, board=None): + spawns.append(task.id) + return 42 + + with kb.connect() as conn: + for title in ("a", "b", "c"): + kb.create_task(conn, title=title, assignee="alice") + res = kb.dispatch_once(conn, spawn_fn=fake_spawn) + + assert len(spawns) == 3 + assert res.memory_pressure is None + + +def test_dispatch_critical_pressure_still_runs_reclaim_bookkeeping( + kanban_home, all_assignees_spawnable, monkeypatch, +): + """The guard must only stop NEW spawns — reclaim/promotion still run.""" + monkeypatch.setattr( + kb, "_system_memory_sample", lambda: _pressure_sample("critical") + ) + with kb.connect() as conn: + parent = kb.create_task(conn, title="parent", assignee="alice") + child = kb.create_task( + conn, title="child", assignee="alice", parents=[parent], + ) + conn.execute("UPDATE tasks SET status = 'done' WHERE id = ?", (parent,)) + res = kb.dispatch_once(conn, spawn_fn=lambda *a, **k: 42) + row = kb.get_task(conn, child) + + # Promotion (todo -> ready once parents are done) happened despite the + # spawn freeze. + assert res.memory_pressure == "critical" + assert row is not None + assert row.status == "ready"