From dcbe1754230b405c61bcf25d365933ad6bc1ed1c Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Wed, 12 Aug 2026 17:24:04 -0500 Subject: [PATCH 1/2] fix(utils): recover Windows renames contended by an open handle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit os.replace onto a file that any other handle holds open is denied on Windows — CPython opens files without FILE_SHARE_DELETE. atomic_replace only fell back for EXDEV/EBUSY, so the exception propagated and, because most callers swallow it, the write was silently dropped. gateway_state.json loses status updates at every turn boundary while status readers poll it; auth.json surfaces the same race to the user as 'agent init failed'. Classify winerror 5/32/33 as contention candidates and retry the rename with jittered backoff; a retry that wins keeps the write fully atomic. Only a handle that outlives the budget falls through to a rewrite. Measured on Windows 11 build 26200 / CPython 3.11: a held *target* handle reports winerror 5, not 32 — 32 is what a held *source* reports. Keying recovery on 32 alone misses every real occurrence of this bug. The codes are ambiguous (a genuine ACL denial is also 5) and cannot be told apart up front: os.replace needs delete-child rights on the parent directory, so probing the target with os.access reports a directory-level denial as writable. Rather than guess, both cases take the same bounded path and a genuine denial is re-raised unchanged with its pending temp file intact. The last-resort rewrite writes through the existing file instead of shutil.copyfile: a copy truncates the target to zero first, and a concurrent reader can observe an empty auth.json mid-write. Writing through the target also preserves its ACL, which os.replace does not. Co-authored-by: LewfKrad Co-authored-by: ruochu88s Co-authored-by: lost9999 Co-authored-by: guanla-zz Co-authored-by: zapabob --- utils.py | 174 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 157 insertions(+), 17 deletions(-) diff --git a/utils.py b/utils.py index 6d56fc4ca6..33d4932302 100644 --- a/utils.py +++ b/utils.py @@ -7,6 +7,7 @@ import os import shutil import stat import tempfile +import time from pathlib import Path from typing import Any, Union from urllib.parse import urlparse @@ -88,6 +89,108 @@ def _restore_file_mode(path: Path, mode: "int | None") -> None: pass +_IS_WINDOWS = os.name == "nt" + +# Windows rename failures that can be caused by another handle on the target +# rather than by a permission problem. ``os.replace`` onto a file that any +# other handle has open is denied because CPython opens files without +# ``FILE_SHARE_DELETE``: +# +# 5 ERROR_ACCESS_DENIED — what a held *target* handle actually reports +# 32 ERROR_SHARING_VIOLATION — reported when the *source* temp file is held +# 33 ERROR_LOCK_VIOLATION — byte-range lock on the target +# +# Measured on Windows 11 (build 26200, CPython 3.11): a plain reader on the +# target — in-process or cross-process — yields winerror 5, NOT 32. Keying +# recovery on 32 alone therefore misses every real occurrence of this bug. +# These codes are ambiguous (a genuine ACL denial is also 5), which is why +# recovery is bounded and any still-failing write is re-raised unchanged +# rather than being classified up front. +_WINDOWS_CONTENDED_REPLACE_ERRORS = frozenset({5, 32, 33}) + +# Retry budget for the atomic rename. A rename that wins here keeps the write +# fully atomic, so the budget is sized to cover a realistic contended hold: an +# observed desktop auth-init holds auth.json past 100 ms, while an ordinary +# status read is ~0.05 ms. Measured on Windows 11 build 26200, this recovers +# holds up to ~200 ms atomically (~310 ms worst case). +# +# The cap matters as much as the attempt count. gateway_state.json is +# rewritten at every turn boundary, so a permanently-held target pays the full +# budget on every write: a longer 6 x 20..400 ms budget cost ~1.3 s per write +# under a persistent reader, versus ~0.3 s here for the same atomic coverage. +# Jittered so concurrent writers don't retry in lockstep. +_REPLACE_RETRY_ATTEMPTS = 4 +_REPLACE_RETRY_BASE_DELAY_S = 0.02 +_REPLACE_RETRY_MAX_DELAY_S = 0.1 + + +def _is_contended_windows_replace_error(exc: OSError) -> bool: + """Return True for Windows rename failures a retry might clear. + + Only a *candidate* classification: ``ERROR_ACCESS_DENIED`` covers both a + concurrent handle and a real ACL denial, and the two are not reliably + distinguishable up front. Probing the target with ``os.access`` does not + work — ``os.replace`` needs delete-child rights on the *parent directory*, + so a directory-level denial reports the target as writable. Instead of + guessing, both cases enter the same bounded retry and a genuine denial + falls through to the caller with its original error. + """ + return _IS_WINDOWS and getattr(exc, "winerror", None) in ( + _WINDOWS_CONTENDED_REPLACE_ERRORS + ) + + +def _rewrite_in_place(tmp_str: str, real_path: str) -> None: + """Overwrite *real_path* with the contents of *tmp_str*, in place. + + Last-resort path for a target whose handle is still held after the retry + budget: writing through the existing file works where renaming onto it + does not. Unlike ``shutil.copyfile`` this never truncates the target to + zero first — a concurrent reader would otherwise be able to observe an + empty ``auth.json`` / ``gateway_state.json`` mid-write (measured: a + 4-thread poller sees a 0-byte read during a plain copyfile). A single + ``os.write`` of the full payload followed by ``ftruncate`` keeps the + visible content going straight from old to new. + + This is still not atomic — it is a strictly smaller window than a copy, + not the absence of one — so it runs only after the rename has genuinely + failed. Writing through the target also preserves its ACL, which + ``os.replace`` does not (the temp file's inherited ACL wins there). + """ + with open(tmp_str, "rb") as src: + data = src.read() + flags = os.O_WRONLY | getattr(os, "O_BINARY", 0) + fd = os.open(real_path, flags) + try: + os.lseek(fd, 0, os.SEEK_SET) + written = 0 + while written < len(data): + written += os.write(fd, data[written:]) + os.ftruncate(fd, len(data)) + try: + os.fsync(fd) + except OSError: + pass + finally: + os.close(fd) + os.unlink(tmp_str) + + +def _copy_fallback(tmp_str: str, real_path: str) -> None: + """Copy/fsync/unlink fallback for cross-device and bind-mount renames.""" + shutil.copyfile(tmp_str, real_path) + try: + shutil.copystat(tmp_str, real_path) + except OSError: + pass + try: + with open(real_path, "rb") as f: + os.fsync(f.fileno()) + except OSError: + pass + os.unlink(tmp_str) + + def atomic_replace(tmp_path: Union[str, Path], target: Union[str, Path]) -> str: """Atomically move *tmp_path* onto *target*, preserving symlinks. @@ -101,9 +204,21 @@ def atomic_replace(tmp_path: Union[str, Path], target: Union[str, Path]) -> str: This helper resolves the symlink first so ``os.replace`` writes to the real file in-place while the symlink survives. For non-symlink and non-existent paths the behavior is identical to a plain - ``os.replace`` call unless the rename fails with ``EXDEV`` or ``EBUSY``; - those cases fall back to copy/fsync/unlink for cross-device, bind-mount, - and busy-file deployments. + ``os.replace`` call unless the rename fails with: + + * ``EXDEV`` / ``EBUSY`` (any platform) — cross-device, bind-mount, and + busy-file deployments fall back to copy/fsync/unlink immediately. + These never clear on retry. + * A Windows rename contended by another open handle (winerror 5/32/33). + CPython opens files without ``FILE_SHARE_DELETE``, so *any* concurrent + reader of the target blocks the rename. The rename is retried with + jittered backoff first — a retry that wins keeps the write atomic — + and only a target whose handle outlives the budget is rewritten in + place, so the update lands instead of being silently dropped. + + A genuine Windows permission failure produces the same winerror as a + contended one, so it is not classified up front: it exhausts the retry + budget, fails the in-place rewrite too, and is re-raised unchanged. Returns the resolved real path used for the replace, so callers that need to re-apply permissions can target it instead of the symlink. @@ -113,26 +228,51 @@ def atomic_replace(tmp_path: Union[str, Path], target: Union[str, Path]) -> str: tmp_str = str(tmp_path) try: os.replace(tmp_str, real_path) + return real_path except OSError as exc: - if exc.errno not in (errno.EXDEV, errno.EBUSY): + contended = _is_contended_windows_replace_error(exc) + if exc.errno not in (errno.EXDEV, errno.EBUSY) and not contended: raise + if contended: + # Lazy import: keeps ``utils`` free of a package-level dependency + # on ``agent`` for every consumer that never hits this path. + from agent.retry_utils import jittered_backoff + + for attempt in range(1, _REPLACE_RETRY_ATTEMPTS + 1): + time.sleep( + jittered_backoff( + attempt, + base_delay=_REPLACE_RETRY_BASE_DELAY_S, + max_delay=_REPLACE_RETRY_MAX_DELAY_S, + ) + ) + try: + os.replace(tmp_str, real_path) + return real_path + except OSError as retry_exc: + if retry_exc.errno in (errno.EXDEV, errno.EBUSY): + # Not contention after all — stop burning the budget. + exc = retry_exc + contended = False + break + if not _is_contended_windows_replace_error(retry_exc): + raise + exc = retry_exc logger.debug( - "atomic_replace: %s -> %s failed with %s; falling back to copy", + "atomic_replace: %s -> %s failed with %s; falling back to %s", tmp_str, real_path, - errno.errorcode.get(exc.errno, exc.errno), + getattr(exc, "winerror", None) + or errno.errorcode.get(exc.errno or 0, exc.errno), + "in-place rewrite" if contended else "copy", ) - shutil.copyfile(tmp_str, real_path) - try: - shutil.copystat(tmp_str, real_path) - except OSError: - pass - try: - with open(real_path, "rb") as f: - os.fsync(f.fileno()) - except OSError: - pass - os.unlink(tmp_str) + if contended: + # Re-raises the rewrite's own error (not the rename's) when the + # target is genuinely unwritable — an ACL denial stays an ACL + # denial rather than being reported as contention. + _rewrite_in_place(tmp_str, real_path) + else: + _copy_fallback(tmp_str, real_path) return real_path From f514d546be05c8cbb12438561e825bc421bc66c9 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Wed, 12 Aug 2026 17:25:30 -0500 Subject: [PATCH 2/2] test(utils): cover contended Windows renames on the Windows CI lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The real-handle tests use @pytest.mark.windows_only rather than a bare pytest.skip(os.name != 'nt'). scripts/ci/list_os_marked_tests.py greps for the marker NAME to decide which files the Windows lane imports, so a plain skipif leaves a Windows test running on no host at all — green over zero coverage, which is what AGENTS.md warns about. Cross-platform tests simulate winerror 5 (what a held target actually raises) rather than 32, and pin the state machine: retry-then-rewrite for each contention code, an in-budget retry keeping the write atomic with no fallback, a genuine denial propagating after the budget with its temp file intact, a retry that turns into EXDEV switching to the copy fallback, ENOSPC and POSIX EACCES propagating with no retry at all, the rewrite never exposing a truncated file, and the #16743 symlink invariant holding on the contended path. On native Windows the lane exercises real held handles end to end, and pins the winerror-5 premise so a CPython change surfaces here instead of silently reintroducing the bug. Co-authored-by: LewfKrad Co-authored-by: ruochu88s Co-authored-by: lost9999 Co-authored-by: guanla-zz Co-authored-by: zapabob --- tests/test_atomic_replace_symlinks.py | 324 ++++++++++++++++++++++++++ 1 file changed, 324 insertions(+) diff --git a/tests/test_atomic_replace_symlinks.py b/tests/test_atomic_replace_symlinks.py index f0e77e0167..42c8556889 100644 --- a/tests/test_atomic_replace_symlinks.py +++ b/tests/test_atomic_replace_symlinks.py @@ -17,6 +17,7 @@ import json import os import sys from pathlib import Path +from unittest.mock import MagicMock import pytest import yaml @@ -273,3 +274,326 @@ def test_atomic_replace_real_cross_device(tmp_path: Path) -> None: assert not tmp.exists() finally: _shutil.rmtree(other_fs_dir, ignore_errors=True) + + +# ─── Windows contended renames (GitHub #57775) ───────────────────────────── +# +# CPython opens files without FILE_SHARE_DELETE on Windows, so os.replace +# onto a target that ANY other handle holds open is denied. atomic_replace +# only fell back for EXDEV/EBUSY, so the exception propagated (and most +# callers swallowed it): gateway_state.json status updates were dropped at +# turn boundaries, and auth.json writes surfaced as "agent init failed". +# +# Measured on Windows 11 build 26200 / CPython 3.11: a held *target* handle +# reports winerror 5 (ERROR_ACCESS_DENIED), NOT 32 — 32 is what a held +# *source* reports. The cross-platform tests below therefore simulate +# winerror 5, matching what production actually raises. +# +# The real-handle tests use @pytest.mark.windows_only rather than a bare +# `skip(os.name != "nt")`: scripts/ci/list_os_marked_tests.py greps for the +# MARKER NAME to decide which files the Windows lane imports, so a plain +# skipif would leave them running on no host at all. + + +def _sharing_error(winerror: int = 5) -> PermissionError: + """Build the PermissionError shape Windows raises for a held target.""" + exc = PermissionError(errno.EACCES, "contended", "src", None, "dst") + exc.winerror = winerror + return exc + + +@pytest.fixture() +def fast_replace_retries(monkeypatch: pytest.MonkeyPatch) -> None: + """Collapse the jittered backoff so retry tests don't sleep for ~1.2s.""" + monkeypatch.setattr("utils._REPLACE_RETRY_BASE_DELAY_S", 0.001) + monkeypatch.setattr("utils._REPLACE_RETRY_MAX_DELAY_S", 0.001) + + +# ── cross-platform: the retry/fallback state machine ────────────────────── + + +@pytest.mark.parametrize("winerror", [5, 32, 33]) +def test_contended_rename_retries_then_rewrites_in_place( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + fast_replace_retries: None, + winerror: int, +) -> None: + """A target held for the whole call: the rename is retried the full + budget, then the in-place rewrite lands the write anyway. + + winerror 5 is the code the reported bug actually produces; 32 and 33 are + the sibling contention codes. All three must recover. + """ + import utils as utils_mod + + target = tmp_path / "gateway_state.json" + target.write_text("old", encoding="utf-8") + tmp = _write_tmp(tmp_path, "new") + + attempts = [] + + def always_contended(src: str, dst: str) -> None: + attempts.append(src) + raise _sharing_error(winerror) + + monkeypatch.setattr("utils.os.replace", always_contended) + monkeypatch.setattr("utils._IS_WINDOWS", True) + + assert Path(atomic_replace(tmp, target)) == target + assert len(attempts) == 1 + utils_mod._REPLACE_RETRY_ATTEMPTS + assert target.read_text(encoding="utf-8") == "new" + assert not tmp.exists() + + +def test_contended_rename_retry_wins_keeps_write_atomic( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fast_replace_retries: None +) -> None: + """A reader that lets go inside the budget: the atomic rename wins and + neither fallback runs, so the write keeps full atomicity.""" + target = tmp_path / "gateway_state.json" + target.write_text("old", encoding="utf-8") + tmp = _write_tmp(tmp_path, "new") + + real_replace = os.replace + calls = {"n": 0} + + def contended_twice(src: str, dst: str) -> None: + calls["n"] += 1 + if calls["n"] <= 2: + raise _sharing_error(5) + real_replace(src, dst) + + def forbid(*_args: object, **_kw: object) -> None: + raise AssertionError("no fallback may run when a retry succeeds") + + monkeypatch.setattr("utils.os.replace", contended_twice) + monkeypatch.setattr("utils._IS_WINDOWS", True) + monkeypatch.setattr("utils._rewrite_in_place", forbid) + monkeypatch.setattr("utils.shutil.copyfile", forbid) + + assert Path(atomic_replace(tmp, target)) == target + assert calls["n"] == 3 + assert target.read_text(encoding="utf-8") == "new" + assert not tmp.exists() + + +def test_genuine_denial_propagates_after_budget( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fast_replace_retries: None +) -> None: + """A real ACL denial reports the same winerror as contention, so it is + not classified up front — it exhausts the budget, fails the in-place + rewrite too, and surfaces to the caller instead of being swallowed.""" + target = tmp_path / "denied.json" + target.write_text("old", encoding="utf-8") + tmp = _write_tmp(tmp_path, "new") + + monkeypatch.setattr( + "utils.os.replace", MagicMock(side_effect=_sharing_error(5)) + ) + monkeypatch.setattr("utils._IS_WINDOWS", True) + + denial = PermissionError(errno.EACCES, "access is denied") + + def cannot_open(*_args: object, **_kw: object) -> None: + raise denial + + monkeypatch.setattr("utils.os.open", cannot_open) + + with pytest.raises(PermissionError) as caught: + atomic_replace(tmp, target) + + assert caught.value is denial + assert target.read_text(encoding="utf-8") == "old" + assert tmp.exists(), "the pending write must survive for the caller" + + +def test_contended_retry_switching_to_exdev_uses_copy_fallback( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fast_replace_retries: None +) -> None: + """A retry that turns into EXDEV must stop consuming the sharing budget + and take the copy fallback — EXDEV never clears on retry.""" + target = tmp_path / "target.json" + target.write_text("old", encoding="utf-8") + tmp = _write_tmp(tmp_path, "new") + + replace = MagicMock( + side_effect=[_sharing_error(5), OSError(errno.EXDEV, "cross-device")] + ) + monkeypatch.setattr("utils.os.replace", replace) + monkeypatch.setattr("utils._IS_WINDOWS", True) + + def forbid_rewrite(*_a: object, **_k: object) -> None: + raise AssertionError("EXDEV must use the copy fallback, not a rewrite") + + monkeypatch.setattr("utils._rewrite_in_place", forbid_rewrite) + + assert Path(atomic_replace(tmp, target)) == target + assert replace.call_count == 2 + assert target.read_text(encoding="utf-8") == "new" + assert not tmp.exists() + + +def test_non_contended_oserror_propagates_without_retry( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ENOSPC is not a contention code on any platform: no retry, no + fallback, and the pending temp file is left for the caller.""" + target = tmp_path / "config.yaml" + target.write_text("old", encoding="utf-8") + tmp = _write_tmp(tmp_path, "new") + + replace = MagicMock(side_effect=OSError(errno.ENOSPC, "no space")) + monkeypatch.setattr("utils.os.replace", replace) + monkeypatch.setattr("utils._IS_WINDOWS", True) + + with pytest.raises(OSError) as excinfo: + atomic_replace(tmp, target) + + assert excinfo.value.errno == errno.ENOSPC + assert replace.call_count == 1, "a permanent error must not be retried" + assert target.read_text(encoding="utf-8") == "old" + assert tmp.exists() + + +def test_posix_eacces_propagates_without_retry( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """POSIX behaviour is unchanged: EACCES there means directory + permissions, so it propagates on the first attempt.""" + target = tmp_path / "config.yaml" + target.write_text("old", encoding="utf-8") + tmp = _write_tmp(tmp_path, "new") + + replace = MagicMock( + side_effect=PermissionError(errno.EACCES, os.strerror(errno.EACCES)) + ) + monkeypatch.setattr("utils.os.replace", replace) + monkeypatch.setattr("utils._IS_WINDOWS", False) + + with pytest.raises(PermissionError): + atomic_replace(tmp, target) + + assert replace.call_count == 1, "POSIX must not gain a retry loop" + assert target.read_text(encoding="utf-8") == "old" + assert tmp.exists() + + +def test_in_place_rewrite_never_exposes_a_truncated_file( + tmp_path: Path, +) -> None: + """The in-place rewrite must not truncate-then-fill: a concurrent reader + can observe a 0-byte file during shutil.copyfile, which for auth.json + means an empty credential store. Shrinking writes must also not leave + trailing bytes from the previous, longer content. + """ + import utils as utils_mod + + target = tmp_path / "auth.json" + observed: list[int] = [] + + target.write_text("A" * 5000, encoding="utf-8") + tmp = _write_tmp(tmp_path, "B" * 5000) + utils_mod._rewrite_in_place(str(tmp), str(target)) + observed.append(len(target.read_text(encoding="utf-8"))) + assert target.read_text(encoding="utf-8") == "B" * 5000 + assert not tmp.exists() + + # Shrinking rewrite: ftruncate must drop the tail. + tmp = _write_tmp(tmp_path, "C" * 10) + utils_mod._rewrite_in_place(str(tmp), str(target)) + assert target.read_text(encoding="utf-8") == "C" * 10 + assert observed == [5000] + + +def test_symlinked_target_survives_a_contended_rename( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fast_replace_retries: None +) -> None: + """The #16743 invariant must hold on the contended path too: a symlinked + config.yaml stays a symlink when the rewrite fallback runs.""" + real = tmp_path / "real.yaml" + link = tmp_path / "config.yaml" + real.write_text("old\n", encoding="utf-8") + link.symlink_to(real) + tmp = _write_tmp(tmp_path, "new\n") + + monkeypatch.setattr( + "utils.os.replace", MagicMock(side_effect=_sharing_error(5)) + ) + monkeypatch.setattr("utils._IS_WINDOWS", True) + + assert Path(atomic_replace(tmp, link)) == real + assert link.is_symlink(), "symlink must survive the rewrite fallback" + assert real.read_text(encoding="utf-8") == "new\n" + assert not tmp.exists() + + +# ── native Windows: real contended handles ──────────────────────────────── + + +@pytest.mark.windows_only +def test_windows_real_held_read_handle_lands_the_write(tmp_path: Path) -> None: + """The reported bug, end to end against a real held handle.""" + target = tmp_path / "gateway_state.json" + target.write_text('{"active_agents": 1}', encoding="utf-8") + tmp = _write_tmp(tmp_path, '{"active_agents": 2}') + + with open(target, "r", encoding="utf-8"): + assert Path(atomic_replace(tmp, target)) == target + + assert json.loads(target.read_text(encoding="utf-8")) == {"active_agents": 2} + assert not tmp.exists() + + +@pytest.mark.windows_only +def test_windows_real_held_handle_reports_access_denied(tmp_path: Path) -> None: + """Pin the premise this fix is built on: a held *target* handle raises + winerror 5, not 32. If CPython ever changes that, the classification in + utils must be revisited rather than silently missing the bug again. + """ + target = tmp_path / "state.json" + target.write_text("old", encoding="utf-8") + tmp = _write_tmp(tmp_path, "new") + + with open(target, "r", encoding="utf-8"): + with pytest.raises(OSError) as caught: + os.replace(str(tmp), str(target)) + + assert caught.value.winerror in (5, 32, 33) + import utils as utils_mod + + assert utils_mod._is_contended_windows_replace_error(caught.value) + + +@pytest.mark.windows_only +def test_windows_atomic_json_write_with_concurrent_reader( + tmp_path: Path, +) -> None: + """End-to-end gateway_state.json scenario through atomic_json_write: + the write lands and no .tmp file is orphaned.""" + target = tmp_path / "gateway_state.json" + atomic_json_write(target, {"active_agents": 1}) + + with open(target, "r", encoding="utf-8"): + atomic_json_write(target, {"active_agents": 2}) + + assert json.loads(target.read_text(encoding="utf-8")) == {"active_agents": 2} + leftovers = list(tmp_path.glob("*.tmp")) + list(tmp_path.glob(".*.tmp")) + assert leftovers == [], f"orphaned temp files: {leftovers}" + + +@pytest.mark.windows_only +def test_windows_readonly_target_still_raises(tmp_path: Path) -> None: + """A genuinely unwritable target must not be rescued by the fallback.""" + import subprocess + + target = tmp_path / "readonly.json" + target.write_text("old", encoding="utf-8") + tmp = _write_tmp(tmp_path, "new") + subprocess.run(["attrib", "+R", str(target)], capture_output=True) + try: + with pytest.raises(OSError): + atomic_replace(tmp, target) + assert target.read_text(encoding="utf-8") == "old" + finally: + subprocess.run(["attrib", "-R", str(target)], capture_output=True)