Merge pull request #84852 from NousResearch/bb/atomic-replace-windows-sharing

fix(utils): recover Windows renames contended by an open handle
This commit is contained in:
brooklyn!
2026-08-12 17:31:30 -05:00
committed by GitHub
2 changed files with 481 additions and 17 deletions
+324
View File
@@ -17,6 +17,7 @@ import json
import os
import sys
from pathlib import Path
from unittest.mock import MagicMock
import pytest
import yaml
@@ -273,3 +274,326 @@ def test_atomic_replace_real_cross_device(tmp_path: Path) -> None:
assert not tmp.exists()
finally:
_shutil.rmtree(other_fs_dir, ignore_errors=True)
# ─── Windows contended renames (GitHub #57775) ─────────────────────────────
#
# CPython opens files without FILE_SHARE_DELETE on Windows, so os.replace
# onto a target that ANY other handle holds open is denied. atomic_replace
# only fell back for EXDEV/EBUSY, so the exception propagated (and most
# callers swallowed it): gateway_state.json status updates were dropped at
# turn boundaries, and auth.json writes surfaced as "agent init failed".
#
# Measured on Windows 11 build 26200 / CPython 3.11: a held *target* handle
# reports winerror 5 (ERROR_ACCESS_DENIED), NOT 32 — 32 is what a held
# *source* reports. The cross-platform tests below therefore simulate
# winerror 5, matching what production actually raises.
#
# The real-handle tests use @pytest.mark.windows_only rather than a bare
# `skip(os.name != "nt")`: scripts/ci/list_os_marked_tests.py greps for the
# MARKER NAME to decide which files the Windows lane imports, so a plain
# skipif would leave them running on no host at all.
def _sharing_error(winerror: int = 5) -> PermissionError:
"""Build the PermissionError shape Windows raises for a held target."""
exc = PermissionError(errno.EACCES, "contended", "src", None, "dst")
exc.winerror = winerror
return exc
@pytest.fixture()
def fast_replace_retries(monkeypatch: pytest.MonkeyPatch) -> None:
"""Collapse the jittered backoff so retry tests don't sleep for ~1.2s."""
monkeypatch.setattr("utils._REPLACE_RETRY_BASE_DELAY_S", 0.001)
monkeypatch.setattr("utils._REPLACE_RETRY_MAX_DELAY_S", 0.001)
# ── cross-platform: the retry/fallback state machine ──────────────────────
@pytest.mark.parametrize("winerror", [5, 32, 33])
def test_contended_rename_retries_then_rewrites_in_place(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
fast_replace_retries: None,
winerror: int,
) -> None:
"""A target held for the whole call: the rename is retried the full
budget, then the in-place rewrite lands the write anyway.
winerror 5 is the code the reported bug actually produces; 32 and 33 are
the sibling contention codes. All three must recover.
"""
import utils as utils_mod
target = tmp_path / "gateway_state.json"
target.write_text("old", encoding="utf-8")
tmp = _write_tmp(tmp_path, "new")
attempts = []
def always_contended(src: str, dst: str) -> None:
attempts.append(src)
raise _sharing_error(winerror)
monkeypatch.setattr("utils.os.replace", always_contended)
monkeypatch.setattr("utils._IS_WINDOWS", True)
assert Path(atomic_replace(tmp, target)) == target
assert len(attempts) == 1 + utils_mod._REPLACE_RETRY_ATTEMPTS
assert target.read_text(encoding="utf-8") == "new"
assert not tmp.exists()
def test_contended_rename_retry_wins_keeps_write_atomic(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fast_replace_retries: None
) -> None:
"""A reader that lets go inside the budget: the atomic rename wins and
neither fallback runs, so the write keeps full atomicity."""
target = tmp_path / "gateway_state.json"
target.write_text("old", encoding="utf-8")
tmp = _write_tmp(tmp_path, "new")
real_replace = os.replace
calls = {"n": 0}
def contended_twice(src: str, dst: str) -> None:
calls["n"] += 1
if calls["n"] <= 2:
raise _sharing_error(5)
real_replace(src, dst)
def forbid(*_args: object, **_kw: object) -> None:
raise AssertionError("no fallback may run when a retry succeeds")
monkeypatch.setattr("utils.os.replace", contended_twice)
monkeypatch.setattr("utils._IS_WINDOWS", True)
monkeypatch.setattr("utils._rewrite_in_place", forbid)
monkeypatch.setattr("utils.shutil.copyfile", forbid)
assert Path(atomic_replace(tmp, target)) == target
assert calls["n"] == 3
assert target.read_text(encoding="utf-8") == "new"
assert not tmp.exists()
def test_genuine_denial_propagates_after_budget(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fast_replace_retries: None
) -> None:
"""A real ACL denial reports the same winerror as contention, so it is
not classified up front — it exhausts the budget, fails the in-place
rewrite too, and surfaces to the caller instead of being swallowed."""
target = tmp_path / "denied.json"
target.write_text("old", encoding="utf-8")
tmp = _write_tmp(tmp_path, "new")
monkeypatch.setattr(
"utils.os.replace", MagicMock(side_effect=_sharing_error(5))
)
monkeypatch.setattr("utils._IS_WINDOWS", True)
denial = PermissionError(errno.EACCES, "access is denied")
def cannot_open(*_args: object, **_kw: object) -> None:
raise denial
monkeypatch.setattr("utils.os.open", cannot_open)
with pytest.raises(PermissionError) as caught:
atomic_replace(tmp, target)
assert caught.value is denial
assert target.read_text(encoding="utf-8") == "old"
assert tmp.exists(), "the pending write must survive for the caller"
def test_contended_retry_switching_to_exdev_uses_copy_fallback(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fast_replace_retries: None
) -> None:
"""A retry that turns into EXDEV must stop consuming the sharing budget
and take the copy fallback — EXDEV never clears on retry."""
target = tmp_path / "target.json"
target.write_text("old", encoding="utf-8")
tmp = _write_tmp(tmp_path, "new")
replace = MagicMock(
side_effect=[_sharing_error(5), OSError(errno.EXDEV, "cross-device")]
)
monkeypatch.setattr("utils.os.replace", replace)
monkeypatch.setattr("utils._IS_WINDOWS", True)
def forbid_rewrite(*_a: object, **_k: object) -> None:
raise AssertionError("EXDEV must use the copy fallback, not a rewrite")
monkeypatch.setattr("utils._rewrite_in_place", forbid_rewrite)
assert Path(atomic_replace(tmp, target)) == target
assert replace.call_count == 2
assert target.read_text(encoding="utf-8") == "new"
assert not tmp.exists()
def test_non_contended_oserror_propagates_without_retry(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""ENOSPC is not a contention code on any platform: no retry, no
fallback, and the pending temp file is left for the caller."""
target = tmp_path / "config.yaml"
target.write_text("old", encoding="utf-8")
tmp = _write_tmp(tmp_path, "new")
replace = MagicMock(side_effect=OSError(errno.ENOSPC, "no space"))
monkeypatch.setattr("utils.os.replace", replace)
monkeypatch.setattr("utils._IS_WINDOWS", True)
with pytest.raises(OSError) as excinfo:
atomic_replace(tmp, target)
assert excinfo.value.errno == errno.ENOSPC
assert replace.call_count == 1, "a permanent error must not be retried"
assert target.read_text(encoding="utf-8") == "old"
assert tmp.exists()
def test_posix_eacces_propagates_without_retry(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""POSIX behaviour is unchanged: EACCES there means directory
permissions, so it propagates on the first attempt."""
target = tmp_path / "config.yaml"
target.write_text("old", encoding="utf-8")
tmp = _write_tmp(tmp_path, "new")
replace = MagicMock(
side_effect=PermissionError(errno.EACCES, os.strerror(errno.EACCES))
)
monkeypatch.setattr("utils.os.replace", replace)
monkeypatch.setattr("utils._IS_WINDOWS", False)
with pytest.raises(PermissionError):
atomic_replace(tmp, target)
assert replace.call_count == 1, "POSIX must not gain a retry loop"
assert target.read_text(encoding="utf-8") == "old"
assert tmp.exists()
def test_in_place_rewrite_never_exposes_a_truncated_file(
tmp_path: Path,
) -> None:
"""The in-place rewrite must not truncate-then-fill: a concurrent reader
can observe a 0-byte file during shutil.copyfile, which for auth.json
means an empty credential store. Shrinking writes must also not leave
trailing bytes from the previous, longer content.
"""
import utils as utils_mod
target = tmp_path / "auth.json"
observed: list[int] = []
target.write_text("A" * 5000, encoding="utf-8")
tmp = _write_tmp(tmp_path, "B" * 5000)
utils_mod._rewrite_in_place(str(tmp), str(target))
observed.append(len(target.read_text(encoding="utf-8")))
assert target.read_text(encoding="utf-8") == "B" * 5000
assert not tmp.exists()
# Shrinking rewrite: ftruncate must drop the tail.
tmp = _write_tmp(tmp_path, "C" * 10)
utils_mod._rewrite_in_place(str(tmp), str(target))
assert target.read_text(encoding="utf-8") == "C" * 10
assert observed == [5000]
def test_symlinked_target_survives_a_contended_rename(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fast_replace_retries: None
) -> None:
"""The #16743 invariant must hold on the contended path too: a symlinked
config.yaml stays a symlink when the rewrite fallback runs."""
real = tmp_path / "real.yaml"
link = tmp_path / "config.yaml"
real.write_text("old\n", encoding="utf-8")
link.symlink_to(real)
tmp = _write_tmp(tmp_path, "new\n")
monkeypatch.setattr(
"utils.os.replace", MagicMock(side_effect=_sharing_error(5))
)
monkeypatch.setattr("utils._IS_WINDOWS", True)
assert Path(atomic_replace(tmp, link)) == real
assert link.is_symlink(), "symlink must survive the rewrite fallback"
assert real.read_text(encoding="utf-8") == "new\n"
assert not tmp.exists()
# ── native Windows: real contended handles ────────────────────────────────
@pytest.mark.windows_only
def test_windows_real_held_read_handle_lands_the_write(tmp_path: Path) -> None:
"""The reported bug, end to end against a real held handle."""
target = tmp_path / "gateway_state.json"
target.write_text('{"active_agents": 1}', encoding="utf-8")
tmp = _write_tmp(tmp_path, '{"active_agents": 2}')
with open(target, "r", encoding="utf-8"):
assert Path(atomic_replace(tmp, target)) == target
assert json.loads(target.read_text(encoding="utf-8")) == {"active_agents": 2}
assert not tmp.exists()
@pytest.mark.windows_only
def test_windows_real_held_handle_reports_access_denied(tmp_path: Path) -> None:
"""Pin the premise this fix is built on: a held *target* handle raises
winerror 5, not 32. If CPython ever changes that, the classification in
utils must be revisited rather than silently missing the bug again.
"""
target = tmp_path / "state.json"
target.write_text("old", encoding="utf-8")
tmp = _write_tmp(tmp_path, "new")
with open(target, "r", encoding="utf-8"):
with pytest.raises(OSError) as caught:
os.replace(str(tmp), str(target))
assert caught.value.winerror in (5, 32, 33)
import utils as utils_mod
assert utils_mod._is_contended_windows_replace_error(caught.value)
@pytest.mark.windows_only
def test_windows_atomic_json_write_with_concurrent_reader(
tmp_path: Path,
) -> None:
"""End-to-end gateway_state.json scenario through atomic_json_write:
the write lands and no .tmp file is orphaned."""
target = tmp_path / "gateway_state.json"
atomic_json_write(target, {"active_agents": 1})
with open(target, "r", encoding="utf-8"):
atomic_json_write(target, {"active_agents": 2})
assert json.loads(target.read_text(encoding="utf-8")) == {"active_agents": 2}
leftovers = list(tmp_path.glob("*.tmp")) + list(tmp_path.glob(".*.tmp"))
assert leftovers == [], f"orphaned temp files: {leftovers}"
@pytest.mark.windows_only
def test_windows_readonly_target_still_raises(tmp_path: Path) -> None:
"""A genuinely unwritable target must not be rescued by the fallback."""
import subprocess
target = tmp_path / "readonly.json"
target.write_text("old", encoding="utf-8")
tmp = _write_tmp(tmp_path, "new")
subprocess.run(["attrib", "+R", str(target)], capture_output=True)
try:
with pytest.raises(OSError):
atomic_replace(tmp, target)
assert target.read_text(encoding="utf-8") == "old"
finally:
subprocess.run(["attrib", "-R", str(target)], capture_output=True)
+157 -17
View File
@@ -7,6 +7,7 @@ import os
import shutil
import stat
import tempfile
import time
from pathlib import Path
from typing import Any, Union
from urllib.parse import urlparse
@@ -88,6 +89,108 @@ def _restore_file_mode(path: Path, mode: "int | None") -> None:
pass
_IS_WINDOWS = os.name == "nt"
# Windows rename failures that can be caused by another handle on the target
# rather than by a permission problem. ``os.replace`` onto a file that any
# other handle has open is denied because CPython opens files without
# ``FILE_SHARE_DELETE``:
#
# 5 ERROR_ACCESS_DENIED — what a held *target* handle actually reports
# 32 ERROR_SHARING_VIOLATION — reported when the *source* temp file is held
# 33 ERROR_LOCK_VIOLATION — byte-range lock on the target
#
# Measured on Windows 11 (build 26200, CPython 3.11): a plain reader on the
# target — in-process or cross-process — yields winerror 5, NOT 32. Keying
# recovery on 32 alone therefore misses every real occurrence of this bug.
# These codes are ambiguous (a genuine ACL denial is also 5), which is why
# recovery is bounded and any still-failing write is re-raised unchanged
# rather than being classified up front.
_WINDOWS_CONTENDED_REPLACE_ERRORS = frozenset({5, 32, 33})
# Retry budget for the atomic rename. A rename that wins here keeps the write
# fully atomic, so the budget is sized to cover a realistic contended hold: an
# observed desktop auth-init holds auth.json past 100 ms, while an ordinary
# status read is ~0.05 ms. Measured on Windows 11 build 26200, this recovers
# holds up to ~200 ms atomically (~310 ms worst case).
#
# The cap matters as much as the attempt count. gateway_state.json is
# rewritten at every turn boundary, so a permanently-held target pays the full
# budget on every write: a longer 6 x 20..400 ms budget cost ~1.3 s per write
# under a persistent reader, versus ~0.3 s here for the same atomic coverage.
# Jittered so concurrent writers don't retry in lockstep.
_REPLACE_RETRY_ATTEMPTS = 4
_REPLACE_RETRY_BASE_DELAY_S = 0.02
_REPLACE_RETRY_MAX_DELAY_S = 0.1
def _is_contended_windows_replace_error(exc: OSError) -> bool:
"""Return True for Windows rename failures a retry might clear.
Only a *candidate* classification: ``ERROR_ACCESS_DENIED`` covers both a
concurrent handle and a real ACL denial, and the two are not reliably
distinguishable up front. Probing the target with ``os.access`` does not
work — ``os.replace`` needs delete-child rights on the *parent directory*,
so a directory-level denial reports the target as writable. Instead of
guessing, both cases enter the same bounded retry and a genuine denial
falls through to the caller with its original error.
"""
return _IS_WINDOWS and getattr(exc, "winerror", None) in (
_WINDOWS_CONTENDED_REPLACE_ERRORS
)
def _rewrite_in_place(tmp_str: str, real_path: str) -> None:
"""Overwrite *real_path* with the contents of *tmp_str*, in place.
Last-resort path for a target whose handle is still held after the retry
budget: writing through the existing file works where renaming onto it
does not. Unlike ``shutil.copyfile`` this never truncates the target to
zero first — a concurrent reader would otherwise be able to observe an
empty ``auth.json`` / ``gateway_state.json`` mid-write (measured: a
4-thread poller sees a 0-byte read during a plain copyfile). A single
``os.write`` of the full payload followed by ``ftruncate`` keeps the
visible content going straight from old to new.
This is still not atomic — it is a strictly smaller window than a copy,
not the absence of one — so it runs only after the rename has genuinely
failed. Writing through the target also preserves its ACL, which
``os.replace`` does not (the temp file's inherited ACL wins there).
"""
with open(tmp_str, "rb") as src:
data = src.read()
flags = os.O_WRONLY | getattr(os, "O_BINARY", 0)
fd = os.open(real_path, flags)
try:
os.lseek(fd, 0, os.SEEK_SET)
written = 0
while written < len(data):
written += os.write(fd, data[written:])
os.ftruncate(fd, len(data))
try:
os.fsync(fd)
except OSError:
pass
finally:
os.close(fd)
os.unlink(tmp_str)
def _copy_fallback(tmp_str: str, real_path: str) -> None:
"""Copy/fsync/unlink fallback for cross-device and bind-mount renames."""
shutil.copyfile(tmp_str, real_path)
try:
shutil.copystat(tmp_str, real_path)
except OSError:
pass
try:
with open(real_path, "rb") as f:
os.fsync(f.fileno())
except OSError:
pass
os.unlink(tmp_str)
def atomic_replace(tmp_path: Union[str, Path], target: Union[str, Path]) -> str:
"""Atomically move *tmp_path* onto *target*, preserving symlinks.
@@ -101,9 +204,21 @@ def atomic_replace(tmp_path: Union[str, Path], target: Union[str, Path]) -> str:
This helper resolves the symlink first so ``os.replace`` writes to
the real file in-place while the symlink survives. For non-symlink
and non-existent paths the behavior is identical to a plain
``os.replace`` call unless the rename fails with ``EXDEV`` or ``EBUSY``;
those cases fall back to copy/fsync/unlink for cross-device, bind-mount,
and busy-file deployments.
``os.replace`` call unless the rename fails with:
* ``EXDEV`` / ``EBUSY`` (any platform) — cross-device, bind-mount, and
busy-file deployments fall back to copy/fsync/unlink immediately.
These never clear on retry.
* A Windows rename contended by another open handle (winerror 5/32/33).
CPython opens files without ``FILE_SHARE_DELETE``, so *any* concurrent
reader of the target blocks the rename. The rename is retried with
jittered backoff first — a retry that wins keeps the write atomic —
and only a target whose handle outlives the budget is rewritten in
place, so the update lands instead of being silently dropped.
A genuine Windows permission failure produces the same winerror as a
contended one, so it is not classified up front: it exhausts the retry
budget, fails the in-place rewrite too, and is re-raised unchanged.
Returns the resolved real path used for the replace, so callers that
need to re-apply permissions can target it instead of the symlink.
@@ -113,26 +228,51 @@ def atomic_replace(tmp_path: Union[str, Path], target: Union[str, Path]) -> str:
tmp_str = str(tmp_path)
try:
os.replace(tmp_str, real_path)
return real_path
except OSError as exc:
if exc.errno not in (errno.EXDEV, errno.EBUSY):
contended = _is_contended_windows_replace_error(exc)
if exc.errno not in (errno.EXDEV, errno.EBUSY) and not contended:
raise
if contended:
# Lazy import: keeps ``utils`` free of a package-level dependency
# on ``agent`` for every consumer that never hits this path.
from agent.retry_utils import jittered_backoff
for attempt in range(1, _REPLACE_RETRY_ATTEMPTS + 1):
time.sleep(
jittered_backoff(
attempt,
base_delay=_REPLACE_RETRY_BASE_DELAY_S,
max_delay=_REPLACE_RETRY_MAX_DELAY_S,
)
)
try:
os.replace(tmp_str, real_path)
return real_path
except OSError as retry_exc:
if retry_exc.errno in (errno.EXDEV, errno.EBUSY):
# Not contention after all — stop burning the budget.
exc = retry_exc
contended = False
break
if not _is_contended_windows_replace_error(retry_exc):
raise
exc = retry_exc
logger.debug(
"atomic_replace: %s -> %s failed with %s; falling back to copy",
"atomic_replace: %s -> %s failed with %s; falling back to %s",
tmp_str,
real_path,
errno.errorcode.get(exc.errno, exc.errno),
getattr(exc, "winerror", None)
or errno.errorcode.get(exc.errno or 0, exc.errno),
"in-place rewrite" if contended else "copy",
)
shutil.copyfile(tmp_str, real_path)
try:
shutil.copystat(tmp_str, real_path)
except OSError:
pass
try:
with open(real_path, "rb") as f:
os.fsync(f.fileno())
except OSError:
pass
os.unlink(tmp_str)
if contended:
# Re-raises the rewrite's own error (not the rename's) when the
# target is genuinely unwritable — an ACL denial stays an ACL
# denial rather than being reported as contention.
_rewrite_in_place(tmp_str, real_path)
else:
_copy_fallback(tmp_str, real_path)
return real_path