fix(worktree): pruner reaps rebase-merged trees via PR state; worktree add survives disk contention

Two gaps behind the recurring 'hermes -w timed out after 30 seconds':

1. Rebase-merge leak: git cherry only catches patch-identical commits.
   Salvage flows routinely change the diff (conflict resolution, follow-up
   commits), so 12 of 22 'unpushed' trees on the incident box had MERGED
   PRs and were preserved forever. The pruner now falls back to
   'gh pr list --head <branch> --state merged' — authoritative, memoized
   on (branch, head_sha) with True-only caching, fail-safe to preserve.

2. Creation timeout 30s -> 120s: the ~10k-file checkout measured 113s at
   near-zero CPU under multi-agent disk contention vs 1.2s idle. 30s
   killed legitimate creates and threw away completed work.
This commit is contained in:
Teknium
2026-08-19 23:39:40 -07:00
parent f309f92d30
commit fc9b4186a0
2 changed files with 180 additions and 2 deletions
+78 -2
View File
@@ -1866,9 +1866,14 @@ def _setup_worktree(repo_root: str = None, sync_base: bool = True,
"-c", "checkout.thresholdForParallelism=100",
]
try:
# 120s, not 30: on a multi-agent box the ~10k-file materialization
# contends with sibling sessions' checkouts/fetches/Electron dev
# builds for the same disk — measured 113s wall at near-zero CPU
# under load vs 1.2s idle (Aug 2026). A too-tight timeout kills a
# legitimately slow create and wastes the work already done.
result = subprocess.run(
["git", *_wt_add_cfg, "worktree", "add", str(wt_path), "-b", branch_name, base_ref],
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, cwd=repo_root,
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=120, cwd=repo_root,
)
if result.returncode != 0:
# If branching from the resolved remote ref failed for any reason
@@ -1883,7 +1888,7 @@ def _setup_worktree(repo_root: str = None, sync_base: bool = True,
base_ref, base_label = "HEAD", "HEAD (fallback — remote base failed)"
result = subprocess.run(
["git", "worktree", "add", str(wt_path), "-b", branch_name, base_ref],
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30, cwd=repo_root,
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=120, cwd=repo_root,
)
if result.returncode != 0:
_cleanup_failed_worktree_add(repo_root, wt_path, branch_name)
@@ -2292,6 +2297,69 @@ def _worktree_commits_all_merged_upstream(
return False
def _worktree_branch_pr_merged(
worktree_path: str,
timeout: int = 15,
cache: Optional[Dict[str, bool]] = None,
) -> bool:
"""Return whether the worktree branch's PR is MERGED on GitHub.
Escape hatch for the case ``git cherry`` cannot catch: a rebase-merge that
altered the diff (conflict resolution against a moved base, follow-up
commits added during salvage/CI-fix) changes the patch-id, so the local
commits are no longer patch-equivalent to anything upstream even though
the PR merged. Those trees survive the cherry check forever (Aug 2026:
12 of 22 "unpushed" trees on a loaded box had MERGED PRs).
GitHub's PR state is the authoritative merge signal, so a clean tree
whose branch has a MERGED PR is reaped. Verdicts are memoized keyed on
``(branch, head_sha)`` — MERGED is monotonic, so a True verdict is cached
permanently; False is never cached (the PR may merge later without new
local commits, which would leave the key unchanged).
Fails SAFE toward False (preserve): no gh binary, offline, rate-limited,
detached HEAD, or any parse failure keeps the tree.
"""
import subprocess
try:
head = subprocess.run(
["git", "rev-parse", "--abbrev-ref", "HEAD"],
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout, cwd=worktree_path,
)
if head.returncode != 0:
return False
branch = head.stdout.strip()
if not branch or branch == "HEAD": # detached — no PR to look up
return False
cache_key = None
if cache is not None:
sha = subprocess.run(
["git", "rev-parse", "HEAD"],
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout, cwd=worktree_path,
)
if sha.returncode == 0 and sha.stdout.strip():
cache_key = f"pr-merged:{branch}:{sha.stdout.strip()}"
if cache.get(cache_key) is True:
return True
result = subprocess.run(
["gh", "pr", "list", "--head", branch, "--state", "merged",
"--json", "number", "--limit", "1"],
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout, cwd=worktree_path,
)
if result.returncode != 0:
return False
prs = json.loads(result.stdout or "[]")
merged = isinstance(prs, list) and len(prs) > 0
if merged and cache is not None and cache_key is not None:
cache[cache_key] = True
return merged
except Exception:
return False
def _worktree_lock_is_live(repo_root: str, worktree_path: str, timeout: int = 10):
"""Classify a worktree's git lock as live, dead, or absent.
@@ -2652,6 +2720,14 @@ def _prune_stale_worktrees(repo_root: str, max_age_hours: int = 24) -> None:
merged = _worktree_commits_all_merged_upstream(
str(entry), timeout=30, cache=snapshot
)
if not merged:
# Rebase-merge escape hatch: conflict resolution or follow-up
# commits change the patch-id, so cherry misses them — but
# GitHub knows the PR merged. Authoritative and cheap (~0.3s,
# memoized on (branch, head_sha) so it's paid once per tree).
merged = _worktree_branch_pr_merged(
str(entry), timeout=15, cache=snapshot
)
with cache_lock:
merge_cache.update(snapshot)
if not merged: