fix(engine): detect wedged buildx + surface recovery hint on deploy

When Docker's buildx leaks bind-mounts from a failed build it starts
reporting 'read-only file system' on its own activity file, even
though nothing is actually read-only. The user's host had 20+
leaked mounts before we noticed — each retry compounds the leak.

_compose_with_retry now:
 * Pre-flight counts /var/lib/docker/tmp/buildkit-mount* entries in
   /proc/self/mounts; if >= 10 and the command is a build, refuses
   to start and returns a clean recovery recipe instead of retrying.
 * On mid-build failures that match the wedge signature
   ('failed to update builder last activity time' or the activity-dir
   path in stderr), short-circuits the retry loop with the same
   recipe. The first occurrence no longer needs a pre-flight; the
   pre-flight catches repeat attempts.

Recipe points at 'docker buildx prune -af && sudo systemctl restart
docker', which is what actually clears the leaked mounts.

Tests cover all three paths: wedge preflight blocks builds, non-build
commands (down/stop) ignore the preflight, mid-build signature
detection kills the retry loop. A new autouse fixture stubs the
wedge-detector to 0 so dev-host state doesn't poison the mocked
subprocess tests.

Wiki companion commit adds Troubleshooting → 'Buildx leaked mounts'.
This commit is contained in:
2026-04-24 19:25:45 -04:00
parent a8356407c5
commit 86b9decf80
2 changed files with 121 additions and 0 deletions

View File

@@ -132,6 +132,52 @@ _PERMANENT_ERRORS = (
"repository does not exist",
)
# Signatures of a wedged buildx: the daemon has leaked bind mounts and
# can no longer update its activity file, which surfaces as "read-only
# file system" even though nothing is actually read-only. Retrying the
# compose build just accumulates more leaked mounts — we bail early
# with a clear recovery recipe.
_BUILDX_WEDGE_PATTERNS = (
"failed to update builder last activity time",
".docker/buildx/activity/",
)
# Count above which we consider buildx's bind-mount table pathological.
# A healthy daemon has 0; a couple is transient during a build. Past
# 10 you're seeing accumulation from a previous failed run.
_BUILDKIT_MOUNT_THRESHOLD = 10
def _count_leaked_buildkit_mounts() -> int:
"""How many orphaned buildkit bind-mounts is the daemon holding?
Best-effort: reads /proc/self/mounts and greps for the known
buildkit tmp pattern. Returns 0 if the file can't be read so we
never block a deploy over our own diagnostic.
"""
try:
with open("/proc/self/mounts", "r", encoding="utf-8") as fh:
return sum(1 for line in fh if "/var/lib/docker/tmp/buildkit-mount" in line)
except OSError:
return 0
def _buildx_recovery_hint(extra: str = "") -> str:
head = (
"Buildx is wedged — Docker's build driver has leaked bind "
"mounts from a previous failed run and can no longer write "
"its activity file. This surfaces as a spurious "
"'read-only file system' error."
)
fix = (
"Recovery:\n"
" 1. docker buildx prune -af\n"
" 2. sudo systemctl restart docker\n"
" 3. Retry the deploy.\n"
"See wiki: Troubleshooting → 'Buildx leaked mounts'."
)
return f"{head}\n\n{fix}{(' ' + extra) if extra else ''}"
@_traced("engine.compose_with_retry")
def _compose_with_retry(
@@ -150,6 +196,21 @@ def _compose_with_retry(
# "project name must not be empty".
cmd = ["docker", "compose", "-p", "decnet", "-f", str(compose_file), *args]
merged = {**os.environ, **(env or {})}
# Preflight: if buildx already looks wedged before the first attempt,
# refuse to start — retrying just leaks more mounts. Only applies to
# build-bearing invocations ("up --build", "build"); "down" etc. are
# unaffected by buildx state.
is_build_cmd = any(a in args for a in ("--build", "build"))
if is_build_cmd:
leaked = _count_leaked_buildkit_mounts()
if leaked >= _BUILDKIT_MOUNT_THRESHOLD:
hint = _buildx_recovery_hint(f"(Detected {leaked} leaked buildkit mounts.)")
log.error("preflight: buildx wedge detected (%d mounts) — refusing to deploy", leaked)
raise subprocess.CalledProcessError(
returncode=1, cmd=cmd, output="", stderr=hint,
)
for attempt in range(1, retries + 1):
result = subprocess.run(cmd, capture_output=True, text=True, env=merged) # nosec B603
if result.returncode == 0:
@@ -163,6 +224,15 @@ def _compose_with_retry(
if any(pat in stderr_lower for pat in _PERMANENT_ERRORS):
console.print(f"[red]Permanent Docker error — not retrying:[/]\n{result.stderr.strip()}")
raise last_exc
if any(pat in stderr_lower for pat in _BUILDX_WEDGE_PATTERNS):
leaked = _count_leaked_buildkit_mounts()
hint = _buildx_recovery_hint(f"(Detected {leaked} leaked buildkit mounts.)")
console.print(f"[red]{hint}[/]")
log.error("buildx wedge detected mid-build (%d mounts) — not retrying", leaked)
raise subprocess.CalledProcessError(
returncode=result.returncode, cmd=cmd,
output=result.stdout, stderr=hint,
)
if attempt < retries:
console.print(
f"[yellow]docker compose {' '.join(args)} failed "