feat(api): /deckies/deploy and /mutate become 202 fire-and-forget

This is the unblock for the wizard hang. Both endpoints used to run
docker compose synchronously inside the HTTP handler -- on master
(unihost) or via asyncio.gather of worker /deploy POSTs at 600s
timeout each (swarm) -- blocking every other API request.

New flow:
  1. Commit the new config shape to repo state (fast).
  2. Create one DeckyLifecycle row per decky (status=pending).
  3. Spawn asyncio.create_task(run_deploy / run_mutate) -- the
     lifecycle runner drives rows through running -> succeeded|failed
     and emits decky.<name>.lifecycle on the bus.
  4. Return 202 with {lifecycle_ids: [...]}. Wizard polls
     GET /deckies/lifecycle?ids=... (next commit).

mutator/engine.py gains pick_new_services() -- shared between the
async API path and the watch-loop's synchronous mutate_decky().

DeployResponse grows lifecycle_ids[]. The old dispatch_decnet_config
helper still exists for the CLI swarm-deploy command path; it just
isn't called from the API handler anymore.

Test changes: 200 -> 202, drop dispatch_decnet_config mocks (handler
no longer calls it), assert lifecycle_ids in response + committed
state matches expectations.

decnet/mutator/engine.py

@@ -37,6 +37,37 @@ log = get_logger("mutator")
 console = Console()
 
 
+def pick_new_services(decky: DeckyConfig) -> list[str] | None:
+    """Pick a fresh service list for *decky* using its archetype pool
+    (or the global pool when no archetype is set).  Returns ``None`` if
+    no services are available to pick from.
+
+    Pure: does not touch the repo, file system, or docker.  Shared by
+    the mutator watch loop and the async API handler.
+    """
+    if decky.archetype:
+        try:
+            arch = get_archetype(decky.archetype)
+            svc_pool = list(arch.services)
+        except ValueError:
+            svc_pool = all_service_names()
+    else:
+        svc_pool = all_service_names()
+
+    if not svc_pool:
+        return None
+
+    current_services = set(decky.services)
+    attempts = 0
+    while True:
+        count = random.randint(1, min(3, len(svc_pool)))  # nosec B311
+        chosen = set(random.sample(svc_pool, count))  # nosec B311
+        attempts += 1
+        if chosen != current_services or attempts > 20:
+            break
+    return list(chosen)
+
+
 @_traced("mutator.mutate_decky")
 async def mutate_decky(
     decky_name: str,

decnet/web/db/models/deploy.py

@@ -20,8 +20,12 @@ class DeployIniRequest(BaseModel):
 
 
 class DeployResponse(BaseModel):
+    """202-Accepted response: deploy spawned in background, client polls
+    GET /deckies/lifecycle?ids=... until each row reaches a terminal
+    status."""
     message: str
     mode: str
+    lifecycle_ids: list[str] = PydanticField(default_factory=list)
 
 
 class PurgeResponse(BaseModel):

decnet/web/router/fleet/api_deploy_deckies.py

@@ -1,16 +1,17 @@
+import asyncio
 import os
 
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, status
 
+from decnet.bus.factory import get_bus
+from decnet.lifecycle.runner import run_deploy
 from decnet.logging import get_logger
 from decnet.telemetry import traced as _traced
 from decnet.config import DEFAULT_MUTATE_INTERVAL, DecnetConfig, _ROOT
-from decnet.engine import deploy as _deploy
 from decnet.ini_loader import load_ini_from_string
 from decnet.network import detect_interface, detect_subnet, get_host_ip
 from decnet.web.dependencies import require_admin, repo
 from decnet.web.db.models import DeployIniRequest, DeployResponse
-from decnet.web.router.swarm.api_deploy_swarm import dispatch_decnet_config
 
 log = get_logger("api")
 
@@ -20,19 +21,19 @@ router = APIRouter()
 @router.post(
     "/deckies/deploy",
     tags=["Fleet Management"],
+    status_code=status.HTTP_202_ACCEPTED,
     response_model=DeployResponse,
     responses={
+        202: {"description": "Deploy accepted; poll GET /deckies/lifecycle?ids=... for terminal status"},
         400: {"description": "Bad Request (e.g. malformed JSON)"},
         401: {"description": "Could not validate credentials"},
         403: {"description": "Insufficient permissions"},
         409: {"description": "Configuration conflict (e.g. invalid IP allocation or network mismatch)"},
         422: {"description": "Invalid INI config or schema validation error"},
-        500: {"description": "Deployment failed"},
-        502: {"description": "Partial swarm deploy failure — one or more worker hosts returned an error"},
     }
 )
 @_traced("api.deploy_deckies")
-async def api_deploy_deckies(req: DeployIniRequest, admin: dict = Depends(require_admin)) -> dict[str, str]:
+async def api_deploy_deckies(req: DeployIniRequest, admin: dict = Depends(require_admin)) -> dict:
     from decnet.fleet import build_deckies_from_ini
 
     try:
@@ -136,46 +137,46 @@ async def api_deploy_deckies(req: DeployIniRequest, admin: dict = Depends(requir
         for i, d in enumerate(unassigned):
             d.host_uuid = swarm_hosts[i % len(swarm_hosts)]["uuid"]
         config = config.model_copy(update={"mode": "swarm"})
+        mode = "swarm"
+    else:
+        mode = "unihost"
 
-        try:
+    # Commit the new shape before spawning so the wizard / dashboard
-            result = await dispatch_decnet_config(config, repo, dry_run=False, no_cache=False)
+    # observe the intended fleet immediately; lifecycle rows track the
-        except HTTPException:
+    # operation's progress separately.
-            raise
+    new_state_payload = {
-        except Exception as e:
+        "config": config.model_dump(),
-            log.exception("swarm-auto deploy dispatch failed: %s", e)
+        "compose_path": state_dict["compose_path"] if state_dict else str(
-            raise HTTPException(status_code=500, detail="Swarm dispatch failed. Check server logs.")
+            _ROOT / "docker-compose.yml",
+        ),
+    }
+    await repo.set_state("deployment", new_state_payload)
 
-        await repo.set_state("deployment", {
+    lifecycle_ids: dict[str, str] = {}
-            "config": config.model_dump(),
+    for d in config.deckies:
-            "compose_path": state_dict["compose_path"] if state_dict else "",
+        lid = await repo.create_lifecycle({
+            "decky_name": d.name,
+            "host_uuid": d.host_uuid,
+            "operation": "deploy",
         })
+        lifecycle_ids[d.name] = lid
 
-        failed = [r for r in result.results if not r.ok]
-        if failed:
-            detail = "; ".join(f"{r.host_name}: {r.detail}" for r in failed)
-            raise HTTPException(status_code=502, detail=f"Partial swarm deploy failure — {detail}")
-        return {
-            "message": f"Deckies deployed across {len(result.results)} swarm host(s)",
-            "mode": "swarm",
-        }
-
-    # Unihost path — docker-compose on the master itself.
-    # NB: the JSON state file (decnet-state.json) and fleet_deckies DB rows
-    # are both written *inside* _deploy(config) — engine.deployer is the
-    # single shared sink for every fleet-creation path (CLI deploy, this
-    # unihost API path, and per-worker SWARM agent deploys).  Do not
-    # duplicate save_state / fleet upserts here.
     try:
-        if os.environ.get("DECNET_CONTRACT_TEST") != "true":
+        bus = get_bus(client_name="api.deploy")
-            _deploy(config)
+    except Exception:
+        bus = None
 
-        new_state_payload = {
+    if os.environ.get("DECNET_CONTRACT_TEST") != "true":
-            "config": config.model_dump(),
+        asyncio.create_task(
-            "compose_path": str(_ROOT / "docker-compose.yml") if not state_dict else state_dict["compose_path"]
+            run_deploy(repo, bus, lifecycle_ids=lifecycle_ids, config=config),
-        }
+            name=f"deploy-{mode}-{len(config.deckies)}",
-        await repo.set_state("deployment", new_state_payload)
+        )
-    except Exception as e:
-        log.exception("Deployment failed: %s", e)
-        raise HTTPException(status_code=500, detail="Deployment failed. Check server logs for details.")
 
-    return {"message": "Deckies deployed successfully", "mode": "unihost"}
+    return {
+        "message": (
+            f"Deploy accepted ({len(config.deckies)} decky/ies, mode={mode}). "
+            f"Poll /deckies/lifecycle?ids=... for completion."
+        ),
+        "mode": mode,
+        "lifecycle_ids": list(lifecycle_ids.values()),
+    }

decnet/web/router/fleet/api_mutate_decky.py

@@ -1,34 +1,101 @@
-import os
+"""POST /deckies/{name}/mutate — operator-triggered single-decky mutate.
-from fastapi import APIRouter, Depends, HTTPException, Path
 
+Returns 202 Accepted with one ``lifecycle_id`` per mutated decky.  The
+real compose work runs in an ``asyncio.create_task``; the wizard polls
+``GET /deckies/lifecycle?ids=...`` until terminal.
+
+Auto-mutate (the watch-loop path) still goes through
+``decnet.mutator.mutate_decky`` and is synchronous within that loop —
+it's a background process, not an HTTP request, so it doesn't need
+fire-and-forget.
+"""
+import asyncio
+import os
+import time
+from pathlib import Path
+
+from fastapi import APIRouter, Depends, HTTPException, Path as PathParam, status
+
+from decnet.bus.factory import get_bus
+from decnet.config import DecnetConfig
+from decnet.lifecycle.runner import run_mutate
+from decnet.logging import get_logger
+from decnet.mutator.engine import pick_new_services
 from decnet.telemetry import traced as _traced
-from decnet.mutator import mutate_decky
+from decnet.web.db.models import LifecycleAcceptedResponse
-from decnet.web.db.models import MessageResponse
 from decnet.web.dependencies import require_admin, repo
 
+log = get_logger("api.mutate")
+
 router = APIRouter()
 
 
 @router.post(
     "/deckies/{decky_name}/mutate",
     tags=["Fleet Management"],
-    response_model=MessageResponse,
+    status_code=status.HTTP_202_ACCEPTED,
+    response_model=LifecycleAcceptedResponse,
     responses={
+        202: {"description": "Mutate accepted; poll GET /deckies/lifecycle?ids=..."},
         401: {"description": "Could not validate credentials"},
         403: {"description": "Insufficient permissions"},
-        404: {"description": "Decky not found"},
+        404: {"description": "No active deployment, or decky not found, or no services available"},
         422: {"description": "Path parameter validation error (decky_name must match ^[a-z0-9\\-]{1,64}$)"},
-    }
+    },
 )
 @_traced("api.mutate_decky")
 async def api_mutate_decky(
-    decky_name: str = Path(..., pattern=r"^[a-z0-9\-]{1,64}$"),
+    decky_name: str = PathParam(..., pattern=r"^[a-z0-9\-]{1,64}$"),
     admin: dict = Depends(require_admin),
-) -> dict[str, str]:
+) -> dict:
     if os.environ.get("DECNET_CONTRACT_TEST") == "true":
-        return {"message": f"Successfully mutated {decky_name} (Contract Test Mock)"}
+        return {"lifecycle_ids": ["contract-test"]}
 
-    success = await mutate_decky(decky_name, repo=repo)
+    state_dict = await repo.get_state("deployment")
-    if success:
+    if state_dict is None:
-        return {"message": f"Successfully mutated {decky_name}"}
+        raise HTTPException(status_code=404, detail="No active deployment")
-    raise HTTPException(status_code=404, detail=f"Decky {decky_name} not found or failed to mutate")
+    config = DecnetConfig(**state_dict["config"])
+    compose_path = Path(state_dict["compose_path"])
+    decky = next((d for d in config.deckies if d.name == decky_name), None)
+    if decky is None:
+        raise HTTPException(status_code=404, detail=f"Decky {decky_name} not found")
+
+    new_services = pick_new_services(decky)
+    if new_services is None:
+        raise HTTPException(
+            status_code=404,
+            detail=f"No services available to mutate {decky_name}",
+        )
+
+    # Commit the new shape to the DB before spawning, so observers
+    # don't see a half-applied mutation if the master crashes mid-task.
+    decky.services = list(new_services)
+    decky.last_mutated = time.time()
+    await repo.set_state(
+        "deployment",
+        {"config": config.model_dump(), "compose_path": str(compose_path)},
+    )
+
+    lifecycle_id = await repo.create_lifecycle({
+        "decky_name": decky.name,
+        "host_uuid": decky.host_uuid,
+        "operation": "mutate",
+    })
+
+    try:
+        bus = get_bus(client_name="api.mutate")
+    except Exception:
+        bus = None
+
+    asyncio.create_task(
+        run_mutate(
+            repo, bus,
+            lifecycle_id=lifecycle_id,
+            decky=decky,
+            services=list(new_services),
+            full_config=config,
+            compose_path=compose_path,
+        ),
+        name=f"mutate-{decky.name}",
+    )
+    return {"lifecycle_ids": [lifecycle_id]}

tests/api/config/test_deploy_limit.py

@@ -57,7 +57,7 @@ services = ssh
         json={"ini_content": ini},
         headers={"Authorization": f"Bearer {auth_token}"},
     )
-    assert resp.status_code == 200
+    assert resp.status_code == 202
     persisted = await repo.get_state("deployment")
     names = [d["name"] for d in persisted["config"]["deckies"]]
     assert names == ["only-decky"]
@@ -81,4 +81,4 @@ services = ssh
     if resp.status_code == 409:
         assert "limit" not in resp.json()["detail"].lower()
     else:
-        assert resp.status_code == 200
+        assert resp.status_code == 202

tests/api/fleet/test_deploy_automode.py

@@ -24,7 +24,7 @@ def mock_network():
 
 @pytest.mark.anyio
 async def test_deploy_automode_unihost_when_no_swarm_hosts(client, auth_token, monkeypatch):
-    """No swarm hosts enrolled → local unihost deploy."""
+    """No swarm hosts enrolled → local unihost deploy returns 202 with lifecycle ids."""
     monkeypatch.setenv("DECNET_MODE", "master")
     for row in await repo.list_swarm_hosts():
         await repo.delete_swarm_host(row["uuid"])
@@ -36,13 +36,21 @@ async def test_deploy_automode_unihost_when_no_swarm_hosts(client, auth_token, m
         json={"ini_content": ini},
         headers={"Authorization": f"Bearer {auth_token}"},
     )
-    assert resp.status_code == 200, resp.text
+    assert resp.status_code == 202, resp.text
-    assert resp.json()["mode"] == "unihost"
+    body = resp.json()
+    assert body["mode"] == "unihost"
+    assert len(body["lifecycle_ids"]) == 1
 
 
 @pytest.mark.anyio
 async def test_deploy_automode_shards_when_swarm_host_enrolled(client, auth_token, monkeypatch):
-    """Master + one active swarm host → swarm mode, dispatch invoked."""
+    """Master + one active swarm host → swarm mode, lifecycle rows + 202.
+
+    The handler no longer awaits dispatch synchronously — it commits the
+    new shape, creates lifecycle rows, and spawns the runner.  We
+    verify the commit + the per-decky host_uuid assignment via the
+    committed deployment state, and that 202 carries one lifecycle id
+    per decky."""
     monkeypatch.setenv("DECNET_MODE", "master")
     await repo.set_state("deployment", None)
 
@@ -63,27 +71,23 @@ async def test_deploy_automode_shards_when_swarm_host_enrolled(client, auth_toke
         "notes": "",
     })
 
-    fake_response = SwarmDeployResponse(results=[
+    ini = "[decky-01]\nservices = ssh\n[decky-02]\nservices = http\n"
-        SwarmHostResult(host_uuid="host-A", host_name="worker-a", ok=True, detail={})
+    resp = await client.post(
-    ])
+        "/api/v1/deckies/deploy",
+        json={"ini_content": ini},
+        headers={"Authorization": f"Bearer {auth_token}"},
+    )
 
-    with patch(
+    assert resp.status_code == 202, resp.text
-        "decnet.web.router.fleet.api_deploy_deckies.dispatch_decnet_config",
+    body = resp.json()
-        new=AsyncMock(return_value=fake_response),
+    assert body["mode"] == "swarm"
-    ) as mock_dispatch:
+    assert len(body["lifecycle_ids"]) == 2
-        ini = "[decky-01]\nservices = ssh\n[decky-02]\nservices = http\n"
-        resp = await client.post(
-            "/api/v1/deckies/deploy",
-            json={"ini_content": ini},
-            headers={"Authorization": f"Bearer {auth_token}"},
-        )
 
-    assert resp.status_code == 200, resp.text
+    committed = await repo.get_state("deployment")
-    assert resp.json()["mode"] == "swarm"
+    assert committed is not None
-    assert mock_dispatch.await_count == 1
+    cfg = committed["config"]
-    dispatched_config = mock_dispatch.await_args.args[0]
+    assert cfg["mode"] == "swarm"
-    assert dispatched_config.mode == "swarm"
+    assert {d["host_uuid"] for d in cfg["deckies"]} == {"host-A"}
-    assert all(d.host_uuid == "host-A" for d in dispatched_config.deckies)
 
     await repo.delete_swarm_host("host-A")
 
@@ -130,25 +134,19 @@ async def test_deploy_automode_resets_stale_host_uuid(client, auth_token, monkey
         "compose_path": "",
     })
 
-    fake_response = SwarmDeployResponse(results=[
+    ini = "[decky-new]\nservices = ssh\n"
-        SwarmHostResult(host_uuid="host-LIVE", host_name="live", ok=True, detail={})
+    resp = await client.post(
-    ])
+        "/api/v1/deckies/deploy",
+        json={"ini_content": ini},
+        headers={"Authorization": f"Bearer {auth_token}"},
+    )
 
-    with patch(
+    assert resp.status_code == 202, resp.text
-        "decnet.web.router.fleet.api_deploy_deckies.dispatch_decnet_config",
+    committed = await repo.get_state("deployment")
-        new=AsyncMock(return_value=fake_response),
+    assert committed is not None
-    ) as mock_dispatch:
+    cfg = committed["config"]
-        ini = "[decky-new]\nservices = ssh\n"
+    # The carried-over decky and the new one must both point at the live host.
-        resp = await client.post(
+    assert {d["host_uuid"] for d in cfg["deckies"]} == {"host-LIVE"}
-            "/api/v1/deckies/deploy",
-            json={"ini_content": ini},
-            headers={"Authorization": f"Bearer {auth_token}"},
-        )
-
-    assert resp.status_code == 200, resp.text
-    dispatched = mock_dispatch.await_args.args[0]
-    # Both the carried-over decky and the new one must point at the live host.
-    assert {d.host_uuid for d in dispatched.deckies} == {"host-LIVE"}
 
     await repo.delete_swarm_host("host-LIVE")
     await repo.set_state("deployment", None)

tests/api/fleet/test_mutate_decky.py

@@ -1,10 +1,21 @@
 """
-Tests for the mutate decky API endpoint.
+Tests for the mutate decky API endpoint — now 202 fire-and-forget.
-"""
+
+The handler must:
+1. Reject anonymous callers (401).
+2. 404 when no active deployment exists.
+3. 404 when the named decky isn't in the current state.
+4. 422 when decky_name pattern fails validation.
+5. On the happy path, create a DeckyLifecycle row, spawn a background
+   task, return 202 with the row's id.
+"""
+from __future__ import annotations
+
+import asyncio
+from unittest.mock import AsyncMock, patch
 
-import pytest
 import httpx
-from unittest.mock import patch
+import pytest
 
 
 class TestMutateDecky:
@@ -14,30 +25,139 @@ class TestMutateDecky:
         assert resp.status_code == 401
 
     @pytest.mark.asyncio
-    async def test_successful_mutation(self, client: httpx.AsyncClient, auth_token: str, monkeypatch: pytest.MonkeyPatch):
+    async def test_no_deployment_returns_404(
+        self, client: httpx.AsyncClient, auth_token: str,
+        monkeypatch: pytest.MonkeyPatch,
+    ):
         monkeypatch.delenv("DECNET_CONTRACT_TEST", raising=False)
-        with patch("decnet.web.router.fleet.api_mutate_decky.mutate_decky", return_value=True):
+        with patch(
-            resp = await client.post(
+            "decnet.web.router.fleet.api_mutate_decky.repo.get_state",
-                "/api/v1/deckies/decky-01/mutate",
+            new_callable=AsyncMock, return_value=None,
-                headers={"Authorization": f"Bearer {auth_token}"},
+        ):
-            )
-        assert resp.status_code == 200
-        assert "Successfully mutated" in resp.json()["message"]
-
-    @pytest.mark.asyncio
-    async def test_failed_mutation_returns_404(self, client: httpx.AsyncClient, auth_token: str, monkeypatch: pytest.MonkeyPatch):
-        monkeypatch.delenv("DECNET_CONTRACT_TEST", raising=False)
-        with patch("decnet.web.router.fleet.api_mutate_decky.mutate_decky", return_value=False):
             resp = await client.post(
                 "/api/v1/deckies/decky-01/mutate",
                 headers={"Authorization": f"Bearer {auth_token}"},
             )
         assert resp.status_code == 404
+        assert "No active deployment" in resp.json()["detail"]
 
     @pytest.mark.asyncio
-    async def test_invalid_decky_name_returns_422(self, client: httpx.AsyncClient, auth_token: str):
+    async def test_unknown_decky_returns_404(
+        self, client: httpx.AsyncClient, auth_token: str,
+        monkeypatch: pytest.MonkeyPatch,
+    ):
+        monkeypatch.delenv("DECNET_CONTRACT_TEST", raising=False)
+        from decnet.config import DecnetConfig, DeckyConfig
+        cfg = DecnetConfig(
+            mode="unihost", interface="eth0",
+            subnet="10.0.0.0/24", gateway="10.0.0.1",
+            deckies=[DeckyConfig(
+                name="decky-existing", ip="10.0.0.10",
+                services=["ssh"], distro="debian",
+                base_image="debian:bookworm-slim", hostname="d01",
+            )],
+        )
+        with patch(
+            "decnet.web.router.fleet.api_mutate_decky.repo.get_state",
+            new_callable=AsyncMock,
+            return_value={"config": cfg.model_dump(), "compose_path": "c.yml"},
+        ):
+            resp = await client.post(
+                "/api/v1/deckies/decky-missing/mutate",
+                headers={"Authorization": f"Bearer {auth_token}"},
+            )
+        assert resp.status_code == 404
+        assert "not found" in resp.json()["detail"]
+
+    @pytest.mark.asyncio
+    async def test_invalid_decky_name_returns_422(
+        self, client: httpx.AsyncClient, auth_token: str,
+    ):
         resp = await client.post(
             "/api/v1/deckies/INVALID NAME!!/mutate",
             headers={"Authorization": f"Bearer {auth_token}"},
         )
         assert resp.status_code == 422
+
+    @pytest.mark.asyncio
+    async def test_successful_mutate_returns_202_with_lifecycle_id(
+        self, client: httpx.AsyncClient, auth_token: str,
+        monkeypatch: pytest.MonkeyPatch,
+    ):
+        monkeypatch.delenv("DECNET_CONTRACT_TEST", raising=False)
+        from decnet.config import DecnetConfig, DeckyConfig
+        cfg = DecnetConfig(
+            mode="unihost", interface="eth0",
+            subnet="10.0.0.0/24", gateway="10.0.0.1",
+            deckies=[DeckyConfig(
+                name="decky-01", ip="10.0.0.10",
+                services=["ssh"], distro="debian",
+                base_image="debian:bookworm-slim", hostname="d01",
+            )],
+        )
+
+        spawned: list[str] = []
+        real_create_task = asyncio.create_task
+
+        def _capture(coro, **kw):
+            spawned.append(kw.get("name", ""))
+            coro.close()
+            async def _noop(): return None
+            return real_create_task(_noop())
+
+        with patch(
+            "decnet.web.router.fleet.api_mutate_decky.repo.get_state",
+            new_callable=AsyncMock,
+            return_value={"config": cfg.model_dump(), "compose_path": "c.yml"},
+        ), patch(
+            "decnet.web.router.fleet.api_mutate_decky.repo.set_state",
+            new_callable=AsyncMock, return_value=None,
+        ), patch(
+            "decnet.web.router.fleet.api_mutate_decky.repo.create_lifecycle",
+            new_callable=AsyncMock, return_value="lid-abc",
+        ), patch(
+            "decnet.web.router.fleet.api_mutate_decky.pick_new_services",
+            return_value=["http", "ftp"],
+        ), patch(
+            "decnet.web.router.fleet.api_mutate_decky.asyncio.create_task",
+            side_effect=_capture,
+        ):
+            resp = await client.post(
+                "/api/v1/deckies/decky-01/mutate",
+                headers={"Authorization": f"Bearer {auth_token}"},
+            )
+        assert resp.status_code == 202, resp.text
+        body = resp.json()
+        assert body["lifecycle_ids"] == ["lid-abc"]
+        assert spawned and spawned[0].startswith("mutate-")
+
+    @pytest.mark.asyncio
+    async def test_no_services_available_returns_404(
+        self, client: httpx.AsyncClient, auth_token: str,
+        monkeypatch: pytest.MonkeyPatch,
+    ):
+        monkeypatch.delenv("DECNET_CONTRACT_TEST", raising=False)
+        from decnet.config import DecnetConfig, DeckyConfig
+        cfg = DecnetConfig(
+            mode="unihost", interface="eth0",
+            subnet="10.0.0.0/24", gateway="10.0.0.1",
+            deckies=[DeckyConfig(
+                name="decky-01", ip="10.0.0.10",
+                services=["ssh"], distro="debian",
+                base_image="debian:bookworm-slim", hostname="d01",
+            )],
+        )
+        with patch(
+            "decnet.web.router.fleet.api_mutate_decky.repo.get_state",
+            new_callable=AsyncMock,
+            return_value={"config": cfg.model_dump(), "compose_path": "c.yml"},
+        ), patch(
+            "decnet.web.router.fleet.api_mutate_decky.pick_new_services",
+            return_value=None,
+        ):
+            resp = await client.post(
+                "/api/v1/deckies/decky-01/mutate",
+                headers={"Authorization": f"Bearer {auth_token}"},
+            )
+        assert resp.status_code == 404
+        assert "No services available" in resp.json()["detail"]