merge: testing → main (reconcile 2-week divergence)

decnet/web/router/swarm_mgmt/api_teardown_host.py

@@ -0,0 +1,150 @@
+"""POST /swarm/hosts/{uuid}/teardown — remote teardown on a swarm worker.
+
+Body: ``{"decky_id": "..."}`` (optional). When ``decky_id`` is null/omitted
+the agent tears down the entire host (all deckies + network); otherwise it
+tears down that single decky.
+
+Async-by-default: the endpoint returns 202 the moment the request is
+accepted and runs the actual agent call + DB cleanup in a background task.
+That lets the operator queue multiple teardowns in parallel without
+blocking on slow docker-compose-down cycles on the worker.
+"""
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from pydantic import BaseModel
+
+from decnet.logging import get_logger
+from decnet.swarm.client import AgentClient
+from decnet.web.db.repository import BaseRepository
+from decnet.web.dependencies import get_repo, require_admin
+
+log = get_logger("swarm.teardown")
+router = APIRouter()
+
+# Track spawned background tasks so (a) they're not GC'd mid-flight and
+# (b) tests can drain them deterministically via ``await drain_pending()``.
+_PENDING: "set[asyncio.Task]" = set()
+
+
+def _spawn(coro) -> asyncio.Task:
+    task = asyncio.create_task(coro)
+    _PENDING.add(task)
+    task.add_done_callback(_PENDING.discard)
+    return task
+
+
+async def drain_pending() -> None:
+    """Await all outstanding teardown tasks. Used by tests."""
+    while _PENDING:
+        await asyncio.gather(*list(_PENDING), return_exceptions=True)
+
+
+class TeardownHostRequest(BaseModel):
+    decky_id: Optional[str] = None
+
+
+class TeardownHostResponse(BaseModel):
+    host_uuid: str
+    host_name: str
+    decky_id: Optional[str] = None
+    accepted: bool
+    detail: str
+
+
+async def _mark_tearing_down(
+    repo: BaseRepository, host_uuid: str, decky_id: Optional[str]
+) -> None:
+    """Flip affected shards to state='tearing_down' so the UI can show
+    progress immediately while the background task runs."""
+    shards = await repo.list_decky_shards(host_uuid)
+    for s in shards:
+        if decky_id and s.get("decky_name") != decky_id:
+            continue
+        await repo.upsert_decky_shard({
+            **s,
+            "state": "tearing_down",
+            "last_error": None,
+        })
+
+
+async def _run_teardown(
+    host: dict[str, Any], repo: BaseRepository, decky_id: Optional[str]
+) -> None:
+    """Fire the remote teardown + DB cleanup. Exceptions are logged and
+    reflected on the shard so the UI surfaces them — never re-raised,
+    since nothing is awaiting us."""
+    try:
+        async with AgentClient(host=host) as agent:
+            await agent.teardown(decky_id)
+    except Exception as exc:
+        log.exception(
+            "swarm.teardown background task failed host=%s decky=%s",
+            host.get("name"), decky_id,
+        )
+        # Reflect the failure on the shard(s) — don't delete on failure,
+        # the operator needs to see what went wrong and retry.
+        try:
+            shards = await repo.list_decky_shards(host["uuid"])
+            for s in shards:
+                if decky_id and s.get("decky_name") != decky_id:
+                    continue
+                await repo.upsert_decky_shard({
+                    **s,
+                    "state": "teardown_failed",
+                    "last_error": str(exc)[:512],
+                })
+        except Exception:
+            log.exception("swarm.teardown failed to record shard failure")
+        return
+
+    try:
+        if decky_id:
+            await repo.delete_decky_shard(decky_id)
+        else:
+            await repo.delete_decky_shards_for_host(host["uuid"])
+    except Exception:
+        log.exception("swarm.teardown DB cleanup failed (agent call succeeded)")
+
+
+@router.post(
+    "/hosts/{uuid}/teardown",
+    response_model=TeardownHostResponse,
+    status_code=status.HTTP_202_ACCEPTED,
+    tags=["Swarm Management"],
+    responses={
+        400: {"description": "Bad Request (malformed JSON body)"},
+        401: {"description": "Could not validate credentials"},
+        403: {"description": "Insufficient permissions"},
+        404: {"description": "Host not found"},
+        422: {"description": "Request body or path parameter validation error"},
+    },
+)
+async def teardown_host(
+    uuid: str,
+    req: TeardownHostRequest,
+    admin: dict = Depends(require_admin),
+    repo: BaseRepository = Depends(get_repo),
+) -> TeardownHostResponse:
+    host = await repo.get_swarm_host_by_uuid(uuid)
+    if host is None:
+        raise HTTPException(status_code=404, detail="host not found")
+
+    await _mark_tearing_down(repo, uuid, req.decky_id)
+
+    # Fire-and-forget: asyncio.create_task (not BackgroundTasks) so the
+    # task runs independently of this request's lifecycle — the operator
+    # can queue another teardown the moment this one returns 202 without
+    # waiting for any per-request cleanup phase.
+    _spawn(_run_teardown(host, repo, req.decky_id))
+
+    return TeardownHostResponse(
+        host_uuid=uuid,
+        host_name=host.get("name") or "",
+        decky_id=req.decky_id,
+        accepted=True,
+        detail="teardown queued",
+    )