feat(swarm): heartbeat-driven topology resync for agent-pinned deployments
Agent heartbeats now carry an applied-topology snapshot. The master heartbeat handler compares the reported version_hash against what canonical_hash yields for the hydrated topology pinned to that host and flags Topology.needs_resync on divergence (or when the agent reports no topology at all while master expects one). The mutator watch loop gains reconcile_agent_resyncs, which re-pushes the current hydrated blob via AgentClient.apply_topology without touching status, then clears the flag on success. Push failures leave the flag set so the next tick retries.
This commit is contained in:
@@ -225,6 +225,11 @@ class Topology(SQLModel, table=True):
|
||||
# the topology or any child row when an expected_version is supplied.
|
||||
# Callers pass their last-seen version; mismatch raises VersionConflict.
|
||||
version: int = Field(default=1, nullable=False)
|
||||
# Set by the heartbeat handler when an agent's reported
|
||||
# ``applied_version_hash`` diverges from what we expect it to be
|
||||
# running. Drained by the mutator watch loop, which re-pushes via
|
||||
# AgentClient and clears the flag. NULL for unihost topologies.
|
||||
needs_resync: bool = Field(default=False, nullable=False)
|
||||
|
||||
|
||||
class LAN(SQLModel, table=True):
|
||||
|
||||
@@ -268,6 +268,12 @@ class BaseRepository(ABC):
|
||||
async def delete_topology_cascade(self, topology_id: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
async def set_topology_resync(self, topology_id: str, value: bool) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def list_topologies_needing_resync(self) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def add_lan(self, data: dict[str, Any]) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -1013,6 +1013,30 @@ class SQLModelRepository(BaseRepository):
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def set_topology_resync(self, topology_id: str, value: bool) -> None:
|
||||
async with self._session() as session:
|
||||
result = await session.execute(
|
||||
select(Topology).where(Topology.id == topology_id)
|
||||
)
|
||||
topo = result.scalar_one_or_none()
|
||||
if topo is None:
|
||||
return
|
||||
topo.needs_resync = bool(value)
|
||||
session.add(topo)
|
||||
await session.commit()
|
||||
|
||||
async def list_topologies_needing_resync(self) -> list[dict[str, Any]]:
|
||||
async with self._session() as session:
|
||||
result = await session.execute(
|
||||
select(Topology).where(Topology.needs_resync == True) # noqa: E712
|
||||
)
|
||||
return [
|
||||
self._deserialize_json_fields(
|
||||
r.model_dump(mode="json"), ("config_snapshot",)
|
||||
)
|
||||
for r in result.scalars().all()
|
||||
]
|
||||
|
||||
async def delete_topology_cascade(self, topology_id: str) -> bool:
|
||||
"""Delete topology and all children. No portable ON DELETE CASCADE."""
|
||||
async with self._session() as session:
|
||||
|
||||
@@ -35,6 +35,7 @@ class HeartbeatRequest(BaseModel):
|
||||
host_uuid: str
|
||||
agent_version: Optional[str] = None
|
||||
status: dict[str, Any]
|
||||
topology: Optional[dict[str, Any]] = None
|
||||
|
||||
|
||||
def _extract_peer_fingerprint(scope: dict[str, Any]) -> Optional[str]:
|
||||
@@ -96,6 +97,67 @@ async def _verify_peer_matches_host(
|
||||
return host
|
||||
|
||||
|
||||
async def _reconcile_topology_report(
|
||||
repo: BaseRepository,
|
||||
host_uuid: str,
|
||||
reported: Optional[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Compare the agent's reported applied_version_hash against what
|
||||
master expects for any topology pinned to *host_uuid*.
|
||||
|
||||
Sets ``needs_resync=True`` when:
|
||||
- master has an ACTIVE topology targeted here but the agent reports
|
||||
a different hash, OR
|
||||
- master has an ACTIVE topology targeted here but the agent reports
|
||||
no topology at all (fresh boot / wiped cache).
|
||||
|
||||
The actual re-push is handled by the mutator reconcile loop so the
|
||||
heartbeat endpoint stays cheap.
|
||||
"""
|
||||
from decnet.topology.hashing import canonical_hash
|
||||
from decnet.topology.persistence import hydrate
|
||||
from decnet.topology.status import TopologyStatus
|
||||
|
||||
try:
|
||||
topos = await repo.list_topologies(status=TopologyStatus.ACTIVE)
|
||||
except Exception:
|
||||
log.exception("heartbeat: could not list active topologies")
|
||||
return
|
||||
mine = [t for t in topos if t.get("target_host_uuid") == host_uuid]
|
||||
if not mine:
|
||||
return
|
||||
|
||||
reported_id = (reported or {}).get("topology_id")
|
||||
reported_hash = (reported or {}).get("applied_version_hash")
|
||||
|
||||
for topo in mine:
|
||||
tid = topo["id"]
|
||||
if topo.get("needs_resync"):
|
||||
continue
|
||||
expected: Optional[str] = None
|
||||
if reported_id == tid and reported_hash:
|
||||
try:
|
||||
hydrated = await hydrate(repo, tid)
|
||||
except Exception:
|
||||
log.exception("heartbeat: hydrate failed tid=%s", tid)
|
||||
continue
|
||||
if hydrated is None:
|
||||
continue
|
||||
expected = canonical_hash(hydrated)
|
||||
if expected == reported_hash:
|
||||
continue
|
||||
# Either mismatch or agent reports no/other topology — flag it.
|
||||
try:
|
||||
await repo.set_topology_resync(tid, True)
|
||||
log.info(
|
||||
"heartbeat: flagged topology %s for resync (host=%s "
|
||||
"reported_id=%s reported_hash=%s expected=%s)",
|
||||
tid, host_uuid, reported_id, reported_hash, expected,
|
||||
)
|
||||
except Exception:
|
||||
log.exception("heartbeat: failed to flag resync tid=%s", tid)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/heartbeat",
|
||||
status_code=204,
|
||||
@@ -120,6 +182,8 @@ async def heartbeat(
|
||||
{"status": "active", "last_heartbeat": now},
|
||||
)
|
||||
|
||||
await _reconcile_topology_report(repo, req.host_uuid, req.topology)
|
||||
|
||||
status_body = req.status or {}
|
||||
if not status_body.get("deployed"):
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user