feat(mazenet): step 7 — topology_mutations queue + mutator reconciler
Adds the live-mutation pipeline for active/degraded topologies: * TopologyMutation table with composite index (state, topology_id) so the watch-loop guard query stays O(log n). * claim_next_mutation is a single atomic UPDATE ... WHERE state='pending' so racing reconcilers deterministically pick one winner; losers see rowcount=0 and skip. * reconcile_topologies drains pending rows per live topology, applies via decnet.mutator.ops.dispatch, and on failure marks the mutation failed + transitions topology to degraded. * run_watch_loop gains a gated branch: flat-fleet mutate_all runs every tick unchanged; the reconciler only enters when the cheap has_pending_topology_mutation guard returns True. * apply_* ops re-check hard invariants (names, IP collisions, subnet overlap, known services, service_config shape) after every mutation so the repo never lands in an invalid state. * CLI: 'decnet topology mutate' / 'mutations' subcommands.
This commit is contained in:
@@ -133,14 +133,89 @@ async def mutate_all(repo: BaseRepository, force: bool = False) -> None:
|
||||
log.info("mutate_all: complete mutated_count=%d", mutated_count)
|
||||
|
||||
|
||||
@_traced("mutator.reconcile_topologies")
|
||||
async def reconcile_topologies(repo: BaseRepository) -> int:
|
||||
"""Drain pending ``topology_mutations`` rows against live topologies.
|
||||
|
||||
For every topology in ``active|degraded`` with at least one pending
|
||||
mutation, atomically claim the oldest via
|
||||
:meth:`BaseRepository.claim_next_mutation`, dispatch to the matching
|
||||
``apply_<op>`` in :mod:`decnet.mutator.ops`, and write the outcome
|
||||
back (``applied`` or ``failed``).
|
||||
|
||||
On ``MutationError`` the topology is flipped to ``degraded`` — the
|
||||
same state the future Healer will target — so operators can see that
|
||||
a requested change was rejected without the repo drifting into an
|
||||
inconsistent state.
|
||||
|
||||
Returns the number of mutations drained this tick.
|
||||
"""
|
||||
# Local imports keep the flat-fleet hot path free of MazeNET cost.
|
||||
from decnet.mutator.ops import MutationError, dispatch as _op_dispatch
|
||||
from decnet.topology.persistence import transition_status
|
||||
from decnet.topology.status import TopologyStatus, TopologyStatusError
|
||||
|
||||
drained = 0
|
||||
for tid in await repo.list_live_topology_ids():
|
||||
while True:
|
||||
mut = await repo.claim_next_mutation(tid)
|
||||
if mut is None:
|
||||
break # no more work for this topology this tick.
|
||||
try:
|
||||
await _op_dispatch(repo, tid, mut["op"], mut["payload"])
|
||||
await repo.mark_mutation_applied(mut["id"])
|
||||
drained += 1
|
||||
log.info(
|
||||
"topology %s mutation %s applied op=%s",
|
||||
tid, mut["id"], mut["op"],
|
||||
)
|
||||
except (MutationError, Exception) as exc: # noqa: BLE001
|
||||
reason = f"{type(exc).__name__}: {exc}"
|
||||
await repo.mark_mutation_failed(mut["id"], reason)
|
||||
log.warning(
|
||||
"topology %s mutation %s failed: %s",
|
||||
tid, mut["id"], reason,
|
||||
)
|
||||
try:
|
||||
await transition_status(
|
||||
repo, tid, TopologyStatus.DEGRADED, reason=reason,
|
||||
)
|
||||
except TopologyStatusError:
|
||||
# Already degraded / in a state that can't degrade
|
||||
# further — leave as is.
|
||||
pass
|
||||
# Stop draining this topology on first failure so the
|
||||
# operator can inspect before a cascade.
|
||||
break
|
||||
return drained
|
||||
|
||||
|
||||
@_traced("mutator.watch_loop")
|
||||
async def run_watch_loop(repo: BaseRepository, poll_interval_secs: int = 10) -> None:
|
||||
"""Run an infinite loop checking for deckies that need mutation."""
|
||||
"""Run an infinite loop checking for deckies that need mutation.
|
||||
|
||||
Two independent responsibilities, in strict order per tick:
|
||||
|
||||
1. Flat-fleet service rotation (``mutate_all``) — runs every tick
|
||||
regardless of MazeNET state, preserving phase-1 timing.
|
||||
2. MazeNET live-mutation reconciliation — runs only when the cheap
|
||||
guard ``has_pending_topology_mutation`` (indexed composite
|
||||
lookup) returns True. Zero-topology and idle-topology hosts pay
|
||||
exactly one indexed query per tick.
|
||||
"""
|
||||
log.info("mutator watch loop started poll_interval_secs=%d", poll_interval_secs)
|
||||
console.print(f"[green]DECNET Mutator Watcher started (polling every {poll_interval_secs}s).[/]")
|
||||
try:
|
||||
while True:
|
||||
await mutate_all(force=False, repo=repo)
|
||||
# Gate reconciler on the O(log n) guard query — avoids
|
||||
# entering the dispatch body when there's nothing to do.
|
||||
try:
|
||||
if await repo.has_pending_topology_mutation():
|
||||
await reconcile_topologies(repo)
|
||||
except NotImplementedError:
|
||||
# Backend without MazeNET support — nothing to reconcile.
|
||||
pass
|
||||
await asyncio.sleep(poll_interval_secs)
|
||||
except KeyboardInterrupt:
|
||||
log.info("mutator watch loop stopped")
|
||||
|
||||
Reference in New Issue
Block a user