feat(workers): bus-backed Workers panel (registry, control, installed flag)
Ships the backend half of Config → Workers:
* Worker registry aggregates `system.*.health` + `system.bus.health`
heartbeats into a last-seen dict; OK / STALE / UNKNOWN tiers drop
out of a 90s window (3× the 30s heartbeat interval).
* `GET /api/v1/workers` returns the snapshot plus `bus_connected`
(so the UI can explain "all UNKNOWN" when the bus socket is down)
and a per-row `installed` flag populated from
`systemctl list-unit-files decnet-*.service` (cached 30s).
* `POST /api/v1/workers/{name}/stop` publishes a stop intent on
`system.<name>.control`; workers listen via the shared control
listener in `bus/publish.py`.
* Heartbeat + control listener wired into collector / profiler /
sniffer / prober / mutator worker loops. API self-heartbeats too
so the panel always has one ground-truth row.
* Topic helper `system_control(name)` + tests covering builder
validation, control listener shutdown path, and the API surface
(auth gating, bus-connected field, unknown-name 404).
Adds `StartFailure` / `StartAllResponse` models in anticipation of
the upcoming start endpoints (DEBT-034).
This commit is contained in:
0
tests/api/workers/__init__.py
Normal file
0
tests/api/workers/__init__.py
Normal file
171
tests/api/workers/test_workers_api.py
Normal file
171
tests/api/workers/test_workers_api.py
Normal file
@@ -0,0 +1,171 @@
|
||||
"""Tests for the Workers panel API endpoints.
|
||||
|
||||
Covers ``GET /api/v1/workers`` (viewer-readable, always surfaces every
|
||||
known worker) and ``POST /api/v1/workers/{name}/stop`` (admin-only,
|
||||
publishes a stop intent on the bus).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from decnet.bus import topics as _topics
|
||||
from decnet.bus.fake import FakeBus
|
||||
from decnet.web import worker_registry as _wr
|
||||
from decnet.web.router.workers import api_control_worker as _ctl
|
||||
from decnet.web.router.workers import api_list_workers as _list
|
||||
from decnet.web.worker_registry import KNOWN_WORKERS
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_registry() -> None:
|
||||
_wr.reset_registry_for_tests()
|
||||
yield
|
||||
_wr.reset_registry_for_tests()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def fake_bus(monkeypatch) -> FakeBus:
|
||||
bus = FakeBus()
|
||||
await bus.connect()
|
||||
|
||||
async def _stub_get_app_bus() -> FakeBus:
|
||||
return bus
|
||||
|
||||
# Patch the symbol the control endpoint imported into its namespace.
|
||||
monkeypatch.setattr(_ctl, "get_app_bus", _stub_get_app_bus)
|
||||
yield bus
|
||||
await bus.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_workers_viewer_sees_all_unknown(
|
||||
client: httpx.AsyncClient, viewer_token: str,
|
||||
) -> None:
|
||||
resp = await client.get(
|
||||
"/api/v1/workers",
|
||||
headers={"Authorization": f"Bearer {viewer_token}"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
names = {w["name"] for w in body["workers"]}
|
||||
assert names == set(KNOWN_WORKERS)
|
||||
# No heartbeats have arrived in the test harness, so every row is unknown.
|
||||
for w in body["workers"]:
|
||||
assert w["status"] == "unknown"
|
||||
assert w["last_heartbeat_ts"] is None
|
||||
assert w["seconds_since"] is None
|
||||
assert "bus_connected" in body
|
||||
assert isinstance(body["bus_connected"], bool)
|
||||
# `installed` flag is always present + boolean.
|
||||
for w in body["workers"]:
|
||||
assert "installed" in w
|
||||
assert isinstance(w["installed"], bool)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_workers_requires_auth(client: httpx.AsyncClient) -> None:
|
||||
resp = await client.get("/api/v1/workers")
|
||||
assert resp.status_code == 401
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_workers_reports_bus_connected_false_when_no_bus(
|
||||
client: httpx.AsyncClient, viewer_token: str, monkeypatch,
|
||||
) -> None:
|
||||
async def _no_bus() -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(_list, "get_app_bus", _no_bus)
|
||||
resp = await client.get(
|
||||
"/api/v1/workers",
|
||||
headers={"Authorization": f"Bearer {viewer_token}"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["bus_connected"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_workers_reports_bus_connected_true_with_fake_bus(
|
||||
client: httpx.AsyncClient, viewer_token: str, monkeypatch,
|
||||
) -> None:
|
||||
bus = FakeBus()
|
||||
await bus.connect()
|
||||
|
||||
async def _fake_bus() -> FakeBus:
|
||||
return bus
|
||||
|
||||
monkeypatch.setattr(_list, "get_app_bus", _fake_bus)
|
||||
try:
|
||||
resp = await client.get(
|
||||
"/api/v1/workers",
|
||||
headers={"Authorization": f"Bearer {viewer_token}"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["bus_connected"] is True
|
||||
finally:
|
||||
await bus.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_worker_admin_publishes_on_bus(
|
||||
client: httpx.AsyncClient, auth_token: str, fake_bus: FakeBus,
|
||||
) -> None:
|
||||
topic = _topics.system_control("mutator")
|
||||
received: list[Any] = []
|
||||
|
||||
sub = fake_bus.subscribe(topic)
|
||||
await sub.__aenter__()
|
||||
|
||||
async def _drain() -> None:
|
||||
async for event in sub:
|
||||
received.append(event)
|
||||
return
|
||||
|
||||
import asyncio
|
||||
reader = asyncio.create_task(_drain())
|
||||
# Give the subscribe a tick so the publish lands on a live reader.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
try:
|
||||
resp = await client.post(
|
||||
"/api/v1/workers/mutator/stop",
|
||||
headers={"Authorization": f"Bearer {auth_token}"},
|
||||
)
|
||||
assert resp.status_code == 202
|
||||
body = resp.json()
|
||||
assert body == {"accepted": True, "worker": "mutator", "action": "stop"}
|
||||
|
||||
await asyncio.wait_for(reader, timeout=1.0)
|
||||
assert len(received) == 1
|
||||
ev = received[0]
|
||||
assert ev.topic == topic
|
||||
assert ev.payload["action"] == _topics.WORKER_CONTROL_STOP
|
||||
assert "requested_by" in ev.payload
|
||||
assert "ts" in ev.payload
|
||||
finally:
|
||||
await sub.__aexit__(None, None, None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_worker_viewer_forbidden(
|
||||
client: httpx.AsyncClient, viewer_token: str, fake_bus: FakeBus,
|
||||
) -> None:
|
||||
resp = await client.post(
|
||||
"/api/v1/workers/mutator/stop",
|
||||
headers={"Authorization": f"Bearer {viewer_token}"},
|
||||
)
|
||||
assert resp.status_code == 403
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_worker_unknown_name_404(
|
||||
client: httpx.AsyncClient, auth_token: str, fake_bus: FakeBus,
|
||||
) -> None:
|
||||
resp = await client.post(
|
||||
"/api/v1/workers/nonsense/stop",
|
||||
headers={"Authorization": f"Bearer {auth_token}"},
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
77
tests/bus/test_control_listener.py
Normal file
77
tests/bus/test_control_listener.py
Normal file
@@ -0,0 +1,77 @@
|
||||
"""Tests for :func:`run_control_listener`.
|
||||
|
||||
The listener is the worker-side half of the Workers panel stop flow:
|
||||
consume ``system.<worker>.control`` messages, set a shutdown event on a
|
||||
well-formed ``{"action": "stop"}``, and ignore everything else without
|
||||
raising.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from decnet.bus import topics as _topics
|
||||
from decnet.bus.fake import FakeBus
|
||||
from decnet.bus.publish import run_control_listener
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_control_listener_sets_shutdown_on_stop() -> None:
|
||||
bus = FakeBus()
|
||||
await bus.connect()
|
||||
shutdown = asyncio.Event()
|
||||
try:
|
||||
task = asyncio.create_task(run_control_listener(bus, "mutator", shutdown))
|
||||
# Give the subscribe() call a tick to register before we publish.
|
||||
await asyncio.sleep(0)
|
||||
await bus.publish(
|
||||
_topics.system_control("mutator"),
|
||||
{"action": _topics.WORKER_CONTROL_STOP, "requested_by": "admin"},
|
||||
event_type="control",
|
||||
)
|
||||
await asyncio.wait_for(task, timeout=1.0)
|
||||
assert shutdown.is_set()
|
||||
finally:
|
||||
await bus.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_control_listener_ignores_malformed() -> None:
|
||||
bus = FakeBus()
|
||||
await bus.connect()
|
||||
shutdown = asyncio.Event()
|
||||
try:
|
||||
task = asyncio.create_task(run_control_listener(bus, "mutator", shutdown))
|
||||
await asyncio.sleep(0)
|
||||
# Unknown action, non-dict-ish field, missing action — none of
|
||||
# these should raise or trigger shutdown.
|
||||
await bus.publish(
|
||||
_topics.system_control("mutator"),
|
||||
{"action": "bogus"}, event_type="control",
|
||||
)
|
||||
await bus.publish(
|
||||
_topics.system_control("mutator"),
|
||||
{"requested_by": "admin"}, event_type="control",
|
||||
)
|
||||
# Now send a real stop to unblock the task so the test terminates.
|
||||
await bus.publish(
|
||||
_topics.system_control("mutator"),
|
||||
{"action": _topics.WORKER_CONTROL_STOP}, event_type="control",
|
||||
)
|
||||
await asyncio.wait_for(task, timeout=1.0)
|
||||
assert shutdown.is_set()
|
||||
finally:
|
||||
await bus.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_control_listener_none_bus_awaits_shutdown() -> None:
|
||||
# With bus=None the listener degrades to awaiting the shutdown event
|
||||
# directly — callers can create_task() unconditionally.
|
||||
shutdown = asyncio.Event()
|
||||
task = asyncio.create_task(run_control_listener(None, "mutator", shutdown))
|
||||
await asyncio.sleep(0)
|
||||
assert not task.done()
|
||||
shutdown.set()
|
||||
await asyncio.wait_for(task, timeout=1.0)
|
||||
@@ -61,3 +61,14 @@ def test_attacker_builder_rejects_empty() -> None:
|
||||
def test_system_health_builder() -> None:
|
||||
assert topics.system_health("sniffer") == "system.sniffer.health"
|
||||
assert topics.system_health("mutator") == "system.mutator.health"
|
||||
|
||||
|
||||
def test_system_control_builder() -> None:
|
||||
assert topics.system_control("mutator") == "system.mutator.control"
|
||||
assert topics.system_control("collector") == "system.collector.control"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad", ["", "has.dot", "has*wildcard", "has>wild", "with space", "with\ttab"])
|
||||
def test_system_control_rejects_bad_segments(bad: str) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
topics.system_control(bad)
|
||||
|
||||
Reference in New Issue
Block a user