anti/DECNET
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(orchestrator): collapse decnet-emailgen.service into orchestrator

Browse Source 
Stage 5 of the realism migration. Email generation is no longer a
separate worker / systemd unit / CLI subcommand — the orchestrator's
single tick loop covers SSH traffic, file plants, and email drops.
Going from 21 services to 20.

Worker:
- _one_tick rolls between traffic / file / email (45/45/10 weights).
  The 10% email weight at a 60s orchestrator interval produces ~one
  email per 10 minutes, close to the pre-collapse 5-minute cadence.
- get_driver_for(action) (stage 4) handles SSH vs Email dispatch.
- Quiet branches fall through so a (decky-set, persona-pool,
  mail-decky) shape that silences one branch doesn't waste the tick.
- Periodic prune covers both orchestrator_events and
  orchestrator_emails tables.

Deletions:
- deploy/decnet-emailgen.service.j2
- decnet/orchestrator/emailgen/worker.py
- decnet/cli/emailgen.py
- tests/orchestrator/emailgen/test_worker_integration.py

Renames (history-preserving):
- decnet/web/router/emailgen/ -> decnet/web/router/realism/
- tests/api/emailgen/        -> tests/api/realism/
- tests/cli/test_emailgen_*  -> tests/cli/test_realism_*

Public surface changes (clean break, pre-v1):
- API URL /api/v1/emailgen/personas -> /api/v1/realism/personas
- CLI `decnet emailgen import-personas` -> `decnet realism
  import-personas`. `decnet emailgen run` is gone — the orchestrator
  covers it.
- gating.py: emailgen master-only group replaced by realism.
- decnet-orchestrator.service.j2: DECNET_REALISM_* env block added.
- decnet.target: decnet-emailgen.service entry removed.
- frontend: PersonaGeneration.tsx fetches /realism/personas.
This commit is contained in:
anti
2026-04-27 16:33:04 -04:00
parent cb1872c52f
commit 32eeb0c813
24 changed files with 1334 additions and 1397 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
tests/api/emailgen/__init__.py → tests/api/realism/__init__.py
View File

4
tests/api/emailgen/test_personas_api.py → tests/api/realism/test_personas_api.py
View File

@@ -1,4 +1,4 @@
"""GET/PUT /api/v1/emailgen/personas — global persona pool CRUD."""
"""GET/PUT /api/v1/realism/personas — global persona pool CRUD."""
from __future__ import annotations
import json
@@ -6,7 +6,7 @@ import json
import pytest
from decnet.realism import personas_pool as global_pool
from decnet.web.router.emailgen.api_personas import (
from decnet.web.router.realism.api_personas import (
list_personas,
replace_personas,
)

180
tests/api/topology/test_personas_api.py Normal file
View File

@@ -0,0 +1,180 @@
"""Per-topology persona endpoints — GET/PUT /topologies/{id}/personas."""
from __future__ import annotations
import json
import pytest
from decnet.topology.config import TopologyConfig
from decnet.topology.generator import generate
from decnet.topology.persistence import persist
from decnet.web.dependencies import repo as _repo
_V1 = "/api/v1/topologies"
def _cfg(name: str = "personas") -> TopologyConfig:
return TopologyConfig(
name=name,
depth=1,
branching_factor=1,
deckies_per_lan_min=1,
deckies_per_lan_max=1,
services_explicit=["ssh"],
randomize_services=False,
seed=0,
)
async def _seed(name: str = "personas") -> str:
return await persist(_repo, generate(_cfg(name)))
def _persona(email: str, name: str = "Jane Doe") -> dict:
return {
"name": name,
"email": email,
"role": "Admin",
"tone": "formal",
"mannerisms": ["uses bullet points"],
}
@pytest.mark.anyio
async def test_get_default_empty(client, auth_token):
tid = await _seed("get-empty")
r = await client.get(
f"{_V1}/{tid}/personas",
headers={"Authorization": f"Bearer {auth_token}"},
)
assert r.status_code == 200, r.text
body = r.json()
assert body["topology_id"] == tid
assert body["personas"] == []
assert body["language_default"] == "en"
@pytest.mark.anyio
async def test_get_404(client, auth_token):
r = await client.get(
f"{_V1}/does-not-exist/personas",
headers={"Authorization": f"Bearer {auth_token}"},
)
assert r.status_code == 404
@pytest.mark.anyio
async def test_put_then_get(client, auth_token):
tid = await _seed("put-roundtrip")
payload = {"personas": [
_persona("a@example.com", "Alice"),
_persona("b@example.com", "Bob"),
]}
r = await client.put(
f"{_V1}/{tid}/personas",
json=payload,
headers={"Authorization": f"Bearer {auth_token}"},
)
assert r.status_code == 200, r.text
assert len(r.json()["personas"]) == 2
r2 = await client.get(
f"{_V1}/{tid}/personas",
headers={"Authorization": f"Bearer {auth_token}"},
)
assert r2.status_code == 200
emails = [p["email"] for p in r2.json()["personas"]]
assert emails == ["a@example.com", "b@example.com"]
# Persisted as JSON string in the topology row.
topo = await _repo.get_topology(tid)
assert isinstance(topo["email_personas"], str)
stored = json.loads(topo["email_personas"])
assert {p["email"] for p in stored} == {"a@example.com", "b@example.com"}
@pytest.mark.anyio
async def test_put_empty_clears(client, auth_token):
tid = await _seed("put-empty")
await client.put(
f"{_V1}/{tid}/personas",
json={"personas": [_persona("x@example.com")]},
headers={"Authorization": f"Bearer {auth_token}"},
)
r = await client.put(
f"{_V1}/{tid}/personas",
json={"personas": []},
headers={"Authorization": f"Bearer {auth_token}"},
)
assert r.status_code == 200
assert r.json()["personas"] == []
@pytest.mark.anyio
async def test_put_non_list_400(client, auth_token):
tid = await _seed("put-non-list")
r = await client.put(
f"{_V1}/{tid}/personas",
json={"personas": "not a list"},
headers={"Authorization": f"Bearer {auth_token}"},
)
assert r.status_code == 400
@pytest.mark.anyio
async def test_put_all_invalid_400(client, auth_token):
tid = await _seed("put-all-bad")
r = await client.put(
f"{_V1}/{tid}/personas",
json={"personas": [{"email": "no-at-sign"}, {"name": "no-email"}]},
headers={"Authorization": f"Bearer {auth_token}"},
)
assert r.status_code == 400
@pytest.mark.anyio
async def test_put_partial_invalid_keeps_valid(client, auth_token):
"""Mirror the global-pool drop-invalid semantics.
The endpoint silently drops bad entries; operators discover what
landed by reading back the GET.
"""
tid = await _seed("put-partial")
r = await client.put(
f"{_V1}/{tid}/personas",
json={"personas": [
_persona("good@example.com"),
{"name": "missing email"},
]},
headers={"Authorization": f"Bearer {auth_token}"},
)
assert r.status_code == 200
body = r.json()
assert [p["email"] for p in body["personas"]] == ["good@example.com"]
@pytest.mark.anyio
async def test_put_404_on_missing_topology(client, auth_token):
r = await client.put(
f"{_V1}/does-not-exist/personas",
json={"personas": [_persona("x@example.com")]},
headers={"Authorization": f"Bearer {auth_token}"},
)
assert r.status_code == 404
@pytest.mark.anyio
async def test_get_does_not_shadow_existing_topology_id(client, auth_token):
"""Ensure the personas subroute is registered before the bare /{id}.
If the literal `/personas` segment got shadowed by the parameterized
`/{id}` route, GET would return the topology body instead of 404 for
a missing personas resource. Sanity-check the order.
"""
tid = await _seed("shadow-check")
r = await client.get(
f"{_V1}/{tid}/personas",
headers={"Authorization": f"Bearer {auth_token}"},
)
assert r.status_code == 200
assert "personas" in r.json()

40
tests/cli/test_emailgen_gating.py → tests/cli/test_realism_gating.py
View File

@@ -1,9 +1,9 @@
"""``decnet emailgen`` is master-only.
"""``decnet realism`` is master-only.
Two layers per CLAUDE.md:
* registration-time hide via :data:`MASTER_ONLY_GROUPS` so agents don't
see ``decnet emailgen`` in ``--help`` at all,
see ``decnet realism`` in ``--help`` at all,
* body-guard ``_require_master_mode()`` so a direct callable import (e.g.
from a third-party tool) still bails on agent hosts.
"""
@@ -17,7 +17,6 @@ import sys
from pathlib import Path
import pytest
from typer.testing import CliRunner
REPO = pathlib.Path(__file__).resolve().parent.parent.parent
@@ -32,7 +31,7 @@ def _clean_env(**overrides: str) -> dict[str, str]:
return base
def test_emailgen_visible_in_master_mode():
def test_realism_visible_in_master_mode():
result = subprocess.run(
[str(DECNET_BIN), "--help"],
env=_clean_env(DECNET_MODE="master"),
@@ -40,10 +39,10 @@ def test_emailgen_visible_in_master_mode():
capture_output=True, text=True, timeout=20,
)
assert result.returncode == 0
assert "emailgen" in result.stdout
assert "realism" in result.stdout
def test_emailgen_hidden_in_agent_mode():
def test_realism_hidden_in_agent_mode():
result = subprocess.run(
[str(DECNET_BIN), "--help"],
env=_clean_env(DECNET_MODE="agent", DECNET_DISALLOW_MASTER="true"),
@@ -51,25 +50,12 @@ def test_emailgen_hidden_in_agent_mode():
capture_output=True, text=True, timeout=20,
)
assert result.returncode == 0
# The sub-app's help string must be gone too — bare "emailgen" can
# The sub-app's help string must be gone too — bare "realism" can
# appear in other command descriptions.
assert "Drip persona-driven fake corporate email" not in result.stdout
assert "realism content engine" not in result.stdout
def test_emailgen_subprocess_run_rejects_in_agent_mode():
"""Subprocess-level: a fresh Python invocation of `decnet emailgen
run` under DECNET_MODE=agent must exit non-zero (gate hides the
sub-app, so the command is unknown to Typer)."""
result = subprocess.run(
[str(DECNET_BIN), "emailgen", "run"],
env=_clean_env(DECNET_MODE="agent", DECNET_DISALLOW_MASTER="true"),
cwd=str(REPO),
capture_output=True, text=True, timeout=20,
)
assert result.returncode != 0
def test_emailgen_subprocess_import_personas_rejects_in_agent_mode(tmp_path):
def test_realism_subprocess_import_personas_rejects_in_agent_mode(tmp_path):
src = tmp_path / "personas.json"
src.write_text(json.dumps([{
"name": "X", "email": "x@y.com", "role": "X", "tone": "formal",
@@ -79,7 +65,7 @@ def test_emailgen_subprocess_import_personas_rejects_in_agent_mode(tmp_path):
"mannerisms": [],
}]))
result = subprocess.run(
[str(DECNET_BIN), "emailgen", "import-personas", str(src)],
[str(DECNET_BIN), "realism", "import-personas", str(src)],
env=_clean_env(DECNET_MODE="agent", DECNET_DISALLOW_MASTER="true"),
cwd=str(REPO),
capture_output=True, text=True, timeout=20,
@@ -89,7 +75,7 @@ def test_emailgen_subprocess_import_personas_rejects_in_agent_mode(tmp_path):
def test_require_master_mode_body_guard_fires_directly(monkeypatch):
"""Defence-in-depth: even bypassing Typer registration, the body-level
``_require_master_mode('emailgen ...')`` raises ``typer.Exit``. Same
``_require_master_mode('realism ...')`` raises ``typer.Exit``. Same
mechanism is verified for `api`/`deploy` in test_mode_gating.py."""
import typer
@@ -99,13 +85,13 @@ def test_require_master_mode_body_guard_fires_directly(monkeypatch):
monkeypatch.setenv("DECNET_DISALLOW_MASTER", "true")
with pytest.raises(typer.Exit):
_require_master_mode("emailgen run")
_require_master_mode("realism import-personas")
def test_master_mode_falls_through_body_guard(monkeypatch):
"""In master mode the guard is a no-op (raises nothing)."""
from decnet.cli.gating import _require_master_mode
from decnet.cli.gating import _require_master_mode # noqa: F401
monkeypatch.setenv("DECNET_MODE", "master")
# Should simply return.
_require_master_mode("emailgen run")
_require_master_mode("realism import-personas")

16
tests/cli/test_emailgen_import_personas.py → tests/cli/test_realism_import_personas.py
View File

@@ -1,4 +1,4 @@
"""``decnet emailgen import-personas`` CLI command."""
"""``decnet realism import-personas`` CLI command."""
from __future__ import annotations
import json
@@ -42,7 +42,7 @@ def test_import_personas_writes_canonical_file(tmp_path, monkeypatch):
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(dest))
result = CliRunner().invoke(
app, ["emailgen", "import-personas", str(src)]
app, ["realism", "import-personas", str(src)]
)
assert result.exit_code == 0, result.stdout
assert dest.exists()
@@ -59,7 +59,7 @@ def test_import_personas_explicit_output_overrides_env(tmp_path, monkeypatch):
result = CliRunner().invoke(
app,
["emailgen", "import-personas", str(src), "--output", str(explicit)],
["realism", "import-personas", str(src), "--output", str(explicit)],
)
assert result.exit_code == 0, result.stdout
assert explicit.exists()
@@ -70,7 +70,7 @@ def test_import_personas_rejects_invalid_json(tmp_path):
src = tmp_path / "src.json"
src.write_text("{not valid")
result = CliRunner().invoke(
app, ["emailgen", "import-personas", str(src)]
app, ["realism", "import-personas", str(src)]
)
assert result.exit_code != 0
assert "Invalid JSON" in result.stdout
@@ -81,7 +81,7 @@ def test_import_personas_rejects_non_list(tmp_path, monkeypatch):
src.write_text(json.dumps({"not": "a list"}))
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(tmp_path / "out.json"))
result = CliRunner().invoke(
app, ["emailgen", "import-personas", str(src)]
app, ["realism", "import-personas", str(src)]
)
assert result.exit_code != 0
assert "list" in result.stdout.lower()
@@ -94,7 +94,7 @@ def test_import_personas_rejects_all_invalid_entries(tmp_path, monkeypatch):
]))
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(tmp_path / "out.json"))
result = CliRunner().invoke(
app, ["emailgen", "import-personas", str(src)]
app, ["realism", "import-personas", str(src)]
)
assert result.exit_code != 0
assert "No valid personas" in result.stdout
@@ -106,7 +106,7 @@ def test_import_personas_warns_on_single_persona(tmp_path, monkeypatch):
dest = tmp_path / "out.json"
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(dest))
result = CliRunner().invoke(
app, ["emailgen", "import-personas", str(src)]
app, ["realism", "import-personas", str(src)]
)
assert result.exit_code == 0, result.stdout
assert "Warning" in result.stdout
@@ -120,7 +120,7 @@ def test_imported_personas_load_via_global_pool(tmp_path, monkeypatch):
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(dest))
result = CliRunner().invoke(
app, ["emailgen", "import-personas", str(src)]
app, ["realism", "import-personas", str(src)]
)
assert result.exit_code == 0, result.stdout

141
tests/orchestrator/emailgen/test_worker_integration.py
View File

@@ -1,141 +0,0 @@
"""End-to-end-ish: one emailgen tick against a real SQLite repo + FakeBus,
with the Ollama + docker-exec subprocess stubbed."""
from __future__ import annotations
import json
import pytest
import pytest_asyncio
from decnet.bus.fake import FakeBus
from decnet.orchestrator.drivers import email as email_driver
from decnet.orchestrator.emailgen import worker as eg_worker
from decnet.orchestrator.emailgen.scheduler import EmailAction # noqa: F401
from decnet.realism.llm.impl.fake import FakeBackend
from decnet.web.db.models import Topology, TopologyDecky
from decnet.web.db.sqlite.repository import SQLiteRepository
_PERSONAS = [
{
"name": "John Smith",
"email": "john@corp.com",
"role": "COO",
"tone": "formal",
"mannerisms": ["uses 'Best regards'"],
"active_hours": "00:00-00:00", # always-on so test is hour-independent
},
{
"name": "Sarah Johnson",
"email": "sarah@corp.com",
"role": "PM",
"tone": "direct",
"mannerisms": ["uses bullets"],
"active_hours": "00:00-00:00",
},
]
@pytest_asyncio.fixture
async def repo(tmp_path):
r = SQLiteRepository(db_path=str(tmp_path / "decnet.db"))
await r.initialize()
yield r
await r.engine.dispose()
@pytest_asyncio.fixture
async def fake_bus():
bus = FakeBus()
await bus.connect()
try:
yield bus
finally:
await bus.close()
async def _seed_mail_topology(repo: SQLiteRepository) -> str:
async with repo._session() as session:
topo = Topology(
name="t-mail",
config_snapshot="{}",
status="active",
email_personas=json.dumps(_PERSONAS),
language_default="en",
)
session.add(topo)
await session.commit()
await session.refresh(topo)
decky = TopologyDecky(
topology_id=topo.id,
name="mailhost",
services=json.dumps(["imap"]),
ip="10.0.0.10",
state="running",
)
session.add(decky)
await session.commit()
await session.refresh(decky)
return decky.uuid
@pytest.mark.asyncio
async def test_one_tick_records_and_publishes(repo, fake_bus, monkeypatch):
decky_uuid = await _seed_mail_topology(repo)
# Stub only the docker exec subprocess; the LLM call goes through
# an injected FakeBackend with deterministic output.
async def fake_run_capture(argv, *, stdin_data=None, timeout=8.0):
return 0, "", ""
monkeypatch.setattr(email_driver, "_run_capture", fake_run_capture)
received: list = []
async def collect():
async with fake_bus.subscribe(f"orchestrator.email.{decky_uuid}") as sub:
async for ev in sub:
received.append(ev)
return
import asyncio
collector = asyncio.create_task(collect())
await asyncio.sleep(0)
driver = email_driver.EmailDriver(
llm=FakeBackend(output="Subject: Hi\n\nBody here.\n"),
)
await eg_worker._one_tick(repo, driver, fake_bus)
await asyncio.wait_for(collector, timeout=2.0)
rows = await repo.list_orchestrator_emails()
assert len(rows) == 1
row = rows[0]
assert row["success"] is True
assert row["mail_decky_uuid"] == decky_uuid
assert row["subject"] == "Hi"
assert row["language"] == "en"
assert len(received) == 1
assert received[0].topic == f"orchestrator.email.{decky_uuid}"
assert received[0].payload["kind"] == "email"
assert received[0].payload["success"] is True
@pytest.mark.asyncio
async def test_one_tick_noop_when_no_mail_decky(repo, fake_bus, monkeypatch):
called = False
async def fake_run_capture(argv, *, stdin_data=None, timeout=8.0):
nonlocal called
called = True
return 0, "", ""
monkeypatch.setattr(email_driver, "_run_capture", fake_run_capture)
driver = email_driver.EmailDriver(
llm=FakeBackend(output="Subject: x\n\nb\n"),
)
await eg_worker._one_tick(repo, driver, fake_bus)
assert called is False
assert await repo.list_orchestrator_emails() == []

30
tests/orchestrator/test_worker_integration.py
View File

@@ -73,6 +73,13 @@ async def test_one_tick_records_event_and_publishes(repo, fake_bus, monkeypatch)
monkeypatch.setattr(ssh_driver, "_run", fake_run)
async def fake_run_with_stdin(argv, stdin_bytes):
# plant_file takes the base64-streaming path; treat any docker
# exec write as a successful no-op for the integration test.
return 0, "", ""
monkeypatch.setattr(ssh_driver, "_run_with_stdin", fake_run_with_stdin)
received: list = []
async def collect():
@@ -87,8 +94,7 @@ async def test_one_tick_records_event_and_publishes(repo, fake_bus, monkeypatch)
# Yield once so the subscription is registered before we publish.
await asyncio.sleep(0)
driver = ssh_driver.SSHDriver()
await orch_worker._one_tick(repo, driver, fake_bus)
await orch_worker._one_tick(repo, fake_bus)
await asyncio.wait_for(collector, timeout=2.0)
@@ -134,8 +140,14 @@ async def test_one_tick_picks_fleet_deckies(repo, fake_bus, monkeypatch):
monkeypatch.setattr(ssh_driver, "_run", fake_run)
driver = ssh_driver.SSHDriver()
await orch_worker._one_tick(repo, driver, fake_bus)
async def fake_run_with_stdin(argv, stdin_bytes):
# plant_file takes the base64-streaming path; treat any docker
# exec write as a successful no-op for the integration test.
return 0, "", ""
monkeypatch.setattr(ssh_driver, "_run_with_stdin", fake_run_with_stdin)
await orch_worker._one_tick(repo, fake_bus)
rows = await repo.list_orchestrator_events(limit=10)
assert len(rows) == 1
@@ -154,8 +166,14 @@ async def test_tick_is_noop_when_no_running_deckies(repo, fake_bus, monkeypatch)
return 0, "SSH-2.0-foo", ""
monkeypatch.setattr(ssh_driver, "_run", fake_run)
driver = ssh_driver.SSHDriver()
await orch_worker._one_tick(repo, driver, fake_bus)
async def fake_run_with_stdin(argv, stdin_bytes):
# plant_file takes the base64-streaming path; treat any docker
# exec write as a successful no-op for the integration test.
return 0, "", ""
monkeypatch.setattr(ssh_driver, "_run_with_stdin", fake_run_with_stdin)
await orch_worker._one_tick(repo, fake_bus)
assert called is False
assert await repo.list_orchestrator_events(limit=10) == []