refactor(orchestrator): collapse decnet-emailgen.service into orchestrator
Stage 5 of the realism migration. Email generation is no longer a separate worker / systemd unit / CLI subcommand — the orchestrator's single tick loop covers SSH traffic, file plants, and email drops. Going from 21 services to 20. Worker: - _one_tick rolls between traffic / file / email (45/45/10 weights). The 10% email weight at a 60s orchestrator interval produces ~one email per 10 minutes, close to the pre-collapse 5-minute cadence. - get_driver_for(action) (stage 4) handles SSH vs Email dispatch. - Quiet branches fall through so a (decky-set, persona-pool, mail-decky) shape that silences one branch doesn't waste the tick. - Periodic prune covers both orchestrator_events and orchestrator_emails tables. Deletions: - deploy/decnet-emailgen.service.j2 - decnet/orchestrator/emailgen/worker.py - decnet/cli/emailgen.py - tests/orchestrator/emailgen/test_worker_integration.py Renames (history-preserving): - decnet/web/router/emailgen/ -> decnet/web/router/realism/ - tests/api/emailgen/ -> tests/api/realism/ - tests/cli/test_emailgen_* -> tests/cli/test_realism_* Public surface changes (clean break, pre-v1): - API URL /api/v1/emailgen/personas -> /api/v1/realism/personas - CLI `decnet emailgen import-personas` -> `decnet realism import-personas`. `decnet emailgen run` is gone — the orchestrator covers it. - gating.py: emailgen master-only group replaced by realism. - decnet-orchestrator.service.j2: DECNET_REALISM_* env block added. - decnet.target: decnet-emailgen.service entry removed. - frontend: PersonaGeneration.tsx fetches /realism/personas.
This commit is contained in:
@@ -1,33 +1,20 @@
|
||||
"""Emailgen — second orchestrator worker.
|
||||
"""Emailgen — email-specific delivery, scheduling, and threading.
|
||||
|
||||
Generates fake corporate emails (multi-language, threaded, persona-driven)
|
||||
and drops them into mail-decky maildirs so attackers landing on
|
||||
IMAP/POP3 honeypots find believable mailboxes instead of empty inboxes.
|
||||
After stage 5 of the realism migration, ``emailgen`` is no longer a
|
||||
separate worker / systemd unit / CLI subcommand. It exposes:
|
||||
|
||||
The module is intentionally a sibling of :mod:`decnet.orchestrator` (not
|
||||
a flag on it) — separate worker, separate CLI command
|
||||
(``decnet emailgen``), separate systemd-supervised lifecycle. Shares the
|
||||
heartbeat / control-listener scaffolding via :mod:`decnet.bus.publish`.
|
||||
* :mod:`decnet.orchestrator.emailgen.scheduler` — the
|
||||
``EmailAction`` shape and the ``pick(repo)`` policy that decides
|
||||
which mail decky / sender / recipient / thread an email belongs to.
|
||||
* :mod:`decnet.orchestrator.emailgen.threads` — RFC 2822 thread chain
|
||||
helpers (Message-ID generation, Re: / In-Reply-To bookkeeping).
|
||||
* :mod:`decnet.orchestrator.emailgen.events` — DB-row + bus-topic
|
||||
builders for email events.
|
||||
|
||||
Lazy worker re-export: :func:`emailgen_worker` is loaded on first
|
||||
attribute access so that submodules can import package-level names
|
||||
(``decnet.orchestrator.emailgen.events``) without triggering an eager
|
||||
load of the worker — and through it, the email driver, which imports
|
||||
back into this package. Without lazy loading the package + driver +
|
||||
worker form a cycle.
|
||||
The orchestrator's main worker (:mod:`decnet.orchestrator.worker`)
|
||||
calls into these modules per tick. LLM glue, persona schema, prompt
|
||||
builder, and the global persona pool moved to :mod:`decnet.realism`
|
||||
in stage 2 of the migration; this package keeps only the
|
||||
email-specific delivery surface.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover - typing only
|
||||
from decnet.orchestrator.emailgen.worker import emailgen_worker # noqa: F401
|
||||
|
||||
__all__ = ["emailgen_worker"]
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
if name == "emailgen_worker":
|
||||
from decnet.orchestrator.emailgen.worker import emailgen_worker as _w
|
||||
return _w
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
"""Emailgen main loop.
|
||||
|
||||
Mirrors :mod:`decnet.orchestrator.worker` shape: same heartbeat, same
|
||||
control listener, same fire-and-forget bus publish, same prune knob.
|
||||
A wedged ollama call stalls only this worker, never the SSH-flavoured
|
||||
orchestrator running alongside.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
|
||||
from decnet.bus.factory import get_bus
|
||||
from decnet.bus.publish import (
|
||||
publish_safely,
|
||||
run_control_listener,
|
||||
run_health_heartbeat,
|
||||
)
|
||||
from decnet.logging import get_logger
|
||||
from decnet.orchestrator.drivers.email import EmailDriver
|
||||
from decnet.orchestrator.emailgen import events, scheduler
|
||||
from decnet.web.db.repository import BaseRepository
|
||||
|
||||
logger = get_logger("orchestrator.emailgen")
|
||||
|
||||
# Periodic-prune knobs — same shape as orchestrator/worker.py.
|
||||
_PRUNE_EVERY_TICKS = 100
|
||||
_PRUNE_PER_DECKY_CAP = 5000
|
||||
|
||||
|
||||
async def emailgen_worker(
|
||||
repo: BaseRepository,
|
||||
*,
|
||||
interval: int = 300,
|
||||
model: str | None = None,
|
||||
) -> None:
|
||||
"""Periodically generate one fake email into a running mail decky.
|
||||
|
||||
Default interval is 5 minutes — emails are expensive (LLM round
|
||||
trip) and don't need to fire every minute to look natural. Honors
|
||||
``system.emailgen.control`` for graceful shutdown.
|
||||
"""
|
||||
logger.info("emailgen worker started interval=%ds model=%s", interval, model)
|
||||
|
||||
bus = None
|
||||
try:
|
||||
bus = get_bus(client_name="emailgen")
|
||||
await bus.connect()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"emailgen: bus unavailable, continuing without publish: %s", exc
|
||||
)
|
||||
bus = None
|
||||
|
||||
driver = EmailDriver(model=model) if model else EmailDriver()
|
||||
shutdown = asyncio.Event()
|
||||
heartbeat_task = asyncio.create_task(run_health_heartbeat(bus, "emailgen"))
|
||||
control_task = asyncio.create_task(
|
||||
run_control_listener(bus, "emailgen", shutdown),
|
||||
)
|
||||
tick_n = 0
|
||||
try:
|
||||
while not shutdown.is_set():
|
||||
try:
|
||||
await asyncio.wait_for(shutdown.wait(), timeout=interval)
|
||||
except asyncio.TimeoutError:
|
||||
pass # normal tick
|
||||
if shutdown.is_set():
|
||||
break
|
||||
try:
|
||||
await _one_tick(repo, driver, bus)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("emailgen tick failed: %s", exc)
|
||||
tick_n += 1
|
||||
if tick_n % _PRUNE_EVERY_TICKS == 0:
|
||||
try:
|
||||
deleted = await repo.prune_orchestrator_emails(
|
||||
per_decky_cap=_PRUNE_PER_DECKY_CAP,
|
||||
)
|
||||
if deleted:
|
||||
logger.info(
|
||||
"emailgen prune deleted=%d cap=%d",
|
||||
deleted, _PRUNE_PER_DECKY_CAP,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("emailgen prune failed: %s", exc)
|
||||
finally:
|
||||
for t in (heartbeat_task, control_task):
|
||||
t.cancel()
|
||||
with contextlib.suppress(Exception, asyncio.CancelledError):
|
||||
await t
|
||||
if bus is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await bus.close()
|
||||
|
||||
|
||||
async def _one_tick(repo: BaseRepository, driver: EmailDriver, bus) -> None:
|
||||
action = await scheduler.pick(repo)
|
||||
if action is None:
|
||||
logger.debug("emailgen: no actionable mail decky / personas this tick")
|
||||
return
|
||||
|
||||
result = await driver.run(action)
|
||||
row = events.to_row(action, result)
|
||||
await repo.record_orchestrator_email(row)
|
||||
|
||||
if bus is not None:
|
||||
topic = events.topic_for(action)
|
||||
# Mirror the orchestrator-event SSE-friendly payload shape: ts
|
||||
# as iso8601, payload as already-serialised dict.
|
||||
bus_payload = {
|
||||
"kind": "email",
|
||||
"mail_decky_uuid": row["mail_decky_uuid"],
|
||||
"thread_id": row["thread_id"],
|
||||
"message_id": row["message_id"],
|
||||
"in_reply_to": row["in_reply_to"],
|
||||
"sender_email": row["sender_email"],
|
||||
"recipient_email": row["recipient_email"],
|
||||
"subject": row["subject"],
|
||||
"language": row["language"],
|
||||
"success": row["success"],
|
||||
"ts": row["ts"].isoformat(),
|
||||
}
|
||||
await publish_safely(
|
||||
bus, topic, bus_payload, event_type=events.event_type_for(action),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"emailgen tick mail_decky=%s thread=%s success=%s reply=%s",
|
||||
row["mail_decky_uuid"], row["thread_id"], row["success"], action.is_reply,
|
||||
)
|
||||
Reference in New Issue
Block a user