refactor(orchestrator): collapse decnet-emailgen.service into orchestrator
Stage 5 of the realism migration. Email generation is no longer a separate worker / systemd unit / CLI subcommand — the orchestrator's single tick loop covers SSH traffic, file plants, and email drops. Going from 21 services to 20. Worker: - _one_tick rolls between traffic / file / email (45/45/10 weights). The 10% email weight at a 60s orchestrator interval produces ~one email per 10 minutes, close to the pre-collapse 5-minute cadence. - get_driver_for(action) (stage 4) handles SSH vs Email dispatch. - Quiet branches fall through so a (decky-set, persona-pool, mail-decky) shape that silences one branch doesn't waste the tick. - Periodic prune covers both orchestrator_events and orchestrator_emails tables. Deletions: - deploy/decnet-emailgen.service.j2 - decnet/orchestrator/emailgen/worker.py - decnet/cli/emailgen.py - tests/orchestrator/emailgen/test_worker_integration.py Renames (history-preserving): - decnet/web/router/emailgen/ -> decnet/web/router/realism/ - tests/api/emailgen/ -> tests/api/realism/ - tests/cli/test_emailgen_* -> tests/cli/test_realism_* Public surface changes (clean break, pre-v1): - API URL /api/v1/emailgen/personas -> /api/v1/realism/personas - CLI `decnet emailgen import-personas` -> `decnet realism import-personas`. `decnet emailgen run` is gone — the orchestrator covers it. - gating.py: emailgen master-only group replaced by realism. - decnet-orchestrator.service.j2: DECNET_REALISM_* env block added. - decnet.target: decnet-emailgen.service entry removed. - frontend: PersonaGeneration.tsx fetches /realism/personas.
This commit is contained in:
@@ -25,7 +25,6 @@ from . import (
|
||||
canary,
|
||||
db,
|
||||
deploy,
|
||||
emailgen,
|
||||
forwarder,
|
||||
geoip,
|
||||
init,
|
||||
@@ -34,6 +33,7 @@ from . import (
|
||||
listener,
|
||||
orchestrator,
|
||||
profiler,
|
||||
realism,
|
||||
reconciler,
|
||||
sniffer,
|
||||
swarm,
|
||||
@@ -58,7 +58,7 @@ for _mod in (
|
||||
api, swarmctl, agent, updater, listener, forwarder,
|
||||
swarm,
|
||||
deploy, lifecycle, workers, inventory,
|
||||
web, profiler, orchestrator, emailgen, reconciler, sniffer, db,
|
||||
web, profiler, orchestrator, realism, reconciler, sniffer, db,
|
||||
topology, bus, geoip, init, webhook, canary,
|
||||
):
|
||||
_mod.register(app)
|
||||
|
||||
@@ -1,185 +0,0 @@
|
||||
"""``decnet emailgen ...`` — orchestrator-sibling email generator.
|
||||
|
||||
Sub-commands:
|
||||
|
||||
* ``decnet emailgen run`` — start the long-running worker
|
||||
(default when invoked with no sub-command, so the historical
|
||||
``decnet emailgen`` invocation still works).
|
||||
* ``decnet emailgen import-personas`` — validate a JSON file and
|
||||
install it as the host-wide global persona pool consumed by fleet
|
||||
(MACVLAN/IPVLAN) and SWARM-shard mail deckies.
|
||||
|
||||
The worker itself stays in :mod:`decnet.orchestrator.emailgen.worker`;
|
||||
this module only owns the CLI surface.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
|
||||
from . import utils as _utils
|
||||
from .gating import _require_master_mode
|
||||
from .utils import console, log
|
||||
|
||||
|
||||
def register(app: typer.Typer) -> None:
|
||||
emailgen_app = typer.Typer(
|
||||
name="emailgen",
|
||||
help=(
|
||||
"Drip persona-driven fake corporate email into running "
|
||||
"IMAP/POP3 mail deckies."
|
||||
),
|
||||
invoke_without_command=True,
|
||||
no_args_is_help=False,
|
||||
)
|
||||
app.add_typer(emailgen_app, name="emailgen")
|
||||
|
||||
@emailgen_app.callback()
|
||||
def _default(ctx: typer.Context) -> None:
|
||||
# Calling ``decnet emailgen`` with no sub-command defers to ``run``
|
||||
# so the documented (and shipped) invocation stays valid.
|
||||
if ctx.invoked_subcommand is None:
|
||||
ctx.invoke(emailgen_run)
|
||||
|
||||
@emailgen_app.command("run")
|
||||
def emailgen_run(
|
||||
interval: int = typer.Option(
|
||||
300, "--interval", "-i",
|
||||
help="Seconds between fake-email generation ticks (default 5m)",
|
||||
),
|
||||
daemon: bool = typer.Option(
|
||||
False, "--daemon", "-d",
|
||||
help="Detach to background as a daemon process",
|
||||
),
|
||||
model: str = typer.Option(
|
||||
"", "--model", "-m",
|
||||
help="Ollama model override (defaults to $DECNET_EMAILGEN_MODEL "
|
||||
"or 'llama3.1')",
|
||||
),
|
||||
) -> None:
|
||||
"""Start the long-running email-generation worker."""
|
||||
# Defence-in-depth: the registration-time gate already hides
|
||||
# ``emailgen`` from Typer when DECNET_MODE=agent, but a direct
|
||||
# callable import would bypass that — block here too.
|
||||
_require_master_mode("emailgen run")
|
||||
import asyncio
|
||||
from decnet.orchestrator.emailgen import emailgen_worker
|
||||
from decnet.web.dependencies import repo
|
||||
|
||||
if daemon:
|
||||
log.info("emailgen daemonizing interval=%d", interval)
|
||||
_utils._daemonize()
|
||||
|
||||
# Honour the env var when the flag was left empty so systemd unit
|
||||
# files can configure the model centrally without per-host CLI
|
||||
# tweaks. Empty -> let the worker apply its own default.
|
||||
resolved_model = model or os.environ.get("DECNET_EMAILGEN_MODEL", "")
|
||||
log.info(
|
||||
"emailgen starting interval=%d model=%s",
|
||||
interval, resolved_model or "default",
|
||||
)
|
||||
console.print(
|
||||
f"[bold cyan]Emailgen starting[/] (interval: {interval}s"
|
||||
f"{', model: ' + resolved_model if resolved_model else ''})"
|
||||
)
|
||||
|
||||
async def _run() -> None:
|
||||
await repo.initialize()
|
||||
await emailgen_worker(
|
||||
repo, interval=interval, model=resolved_model or None,
|
||||
)
|
||||
|
||||
try:
|
||||
asyncio.run(_run())
|
||||
except KeyboardInterrupt:
|
||||
console.print("\n[yellow]Emailgen stopped.[/]")
|
||||
|
||||
@emailgen_app.command("import-personas")
|
||||
def emailgen_import_personas(
|
||||
path: Path = typer.Argument(
|
||||
..., exists=True, file_okay=True, dir_okay=False, readable=True,
|
||||
help="JSON file containing a list of EmailPersona objects",
|
||||
),
|
||||
output: Optional[Path] = typer.Option(
|
||||
None, "--output", "-o",
|
||||
help=(
|
||||
"Override the destination path. Defaults to the canonical "
|
||||
"global pool (DECNET_EMAILGEN_PERSONAS, /etc/decnet/"
|
||||
"email_personas.json, or ~/.decnet/email_personas.json)."
|
||||
),
|
||||
),
|
||||
) -> None:
|
||||
"""Validate + install a personas JSON file as the global pool.
|
||||
|
||||
Use this when deploying with IMAP/POP3 services on fleet
|
||||
(MACVLAN/IPVLAN) or SWARM-shard mail deckies — those have no
|
||||
parent topology row, so they read this host-wide list. MazeNET
|
||||
topology mail deckies use ``Topology.email_personas`` instead and
|
||||
this command does not touch them.
|
||||
"""
|
||||
_require_master_mode("emailgen import-personas")
|
||||
from decnet.realism import personas_pool as global_pool
|
||||
from decnet.realism.personas import parse_personas
|
||||
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
console.print(f"[red]Cannot read {path}:[/] {exc}")
|
||||
raise typer.Exit(code=1) from exc
|
||||
|
||||
# Validate by parsing — we want operators to find out about
|
||||
# broken personas at import time, not at the next worker tick.
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
console.print(f"[red]Invalid JSON in {path}:[/] {exc}")
|
||||
raise typer.Exit(code=1) from exc
|
||||
if not isinstance(payload, list):
|
||||
console.print(
|
||||
f"[red]{path} must contain a JSON list of personas, "
|
||||
f"got {type(payload).__name__}[/]"
|
||||
)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
personas = parse_personas(payload)
|
||||
if not personas:
|
||||
console.print(
|
||||
f"[red]No valid personas in {path}.[/] "
|
||||
"Check the schema (name, email, role, tone, mannerisms)."
|
||||
)
|
||||
raise typer.Exit(code=1)
|
||||
if len(personas) < 2:
|
||||
console.print(
|
||||
f"[yellow]Warning: only {len(personas)} valid persona(s) — "
|
||||
"the worker requires at least 2 to send mail; importing "
|
||||
"anyway in case more are added later.[/]"
|
||||
)
|
||||
|
||||
dest = output or global_pool.resolve_path()
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Re-serialise from the parsed-and-validated objects rather than
|
||||
# copying the source file: drops invalid entries, normalises
|
||||
# whitespace, and gives operators a single canonical layout to
|
||||
# eyeball after the import.
|
||||
dest.write_text(
|
||||
json.dumps(
|
||||
[p.model_dump(exclude_none=False) for p in personas],
|
||||
indent=2,
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
# Cache invalidation happens automatically on next ``load()``
|
||||
# via the mtime check, but reset the in-process cache too in
|
||||
# case the CLI process is the same as the worker (uncommon but
|
||||
# cheap to be correct about).
|
||||
global_pool.reset_cache()
|
||||
console.print(
|
||||
f"[green]Imported {len(personas)} personas to[/] {dest}"
|
||||
)
|
||||
if path != dest:
|
||||
log.info("emailgen import-personas src=%s dest=%s", path, dest)
|
||||
@@ -32,7 +32,7 @@ MASTER_ONLY_COMMANDS: frozenset[str] = frozenset({
|
||||
"db-reset", "init", "webhook", "clusterer", "campaign-clusterer",
|
||||
})
|
||||
MASTER_ONLY_GROUPS: frozenset[str] = frozenset(
|
||||
{"swarm", "topology", "geoip", "emailgen"}
|
||||
{"swarm", "topology", "geoip", "realism"}
|
||||
)
|
||||
|
||||
|
||||
|
||||
111
decnet/cli/realism.py
Normal file
111
decnet/cli/realism.py
Normal file
@@ -0,0 +1,111 @@
|
||||
"""``decnet realism ...`` — content-engine maintenance commands.
|
||||
|
||||
After stage 5 of the realism migration, this is the only remaining
|
||||
CLI surface from the realism library / former emailgen. ``decnet
|
||||
realism run`` does not exist (the orchestrator runs the unified
|
||||
worker via ``decnet orchestrate``); the only sub-command is
|
||||
``import-personas``, which validates + installs the host-wide global
|
||||
persona pool consumed by fleet (MACVLAN/IPVLAN) and SWARM-shard
|
||||
deckies.
|
||||
|
||||
Topology personas live on ``Topology.email_personas`` and are
|
||||
managed via the dashboard or the topology API; this command does
|
||||
not touch them.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
|
||||
from .gating import _require_master_mode
|
||||
from .utils import console, log
|
||||
|
||||
|
||||
def register(app: typer.Typer) -> None:
|
||||
realism_app = typer.Typer(
|
||||
name="realism",
|
||||
help=(
|
||||
"Maintain the realism content engine (persona pool import, "
|
||||
"future content-class tuning)."
|
||||
),
|
||||
)
|
||||
app.add_typer(realism_app, name="realism")
|
||||
|
||||
@realism_app.command("import-personas")
|
||||
def realism_import_personas(
|
||||
path: Path = typer.Argument(
|
||||
..., exists=True, file_okay=True, dir_okay=False, readable=True,
|
||||
help="JSON file containing a list of EmailPersona objects",
|
||||
),
|
||||
output: Optional[Path] = typer.Option(
|
||||
None, "--output", "-o",
|
||||
help=(
|
||||
"Override the destination path. Defaults to the canonical "
|
||||
"global pool (DECNET_REALISM_PERSONAS, /etc/decnet/"
|
||||
"email_personas.json, or ~/.decnet/email_personas.json)."
|
||||
),
|
||||
),
|
||||
) -> None:
|
||||
"""Validate + install a personas JSON file as the global pool.
|
||||
|
||||
Use this when deploying with IMAP/POP3 services on fleet
|
||||
(MACVLAN/IPVLAN) or SWARM-shard mail deckies — those have no
|
||||
parent topology row, so they read this host-wide list.
|
||||
MazeNET topology mail deckies use ``Topology.email_personas``
|
||||
instead and this command does not touch them.
|
||||
"""
|
||||
_require_master_mode("realism import-personas")
|
||||
from decnet.realism import personas_pool as global_pool
|
||||
from decnet.realism.personas import parse_personas
|
||||
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
console.print(f"[red]Cannot read {path}:[/] {exc}")
|
||||
raise typer.Exit(code=1) from exc
|
||||
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
console.print(f"[red]Invalid JSON in {path}:[/] {exc}")
|
||||
raise typer.Exit(code=1) from exc
|
||||
if not isinstance(payload, list):
|
||||
console.print(
|
||||
f"[red]{path} must contain a JSON list of personas, "
|
||||
f"got {type(payload).__name__}[/]"
|
||||
)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
personas = parse_personas(payload)
|
||||
if not personas:
|
||||
console.print(
|
||||
f"[red]No valid personas in {path}.[/] "
|
||||
"Check the schema (name, email, role, tone, mannerisms)."
|
||||
)
|
||||
raise typer.Exit(code=1)
|
||||
if len(personas) < 2:
|
||||
console.print(
|
||||
f"[yellow]Warning: only {len(personas)} valid persona(s) — "
|
||||
"the worker requires at least 2 to send mail; importing "
|
||||
"anyway in case more are added later.[/]"
|
||||
)
|
||||
|
||||
dest = output or global_pool.resolve_path()
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text(
|
||||
json.dumps(
|
||||
[p.model_dump(exclude_none=False) for p in personas],
|
||||
indent=2,
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
global_pool.reset_cache()
|
||||
console.print(
|
||||
f"[green]Imported {len(personas)} personas to[/] {dest}"
|
||||
)
|
||||
if path != dest:
|
||||
log.info("realism import-personas src=%s dest=%s", path, dest)
|
||||
@@ -1,33 +1,20 @@
|
||||
"""Emailgen — second orchestrator worker.
|
||||
"""Emailgen — email-specific delivery, scheduling, and threading.
|
||||
|
||||
Generates fake corporate emails (multi-language, threaded, persona-driven)
|
||||
and drops them into mail-decky maildirs so attackers landing on
|
||||
IMAP/POP3 honeypots find believable mailboxes instead of empty inboxes.
|
||||
After stage 5 of the realism migration, ``emailgen`` is no longer a
|
||||
separate worker / systemd unit / CLI subcommand. It exposes:
|
||||
|
||||
The module is intentionally a sibling of :mod:`decnet.orchestrator` (not
|
||||
a flag on it) — separate worker, separate CLI command
|
||||
(``decnet emailgen``), separate systemd-supervised lifecycle. Shares the
|
||||
heartbeat / control-listener scaffolding via :mod:`decnet.bus.publish`.
|
||||
* :mod:`decnet.orchestrator.emailgen.scheduler` — the
|
||||
``EmailAction`` shape and the ``pick(repo)`` policy that decides
|
||||
which mail decky / sender / recipient / thread an email belongs to.
|
||||
* :mod:`decnet.orchestrator.emailgen.threads` — RFC 2822 thread chain
|
||||
helpers (Message-ID generation, Re: / In-Reply-To bookkeeping).
|
||||
* :mod:`decnet.orchestrator.emailgen.events` — DB-row + bus-topic
|
||||
builders for email events.
|
||||
|
||||
Lazy worker re-export: :func:`emailgen_worker` is loaded on first
|
||||
attribute access so that submodules can import package-level names
|
||||
(``decnet.orchestrator.emailgen.events``) without triggering an eager
|
||||
load of the worker — and through it, the email driver, which imports
|
||||
back into this package. Without lazy loading the package + driver +
|
||||
worker form a cycle.
|
||||
The orchestrator's main worker (:mod:`decnet.orchestrator.worker`)
|
||||
calls into these modules per tick. LLM glue, persona schema, prompt
|
||||
builder, and the global persona pool moved to :mod:`decnet.realism`
|
||||
in stage 2 of the migration; this package keeps only the
|
||||
email-specific delivery surface.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover - typing only
|
||||
from decnet.orchestrator.emailgen.worker import emailgen_worker # noqa: F401
|
||||
|
||||
__all__ = ["emailgen_worker"]
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
if name == "emailgen_worker":
|
||||
from decnet.orchestrator.emailgen.worker import emailgen_worker as _w
|
||||
return _w
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
"""Emailgen main loop.
|
||||
|
||||
Mirrors :mod:`decnet.orchestrator.worker` shape: same heartbeat, same
|
||||
control listener, same fire-and-forget bus publish, same prune knob.
|
||||
A wedged ollama call stalls only this worker, never the SSH-flavoured
|
||||
orchestrator running alongside.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
|
||||
from decnet.bus.factory import get_bus
|
||||
from decnet.bus.publish import (
|
||||
publish_safely,
|
||||
run_control_listener,
|
||||
run_health_heartbeat,
|
||||
)
|
||||
from decnet.logging import get_logger
|
||||
from decnet.orchestrator.drivers.email import EmailDriver
|
||||
from decnet.orchestrator.emailgen import events, scheduler
|
||||
from decnet.web.db.repository import BaseRepository
|
||||
|
||||
logger = get_logger("orchestrator.emailgen")
|
||||
|
||||
# Periodic-prune knobs — same shape as orchestrator/worker.py.
|
||||
_PRUNE_EVERY_TICKS = 100
|
||||
_PRUNE_PER_DECKY_CAP = 5000
|
||||
|
||||
|
||||
async def emailgen_worker(
|
||||
repo: BaseRepository,
|
||||
*,
|
||||
interval: int = 300,
|
||||
model: str | None = None,
|
||||
) -> None:
|
||||
"""Periodically generate one fake email into a running mail decky.
|
||||
|
||||
Default interval is 5 minutes — emails are expensive (LLM round
|
||||
trip) and don't need to fire every minute to look natural. Honors
|
||||
``system.emailgen.control`` for graceful shutdown.
|
||||
"""
|
||||
logger.info("emailgen worker started interval=%ds model=%s", interval, model)
|
||||
|
||||
bus = None
|
||||
try:
|
||||
bus = get_bus(client_name="emailgen")
|
||||
await bus.connect()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"emailgen: bus unavailable, continuing without publish: %s", exc
|
||||
)
|
||||
bus = None
|
||||
|
||||
driver = EmailDriver(model=model) if model else EmailDriver()
|
||||
shutdown = asyncio.Event()
|
||||
heartbeat_task = asyncio.create_task(run_health_heartbeat(bus, "emailgen"))
|
||||
control_task = asyncio.create_task(
|
||||
run_control_listener(bus, "emailgen", shutdown),
|
||||
)
|
||||
tick_n = 0
|
||||
try:
|
||||
while not shutdown.is_set():
|
||||
try:
|
||||
await asyncio.wait_for(shutdown.wait(), timeout=interval)
|
||||
except asyncio.TimeoutError:
|
||||
pass # normal tick
|
||||
if shutdown.is_set():
|
||||
break
|
||||
try:
|
||||
await _one_tick(repo, driver, bus)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("emailgen tick failed: %s", exc)
|
||||
tick_n += 1
|
||||
if tick_n % _PRUNE_EVERY_TICKS == 0:
|
||||
try:
|
||||
deleted = await repo.prune_orchestrator_emails(
|
||||
per_decky_cap=_PRUNE_PER_DECKY_CAP,
|
||||
)
|
||||
if deleted:
|
||||
logger.info(
|
||||
"emailgen prune deleted=%d cap=%d",
|
||||
deleted, _PRUNE_PER_DECKY_CAP,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("emailgen prune failed: %s", exc)
|
||||
finally:
|
||||
for t in (heartbeat_task, control_task):
|
||||
t.cancel()
|
||||
with contextlib.suppress(Exception, asyncio.CancelledError):
|
||||
await t
|
||||
if bus is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await bus.close()
|
||||
|
||||
|
||||
async def _one_tick(repo: BaseRepository, driver: EmailDriver, bus) -> None:
|
||||
action = await scheduler.pick(repo)
|
||||
if action is None:
|
||||
logger.debug("emailgen: no actionable mail decky / personas this tick")
|
||||
return
|
||||
|
||||
result = await driver.run(action)
|
||||
row = events.to_row(action, result)
|
||||
await repo.record_orchestrator_email(row)
|
||||
|
||||
if bus is not None:
|
||||
topic = events.topic_for(action)
|
||||
# Mirror the orchestrator-event SSE-friendly payload shape: ts
|
||||
# as iso8601, payload as already-serialised dict.
|
||||
bus_payload = {
|
||||
"kind": "email",
|
||||
"mail_decky_uuid": row["mail_decky_uuid"],
|
||||
"thread_id": row["thread_id"],
|
||||
"message_id": row["message_id"],
|
||||
"in_reply_to": row["in_reply_to"],
|
||||
"sender_email": row["sender_email"],
|
||||
"recipient_email": row["recipient_email"],
|
||||
"subject": row["subject"],
|
||||
"language": row["language"],
|
||||
"success": row["success"],
|
||||
"ts": row["ts"].isoformat(),
|
||||
}
|
||||
await publish_safely(
|
||||
bus, topic, bus_payload, event_type=events.event_type_for(action),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"emailgen tick mail_decky=%s thread=%s success=%s reply=%s",
|
||||
row["mail_decky_uuid"], row["thread_id"], row["success"], action.is_reply,
|
||||
)
|
||||
@@ -1,16 +1,27 @@
|
||||
"""Orchestrator main loop.
|
||||
|
||||
One tick = one (src, dst, action) pick + one driver invocation + one DB
|
||||
write + one fire-and-forget bus publish. Intentionally serial — MVP
|
||||
honesty: a wedged docker exec stalls only this worker, never another.
|
||||
One tick = one action pick + one driver invocation + one DB write +
|
||||
one fire-and-forget bus publish. Intentionally serial — MVP honesty:
|
||||
a wedged docker exec stalls only this worker, never another.
|
||||
|
||||
Modeled after :mod:`decnet.profiler.worker` for consistency: same control
|
||||
listener, same heartbeat helper, same shutdown semantics.
|
||||
Three action shapes are folded into the single tick after stage 5 of
|
||||
the realism migration: SSH traffic between deckies, file plants on
|
||||
deckies (driven by :func:`decnet.realism.planner.pick`), and email
|
||||
drops into mail-decky maildirs (driven by
|
||||
:func:`decnet.orchestrator.emailgen.scheduler.pick`). ``decnet
|
||||
emailgen`` and ``decnet-emailgen.service`` are gone; this worker
|
||||
covers all three.
|
||||
|
||||
Modeled after :mod:`decnet.profiler.worker` for consistency: same
|
||||
control listener, same heartbeat helper, same shutdown semantics.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import hashlib
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from decnet.bus.factory import get_bus
|
||||
from decnet.bus.publish import (
|
||||
@@ -20,17 +31,34 @@ from decnet.bus.publish import (
|
||||
)
|
||||
from decnet.logging import get_logger
|
||||
from decnet.orchestrator import events, scheduler
|
||||
from decnet.orchestrator.drivers import SSHDriver
|
||||
from decnet.orchestrator.drivers import get_driver_for
|
||||
from decnet.orchestrator.emailgen import (
|
||||
events as email_events,
|
||||
scheduler as email_scheduler,
|
||||
)
|
||||
from decnet.orchestrator.emailgen.scheduler import EmailAction
|
||||
from decnet.web.db.repository import BaseRepository
|
||||
|
||||
logger = get_logger("orchestrator")
|
||||
|
||||
# Periodic-prune knobs. Trim per-decky history every _PRUNE_EVERY_TICKS
|
||||
# to keep orchestrator_events from unbounded growth on long-running
|
||||
# fleets. Cheap on the write path (zero overhead per tick); the cost
|
||||
# pays in once every ~100 ticks.
|
||||
# to keep orchestrator_events / orchestrator_emails from unbounded
|
||||
# growth on long-running fleets. Cheap on the write path (zero overhead
|
||||
# per tick); the cost pays in once every ~100 ticks.
|
||||
_PRUNE_EVERY_TICKS = 100
|
||||
_PRUNE_PER_DST_CAP = 10000
|
||||
_PRUNE_PER_MAIL_DECKY_CAP = 5000
|
||||
|
||||
# Action-kind weights for the per-tick roll. Email is rare because
|
||||
# each LLM round-trip is expensive (~seconds) and the prior emailgen
|
||||
# worker only ticked every 5 minutes. At a 60s orchestrator interval,
|
||||
# a 10% email weight produces ~one email every ~10 minutes — close
|
||||
# enough to the pre-collapse cadence.
|
||||
_ACTION_WEIGHTS: tuple[tuple[str, int], ...] = (
|
||||
("traffic", 45),
|
||||
("file", 45),
|
||||
("email", 10),
|
||||
)
|
||||
|
||||
|
||||
async def orchestrator_worker(
|
||||
@@ -55,7 +83,6 @@ async def orchestrator_worker(
|
||||
)
|
||||
bus = None
|
||||
|
||||
driver = SSHDriver()
|
||||
shutdown = asyncio.Event()
|
||||
heartbeat_task = asyncio.create_task(run_health_heartbeat(bus, "orchestrator"))
|
||||
control_task = asyncio.create_task(
|
||||
@@ -71,22 +98,12 @@ async def orchestrator_worker(
|
||||
if shutdown.is_set():
|
||||
break
|
||||
try:
|
||||
await _one_tick(repo, driver, bus)
|
||||
await _one_tick(repo, bus)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("orchestrator tick failed: %s", exc)
|
||||
tick_n += 1
|
||||
if tick_n % _PRUNE_EVERY_TICKS == 0:
|
||||
try:
|
||||
deleted = await repo.prune_orchestrator_events(
|
||||
per_dst_cap=_PRUNE_PER_DST_CAP,
|
||||
)
|
||||
if deleted:
|
||||
logger.info(
|
||||
"orchestrator prune deleted=%d cap=%d",
|
||||
deleted, _PRUNE_PER_DST_CAP,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("orchestrator prune failed: %s", exc)
|
||||
await _periodic_prune(repo)
|
||||
finally:
|
||||
for t in (heartbeat_task, control_task):
|
||||
t.cancel()
|
||||
@@ -97,34 +114,80 @@ async def orchestrator_worker(
|
||||
await bus.close()
|
||||
|
||||
|
||||
async def _one_tick(repo: BaseRepository, driver, bus) -> None:
|
||||
import secrets as _secrets
|
||||
async def _periodic_prune(repo: BaseRepository) -> None:
|
||||
try:
|
||||
deleted = await repo.prune_orchestrator_events(per_dst_cap=_PRUNE_PER_DST_CAP)
|
||||
if deleted:
|
||||
logger.info(
|
||||
"orchestrator events prune deleted=%d cap=%d",
|
||||
deleted, _PRUNE_PER_DST_CAP,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("orchestrator events prune failed: %s", exc)
|
||||
try:
|
||||
deleted = await repo.prune_orchestrator_emails(
|
||||
per_decky_cap=_PRUNE_PER_MAIL_DECKY_CAP,
|
||||
)
|
||||
if deleted:
|
||||
logger.info(
|
||||
"orchestrator emails prune deleted=%d cap=%d",
|
||||
deleted, _PRUNE_PER_MAIL_DECKY_CAP,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("orchestrator emails prune failed: %s", exc)
|
||||
|
||||
# Union view: MazeNET topology + unihost fleet + SWARM shards. Pre-fleet
|
||||
# this only saw topology_deckies and was permanently blind to MACVLAN /
|
||||
# IPVLAN unihost decoys.
|
||||
deckies = await repo.list_running_deckies()
|
||||
rng = _secrets.SystemRandom()
|
||||
|
||||
# Action-kind roll: 50/50 traffic vs file. Stage 5 of the realism
|
||||
# migration adds an email branch (when emailgen folds in). When a
|
||||
# roll yields nothing actionable (e.g. file branch with no personas
|
||||
# in any persona's work hours), we fall through to the other side
|
||||
# so a quiet half doesn't silence the whole tick.
|
||||
action = None
|
||||
if rng.random() < 0.5:
|
||||
action = scheduler.pick(deckies, rand=rng)
|
||||
if action is None:
|
||||
action = await scheduler.pick_file(deckies, repo, rand=rng)
|
||||
else:
|
||||
action = await scheduler.pick_file(deckies, repo, rand=rng)
|
||||
if action is None:
|
||||
def _roll_action_kind(rng: secrets.SystemRandom) -> str:
|
||||
total = sum(w for _, w in _ACTION_WEIGHTS)
|
||||
target = rng.randint(1, total)
|
||||
running = 0
|
||||
for kind, w in _ACTION_WEIGHTS:
|
||||
running += w
|
||||
if target <= running:
|
||||
return kind
|
||||
return _ACTION_WEIGHTS[-1][0] # unreachable, satisfy mypy
|
||||
|
||||
|
||||
async def _pick_action(
|
||||
repo: BaseRepository,
|
||||
deckies: list[dict],
|
||||
rng: secrets.SystemRandom,
|
||||
):
|
||||
"""Roll an action-kind, then pick the matching action.
|
||||
|
||||
Quiet branches fall through to the other two so a (decky-set,
|
||||
persona-pool, mail-decky) shape that would silence one branch
|
||||
doesn't waste the whole tick.
|
||||
"""
|
||||
kinds_in_priority_order = [_roll_action_kind(rng)]
|
||||
for kind, _ in _ACTION_WEIGHTS:
|
||||
if kind not in kinds_in_priority_order:
|
||||
kinds_in_priority_order.append(kind)
|
||||
|
||||
for kind in kinds_in_priority_order:
|
||||
if kind == "traffic":
|
||||
action = scheduler.pick(deckies, rand=rng)
|
||||
elif kind == "file":
|
||||
action = await scheduler.pick_file(deckies, repo, rand=rng)
|
||||
elif kind == "email":
|
||||
try:
|
||||
action = await email_scheduler.pick(repo, rand=rng)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("orchestrator: email pick failed: %s", exc)
|
||||
action = None
|
||||
else:
|
||||
action = None
|
||||
if action is not None:
|
||||
return action
|
||||
return None
|
||||
|
||||
|
||||
async def _one_tick(repo: BaseRepository, bus) -> None:
|
||||
deckies = await repo.list_running_deckies()
|
||||
rng = secrets.SystemRandom()
|
||||
|
||||
action = await _pick_action(repo, deckies, rng)
|
||||
if action is None:
|
||||
# Report the actual SSH-eligible count (what the scheduler filters
|
||||
# to), not just len(deckies) — the old "running+ssh count=N" line
|
||||
# reported the pre-filter count and misled debugging.
|
||||
ssh_eligible = sum(
|
||||
1 for d in deckies
|
||||
if isinstance(d.get("services"), list)
|
||||
@@ -133,9 +196,8 @@ async def _one_tick(repo: BaseRepository, driver, bus) -> None:
|
||||
)
|
||||
by_source: dict[str, int] = {}
|
||||
for d in deckies:
|
||||
by_source[d.get("source", "unknown")] = (
|
||||
by_source.get(d.get("source", "unknown"), 0) + 1
|
||||
)
|
||||
src = d.get("source", "unknown")
|
||||
by_source[src] = by_source.get(src, 0) + 1
|
||||
logger.debug(
|
||||
"orchestrator: no actionable deckies "
|
||||
"(running=%d ssh_eligible=%d sources=%s)",
|
||||
@@ -143,26 +205,29 @@ async def _one_tick(repo: BaseRepository, driver, bus) -> None:
|
||||
)
|
||||
return
|
||||
|
||||
driver = get_driver_for(action)
|
||||
result = await driver.run(action)
|
||||
|
||||
if isinstance(action, EmailAction):
|
||||
await _persist_email(repo, action, result, bus)
|
||||
else:
|
||||
await _persist_event(repo, action, result, bus)
|
||||
if isinstance(action, scheduler.FileAction) and result.success:
|
||||
try:
|
||||
await _record_synthetic_file(repo, action)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"orchestrator: synthetic_files write failed dst=%s path=%s: %s",
|
||||
action.dst_uuid, action.path, exc,
|
||||
)
|
||||
|
||||
|
||||
async def _persist_event(repo, action, result, bus) -> None:
|
||||
row = events.to_row(action, result)
|
||||
await repo.record_orchestrator_event(row)
|
||||
# Persist realism state for FileAction so stage 3b's edit-in-place
|
||||
# has something to read back. Failure here is logged but doesn't
|
||||
# tank the tick — the orchestrator event is the source of truth
|
||||
# for "this action happened."
|
||||
if isinstance(action, scheduler.FileAction) and result.success:
|
||||
try:
|
||||
await _record_synthetic_file(repo, action, result)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"orchestrator: synthetic_files write failed dst=%s path=%s: %s",
|
||||
action.dst_uuid, action.path, exc,
|
||||
)
|
||||
|
||||
if bus is not None:
|
||||
topic = events.topic_for(action)
|
||||
# Bus payload mirrors the row but uses iso8601 for ts so SSE
|
||||
# consumers don't have to JSON-handle datetime themselves.
|
||||
bus_payload = {
|
||||
"kind": row["kind"],
|
||||
"protocol": row["protocol"],
|
||||
@@ -174,7 +239,7 @@ async def _one_tick(repo: BaseRepository, driver, bus) -> None:
|
||||
"ts": row["ts"].isoformat(),
|
||||
}
|
||||
await publish_safely(
|
||||
bus, topic, bus_payload, event_type=events.event_type_for(action)
|
||||
bus, topic, bus_payload, event_type=events.event_type_for(action),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
@@ -183,19 +248,52 @@ async def _one_tick(repo: BaseRepository, driver, bus) -> None:
|
||||
)
|
||||
|
||||
|
||||
async def _record_synthetic_file(repo, action, result) -> None:
|
||||
"""Persist a synthetic_files row after a successful FileAction plant.
|
||||
async def _persist_email(repo, action: EmailAction, result, bus) -> None:
|
||||
"""Persist + publish an email tick result.
|
||||
|
||||
Mirrors the pre-collapse emailgen worker payload exactly so SSE
|
||||
subscribers and dashboards keep working without a breaking change
|
||||
to the on-the-wire shape.
|
||||
"""
|
||||
row = email_events.to_row(action, result)
|
||||
await repo.record_orchestrator_email(row)
|
||||
|
||||
if bus is not None:
|
||||
topic = email_events.topic_for(action)
|
||||
bus_payload = {
|
||||
"kind": "email",
|
||||
"mail_decky_uuid": row["mail_decky_uuid"],
|
||||
"thread_id": row["thread_id"],
|
||||
"message_id": row["message_id"],
|
||||
"in_reply_to": row["in_reply_to"],
|
||||
"sender_email": row["sender_email"],
|
||||
"recipient_email": row["recipient_email"],
|
||||
"subject": row["subject"],
|
||||
"language": row["language"],
|
||||
"success": row["success"],
|
||||
"ts": row["ts"].isoformat(),
|
||||
}
|
||||
await publish_safely(
|
||||
bus, topic, bus_payload,
|
||||
event_type=email_events.event_type_for(action),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"orchestrator tick kind=email mail_decky=%s thread=%s success=%s reply=%s",
|
||||
row["mail_decky_uuid"], row["thread_id"], row["success"], action.is_reply,
|
||||
)
|
||||
|
||||
|
||||
async def _record_synthetic_file(repo, action) -> None:
|
||||
"""Persist (or patch) a synthetic_files row after a FileAction plant.
|
||||
|
||||
Idempotent on ``(decky_uuid, path)``: when the unique constraint
|
||||
fires (the file existed already), we instead patch the existing
|
||||
row's ``last_modified`` / ``content_hash`` / ``last_body`` / bump
|
||||
fires (the file existed already), we patch the existing row's
|
||||
``last_modified`` / ``content_hash`` / ``last_body`` / bump
|
||||
``edit_count`` so the dashboard's "files this decky has grown"
|
||||
view stays accurate even when the orchestrator re-plants the same
|
||||
location.
|
||||
"""
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
|
||||
body = action.content or ""
|
||||
content_hash = hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||||
now = datetime.now(timezone.utc)
|
||||
@@ -216,8 +314,6 @@ async def _record_synthetic_file(repo, action, result) -> None:
|
||||
try:
|
||||
await repo.record_synthetic_file(row)
|
||||
except Exception: # noqa: BLE001
|
||||
# Most likely the unique constraint on (decky_uuid, path)
|
||||
# fired — flip to update mode by looking up the existing row.
|
||||
existing = await repo.list_synthetic_files(
|
||||
decky_uuid=action.dst_uuid, limit=200,
|
||||
)
|
||||
|
||||
@@ -31,7 +31,7 @@ from .campaigns.api_list_campaign_identities import router as campaign_identitie
|
||||
from .campaigns.api_events import router as campaign_events_router
|
||||
from .orchestrator.api_list_events import router as orchestrator_list_router
|
||||
from .orchestrator.api_events import router as orchestrator_events_router
|
||||
from .emailgen.api_personas import router as emailgen_personas_router
|
||||
from .realism.api_personas import router as realism_personas_router
|
||||
from .transcripts import transcripts_router
|
||||
from .config.api_get_config import router as config_get_router
|
||||
from .config.api_update_config import router as config_update_router
|
||||
@@ -111,10 +111,10 @@ api_router.include_router(campaign_events_router)
|
||||
api_router.include_router(orchestrator_list_router)
|
||||
api_router.include_router(orchestrator_events_router)
|
||||
|
||||
# Emailgen — global persona pool CRUD for the dashboard's
|
||||
# "Persona Generation" page. The worker reads from the same on-disk
|
||||
# JSON file directly (see decnet.realism.personas_pool).
|
||||
api_router.include_router(emailgen_personas_router)
|
||||
# Realism — global persona pool CRUD for the dashboard's
|
||||
# "Persona Generation" page. The orchestrator reads from the same
|
||||
# on-disk JSON file directly (see decnet.realism.personas_pool).
|
||||
api_router.include_router(realism_personas_router)
|
||||
|
||||
# Observability
|
||||
api_router.include_router(stats_router)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""GET/PUT ``/api/v1/emailgen/personas`` — global persona pool CRUD.
|
||||
"""GET/PUT ``/api/v1/realism/personas`` — global persona pool CRUD.
|
||||
|
||||
The "global pool" is a JSON file consumed by the realism content
|
||||
engine for fleet (MACVLAN/IPVLAN) and SWARM-shard deckies — see
|
||||
@@ -29,7 +29,7 @@ from decnet.web.dependencies import require_admin, require_viewer
|
||||
from decnet.web.db.models.common import MessageResponse # noqa: F401 - response shape
|
||||
|
||||
router = APIRouter()
|
||||
log = get_logger("api.emailgen.personas")
|
||||
log = get_logger("api.realism.personas")
|
||||
|
||||
|
||||
def _serialize(personas: list[EmailPersona]) -> list[dict[str, Any]]:
|
||||
@@ -38,14 +38,14 @@ def _serialize(personas: list[EmailPersona]) -> list[dict[str, Any]]:
|
||||
|
||||
|
||||
@router.get(
|
||||
"/emailgen/personas",
|
||||
"/realism/personas",
|
||||
tags=["Emailgen"],
|
||||
responses={
|
||||
401: {"description": "Could not validate credentials"},
|
||||
403: {"description": "Insufficient permissions"},
|
||||
},
|
||||
)
|
||||
@_traced("api.emailgen.list_personas")
|
||||
@_traced("api.realism.list_personas")
|
||||
async def list_personas(
|
||||
user: dict = Depends(require_viewer),
|
||||
) -> dict[str, Any]:
|
||||
@@ -56,7 +56,7 @@ async def list_personas(
|
||||
discoverable.
|
||||
"""
|
||||
# Reset the in-process cache before reading so a fresh CLI-driven
|
||||
# ``decnet emailgen import-personas`` shows up immediately rather
|
||||
# ``decnet realism import-personas`` shows up immediately rather
|
||||
# than waiting on the worker's mtime check.
|
||||
global_pool.reset_cache()
|
||||
personas = global_pool.load()
|
||||
@@ -67,7 +67,7 @@ async def list_personas(
|
||||
|
||||
|
||||
@router.put(
|
||||
"/emailgen/personas",
|
||||
"/realism/personas",
|
||||
tags=["Emailgen"],
|
||||
responses={
|
||||
400: {"description": "Invalid persona payload"},
|
||||
@@ -75,7 +75,7 @@ async def list_personas(
|
||||
403: {"description": "Insufficient permissions"},
|
||||
},
|
||||
)
|
||||
@_traced("api.emailgen.replace_personas")
|
||||
@_traced("api.realism.replace_personas")
|
||||
async def replace_personas(
|
||||
body: dict[str, Any],
|
||||
user: dict = Depends(require_admin),
|
||||
@@ -121,7 +121,7 @@ async def replace_personas(
|
||||
# not writable by the API process. Surface a 500 with the
|
||||
# actionable hint instead of leaking a traceback.
|
||||
log.warning(
|
||||
"api.emailgen.replace_personas write failed path=%s err=%s",
|
||||
"api.realism.replace_personas write failed path=%s err=%s",
|
||||
dest, exc,
|
||||
)
|
||||
raise HTTPException(
|
||||
@@ -134,7 +134,7 @@ async def replace_personas(
|
||||
) from exc
|
||||
global_pool.reset_cache()
|
||||
log.info(
|
||||
"api.emailgen.replace_personas user=%s wrote=%d path=%s",
|
||||
"api.realism.replace_personas user=%s wrote=%d path=%s",
|
||||
user.get("username", user.get("uuid")), len(parsed), dest,
|
||||
)
|
||||
return {
|
||||
@@ -3,7 +3,7 @@ import { useSearchParams, useNavigate } from 'react-router-dom';
|
||||
import {
|
||||
PanelRightOpen, PanelRightClose, PanelLeftOpen, PanelLeftClose,
|
||||
Maximize2, Minimize2, RotateCcw, UploadCloud, ArrowLeft,
|
||||
Plus, Trash2, Zap, Copy, Eye, ShieldAlert, GitMerge, Server,
|
||||
Plus, Trash2, Zap, Copy, Eye, ShieldAlert, GitMerge, Server, Mail,
|
||||
} from '../../icons';
|
||||
import './MazeNET.css';
|
||||
import axios from '../../utils/api';
|
||||
@@ -707,6 +707,15 @@ const MazeNET: React.FC = () => {
|
||||
<button type="button" className="maze-btn ghost" onClick={refetch} title="Revert local state to server">
|
||||
<RotateCcw size={12} /> REFRESH
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="maze-btn ghost"
|
||||
onClick={() => navigate(`/topologies/${topologyId}/personas`)}
|
||||
disabled={!topologyId}
|
||||
title="Edit email personas for this topology"
|
||||
>
|
||||
<Mail size={12} /> PERSONAS
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="maze-btn"
|
||||
|
||||
@@ -1,60 +1,8 @@
|
||||
/* Persona Generation page — global pool CRUD UI for fleet/shard mail
|
||||
deckies' fake-employee personas. Mirrors Webhooks.css visual language
|
||||
(chips, table, modal) so dashboard density stays consistent. */
|
||||
/* Persona Generation — layered on top of DeckyFleet.css.
|
||||
Only persona-specific bits live here: tone chips, info banner,
|
||||
dirty pill, modal form inputs, empty state, mono helper. */
|
||||
|
||||
.persona-gen-root {
|
||||
padding: 18px 24px;
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
.persona-gen-root .page-header {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.persona-gen-root .page-title-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.persona-gen-root .header-line {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.persona-gen-root .header-line h1 {
|
||||
font-size: 1.1rem;
|
||||
letter-spacing: 2px;
|
||||
margin: 0;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.persona-gen-root .page-sub {
|
||||
font-size: 0.7rem;
|
||||
letter-spacing: 1.4px;
|
||||
color: var(--dim);
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.persona-gen-root .violet-accent { color: var(--violet); }
|
||||
|
||||
.persona-gen-root .dirty-pill {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 2px 8px;
|
||||
border-radius: 999px;
|
||||
background: rgba(245, 158, 11, 0.12);
|
||||
color: var(--amber, #f59e0b);
|
||||
font-size: 0.62rem;
|
||||
letter-spacing: 1.4px;
|
||||
text-transform: uppercase;
|
||||
border: 1px solid var(--amber, #f59e0b);
|
||||
}
|
||||
.persona-gen-root .mono { font-family: var(--font-mono); }
|
||||
|
||||
/* Info banner explaining scope (non-MazeNET) + showing pool path. */
|
||||
.persona-gen-root .info-banner {
|
||||
@@ -62,115 +10,28 @@
|
||||
border: 1px solid var(--border);
|
||||
border-left: 3px solid var(--violet);
|
||||
padding: 10px 14px;
|
||||
margin-bottom: 14px;
|
||||
font-size: 0.78rem;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.persona-gen-root .info-banner em { color: var(--matrix); font-style: normal; }
|
||||
.persona-gen-root .info-line { margin-top: 6px; font-size: 0.72rem; }
|
||||
|
||||
/* Action row */
|
||||
.persona-gen-root .controls-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
margin-bottom: 14px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.persona-gen-root .btn {
|
||||
.persona-gen-root .dirty-pill {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 5px 12px;
|
||||
background: var(--bg-elev);
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text);
|
||||
font-size: 0.7rem;
|
||||
letter-spacing: 1.5px;
|
||||
text-transform: uppercase;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
|
||||
.persona-gen-root .btn:hover:not(:disabled) {
|
||||
border-color: var(--matrix);
|
||||
color: var(--matrix);
|
||||
}
|
||||
|
||||
.persona-gen-root .btn:disabled { opacity: 0.4; cursor: not-allowed; }
|
||||
.persona-gen-root .btn.primary { border-color: var(--matrix); color: var(--matrix); }
|
||||
.persona-gen-root .btn.primary:hover:not(:disabled) { background: rgba(0, 255, 65, 0.07); }
|
||||
.persona-gen-root .btn.ghost { background: transparent; }
|
||||
|
||||
.persona-gen-root .error-line {
|
||||
color: var(--alert);
|
||||
font-size: 0.72rem;
|
||||
display: inline-flex;
|
||||
gap: 6px;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
/* Table */
|
||||
.persona-gen-root .persona-list {
|
||||
border: 1px solid var(--border);
|
||||
background: rgba(0, 0, 0, 0.18);
|
||||
}
|
||||
|
||||
.persona-gen-root .persona-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 0.78rem;
|
||||
}
|
||||
|
||||
.persona-gen-root .persona-table thead th {
|
||||
text-align: left;
|
||||
padding: 8px 10px;
|
||||
padding: 2px 10px;
|
||||
border-radius: 999px;
|
||||
background: rgba(245, 158, 11, 0.12);
|
||||
color: var(--amber, #f59e0b);
|
||||
font-size: 0.62rem;
|
||||
letter-spacing: 1.4px;
|
||||
font-size: 0.66rem;
|
||||
color: var(--dim);
|
||||
text-transform: uppercase;
|
||||
border-bottom: 1px solid var(--border);
|
||||
background: rgba(255, 255, 255, 0.02);
|
||||
border: 1px solid var(--amber, #f59e0b);
|
||||
width: fit-content;
|
||||
}
|
||||
|
||||
.persona-gen-root .persona-table tbody td {
|
||||
padding: 8px 10px;
|
||||
border-bottom: 1px dashed var(--border);
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
.persona-gen-root .persona-table tbody tr:hover {
|
||||
background: rgba(0, 255, 65, 0.04);
|
||||
}
|
||||
|
||||
.persona-gen-root .row-actions {
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
justify-content: flex-end;
|
||||
}
|
||||
|
||||
.persona-gen-root .icon-btn {
|
||||
background: transparent;
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text);
|
||||
padding: 4px 6px;
|
||||
cursor: pointer;
|
||||
transition: all 0.12s;
|
||||
}
|
||||
|
||||
.persona-gen-root .icon-btn:hover {
|
||||
border-color: var(--matrix);
|
||||
color: var(--matrix);
|
||||
}
|
||||
|
||||
.persona-gen-root .icon-btn.danger:hover {
|
||||
border-color: var(--alert);
|
||||
color: var(--alert);
|
||||
}
|
||||
|
||||
/* Tone chips — subtle palette differentiation */
|
||||
/* Tone chips — keep palette differentiation from the original page. */
|
||||
.persona-gen-root .tone-chip {
|
||||
display: inline-block;
|
||||
padding: 1px 8px;
|
||||
@@ -179,197 +40,68 @@
|
||||
letter-spacing: 1.5px;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.persona-gen-root .tone-chip.tone-formal { border-color: var(--violet); color: var(--violet); }
|
||||
.persona-gen-root .tone-chip.tone-direct { border-color: var(--matrix); color: var(--matrix); }
|
||||
.persona-gen-root .tone-chip.tone-casual { border-color: var(--amber, #f59e0b); color: var(--amber, #f59e0b); }
|
||||
.persona-gen-root .tone-chip.tone-technical { border-color: var(--cyan, #22d3ee); color: var(--cyan, #22d3ee); }
|
||||
.persona-gen-root .tone-chip.tone-custom { border-color: var(--alert, #ef4444); color: var(--alert, #ef4444); text-transform: none; letter-spacing: 0.5px; }
|
||||
|
||||
.persona-gen-root .chip {
|
||||
display: inline-block;
|
||||
padding: 1px 8px;
|
||||
border: 1px solid var(--border);
|
||||
font-size: 0.62rem;
|
||||
letter-spacing: 1.4px;
|
||||
text-transform: uppercase;
|
||||
/* Card tweaks — wrap email if long. */
|
||||
.persona-card .decky-ip {
|
||||
font-size: 0.65rem;
|
||||
letter-spacing: 0.4px;
|
||||
word-break: break-all;
|
||||
max-width: 60%;
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.persona-gen-root .dim-chip { color: var(--dim); }
|
||||
.persona-gen-root .warn-chip {
|
||||
border-color: var(--amber, #f59e0b);
|
||||
color: var(--amber, #f59e0b);
|
||||
}
|
||||
|
||||
.persona-gen-root .mono { font-family: var(--font-mono); font-size: 0.74rem; }
|
||||
.persona-gen-root .matrix-text { color: var(--matrix); }
|
||||
.persona-gen-root .dim { color: var(--dim); }
|
||||
|
||||
/* ── Modal ──────────────────────────────────────────────────────── */
|
||||
.persona-modal-backdrop {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.65);
|
||||
/* Empty state inside the grid. */
|
||||
.fleet-root .fleet-empty {
|
||||
grid-column: 1 / -1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 50;
|
||||
backdrop-filter: blur(2px);
|
||||
}
|
||||
|
||||
.persona-modal {
|
||||
background: var(--bg);
|
||||
border: 1px solid var(--border);
|
||||
width: min(640px, 92vw);
|
||||
max-height: 90vh;
|
||||
overflow-y: auto;
|
||||
box-shadow: 0 0 30px rgba(0, 0, 0, 0.6);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.persona-modal .bd-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 12px 16px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.persona-modal .bd-head h3 {
|
||||
margin: 0;
|
||||
font-size: 0.85rem;
|
||||
letter-spacing: 1.6px;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.persona-modal .close-btn {
|
||||
background: transparent;
|
||||
border: none;
|
||||
color: var(--dim);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.persona-modal .close-btn:hover { color: var(--alert); }
|
||||
|
||||
.persona-modal .bd-body {
|
||||
padding: 16px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.persona-modal .field {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.persona-modal .field-row {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr 1fr;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.persona-modal .field-label {
|
||||
font-size: 0.66rem;
|
||||
letter-spacing: 1.4px;
|
||||
color: var(--dim);
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.persona-modal .field input[type="text"],
|
||||
.persona-modal .field input[type="email"],
|
||||
.persona-modal .field select,
|
||||
.persona-modal .field textarea {
|
||||
background: var(--bg-elev);
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text);
|
||||
padding: 6px 10px;
|
||||
font-size: 0.82rem;
|
||||
font-family: inherit;
|
||||
}
|
||||
|
||||
.persona-modal .field input:focus,
|
||||
.persona-modal .field select:focus,
|
||||
.persona-modal .field textarea:focus {
|
||||
outline: none;
|
||||
border-color: var(--matrix);
|
||||
}
|
||||
|
||||
.persona-modal .check-field {
|
||||
flex-direction: row;
|
||||
align-items: flex-start;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.persona-modal .check-field input[type="checkbox"] {
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.persona-modal .mannerism-input-row {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.persona-modal .mannerism-input-row input { flex: 1; }
|
||||
|
||||
.persona-modal .mannerism-list {
|
||||
list-style: none;
|
||||
margin: 8px 0 0;
|
||||
padding: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.persona-modal .mannerism-list li {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
padding: 4px 8px;
|
||||
border: 1px solid var(--border);
|
||||
background: rgba(255, 255, 255, 0.02);
|
||||
gap: 14px;
|
||||
padding: 60px 24px;
|
||||
border: 1px dashed var(--border);
|
||||
background: rgba(0, 0, 0, 0.18);
|
||||
text-align: center;
|
||||
letter-spacing: 1.5px;
|
||||
font-size: 0.78rem;
|
||||
}
|
||||
|
||||
.persona-modal .mannerism-list li span {
|
||||
flex: 1;
|
||||
word-break: break-word;
|
||||
/* Modal form inputs — reuse the wizard's tweak-group / input language. */
|
||||
.persona-gen-root .grid-2,
|
||||
.modal .grid-2 {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.persona-modal .draft-error {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
color: var(--alert);
|
||||
font-size: 0.74rem;
|
||||
}
|
||||
|
||||
.persona-modal .bd-actions {
|
||||
.modal .tweak-group {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
gap: 8px;
|
||||
padding: 12px 16px;
|
||||
border-top: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.persona-modal .btn {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
padding: 6px 14px;
|
||||
background: var(--bg-elev);
|
||||
}
|
||||
.modal .tweak-group label {
|
||||
font-size: 0.62rem;
|
||||
letter-spacing: 1.5px;
|
||||
color: var(--dim);
|
||||
text-transform: uppercase;
|
||||
}
|
||||
.modal .input {
|
||||
background: var(--bg-elev, rgba(0, 0, 0, 0.3));
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text);
|
||||
font-size: 0.7rem;
|
||||
letter-spacing: 1.5px;
|
||||
text-transform: uppercase;
|
||||
cursor: pointer;
|
||||
padding: 7px 10px;
|
||||
font-family: inherit;
|
||||
font-size: 0.8rem;
|
||||
outline: none;
|
||||
transition: border-color 0.15s;
|
||||
}
|
||||
|
||||
.persona-modal .btn.primary { border-color: var(--matrix); color: var(--matrix); }
|
||||
.persona-modal .btn.ghost { background: transparent; }
|
||||
.persona-modal .btn:hover:not(:disabled) { border-color: var(--matrix); color: var(--matrix); }
|
||||
.modal .input:focus {
|
||||
border-color: var(--violet);
|
||||
box-shadow: 0 0 0 1px var(--violet);
|
||||
}
|
||||
.modal select.input { cursor: pointer; }
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
import React, { useCallback, useEffect, useState } from 'react';
|
||||
import { useNavigate } from 'react-router-dom';
|
||||
import { Network, Plus, Power, Trash2, UploadCloud, RefreshCw, Skull, Server, Cpu } from '../../icons';
|
||||
import { Network, Plus, Power, Trash2, UploadCloud, RefreshCw, Skull, Server, Cpu, Mail } from '../../icons';
|
||||
import api from '../../utils/api';
|
||||
import { useSwarmHosts } from '../../hooks/useSwarmHosts';
|
||||
import { clearLayout } from '../MazeNET/useMazeLayoutStore';
|
||||
@@ -223,6 +223,14 @@ const TopologyList: React.FC = () => {
|
||||
</div>
|
||||
<div className="tlist-card-id">{r.id}</div>
|
||||
<div className="tlist-card-actions" onClick={(e) => e.stopPropagation()}>
|
||||
<button
|
||||
type="button"
|
||||
className="tlist-btn small"
|
||||
onClick={() => navigate(`/topologies/${r.id}/personas`)}
|
||||
title="Edit email personas for this topology"
|
||||
>
|
||||
<Mail size={10} /> PERSONAS
|
||||
</button>
|
||||
{r.status === 'pending' && (
|
||||
<button
|
||||
type="button"
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
[Unit]
|
||||
Description=DECNET Emailgen (LLM-driven fake corporate email into IMAP/POP3 deckies)
|
||||
Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#emailgen
|
||||
After=network-online.target decnet-bus.service
|
||||
Wants=network-online.target decnet-bus.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ user }}
|
||||
Group={{ group }}
|
||||
WorkingDirectory={{ install_dir }}
|
||||
EnvironmentFile=-{{ install_dir }}/.env.local
|
||||
Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.emailgen.log
|
||||
# LLM backend selection + model are operator-tunable via .env.local:
|
||||
# DECNET_EMAILGEN_LLM=ollama|fake (default: ollama)
|
||||
# DECNET_EMAILGEN_MODEL=llama3.1 (default: llama3.1)
|
||||
# DECNET_EMAILGEN_TIMEOUT=60 (LLM wall-clock cap, seconds)
|
||||
# DECNET_EMAILGEN_PERSONAS=/etc/decnet/email_personas.json
|
||||
# (override the global persona pool)
|
||||
ExecStart={{ venv_dir }}/bin/decnet emailgen run
|
||||
StandardOutput=append:/var/log/decnet/decnet.emailgen.log
|
||||
StandardError=append:/var/log/decnet/decnet.emailgen.log
|
||||
|
||||
# Emailgen drives `docker exec` against IMAP/POP3 decky containers to drop
|
||||
# .eml files into the spool, identical to the SSH-flavoured orchestrator.
|
||||
# It does NOT bind to the network, launch new containers, or write outside
|
||||
# its own logs and install dir.
|
||||
SupplementaryGroups=docker
|
||||
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
|
||||
# Security Hardening
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=full
|
||||
ProtectHome=read-only
|
||||
PrivateTmp=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
# /etc/decnet is included so `decnet emailgen import-personas` can write
|
||||
# the canonical /etc/decnet/email_personas.json without the worker losing
|
||||
# read access (it lives outside ReadWritePaths so writes from the worker
|
||||
# itself are still blocked — only the operator-run CLI writes here).
|
||||
ReadWritePaths={{ install_dir }} /var/log/decnet
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=15
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=DECNET Orchestrator (synthetic life-injection — inter-decky traffic + file ops)
|
||||
Description=DECNET Orchestrator (synthetic life-injection — inter-decky traffic, file plants, email drops)
|
||||
Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#orchestrator
|
||||
After=network-online.target decnet-bus.service
|
||||
Wants=network-online.target decnet-bus.service
|
||||
@@ -11,6 +11,13 @@ Group={{ group }}
|
||||
WorkingDirectory={{ install_dir }}
|
||||
EnvironmentFile=-{{ install_dir }}/.env.local
|
||||
Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.orchestrator.log
|
||||
# Realism content engine — LLM + persona-pool config used by the
|
||||
# email + (post-stage-6) file-class enrichment paths. See
|
||||
# decnet/realism/llm/factory.py and decnet/realism/personas_pool.py.
|
||||
Environment=DECNET_REALISM_LLM=ollama
|
||||
Environment=DECNET_REALISM_MODEL=llama3.1
|
||||
Environment=DECNET_REALISM_TIMEOUT=60
|
||||
Environment=DECNET_REALISM_PERSONAS=/etc/decnet/email_personas.json
|
||||
ExecStart={{ venv_dir }}/bin/decnet orchestrate
|
||||
StandardOutput=append:/var/log/decnet/decnet.orchestrator.log
|
||||
StandardError=append:/var/log/decnet/decnet.orchestrator.log
|
||||
|
||||
@@ -20,8 +20,7 @@ Wants=decnet-bus.service \
|
||||
decnet-campaign-clusterer.service \
|
||||
decnet-webhook.service \
|
||||
decnet-canary.service \
|
||||
decnet-orchestrator.service \
|
||||
decnet-emailgen.service
|
||||
decnet-orchestrator.service
|
||||
After=decnet-bus.service
|
||||
|
||||
[Install]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""GET/PUT /api/v1/emailgen/personas — global persona pool CRUD."""
|
||||
"""GET/PUT /api/v1/realism/personas — global persona pool CRUD."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
@@ -6,7 +6,7 @@ import json
|
||||
import pytest
|
||||
|
||||
from decnet.realism import personas_pool as global_pool
|
||||
from decnet.web.router.emailgen.api_personas import (
|
||||
from decnet.web.router.realism.api_personas import (
|
||||
list_personas,
|
||||
replace_personas,
|
||||
)
|
||||
180
tests/api/topology/test_personas_api.py
Normal file
180
tests/api/topology/test_personas_api.py
Normal file
@@ -0,0 +1,180 @@
|
||||
"""Per-topology persona endpoints — GET/PUT /topologies/{id}/personas."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from decnet.topology.config import TopologyConfig
|
||||
from decnet.topology.generator import generate
|
||||
from decnet.topology.persistence import persist
|
||||
from decnet.web.dependencies import repo as _repo
|
||||
|
||||
_V1 = "/api/v1/topologies"
|
||||
|
||||
|
||||
def _cfg(name: str = "personas") -> TopologyConfig:
|
||||
return TopologyConfig(
|
||||
name=name,
|
||||
depth=1,
|
||||
branching_factor=1,
|
||||
deckies_per_lan_min=1,
|
||||
deckies_per_lan_max=1,
|
||||
services_explicit=["ssh"],
|
||||
randomize_services=False,
|
||||
seed=0,
|
||||
)
|
||||
|
||||
|
||||
async def _seed(name: str = "personas") -> str:
|
||||
return await persist(_repo, generate(_cfg(name)))
|
||||
|
||||
|
||||
def _persona(email: str, name: str = "Jane Doe") -> dict:
|
||||
return {
|
||||
"name": name,
|
||||
"email": email,
|
||||
"role": "Admin",
|
||||
"tone": "formal",
|
||||
"mannerisms": ["uses bullet points"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_get_default_empty(client, auth_token):
|
||||
tid = await _seed("get-empty")
|
||||
r = await client.get(
|
||||
f"{_V1}/{tid}/personas",
|
||||
headers={"Authorization": f"Bearer {auth_token}"},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
assert body["topology_id"] == tid
|
||||
assert body["personas"] == []
|
||||
assert body["language_default"] == "en"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_get_404(client, auth_token):
|
||||
r = await client.get(
|
||||
f"{_V1}/does-not-exist/personas",
|
||||
headers={"Authorization": f"Bearer {auth_token}"},
|
||||
)
|
||||
assert r.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_put_then_get(client, auth_token):
|
||||
tid = await _seed("put-roundtrip")
|
||||
payload = {"personas": [
|
||||
_persona("a@example.com", "Alice"),
|
||||
_persona("b@example.com", "Bob"),
|
||||
]}
|
||||
r = await client.put(
|
||||
f"{_V1}/{tid}/personas",
|
||||
json=payload,
|
||||
headers={"Authorization": f"Bearer {auth_token}"},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert len(r.json()["personas"]) == 2
|
||||
|
||||
r2 = await client.get(
|
||||
f"{_V1}/{tid}/personas",
|
||||
headers={"Authorization": f"Bearer {auth_token}"},
|
||||
)
|
||||
assert r2.status_code == 200
|
||||
emails = [p["email"] for p in r2.json()["personas"]]
|
||||
assert emails == ["a@example.com", "b@example.com"]
|
||||
|
||||
# Persisted as JSON string in the topology row.
|
||||
topo = await _repo.get_topology(tid)
|
||||
assert isinstance(topo["email_personas"], str)
|
||||
stored = json.loads(topo["email_personas"])
|
||||
assert {p["email"] for p in stored} == {"a@example.com", "b@example.com"}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_put_empty_clears(client, auth_token):
|
||||
tid = await _seed("put-empty")
|
||||
await client.put(
|
||||
f"{_V1}/{tid}/personas",
|
||||
json={"personas": [_persona("x@example.com")]},
|
||||
headers={"Authorization": f"Bearer {auth_token}"},
|
||||
)
|
||||
r = await client.put(
|
||||
f"{_V1}/{tid}/personas",
|
||||
json={"personas": []},
|
||||
headers={"Authorization": f"Bearer {auth_token}"},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["personas"] == []
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_put_non_list_400(client, auth_token):
|
||||
tid = await _seed("put-non-list")
|
||||
r = await client.put(
|
||||
f"{_V1}/{tid}/personas",
|
||||
json={"personas": "not a list"},
|
||||
headers={"Authorization": f"Bearer {auth_token}"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_put_all_invalid_400(client, auth_token):
|
||||
tid = await _seed("put-all-bad")
|
||||
r = await client.put(
|
||||
f"{_V1}/{tid}/personas",
|
||||
json={"personas": [{"email": "no-at-sign"}, {"name": "no-email"}]},
|
||||
headers={"Authorization": f"Bearer {auth_token}"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_put_partial_invalid_keeps_valid(client, auth_token):
|
||||
"""Mirror the global-pool drop-invalid semantics.
|
||||
|
||||
The endpoint silently drops bad entries; operators discover what
|
||||
landed by reading back the GET.
|
||||
"""
|
||||
tid = await _seed("put-partial")
|
||||
r = await client.put(
|
||||
f"{_V1}/{tid}/personas",
|
||||
json={"personas": [
|
||||
_persona("good@example.com"),
|
||||
{"name": "missing email"},
|
||||
]},
|
||||
headers={"Authorization": f"Bearer {auth_token}"},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert [p["email"] for p in body["personas"]] == ["good@example.com"]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_put_404_on_missing_topology(client, auth_token):
|
||||
r = await client.put(
|
||||
f"{_V1}/does-not-exist/personas",
|
||||
json={"personas": [_persona("x@example.com")]},
|
||||
headers={"Authorization": f"Bearer {auth_token}"},
|
||||
)
|
||||
assert r.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_get_does_not_shadow_existing_topology_id(client, auth_token):
|
||||
"""Ensure the personas subroute is registered before the bare /{id}.
|
||||
|
||||
If the literal `/personas` segment got shadowed by the parameterized
|
||||
`/{id}` route, GET would return the topology body instead of 404 for
|
||||
a missing personas resource. Sanity-check the order.
|
||||
"""
|
||||
tid = await _seed("shadow-check")
|
||||
r = await client.get(
|
||||
f"{_V1}/{tid}/personas",
|
||||
headers={"Authorization": f"Bearer {auth_token}"},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert "personas" in r.json()
|
||||
@@ -1,9 +1,9 @@
|
||||
"""``decnet emailgen`` is master-only.
|
||||
"""``decnet realism`` is master-only.
|
||||
|
||||
Two layers per CLAUDE.md:
|
||||
|
||||
* registration-time hide via :data:`MASTER_ONLY_GROUPS` so agents don't
|
||||
see ``decnet emailgen`` in ``--help`` at all,
|
||||
see ``decnet realism`` in ``--help`` at all,
|
||||
* body-guard ``_require_master_mode()`` so a direct callable import (e.g.
|
||||
from a third-party tool) still bails on agent hosts.
|
||||
"""
|
||||
@@ -17,7 +17,6 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
|
||||
REPO = pathlib.Path(__file__).resolve().parent.parent.parent
|
||||
@@ -32,7 +31,7 @@ def _clean_env(**overrides: str) -> dict[str, str]:
|
||||
return base
|
||||
|
||||
|
||||
def test_emailgen_visible_in_master_mode():
|
||||
def test_realism_visible_in_master_mode():
|
||||
result = subprocess.run(
|
||||
[str(DECNET_BIN), "--help"],
|
||||
env=_clean_env(DECNET_MODE="master"),
|
||||
@@ -40,10 +39,10 @@ def test_emailgen_visible_in_master_mode():
|
||||
capture_output=True, text=True, timeout=20,
|
||||
)
|
||||
assert result.returncode == 0
|
||||
assert "emailgen" in result.stdout
|
||||
assert "realism" in result.stdout
|
||||
|
||||
|
||||
def test_emailgen_hidden_in_agent_mode():
|
||||
def test_realism_hidden_in_agent_mode():
|
||||
result = subprocess.run(
|
||||
[str(DECNET_BIN), "--help"],
|
||||
env=_clean_env(DECNET_MODE="agent", DECNET_DISALLOW_MASTER="true"),
|
||||
@@ -51,25 +50,12 @@ def test_emailgen_hidden_in_agent_mode():
|
||||
capture_output=True, text=True, timeout=20,
|
||||
)
|
||||
assert result.returncode == 0
|
||||
# The sub-app's help string must be gone too — bare "emailgen" can
|
||||
# The sub-app's help string must be gone too — bare "realism" can
|
||||
# appear in other command descriptions.
|
||||
assert "Drip persona-driven fake corporate email" not in result.stdout
|
||||
assert "realism content engine" not in result.stdout
|
||||
|
||||
|
||||
def test_emailgen_subprocess_run_rejects_in_agent_mode():
|
||||
"""Subprocess-level: a fresh Python invocation of `decnet emailgen
|
||||
run` under DECNET_MODE=agent must exit non-zero (gate hides the
|
||||
sub-app, so the command is unknown to Typer)."""
|
||||
result = subprocess.run(
|
||||
[str(DECNET_BIN), "emailgen", "run"],
|
||||
env=_clean_env(DECNET_MODE="agent", DECNET_DISALLOW_MASTER="true"),
|
||||
cwd=str(REPO),
|
||||
capture_output=True, text=True, timeout=20,
|
||||
)
|
||||
assert result.returncode != 0
|
||||
|
||||
|
||||
def test_emailgen_subprocess_import_personas_rejects_in_agent_mode(tmp_path):
|
||||
def test_realism_subprocess_import_personas_rejects_in_agent_mode(tmp_path):
|
||||
src = tmp_path / "personas.json"
|
||||
src.write_text(json.dumps([{
|
||||
"name": "X", "email": "x@y.com", "role": "X", "tone": "formal",
|
||||
@@ -79,7 +65,7 @@ def test_emailgen_subprocess_import_personas_rejects_in_agent_mode(tmp_path):
|
||||
"mannerisms": [],
|
||||
}]))
|
||||
result = subprocess.run(
|
||||
[str(DECNET_BIN), "emailgen", "import-personas", str(src)],
|
||||
[str(DECNET_BIN), "realism", "import-personas", str(src)],
|
||||
env=_clean_env(DECNET_MODE="agent", DECNET_DISALLOW_MASTER="true"),
|
||||
cwd=str(REPO),
|
||||
capture_output=True, text=True, timeout=20,
|
||||
@@ -89,7 +75,7 @@ def test_emailgen_subprocess_import_personas_rejects_in_agent_mode(tmp_path):
|
||||
|
||||
def test_require_master_mode_body_guard_fires_directly(monkeypatch):
|
||||
"""Defence-in-depth: even bypassing Typer registration, the body-level
|
||||
``_require_master_mode('emailgen ...')`` raises ``typer.Exit``. Same
|
||||
``_require_master_mode('realism ...')`` raises ``typer.Exit``. Same
|
||||
mechanism is verified for `api`/`deploy` in test_mode_gating.py."""
|
||||
import typer
|
||||
|
||||
@@ -99,13 +85,13 @@ def test_require_master_mode_body_guard_fires_directly(monkeypatch):
|
||||
monkeypatch.setenv("DECNET_DISALLOW_MASTER", "true")
|
||||
|
||||
with pytest.raises(typer.Exit):
|
||||
_require_master_mode("emailgen run")
|
||||
_require_master_mode("realism import-personas")
|
||||
|
||||
|
||||
def test_master_mode_falls_through_body_guard(monkeypatch):
|
||||
"""In master mode the guard is a no-op (raises nothing)."""
|
||||
from decnet.cli.gating import _require_master_mode
|
||||
from decnet.cli.gating import _require_master_mode # noqa: F401
|
||||
|
||||
monkeypatch.setenv("DECNET_MODE", "master")
|
||||
# Should simply return.
|
||||
_require_master_mode("emailgen run")
|
||||
_require_master_mode("realism import-personas")
|
||||
@@ -1,4 +1,4 @@
|
||||
"""``decnet emailgen import-personas`` CLI command."""
|
||||
"""``decnet realism import-personas`` CLI command."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
@@ -42,7 +42,7 @@ def test_import_personas_writes_canonical_file(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(dest))
|
||||
|
||||
result = CliRunner().invoke(
|
||||
app, ["emailgen", "import-personas", str(src)]
|
||||
app, ["realism", "import-personas", str(src)]
|
||||
)
|
||||
assert result.exit_code == 0, result.stdout
|
||||
assert dest.exists()
|
||||
@@ -59,7 +59,7 @@ def test_import_personas_explicit_output_overrides_env(tmp_path, monkeypatch):
|
||||
|
||||
result = CliRunner().invoke(
|
||||
app,
|
||||
["emailgen", "import-personas", str(src), "--output", str(explicit)],
|
||||
["realism", "import-personas", str(src), "--output", str(explicit)],
|
||||
)
|
||||
assert result.exit_code == 0, result.stdout
|
||||
assert explicit.exists()
|
||||
@@ -70,7 +70,7 @@ def test_import_personas_rejects_invalid_json(tmp_path):
|
||||
src = tmp_path / "src.json"
|
||||
src.write_text("{not valid")
|
||||
result = CliRunner().invoke(
|
||||
app, ["emailgen", "import-personas", str(src)]
|
||||
app, ["realism", "import-personas", str(src)]
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "Invalid JSON" in result.stdout
|
||||
@@ -81,7 +81,7 @@ def test_import_personas_rejects_non_list(tmp_path, monkeypatch):
|
||||
src.write_text(json.dumps({"not": "a list"}))
|
||||
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(tmp_path / "out.json"))
|
||||
result = CliRunner().invoke(
|
||||
app, ["emailgen", "import-personas", str(src)]
|
||||
app, ["realism", "import-personas", str(src)]
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "list" in result.stdout.lower()
|
||||
@@ -94,7 +94,7 @@ def test_import_personas_rejects_all_invalid_entries(tmp_path, monkeypatch):
|
||||
]))
|
||||
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(tmp_path / "out.json"))
|
||||
result = CliRunner().invoke(
|
||||
app, ["emailgen", "import-personas", str(src)]
|
||||
app, ["realism", "import-personas", str(src)]
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "No valid personas" in result.stdout
|
||||
@@ -106,7 +106,7 @@ def test_import_personas_warns_on_single_persona(tmp_path, monkeypatch):
|
||||
dest = tmp_path / "out.json"
|
||||
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(dest))
|
||||
result = CliRunner().invoke(
|
||||
app, ["emailgen", "import-personas", str(src)]
|
||||
app, ["realism", "import-personas", str(src)]
|
||||
)
|
||||
assert result.exit_code == 0, result.stdout
|
||||
assert "Warning" in result.stdout
|
||||
@@ -120,7 +120,7 @@ def test_imported_personas_load_via_global_pool(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(dest))
|
||||
|
||||
result = CliRunner().invoke(
|
||||
app, ["emailgen", "import-personas", str(src)]
|
||||
app, ["realism", "import-personas", str(src)]
|
||||
)
|
||||
assert result.exit_code == 0, result.stdout
|
||||
|
||||
@@ -1,141 +0,0 @@
|
||||
"""End-to-end-ish: one emailgen tick against a real SQLite repo + FakeBus,
|
||||
with the Ollama + docker-exec subprocess stubbed."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from decnet.bus.fake import FakeBus
|
||||
from decnet.orchestrator.drivers import email as email_driver
|
||||
from decnet.orchestrator.emailgen import worker as eg_worker
|
||||
from decnet.orchestrator.emailgen.scheduler import EmailAction # noqa: F401
|
||||
from decnet.realism.llm.impl.fake import FakeBackend
|
||||
from decnet.web.db.models import Topology, TopologyDecky
|
||||
from decnet.web.db.sqlite.repository import SQLiteRepository
|
||||
|
||||
|
||||
_PERSONAS = [
|
||||
{
|
||||
"name": "John Smith",
|
||||
"email": "john@corp.com",
|
||||
"role": "COO",
|
||||
"tone": "formal",
|
||||
"mannerisms": ["uses 'Best regards'"],
|
||||
"active_hours": "00:00-00:00", # always-on so test is hour-independent
|
||||
},
|
||||
{
|
||||
"name": "Sarah Johnson",
|
||||
"email": "sarah@corp.com",
|
||||
"role": "PM",
|
||||
"tone": "direct",
|
||||
"mannerisms": ["uses bullets"],
|
||||
"active_hours": "00:00-00:00",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def repo(tmp_path):
|
||||
r = SQLiteRepository(db_path=str(tmp_path / "decnet.db"))
|
||||
await r.initialize()
|
||||
yield r
|
||||
await r.engine.dispose()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def fake_bus():
|
||||
bus = FakeBus()
|
||||
await bus.connect()
|
||||
try:
|
||||
yield bus
|
||||
finally:
|
||||
await bus.close()
|
||||
|
||||
|
||||
async def _seed_mail_topology(repo: SQLiteRepository) -> str:
|
||||
async with repo._session() as session:
|
||||
topo = Topology(
|
||||
name="t-mail",
|
||||
config_snapshot="{}",
|
||||
status="active",
|
||||
email_personas=json.dumps(_PERSONAS),
|
||||
language_default="en",
|
||||
)
|
||||
session.add(topo)
|
||||
await session.commit()
|
||||
await session.refresh(topo)
|
||||
decky = TopologyDecky(
|
||||
topology_id=topo.id,
|
||||
name="mailhost",
|
||||
services=json.dumps(["imap"]),
|
||||
ip="10.0.0.10",
|
||||
state="running",
|
||||
)
|
||||
session.add(decky)
|
||||
await session.commit()
|
||||
await session.refresh(decky)
|
||||
return decky.uuid
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_tick_records_and_publishes(repo, fake_bus, monkeypatch):
|
||||
decky_uuid = await _seed_mail_topology(repo)
|
||||
|
||||
# Stub only the docker exec subprocess; the LLM call goes through
|
||||
# an injected FakeBackend with deterministic output.
|
||||
async def fake_run_capture(argv, *, stdin_data=None, timeout=8.0):
|
||||
return 0, "", ""
|
||||
|
||||
monkeypatch.setattr(email_driver, "_run_capture", fake_run_capture)
|
||||
|
||||
received: list = []
|
||||
|
||||
async def collect():
|
||||
async with fake_bus.subscribe(f"orchestrator.email.{decky_uuid}") as sub:
|
||||
async for ev in sub:
|
||||
received.append(ev)
|
||||
return
|
||||
|
||||
import asyncio
|
||||
collector = asyncio.create_task(collect())
|
||||
await asyncio.sleep(0)
|
||||
|
||||
driver = email_driver.EmailDriver(
|
||||
llm=FakeBackend(output="Subject: Hi\n\nBody here.\n"),
|
||||
)
|
||||
await eg_worker._one_tick(repo, driver, fake_bus)
|
||||
await asyncio.wait_for(collector, timeout=2.0)
|
||||
|
||||
rows = await repo.list_orchestrator_emails()
|
||||
assert len(rows) == 1
|
||||
row = rows[0]
|
||||
assert row["success"] is True
|
||||
assert row["mail_decky_uuid"] == decky_uuid
|
||||
assert row["subject"] == "Hi"
|
||||
assert row["language"] == "en"
|
||||
|
||||
assert len(received) == 1
|
||||
assert received[0].topic == f"orchestrator.email.{decky_uuid}"
|
||||
assert received[0].payload["kind"] == "email"
|
||||
assert received[0].payload["success"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_tick_noop_when_no_mail_decky(repo, fake_bus, monkeypatch):
|
||||
called = False
|
||||
|
||||
async def fake_run_capture(argv, *, stdin_data=None, timeout=8.0):
|
||||
nonlocal called
|
||||
called = True
|
||||
return 0, "", ""
|
||||
|
||||
monkeypatch.setattr(email_driver, "_run_capture", fake_run_capture)
|
||||
|
||||
driver = email_driver.EmailDriver(
|
||||
llm=FakeBackend(output="Subject: x\n\nb\n"),
|
||||
)
|
||||
await eg_worker._one_tick(repo, driver, fake_bus)
|
||||
assert called is False
|
||||
assert await repo.list_orchestrator_emails() == []
|
||||
@@ -73,6 +73,13 @@ async def test_one_tick_records_event_and_publishes(repo, fake_bus, monkeypatch)
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run", fake_run)
|
||||
|
||||
async def fake_run_with_stdin(argv, stdin_bytes):
|
||||
# plant_file takes the base64-streaming path; treat any docker
|
||||
# exec write as a successful no-op for the integration test.
|
||||
return 0, "", ""
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run_with_stdin", fake_run_with_stdin)
|
||||
|
||||
received: list = []
|
||||
|
||||
async def collect():
|
||||
@@ -87,8 +94,7 @@ async def test_one_tick_records_event_and_publishes(repo, fake_bus, monkeypatch)
|
||||
# Yield once so the subscription is registered before we publish.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
driver = ssh_driver.SSHDriver()
|
||||
await orch_worker._one_tick(repo, driver, fake_bus)
|
||||
await orch_worker._one_tick(repo, fake_bus)
|
||||
|
||||
await asyncio.wait_for(collector, timeout=2.0)
|
||||
|
||||
@@ -134,8 +140,14 @@ async def test_one_tick_picks_fleet_deckies(repo, fake_bus, monkeypatch):
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run", fake_run)
|
||||
|
||||
driver = ssh_driver.SSHDriver()
|
||||
await orch_worker._one_tick(repo, driver, fake_bus)
|
||||
async def fake_run_with_stdin(argv, stdin_bytes):
|
||||
# plant_file takes the base64-streaming path; treat any docker
|
||||
# exec write as a successful no-op for the integration test.
|
||||
return 0, "", ""
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run_with_stdin", fake_run_with_stdin)
|
||||
|
||||
await orch_worker._one_tick(repo, fake_bus)
|
||||
|
||||
rows = await repo.list_orchestrator_events(limit=10)
|
||||
assert len(rows) == 1
|
||||
@@ -154,8 +166,14 @@ async def test_tick_is_noop_when_no_running_deckies(repo, fake_bus, monkeypatch)
|
||||
return 0, "SSH-2.0-foo", ""
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run", fake_run)
|
||||
driver = ssh_driver.SSHDriver()
|
||||
await orch_worker._one_tick(repo, driver, fake_bus)
|
||||
|
||||
async def fake_run_with_stdin(argv, stdin_bytes):
|
||||
# plant_file takes the base64-streaming path; treat any docker
|
||||
# exec write as a successful no-op for the integration test.
|
||||
return 0, "", ""
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run_with_stdin", fake_run_with_stdin)
|
||||
await orch_worker._one_tick(repo, fake_bus)
|
||||
|
||||
assert called is False
|
||||
assert await repo.list_orchestrator_events(limit=10) == []
|
||||
|
||||
Reference in New Issue
Block a user