merge: testing → main (reconcile 2-week divergence)
This commit is contained in:
0
tests/orchestrator/__init__.py
Normal file
0
tests/orchestrator/__init__.py
Normal file
0
tests/orchestrator/emailgen/__init__.py
Normal file
0
tests/orchestrator/emailgen/__init__.py
Normal file
215
tests/orchestrator/emailgen/test_driver.py
Normal file
215
tests/orchestrator/emailgen/test_driver.py
Normal file
@@ -0,0 +1,215 @@
|
||||
"""EmailDriver: inject a fake LLM backend + stub docker exec; verify
|
||||
EML parse-and-repair and payload metadata."""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from decnet.orchestrator.drivers import email as email_driver
|
||||
from decnet.orchestrator.emailgen.scheduler import EmailAction
|
||||
from decnet.realism.llm.base import LLMResult, LLMTimeout
|
||||
from decnet.realism.llm.impl.fake import FakeBackend
|
||||
from decnet.realism.personas import EmailPersona
|
||||
|
||||
|
||||
class _RaisingBackend:
|
||||
"""Async stub that raises LLMTimeout on every call."""
|
||||
model = "stuck-model"
|
||||
timeout = 0.1
|
||||
|
||||
async def generate(self, prompt: str) -> LLMResult: # noqa: ARG002
|
||||
raise LLMTimeout("stuck")
|
||||
|
||||
|
||||
class _FailingBackend:
|
||||
"""Async stub that returns success=False."""
|
||||
model = "broken-model"
|
||||
timeout = 1.0
|
||||
|
||||
async def generate(self, prompt: str) -> LLMResult: # noqa: ARG002
|
||||
return LLMResult(
|
||||
success=False,
|
||||
text="",
|
||||
model=self.model,
|
||||
latency_ms=5,
|
||||
extra={"rc": 1, "stderr": "model not found"},
|
||||
)
|
||||
|
||||
|
||||
def _persona(name="John", email="john@corp.com"):
|
||||
return EmailPersona(
|
||||
name=name,
|
||||
email=email,
|
||||
role="COO",
|
||||
tone="formal",
|
||||
mannerisms=["uses 'Best regards'"],
|
||||
language="en",
|
||||
)
|
||||
|
||||
|
||||
def _action(is_reply=False):
|
||||
return EmailAction(
|
||||
mail_decky_uuid="d1",
|
||||
mail_decky_name="mailhost",
|
||||
mail_decky_services=("imap",),
|
||||
sender=_persona(),
|
||||
recipient=_persona(name="Sarah", email="sarah@corp.com"),
|
||||
thread_id="thr1",
|
||||
parent_message_id="<old@corp.com>" if is_reply else None,
|
||||
references="" if not is_reply else "<old@corp.com>",
|
||||
subject_hint="Re: budget" if is_reply else None,
|
||||
parent_excerpt=None,
|
||||
context_hint="Q3 budget" if not is_reply else "Re: budget",
|
||||
is_reply=is_reply,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_subject_and_body_extracts_subject_line():
|
||||
out = "Subject: Quick update\n\nHi Sarah,\nNumbers attached.\n"
|
||||
subject, body = email_driver._parse_subject_and_body(out)
|
||||
assert subject == "Quick update"
|
||||
assert body.startswith("Hi Sarah")
|
||||
|
||||
|
||||
def test_parse_subject_strips_code_fences():
|
||||
out = "```\nSubject: Quick update\n\nbody\n```\n"
|
||||
subject, body = email_driver._parse_subject_and_body(out)
|
||||
assert subject == "Quick update"
|
||||
assert body == "body"
|
||||
|
||||
|
||||
def test_parse_subject_falls_back_when_missing():
|
||||
out = "Just a body, no subject\n"
|
||||
subject, body = email_driver._parse_subject_and_body(out)
|
||||
assert subject == "Business Communication"
|
||||
assert "body" in body.lower()
|
||||
|
||||
|
||||
def test_build_eml_includes_required_headers():
|
||||
from datetime import datetime, timezone
|
||||
|
||||
eml = email_driver._build_eml(
|
||||
sender_name="John",
|
||||
sender_email="john@corp.com",
|
||||
recipient_name="Sarah",
|
||||
recipient_email="sarah@corp.com",
|
||||
subject="Q3 budget",
|
||||
body="Hi Sarah,\nNumbers attached.",
|
||||
message_id="<m1@corp.com>",
|
||||
in_reply_to=None,
|
||||
references="",
|
||||
ts=datetime(2026, 4, 26, 12, 0, tzinfo=timezone.utc),
|
||||
).decode("utf-8")
|
||||
assert "From: John <john@corp.com>" in eml
|
||||
assert "To: Sarah <sarah@corp.com>" in eml
|
||||
assert "Subject: Q3 budget" in eml
|
||||
assert "Message-ID: <m1@corp.com>" in eml
|
||||
assert "MIME-Version: 1.0" in eml
|
||||
assert "In-Reply-To" not in eml
|
||||
|
||||
|
||||
def test_build_eml_threads_carry_in_reply_to_and_references():
|
||||
from datetime import datetime, timezone
|
||||
|
||||
eml = email_driver._build_eml(
|
||||
sender_name="John",
|
||||
sender_email="john@corp.com",
|
||||
recipient_name="Sarah",
|
||||
recipient_email="sarah@corp.com",
|
||||
subject="Re: Q3",
|
||||
body="Following up.",
|
||||
message_id="<m2@corp.com>",
|
||||
in_reply_to="<m1@corp.com>",
|
||||
references="<m1@corp.com>",
|
||||
ts=datetime(2026, 4, 26, 12, 0, tzinfo=timezone.utc),
|
||||
).decode("utf-8")
|
||||
assert "In-Reply-To: <m1@corp.com>" in eml
|
||||
assert "References: <m1@corp.com>" in eml
|
||||
|
||||
|
||||
def test_container_for_imap_takes_priority():
|
||||
assert email_driver._container_for("mailhost", ["imap", "pop3"]) == "mailhost-imap"
|
||||
|
||||
|
||||
def test_container_for_pop3_only():
|
||||
assert email_driver._container_for("mailhost", ["pop3"]) == "mailhost-pop3"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_driver_run_success_path(monkeypatch):
|
||||
"""Inject a FakeBackend + stub docker exec; success end-to-end."""
|
||||
docker_calls: list[list[str]] = []
|
||||
|
||||
async def fake_run_capture(argv, *, stdin_data=None, timeout=8.0):
|
||||
docker_calls.append(list(argv))
|
||||
return 0, "", ""
|
||||
|
||||
monkeypatch.setattr(email_driver, "_run_capture", fake_run_capture)
|
||||
|
||||
llm = FakeBackend(
|
||||
model="llama3.1",
|
||||
output="Subject: Q3 budget\n\nHi Sarah,\nNumbers attached.\n",
|
||||
)
|
||||
drv = email_driver.EmailDriver(llm=llm)
|
||||
result = await drv.run(_action())
|
||||
assert result.success is True
|
||||
assert result.payload["model"] == "llama3.1"
|
||||
assert result.payload["subject"] == "Q3 budget"
|
||||
assert result.payload["language"] == "en"
|
||||
assert result.payload["mannerisms_used"]
|
||||
assert result.payload["message_id"].startswith("<")
|
||||
assert result.payload["eml_path"].endswith(".eml")
|
||||
assert result.payload["container"] == "mailhost-imap"
|
||||
# Only docker exec is shelled out now — the LLM call is in-process
|
||||
# via the FakeBackend.
|
||||
assert len(docker_calls) == 1
|
||||
assert docker_calls[0][0] == "docker"
|
||||
docker_sh = docker_calls[0][-1]
|
||||
assert "touch -d" in docker_sh
|
||||
assert "tee" in docker_sh
|
||||
assert docker_sh.index("tee") < docker_sh.index("touch -d")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_driver_run_llm_failure_short_circuits(monkeypatch):
|
||||
"""When the backend reports success=False, no docker exec should fire."""
|
||||
docker_called = False
|
||||
|
||||
async def fake_run_capture(argv, *, stdin_data=None, timeout=8.0):
|
||||
nonlocal docker_called
|
||||
docker_called = True
|
||||
return 0, "", ""
|
||||
|
||||
monkeypatch.setattr(email_driver, "_run_capture", fake_run_capture)
|
||||
|
||||
drv = email_driver.EmailDriver(llm=_FailingBackend())
|
||||
result = await drv.run(_action())
|
||||
assert result.success is False
|
||||
assert result.payload["stage"] == "llm"
|
||||
assert "stderr" in result.payload
|
||||
assert "model not found" in result.payload["stderr"]
|
||||
assert docker_called is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_driver_run_llm_timeout_reported_distinctly(monkeypatch):
|
||||
drv = email_driver.EmailDriver(llm=_RaisingBackend())
|
||||
result = await drv.run(_action())
|
||||
assert result.success is False
|
||||
assert result.payload["stage"] == "llm"
|
||||
assert result.payload["error"] == "timeout"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_driver_run_delivery_failure(monkeypatch):
|
||||
async def fake_run_capture(argv, *, stdin_data=None, timeout=8.0):
|
||||
return 1, "", "no such container"
|
||||
|
||||
monkeypatch.setattr(email_driver, "_run_capture", fake_run_capture)
|
||||
|
||||
drv = email_driver.EmailDriver(
|
||||
llm=FakeBackend(output="Subject: hi\n\nbody\n"),
|
||||
)
|
||||
result = await drv.run(_action())
|
||||
assert result.success is False
|
||||
assert result.payload["stage"] == "delivery"
|
||||
assert "no such container" in result.payload["stderr"]
|
||||
72
tests/orchestrator/emailgen/test_events.py
Normal file
72
tests/orchestrator/emailgen/test_events.py
Normal file
@@ -0,0 +1,72 @@
|
||||
"""events.to_row / topic_for / event_type_for."""
|
||||
from __future__ import annotations
|
||||
|
||||
from decnet.bus import topics as _topics
|
||||
from decnet.orchestrator.drivers.base import ActivityResult
|
||||
from decnet.orchestrator.emailgen import events
|
||||
from decnet.realism.personas import EmailPersona
|
||||
from decnet.orchestrator.emailgen.scheduler import EmailAction
|
||||
|
||||
|
||||
def _persona(email="john@corp.com"):
|
||||
return EmailPersona(
|
||||
name="John", email=email, role="COO", tone="formal",
|
||||
mannerisms=[], language="en",
|
||||
)
|
||||
|
||||
|
||||
def _action():
|
||||
return EmailAction(
|
||||
mail_decky_uuid="d1",
|
||||
mail_decky_name="mailhost",
|
||||
mail_decky_services=("imap",),
|
||||
sender=_persona(),
|
||||
recipient=_persona(email="sarah@corp.com"),
|
||||
thread_id="thr1",
|
||||
parent_message_id=None,
|
||||
references="",
|
||||
subject_hint=None,
|
||||
parent_excerpt=None,
|
||||
context_hint="Q3 budget",
|
||||
is_reply=False,
|
||||
)
|
||||
|
||||
|
||||
def test_to_row_pulls_message_id_subject_from_payload():
|
||||
res = ActivityResult(
|
||||
success=True,
|
||||
payload={
|
||||
"message_id": "<m1@corp.com>",
|
||||
"subject": "Q3 budget",
|
||||
"language": "en",
|
||||
"eml_path": "/var/spool/decnet-emails/thr1/m1.eml",
|
||||
"model": "llama3.1",
|
||||
},
|
||||
)
|
||||
row = events.to_row(_action(), res)
|
||||
assert row["mail_decky_uuid"] == "d1"
|
||||
assert row["thread_id"] == "thr1"
|
||||
assert row["message_id"] == "<m1@corp.com>"
|
||||
assert row["subject"] == "Q3 budget"
|
||||
assert row["sender_email"] == "john@corp.com"
|
||||
assert row["recipient_email"] == "sarah@corp.com"
|
||||
assert row["language"] == "en"
|
||||
assert row["eml_path"].endswith(".eml")
|
||||
assert row["success"] is True
|
||||
assert row["payload"]["model"] == "llama3.1"
|
||||
|
||||
|
||||
def test_to_row_falls_back_to_persona_language():
|
||||
res = ActivityResult(success=True, payload={})
|
||||
row = events.to_row(_action(), res)
|
||||
assert row["language"] == "en"
|
||||
assert row["message_id"] == ""
|
||||
|
||||
|
||||
def test_topic_for_uses_orchestrator_email_root():
|
||||
topic = events.topic_for(_action())
|
||||
assert topic == f"orchestrator.{_topics.ORCHESTRATOR_EMAIL}.d1"
|
||||
|
||||
|
||||
def test_event_type_for_returns_email_constant():
|
||||
assert events.event_type_for(_action()) == _topics.ORCHESTRATOR_EMAIL
|
||||
129
tests/orchestrator/emailgen/test_repo.py
Normal file
129
tests/orchestrator/emailgen/test_repo.py
Normal file
@@ -0,0 +1,129 @@
|
||||
"""record / list / count / prune orchestrator_emails on a real SQLite repo."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from decnet.web.db.sqlite.repository import SQLiteRepository
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def repo(tmp_path):
|
||||
r = SQLiteRepository(db_path=str(tmp_path / "decnet.db"))
|
||||
await r.initialize()
|
||||
yield r
|
||||
await r.engine.dispose()
|
||||
|
||||
|
||||
def _row(
|
||||
mail="d1",
|
||||
thread="thr1",
|
||||
msg="<m1@x>",
|
||||
sender="john@corp.com",
|
||||
recipient="sarah@corp.com",
|
||||
subject="Q3 budget",
|
||||
success=True,
|
||||
in_reply_to=None,
|
||||
ts=None,
|
||||
):
|
||||
return {
|
||||
"ts": ts or datetime.now(timezone.utc),
|
||||
"mail_decky_uuid": mail,
|
||||
"thread_id": thread,
|
||||
"message_id": msg,
|
||||
"in_reply_to": in_reply_to,
|
||||
"sender_email": sender,
|
||||
"recipient_email": recipient,
|
||||
"subject": subject,
|
||||
"language": "en",
|
||||
"eml_path": f"/var/spool/decnet-emails/{thread}/{msg}.eml",
|
||||
"success": success,
|
||||
"payload": {"model": "llama3.1"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_returns_uuid_and_serialises_payload(repo):
|
||||
uuid = await repo.record_orchestrator_email(_row())
|
||||
assert isinstance(uuid, str) and len(uuid) == 36
|
||||
rows = await repo.list_orchestrator_emails()
|
||||
assert len(rows) == 1
|
||||
# payload is stored as JSON text, list endpoint hands it back as the
|
||||
# raw column value — we just verify it round-trips intact.
|
||||
assert json.loads(rows[0]["payload"])["model"] == "llama3.1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_filters_by_thread_and_mail_decky(repo):
|
||||
await repo.record_orchestrator_email(_row(thread="t1", msg="<a@x>"))
|
||||
await repo.record_orchestrator_email(_row(thread="t2", msg="<b@x>"))
|
||||
await repo.record_orchestrator_email(_row(mail="d2", msg="<c@x>"))
|
||||
|
||||
by_thread = await repo.list_orchestrator_emails(thread_id="t1")
|
||||
assert {r["message_id"] for r in by_thread} == {"<a@x>"}
|
||||
|
||||
by_mail = await repo.list_orchestrator_emails(mail_decky_uuid="d1")
|
||||
assert len(by_mail) == 2
|
||||
|
||||
everything = await repo.list_orchestrator_emails()
|
||||
assert len(everything) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_orchestrator_emails(repo):
|
||||
for i in range(3):
|
||||
await repo.record_orchestrator_email(_row(msg=f"<m{i}@x>"))
|
||||
assert await repo.count_orchestrator_emails() == 3
|
||||
assert await repo.count_orchestrator_emails(mail_decky_uuid="d1") == 3
|
||||
assert await repo.count_orchestrator_emails(mail_decky_uuid="other") == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_lookup_only_returns_pair_threads(repo):
|
||||
await repo.record_orchestrator_email(
|
||||
_row(sender="john@corp.com", recipient="sarah@corp.com", msg="<a@x>")
|
||||
)
|
||||
# Reverse direction (Sarah → John) should still match the same pair.
|
||||
await repo.record_orchestrator_email(
|
||||
_row(sender="sarah@corp.com", recipient="john@corp.com", msg="<b@x>")
|
||||
)
|
||||
# Unrelated pair must not match.
|
||||
await repo.record_orchestrator_email(
|
||||
_row(sender="mike@corp.com", recipient="sarah@corp.com", msg="<c@x>")
|
||||
)
|
||||
threads = await repo.list_orchestrator_email_threads(
|
||||
"d1", "john@corp.com", "sarah@corp.com",
|
||||
)
|
||||
assert {t["message_id"] for t in threads} == {"<a@x>", "<b@x>"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_lookup_excludes_failed_rows(repo):
|
||||
await repo.record_orchestrator_email(_row(msg="<ok@x>", success=True))
|
||||
await repo.record_orchestrator_email(_row(msg="<bad@x>", success=False))
|
||||
threads = await repo.list_orchestrator_email_threads(
|
||||
"d1", "john@corp.com", "sarah@corp.com",
|
||||
)
|
||||
assert {t["message_id"] for t in threads} == {"<ok@x>"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prune_caps_per_decky(repo):
|
||||
# Insert 5 rows on d1 with strictly-increasing timestamps so the
|
||||
# prune's "newest-first keep, drop the rest" deterministically picks
|
||||
# the older two.
|
||||
base = datetime.now(timezone.utc) - timedelta(hours=10)
|
||||
for i in range(5):
|
||||
await repo.record_orchestrator_email(
|
||||
_row(msg=f"<m{i}@x>", ts=base + timedelta(minutes=i))
|
||||
)
|
||||
# Cap at 3 — expect 2 deleted.
|
||||
deleted = await repo.prune_orchestrator_emails(per_decky_cap=3)
|
||||
assert deleted == 2
|
||||
remaining = await repo.list_orchestrator_emails()
|
||||
assert len(remaining) == 3
|
||||
# The three newest survived.
|
||||
assert {r["message_id"] for r in remaining} == {"<m2@x>", "<m3@x>", "<m4@x>"}
|
||||
240
tests/orchestrator/emailgen/test_scheduler.py
Normal file
240
tests/orchestrator/emailgen/test_scheduler.py
Normal file
@@ -0,0 +1,240 @@
|
||||
"""Scheduler.pick() — async, takes a repo-shaped object."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from decnet.orchestrator.emailgen import scheduler
|
||||
from decnet.realism import personas_pool as global_pool
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_global_pool():
|
||||
global_pool.reset_cache()
|
||||
yield
|
||||
global_pool.reset_cache()
|
||||
|
||||
|
||||
_PERSONAS_TWO = [
|
||||
{
|
||||
"name": "John Smith",
|
||||
"email": "john@corp.com",
|
||||
"role": "COO",
|
||||
"tone": "formal",
|
||||
"mannerisms": ["uses 'Best regards'"],
|
||||
},
|
||||
{
|
||||
"name": "Sarah Johnson",
|
||||
"email": "sarah@corp.com",
|
||||
"role": "PM",
|
||||
"tone": "direct",
|
||||
"mannerisms": ["uses bullets"],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
class _FakeRepo:
|
||||
"""Minimal repo stub matching the methods scheduler.pick() uses."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
deckies: list[dict[str, Any]] | None = None,
|
||||
topologies: dict[str, dict[str, Any]] | None = None,
|
||||
threads: list[dict[str, Any]] | None = None,
|
||||
):
|
||||
self.deckies = deckies or []
|
||||
self.topologies = topologies or {}
|
||||
self.threads = threads or []
|
||||
self.thread_calls = 0
|
||||
|
||||
async def list_running_deckies(self):
|
||||
return self.deckies
|
||||
|
||||
async def get_topology(self, topology_id: str):
|
||||
return self.topologies.get(topology_id)
|
||||
|
||||
async def list_orchestrator_email_threads(self, *args, **kwargs):
|
||||
self.thread_calls += 1
|
||||
return list(self.threads)
|
||||
|
||||
|
||||
def _decky(
|
||||
uuid="d1",
|
||||
name="mailhost",
|
||||
services=("imap",),
|
||||
topology_id="t1",
|
||||
source="topology",
|
||||
):
|
||||
return {
|
||||
"uuid": uuid,
|
||||
"name": name,
|
||||
"services": list(services),
|
||||
"topology_id": topology_id,
|
||||
"source": source,
|
||||
}
|
||||
|
||||
|
||||
def _topology(personas=_PERSONAS_TWO, language_default="en"):
|
||||
return {
|
||||
"id": "t1",
|
||||
"email_personas": json.dumps(personas),
|
||||
"language_default": language_default,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_no_mail_decky_returns_none():
|
||||
repo = _FakeRepo(deckies=[_decky(services=("ssh",))])
|
||||
assert await scheduler.pick(repo) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_unknown_topology_returns_none():
|
||||
repo = _FakeRepo(deckies=[_decky()])
|
||||
# No topology row for "t1" — scheduler should bail.
|
||||
assert await scheduler.pick(repo) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_topology_with_one_persona_returns_none():
|
||||
repo = _FakeRepo(
|
||||
deckies=[_decky()],
|
||||
topologies={"t1": _topology(personas=_PERSONAS_TWO[:1])},
|
||||
)
|
||||
assert await scheduler.pick(repo) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_returns_action_for_valid_setup():
|
||||
repo = _FakeRepo(
|
||||
deckies=[_decky()],
|
||||
topologies={"t1": _topology()},
|
||||
)
|
||||
action = await scheduler.pick(repo, now=datetime(2026, 4, 26, 12, 0, 0))
|
||||
assert action is not None
|
||||
assert action.mail_decky_uuid == "d1"
|
||||
assert action.sender.email != action.recipient.email
|
||||
assert action.thread_id # populated for both new and reply branches
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_active_hours_filter_kicks_in_at_midnight():
|
||||
repo = _FakeRepo(
|
||||
deckies=[_decky()],
|
||||
topologies={"t1": _topology()},
|
||||
)
|
||||
# Default active_hours is 09:00-18:00; midnight => everyone out of office.
|
||||
action = await scheduler.pick(repo, now=datetime(2026, 4, 26, 3, 0, 0))
|
||||
assert action is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_uses_pop3_decky_too():
|
||||
repo = _FakeRepo(
|
||||
deckies=[_decky(services=("pop3",))],
|
||||
topologies={"t1": _topology()},
|
||||
)
|
||||
action = await scheduler.pick(repo, now=datetime(2026, 4, 26, 12, 0, 0))
|
||||
assert action is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_for_fleet_source_uses_global_pool(tmp_path, monkeypatch):
|
||||
"""Fleet (MACVLAN/IPVLAN) mail decky has no parent topology row;
|
||||
personas come from the host-wide JSON file."""
|
||||
pool_file = tmp_path / "personas.json"
|
||||
pool_file.write_text(json.dumps(_PERSONAS_TWO))
|
||||
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(pool_file))
|
||||
|
||||
repo = _FakeRepo(
|
||||
deckies=[_decky(source="fleet", topology_id=None)],
|
||||
# No topology row — confirms we never walk back to the topology.
|
||||
)
|
||||
action = await scheduler.pick(repo, now=datetime(2026, 4, 26, 12, 0, 0))
|
||||
assert action is not None
|
||||
assert action.mail_decky_uuid == "d1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_for_shard_source_uses_global_pool(tmp_path, monkeypatch):
|
||||
"""SWARM shards are non-topology too — same path as fleet."""
|
||||
pool_file = tmp_path / "personas.json"
|
||||
pool_file.write_text(json.dumps(_PERSONAS_TWO))
|
||||
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(pool_file))
|
||||
|
||||
repo = _FakeRepo(
|
||||
deckies=[_decky(source="shard", topology_id=None)],
|
||||
)
|
||||
action = await scheduler.pick(repo, now=datetime(2026, 4, 26, 12, 0, 0))
|
||||
assert action is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_fleet_with_empty_global_pool_returns_none(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(tmp_path / "missing.json"))
|
||||
repo = _FakeRepo(deckies=[_decky(source="fleet", topology_id=None)])
|
||||
assert await scheduler.pick(repo, now=datetime(2026, 4, 26, 12, 0, 0)) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_topology_personas_isolated_from_global_pool(tmp_path, monkeypatch):
|
||||
"""A topology with its own personas must NOT leak into / pull from
|
||||
the global pool — per-topology richness is the whole point."""
|
||||
pool_file = tmp_path / "personas.json"
|
||||
pool_file.write_text(json.dumps([{
|
||||
"name": "Pool Persona",
|
||||
"email": "pool@corp.com",
|
||||
"role": "Pooler",
|
||||
"tone": "casual",
|
||||
"mannerisms": [],
|
||||
}]))
|
||||
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(pool_file))
|
||||
|
||||
repo = _FakeRepo(
|
||||
deckies=[_decky()],
|
||||
topologies={"t1": _topology()}, # topology has _PERSONAS_TWO
|
||||
)
|
||||
action = await scheduler.pick(repo, now=datetime(2026, 4, 26, 12, 0, 0))
|
||||
assert action is not None
|
||||
# The chosen sender + recipient must come from the topology's pool,
|
||||
# not the global one — pool@corp.com would be a leak.
|
||||
assert action.sender.email != "pool@corp.com"
|
||||
assert action.recipient.email != "pool@corp.com"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_reply_chain_sets_in_reply_to():
|
||||
threads = [{
|
||||
"thread_id": "thr1",
|
||||
"message_id": "<old@corp.com>",
|
||||
"subject": "Q3 budget",
|
||||
}]
|
||||
repo = _FakeRepo(
|
||||
deckies=[_decky()],
|
||||
topologies={"t1": _topology()},
|
||||
threads=threads,
|
||||
)
|
||||
|
||||
# Force the "reply" branch by stubbing the RNG: random() < 0.6 is True.
|
||||
class _Rng:
|
||||
def __init__(self):
|
||||
self.calls = 0
|
||||
|
||||
def choice(self, seq):
|
||||
return seq[0]
|
||||
|
||||
def random(self):
|
||||
return 0.0 # always reply
|
||||
|
||||
action = await scheduler.pick(
|
||||
repo, rand=_Rng(), now=datetime(2026, 4, 26, 12, 0, 0),
|
||||
)
|
||||
assert action is not None
|
||||
assert action.is_reply is True
|
||||
assert action.parent_message_id == "<old@corp.com>"
|
||||
assert action.thread_id == "thr1"
|
||||
assert action.subject_hint == "Re: Q3 budget"
|
||||
61
tests/orchestrator/emailgen/test_threads.py
Normal file
61
tests/orchestrator/emailgen/test_threads.py
Normal file
@@ -0,0 +1,61 @@
|
||||
"""Thread-chain helpers."""
|
||||
from __future__ import annotations
|
||||
|
||||
from decnet.orchestrator.emailgen.threads import (
|
||||
ThreadChain,
|
||||
new_message_id,
|
||||
new_thread_id,
|
||||
references_for_reply,
|
||||
reply_subject,
|
||||
)
|
||||
|
||||
|
||||
def test_new_thread_id_is_uuid_string():
|
||||
tid = new_thread_id()
|
||||
assert len(tid) == 36
|
||||
assert tid.count("-") == 4
|
||||
|
||||
|
||||
def test_new_message_id_format_with_domain():
|
||||
mid = new_message_id("example.com")
|
||||
assert mid.startswith("<") and mid.endswith(">")
|
||||
assert "@example.com" in mid
|
||||
|
||||
|
||||
def test_new_message_id_handles_blank_domain():
|
||||
mid = new_message_id(" ")
|
||||
assert "@localhost" in mid
|
||||
|
||||
|
||||
def test_reply_subject_prepends_re():
|
||||
assert reply_subject("Q3 budget") == "Re: Q3 budget"
|
||||
|
||||
|
||||
def test_reply_subject_collapses_existing_re():
|
||||
assert reply_subject("Re: Re: Q3 budget") == "Re: Q3 budget"
|
||||
assert reply_subject("RE: Q3 budget") == "Re: Q3 budget"
|
||||
|
||||
|
||||
def test_references_for_reply_root_is_empty():
|
||||
assert references_for_reply(None) == ""
|
||||
|
||||
|
||||
def test_references_for_reply_appends_parent():
|
||||
chain = ThreadChain(
|
||||
thread_id="t1",
|
||||
parent_message_id="<m2@x>",
|
||||
references=("<m1@x>",),
|
||||
parent_subject="Re: budget",
|
||||
)
|
||||
refs = references_for_reply(chain)
|
||||
assert refs == "<m1@x> <m2@x>"
|
||||
|
||||
|
||||
def test_references_empty_chain_starts_with_parent_only():
|
||||
chain = ThreadChain(
|
||||
thread_id="t1",
|
||||
parent_message_id="<m1@x>",
|
||||
references=(),
|
||||
parent_subject="budget",
|
||||
)
|
||||
assert references_for_reply(chain) == "<m1@x>"
|
||||
74
tests/orchestrator/test_realism_config_refresh.py
Normal file
74
tests/orchestrator/test_realism_config_refresh.py
Normal file
@@ -0,0 +1,74 @@
|
||||
"""The orchestrator pulls operator-tuned weights from realism_config.
|
||||
|
||||
§3c contract: the planner reads in-memory module globals, but the
|
||||
operator's tuning lives in the DB (admin PUT /api/v1/realism/config).
|
||||
The orchestrator worker bridges the two by calling
|
||||
``_refresh_realism_config(repo)`` at startup and every Nth tick.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from decnet.orchestrator.worker import _refresh_realism_config
|
||||
from decnet.realism import planner
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_planner():
|
||||
yield
|
||||
planner.reset_to_defaults()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_no_row_keeps_defaults():
|
||||
repo = AsyncMock()
|
||||
repo.get_realism_config = AsyncMock(return_value=None)
|
||||
await _refresh_realism_config(repo)
|
||||
assert planner.current_payload()["canary_probability"] == pytest.approx(0.03)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_applies_stored_payload():
|
||||
repo = AsyncMock()
|
||||
repo.get_realism_config = AsyncMock(return_value={
|
||||
"key": "weights",
|
||||
"value": json.dumps({"canary_probability": 0.12}),
|
||||
})
|
||||
await _refresh_realism_config(repo)
|
||||
assert planner.current_payload()["canary_probability"] == pytest.approx(0.12)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_swallows_db_error():
|
||||
"""A wedged DB must not bring down the orchestrator's tick loop."""
|
||||
repo = AsyncMock()
|
||||
repo.get_realism_config = AsyncMock(side_effect=RuntimeError("boom"))
|
||||
await _refresh_realism_config(repo) # does not raise
|
||||
# planner unchanged
|
||||
assert planner.current_payload()["canary_probability"] == pytest.approx(0.03)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_swallows_malformed_json():
|
||||
repo = AsyncMock()
|
||||
repo.get_realism_config = AsyncMock(return_value={
|
||||
"key": "weights",
|
||||
"value": "not-json",
|
||||
})
|
||||
await _refresh_realism_config(repo)
|
||||
assert planner.current_payload()["canary_probability"] == pytest.approx(0.03)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_swallows_invalid_payload():
|
||||
repo = AsyncMock()
|
||||
repo.get_realism_config = AsyncMock(return_value={
|
||||
"key": "weights",
|
||||
"value": json.dumps({"canary_probability": 9.0}),
|
||||
})
|
||||
await _refresh_realism_config(repo)
|
||||
# Planner config not corrupted by a bad refresh.
|
||||
assert planner.current_payload()["canary_probability"] == pytest.approx(0.03)
|
||||
58
tests/orchestrator/test_realism_health_snapshot.py
Normal file
58
tests/orchestrator/test_realism_health_snapshot.py
Normal file
@@ -0,0 +1,58 @@
|
||||
"""LLM status surfaces in the orchestrator's heartbeat ``extra``.
|
||||
|
||||
Exposes the realism subsystem's LLM backend / model / circuit-breaker
|
||||
state so the dashboard can render a status badge without poking
|
||||
worker process memory.
|
||||
|
||||
Pinned by `feedback_push_principled_answer.md`: heartbeat is the
|
||||
canonical worker self-report channel, so this rides the existing
|
||||
``run_health_heartbeat(extra=...)`` extension hook rather than carving
|
||||
a new bus topic.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from decnet.orchestrator.worker import _realism_health_snapshot
|
||||
from decnet.realism.llm.circuit import LLMCircuitBreaker
|
||||
|
||||
|
||||
class _FakeLLM:
|
||||
model = "llama3.1:8b"
|
||||
|
||||
|
||||
def test_snapshot_reports_disabled_when_no_llm():
|
||||
snap = _realism_health_snapshot(llm=None, breaker=None)
|
||||
assert snap == {
|
||||
"llm_enabled": False,
|
||||
"llm_backend": None,
|
||||
"llm_model": None,
|
||||
"llm_breaker_state": None,
|
||||
}
|
||||
|
||||
|
||||
def test_snapshot_carries_backend_model_breaker_state(monkeypatch):
|
||||
monkeypatch.setenv("DECNET_REALISM_LLM", "ollama")
|
||||
breaker = LLMCircuitBreaker(failure_threshold=2, cooldown_seconds=1.0)
|
||||
snap = _realism_health_snapshot(llm=_FakeLLM(), breaker=breaker)
|
||||
assert snap["llm_enabled"] is True
|
||||
assert snap["llm_backend"] == "ollama"
|
||||
assert snap["llm_model"] == "llama3.1:8b"
|
||||
assert snap["llm_breaker_state"] == "closed"
|
||||
|
||||
|
||||
def test_snapshot_reflects_open_breaker(monkeypatch):
|
||||
monkeypatch.setenv("DECNET_REALISM_LLM", "ollama")
|
||||
breaker = LLMCircuitBreaker(failure_threshold=2, cooldown_seconds=60.0)
|
||||
breaker.record_failure()
|
||||
breaker.record_failure()
|
||||
snap = _realism_health_snapshot(llm=_FakeLLM(), breaker=breaker)
|
||||
assert snap["llm_breaker_state"] == "open"
|
||||
|
||||
|
||||
def test_snapshot_handles_llm_without_breaker(monkeypatch):
|
||||
"""Defensive: if init left ``breaker=None`` for any reason, the
|
||||
snapshot still publishes — just without breaker state."""
|
||||
monkeypatch.setenv("DECNET_REALISM_LLM", "fake")
|
||||
snap = _realism_health_snapshot(llm=_FakeLLM(), breaker=None)
|
||||
assert snap["llm_enabled"] is True
|
||||
assert snap["llm_backend"] == "fake"
|
||||
assert snap["llm_breaker_state"] is None
|
||||
107
tests/orchestrator/test_repo_pagination.py
Normal file
107
tests/orchestrator/test_repo_pagination.py
Normal file
@@ -0,0 +1,107 @@
|
||||
"""Pagination + filter + prune for orchestrator_events repo methods."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from decnet.web.db.models import Topology, TopologyDecky
|
||||
from decnet.web.db.sqlite.repository import SQLiteRepository
|
||||
|
||||
|
||||
async def _make_repo(tmp_path, name: str) -> SQLiteRepository:
|
||||
r = SQLiteRepository(db_path=str(tmp_path / name))
|
||||
await r.initialize()
|
||||
return r
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_table_zero_total(tmp_path):
|
||||
repo = await _make_repo(tmp_path, "orch.db")
|
||||
assert await repo.list_orchestrator_events(limit=50, offset=0) == []
|
||||
assert await repo.count_orchestrator_events() == 0
|
||||
|
||||
|
||||
async def _seed_decky(repo: SQLiteRepository, name: str = "d-1") -> str:
|
||||
async with repo._session() as session:
|
||||
topo = Topology(name=f"t-{name}", config_snapshot="{}", status="active")
|
||||
session.add(topo)
|
||||
await session.commit()
|
||||
await session.refresh(topo)
|
||||
d = TopologyDecky(
|
||||
topology_id=topo.id, name=name,
|
||||
services=json.dumps(["ssh"]), ip="10.0.0.2", state="running",
|
||||
)
|
||||
session.add(d)
|
||||
await session.commit()
|
||||
await session.refresh(d)
|
||||
return d.uuid
|
||||
|
||||
|
||||
async def _seed(
|
||||
repo: SQLiteRepository,
|
||||
n: int = 5,
|
||||
kind: str = "traffic",
|
||||
dst: str | None = None,
|
||||
) -> str:
|
||||
if dst is None:
|
||||
dst = await _seed_decky(repo, "decky-A")
|
||||
for i in range(n):
|
||||
await repo.record_orchestrator_event({
|
||||
"kind": kind,
|
||||
"protocol": "ssh",
|
||||
"action": f"exec:{i}",
|
||||
"src_decky_uuid": None,
|
||||
"dst_decky_uuid": dst,
|
||||
"success": True,
|
||||
"payload": {"i": i},
|
||||
})
|
||||
return dst
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pagination_respects_limit_offset(tmp_path):
|
||||
repo = await _make_repo(tmp_path, "p.db")
|
||||
await _seed(repo, n=5)
|
||||
|
||||
assert await repo.count_orchestrator_events() == 5
|
||||
page1 = await repo.list_orchestrator_events(limit=2, offset=0)
|
||||
page2 = await repo.list_orchestrator_events(limit=2, offset=2)
|
||||
assert len(page1) == 2
|
||||
assert len(page2) == 2
|
||||
assert {r["uuid"] for r in page1}.isdisjoint({r["uuid"] for r in page2})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_kind_filter_narrows(tmp_path):
|
||||
repo = await _make_repo(tmp_path, "k.db")
|
||||
dst = await _seed_decky(repo, "decky-K")
|
||||
for i in range(3):
|
||||
await repo.record_orchestrator_event({
|
||||
"kind": "traffic", "protocol": "ssh", "action": f"a{i}",
|
||||
"src_decky_uuid": None, "dst_decky_uuid": dst,
|
||||
"success": True, "payload": {},
|
||||
})
|
||||
for i in range(2):
|
||||
await repo.record_orchestrator_event({
|
||||
"kind": "file", "protocol": "ssh", "action": f"f{i}",
|
||||
"src_decky_uuid": None, "dst_decky_uuid": dst,
|
||||
"success": True, "payload": {},
|
||||
})
|
||||
|
||||
assert await repo.count_orchestrator_events() == 5
|
||||
assert await repo.count_orchestrator_events(kind="traffic") == 3
|
||||
assert await repo.count_orchestrator_events(kind="file") == 2
|
||||
|
||||
only_file = await repo.list_orchestrator_events(limit=50, kind="file")
|
||||
assert {r["kind"] for r in only_file} == {"file"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prune_caps_per_dst(tmp_path):
|
||||
repo = await _make_repo(tmp_path, "prune.db")
|
||||
await _seed(repo, n=10)
|
||||
|
||||
deleted = await repo.prune_orchestrator_events(per_dst_cap=3)
|
||||
assert deleted == 7
|
||||
assert await repo.count_orchestrator_events() == 3
|
||||
204
tests/orchestrator/test_scheduler.py
Normal file
204
tests/orchestrator/test_scheduler.py
Normal file
@@ -0,0 +1,204 @@
|
||||
"""Picker policy tests for the orchestrator scheduler.
|
||||
|
||||
Stage-3 realism split:
|
||||
|
||||
* :func:`scheduler.pick` is now traffic-only — sync, returns
|
||||
:class:`TrafficAction` or ``None``.
|
||||
* :func:`scheduler.pick_file` is async, takes a repo (for persona
|
||||
resolution), and returns a :class:`FileAction` driven by
|
||||
:func:`decnet.realism.planner.pick`.
|
||||
|
||||
Pre-realism behavior (one ``pick()`` returning either kind) is gone;
|
||||
the orchestrator worker rolls per tick.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from decnet.orchestrator import scheduler
|
||||
|
||||
|
||||
def _decky(
|
||||
uuid: str = "u1",
|
||||
name: str = "decky-01",
|
||||
ip: str | None = "10.0.0.1",
|
||||
services: list[str] | str = ("ssh",),
|
||||
*,
|
||||
source: str = "topology",
|
||||
topology_id: str | None = "t1",
|
||||
) -> dict:
|
||||
return {
|
||||
"uuid": uuid,
|
||||
"name": name,
|
||||
"ip": ip,
|
||||
"services": list(services) if not isinstance(services, str) else services,
|
||||
"source": source,
|
||||
"topology_id": topology_id,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sync pick() — traffic only.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_pick_returns_none_when_no_ssh_deckies():
|
||||
deckies = [
|
||||
_decky("u1", services=["http"]),
|
||||
_decky("u2", services=["smb"]),
|
||||
]
|
||||
assert scheduler.pick(deckies) is None
|
||||
|
||||
|
||||
def test_pick_returns_none_with_single_ssh_decky():
|
||||
# Traffic needs a pair; one decky alone can't generate inter-decky
|
||||
# SSH probes. Realism file actions reach this single decky via the
|
||||
# async pick_file() entry point instead.
|
||||
deckies = [_decky()]
|
||||
assert scheduler.pick(deckies) is None
|
||||
|
||||
|
||||
def test_pick_returns_none_when_ssh_decky_has_no_ip():
|
||||
deckies = [_decky(ip=None)]
|
||||
assert scheduler.pick(deckies) is None
|
||||
|
||||
|
||||
def test_pick_traffic_with_two_ssh_deckies():
|
||||
deckies = [
|
||||
_decky("u1", "decky-01", "10.0.0.1", ["ssh"]),
|
||||
_decky("u2", "decky-02", "10.0.0.2", ["ssh"]),
|
||||
]
|
||||
for _ in range(20):
|
||||
action = scheduler.pick(deckies)
|
||||
assert isinstance(action, scheduler.TrafficAction)
|
||||
assert action.src_uuid != action.dst_uuid
|
||||
assert action.dst_ip in {"10.0.0.1", "10.0.0.2"}
|
||||
assert action.protocol == "ssh"
|
||||
|
||||
|
||||
def test_pick_skips_non_deserialised_services():
|
||||
"""If services is still a JSON string (defensive), the decky is excluded."""
|
||||
deckies = [_decky(services='["ssh"]')]
|
||||
assert scheduler.pick(deckies) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Async pick_file() — realism-driven file actions.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_PERSONAS_TWO = [
|
||||
{
|
||||
"name": "admin",
|
||||
"email": "admin@corp.com",
|
||||
"role": "ops",
|
||||
"tone": "direct",
|
||||
"mannerisms": [],
|
||||
"active_hours": "00:00-00:00", # always-on for predictability
|
||||
},
|
||||
{
|
||||
"name": "ubuntu",
|
||||
"email": "ubuntu@corp.com",
|
||||
"role": "service",
|
||||
"tone": "casual",
|
||||
"mannerisms": [],
|
||||
"active_hours": "00:00-00:00",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
class _FakeRepo:
|
||||
"""Minimal repo with just the methods scheduler.pick_file needs."""
|
||||
|
||||
def __init__(self, *, topologies=None, fleet_pool=None):
|
||||
self._topologies = topologies or {}
|
||||
# Fleet/global pool gets read via realism.personas_pool.load();
|
||||
# the test pins the pool path via env in fleet-source tests.
|
||||
|
||||
async def get_topology(self, topology_id):
|
||||
return self._topologies.get(topology_id)
|
||||
|
||||
|
||||
def _topology_row(personas):
|
||||
import json
|
||||
return {
|
||||
"id": "t1",
|
||||
"email_personas": json.dumps(personas),
|
||||
"language_default": "en",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_file_returns_none_when_no_ssh_deckies():
|
||||
repo = _FakeRepo(topologies={"t1": _topology_row(_PERSONAS_TWO)})
|
||||
deckies = [_decky(services=["http"])]
|
||||
assert await scheduler.pick_file(deckies, repo) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_file_returns_none_when_topology_has_no_personas():
|
||||
repo = _FakeRepo(topologies={"t1": _topology_row([])})
|
||||
deckies = [_decky()]
|
||||
assert await scheduler.pick_file(deckies, repo) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_file_produces_file_action_for_topology_decky():
|
||||
import random as _r
|
||||
repo = _FakeRepo(topologies={"t1": _topology_row(_PERSONAS_TWO)})
|
||||
deckies = [_decky()]
|
||||
# Pin the RNG so the 3% canary gate (stage 7) and 10% leave-alone
|
||||
# roll don't flake this test. Seed 1 lands on a vanilla create.
|
||||
action = await scheduler.pick_file(
|
||||
deckies, repo,
|
||||
now=datetime(2026, 4, 27, 12, 0, tzinfo=timezone.utc),
|
||||
rand=_r.Random(1),
|
||||
)
|
||||
assert isinstance(action, scheduler.FileAction)
|
||||
assert action.dst_uuid == "u1"
|
||||
assert action.persona in {"admin", "ubuntu"}
|
||||
assert action.path.startswith("/")
|
||||
assert action.content
|
||||
assert action.mtime is not None
|
||||
# mtime must be in the past (the realism failure today is
|
||||
# wall-clock-now stamps).
|
||||
assert action.mtime < datetime(2026, 4, 27, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_file_skips_decky_when_personas_outside_window():
|
||||
out_of_hours = [{**p, "active_hours": "01:00-02:00"} for p in _PERSONAS_TWO]
|
||||
repo = _FakeRepo(topologies={"t1": _topology_row(out_of_hours)})
|
||||
deckies = [_decky()]
|
||||
action = await scheduler.pick_file(
|
||||
deckies, repo,
|
||||
now=datetime(2026, 4, 27, 12, 0, tzinfo=timezone.utc),
|
||||
)
|
||||
assert action is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_file_uses_global_pool_for_fleet_source(tmp_path, monkeypatch):
|
||||
import json
|
||||
import random as _r
|
||||
pool = tmp_path / "personas.json"
|
||||
pool.write_text(json.dumps(_PERSONAS_TWO))
|
||||
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(pool))
|
||||
|
||||
# Reset the global cache so the new pool path takes effect.
|
||||
from decnet.realism import personas_pool
|
||||
personas_pool.reset_cache()
|
||||
|
||||
repo = _FakeRepo() # no topology rows — fleet path
|
||||
deckies = [_decky(source="fleet", topology_id=None)]
|
||||
|
||||
# Pin the RNG so the canary / leave-alone rolls don't flake.
|
||||
action = await scheduler.pick_file(
|
||||
deckies, repo,
|
||||
now=datetime(2026, 4, 27, 12, 0, tzinfo=timezone.utc),
|
||||
rand=_r.Random(1),
|
||||
)
|
||||
assert isinstance(action, scheduler.FileAction)
|
||||
assert action.dst_uuid == "u1"
|
||||
173
tests/orchestrator/test_ssh_driver.py
Normal file
173
tests/orchestrator/test_ssh_driver.py
Normal file
@@ -0,0 +1,173 @@
|
||||
"""Driver tests with the docker subprocess mocked.
|
||||
|
||||
We don't need a real Docker daemon to validate the driver's contract:
|
||||
it boils down to "build an argv, call _run, classify the result". A
|
||||
dependency-injected ``_run`` keeps the tests hermetic.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from decnet.orchestrator.drivers import ssh as ssh_driver
|
||||
from decnet.orchestrator.drivers.base import ActivityResult
|
||||
from decnet.orchestrator.scheduler import FileAction, TrafficAction
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_traffic_success_classifies_on_ssh_banner(monkeypatch):
|
||||
captured_argv: list[list[str]] = []
|
||||
|
||||
async def fake_run(argv):
|
||||
captured_argv.append(argv)
|
||||
return 0, "SSH-2.0-OpenSSH_9.6\r\n", ""
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run", fake_run)
|
||||
drv = ssh_driver.SSHDriver()
|
||||
action = TrafficAction(
|
||||
src_uuid="u1", src_name="decky-01",
|
||||
dst_uuid="u2", dst_name="decky-02",
|
||||
dst_ip="10.0.0.2",
|
||||
)
|
||||
result = await drv.run(action)
|
||||
assert isinstance(result, ActivityResult)
|
||||
assert result.success is True
|
||||
assert result.payload["banner"].startswith("SSH-2.0-OpenSSH")
|
||||
assert captured_argv[0][:3] == ["docker", "exec", "decky-01-ssh"]
|
||||
assert captured_argv[0][-1] == "10.0.0.2"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_traffic_failure_when_banner_missing(monkeypatch):
|
||||
async def fake_run(argv):
|
||||
return 1, "", "Connection refused"
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run", fake_run)
|
||||
drv = ssh_driver.SSHDriver()
|
||||
action = TrafficAction(
|
||||
src_uuid="u1", src_name="decky-01",
|
||||
dst_uuid="u2", dst_name="decky-02",
|
||||
dst_ip="10.0.0.2",
|
||||
)
|
||||
result = await drv.run(action)
|
||||
assert result.success is False
|
||||
assert result.payload["rc"] == 1
|
||||
assert "Connection refused" in result.payload["stderr"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_action_invokes_docker_exec_on_dst(monkeypatch):
|
||||
captured: list[tuple[list[str], bytes | None]] = []
|
||||
|
||||
async def fake_run_with_stdin(argv, stdin_bytes):
|
||||
captured.append((argv, stdin_bytes))
|
||||
return 0, "", ""
|
||||
|
||||
# plant_file streams base64 content via stdin to avoid ARG_MAX
|
||||
# (mirrors decnet.canary.planter; see commit c17b9e0). The test
|
||||
# patches _run_with_stdin instead of _run because that's the
|
||||
# codepath FileAction now exercises.
|
||||
monkeypatch.setattr(ssh_driver, "_run_with_stdin", fake_run_with_stdin)
|
||||
drv = ssh_driver.SSHDriver()
|
||||
action = FileAction(
|
||||
dst_uuid="u2", dst_name="decky-02",
|
||||
path="/tmp/.cache-1700000000.tmp",
|
||||
content="session=1700000000\n",
|
||||
)
|
||||
result = await drv.run(action)
|
||||
assert result.success is True
|
||||
assert result.payload["bytes"] == len(b"session=1700000000\n")
|
||||
argv, stdin_bytes = captured[0]
|
||||
assert argv[:4] == ["docker", "exec", "-i", "decky-02-ssh"]
|
||||
assert argv[4] == "sh"
|
||||
assert argv[5] == "-c"
|
||||
sh_cmd = argv[6]
|
||||
assert "/tmp/.cache-1700000000.tmp" in sh_cmd
|
||||
assert "base64 -d" in sh_cmd
|
||||
assert "mkdir -p /tmp" in sh_cmd
|
||||
# Content travels base64-encoded on stdin, not interpolated into
|
||||
# argv — that's the ARG_MAX-safe + shell-injection-safe contract.
|
||||
import base64
|
||||
assert stdin_bytes is not None
|
||||
assert base64.b64decode(stdin_bytes) == b"session=1700000000\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_handles_missing_docker_binary(monkeypatch):
|
||||
async def fake_create(*args, **kwargs):
|
||||
raise FileNotFoundError("docker")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"asyncio.create_subprocess_exec", fake_create,
|
||||
)
|
||||
rc, out, err = await ssh_driver._run(["docker", "exec", "x", "true"])
|
||||
assert rc == 127
|
||||
assert "not found" in err
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plant_file_applies_mtime_via_touch_d(monkeypatch):
|
||||
from datetime import datetime, timezone
|
||||
captured: list[tuple[list[str], bytes | None]] = []
|
||||
|
||||
async def fake_run_with_stdin(argv, stdin_bytes):
|
||||
captured.append((argv, stdin_bytes))
|
||||
return 0, "", ""
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run_with_stdin", fake_run_with_stdin)
|
||||
drv = ssh_driver.SSHDriver()
|
||||
mtime = datetime(2026, 4, 20, 11, 30, 0, tzinfo=timezone.utc)
|
||||
result = await drv.plant_file(
|
||||
"decky-03", "/home/admin/TODO.md", b"- [ ] rotate keys\n",
|
||||
mode=0o644, mtime=mtime,
|
||||
)
|
||||
assert result.success is True
|
||||
sh_cmd = captured[0][0][6]
|
||||
# Backdated mtime appears in the touch -d argument.
|
||||
assert "touch -d '2026-04-20 11:30:00 UTC'" in sh_cmd
|
||||
assert "chmod 644" in sh_cmd
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_file_returns_bytes(monkeypatch):
|
||||
async def fake_run(argv):
|
||||
assert argv[:3] == ["docker", "exec", "decky-04-ssh"]
|
||||
assert argv[3] == "cat"
|
||||
return 0, "previous body\n", ""
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run", fake_run)
|
||||
drv = ssh_driver.SSHDriver()
|
||||
body = await drv.read_file("decky-04", "/home/admin/notes.txt")
|
||||
assert body == b"previous body\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_file_raises_file_not_found(monkeypatch):
|
||||
async def fake_run(argv):
|
||||
return 1, "", "cat: /nope: No such file or directory"
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run", fake_run)
|
||||
drv = ssh_driver.SSHDriver()
|
||||
with pytest.raises(FileNotFoundError):
|
||||
await drv.read_file("decky-04", "/nope")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_driver_for_dispatches_by_action_type():
|
||||
from decnet.orchestrator.drivers import get_driver_for, SSHDriver
|
||||
traffic = TrafficAction(
|
||||
src_uuid="u1", src_name="a", dst_uuid="u2", dst_name="b",
|
||||
dst_ip="10.0.0.1",
|
||||
)
|
||||
file_a = FileAction(
|
||||
dst_uuid="u2", dst_name="b", path="/tmp/x", content="y",
|
||||
)
|
||||
assert isinstance(get_driver_for(traffic), SSHDriver)
|
||||
assert isinstance(get_driver_for(file_a), SSHDriver)
|
||||
|
||||
|
||||
def test_get_driver_for_unknown_action_raises():
|
||||
from decnet.orchestrator.drivers import get_driver_for
|
||||
class _Bogus:
|
||||
pass
|
||||
with pytest.raises(TypeError, match="no driver registered"):
|
||||
get_driver_for(_Bogus()) # type: ignore[arg-type]
|
||||
294
tests/orchestrator/test_worker_integration.py
Normal file
294
tests/orchestrator/test_worker_integration.py
Normal file
@@ -0,0 +1,294 @@
|
||||
"""End-to-end-ish: run one orchestrator tick against a real SQLite repo +
|
||||
FakeBus, with the docker subprocess stubbed. Verifies that:
|
||||
|
||||
* :func:`scheduler.pick` reads the deckies the repo returns,
|
||||
* the driver result is persisted to ``orchestrator_events``,
|
||||
* a bus event is published to the right topic.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from decnet.bus.fake import FakeBus
|
||||
from decnet.orchestrator import worker as orch_worker
|
||||
from decnet.orchestrator.drivers import ssh as ssh_driver
|
||||
from decnet.web.db.models import TopologyDecky, Topology
|
||||
from decnet.web.db.sqlite.repository import SQLiteRepository
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def repo(tmp_path):
|
||||
r = SQLiteRepository(db_path=str(tmp_path / "decnet.db"))
|
||||
await r.initialize()
|
||||
yield r
|
||||
await r.engine.dispose()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def fake_bus():
|
||||
bus = FakeBus()
|
||||
await bus.connect()
|
||||
try:
|
||||
yield bus
|
||||
finally:
|
||||
await bus.close()
|
||||
|
||||
|
||||
async def _seed_two_running_ssh_deckies(repo: SQLiteRepository) -> tuple[str, str]:
|
||||
async with repo._session() as session:
|
||||
topo = Topology(name="t1", config_snapshot="{}", status="active")
|
||||
session.add(topo)
|
||||
await session.commit()
|
||||
await session.refresh(topo)
|
||||
d1 = TopologyDecky(
|
||||
topology_id=topo.id, name="decky-01",
|
||||
services=json.dumps(["ssh"]), ip="10.0.0.1", state="running",
|
||||
)
|
||||
d2 = TopologyDecky(
|
||||
topology_id=topo.id, name="decky-02",
|
||||
services=json.dumps(["ssh"]), ip="10.0.0.2", state="running",
|
||||
)
|
||||
session.add(d1)
|
||||
session.add(d2)
|
||||
await session.commit()
|
||||
await session.refresh(d1)
|
||||
await session.refresh(d2)
|
||||
return d1.uuid, d2.uuid
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_tick_records_event_and_publishes(repo, fake_bus, monkeypatch):
|
||||
await _seed_two_running_ssh_deckies(repo)
|
||||
|
||||
# Pretend every docker exec succeeds with an SSH banner; that lets
|
||||
# both action kinds (traffic + file) land as success rows so the
|
||||
# assertions below don't have to care which one the scheduler picked.
|
||||
async def fake_run(argv):
|
||||
if argv[3] == "python3":
|
||||
return 0, "SSH-2.0-OpenSSH_9.6\r\n", ""
|
||||
return 0, "", ""
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run", fake_run)
|
||||
|
||||
async def fake_run_with_stdin(argv, stdin_bytes):
|
||||
# plant_file takes the base64-streaming path; treat any docker
|
||||
# exec write as a successful no-op for the integration test.
|
||||
return 0, "", ""
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run_with_stdin", fake_run_with_stdin)
|
||||
|
||||
received: list = []
|
||||
|
||||
async def collect():
|
||||
async with fake_bus.subscribe("orchestrator.>") as sub:
|
||||
async for ev in sub:
|
||||
received.append(ev)
|
||||
if len(received) >= 1:
|
||||
return
|
||||
|
||||
import asyncio
|
||||
collector = asyncio.create_task(collect())
|
||||
# Yield once so the subscription is registered before we publish.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
await orch_worker._one_tick(repo, fake_bus)
|
||||
|
||||
await asyncio.wait_for(collector, timeout=2.0)
|
||||
|
||||
rows = await repo.list_orchestrator_events(limit=10)
|
||||
assert len(rows) == 1
|
||||
row = rows[0]
|
||||
assert row["success"] is True
|
||||
assert row["protocol"] == "ssh"
|
||||
assert row["kind"] in {"traffic", "file"}
|
||||
|
||||
assert len(received) == 1
|
||||
ev = received[0]
|
||||
assert ev.topic.startswith("orchestrator.")
|
||||
assert ev.payload["success"] is True
|
||||
assert ev.payload["kind"] == row["kind"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_tick_picks_fleet_deckies(repo, fake_bus, monkeypatch):
|
||||
"""Regression: orchestrator was permanently blind to unihost MACVLAN /
|
||||
IPVLAN deckies because list_running_topology_deckies only scans
|
||||
topology_deckies. The new union view (list_running_deckies) must
|
||||
pull in fleet_deckies rows too."""
|
||||
await repo.upsert_fleet_decky({
|
||||
"host_uuid": "local",
|
||||
"name": "fleet-d1",
|
||||
"services": ["ssh"],
|
||||
"decky_ip": "10.0.0.50",
|
||||
"state": "running",
|
||||
})
|
||||
await repo.upsert_fleet_decky({
|
||||
"host_uuid": "local",
|
||||
"name": "fleet-d2",
|
||||
"services": ["ssh"],
|
||||
"decky_ip": "10.0.0.51",
|
||||
"state": "running",
|
||||
})
|
||||
|
||||
async def fake_run(argv):
|
||||
if argv[3] == "python3":
|
||||
return 0, "SSH-2.0-OpenSSH_9.6\r\n", ""
|
||||
return 0, "", ""
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run", fake_run)
|
||||
|
||||
async def fake_run_with_stdin(argv, stdin_bytes):
|
||||
# plant_file takes the base64-streaming path; treat any docker
|
||||
# exec write as a successful no-op for the integration test.
|
||||
return 0, "", ""
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run_with_stdin", fake_run_with_stdin)
|
||||
|
||||
await orch_worker._one_tick(repo, fake_bus)
|
||||
|
||||
rows = await repo.list_orchestrator_events(limit=10)
|
||||
assert len(rows) == 1
|
||||
# The dst_decky_uuid is our composite "host_uuid:name" identifier
|
||||
# for fleet-source rows (see SQLModelRepository.list_running_deckies).
|
||||
assert rows[0]["dst_decky_uuid"].startswith("local:fleet-")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_tick_email_branch_records_orchestrator_email(
|
||||
repo, fake_bus, monkeypatch,
|
||||
):
|
||||
"""Stage 5 contract: email actions land via the unified orchestrator.
|
||||
|
||||
The pre-collapse path was a separate ``decnet emailgen run`` worker;
|
||||
after the realism migration the orchestrator's tick handles email
|
||||
drops alongside traffic + file via the action-kind roll. This test
|
||||
seeds a topology with a mail decky + two personas, forces the
|
||||
action roll to ``email``, stubs the LLM + docker-exec write paths,
|
||||
and verifies an ``orchestrator_emails`` row + bus event land.
|
||||
"""
|
||||
import json
|
||||
from decnet.orchestrator.drivers import email as email_driver
|
||||
from decnet.realism.llm.impl.fake import FakeBackend
|
||||
|
||||
personas = [
|
||||
{
|
||||
"name": "John Smith", "email": "john@corp.com", "role": "COO",
|
||||
"tone": "formal", "mannerisms": ["uses 'Best regards'"],
|
||||
"active_hours": "00:00-00:00",
|
||||
},
|
||||
{
|
||||
"name": "Sarah Johnson", "email": "sarah@corp.com", "role": "PM",
|
||||
"tone": "direct", "mannerisms": ["uses bullets"],
|
||||
"active_hours": "00:00-00:00",
|
||||
},
|
||||
]
|
||||
async with repo._session() as session:
|
||||
topo = Topology(
|
||||
name="t-email", config_snapshot="{}", status="active",
|
||||
email_personas=json.dumps(personas),
|
||||
)
|
||||
session.add(topo)
|
||||
await session.commit()
|
||||
await session.refresh(topo)
|
||||
mail_decky = TopologyDecky(
|
||||
topology_id=topo.id, name="mailhost",
|
||||
services=json.dumps(["imap"]), ip="10.0.0.5", state="running",
|
||||
)
|
||||
session.add(mail_decky)
|
||||
await session.commit()
|
||||
|
||||
# Force the worker's action roll to the email branch — no SSH-capable
|
||||
# deckies exist in this seed (only IMAP), so traffic/file drop to
|
||||
# None and email is the only viable branch anyway, but we pin the
|
||||
# roll for determinism.
|
||||
monkeypatch.setattr(orch_worker, "_roll_action_kind", lambda _rng: "email")
|
||||
|
||||
# Stub the LLM so we don't shell out to ollama. The driver
|
||||
# constructs its own backend in __init__; we patch get_driver_for
|
||||
# to return a driver with a FakeBackend pre-injected.
|
||||
fake_eml = (
|
||||
"Subject: Q3 ops review\n\n"
|
||||
"Hi Sarah,\n\nQuick note on the Q3 review.\n\nBest regards,\nJohn\n"
|
||||
)
|
||||
fake_llm = FakeBackend(output=fake_eml)
|
||||
fake_driver = email_driver.EmailDriver(llm=fake_llm)
|
||||
|
||||
def _factory(action):
|
||||
from decnet.orchestrator.emailgen.scheduler import EmailAction as _EA
|
||||
if isinstance(action, _EA):
|
||||
return fake_driver
|
||||
from decnet.orchestrator.drivers import get_driver_for as _real
|
||||
return _real(action)
|
||||
|
||||
monkeypatch.setattr(orch_worker, "get_driver_for", _factory)
|
||||
|
||||
# Stub the docker-exec write path on the email driver — same trick
|
||||
# the SSH driver tests use, but EmailDriver shells out via plain
|
||||
# asyncio.create_subprocess_exec.
|
||||
async def fake_create(*args, **kwargs):
|
||||
class _Stub:
|
||||
returncode = 0
|
||||
async def communicate(self, _stdin=None):
|
||||
return b"", b""
|
||||
return _Stub()
|
||||
|
||||
import asyncio as _asyncio
|
||||
monkeypatch.setattr(_asyncio, "create_subprocess_exec", fake_create)
|
||||
|
||||
received: list = []
|
||||
async def collect():
|
||||
async with fake_bus.subscribe("orchestrator.>") as sub:
|
||||
async for ev in sub:
|
||||
received.append(ev)
|
||||
if len(received) >= 1:
|
||||
return
|
||||
collector = _asyncio.create_task(collect())
|
||||
await _asyncio.sleep(0)
|
||||
|
||||
await orch_worker._one_tick(repo, fake_bus)
|
||||
await _asyncio.wait_for(collector, timeout=2.0)
|
||||
|
||||
# The email branch lands in orchestrator_emails, NOT
|
||||
# orchestrator_events — separate table, separate kind discriminant.
|
||||
emails = await repo.list_orchestrator_emails(limit=10)
|
||||
assert len(emails) == 1
|
||||
row = emails[0]
|
||||
assert row["mail_decky_uuid"] == mail_decky.uuid
|
||||
assert row["sender_email"] in {"john@corp.com", "sarah@corp.com"}
|
||||
assert row["recipient_email"] in {"john@corp.com", "sarah@corp.com"}
|
||||
assert row["sender_email"] != row["recipient_email"]
|
||||
assert row["subject"]
|
||||
assert row["success"] is True
|
||||
|
||||
# Bus event topic discriminator + payload kind agree.
|
||||
assert len(received) == 1
|
||||
ev = received[0]
|
||||
assert ev.topic.startswith("orchestrator.email.")
|
||||
assert ev.payload["kind"] == "email"
|
||||
assert ev.payload["mail_decky_uuid"] == mail_decky.uuid
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tick_is_noop_when_no_running_deckies(repo, fake_bus, monkeypatch):
|
||||
called = False
|
||||
|
||||
async def fake_run(argv):
|
||||
nonlocal called
|
||||
called = True
|
||||
return 0, "SSH-2.0-foo", ""
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run", fake_run)
|
||||
|
||||
async def fake_run_with_stdin(argv, stdin_bytes):
|
||||
# plant_file takes the base64-streaming path; treat any docker
|
||||
# exec write as a successful no-op for the integration test.
|
||||
return 0, "", ""
|
||||
|
||||
monkeypatch.setattr(ssh_driver, "_run_with_stdin", fake_run_with_stdin)
|
||||
await orch_worker._one_tick(repo, fake_bus)
|
||||
|
||||
assert called is False
|
||||
assert await repo.list_orchestrator_events(limit=10) == []
|
||||
Reference in New Issue
Block a user