anti/DECNET
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(realism): synthetic_files table + planner wiring + scheduler swap

Browse Source 
Stage 3 of the realism migration. Replaces orchestrator/scheduler.py's
hardcoded _FILE_TEMPLATES/_USERS (3 templates emitting epoch-suffixed
filenames like notes-1777315854.txt with identical bodies per
template) with a persona-driven realism engine.

New surface:

- SyntheticFile SQLModel (synthetic_files table, UNIQUE on
  decky_uuid+path) — per-(decky, path) state for the future
  edit-in-place flow. Pre-v1, no _migrate_* helper.
- BaseRepository methods: record_synthetic_file,
  update_synthetic_file, list_synthetic_files,
  pick_random_synthetic_file_for_edit (used by stage 3b).
- realism/naming.py: per-content-class filename templates,
  persona-conditioned. /var/log/cron.log + logrotate skeleton for
  system-class; /home/<persona>/TODO.md, scratch.md, etc. for
  user-class. Anti-regression test pins "no 8+ digit decimals in
  basenames" (the realism failure today).
- realism/bodies.py: deterministic body templates per content_class.
  TODO body uses checkbox markdown, script body has a shebang, cron
  body matches syslog cron shape ("CRON[PID]: (user) CMD (...)").
- realism/planner.py: pick(deckies, now, rng) returns a Plan.
  Diurnal-gated, weighted user/system content split (70/30 user
  bias). Create-only in stage 3; edit branch lands in stage 3b.

Scheduler split:

- scheduler.pick is now traffic-only (sync).
- scheduler.pick_file is async, takes a repo, resolves personas
  (Topology.email_personas for topology-source deckies; global
  realism.personas_pool otherwise), and maps Plan -> FileAction.
- FileAction gains persona/content_class/mtime fields.

Worker:

- _one_tick rolls 50/50 between traffic and file each tick. After a
  successful FileAction plant, _record_synthetic_file persists or
  patches the synthetic_files row (catching the unique-constraint
  collision on re-plant of the same path).
- SSHDriver._run_file passes action.mtime through to plant_file so
  files don't all stamp at wall-clock-now.
This commit is contained in:
anti
2026-04-27 16:22:07 -04:00
parent 636c057cc5
commit cb1872c52f
15 changed files with 1541 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

197
tests/orchestrator/test_scheduler.py
View File

@@ -1,60 +1,197 @@
"""Picker policy tests for the orchestrator scheduler."""
"""Picker policy tests for the orchestrator scheduler.
Stage-3 realism split:
* :func:`scheduler.pick` is now traffic-only — sync, returns
:class:`TrafficAction` or ``None``.
* :func:`scheduler.pick_file` is async, takes a repo (for persona
resolution), and returns a :class:`FileAction` driven by
:func:`decnet.realism.planner.pick`.
Pre-realism behavior (one ``pick()`` returning either kind) is gone;
the orchestrator worker rolls per tick.
"""
from __future__ import annotations
import secrets
from datetime import datetime, timezone
import pytest
from decnet.orchestrator import scheduler
def _decky(uuid: str, name: str, ip: str | None, services: list[str] | str):
return {"uuid": uuid, "name": name, "ip": ip, "services": services}
def _decky(
uuid: str = "u1",
name: str = "decky-01",
ip: str | None = "10.0.0.1",
services: list[str] | str = ("ssh",),
*,
source: str = "topology",
topology_id: str | None = "t1",
) -> dict:
return {
"uuid": uuid,
"name": name,
"ip": ip,
"services": list(services) if not isinstance(services, str) else services,
"source": source,
"topology_id": topology_id,
}
# ---------------------------------------------------------------------------
# Sync pick() — traffic only.
# ---------------------------------------------------------------------------
def test_pick_returns_none_when_no_ssh_deckies():
deckies = [
_decky("u1", "decky-01", "10.0.0.1", ["http"]),
_decky("u2", "decky-02", "10.0.0.2", ["smb"]),
_decky("u1", services=["http"]),
_decky("u2", services=["smb"]),
]
assert scheduler.pick(deckies) is None
def test_pick_returns_none_with_single_ssh_decky():
# Traffic needs a pair; one decky alone can't generate inter-decky
# SSH probes. Realism file actions reach this single decky via the
# async pick_file() entry point instead.
deckies = [_decky()]
assert scheduler.pick(deckies) is None
def test_pick_returns_none_when_ssh_decky_has_no_ip():
deckies = [_decky("u1", "decky-01", None, ["ssh"])]
deckies = [_decky(ip=None)]
assert scheduler.pick(deckies) is None
def test_pick_file_action_with_single_ssh_decky():
deckies = [_decky("u1", "decky-01", "10.0.0.1", ["ssh"])]
rng = secrets.SystemRandom()
rng.seed = lambda *_: None # SystemRandom doesn't seed; ignore
action = scheduler.pick(deckies, rand=rng)
assert isinstance(action, scheduler.FileAction)
assert action.dst_uuid == "u1"
assert action.path.startswith("/")
assert action.content
def test_pick_traffic_or_file_with_two_ssh_deckies():
def test_pick_traffic_with_two_ssh_deckies():
deckies = [
_decky("u1", "decky-01", "10.0.0.1", ["ssh"]),
_decky("u2", "decky-02", "10.0.0.2", ["ssh"]),
]
seen_kinds: set[str] = set()
# 50/50 split — 40 trials makes both kinds essentially certain
for _ in range(40):
for _ in range(20):
action = scheduler.pick(deckies)
assert action is not None
seen_kinds.add("traffic" if isinstance(action, scheduler.TrafficAction) else "file")
if isinstance(action, scheduler.TrafficAction):
assert action.src_uuid != action.dst_uuid
assert action.dst_ip in {"10.0.0.1", "10.0.0.2"}
assert action.protocol == "ssh"
assert seen_kinds == {"traffic", "file"}
assert isinstance(action, scheduler.TrafficAction)
assert action.src_uuid != action.dst_uuid
assert action.dst_ip in {"10.0.0.1", "10.0.0.2"}
assert action.protocol == "ssh"
def test_pick_skips_non_deserialised_services():
"""If services is still a JSON string (defensive), the decky is excluded."""
deckies = [_decky("u1", "decky-01", "10.0.0.1", '["ssh"]')]
deckies = [_decky(services='["ssh"]')]
assert scheduler.pick(deckies) is None
# ---------------------------------------------------------------------------
# Async pick_file() — realism-driven file actions.
# ---------------------------------------------------------------------------
_PERSONAS_TWO = [
{
"name": "admin",
"email": "admin@corp.com",
"role": "ops",
"tone": "direct",
"mannerisms": [],
"active_hours": "00:00-00:00", # always-on for predictability
},
{
"name": "ubuntu",
"email": "ubuntu@corp.com",
"role": "service",
"tone": "casual",
"mannerisms": [],
"active_hours": "00:00-00:00",
},
]
class _FakeRepo:
"""Minimal repo with just the methods scheduler.pick_file needs."""
def __init__(self, *, topologies=None, fleet_pool=None):
self._topologies = topologies or {}
# Fleet/global pool gets read via realism.personas_pool.load();
# the test pins the pool path via env in fleet-source tests.
async def get_topology(self, topology_id):
return self._topologies.get(topology_id)
def _topology_row(personas):
import json
return {
"id": "t1",
"email_personas": json.dumps(personas),
"language_default": "en",
}
@pytest.mark.asyncio
async def test_pick_file_returns_none_when_no_ssh_deckies():
repo = _FakeRepo(topologies={"t1": _topology_row(_PERSONAS_TWO)})
deckies = [_decky(services=["http"])]
assert await scheduler.pick_file(deckies, repo) is None
@pytest.mark.asyncio
async def test_pick_file_returns_none_when_topology_has_no_personas():
repo = _FakeRepo(topologies={"t1": _topology_row([])})
deckies = [_decky()]
assert await scheduler.pick_file(deckies, repo) is None
@pytest.mark.asyncio
async def test_pick_file_produces_file_action_for_topology_decky():
repo = _FakeRepo(topologies={"t1": _topology_row(_PERSONAS_TWO)})
deckies = [_decky()]
action = await scheduler.pick_file(
deckies, repo,
now=datetime(2026, 4, 27, 12, 0, tzinfo=timezone.utc),
)
assert isinstance(action, scheduler.FileAction)
assert action.dst_uuid == "u1"
assert action.persona in {"admin", "ubuntu"}
assert action.path.startswith("/")
assert action.content
assert action.mtime is not None
# mtime must be in the past (the realism failure today is
# wall-clock-now stamps).
assert action.mtime < datetime(2026, 4, 27, 12, 0, tzinfo=timezone.utc)
@pytest.mark.asyncio
async def test_pick_file_skips_decky_when_personas_outside_window():
out_of_hours = [{**p, "active_hours": "01:00-02:00"} for p in _PERSONAS_TWO]
repo = _FakeRepo(topologies={"t1": _topology_row(out_of_hours)})
deckies = [_decky()]
action = await scheduler.pick_file(
deckies, repo,
now=datetime(2026, 4, 27, 12, 0, tzinfo=timezone.utc),
)
assert action is None
@pytest.mark.asyncio
async def test_pick_file_uses_global_pool_for_fleet_source(tmp_path, monkeypatch):
import json
pool = tmp_path / "personas.json"
pool.write_text(json.dumps(_PERSONAS_TWO))
monkeypatch.setenv("DECNET_REALISM_PERSONAS", str(pool))
# Reset the global cache so the new pool path takes effect.
from decnet.realism import personas_pool
personas_pool.reset_cache()
repo = _FakeRepo() # no topology rows — fleet path
deckies = [_decky(source="fleet", topology_id=None)]
action = await scheduler.pick_file(
deckies, repo,
now=datetime(2026, 4, 27, 12, 0, tzinfo=timezone.utc),
)
assert isinstance(action, scheduler.FileAction)
assert action.dst_uuid == "u1"

68
tests/realism/test_bodies.py Normal file
View File

@@ -0,0 +1,68 @@
"""Body templates produce realistic, non-empty output per content class."""
from __future__ import annotations
import secrets
import pytest
from decnet.realism.bodies import make_body
from decnet.realism.taxonomy import ContentClass
_INERT_CLASSES = (
ContentClass.NOTE,
ContentClass.TODO,
ContentClass.DRAFT,
ContentClass.SCRIPT,
ContentClass.LOG_CRON,
ContentClass.LOG_DAEMON,
ContentClass.CACHE_TMP,
)
@pytest.mark.parametrize("cls", _INERT_CLASSES)
def test_body_is_nonempty(cls: ContentClass) -> None:
body = make_body(cls, "admin", rand=secrets.SystemRandom())
assert isinstance(body, str)
assert body.strip()
def test_todo_body_uses_checkbox_markdown() -> None:
body = make_body(ContentClass.TODO, "admin")
# Each line should look like a markdown checkbox; we don't pin the
# exact distribution because the % checked is randomised.
for line in body.strip().splitlines():
assert line.startswith("- [")
def test_script_body_starts_with_shebang() -> None:
seen_shebangs: set[str] = set()
rng = secrets.SystemRandom()
for _ in range(20):
body = make_body(ContentClass.SCRIPT, "admin", rand=rng)
assert body.startswith("#!")
seen_shebangs.add(body.splitlines()[0])
# We should pick from at least two interpreter shebangs across 20
# trials; if not, the template list collapsed.
assert len(seen_shebangs) >= 2
def test_log_cron_body_has_cron_syslog_shape() -> None:
body = make_body(ContentClass.LOG_CRON, "admin", rand=secrets.SystemRandom())
for line in body.strip().splitlines():
assert "CRON[" in line
assert "CMD (" in line
@pytest.mark.parametrize(
"cls",
[c for c in ContentClass if c.value.startswith("canary_")],
)
def test_canary_classes_raise_in_bodies(cls: ContentClass) -> None:
with pytest.raises(NotImplementedError, match="canary"):
make_body(cls, "admin")
def test_email_class_raises_in_bodies() -> None:
with pytest.raises(NotImplementedError, match="email"):
make_body(ContentClass.EMAIL, "admin")

95
tests/realism/test_naming.py Normal file
View File

@@ -0,0 +1,95 @@
"""Filename realism contracts.
The pre-realism orchestrator emitted ``notes-1777315854.txt`` —
unix-epoch suffix, instant tell. This file pins the anti-regression:
no namer is allowed to drop a raw decimal timestamp into a filename.
"""
from __future__ import annotations
import re
import secrets
import pytest
from decnet.realism.naming import make_path
from decnet.realism.taxonomy import ContentClass
_USER_CLASSES = (
ContentClass.NOTE,
ContentClass.TODO,
ContentClass.DRAFT,
ContentClass.SCRIPT,
)
_SYSTEM_CLASSES = (
ContentClass.LOG_CRON,
ContentClass.LOG_DAEMON,
ContentClass.CACHE_TMP,
)
@pytest.mark.parametrize("cls", _USER_CLASSES)
def test_user_class_paths_live_under_persona_home(cls: ContentClass) -> None:
p = make_path(cls, "admin", rand=secrets.SystemRandom())
assert p.startswith("/home/admin/"), p
@pytest.mark.parametrize("cls", _SYSTEM_CLASSES)
def test_system_class_paths_have_no_epoch_suffix(cls: ContentClass) -> None:
rng = secrets.SystemRandom()
for _ in range(20):
p = make_path(cls, "admin", rand=rng)
# The realism failure today: filenames carry raw unix epochs.
# 8+ consecutive digits in the basename is the tell.
basename = p.rsplit("/", 1)[-1]
assert not re.search(r"\d{8,}", basename), (
f"epoch-shaped suffix found in {p!r}"
)
def test_log_cron_uses_logrotate_skeleton() -> None:
seen: set[str] = set()
rng = secrets.SystemRandom()
for _ in range(40):
seen.add(make_path(ContentClass.LOG_CRON, "admin", rand=rng))
# Real cron only ever writes a fixed set of names; anything outside
# the logrotate cycle is a realism bug.
expected = {"/var/log/cron.log", "/var/log/cron.log.1", "/var/log/cron.log.2.gz"}
assert seen <= expected
# And we should see at least the canonical name across 40 trials.
assert "/var/log/cron.log" in seen
def test_cache_tmp_uses_mkstemp_shape() -> None:
p = make_path(ContentClass.CACHE_TMP, "admin")
assert re.match(r"^/tmp/\.cache-[a-z0-9]{6}$", p), p
@pytest.mark.parametrize(
"cls",
[c for c in ContentClass if c.value.startswith("canary_")],
)
def test_canary_classes_raise_in_naming(cls: ContentClass) -> None:
with pytest.raises(NotImplementedError, match="canary"):
make_path(cls, "admin")
def test_email_class_raises_in_naming() -> None:
with pytest.raises(NotImplementedError, match="email"):
make_path(ContentClass.EMAIL, "admin")
def test_persona_with_spaces_normalises_to_login() -> None:
# "John Smith" → "johnsmith" is a plausible login, so the namer
# collapses spaces rather than falling back. This pins that
# behaviour against a future overcorrection.
p = make_path(ContentClass.NOTE, "John Smith")
assert p.startswith("/home/johnsmith/")
def test_persona_with_punctuation_falls_back_to_user_home() -> None:
# A persona name with punctuation (or non-ASCII letters) can't
# cleanly become a username; the namer must fall back to
# /home/user rather than leak weird chars onto the filesystem.
p = make_path(ContentClass.NOTE, "C-3PO!")
assert p.startswith("/home/user/")

101
tests/realism/test_planner.py Normal file
View File

@@ -0,0 +1,101 @@
"""Realism planner — picks (decky, persona, class, action, mtime).
Stage 3 ships create-only plans; the edit branch lands in 3b. Tests
pin the diurnal gate, the eligibility filter, and the create
contract.
"""
from __future__ import annotations
import random
from datetime import datetime, timezone
import pytest
from decnet.realism.personas import EmailPersona
from decnet.realism.planner import pick
from decnet.realism.taxonomy import ContentClass
def _persona(name: str = "admin", window: str = "00:00-00:00") -> EmailPersona:
return EmailPersona(
name=name,
email=f"{name}@corp.com",
role="ops",
tone="direct",
active_hours=window,
)
def _decky(uuid: str = "u1", name: str = "decky-01", personas=None) -> dict:
return {
"uuid": uuid,
"name": name,
"_realism_personas": personas or [_persona()],
}
_NOW = datetime(2026, 4, 27, 14, 0, tzinfo=timezone.utc)
def test_pick_returns_none_when_no_deckies() -> None:
assert pick([], _NOW) is None
def test_pick_returns_none_when_decky_has_no_personas() -> None:
assert pick([{"uuid": "u1", "name": "d", "_realism_personas": []}], _NOW) is None
def test_pick_filters_personas_outside_window() -> None:
# A persona pegged to 01:00-02:00 with now=14:00 must not be picked.
out_of_hours = _persona(window="01:00-02:00")
deckies = [_decky(personas=[out_of_hours])]
assert pick(deckies, _NOW) is None
def test_pick_returns_create_plan_with_mtime_in_past() -> None:
deckies = [_decky()]
plan = pick(deckies, _NOW, rand=random.Random(0))
assert plan is not None
assert plan.action == "create"
assert plan.decky_uuid == "u1"
assert plan.persona == "admin"
assert plan.target_path.startswith("/")
assert plan.body_hint
assert plan.mtime < _NOW
def test_pick_distributes_across_user_and_system_classes() -> None:
deckies = [_decky()]
seen: set[ContentClass] = set()
for seed in range(80):
plan = pick(deckies, _NOW, rand=random.Random(seed))
if plan is not None:
seen.add(plan.content_class)
# Across 80 seeds we should hit both buckets — at least one user
# class and at least one system class — otherwise the weights or
# the 70/30 split is broken.
user_classes = {c for c in seen if c.is_user_class()}
system_classes = {c for c in seen if c.is_system_class()}
assert user_classes, f"no user-class plans in 80 trials: {seen}"
assert system_classes, f"no system-class plans in 80 trials: {seen}"
def test_pick_never_returns_canary_class_in_stage3() -> None:
deckies = [_decky()]
for seed in range(40):
plan = pick(deckies, _NOW, rand=random.Random(seed))
if plan is None:
continue
assert not plan.content_class.is_canary(), (
"canary class slipped into the realism planner; cultivator "
"lands in stage 7"
)
def test_pick_persists_persona_window_in_notes() -> None:
plan = pick([_decky()], _NOW, rand=random.Random(0))
assert plan is not None
# The plan's notes carry the persona name and the window — useful
# for the dashboard's "why this file" inspector.
assert any("persona=admin" in n for n in plan.notes)
assert any("window=" in n for n in plan.notes)

116
tests/realism/test_synthetic_files_repo.py Normal file
View File

@@ -0,0 +1,116 @@
"""record / update / list / pick-for-edit on the synthetic_files table.
Stage 3 of the realism migration introduces the synthetic_files
table for per-(decky, path) state. Tests pin the contract on a
real :class:`SQLiteRepository` so SQLModel schema bugs surface here
rather than in production.
"""
from __future__ import annotations
import hashlib
from datetime import datetime, timedelta, timezone
import pytest
import pytest_asyncio
from decnet.web.db.sqlite.repository import SQLiteRepository
@pytest_asyncio.fixture
async def repo(tmp_path):
r = SQLiteRepository(db_path=str(tmp_path / "decnet.db"))
await r.initialize()
yield r
await r.engine.dispose()
def _row(
decky: str = "d1",
path: str = "/home/admin/TODO.md",
persona: str = "admin",
cls: str = "todo",
body: str = "- [ ] rotate keys\n",
ts: datetime | None = None,
) -> dict:
now = ts or datetime.now(timezone.utc)
return {
"decky_uuid": decky,
"path": path,
"persona": persona,
"content_class": cls,
"created_at": now,
"last_modified": now,
"edit_count": 0,
"content_hash": hashlib.sha256(body.encode()).hexdigest(),
"last_body": body,
}
@pytest.mark.asyncio
async def test_record_returns_uuid(repo):
uuid = await repo.record_synthetic_file(_row())
assert isinstance(uuid, str) and uuid
@pytest.mark.asyncio
async def test_unique_constraint_on_decky_path(repo):
await repo.record_synthetic_file(_row())
with pytest.raises(Exception):
await repo.record_synthetic_file(_row())
@pytest.mark.asyncio
async def test_update_synthetic_file_patches_fields(repo):
uuid = await repo.record_synthetic_file(_row())
await repo.update_synthetic_file(
uuid,
{"edit_count": 1, "last_body": "- [x] rotate keys\n"},
)
listing = await repo.list_synthetic_files(decky_uuid="d1")
assert len(listing) == 1
assert listing[0]["edit_count"] == 1
assert listing[0]["last_body"].startswith("- [x]")
@pytest.mark.asyncio
async def test_list_filters_by_decky_and_persona(repo):
await repo.record_synthetic_file(_row(decky="d1", path="/a"))
await repo.record_synthetic_file(_row(decky="d1", path="/b", persona="ubuntu"))
await repo.record_synthetic_file(_row(decky="d2", path="/c"))
by_decky = await repo.list_synthetic_files(decky_uuid="d1")
assert {r["path"] for r in by_decky} == {"/a", "/b"}
by_persona = await repo.list_synthetic_files(decky_uuid="d1", persona="ubuntu")
assert {r["path"] for r in by_persona} == {"/b"}
@pytest.mark.asyncio
async def test_pick_random_returns_none_when_empty(repo):
assert await repo.pick_random_synthetic_file_for_edit("d-empty") is None
@pytest.mark.asyncio
async def test_pick_random_excludes_canary_classes(repo):
# Canary-class files are stored on the same table (stage 7) but
# the editor must skip them — their bodies are binary blobs.
await repo.record_synthetic_file(_row(cls="canary_aws_creds"))
picked = await repo.pick_random_synthetic_file_for_edit("d1")
assert picked is None
@pytest.mark.asyncio
async def test_pick_random_excludes_too_old_rows(repo):
old = datetime.now(timezone.utc) - timedelta(days=120)
await repo.record_synthetic_file(_row(ts=old))
picked = await repo.pick_random_synthetic_file_for_edit("d1", max_age_days=30)
assert picked is None
@pytest.mark.asyncio
async def test_pick_random_returns_eligible_row(repo):
await repo.record_synthetic_file(_row(cls="todo"))
picked = await repo.pick_random_synthetic_file_for_edit("d1")
assert picked is not None
assert picked["content_class"] == "todo"
assert picked["path"] == "/home/admin/TODO.md"