merge: testing → main (reconcile 2-week divergence)

decnet/web/router/realism/api_config.py

@@ -0,0 +1,115 @@
+"""GET/PUT ``/api/v1/realism/config`` — operator-tunable realism knobs.
+
+Today only the planner's content-class weights + canary probability
+are exposed. The wire shape mirrors what
+:func:`decnet.realism.planner.current_payload` produces and
+:func:`decnet.realism.planner.apply_payload` consumes.
+
+Reads accept viewer; writes are admin (writes mutate sampling
+behaviour across the whole orchestrator fleet, same trust level as
+the persona-pool surface).
+
+The orchestrator worker periodically re-loads from the
+``realism_config`` table; the API process applies overrides locally
+on PUT so the GET-after-PUT round-trip reflects the change without
+waiting for the orchestrator's next refresh tick.
+"""
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from decnet.logging import get_logger
+from decnet.realism import planner
+from decnet.telemetry import traced as _traced
+from decnet.web.dependencies import repo, require_admin, require_viewer
+
+router = APIRouter()
+log = get_logger("api.realism.config")
+
+_CONFIG_KEY = "weights"
+
+
+@router.get(
+    "/realism/config",
+    tags=["Realism"],
+    responses={
+        401: {"description": "Could not validate credentials"},
+        403: {"description": "Insufficient permissions"},
+    },
+)
+@_traced("api.realism.get_config")
+async def get_config(
+    user: dict = Depends(require_viewer),
+) -> dict[str, Any]:
+    """Return the live planner config in this API process.
+
+    Note: the API process and the orchestrator worker each carry their
+    own in-memory copy of the planner config. After a fresh API
+    restart the ``realism_config`` row is loaded into this process the
+    first time GET is called; subsequent reads are local.
+    """
+    # Lazy hydration — first call after restart pulls from DB so the
+    # admin sees what the orchestrator is actually using, not the
+    # baked-in defaults.
+    row = await repo.get_realism_config(_CONFIG_KEY)
+    if row is not None:
+        try:
+            stored = json.loads(row.get("value") or "{}")
+            if isinstance(stored, dict):
+                planner.apply_payload(stored)
+        except (json.JSONDecodeError, ValueError) as exc:
+            log.warning(
+                "api.realism.get_config: stored payload invalid, "
+                "serving defaults: %s", exc,
+            )
+    return planner.current_payload()
+
+
+@router.put(
+    "/realism/config",
+    tags=["Realism"],
+    responses={
+        400: {"description": "Invalid config payload"},
+        401: {"description": "Could not validate credentials"},
+        403: {"description": "Insufficient permissions"},
+    },
+)
+@_traced("api.realism.put_config")
+async def put_config(
+    body: dict[str, Any],
+    user: dict = Depends(require_admin),
+) -> dict[str, Any]:
+    """Replace (partial) planner config and persist to ``realism_config``.
+
+    Body shape (all fields optional — unset fields keep current value):
+
+    * ``user_class_weights``: ``[{"content_class": "note", "weight": 30}, ...]``
+    * ``system_class_weights``: same shape
+    * ``canary_class_weights``: same shape
+    * ``canary_probability``: float in [0.0, 1.0]
+
+    Validation: any structural failure raises 400 *before* the rebind,
+    so the live config never goes torn.
+    """
+    if not isinstance(body, dict):
+        raise HTTPException(status_code=400, detail="body must be an object")
+
+    try:
+        planner.apply_payload(body)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    # Persist what the planner now reflects (keeps DB in sync with the
+    # in-memory state — partial bodies merge into prior config).
+    snapshot = planner.current_payload()
+    await repo.set_realism_config(_CONFIG_KEY, json.dumps(snapshot))
+
+    log.info(
+        "api.realism.put_config user=%s canary_probability=%.4f",
+        user.get("username", user.get("uuid")),
+        snapshot["canary_probability"],
+    )
+    return snapshot

decnet/web/router/realism/api_personas.py

@@ -0,0 +1,143 @@
+"""GET/PUT ``/api/v1/realism/personas`` — global persona pool CRUD.
+
+The "global pool" is a JSON file consumed by the realism content
+engine for fleet (MACVLAN/IPVLAN) and SWARM-shard deckies — see
+:mod:`decnet.realism.personas_pool`.  MazeNET topology deckies use
+``Topology.email_personas`` instead and are configured per-topology
+elsewhere.
+
+This endpoint is the API surface behind the dashboard's "Persona
+Generation" page.  Reads accept admin or viewer; writes are admin-only
+because the persistence target is a config file the worker reads on
+its hot path.
+
+Concurrency: last-write-wins.  The pool is operator-curated and small
+(<50 entries typically); the cost of a stronger model isn't justified.
+"""
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from decnet.logging import get_logger
+from decnet.realism import personas_pool as global_pool
+from decnet.realism.personas import EmailPersona, parse_personas
+from decnet.telemetry import traced as _traced
+from decnet.web.dependencies import require_admin, require_viewer
+from decnet.web.db.models.common import MessageResponse  # noqa: F401 - response shape
+
+router = APIRouter()
+log = get_logger("api.realism.personas")
+
+
+def _serialize(personas: list[EmailPersona]) -> list[dict[str, Any]]:
+    """Pydantic → plain dicts for the response body."""
+    return [p.model_dump(exclude_none=False) for p in personas]
+
+
+@router.get(
+    "/realism/personas",
+    tags=["Emailgen"],
+    responses={
+        401: {"description": "Could not validate credentials"},
+        403: {"description": "Insufficient permissions"},
+    },
+)
+@_traced("api.realism.list_personas")
+async def list_personas(
+    user: dict = Depends(require_viewer),
+) -> dict[str, Any]:
+    """Return the current global persona pool + the resolved file path.
+
+    The ``path`` field lets the dashboard show operators where the file
+    lives on disk so a CLI-driven backup / git-tracked workflow stays
+    discoverable.
+    """
+    # Reset the in-process cache before reading so a fresh CLI-driven
+    # ``decnet realism import-personas`` shows up immediately rather
+    # than waiting on the worker's mtime check.
+    global_pool.reset_cache()
+    personas = global_pool.load()
+    return {
+        "path": str(global_pool.resolve_path()),
+        "personas": _serialize(personas),
+    }
+
+
+@router.put(
+    "/realism/personas",
+    tags=["Emailgen"],
+    responses={
+        400: {"description": "Invalid persona payload"},
+        401: {"description": "Could not validate credentials"},
+        403: {"description": "Insufficient permissions"},
+    },
+)
+@_traced("api.realism.replace_personas")
+async def replace_personas(
+    body: dict[str, Any],
+    user: dict = Depends(require_admin),
+) -> dict[str, Any]:
+    """Replace the entire global pool with the supplied list.
+
+    Body shape: ``{"personas": [<EmailPersona>, ...]}``.
+
+    Validation is the same path the worker uses (``parse_personas``):
+    invalid entries are dropped with a warning rather than failing the
+    whole request — operators see exactly what landed by reading back
+    the GET response.  An entirely-invalid payload returns 400.
+    """
+    raw = body.get("personas")
+    if not isinstance(raw, list):
+        raise HTTPException(
+            status_code=400,
+            detail="body.personas must be a list",
+        )
+
+    parsed = parse_personas(raw)
+    if raw and not parsed:
+        # Operator sent a non-empty list and *every* entry was invalid —
+        # almost certainly a schema mistake on their side; fail loudly
+        # rather than silently writing an empty pool.
+        raise HTTPException(
+            status_code=400,
+            detail=(
+                "All persona entries failed validation. Required fields: "
+                "name, email (user@host.tld), role, tone, mannerisms."
+            ),
+        )
+
+    dest = global_pool.resolve_path()
+    try:
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_text(
+            json.dumps(_serialize(parsed), indent=2, ensure_ascii=False),
+            encoding="utf-8",
+        )
+    except OSError as exc:
+        # Most common cause on dev boxes: ``/etc/decnet`` exists but is
+        # not writable by the API process.  Surface a 500 with the
+        # actionable hint instead of leaking a traceback.
+        log.warning(
+            "api.realism.replace_personas write failed path=%s err=%s",
+            dest, exc,
+        )
+        raise HTTPException(
+            status_code=500,
+            detail=(
+                f"Could not write persona pool at {dest}: {exc.strerror or exc}. "
+                f"Set DECNET_EMAILGEN_PERSONAS to a writable path "
+                f"(e.g. ~/.decnet/email_personas.json) and restart the API."
+            ),
+        ) from exc
+    global_pool.reset_cache()
+    log.info(
+        "api.realism.replace_personas user=%s wrote=%d path=%s",
+        user.get("username", user.get("uuid")), len(parsed), dest,
+    )
+    return {
+        "path": str(dest),
+        "personas": _serialize(parsed),
+    }

decnet/web/router/realism/api_synthetic_files.py

@@ -0,0 +1,99 @@
+"""GET ``/api/v1/realism/synthetic-files`` — browse planted realism files.
+
+The orchestrator's realism worker grows synthetic files on each decky
+(notes, TODOs, drafts, scripts, log lines, canary artifacts). The
+:class:`~decnet.web.db.models.realism.SyntheticFile` table is the
+canonical record of what's been planted where; this endpoint lets
+operators inspect the lineage without ssh'ing into a decky.
+
+Read-only. No writes — the orchestrator is the sole writer; the
+dashboard is observation surface only.
+
+The body preview (``last_body``) is repo-clipped at 64 KB
+(:data:`SYNTHETIC_FILE_BODY_LIMIT`); when the original was larger the
+detail response carries ``truncated: true`` so the operator knows what
+they're looking at.
+"""
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+
+from decnet.telemetry import traced as _traced
+from decnet.web.db.models.realism import SYNTHETIC_FILE_BODY_LIMIT
+from decnet.web.dependencies import repo, require_viewer
+
+router = APIRouter()
+
+
+@router.get(
+    "/realism/synthetic-files",
+    tags=["Realism"],
+    responses={
+        401: {"description": "Could not validate credentials"},
+        403: {"description": "Insufficient permissions"},
+        422: {"description": "Validation error"},
+    },
+)
+@_traced("api.realism.list_synthetic_files")
+async def list_synthetic_files(
+    limit: int = Query(50, ge=1, le=500),
+    offset: int = Query(0, ge=0, le=2147483647),
+    decky_uuid: Optional[str] = Query(None, max_length=64),
+    persona: Optional[str] = Query(None, max_length=128),
+    content_class: Optional[str] = Query(None, max_length=32),
+    user: dict = Depends(require_viewer),
+) -> dict[str, Any]:
+    """Paginated synthetic_files newest-first.
+
+    Filters: ``decky_uuid``, ``persona``, ``content_class``. The list
+    response strips ``last_body`` to keep the payload bounded — fetch
+    the detail endpoint for the body preview.
+    """
+    rows = await repo.list_synthetic_files(
+        decky_uuid=decky_uuid,
+        persona=persona,
+        content_class=content_class,
+        limit=limit,
+        offset=offset,
+    )
+    total = await repo.count_synthetic_files(
+        decky_uuid=decky_uuid,
+        persona=persona,
+        content_class=content_class,
+    )
+    # The list view doesn't need bodies; drop them so the response stays
+    # small even when 50 rows each carry ~64 KB. Detail endpoint returns
+    # the body.
+    for r in rows:
+        r.pop("last_body", None)
+    return {"total": total, "limit": limit, "offset": offset, "data": rows}
+
+
+@router.get(
+    "/realism/synthetic-files/{uuid}",
+    tags=["Realism"],
+    responses={
+        401: {"description": "Could not validate credentials"},
+        403: {"description": "Insufficient permissions"},
+        404: {"description": "Synthetic file not found"},
+    },
+)
+@_traced("api.realism.get_synthetic_file")
+async def get_synthetic_file(
+    uuid: str,
+    user: dict = Depends(require_viewer),
+) -> dict[str, Any]:
+    """Return one synthetic_files row including the body preview.
+
+    ``truncated`` is true when the stored body is at the cap — the
+    decky filesystem holds the canonical bytes; the master view is a
+    snapshot.
+    """
+    row = await repo.get_synthetic_file(uuid)
+    if row is None:
+        raise HTTPException(status_code=404, detail="synthetic file not found")
+    body = row.get("last_body") or ""
+    row["truncated"] = len(body) >= SYNTHETIC_FILE_BODY_LIMIT
+    return row