anti/DECNET
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: testing → main (reconcile 2-week divergence)

Browse Source
This commit is contained in:
anti
2026-04-28 18:36:00 -04:00
parent 499836c9e4
commit 862e4dbb31
1235 changed files with 160255 additions and 7996 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
decnet/web/_uvicorn_tls_scope.py Normal file
View File

@@ -0,0 +1,72 @@
"""Inject the TLS peer cert into ASGI scope — uvicorn ≤ 0.44 does not.
Uvicorn's h11/httptools HTTP protocols build the ASGI ``scope`` dict
without any ``extensions.tls`` entry, so per-request cert pinning
handlers (like POST /swarm/heartbeat) can't see the client cert that
CERT_REQUIRED already validated at handshake.
We patch ``RequestResponseCycle.__init__`` on both protocol modules to
read the peer cert off the asyncio transport (which *does* carry it)
and write the DER bytes into
``scope["extensions"]["tls"]["client_cert_chain"]``. This is the same
key the ASGI TLS extension proposal uses, so the application code will
keep working unchanged if a future uvicorn populates it natively.
Import this module once at app startup time (before uvicorn starts
accepting connections). Idempotent — subsequent imports are no-ops.
"""
from __future__ import annotations
from typing import Any
_PATCHED = False
def _wrap_cycle_init(cycle_cls) -> None:
original = cycle_cls.__init__
def _patched_init(self, *args: Any, **kwargs: Any) -> None:
original(self, *args, **kwargs)
transport = kwargs.get("transport") or getattr(self, "transport", None)
if transport is None:
return
ssl_obj = transport.get_extra_info("ssl_object")
if ssl_obj is None:
return
try:
der = ssl_obj.getpeercert(binary_form=True)
except Exception:
return
if not der:
return
# scope is a mutable dict uvicorn stores here; Starlette forwards
# it to handlers as request.scope. Use setdefault so we don't clobber
# any future native extension entries from uvicorn itself.
scope = self.scope
extensions = scope.setdefault("extensions", {})
extensions.setdefault("tls", {"client_cert_chain": [der]})
cycle_cls.__init__ = _patched_init
def install() -> None:
"""Patch uvicorn's HTTP cycle classes. Safe to call multiple times."""
global _PATCHED
if _PATCHED:
return
try:
from uvicorn.protocols.http import h11_impl
_wrap_cycle_init(h11_impl.RequestResponseCycle)
except Exception: # nosec B110 - optional uvicorn impl may be unavailable
pass
try:
from uvicorn.protocols.http import httptools_impl
_wrap_cycle_init(httptools_impl.RequestResponseCycle)
except Exception: # nosec B110 - optional uvicorn impl may be unavailable
pass
_PATCHED = True
# Auto-install on import so simply importing this module patches uvicorn.
install()

243
decnet/web/api.py
View File

@@ -1,59 +1,197 @@
import asyncio
import logging
# Local binding for the DB-retry sleep so tests can patch it without
# affecting `asyncio.sleep` globally (which would otherwise starve the
# heartbeat / worker loops that share the interpreter's asyncio module).
from asyncio import sleep as _retry_sleep
import os
import traceback
import uuid
from contextlib import asynccontextmanager
from typing import Any, AsyncGenerator, Optional
from fastapi import FastAPI, Request, status
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from fastapi.responses import ORJSONResponse
from pydantic import ValidationError
from fastapi.middleware.cors import CORSMiddleware
from decnet.env import DECNET_CORS_ORIGINS, DECNET_DEVELOPER, DECNET_INGEST_LOG_FILE
from decnet.env import (
DECNET_CORS_ORIGINS,
DECNET_DEVELOPER,
DECNET_EMBED_COLLECTOR,
DECNET_EMBED_PROFILER,
DECNET_EMBED_SNIFFER,
DECNET_INGEST_LOG_FILE,
DECNET_PROFILE_DIR,
DECNET_PROFILE_REQUESTS,
validate_public_binding,
)
from decnet.logging import get_logger
from decnet.web.dependencies import repo
from decnet.collector import log_collector_worker
from decnet.web.ingester import log_ingestion_worker
from decnet.profiler import attacker_profile_worker
from decnet.web.limiter import limiter
from decnet.web.router import api_router
from slowapi import _rate_limit_exceeded_handler
from slowapi.errors import RateLimitExceeded
from slowapi.middleware import SlowAPIMiddleware
log = logging.getLogger(__name__)
log = get_logger("api")
ingestion_task: Optional[asyncio.Task[Any]] = None
collector_task: Optional[asyncio.Task[Any]] = None
attacker_task: Optional[asyncio.Task[Any]] = None
sniffer_task: Optional[asyncio.Task[Any]] = None
heartbeat_task: Optional[asyncio.Task[Any]] = None
def get_background_tasks() -> dict[str, Optional[asyncio.Task[Any]]]:
"""Expose background task handles for the health endpoint."""
return {
"ingestion_worker": ingestion_task,
"collector_worker": collector_task,
"attacker_worker": attacker_task,
"sniffer_worker": sniffer_task,
}
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
global ingestion_task, collector_task
global ingestion_task, collector_task, attacker_task, sniffer_task
global heartbeat_task
import resource
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
if soft < 4096:
log.warning(
"Low open-file limit detected (ulimit -n = %d). "
"High-traffic deployments may hit 'Too many open files' errors. "
"Raise it with: ulimit -n 65536 (session) or LimitNOFILE=65536 (systemd)",
soft,
)
# Refuse to come up with a footgun config on a public binding (loopback
# CORS origin while bound to 0.0.0.0, plaintext canary base, etc.).
# Raises ValueError with an actionable message; uvicorn surfaces it.
validate_public_binding()
# Defence-in-depth on top of the CLI mode gating. Typer hides master-only
# commands when DECNET_MODE=agent, but a misconfigured systemd unit or
# a direct `python -m uvicorn decnet.web.api:app` call would bypass that.
# This raises before any worker / DB / bus comes up.
_mode = os.environ.get("DECNET_MODE", "master").lower()
_disallow = os.environ.get("DECNET_DISALLOW_MASTER", "true").lower() == "true"
if _mode == "agent" and _disallow:
raise RuntimeError(
"decnet.web.api refuses to start with DECNET_MODE=agent. "
"The master API is master-only; agents run `decnet agent` instead. "
"If this host genuinely plays both roles, set DECNET_DISALLOW_MASTER=false."
)
# Resolve DECNET_JWT_SECRET eagerly so a missing/insecure secret fails
# at boot rather than on the first request that hits an auth-gated
# endpoint. The lazy-load shape stays useful for non-master CLIs.
from decnet import env as _env
_ = _env.DECNET_JWT_SECRET # raises ValueError on missing/bad
log.info("API startup initialising database")
for attempt in range(1, 6):
try:
await repo.initialize()
log.debug("API startup DB initialised attempt=%d", attempt)
break
except Exception as exc:
log.warning("DB init attempt %d/5 failed: %s", attempt, exc)
if attempt == 5:
log.error("DB failed to initialize after 5 attempts — startup may be degraded")
await asyncio.sleep(0.5)
await _retry_sleep(0.5)
# Conditionally enable OpenTelemetry tracing
from decnet.telemetry import setup_tracing
setup_tracing(app)
# Start background tasks only if not in contract test mode
if os.environ.get("DECNET_CONTRACT_TEST") != "true":
# Start background ingestion task
if ingestion_task is None or ingestion_task.done():
ingestion_task = asyncio.create_task(log_ingestion_worker(repo))
log.debug("API startup ingest worker started")
# Start Docker log collector (writes to log file; ingester reads from it)
# Start Docker log collector (writes to log file; ingester reads from it).
# Gated on DECNET_EMBED_COLLECTOR: when `decnet-collector.service` (or
# any other standalone collector) is running, embedding a second tailer
# here writes every container line twice — the ingester then inserts
# the same event into the DB twice, which surfaces as duplicate rows
# on the dashboard.
_log_file = os.environ.get("DECNET_INGEST_LOG_FILE", DECNET_INGEST_LOG_FILE)
if _log_file and (collector_task is None or collector_task.done()):
collector_task = asyncio.create_task(log_collector_worker(_log_file))
elif not _log_file:
log.warning("DECNET_INGEST_LOG_FILE not set — Docker log collection disabled.")
if DECNET_EMBED_COLLECTOR:
if _log_file and (collector_task is None or collector_task.done()):
collector_task = asyncio.create_task(log_collector_worker(_log_file))
log.info(
"API startup: embedded collector started "
"(DECNET_EMBED_COLLECTOR=true) log_file=%s",
_log_file,
)
elif not _log_file:
log.warning("DECNET_INGEST_LOG_FILE not set — embedded collector disabled.")
else:
log.debug("API startup: collector not embedded — expecting standalone daemon")
# Start attacker profile rebuild worker only when explicitly requested.
# Default is OFF because `decnet deploy` always starts a standalone
# `decnet profiler --daemon` process. Running both against the same
# DB cursor causes events to be skipped or double-processed.
if DECNET_EMBED_PROFILER:
if attacker_task is None or attacker_task.done():
attacker_task = asyncio.create_task(attacker_profile_worker(repo))
log.info("API startup: embedded profiler started (DECNET_EMBED_PROFILER=true)")
else:
log.debug("API startup: profiler not embedded — expecting standalone daemon")
# Start fleet-wide MACVLAN sniffer only when explicitly requested.
# Default is OFF because `decnet deploy` always starts a standalone
# `decnet sniffer --daemon` process. Running both against the same
# interface produces duplicated events and wastes CPU.
if DECNET_EMBED_SNIFFER:
try:
from decnet.sniffer import sniffer_worker
if sniffer_task is None or sniffer_task.done():
sniffer_task = asyncio.create_task(sniffer_worker(_log_file))
log.info("API startup: embedded sniffer started (DECNET_EMBED_SNIFFER=true)")
except Exception as exc:
log.warning("Sniffer worker failed to start — API continues without sniffing: %s", exc)
else:
log.debug("API startup: sniffer not embedded — expecting standalone daemon")
else:
log.info("Contract Test Mode: skipping background worker startup")
# Worker registry + API self-heartbeat — always on, even under
# contract-test mode, so the Workers panel can render the process
# without the dev needing to run a full stack. A missing bus turns
# both into no-ops inside the helpers.
try:
from decnet.bus.app import get_app_bus
from decnet.bus.publish import run_health_heartbeat
from decnet.web.worker_registry import get_registry
_bus = await get_app_bus()
await get_registry().start(_bus)
if heartbeat_task is None or heartbeat_task.done():
heartbeat_task = asyncio.create_task(
run_health_heartbeat(_bus, "api"),
)
except Exception as exc: # noqa: BLE001
log.warning("worker registry bootstrap failed: %s", exc)
yield
# Shutdown background tasks
for task in (ingestion_task, collector_task):
log.info("API shutdown cancelling background tasks")
try:
from decnet.web.worker_registry import get_registry
await get_registry().stop()
except Exception as exc: # noqa: BLE001
log.warning("worker registry stop raised: %s", exc)
for task in (ingestion_task, collector_task, attacker_task, sniffer_task, heartbeat_task):
if task and not task.done():
task.cancel()
try:
@@ -62,31 +200,66 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
pass
except Exception as exc:
log.warning("Task shutdown error: %s", exc)
from decnet.bus.app import close_app_bus
await close_app_bus()
from decnet.telemetry import shutdown_tracing
shutdown_tracing()
log.info("API shutdown complete")
app: FastAPI = FastAPI(
title="DECNET Web Dashboard API",
version="1.0.0",
lifespan=lifespan,
default_response_class=ORJSONResponse,
docs_url="/docs" if DECNET_DEVELOPER else None,
redoc_url="/redoc" if DECNET_DEVELOPER else None,
openapi_url="/openapi.json" if DECNET_DEVELOPER else None
)
app.state.limiter = limiter
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
app.add_middleware(SlowAPIMiddleware)
app.add_middleware(
CORSMiddleware,
allow_origins=DECNET_CORS_ORIGINS,
allow_credentials=False,
allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
allow_headers=["Authorization", "Content-Type", "Last-Event-ID"],
)
if DECNET_PROFILE_REQUESTS:
import time
from pathlib import Path
from pyinstrument import Profiler
from starlette.middleware.base import BaseHTTPMiddleware
_profile_dir = Path(DECNET_PROFILE_DIR)
_profile_dir.mkdir(parents=True, exist_ok=True)
class PyinstrumentMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
profiler = Profiler(async_mode="enabled")
profiler.start()
try:
response = await call_next(request)
finally:
profiler.stop()
slug = request.url.path.strip("/").replace("/", "_") or "root"
out = _profile_dir / f"{int(time.time() * 1000)}-{request.method}-{slug}.html"
out.write_text(profiler.output_html())
return response
app.add_middleware(PyinstrumentMiddleware)
log.info("Pyinstrument middleware mounted — flamegraphs -> %s", _profile_dir)
# Include the modular API router
app.include_router(api_router, prefix="/api/v1")
@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
async def validation_exception_handler(request: Request, exc: RequestValidationError) -> ORJSONResponse:
"""
Handle validation errors with targeted status codes to satisfy contract tests.
Tiered Prioritization:
@@ -106,7 +279,7 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
for err in errors
)
if is_structural_violation:
return JSONResponse(
return ORJSONResponse(
status_code=status.HTTP_400_BAD_REQUEST,
content={"detail": "Bad Request: Schema structural violation (wrong type, extra fields, or invalid length)."},
)
@@ -117,7 +290,7 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
# Empty INI content (Valid string but semantically empty)
is_ini_empty = any("INI content is empty" in err.get("msg", "") for err in errors)
if is_ini_empty:
return JSONResponse(
return ORJSONResponse(
status_code=status.HTTP_409_CONFLICT,
content={"detail": "Configuration conflict: INI content is empty."},
)
@@ -126,7 +299,7 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
# Mapping to 409 for Positive Data compliance.
is_invalid_characters = any("Invalid INI format" in err.get("msg", "") for err in errors)
if is_invalid_characters:
return JSONResponse(
return ORJSONResponse(
status_code=status.HTTP_409_CONFLICT,
content={"detail": "Configuration conflict: INI syntax or characters are invalid."},
)
@@ -134,7 +307,7 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
# Logical invalidity (Valid string, valid syntax, but missing required DECNET logic like sections)
is_ini_invalid_logic = any("at least one section" in err.get("msg", "") for err in errors)
if is_ini_invalid_logic:
return JSONResponse(
return ORJSONResponse(
status_code=status.HTTP_409_CONFLICT,
content={"detail": "Invalid INI config structure: No decky sections found."},
)
@@ -149,22 +322,48 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
if "/deckies/deploy" in request.url.path:
message = "Invalid INI config"
return JSONResponse(
return ORJSONResponse(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
content={"detail": message},
)
@app.exception_handler(ValidationError)
async def pydantic_validation_exception_handler(request: Request, exc: ValidationError) -> JSONResponse:
async def pydantic_validation_exception_handler(request: Request, exc: ValidationError) -> ORJSONResponse:
"""
Handle Pydantic errors that occur during manual model instantiation (e.g. state hydration).
Prevents 500 errors when the database contains inconsistent or outdated schema data.
"""
log.error("Internal Pydantic validation error: %s", exc)
return JSONResponse(
return ORJSONResponse(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
content={
"detail": "Internal data consistency error",
"type": "internal_validation_error"
},
)
@app.exception_handler(Exception)
async def unhandled_exception_handler(request: Request, exc: Exception) -> ORJSONResponse:
"""Catch-all for uncaught exceptions in route handlers and dependencies.
Prod: opaque 500 with an ``error_id``; full traceback goes ONLY to server
logs. Dev (``DECNET_DEVELOPER=True``): same response plus ``exception_type``
and ``traceback`` fields so failures are debuggable without tailing logs.
The ``error_id`` lets operators correlate a user's 500 report with the full
traceback in server logs (``grep <error_id> /var/log/decnet.log``).
FastAPI's own ``HTTPException`` routing still takes precedence — this
handler only fires on genuinely-uncaught exceptions.
"""
error_id = uuid.uuid4().hex
log.exception(
"unhandled exception on %s %s [error_id=%s]",
request.method, request.url.path, error_id,
)
body: dict[str, Any] = {"detail": "Internal Server Error", "error_id": error_id}
if DECNET_DEVELOPER:
body["exception_type"] = type(exc).__name__
body["traceback"] = traceback.format_exc()
return ORJSONResponse(status_code=500, content=body)

10
decnet/web/auth.py
View File

@@ -1,3 +1,4 @@
import asyncio
from datetime import datetime, timedelta, timezone
from typing import Optional, Any
import jwt
@@ -24,6 +25,15 @@ def get_password_hash(password: str) -> str:
return _hashed.decode("utf-8")
async def averify_password(plain_password: str, hashed_password: str) -> bool:
# bcrypt is CPU-bound and ~250ms/call; keep it off the event loop.
return await asyncio.to_thread(verify_password, plain_password, hashed_password)
async def ahash_password(password: str) -> str:
return await asyncio.to_thread(get_password_hash, password)
def create_access_token(data: dict[str, Any], expires_delta: Optional[timedelta] = None) -> str:
_to_encode: dict[str, Any] = data.copy()
_expire: datetime

29
decnet/web/db/factory.py
View File

@@ -1,18 +1,33 @@
"""
Repository factory — selects a :class:`BaseRepository` implementation based on
``DECNET_DB_TYPE`` (``sqlite`` or ``mysql``).
"""
from __future__ import annotations
import os
from typing import Any
from decnet.env import os
from decnet.web.db.repository import BaseRepository
def get_repository(**kwargs: Any) -> BaseRepository:
"""Factory function to instantiate the correct repository implementation based on environment."""
"""Instantiate the repository implementation selected by ``DECNET_DB_TYPE``.
Keyword arguments are forwarded to the concrete implementation:
* SQLite accepts ``db_path``.
* MySQL accepts ``url`` and engine tuning knobs (``pool_size``, …).
"""
db_type = os.environ.get("DECNET_DB_TYPE", "sqlite").lower()
if db_type == "sqlite":
from decnet.web.db.sqlite.repository import SQLiteRepository
return SQLiteRepository(**kwargs)
repo = SQLiteRepository(**kwargs)
elif db_type == "mysql":
# Placeholder for future implementation
# from decnet.web.db.mysql.repository import MySQLRepository
# return MySQLRepository()
raise NotImplementedError("MySQL support is planned but not yet implemented.")
from decnet.web.db.mysql.repository import MySQLRepository
repo = MySQLRepository(**kwargs)
else:
raise ValueError(f"Unsupported database type: {db_type}")
from decnet.telemetry import wrap_repository
return wrap_repository(repo)

95
decnet/web/db/models.py
View File

@@ -1,95 +0,0 @@
from datetime import datetime, timezone
from typing import Optional, Any, List, Annotated
from sqlmodel import SQLModel, Field
from pydantic import BaseModel, ConfigDict, Field as PydanticField, BeforeValidator
from decnet.models import IniContent
def _normalize_null(v: Any) -> Any:
if isinstance(v, str) and v.lower() in ("null", "undefined", ""):
return None
return v
NullableDatetime = Annotated[Optional[datetime], BeforeValidator(_normalize_null)]
NullableString = Annotated[Optional[str], BeforeValidator(_normalize_null)]
# --- Database Tables (SQLModel) ---
class User(SQLModel, table=True):
__tablename__ = "users"
uuid: str = Field(primary_key=True)
username: str = Field(index=True, unique=True)
password_hash: str
role: str = Field(default="viewer")
must_change_password: bool = Field(default=False)
class Log(SQLModel, table=True):
__tablename__ = "logs"
id: Optional[int] = Field(default=None, primary_key=True)
timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc), index=True)
decky: str = Field(index=True)
service: str = Field(index=True)
event_type: str = Field(index=True)
attacker_ip: str = Field(index=True)
raw_line: str
fields: str
msg: Optional[str] = None
class Bounty(SQLModel, table=True):
__tablename__ = "bounty"
id: Optional[int] = Field(default=None, primary_key=True)
timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc), index=True)
decky: str = Field(index=True)
service: str = Field(index=True)
attacker_ip: str = Field(index=True)
bounty_type: str = Field(index=True)
payload: str
class State(SQLModel, table=True):
__tablename__ = "state"
key: str = Field(primary_key=True)
value: str # Stores JSON serialized DecnetConfig or other state blobs
# --- API Request/Response Models (Pydantic) ---
class Token(BaseModel):
access_token: str
token_type: str
must_change_password: bool = False
class LoginRequest(BaseModel):
username: str
password: str = PydanticField(..., max_length=72)
class ChangePasswordRequest(BaseModel):
old_password: str = PydanticField(..., max_length=72)
new_password: str = PydanticField(..., max_length=72)
class LogsResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class BountyResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class StatsResponse(BaseModel):
total_logs: int
unique_attackers: int
active_deckies: int
deployed_deckies: int
class MutateIntervalRequest(BaseModel):
# Human-readable duration: <number><unit> where unit is m(inutes), d(ays), M(onths), y/Y(ears).
# Minimum granularity is 1 minute. Seconds are not accepted.
mutate_interval: Optional[str] = PydanticField(None, pattern=r"^[1-9]\d*[mdMyY]$")
class DeployIniRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
# This field now enforces strict INI structure during Pydantic initialization.
# The OpenAPI schema correctly shows it as a required string.
ini_content: IniContent = PydanticField(..., description="A valid INI formatted string")

319
decnet/web/db/models/__init__.py Normal file
View File

@@ -0,0 +1,319 @@
"""
Database tables (SQLModel) and HTTP request/response shapes (Pydantic).
Split into topical modules for readability, but every symbol is re-exported
from this package so ``from decnet.web.db.models import X`` keeps working
everywhere — no importer needs to know which submodule a class lives in.
"""
from ._base import (
NullableDatetime,
NullableString,
_BIG_TEXT,
_normalize_null,
)
from .common import (
MessageResponse,
)
from .canary import (
CanaryBlob,
CanaryBlobResponse,
CanaryBlobsResponse,
CanaryKind,
CanaryState,
CanaryToken,
CanaryTokenCreateRequest,
CanaryTokenResponse,
CanaryTokensResponse,
CanaryTrigger,
CanaryTriggerResponse,
CanaryTriggersResponse,
)
from .auth import (
AdminConfigResponse,
ChangePasswordRequest,
ConfigResponse,
CreateUserRequest,
DeploymentLimitRequest,
GlobalMutationIntervalRequest,
LoginRequest,
ResetUserPasswordRequest,
Token,
UpdateUserRoleRequest,
User,
UserResponse,
)
from .attackers import (
Attacker,
AttackerBehavior,
AttackerIdentity,
AttackersResponse,
SessionProfile,
SmtpTarget,
)
from .attacker_intel import (
AttackerIntel,
)
from .campaigns import (
Campaign,
CampaignsResponse,
)
from .deploy import (
DeployIniRequest,
DeployResponse,
MutateIntervalRequest,
PurgeResponse,
)
from .fleet import (
LOCAL_HOST_SENTINEL,
FleetDecky,
)
from .health import (
ComponentHealth,
HealthResponse,
)
from .orchestrator import (
OrchestratorEmail,
OrchestratorEmailsResponse,
OrchestratorEvent,
OrchestratorEventsResponse,
)
from .realism import (
RealismConfig,
SyntheticFile,
SyntheticFilesResponse,
)
from .logs import (
Bounty,
BountyResponse,
Credential,
CredentialReuse,
CredentialReuseResponse,
CredentialsResponse,
Log,
LogsResponse,
State,
StatsResponse,
)
from .swarm import (
DeckyShard,
DeckyShardView,
SwarmCheckResponse,
SwarmDeployRequest,
SwarmDeployResponse,
SwarmEnrolledBundle,
SwarmEnrollRequest,
SwarmHost,
SwarmHostHealth,
SwarmHostResult,
SwarmHostView,
SwarmTeardownRequest,
SwarmUpdaterBundle,
)
from .topology import (
LAN,
ArchetypeCatalogResponse,
ArchetypeEntry,
DeckyCreateRequest,
DeckyRow,
DeckyUpdateRequest,
DeployAcceptedResponse,
EdgeCreateRequest,
EdgeRow,
LANCreateRequest,
LANRow,
LANUpdateRequest,
MutationEnqueueRequest,
MutationEnqueueResponse,
MutationRow,
NextIPResponse,
NextSubnetResponse,
NotEditableResponse,
ReapReportResponse,
ServiceCatalogResponse,
Topology,
TopologyDecky,
TopologyDetail,
TopologyEdge,
TopologyGenerateRequest,
TopologyListResponse,
TopologyMutation,
TopologyStatusEvent,
TopologyStatusEventRow,
TopologySummary,
ValidationErrorResponse,
ValidationIssueResponse,
VersionConflictResponse,
)
from .updater import (
HostReleaseInfo,
HostReleasesResponse,
PushUpdateRequest,
PushUpdateResponse,
PushUpdateResult,
RollbackRequest,
RollbackResponse,
)
from .webhooks import (
SimpleEvent,
WebhookCreateRequest,
WebhookCreateResponse,
WebhookResponse,
WebhookSubscription,
WebhookTestResponse,
WebhookUpdateRequest,
)
from .workers import (
StartAllResponse,
StartFailure,
WorkerControlResponse,
WorkersResponse,
WorkerStatus,
)
__all__ = [
# _base
"NullableDatetime",
"NullableString",
"_BIG_TEXT",
"_normalize_null",
# common
"MessageResponse",
# canary
"CanaryBlob",
"CanaryBlobResponse",
"CanaryBlobsResponse",
"CanaryKind",
"CanaryState",
"CanaryToken",
"CanaryTokenCreateRequest",
"CanaryTokenResponse",
"CanaryTokensResponse",
"CanaryTrigger",
"CanaryTriggerResponse",
"CanaryTriggersResponse",
# auth
"AdminConfigResponse",
"ChangePasswordRequest",
"ConfigResponse",
"CreateUserRequest",
"DeploymentLimitRequest",
"GlobalMutationIntervalRequest",
"LoginRequest",
"ResetUserPasswordRequest",
"Token",
"UpdateUserRoleRequest",
"User",
"UserResponse",
# attackers
"Attacker",
"AttackerBehavior",
"AttackerIdentity",
"AttackerIntel",
"AttackersResponse",
"SessionProfile",
"SmtpTarget",
# campaigns
"Campaign",
"CampaignsResponse",
# deploy
"DeployIniRequest",
"DeployResponse",
"MutateIntervalRequest",
"PurgeResponse",
# fleet
"LOCAL_HOST_SENTINEL",
"FleetDecky",
# health
"ComponentHealth",
"HealthResponse",
# orchestrator
"OrchestratorEmail",
"OrchestratorEmailsResponse",
"OrchestratorEvent",
"OrchestratorEventsResponse",
# realism
"RealismConfig",
"SyntheticFile",
"SyntheticFilesResponse",
# logs
"Bounty",
"BountyResponse",
"Credential",
"CredentialReuse",
"CredentialReuseResponse",
"CredentialsResponse",
"Log",
"LogsResponse",
"State",
"StatsResponse",
# swarm
"DeckyShard",
"DeckyShardView",
"SwarmCheckResponse",
"SwarmDeployRequest",
"SwarmDeployResponse",
"SwarmEnrolledBundle",
"SwarmEnrollRequest",
"SwarmHost",
"SwarmHostHealth",
"SwarmHostResult",
"SwarmHostView",
"SwarmTeardownRequest",
"SwarmUpdaterBundle",
# topology
"LAN",
"ArchetypeCatalogResponse",
"ArchetypeEntry",
"DeckyCreateRequest",
"DeckyRow",
"DeckyUpdateRequest",
"DeployAcceptedResponse",
"EdgeCreateRequest",
"EdgeRow",
"LANCreateRequest",
"LANRow",
"LANUpdateRequest",
"MutationEnqueueRequest",
"MutationEnqueueResponse",
"MutationRow",
"NextIPResponse",
"NextSubnetResponse",
"NotEditableResponse",
"ReapReportResponse",
"ServiceCatalogResponse",
"Topology",
"TopologyDecky",
"TopologyDetail",
"TopologyEdge",
"TopologyGenerateRequest",
"TopologyListResponse",
"TopologyMutation",
"TopologyStatusEvent",
"TopologyStatusEventRow",
"TopologySummary",
"ValidationErrorResponse",
"ValidationIssueResponse",
"VersionConflictResponse",
# updater
"HostReleaseInfo",
"HostReleasesResponse",
"PushUpdateRequest",
"PushUpdateResponse",
"PushUpdateResult",
"RollbackRequest",
"RollbackResponse",
# webhooks
"SimpleEvent",
"WebhookCreateRequest",
"WebhookCreateResponse",
"WebhookResponse",
"WebhookSubscription",
"WebhookTestResponse",
"WebhookUpdateRequest",
# workers
"StartAllResponse",
"StartFailure",
"WorkerControlResponse",
"WorkersResponse",
"WorkerStatus",
]

23
decnet/web/db/models/_base.py Normal file
View File

@@ -0,0 +1,23 @@
"""Shared column/validator helpers used across model domain modules."""
from datetime import datetime
from typing import Annotated, Any, Optional
from pydantic import BeforeValidator
from sqlalchemy import Text
from sqlalchemy.dialects.mysql import MEDIUMTEXT
# Use on columns that accumulate over an attacker's lifetime (commands,
# fingerprints, state blobs). TEXT on MySQL caps at 64 KiB; MEDIUMTEXT
# stretches to 16 MiB. SQLite has no fixed-width text types so Text()
# stays unchanged there.
_BIG_TEXT = Text().with_variant(MEDIUMTEXT(), "mysql")
def _normalize_null(v: Any) -> Any:
if isinstance(v, str) and v.lower() in ("null", "undefined", ""):
return None
return v
NullableDatetime = Annotated[Optional[datetime], BeforeValidator(_normalize_null)]
NullableString = Annotated[Optional[str], BeforeValidator(_normalize_null)]

93
decnet/web/db/models/attacker_intel.py Normal file
View File

@@ -0,0 +1,93 @@
"""Threat-intel enrichment row — one per attacker IP, TTL-cached."""
from datetime import datetime, timezone
from typing import Optional
from sqlalchemy import Column
from sqlmodel import Field, SQLModel
from ._base import _BIG_TEXT
class AttackerIntel(SQLModel, table=True):
"""Aggregated threat-intel verdict for a single attacker IP.
Populated by the ``decnet enrich`` worker, which queries multiple
free-tier intel providers (GreyNoise Community, AbuseIPDB,
abuse.ch Feodo Tracker + ThreatFox) and writes one row per
attacker IP. The row is TTL-cached via ``expires_at`` so re-firings
inside the cache window short-circuit before any HTTP egress.
Per-provider columns are nullable until each provider has answered;
the enrichment pass writes whichever providers succeeded and leaves
the rest unchanged on a partial failure.
``schema_version`` is committed to storage from day one — federation
gossip in v2/v3 requires cross-operator compatibility, and
retrofitting a version column after rows exist is painful. Mirrors
the rationale on :class:`SessionProfile`.
"""
__tablename__ = "attacker_intel"
uuid: str = Field(primary_key=True) # uuid.uuid4().hex, generated by writer
# Canonical key. One intel row per attacker UUID; FK guarantees no orphan
# rows when an attacker is deleted, and UNIQUE keeps upserts honest.
attacker_uuid: str = Field(
foreign_key="attackers.uuid",
unique=True,
index=True,
)
# DENORMALISED — NOT a key. The IP the worker queried providers with at
# write time. Useful for SIEM payloads and audit lookups; updated on every
# upsert if the attacker rotates IPs. Never use this column as a lookup
# key; ``attacker_uuid`` is the only canonical identifier here.
attacker_ip: str = Field(index=True)
schema_version: int = Field(default=1)
# ── GreyNoise Community ─────────────────────────────────────────────
# classification ∈ {"benign", "malicious", "suspicious", "unknown"}
greynoise_classification: Optional[str] = Field(default=None, max_length=32)
greynoise_raw: str = Field(
default="{}",
sa_column=Column("greynoise_raw", _BIG_TEXT, nullable=False, default="{}"),
)
greynoise_queried_at: Optional[datetime] = Field(default=None)
# ── AbuseIPDB ────────────────────────────────────────────────────────
# 0..100 abuse confidence score
abuseipdb_score: Optional[int] = Field(default=None)
abuseipdb_raw: str = Field(
default="{}",
sa_column=Column("abuseipdb_raw", _BIG_TEXT, nullable=False, default="{}"),
)
abuseipdb_queried_at: Optional[datetime] = Field(default=None)
# ── abuse.ch Feodo Tracker ───────────────────────────────────────────
feodo_listed: Optional[bool] = Field(default=None)
feodo_raw: str = Field(
default="{}",
sa_column=Column("feodo_raw", _BIG_TEXT, nullable=False, default="{}"),
)
feodo_queried_at: Optional[datetime] = Field(default=None)
# ── abuse.ch ThreatFox ───────────────────────────────────────────────
threatfox_listed: Optional[bool] = Field(default=None)
threatfox_raw: str = Field(
default="{}",
sa_column=Column("threatfox_raw", _BIG_TEXT, nullable=False, default="{}"),
)
threatfox_queried_at: Optional[datetime] = Field(default=None)
# ── Aggregate verdict ────────────────────────────────────────────────
# Synthesised from per-provider columns. ∈ {"malicious", "suspicious",
# "benign", "unknown"}. Used by the dashboard and webhook consumers
# that don't want to reason over four provider columns.
aggregate_verdict: Optional[str] = Field(
default=None, max_length=32, index=True
)
# ── TTL bookkeeping ──────────────────────────────────────────────────
cached_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
expires_at: datetime = Field(index=True)

414
decnet/web/db/models/attackers.py Normal file
View File

@@ -0,0 +1,414 @@
"""Attacker core + per-attacker behavioral and per-session profile rows."""
from datetime import datetime, timezone
from typing import Any, List, Optional
from pydantic import BaseModel
from sqlalchemy import BINARY, Column, Text, UniqueConstraint
from sqlmodel import Field, SQLModel
from ._base import _BIG_TEXT
# ─── Keystroke-dynamics tuning constants ──────────────────────────────────────
#
# These are the semantic thresholds the session-profile ingester (DEBT-036)
# uses to bucket IATs and decide what "started a new action" means. Keeping
# them here (not inline in the ingester) so that:
# * the schema docstrings below can reference exact boundaries instead of
# copy-pasted magic numbers, and
# * a future calibration pass against real honeypot session data only has
# to touch one place.
# All values in seconds.
KD_PAUSE_BURST_MAX_S: float = 0.2 # IAT < this = muscle-memory digraph
KD_PAUSE_THINK_MAX_S: float = 1.5 # IAT < this = semantic / context-switch pause
# everything ≥ this lands in the distracted bucket
KD_START_OF_ACTION_IDLE_S: float = 2.0 # idle gap that counts as "new action"
# raised from 1s — 1s still catches a lot of
# mid-command hesitation, 2s is closer to
# empirical "meaningfully new action"
class Attacker(SQLModel, table=True):
"""
Per-IP **observation** row. Every distinct source IP we observe gets
one of these. The semantic role is "observation event," not "actor
identity" — an actor rotating across N IPs produces N rows here.
The deduped actor view lives in ``AttackerIdentity`` (one identity
per actor; many observations per identity); the per-operation view
lives in ``Campaign``. ``identity_id`` is set by the clusterer
worker once it resolves which observations are the same hands.
NULL while the clusterer hasn't run on this row yet.
See ``development/IDENTITY_RESOLUTION.md`` for the three-level
hierarchy rationale.
"""
__tablename__ = "attackers"
uuid: str = Field(primary_key=True)
ip: str = Field(index=True)
identity_id: Optional[str] = Field(
default=None,
foreign_key="attacker_identities.uuid",
index=True,
)
first_seen: datetime = Field(index=True)
last_seen: datetime = Field(index=True)
event_count: int = Field(default=0)
service_count: int = Field(default=0)
decky_count: int = Field(default=0)
# JSON blobs — these grow over the attacker's lifetime. Use MEDIUMTEXT on
# MySQL (16 MiB) for the fields that accumulate (fingerprints, commands,
# and the deckies/services lists that are unbounded in principle).
services: str = Field(
default="[]", sa_column=Column("services", _BIG_TEXT, nullable=False, default="[]")
) # JSON list[str]
deckies: str = Field(
default="[]", sa_column=Column("deckies", _BIG_TEXT, nullable=False, default="[]")
) # JSON list[str], first-contact ordered
traversal_path: Optional[str] = Field(
default=None, sa_column=Column("traversal_path", Text, nullable=True)
) # "decky-01 → decky-03 → decky-05"
is_traversal: bool = Field(default=False)
bounty_count: int = Field(default=0)
credential_count: int = Field(default=0)
fingerprints: str = Field(
default="[]", sa_column=Column("fingerprints", _BIG_TEXT, nullable=False, default="[]")
) # JSON list[dict] — bounty fingerprints
commands: str = Field(
default="[]", sa_column=Column("commands", _BIG_TEXT, nullable=False, default="[]")
) # JSON list[dict] — commands per service/decky
# GeoIP enrichment (populated by the profiler from decnet.geoip.enrich_ip).
# Nullable because private / loopback / IPv6 sources never resolve.
country_code: Optional[str] = Field(default=None, max_length=2, index=True)
country_source: Optional[str] = Field(default=None, max_length=16)
# ASN enrichment (populated by the profiler from decnet.asn.enrich_ip).
# Nullable for the same reasons as country_code, plus IPs not currently
# announced in the global BGP table (e.g. CGNAT, dark space).
asn: Optional[int] = Field(default=None, index=True)
as_name: Optional[str] = Field(default=None, max_length=128)
asn_source: Optional[str] = Field(default=None, max_length=16)
# Reverse-DNS (PTR) name, one-shot resolved by the profiler at first
# sighting. Nullable — many attackers run infra with no rDNS, and
# private/loopback addresses never resolve. 256 chars matches
# RFC 1035 max hostname length.
ptr_record: Optional[str] = Field(default=None, max_length=256)
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
class AttackerIdentity(SQLModel, table=True):
"""
Resolved actor identity — the dedup'd "same hands" row that one or
more ``Attacker`` observations FK into. Populated by the (future)
clusterer worker; NULL on every observation until it runs.
Why a separate table from ``Attacker``: an actor rotating across N
IPs produces N observation rows but only ONE identity row. The
identity is recovered from signals the attacker can't cheaply
rotate — JA3, HASSH, payload hashes, C2 callbacks, and (V2)
keystroke-rhythm SimHash. See ``development/IDENTITY_RESOLUTION.md``.
All clusterer-populated fields are nullable; the table ships empty
in the schema-only PR (commit 1) and stays empty until the
clusterer lands. Empty is valid.
``schema_version`` is non-negotiable from day one. Federation
gossip in V2 will share identity vectors across operators;
bumping feature definitions without a version field silently
poisons receivers.
"""
__tablename__ = "attacker_identities"
uuid: str = Field(primary_key=True)
schema_version: int = Field(default=1)
# Set by the campaign clusterer. The ``campaigns`` table now
# exists; this is a real FK. Nullable until the campaign clusterer
# has run on this identity row.
campaign_id: Optional[str] = Field(
default=None, foreign_key="campaigns.uuid", index=True
)
first_seen_at: Optional[datetime] = Field(default=None, index=True)
last_seen_at: Optional[datetime] = Field(default=None, index=True)
created_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
# Identity-cohesion score from the clusterer. Range [0, 1]; null
# until the clusterer writes. Higher = more confident the
# observations linked to this identity are the same hands.
confidence: Optional[float] = Field(default=None)
# Denormalized count of FK'd Attacker rows. Maintained by the
# clusterer when it links/unlinks. Cheap dashboard read.
observation_count: int = Field(default=0)
# Fingerprint summary columns. JSON-serialized list[str] in TEXT
# because: (a) federation gossip wants this exact shape on the
# wire, (b) MySQL can't index BLOB/TEXT without prefix lengths,
# (c) actors can present multiple JA3/HASSH values across tools
# so a scalar column is wrong.
ja3_hashes: Optional[str] = Field(
default=None, sa_column=Column("ja3_hashes", Text, nullable=True)
)
hassh_hashes: Optional[str] = Field(
default=None, sa_column=Column("hassh_hashes", Text, nullable=True)
)
# JSON list[str] — SHA-256 fingerprints of leaf certs presented by
# attacker-run TLS servers, captured by the active prober alongside
# JARM. Same federation-gossip rationale as ja3_hashes/hassh_hashes:
# a self-signed cert reused across C2 nodes is an instant cluster-link
# signal, and TEXT keeps MySQL indexable via prefix length.
tls_cert_sha256: Optional[str] = Field(
default=None, sa_column=Column("tls_cert_sha256", Text, nullable=True)
)
# Payload SimHash list — 64-bit ints serialized as hex strings.
# SimHashes are Hamming-comparable, which is the entire reason
# they're a list (not a set).
payload_simhashes: Optional[str] = Field(
default=None, sa_column=Column("payload_simhashes", Text, nullable=True)
)
c2_endpoints: Optional[str] = Field(
default=None, sa_column=Column("c2_endpoints", Text, nullable=True)
)
# V2 keystroke-dynamics hook. Same shape as
# SessionProfile.kd_digraph_simhash; this is the centroid (or
# majority vote) across the identity's sessions. BINARY(8) so
# MySQL can index without a prefix length, same as session_profile.
kd_digraph_simhash: Optional[bytes] = Field(
default=None,
sa_column=Column("kd_digraph_simhash", BINARY(8), nullable=True, index=True),
)
# Soft-merge audit trail. When the clusterer collapses two
# identities, the loser's row stays in place with this set to the
# winner's UUID — preserves the audit trail without orphaning FKs
# from any cached subscribers. Resolvers (e.g.
# GET /identities/{uuid}) follow the chain and surface the winner.
merged_into_uuid: Optional[str] = Field(
default=None, foreign_key="attacker_identities.uuid", index=True
)
# Operator-editable free-form notes — annotation surface for human
# analysts ("known APT-XX cluster," "matches MISP event 1234").
notes: Optional[str] = Field(
default=None, sa_column=Column("notes", Text, nullable=True)
)
class AttackerBehavior(SQLModel, table=True):
"""
Timing & behavioral profile for an attacker, joined to Attacker by uuid.
Kept in a separate table so the core Attacker row stays narrow and
behavior data can be updated independently (e.g. as the sniffer observes
more packets) without touching the event-count aggregates.
"""
__tablename__ = "attacker_behavior"
attacker_uuid: str = Field(primary_key=True, foreign_key="attackers.uuid")
# OS / TCP stack fingerprint (rolled up from sniffer events)
os_guess: Optional[str] = None
hop_distance: Optional[int] = None
tcp_fingerprint: str = Field(
default="{}",
sa_column=Column("tcp_fingerprint", Text, nullable=False, default="{}"),
) # JSON: window, wscale, mss, options_sig
# Raw SSH KEX algorithm preference strings observed across HASSH probes
# (one entry per hassh_fingerprint event). Keeping the raw ordered list
# enables post-hoc KEX-order fingerprinting beyond the HASSH hash.
kex_order_raw: Optional[str] = Field(
default=None,
sa_column=Column("kex_order_raw", Text, nullable=True),
) # JSON list[str] — kex_algorithms comma-separated strings
# Sniffer-observed SSH client identification strings (RFC 4253 §4.2),
# deduped in observation order. Captures the attacker's SSH client
# software (e.g. "SSH-2.0-OpenSSH_9.2p1", "SSH-2.0-libssh2_1.10.0").
ssh_client_banners: Optional[str] = Field(
default=None,
sa_column=Column("ssh_client_banners", Text, nullable=True),
) # JSON list[str]
retransmit_count: int = Field(default=0)
# Behavioral (derived by the profiler from log-event timing)
behavior_class: Optional[str] = None # beaconing | interactive | scanning | brute_force | slow_scan | mixed | unknown
beacon_interval_s: Optional[float] = None
beacon_jitter_pct: Optional[float] = None
tool_guesses: Optional[str] = None # JSON list[str] — all matched tools
timing_stats: str = Field(
default="{}",
sa_column=Column("timing_stats", Text, nullable=False, default="{}"),
) # JSON: mean/median/stdev/min/max IAT
phase_sequence: str = Field(
default="{}",
sa_column=Column("phase_sequence", Text, nullable=False, default="{}"),
) # JSON: recon_end/exfil_start/latency
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
class SessionProfile(SQLModel, table=True):
"""
Per-session keystroke-dynamics fingerprint.
One row per recorded interactive session. Pre-v1 the ingestion job
that populates these columns is not yet built (tracked as gap #2 in
SIGNAL_CAPTURE_AUDIT.md); the table ships empty so that:
* downstream correlation/federation work can target a stable schema, and
* `schema_version` is committed to storage from day one — federation
gossip in v2 requires cross-operator compatibility, and retrofitting
a version column after rows exist is painful.
All feature columns are nullable so the empty write path (one row per
closed session) is valid without the behavioral analyzer online yet.
"""
__tablename__ = "session_profile"
sid: str = Field(primary_key=True) # session UUID
log_id: Optional[int] = Field(
default=None, foreign_key="logs.id", index=True
)
schema_version: int = Field(default=1)
# ──────────────────────────────────────────────────────────────────────
# Keystroke-dynamics feature columns (kd_*).
#
# Intended use: session clustering and tooling attribution
# ("is this the same typist?" / "is this a known C2
# framework's paste cadence?").
# Explicitly NOT for: attribution to named individuals, access or
# admission decisions, any ML-driven identity lookup,
# or biometric-login-style user identification. Those
# framings push into legal/ethics territory we don't
# want this project walking into by accident.
# PII discipline: every kd_* column aggregates CHARACTERS and TIMING
# only — never raw input-stream content. Attacker
# passwords typed over SSH must not land here.
# Nulls semantic: a null means "ingester hasn't run on this session
# yet", not "zero events". Consumers should treat
# null as absent, not as a computed zero.
# ──────────────────────────────────────────────────────────────────────
# Inter-key interval timing moments (seconds).
kd_iki_mean: Optional[float] = None
kd_iki_stdev: Optional[float] = None
kd_iki_p50: Optional[float] = None
kd_iki_p95: Optional[float] = None
kd_enter_latency_p50: Optional[float] = None
kd_enter_latency_p95: Optional[float] = None
# Cadence ratios.
kd_burst_ratio: Optional[float] = None
kd_think_ratio: Optional[float] = None
# Control-character rates (events per keystroke).
kd_ctrl_backspace: Optional[float] = None
kd_ctrl_wkill: Optional[float] = None
kd_ctrl_ukill: Optional[float] = None
kd_ctrl_abort: Optional[float] = None
kd_ctrl_eof: Optional[float] = None
kd_arrow_rate: Optional[float] = None
kd_tab_rate: Optional[float] = None
# 8-byte SimHash over keystroke digraphs — Hamming-comparable across sessions.
# Fixed-width BINARY(8) rather than BLOB: MySQL can't index BLOB/TEXT
# columns without a prefix length, and SimHashes are always exactly 8
# bytes so a variable-length type gains nothing here.
#
# PII discipline: the simhash is computed over keystroke CHARACTERS
# (digraph bigrams), never over the raw content of the input stream —
# attacker passwords typed over SSH must never land in this column.
kd_digraph_simhash: Optional[bytes] = Field(
default=None,
sa_column=Column("kd_digraph_simhash", BINARY(8), nullable=True, index=True),
)
# Top-N most-common digraphs with their mean IAT, as JSON.
# Complements kd_digraph_simhash: the simhash answers "same typist?",
# this answers "same typist IN THE SAME MENTAL STATE?" (tired vs rested
# vs distracted shifts bigram-specific IATs measurably). Shape:
# [["th", 47, 0.082], ["in", 31, 0.091], ...] (bigram, count, mean_iat_s)
# Bounded by the ingester to N≤32 to cap row width.
#
# TODO(DEBT-036 upgrade path): JSON-in-TEXT is fine for v1's
# "surface the typist's top digraphs on the attacker page" use
# case, but every similarity query (e.g. "find sessions where the
# 'th' digraph mean IAT is within 20 ms of this one") has to pull
# the string, parse JSON, compare — O(sessions) with a constant
# overhead per row. If that query shape becomes hot, promote to a
# dedicated `session_bigram_stats(sid, bigram, count, mean_iat_s)`
# table with a (bigram, mean_iat_s) index, or a JSONB column on
# Postgres with a GIN index. Either is straightforward, neither
# changes the write-side ingester materially.
kd_top_bigrams: Optional[str] = Field(
default=None, sa_column=Column("kd_top_bigrams", Text, nullable=True),
)
# IAT of the first keystroke following an idle gap >
# KD_START_OF_ACTION_IDLE_S (or the session-start gap before the
# very first keystroke). Separates "initiating a command" from
# "executing a remembered one" — real humans have measurable
# start-of-action latency, bots don't. Median across all such
# initiations in the session, seconds.
#
# Prompt-agnostic on purpose: PS1 / multi-line prompts / sudo
# password prompts make prompt-anchored detection fragile. The
# idle-gap approach conflates post-prompt action-start with
# mid-session think-and-resume — acceptable for a single median
# field; if we later want to split them, feed the concurrent
# output-stream prompt-pattern into the ingester and fall back to
# time-only detection when it misses.
kd_start_of_action_latency: Optional[float] = None
# Three-bucket pause-length histogram, counts (not ratios — raw
# counts preserve the total-keystrokes denominator in the column
# itself). Bucket edges are the KD_PAUSE_* module constants:
# burst : IAT < KD_PAUSE_BURST_MAX_S (muscle-memory digraphs)
# think : KD_PAUSE_BURST_MAX_S ≤ IAT < KD_PAUSE_THINK_MAX_S
# (semantic boundary, context switch)
# distracted: IAT ≥ KD_PAUSE_THINK_MAX_S (went to look something
# up, got paged, reading another window)
# More discriminating than the flat burst_ratio / think_ratio pair:
# C2 operators concentrate in the burst bucket with a thin tail;
# opportunistic humans have a fat think bucket and a long
# distracted tail.
kd_pause_hist_burst: Optional[int] = None
kd_pause_hist_think: Optional[int] = None
kd_pause_hist_distracted: Optional[int] = None
# Longest IAT in the session, seconds. The distracted-bucket count
# alone can't tell "one 3-second pause" from "three 60-second
# pauses" — both contribute 1-3 to the distracted bucket but
# represent different behaviours (brief think vs actual
# disengagement). max_pause_gap carries that signal in one scalar.
kd_max_pause_gap: Optional[float] = None
# Derived totals.
total_keystrokes: Optional[int] = None
session_duration_s: Optional[float] = None
created_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc)
)
class SmtpTarget(SQLModel, table=True):
"""
Per-attacker list of victim domains observed via the SMTP honeypots.
Each row is one (attacker_uuid, domain) pair — an attacker who relays
mail to 500 addresses at acme.com collapses into a single row with
count=500. Only the *domain* is stored; local-parts (the bit before
`@`) are dropped at ingestion, so this table contains no PII beyond
the target organisation's identity.
Shape is designed for future V2 federation gossip: the
`smtp_target_seen(domain)` query returns aggregate counts with zero
cross-org attacker leakage — each operator can answer "have you seen
this domain being targeted?" without exposing *which* attackers did.
"""
__tablename__ = "smtp_targets"
id: Optional[int] = Field(default=None, primary_key=True)
attacker_uuid: str = Field(foreign_key="attackers.uuid", index=True)
domain: str = Field(index=True)
first_seen: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
last_seen: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
# Aggregate counter — one rcpt_to / message_accepted recipient bumps this.
count: int = Field(default=1)
__table_args__ = (
UniqueConstraint("attacker_uuid", "domain", name="uq_smtp_targets_attacker_domain"),
)
class AttackersResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]

73
decnet/web/db/models/auth.py Normal file
View File

@@ -0,0 +1,73 @@
"""Auth + user-management tables and DTOs."""
from typing import List, Literal
from pydantic import BaseModel, Field as PydanticField
from sqlmodel import Field, SQLModel
class User(SQLModel, table=True):
__tablename__ = "users"
uuid: str = Field(primary_key=True)
username: str = Field(index=True, unique=True)
password_hash: str
role: str = Field(default="viewer")
must_change_password: bool = Field(default=False)
# --- API Request/Response Models (Pydantic) ---
class Token(BaseModel):
access_token: str
token_type: str
must_change_password: bool = False
class LoginRequest(BaseModel):
username: str
password: str = PydanticField(..., max_length=72)
class ChangePasswordRequest(BaseModel):
old_password: str = PydanticField(..., max_length=72)
new_password: str = PydanticField(..., max_length=72)
# --- Configuration Models ---
class CreateUserRequest(BaseModel):
username: str = PydanticField(..., min_length=1, max_length=64)
password: str = PydanticField(..., min_length=8, max_length=72)
role: Literal["admin", "viewer"] = "viewer"
class UpdateUserRoleRequest(BaseModel):
role: Literal["admin", "viewer"]
class ResetUserPasswordRequest(BaseModel):
new_password: str = PydanticField(..., min_length=8, max_length=72)
class DeploymentLimitRequest(BaseModel):
deployment_limit: int = PydanticField(..., ge=1, le=500)
class GlobalMutationIntervalRequest(BaseModel):
global_mutation_interval: str = PydanticField(..., pattern=r"^[1-9]\d*[mdMyY]$")
class UserResponse(BaseModel):
uuid: str
username: str
role: str
must_change_password: bool
class ConfigResponse(BaseModel):
role: str
deployment_limit: int
global_mutation_interval: str
class AdminConfigResponse(ConfigResponse):
users: List[UserResponse]

83
decnet/web/db/models/campaigns.py Normal file
View File

@@ -0,0 +1,83 @@
"""Campaign — operation-level grouping of resolved attacker identities."""
from datetime import datetime, timezone
from typing import Any, List, Optional
from pydantic import BaseModel
from sqlalchemy import Column, Text
from sqlmodel import Field, SQLModel
class Campaign(SQLModel, table=True):
"""
Campaign — one operation, one or more identities.
Sits one level above ``AttackerIdentity``: an actor (identity) may
appear in multiple campaigns over time, and a campaign may have
several distinct identities cooperating (e.g. a night-shift and
day-shift operator on the same job — fixture F5 multi_operator).
Populated by the campaign clusterer worker (downstream of identity
resolution). Empty rows are valid; the table ships empty until the
clusterer lands. ``schema_version`` is non-negotiable from day one
for the same federation-gossip reason ``AttackerIdentity`` carries
one — bumping campaign-level feature definitions without a version
field silently poisons cross-operator gossip in V2.
See ``development/CAMPAIGN_CLUSTERING.md`` for the signal taxonomy
(phase-handoff, shared-infra, temporal overlap, cohort).
"""
__tablename__ = "campaigns"
uuid: str = Field(primary_key=True)
schema_version: int = Field(default=1)
first_seen_at: Optional[datetime] = Field(default=None, index=True)
last_seen_at: Optional[datetime] = Field(default=None, index=True)
created_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
# Campaign-cohesion score from the clusterer. Range [0, 1]; null
# until the clusterer writes. Higher = more confident the linked
# identities are part of the same operation.
confidence: Optional[float] = Field(default=None)
# Denormalized count of FK'd ``AttackerIdentity`` rows.
identity_count: int = Field(default=0)
# Aggregated fingerprint summary across member identities. Same
# JSON-serialized list[str] in TEXT shape as
# ``AttackerIdentity.{ja3,hassh,payload_simhashes,c2_endpoints}`` —
# federation gossip wants the same wire shape at every layer.
ja3_hashes: Optional[str] = Field(
default=None, sa_column=Column("ja3_hashes", Text, nullable=True)
)
hassh_hashes: Optional[str] = Field(
default=None, sa_column=Column("hassh_hashes", Text, nullable=True)
)
tls_cert_sha256: Optional[str] = Field(
default=None, sa_column=Column("tls_cert_sha256", Text, nullable=True)
)
payload_simhashes: Optional[str] = Field(
default=None, sa_column=Column("payload_simhashes", Text, nullable=True)
)
c2_endpoints: Optional[str] = Field(
default=None, sa_column=Column("c2_endpoints", Text, nullable=True)
)
# Soft-merge audit trail — same revocable-merge pattern as
# ``AttackerIdentity.merged_into_uuid``. When the clusterer
# collapses two campaigns, the loser's row stays in place with this
# set to the winner's UUID; resolvers follow the chain.
merged_into_uuid: Optional[str] = Field(
default=None, foreign_key="campaigns.uuid", index=True
)
# Operator-editable free-form notes — annotation surface for
# human analysts ("APT-XX Q2 campaign", "matches CTI report 5678").
notes: Optional[str] = Field(
default=None, sa_column=Column("notes", Text, nullable=True)
)
class CampaignsResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]

242
decnet/web/db/models/canary.py Normal file
View File

@@ -0,0 +1,242 @@
"""Canary token tables + CRUD DTOs.
Canary tokens are decoy artifacts (operator-uploaded honeydocs / synthesised
fake configs) planted inside a decky's filesystem. When an attacker exfils
the artifact and uses it, an HTTP slug or DNS subdomain encoded into the
file is hit; the ``decnet canary`` worker observes the callback and
publishes ``canary.{token_id}.triggered`` on the bus. The webhook fanout
+ correlator pick it up the same way they handle any other attacker
event — no canary-specific consumer wiring needed downstream.
Three tables:
* :class:`CanaryBlob` — operator-uploaded source artifact, deduped by
sha256. The original bytes live on disk under
``/var/lib/decnet/canary/blobs/{sha256}``; this row carries metadata
+ refcount-aware deletion.
* :class:`CanaryToken` — one planted artifact in one decky. Either
references a blob (``blob_id``) and an instrumenter, or is a wholly
synthesised fake (e.g. ``aws_creds`` / ``git_config`` from a
generator) and ``blob_id`` is NULL. ``callback_token`` is the short
random slug embedded into HTTP URLs and DNS labels — unique across
the fleet so the worker can resolve a hit to a row in one query.
* :class:`CanaryTrigger` — append-only log of every callback hit.
``attacker_id`` is back-filled by the correlator after it attributes
``src_ip`` to an existing :class:`Attacker`; NULL until then.
We follow the project convention from :mod:`webhooks` and
:mod:`orchestrator`: stringly-typed UUIDs (``str`` PKs via
``str(uuid4())``), no FK to the composite-PK fleet table, indexes on
the join keys. Pydantic request/response shapes live in this same
file (per :mod:`feedback_models_single_source`).
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, List, Literal, Optional
from uuid import uuid4
from pydantic import BaseModel, Field as PydanticField
from sqlalchemy import Column, Index, Text
from sqlmodel import Field, SQLModel
from ._base import _BIG_TEXT
# --- Enum-shaped string literals -------------------------------------------
CanaryKind = Literal["http", "dns", "aws_passive"]
"""Detection mechanism for a token.
* ``http`` — slug embedded in artifact; attacker fetches our HTTP endpoint.
* ``dns`` — subdomain embedded; attacker's resolver looks up our DNS server.
* ``aws_passive`` — fake AWS credentials with no callback wiring. Trips
zero alerts on its own; useful only as bait + as evidence the attacker
read the file when correlated with other timing signals.
"""
CanaryState = Literal["planted", "revoked", "failed"]
"""Lifecycle state of a token row.
* ``planted`` — file is in the decky and the slug/host is live.
* ``revoked`` — operator deleted the token; planter unlinked the file
(best-effort) and the slug/host stops resolving.
* ``failed`` — placement failed (docker exec error, instrumenter
rejected the blob, etc.); surfaced in the UI so the operator can
retry or pick a different kind.
"""
# --- DB tables -------------------------------------------------------------
class CanaryBlob(SQLModel, table=True):
"""Operator-uploaded source artifact, deduped by sha256.
The same bytes uploaded twice produce the same row (insert-or-get
semantics in the repository). We never store the bytes inline —
only the disk path derived from ``sha256``. Deletion is
refcount-aware: ``DELETE`` is rejected while at least one
:class:`CanaryToken` references the blob.
"""
__tablename__ = "canary_blobs"
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
sha256: str = Field(index=True, unique=True)
filename: str # original filename — UI display only, not used for path resolution
content_type: str # sniffed MIME (python-magic); drives instrumenter selection
size_bytes: int
uploaded_by: str = Field(index=True) # User.uuid
uploaded_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
class CanaryToken(SQLModel, table=True):
"""One canary artifact planted inside one decky."""
__tablename__ = "canary_tokens"
__table_args__ = (
Index("ix_canary_tokens_decky", "decky_name", "state"),
)
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
kind: str = Field(index=True) # CanaryKind literal at the API layer
decky_name: str = Field(index=True) # FleetDecky.name; no FK (composite PK)
blob_uuid: Optional[str] = Field(
default=None, foreign_key="canary_blobs.uuid", index=True,
)
# Which instrumenter mutated the blob (``docx``/``xlsx``/``pdf``/``html``/
# ``image``/``plain``/``passthrough``). NULL when the artifact came
# from a synthesizer (``git_config``/``env_file``/``ssh_key``/
# ``aws_creds``/``honeydoc``); ``generator`` carries that name instead.
instrumenter: Optional[str] = Field(default=None)
generator: Optional[str] = Field(default=None)
placement_path: str # absolute path inside the container
# Short random slug (e.g. 16 url-safe bytes). Embedded in HTTP URLs
# *and* DNS labels — same value, different envelope, so both
# detection paths resolve to the same token row.
callback_token: str = Field(unique=True, index=True)
# Stable secret used by re-instrumentation: same blob + same seed
# = same mutated bytes, so re-seeding produces the same on-disk
# artifact and the planter is naturally idempotent.
secret_seed: str
placed_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
last_triggered_at: Optional[datetime] = Field(default=None, index=True)
trigger_count: int = Field(default=0)
created_by: str = Field(index=True) # User.uuid; "system" for baseline-seeded tokens
state: str = Field(default="planted", index=True)
last_error: Optional[str] = Field(
default=None, sa_column=Column("last_error", Text, nullable=True),
)
class CanaryTrigger(SQLModel, table=True):
"""Append-only log of one callback hit."""
__tablename__ = "canary_triggers"
__table_args__ = (
Index("ix_canary_triggers_token_ts", "token_uuid", "occurred_at"),
Index("ix_canary_triggers_attacker", "attacker_id"),
)
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
token_uuid: str = Field(foreign_key="canary_tokens.uuid", index=True)
occurred_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
src_ip: str = Field(index=True)
user_agent: Optional[str] = None
request_path: Optional[str] = None # HTTP path including the slug
dns_qname: Optional[str] = None # DNS qname when the hit came over DNS
# JSON-encoded request headers (HTTP) or empty for DNS. Stored as
# TEXT for cross-dialect portability — same trick as
# :attr:`WebhookSubscription.topic_patterns`.
raw_headers: str = Field(
default="{}",
sa_column=Column("raw_headers", _BIG_TEXT, nullable=False, default="{}"),
)
# Set by the correlator once it attributes ``src_ip`` to an existing
# :class:`Attacker`. NULL until correlation runs (which happens on
# the bus event we publish, so latency is sub-second).
attacker_id: Optional[str] = Field(default=None, index=True)
def headers(self) -> dict[str, Any]:
"""Decode :attr:`raw_headers` JSON; ``{}`` on bad/empty input."""
try:
raw = json.loads(self.raw_headers or "{}")
except (ValueError, TypeError):
return {}
return raw if isinstance(raw, dict) else {}
# --- API request / response shapes -----------------------------------------
class CanaryBlobResponse(BaseModel):
uuid: str
sha256: str
filename: str
content_type: str
size_bytes: int
uploaded_by: str
uploaded_at: datetime
# Number of tokens currently referencing this blob. Surfaces in the
# UI so operators don't try to delete a blob that's still in use,
# and the API uses it to gate ``DELETE`` (returns 409).
token_count: int = 0
class CanaryTokenCreateRequest(BaseModel):
"""Generate + plant a new token.
Exactly one of ``blob_uuid`` (operator-supplied artifact) or
``generator`` (synthesised fake) must be set. Validated in the
router so the 400 carries a clear detail message.
"""
decky_name: str = PydanticField(..., min_length=1)
kind: CanaryKind
placement_path: str = PydanticField(..., min_length=1)
blob_uuid: Optional[str] = None
generator: Optional[str] = None # git_config | env_file | ssh_key | aws_creds | honeydoc
# Optional override for the path-mapping helper — useful when the
# operator wants a specific Windows-shaped path on a windows-persona
# decky. Defaults to placement_path verbatim.
persona_path_hint: Optional[str] = None
class CanaryTokenResponse(BaseModel):
uuid: str
kind: CanaryKind
decky_name: str
blob_uuid: Optional[str]
instrumenter: Optional[str]
generator: Optional[str]
placement_path: str
callback_token: str
placed_at: datetime
last_triggered_at: Optional[datetime]
trigger_count: int
created_by: str
state: CanaryState
last_error: Optional[str]
class CanaryTriggerResponse(BaseModel):
uuid: str
token_uuid: str
occurred_at: datetime
src_ip: str
user_agent: Optional[str]
request_path: Optional[str]
dns_qname: Optional[str]
headers: dict[str, Any] = PydanticField(default_factory=dict)
attacker_id: Optional[str]
class CanaryTokensResponse(BaseModel):
tokens: List[CanaryTokenResponse]
total: int
class CanaryTriggersResponse(BaseModel):
triggers: List[CanaryTriggerResponse]
total: int
class CanaryBlobsResponse(BaseModel):
blobs: List[CanaryBlobResponse]
total: int

15
decnet/web/db/models/common.py Normal file
View File

@@ -0,0 +1,15 @@
"""Generic response shapes used across multiple router domains."""
from __future__ import annotations
from pydantic import BaseModel
class MessageResponse(BaseModel):
"""Standard envelope for mutations whose only payload is a status message.
Pinning the wire shape at the decorator (``response_model=MessageResponse``)
prevents a handler that accidentally returns a richer dict — e.g. a user
row with ``password_hash`` — from leaking extra fields to the client.
"""
message: str

29
decnet/web/db/models/deploy.py Normal file
View File

@@ -0,0 +1,29 @@
"""Fleet deploy + mutate-interval request DTOs."""
from typing import Optional
from pydantic import BaseModel, ConfigDict, Field as PydanticField
from decnet.models import IniContent
class MutateIntervalRequest(BaseModel):
# Human-readable duration: <number><unit> where unit is m(inutes), d(ays), M(onths), y/Y(ears).
# Minimum granularity is 1 minute. Seconds are not accepted.
mutate_interval: Optional[str] = PydanticField(None, pattern=r"^[1-9]\d*[mdMyY]$")
class DeployIniRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
# This field now enforces strict INI structure during Pydantic initialization.
# The OpenAPI schema correctly shows it as a required string.
ini_content: IniContent = PydanticField(..., description="A valid INI formatted string")
class DeployResponse(BaseModel):
message: str
mode: str
class PurgeResponse(BaseModel):
message: str
deleted: dict[str, int]

72
decnet/web/db/models/fleet.py Normal file
View File

@@ -0,0 +1,72 @@
"""Fleet decky table — DB mirror of ``decnet-state.json``.
The legacy unihost / MACVLAN / IPVLAN deploy path persists fleet state to a
JSON file (``/var/lib/decnet/decnet-state.json``) via
:func:`decnet.config.save_state`. That file is consumed directly by
``decnet status``/``decnet teardown``, the sniffer, and the collector — all
host-local CLI / worker code that may run on a box without the API daemon.
The FleetDecky table is a *mirror* of that JSON state inside MySQL/SQLite so
DB-only consumers (the orchestrator, the web dashboard, the REST API) can
see fleet decoys without touching the filesystem.
Both writers — CLI ``decnet deploy`` (``engine.deployer.deploy``) and the
web/API deploy path (``web.router.fleet.api_deploy_deckies``) — write to
*both* surfaces. A reconciler (``decnet.fleet.reconciler``) handles drift.
Schema mirrors :class:`decnet.web.db.models.swarm.DeckyShard` field-for-field
so the dashboard can render fleet rows with the same card shape. The PK is
composite ``(host_uuid, name)`` to future-proof for multi-host motherships
(a master that runs its own local fleet AND swarm-shards onto workers). In
unihost mode ``host_uuid`` defaults to the sentinel
:data:`LOCAL_HOST_SENTINEL`; we deliberately do NOT FK to ``swarm_hosts``
because the local mothership is not enrolled as a swarm worker.
"""
from datetime import datetime, timezone
from typing import Optional
from sqlalchemy import Column, Text
from sqlmodel import Field, SQLModel
from ._base import _BIG_TEXT
LOCAL_HOST_SENTINEL = "local"
class FleetDecky(SQLModel, table=True):
"""A unihost / MACVLAN / IPVLAN decky deployed on the local mothership.
Disjoint from :class:`DeckyShard` (SWARM-only) and :class:`TopologyDecky`
(MazeNET-only). Composite PK lets multiple hosts coexist when a future
mothership runs both a local fleet and acts as a swarm master.
"""
__tablename__ = "fleet_deckies"
host_uuid: str = Field(
default=LOCAL_HOST_SENTINEL, primary_key=True, index=True,
)
name: str = Field(primary_key=True)
# JSON list of service names on this decky (snapshot of assignment).
services: str = Field(
sa_column=Column("services", _BIG_TEXT, nullable=False, default="[]")
)
# Full serialised DeckyConfig — lets the dashboard render the same rich
# card (hostname/distro/archetype/service_config/mutate_interval) without
# round-tripping to load_state() on every page render.
decky_config: Optional[str] = Field(
default=None, sa_column=Column("decky_config", _BIG_TEXT, nullable=True)
)
decky_ip: Optional[str] = Field(default=None)
# pending|running|failed|torn_down|degraded|tearing_down|teardown_failed
state: str = Field(default="pending", index=True)
last_error: Optional[str] = Field(
default=None, sa_column=Column("last_error", Text, nullable=True),
)
compose_hash: Optional[str] = Field(default=None)
# Last reconciler observation (docker inspect) — lets the dashboard show
# "stale" rows whose reconciler hasn't ticked.
last_seen: Optional[datetime] = Field(default=None)
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc)
)

14
decnet/web/db/models/health.py Normal file
View File

@@ -0,0 +1,14 @@
"""Health-endpoint DTOs."""
from typing import Literal, Optional
from pydantic import BaseModel
class ComponentHealth(BaseModel):
status: Literal["ok", "failing"]
detail: Optional[str] = None
class HealthResponse(BaseModel):
status: Literal["healthy", "degraded", "unhealthy"]
components: dict[str, ComponentHealth]

222
decnet/web/db/models/logs.py Normal file
View File

@@ -0,0 +1,222 @@
"""Log / Bounty / Credential / State tables + their list-response DTOs."""
from datetime import datetime, timezone
from typing import Any, List, Optional
from pydantic import BaseModel
from sqlalchemy import Column, Index, Text, UniqueConstraint
from sqlmodel import Field, SQLModel
from ._base import _BIG_TEXT
class Log(SQLModel, table=True):
__tablename__ = "logs"
id: Optional[int] = Field(default=None, primary_key=True)
timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc), index=True)
decky: str = Field(index=True)
service: str = Field(index=True)
event_type: str = Field(index=True)
attacker_ip: str = Field(index=True)
# Long-text columns — use TEXT so MySQL DDL doesn't truncate to VARCHAR(255).
# TEXT is equivalent to plain text in SQLite.
raw_line: str = Field(sa_column=Column("raw_line", Text, nullable=False))
fields: str = Field(sa_column=Column("fields", Text, nullable=False))
msg: Optional[str] = Field(default=None, sa_column=Column("msg", Text, nullable=True))
# OTEL trace context — bridges the collector→ingester trace to the SSE
# read path. Nullable so pre-existing rows and non-traced deployments
# are unaffected.
trace_id: Optional[str] = Field(default=None)
span_id: Optional[str] = Field(default=None)
class Bounty(SQLModel, table=True):
__tablename__ = "bounty"
id: Optional[int] = Field(default=None, primary_key=True)
timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc), index=True)
decky: str = Field(index=True)
service: str = Field(index=True)
attacker_ip: str = Field(index=True)
bounty_type: str = Field(index=True)
payload: str = Field(sa_column=Column("payload", Text, nullable=False))
class Credential(SQLModel, table=True):
"""One observed credential attempt against a decky service.
Forward-compatible across every auth-bearing service in the fleet:
SSH user+pass, Telnet user+pass, SMTP domain+pass, LDAP dn+pass,
Redis password-only, etc. The two universal lossless representations
(``secret_b64`` + ``secret_sha256``) hoist to indexed columns so
cross-service reuse queries don't scan opaque JSON.
Per-service identity (the human-meaningful "who's authenticating")
lives in ``principal`` — username for SSH, domain for SMTP, dn for
LDAP. Nullable for principal-less mechanisms (Redis AUTH, bearer
tokens). Fully service-specific keys ride in ``fields`` JSON.
Dedup contract: same (attacker_ip, decky, service, secret_sha256,
principal_or_empty) tuple → upsert, bumps ``attempt_count`` and
``last_seen``. Different secret or different principal → new row.
``attacker_uuid`` is backfilled by the profiler once an Attacker row
has been minted for the source IP. It is nullable on first write so
the credential ingest path stays decoupled from the profiler.
"""
__tablename__ = "credentials"
__table_args__ = (
Index("ix_credentials_secret_service", "secret_sha256", "service"),
Index("ix_credentials_principal_service", "principal", "service"),
)
id: Optional[int] = Field(default=None, primary_key=True)
# Keyed by attacker IP (not attackers.uuid) on the write path to
# avoid the chicken-and-egg of landing a credential before the
# profiler has minted the Attacker. The profiler backfills
# ``attacker_uuid`` once it knows the IP, so cross-IP reuse queries
# eventually have an indexed FK to traverse.
attacker_ip: str = Field(index=True)
attacker_uuid: Optional[str] = Field(
default=None, foreign_key="attackers.uuid", index=True
)
decky_name: str = Field(index=True)
service: str = Field(index=True)
principal: Optional[str] = Field(default=None, index=True, max_length=256)
# Discriminator for what `secret_b64` actually contains. Default
# ``"plaintext"`` — a recoverable password the attacker sent on the
# wire (SSH/Telnet/FTP/IMAP/POP3/SMTP/Redis/LDAP/MQTT). Other kinds:
# ``"postgres_md5_challenge"`` (md5(md5(pw+user)+salt) hex bytes
# the attacker sent in the Postgres password message — plaintext
# irrecoverable), ``"vnc_des_response"`` (16-byte DES-encrypted
# challenge response — same shape).
#
# Reuse semantics gracefully degrade: same secret_sha256 only
# correlates within a single ``secret_kind``. Cross-kind matches
# are meaningless because different challenges produce different
# bytes for the same plaintext password.
secret_kind: str = Field(default="plaintext", index=True, max_length=32)
# Universal lossless secret representations. For non-plaintext
# kinds, secret_b64 is base64 of the raw attacker-sent bytes (after
# hex-decode for protocols that ship the response as a hex string).
secret_sha256: str = Field(index=True, max_length=64)
secret_b64: Optional[str] = Field(default=None, max_length=2048)
# Best-effort printable form — non-printable bytes collapsed to '?'
# by either auth-helper.c (SSH/Telnet) or the ingester's legacy
# adapter (FTP/POP3/IMAP/SMTP). May be lossy on non-UTF8.
secret_printable: Optional[str] = Field(default=None, max_length=512)
outcome: Optional[str] = Field(default=None, max_length=16) # success|failure|observed
fields: str = Field(
sa_column=Column("fields", _BIG_TEXT, nullable=False, default="{}")
)
first_seen: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
last_seen: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
attempt_count: int = Field(default=1)
class CredentialReuse(SQLModel, table=True):
"""One observed credential reuse pattern across deckies and/or services.
A row here is a *finding* produced by the correlator: the same
``(secret_sha256, secret_kind, principal)`` tuple was observed
against ``target_count`` distinct decky×service pairs. Upserted on
that natural key — the row accumulates new deckies/services/IPs
over time as the credential is reused.
The ``confidence`` column is reserved for a future fuzzy-match pass
(credential variants, e.g. ``hunter2`` vs ``hunter22``); rows
written by the exact-secret correlator are always 1.0.
"""
__tablename__ = "credential_reuse"
__table_args__ = (
UniqueConstraint(
"secret_sha256", "secret_kind", "principal_key",
name="uq_credential_reuse_secret_principal",
),
)
id: str = Field(primary_key=True, max_length=36)
secret_sha256: str = Field(index=True, max_length=64)
secret_kind: str = Field(index=True, max_length=32)
# Optional human-readable principal (e.g. "root"). Nullable — for
# cross-principal reuse rows we leave this null, but we still need
# a unique constraint, so ``principal_key`` is the non-null
# canonicalised form ("" when principal is null) used in the
# uniqueness tuple. SQLite's NULLs-distinct-in-UNIQUE behaviour
# would otherwise let duplicate null-principal rows through.
principal: Optional[str] = Field(default=None, max_length=256)
principal_key: str = Field(default="", max_length=256)
attacker_uuids: str = Field(
default="[]",
sa_column=Column("attacker_uuids", _BIG_TEXT, nullable=False, default="[]"),
) # JSON list[str]
attacker_ips: str = Field(
default="[]",
sa_column=Column("attacker_ips", _BIG_TEXT, nullable=False, default="[]"),
) # JSON list[str]
deckies: str = Field(
default="[]",
sa_column=Column("deckies", _BIG_TEXT, nullable=False, default="[]"),
) # JSON list[str]
services: str = Field(
default="[]",
sa_column=Column("services", _BIG_TEXT, nullable=False, default="[]"),
) # JSON list[str]
# COUNT(DISTINCT decky||':'||service). The discriminative scalar
# for ranking and filtering — a credential seen on 12 targets is
# far more interesting than one seen on 2.
target_count: int = Field(default=0, index=True)
attempt_count: int = Field(default=0)
confidence: float = Field(default=1.0)
first_seen: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
last_seen: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
class CredentialReuseResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class State(SQLModel, table=True):
__tablename__ = "state"
key: str = Field(primary_key=True)
# JSON-serialized DecnetConfig or other state blobs — can be large as
# deckies/services accumulate. MEDIUMTEXT on MySQL (16 MiB ceiling).
value: str = Field(sa_column=Column("value", _BIG_TEXT, nullable=False))
class LogsResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class BountyResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class CredentialsResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class StatsResponse(BaseModel):
total_logs: int
unique_attackers: int
active_deckies: int
deployed_deckies: int

111
decnet/web/db/models/orchestrator.py Normal file
View File

@@ -0,0 +1,111 @@
"""Orchestrator-emitted activity events.
Purpose-built sibling to ``logs.Log`` so attacker-originated events stay
cleanly separable from synthetic life-injection events at query time.
The orchestrator worker is the sole writer.
"""
from datetime import datetime, timezone
from typing import Any, List, Optional
from uuid import uuid4
from pydantic import BaseModel
from sqlalchemy import Column, Index, Text
from sqlmodel import Field, SQLModel
class OrchestratorEvent(SQLModel, table=True):
"""One orchestrator-driven action against a decky.
``kind`` discriminates the two MVP flavours:
* ``"traffic"`` — a protocol-driven interaction (SSH command exec for
MVP). ``src_decky_uuid`` is the *logical* originator and may differ
from the actual TCP source for the duration of the MVP, where the
orchestrator process drives the connection from the host. ``v1``
will execute the connection from inside the source container.
* ``"file"`` — a filesystem touch via ``docker exec`` against the
destination decky. ``src_decky_uuid`` is null.
``payload`` is the per-action JSON envelope: command run, exit code,
stdout/stderr digest, file path, byte counts, etc. Schema is
deliberately loose — the worker can extend it without a migration.
"""
__tablename__ = "orchestrator_events"
__table_args__ = (
Index("ix_orchestrator_events_dst_ts", "dst_decky_uuid", "ts"),
)
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
ts: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
kind: str = Field(index=True, max_length=16) # traffic|file
protocol: str = Field(index=True, max_length=16) # ssh for MVP
action: str = Field(max_length=64) # exec:uptime|file:create|...
# No FK to topology_deckies: dst/src may be a TopologyDecky.uuid
# (MazeNET source), a "host_uuid:name" composite (fleet / SWARM shard
# sources), or — for retired deckies — a row that's already gone. The
# column is an opaque identifier matching whatever
# ``BaseRepository.list_running_deckies`` emits in its ``uuid`` field.
# Index is kept; the FK was misleading and broke fleet-source events.
src_decky_uuid: Optional[str] = Field(default=None, index=True)
dst_decky_uuid: str = Field(index=True)
success: bool = Field(default=False, index=True)
payload: str = Field(
sa_column=Column("payload", Text, nullable=False, default="{}")
)
class OrchestratorEventsResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class OrchestratorEmail(SQLModel, table=True):
"""One fake email generated by the ``decnet emailgen`` worker.
Sibling table to :class:`OrchestratorEvent` — kept disjoint because
email rows carry domain-specific fields (subject, message_id,
in_reply_to, language) that have no analogue in the SSH/file events
and would otherwise bloat ``OrchestratorEvent.payload``.
The mail decky's UUID lives in ``mail_decky_uuid`` (the host serving
the IMAP/POP3 mailbox). ``thread_id`` is a worker-side UUID used to
chain replies; ``in_reply_to`` is the parent email's RFC 2822
Message-ID header value (or ``None`` for thread roots).
``payload`` follows the same loose-JSON convention as
:class:`OrchestratorEvent`: ``bytes``, ``generation_ms``, ``model``,
``mannerisms_used``, etc. The worker can extend it without a
migration.
"""
__tablename__ = "orchestrator_emails"
__table_args__ = (
Index("ix_orchestrator_emails_mail_ts", "mail_decky_uuid", "ts"),
Index("ix_orchestrator_emails_thread", "thread_id"),
)
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
ts: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
mail_decky_uuid: str = Field(index=True)
thread_id: str = Field(index=True)
message_id: str = Field(max_length=255)
in_reply_to: Optional[str] = Field(default=None, max_length=255)
sender_email: str = Field(max_length=255, index=True)
recipient_email: str = Field(max_length=255, index=True)
subject: str = Field(max_length=512)
language: str = Field(max_length=8, default="en")
eml_path: str = Field(max_length=1024)
success: bool = Field(default=False, index=True)
payload: str = Field(
sa_column=Column("payload", Text, nullable=False, default="{}")
)
class OrchestratorEmailsResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]

107
decnet/web/db/models/realism.py Normal file
View File

@@ -0,0 +1,107 @@
"""Realism — synthetic-file state across orchestrator ticks.
The orchestrator's pre-realism file generator forgot every file the
moment it was planted: each tick wrote a brand-new ``notes-{ts}.txt``
with a literal unix-epoch suffix. No edits, no rotation, no diurnal
shape — three of the realism failures the migration is fixing.
:class:`SyntheticFile` is the per-(decky, path) memory that lets the
realism engine read back yesterday's ``TODO.md``, mutate it, write
back the new body, and let the dashboard inspect the lineage.
Pre-v1: schema lives directly in the SQLModel; no ``_migrate_*``
helper (per the project's "no new migrations pre-v1" rule —
``feedback_no_new_migrations_prev1.md``). Alembic lands at v1.
"""
from datetime import datetime, timezone
from typing import Any, List
from uuid import uuid4
from pydantic import BaseModel
from sqlalchemy import Column, Index, Text, UniqueConstraint
from sqlmodel import Field, SQLModel
SYNTHETIC_FILE_BODY_LIMIT = 65536
"""Cap on persisted ``synthetic_files.last_body`` bytes.
Enforced by the repo on both insert and update — callers may pass the
full body; the repo clips. Large blobs (DOCX/PDF, canary artifacts) are
wasted disk on the master side; the decky filesystem holds the canonical
bytes."""
class SyntheticFile(SQLModel, table=True):
"""One realism-planted file on one decky.
The unique key is ``(decky_uuid, path)`` — there's at most one
realism record per location, even if the planter has rotated the
file (rotation updates ``edit_count`` and ``last_modified``, not
a new row).
``last_body`` is capped — large blobs (DOCX/PDF, future canary
artifacts) are truncated at write time. The edit-in-place flow
(stage 3b) only needs the body when the content class supports
body-level mutation (``note``, ``todo``, ``draft``, ``script``),
so storing the canonical bytes for binary blobs would be wasted.
``content_hash`` is sha256 of the *body bytes only* — never of
metadata or wrapper headers — so a hash compare is a cheap
"did the body change?" check across edits.
"""
__tablename__ = "synthetic_files"
__table_args__ = (
UniqueConstraint(
"decky_uuid", "path", name="uq_synthetic_files_decky_path",
),
Index("ix_synthetic_files_decky_modified", "decky_uuid", "last_modified"),
)
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
decky_uuid: str = Field(index=True, max_length=64)
# Capped at 512 so the (decky_uuid, path) unique index fits MySQL's
# 3072-byte utf8mb4 limit: (64+512)*4 = 2304 bytes. Real realism +
# canary paths are well under (longest is
# ``/home/<persona>/Documents/Q3-Operations-Review.docx``, ~70 chars).
path: str = Field(max_length=512)
persona: str = Field(max_length=128) # EmailPersona.name
content_class: str = Field(max_length=32, index=True) # ContentClass enum value
created_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True,
)
last_modified: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc),
)
edit_count: int = Field(default=0)
content_hash: str = Field(max_length=64) # sha256 hex
last_body: str = Field(
sa_column=Column("last_body", Text, nullable=False, default="")
)
class SyntheticFilesResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class RealismConfig(SQLModel, table=True):
"""Operator-tunable realism knobs.
Single-row-per-key schema: each row carries one piece of operator
config (today: ``key="weights"`` → JSON encoding the planner's
user/system/canary weights and canary probability). The planner
reads in-memory module globals; the orchestrator worker refreshes
those globals from this table on a periodic tick.
UUID PK + unique key per ``feedback_uuid_over_natural_keys.md``.
"""
__tablename__ = "realism_config"
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
key: str = Field(max_length=64, unique=True, index=True)
value: str = Field(
sa_column=Column("value", Text, nullable=False, default="{}"),
)
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc),
)

200
decnet/web/db/models/swarm.py Normal file
View File

@@ -0,0 +1,200 @@
"""Swarm host + decky shard tables and their HTTP DTOs."""
from datetime import datetime, timezone
from typing import Annotated, Any, Optional
from pydantic import BaseModel, Field as PydanticField
from sqlalchemy import Column, Text
from sqlmodel import Field, SQLModel
from decnet.models import DecnetConfig
from ._base import _BIG_TEXT
class SwarmHost(SQLModel, table=True):
"""A worker host enrolled into a DECNET swarm.
Rows exist only on the master. Populated by `decnet swarm enroll` and
read by the swarm controller when sharding deckies onto workers.
"""
__tablename__ = "swarm_hosts"
uuid: str = Field(primary_key=True)
name: str = Field(index=True, unique=True)
address: str # IP or hostname reachable by the master
agent_port: int = Field(default=8765)
status: str = Field(default="enrolled", index=True)
# ISO-8601 string of the last successful agent /health probe
last_heartbeat: Optional[datetime] = Field(default=None)
client_cert_fingerprint: str # SHA-256 hex of worker's issued client cert
# SHA-256 hex of the updater-identity cert, if the host was enrolled
# with ``--updater`` / ``issue_updater_bundle``. ``None`` for hosts
# that only have an agent identity.
updater_cert_fingerprint: Optional[str] = Field(default=None)
# Directory on the master where the per-worker cert bundle lives
cert_bundle_path: str
enrolled_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
notes: Optional[str] = Field(default=None, sa_column=Column("notes", Text, nullable=True))
# Per-host driver preference. True => deckies on this host run over IPvlan
# (L2) instead of macvlan — required when the host is a VirtualBox guest
# bridged over Wi-Fi, because Wi-Fi APs only allow one MAC per station
# and macvlan's per-container MACs rotate the VM's DHCP lease.
use_ipvlan: bool = Field(default=False)
class DeckyShard(SQLModel, table=True):
"""Mapping of a single decky to the worker host running it (swarm mode)."""
__tablename__ = "decky_shards"
decky_name: str = Field(primary_key=True)
host_uuid: str = Field(foreign_key="swarm_hosts.uuid", index=True)
# JSON list of service names running on this decky (snapshot of assignment).
services: str = Field(sa_column=Column("services", _BIG_TEXT, nullable=False, default="[]"))
# Full serialised DeckyConfig from the most recent dispatch or heartbeat.
# Lets the dashboard render the same rich card (hostname/distro/archetype/
# service_config/mutate_interval) that the local-fleet view uses, without
# needing a live round-trip to the worker for every page render.
decky_config: Optional[str] = Field(
default=None, sa_column=Column("decky_config", _BIG_TEXT, nullable=True)
)
decky_ip: Optional[str] = Field(default=None)
state: str = Field(default="pending", index=True) # pending|running|failed|torn_down|degraded|tearing_down|teardown_failed
last_error: Optional[str] = Field(default=None, sa_column=Column("last_error", Text, nullable=True))
compose_hash: Optional[str] = Field(default=None)
# Timestamp of the last heartbeat that echoed this shard; lets the UI
# show "stale" decks whose agent has gone silent.
last_seen: Optional[datetime] = Field(default=None)
updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
# --- Swarm API DTOs ---
# Request/response contracts for the master-side swarm controller
# (decnet/web/swarm_api.py). The underlying SQLModel tables — SwarmHost and
# DeckyShard — live above; these are the HTTP-facing shapes.
class SwarmEnrollRequest(BaseModel):
# x509 CommonName is capped at 64 bytes (RFC 5280 UB-common-name) — the
# cert issuer would reject anything longer with a ValueError.
# Pattern: ASCII hostname-safe characters only. The name is embedded
# both in the CN and as a SAN DNS entry; x509.DNSName only accepts
# A-label ASCII, so non-ASCII would blow up at issuance.
name: str = PydanticField(
..., min_length=1, max_length=64,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._\-]*$",
)
address: str = PydanticField(
..., min_length=1, max_length=253,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:\-]*$",
description="IP or DNS the master uses to reach the worker",
)
agent_port: int = PydanticField(default=8765, ge=1, le=65535)
sans: list[
Annotated[
str,
PydanticField(
min_length=1, max_length=253,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:\-]*$",
),
]
] = PydanticField(
default_factory=list,
description="Extra SANs (IPs / hostnames) to embed in the worker cert",
)
notes: Optional[str] = None
issue_updater_bundle: bool = PydanticField(
default=False,
description="If true, also issue an updater cert (CN=updater@<name>) for the remote self-updater",
)
class SwarmUpdaterBundle(BaseModel):
"""Subset of SwarmEnrolledBundle for the updater identity."""
fingerprint: str
updater_cert_pem: str
updater_key_pem: str
class SwarmEnrolledBundle(BaseModel):
"""Cert bundle returned to the operator — must be delivered to the worker."""
host_uuid: str
name: str
address: str
agent_port: int
fingerprint: str
ca_cert_pem: str
worker_cert_pem: str
worker_key_pem: str
updater: Optional[SwarmUpdaterBundle] = None
class SwarmHostView(BaseModel):
uuid: str
name: str
address: str
agent_port: int
status: str
last_heartbeat: Optional[datetime] = None
client_cert_fingerprint: str
updater_cert_fingerprint: Optional[str] = None
enrolled_at: datetime
notes: Optional[str] = None
use_ipvlan: bool = False
class DeckyShardView(BaseModel):
"""One decky → host mapping, enriched with the host's identity for display."""
decky_name: str
decky_ip: Optional[str] = None # resolved from the stored DecnetConfig at read time
host_uuid: str
host_name: str
host_address: str
host_status: str
services: list[str]
state: str
last_error: Optional[str] = None
compose_hash: Optional[str] = None
updated_at: datetime
# Enriched fields lifted from the stored DeckyConfig snapshot so the
# dashboard can render the same card shape as the local-fleet view.
hostname: Optional[str] = None
distro: Optional[str] = None
archetype: Optional[str] = None
service_config: dict[str, dict[str, Any]] = {}
mutate_interval: Optional[int] = None
last_mutated: float = 0.0
last_seen: Optional[datetime] = None
class SwarmDeployRequest(BaseModel):
config: DecnetConfig
dry_run: bool = False
no_cache: bool = False
class SwarmTeardownRequest(BaseModel):
host_uuid: Optional[str] = PydanticField(
default=None,
description="If set, tear down only this worker; otherwise tear down all hosts",
)
decky_id: Optional[str] = None
class SwarmHostResult(BaseModel):
host_uuid: str
host_name: str
ok: bool
detail: Any | None = None
class SwarmDeployResponse(BaseModel):
results: list[SwarmHostResult]
class SwarmHostHealth(BaseModel):
host_uuid: str
name: str
address: str
reachable: bool
detail: Any | None = None
class SwarmCheckResponse(BaseModel):
results: list[SwarmHostHealth]

442
decnet/web/db/models/topology.py Normal file
View File

@@ -0,0 +1,442 @@
"""MazeNET topology tables + the REST DTOs that wrap them."""
from datetime import datetime, timezone
from typing import Annotated, Any, Literal, Optional
from uuid import uuid4
from pydantic import BaseModel, BeforeValidator, ConfigDict, Field as PydanticField
from sqlalchemy import Column, Index, Text, UniqueConstraint
from sqlmodel import Field, SQLModel
from ._base import _BIG_TEXT
# --- MazeNET tables ---
# Nested deception topologies: an arbitrary-depth DAG of LANs connected by
# multi-homed "bridge" deckies. Purpose-built; disjoint from DeckyShard which
# remains SWARM-only.
class Topology(SQLModel, table=True):
__tablename__ = "topologies"
id: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
name: str = Field(index=True, unique=True)
mode: str = Field(default="unihost") # unihost|agent
# When ``mode == "agent"``, pins this topology to a specific enrolled
# worker. ``None`` for unihost topologies (master-local deploy).
target_host_uuid: Optional[str] = Field(
default=None, foreign_key="swarm_hosts.uuid", index=True
)
# Full TopologyConfig snapshot (including seed) used at generation time.
config_snapshot: str = Field(
sa_column=Column("config_snapshot", _BIG_TEXT, nullable=False, default="{}")
)
status: str = Field(
default="pending", index=True
) # pending|deploying|active|degraded|failed|tearing_down|torn_down
status_changed_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc)
)
created_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
# Optimistic-concurrency token. Bumped by repo methods that mutate
# the topology or any child row when an expected_version is supplied.
# Callers pass their last-seen version; mismatch raises VersionConflict.
version: int = Field(default=1, nullable=False)
# Set by the heartbeat handler when an agent's reported
# ``applied_version_hash`` diverges from what we expect it to be
# running. Drained by the mutator watch loop, which re-pushes via
# AgentClient and clears the flag. NULL for unihost topologies.
needs_resync: bool = Field(default=False, nullable=False)
# JSON-serialised list of EmailPersona dicts consumed by the
# ``decnet emailgen`` worker. Empty list = no fake mailbox owners
# configured for this topology, the worker skips it.
email_personas: str = Field(
sa_column=Column(
"email_personas", _BIG_TEXT, nullable=False, default="[]"
)
)
# ISO 639-1 language code applied to any persona that doesn't override
# ``language`` itself. English by default; ANTI's deployments default
# to "es" by editing this column.
language_default: str = Field(default="en", max_length=8)
class LAN(SQLModel, table=True):
__tablename__ = "lans"
__table_args__ = (UniqueConstraint("topology_id", "name", name="uq_lan_topology_name"),)
id: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
topology_id: str = Field(foreign_key="topologies.id", index=True)
name: str
# Populated after the Docker network is created; nullable before deploy.
docker_network_id: Optional[str] = Field(default=None)
subnet: str
is_dmz: bool = Field(default=False)
# Canvas layout coordinates (set by the web editor). Nullable so
# generator-emitted LANs don't need auto-layout at generation time.
x: Optional[float] = Field(default=None)
y: Optional[float] = Field(default=None)
class TopologyDecky(SQLModel, table=True):
"""A decky belonging to a MazeNET topology.
Disjoint from DeckyShard (which is SWARM-only). UUID PK; decky name is
unique only within a topology, so two topologies can both have a
``decky-01`` without colliding.
"""
__tablename__ = "topology_deckies"
__table_args__ = (
UniqueConstraint("topology_id", "name", name="uq_topology_decky_name"),
)
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
topology_id: str = Field(foreign_key="topologies.id", index=True)
name: str
# JSON list[str] of service names on this decky (snapshot of assignment).
services: str = Field(
sa_column=Column("services", _BIG_TEXT, nullable=False, default="[]")
)
# Full serialised DeckyConfig snapshot — lets the dashboard render the
# same card shape as DeckyShard without a live round-trip.
decky_config: Optional[str] = Field(
default=None, sa_column=Column("decky_config", _BIG_TEXT, nullable=True)
)
ip: Optional[str] = Field(default=None)
# Same vocabulary as DeckyShard.state to keep dashboard rendering uniform.
state: str = Field(
default="pending", index=True
) # pending|running|failed|torn_down|degraded|tearing_down|teardown_failed
last_error: Optional[str] = Field(
default=None, sa_column=Column("last_error", Text, nullable=True)
)
compose_hash: Optional[str] = Field(default=None)
last_seen: Optional[datetime] = Field(default=None)
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc)
)
# Canvas layout coordinates (set by the web editor). Nullable so
# generator-emitted deckies don't need auto-layout at generation time.
x: Optional[float] = Field(default=None)
y: Optional[float] = Field(default=None)
class TopologyEdge(SQLModel, table=True):
"""Membership edge: a decky attached to a LAN.
A decky appearing in ≥2 edges is multi-homed (a bridge decky).
"""
__tablename__ = "topology_edges"
id: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
topology_id: str = Field(foreign_key="topologies.id", index=True)
decky_uuid: str = Field(foreign_key="topology_deckies.uuid", index=True)
lan_id: str = Field(foreign_key="lans.id", index=True)
is_bridge: bool = Field(default=False)
forwards_l3: bool = Field(default=False)
class TopologyStatusEvent(SQLModel, table=True):
"""Append-only audit log of topology status transitions."""
__tablename__ = "topology_status_events"
id: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
topology_id: str = Field(foreign_key="topologies.id", index=True)
from_status: str
to_status: str
at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
reason: Optional[str] = Field(
default=None, sa_column=Column("reason", Text, nullable=True)
)
class TopologyMutation(SQLModel, table=True):
"""Operator-requested live mutation for an active MazeNET topology.
Each row is one intent (add LAN, attach decky, etc.). The mutator's
reconciler claims ``pending`` rows atomically (see
``SQLModelRepository.claim_next_mutation``), applies them against
Docker, and writes ``applied`` or ``failed`` back. The ``(state,
topology_id)`` composite index keeps the watch-loop guard query
cheap even with years of mutation history.
"""
__tablename__ = "topology_mutations"
__table_args__ = (
Index(
"ix_topology_mutations_state_topology",
"state",
"topology_id",
),
)
id: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
topology_id: str = Field(foreign_key="topologies.id", index=True)
# add_lan|remove_lan|add_decky|attach_decky|detach_decky|
# remove_decky|update_decky|update_lan
op: str = Field(index=True)
# JSON-serialised op payload (keys depend on ``op``).
payload: str = Field(
sa_column=Column("payload", _BIG_TEXT, nullable=False, default="{}")
)
# pending|applying|applied|failed
state: str = Field(default="pending", index=True)
requested_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
applied_at: Optional[datetime] = Field(default=None)
reason: Optional[str] = Field(
default=None, sa_column=Column("reason", Text, nullable=True)
)
# --- MazeNET Topology REST DTOs (phase 3) ---
# Request/response shapes for /api/v1/topologies. All write paths are
# admin-only; reads accept admin or viewer. Child CRUD is pending-only;
# mutations of active|degraded topologies go through the queue.
class TopologyGenerateRequest(BaseModel):
"""Body for POST /topologies — mirrors the `topology generate` CLI."""
name: str = PydanticField(..., min_length=1, max_length=64)
mode: str = PydanticField(default="unihost", pattern=r"^(unihost|agent)$")
target_host_uuid: Optional[str] = None
depth: int = PydanticField(..., ge=1, le=16)
branching_factor: int = PydanticField(..., ge=1, le=8)
deckies_per_lan_min: int = PydanticField(default=1, ge=0, le=32)
deckies_per_lan_max: int = PydanticField(default=3, ge=1, le=32)
bridge_forward_probability: float = PydanticField(default=1.0, ge=0.0, le=1.0)
cross_edge_probability: float = PydanticField(default=0.0, ge=0.0, le=1.0)
services_explicit: Optional[list[str]] = None
randomize_services: bool = True
seed: Optional[int] = PydanticField(default=None, ge=0)
class TopologySummary(BaseModel):
"""List-row shape for GET /topologies."""
model_config = ConfigDict(extra="ignore")
id: str
name: str
mode: str
target_host_uuid: Optional[str] = None
status: str
version: int
needs_resync: bool = False
created_at: datetime
status_changed_at: Optional[datetime] = None
class TopologyListResponse(BaseModel):
total: int
limit: Optional[int] = None
offset: Optional[int] = None
data: list[TopologySummary]
class LANRow(BaseModel):
model_config = ConfigDict(extra="ignore")
id: str
topology_id: str
name: str
subnet: str
is_dmz: bool = False
docker_network_id: Optional[str] = None
x: Optional[float] = None
y: Optional[float] = None
class DeckyRow(BaseModel):
model_config = ConfigDict(extra="ignore")
uuid: str
topology_id: str
name: str
services: list[str] = PydanticField(default_factory=list)
decky_config: Optional[dict[str, Any]] = None
ip: Optional[str] = None
state: str
last_error: Optional[str] = None
x: Optional[float] = None
y: Optional[float] = None
class EdgeRow(BaseModel):
model_config = ConfigDict(extra="ignore")
id: str
topology_id: str
decky_uuid: str
lan_id: str
is_bridge: bool = False
forwards_l3: bool = False
class TopologyDetail(BaseModel):
"""Hydrated topology — mirrors persistence.hydrate() output.
``topology`` uses :class:`TopologySummary` which already exposes
``target_host_uuid`` — agent-targeted topologies surface their
pinned host through that field.
"""
topology: TopologySummary
lans: list[LANRow]
deckies: list[DeckyRow]
edges: list[EdgeRow]
class TopologyStatusEventRow(BaseModel):
model_config = ConfigDict(extra="ignore")
id: str
topology_id: str
from_status: str
to_status: str
at: datetime
reason: Optional[str] = None
class LANCreateRequest(BaseModel):
name: str = PydanticField(..., min_length=1, max_length=64)
subnet: Optional[str] = None
is_dmz: bool = False
x: Optional[float] = None
y: Optional[float] = None
expected_version: Optional[int] = None
class LANUpdateRequest(BaseModel):
name: Optional[str] = None
subnet: Optional[str] = None
is_dmz: Optional[bool] = None
x: Optional[float] = None
y: Optional[float] = None
expected_version: Optional[int] = None
class DeckyCreateRequest(BaseModel):
name: str = PydanticField(..., min_length=1, max_length=64)
services: list[str] = PydanticField(default_factory=list)
decky_config: Optional[dict[str, Any]] = None
x: Optional[float] = None
y: Optional[float] = None
expected_version: Optional[int] = None
class DeckyUpdateRequest(BaseModel):
name: Optional[str] = None
services: Optional[list[str]] = None
decky_config: Optional[dict[str, Any]] = None
x: Optional[float] = None
y: Optional[float] = None
expected_version: Optional[int] = None
class EdgeCreateRequest(BaseModel):
decky_uuid: str
lan_id: str
is_bridge: bool = False
forwards_l3: bool = False
expected_version: Optional[int] = None
_MUTATION_OPS = Literal[
"add_lan",
"remove_lan",
"add_decky",
"attach_decky",
"detach_decky",
"remove_decky",
"update_decky",
"update_lan",
]
class MutationEnqueueRequest(BaseModel):
op: _MUTATION_OPS
payload: dict[str, Any] = PydanticField(default_factory=dict)
expected_version: Optional[int] = None
def _decode_json_payload(v: Any) -> Any:
"""Accept either a dict or a JSON-encoded string for mutation payloads."""
if isinstance(v, str):
import json as _json
return _json.loads(v) if v else {}
return v
_MutationPayload = Annotated[dict[str, Any], BeforeValidator(_decode_json_payload)]
class MutationRow(BaseModel):
model_config = ConfigDict(extra="ignore")
id: str
topology_id: str
op: str
payload: _MutationPayload = PydanticField(default_factory=dict)
state: str
requested_at: datetime
applied_at: Optional[datetime] = None
reason: Optional[str] = None
class MutationEnqueueResponse(BaseModel):
mutation_id: str
state: str = "pending"
class ValidationIssueResponse(BaseModel):
severity: str
code: str
message: str
target: dict[str, Any] = PydanticField(default_factory=dict)
class ValidationErrorResponse(BaseModel):
detail: str = "Topology validation failed"
issues: list[ValidationIssueResponse]
class VersionConflictResponse(BaseModel):
detail: str = "Topology version conflict"
current: int
expected: int
class NotEditableResponse(BaseModel):
detail: str = "Topology not editable"
status: str
reason: Optional[str] = None
class ServiceCatalogResponse(BaseModel):
services: list[str]
class ArchetypeEntry(BaseModel):
slug: str
display_name: str
description: str
services: list[str]
preferred_distros: list[str]
nmap_os: str
class ArchetypeCatalogResponse(BaseModel):
archetypes: list[ArchetypeEntry]
class NextIPResponse(BaseModel):
subnet: str
ip: str
class NextSubnetResponse(BaseModel):
subnet: str
class DeployAcceptedResponse(BaseModel):
topology_id: str
status: str
dry_run: bool = False
class ReapReportResponse(BaseModel):
live_prefixes: list[str]
orphan_prefixes: list[str]
containers_removed: list[str]
networks_removed: list[str]
errors: list[str]

73
decnet/web/db/models/updater.py Normal file
View File

@@ -0,0 +1,73 @@
"""Remote updates DTOs (master → worker /updater fan-out)."""
from typing import Any, Literal, Optional
from pydantic import BaseModel, Field as PydanticField
# --- Remote Updates (master → worker /updater) DTOs ---
# Powers the dashboard's Remote Updates page. The master dashboard calls
# these (auth-gated) endpoints; internally they fan out to each worker's
# updater daemon over mTLS via UpdaterClient.
class HostReleaseInfo(BaseModel):
host_uuid: str
host_name: str
address: str
reachable: bool
# These fields mirror the updater's /health payload when reachable; they
# are all Optional so an unreachable host still serializes cleanly.
agent_status: Optional[str] = None
current_sha: Optional[str] = None
previous_sha: Optional[str] = None
releases: list[dict[str, Any]] = PydanticField(default_factory=list)
detail: Optional[str] = None # populated when unreachable
class HostReleasesResponse(BaseModel):
hosts: list[HostReleaseInfo]
class PushUpdateRequest(BaseModel):
host_uuids: Optional[list[str]] = PydanticField(
default=None,
description="Target specific hosts; mutually exclusive with 'all'.",
)
all: bool = PydanticField(default=False, description="Target every non-decommissioned host with an updater bundle.")
include_self: bool = PydanticField(
default=False,
description="After a successful /update, also push /update-self to upgrade the updater itself.",
)
exclude: list[str] = PydanticField(
default_factory=list,
description="Additional tarball exclude globs (on top of the built-in defaults).",
)
class PushUpdateResult(BaseModel):
host_uuid: str
host_name: str
# updated = /update 200. rolled-back = /update 409 (auto-recovered).
# failed = transport error or non-200/409 response. self-updated = /update-self succeeded.
status: Literal["updated", "rolled-back", "failed", "self-updated", "self-failed"]
http_status: Optional[int] = None
sha: Optional[str] = None
detail: Optional[str] = None
stderr: Optional[str] = None
class PushUpdateResponse(BaseModel):
sha: str
tarball_bytes: int
results: list[PushUpdateResult]
class RollbackRequest(BaseModel):
host_uuid: str = PydanticField(..., description="Host to roll back to its previous release slot.")
class RollbackResponse(BaseModel):
host_uuid: str
host_name: str
status: Literal["rolled-back", "failed"]
http_status: Optional[int] = None
detail: Optional[str] = None

162
decnet/web/db/models/webhooks.py Normal file
View File

@@ -0,0 +1,162 @@
"""Webhook subscription table + CRUD DTOs.
Webhooks push DECNET bus events out to external SIEM / SOAR stacks
(Wazuh, Shuffle, TheHive, n8n, ...). Each subscription carries a set
of NATS-style topic patterns; the `decnet webhook` worker subscribes
to the union of patterns across all enabled subscriptions and POSTs
matching events to each matching URL with HMAC-SHA256 signing.
Simple mode (UI) exposes a friendly enum (`AttackerDetail`,
`DeckyStatus`, `SystemStatus`) that expands to patterns at save time.
Advanced mode lets an admin set raw patterns directly. Storage is
always the expanded list — the enum is sugar at the router layer.
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, List, Literal, Optional
from uuid import uuid4
from pydantic import BaseModel, Field as PydanticField, HttpUrl
from sqlmodel import Field, SQLModel
SimpleEvent = Literal["AttackerDetail", "DeckyStatus", "SystemStatus"]
class WebhookSubscription(SQLModel, table=True):
__tablename__ = "webhook_subscriptions"
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
name: str = Field(index=True, unique=True)
url: str
secret: str # HMAC-SHA256 key; plaintext pre-v1 (see DEBT-037 §7)
# JSON-encoded list[str] of NATS-style bus topic patterns.
# Storing as TEXT keeps the schema portable across SQLite and MySQL
# without pulling in dialect-specific JSON columns.
topic_patterns: str = Field(default="[]")
enabled: bool = Field(default=True, index=True)
consecutive_failures: int = Field(default=0)
last_success_at: Optional[datetime] = None
last_failure_at: Optional[datetime] = None
last_error: Optional[str] = None
# Set when the circuit breaker auto-disables the subscription after
# too many consecutive failures. NULL means "not tripped" — the
# subscription is either active (enabled=True) or admin-paused
# (enabled=False, auto_disabled_at=NULL). A non-NULL stamp with
# enabled=False means the worker tripped it; the operator clears
# the flag by re-enabling via PATCH.
auto_disabled_at: Optional[datetime] = None
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
def patterns(self) -> list[str]:
"""Decode `topic_patterns` to a list. Returns [] on bad/empty JSON."""
try:
raw = json.loads(self.topic_patterns or "[]")
except (ValueError, TypeError):
return []
return [p for p in raw if isinstance(p, str)]
# --- API Request / Response Models (Pydantic) ---
class WebhookCreateRequest(BaseModel):
name: str = PydanticField(..., min_length=1, max_length=64)
url: HttpUrl
# If secret is omitted, the router generates a secure random one and
# returns it exactly once on the create response. After that, callers
# can only rotate via PATCH.
secret: Optional[str] = PydanticField(None, min_length=16, max_length=256)
# At least one of simple_events / topic_patterns must be non-empty
# (validated in the router, not Pydantic, so the 400 carries a clear
# detail message).
simple_events: List[SimpleEvent] = PydanticField(default_factory=list)
topic_patterns: List[str] = PydanticField(default_factory=list)
enabled: bool = True
class WebhookUpdateRequest(BaseModel):
# Partial update — every field optional; the router diffs against the
# current row and only writes what changed.
name: Optional[str] = PydanticField(None, min_length=1, max_length=64)
url: Optional[HttpUrl] = None
secret: Optional[str] = PydanticField(None, min_length=16, max_length=256)
simple_events: Optional[List[SimpleEvent]] = None
topic_patterns: Optional[List[str]] = None
enabled: Optional[bool] = None
class WebhookResponse(BaseModel):
"""Public shape — deliberately omits `secret`.
The `warnings` field carries non-blocking advisories about the
subscription's configuration — e.g. an `http://` URL is fine but
surfaces a warning so the operator knows the event body is
plaintext on the wire. Empty list when nothing is worth flagging.
"""
uuid: str
name: str
url: str
topic_patterns: List[str]
enabled: bool
consecutive_failures: int
last_success_at: Optional[datetime] = None
last_failure_at: Optional[datetime] = None
last_error: Optional[str] = None
auto_disabled_at: Optional[datetime] = None
created_at: datetime
updated_at: datetime
warnings: List[str] = PydanticField(default_factory=list)
class WebhookCreateResponse(WebhookResponse):
"""Create-path response — carries the secret exactly once, for copy-out."""
secret: str
class WebhookTestResponse(BaseModel):
delivered: bool
status_code: Optional[int] = None
error: Optional[str] = None
def _compute_warnings(url: str) -> List[str]:
"""Non-blocking advisories about a subscription's configuration.
The HMAC signature detects tampering regardless of transport, but an
on-path attacker can still *read* the event body over plaintext HTTP.
We surface the warning and let the admin decide — matches DECNET's
operator-trust posture (see THREAT_MODEL WH-03).
"""
out: List[str] = []
lower = (url or "").lower()
if lower.startswith("http://"):
out.append(
"insecure_url: URL uses http://. Event bodies (including "
"payload fields) traverse the wire in plaintext; HMAC still "
"detects tampering but anyone on-path can read the event. "
"Use https:// in production."
)
return out
def _row_to_response_dict(row: dict[str, Any]) -> dict[str, Any]:
"""Normalize a DB row into the WebhookResponse dict shape.
Used by the CRUD router to decode `topic_patterns` JSON, drop the
`secret` column, and compute any configuration warnings.
"""
out = dict(row)
raw = out.pop("topic_patterns", "[]")
try:
out["topic_patterns"] = json.loads(raw or "[]")
except (ValueError, TypeError):
out["topic_patterns"] = []
out.pop("secret", None)
out["warnings"] = _compute_warnings(out.get("url", ""))
return out

50
decnet/web/db/models/workers.py Normal file
View File

@@ -0,0 +1,50 @@
"""Workers panel DTOs (bus-backed health + control)."""
from typing import Any, Dict, List, Literal, Optional
from pydantic import BaseModel, Field as PydanticField
# --- Workers panel (Config → Workers) ---
# Bus-backed health + control: workers heartbeat on ``system.<name>.health``
# and listen on ``system.<name>.control``. The API aggregates last-seen
# heartbeats via the worker registry; these are the HTTP-facing shapes.
class WorkerStatus(BaseModel):
name: str
# ``ok`` — heartbeat within 90s (3× 30s heartbeat interval)
# ``stale`` — worker was seen before but hasn't pulsed in 90s+
# ``unknown`` — we've never received a heartbeat from this name
status: Literal["ok", "stale", "unknown"]
last_heartbeat_ts: Optional[float] = None
seconds_since: Optional[float] = None
# Whatever the worker's ``extra()`` callback put in the heartbeat;
# opaque to the panel, displayed only if the UI knows the key.
extra: Dict[str, Any] = PydanticField(default_factory=dict)
# True iff a ``decnet-<name>.service`` unit file is present on the
# host. False flips the UI START button to disabled with a
# "Unit not installed" tooltip. Default True for backwards compat
# on clients that pre-date the field.
installed: bool = True
class WorkersResponse(BaseModel):
workers: List[WorkerStatus]
generated_at: float
bus_connected: bool
class WorkerControlResponse(BaseModel):
accepted: bool
worker: str
action: str
class StartFailure(BaseModel):
name: str
reason: str
class StartAllResponse(BaseModel):
started: List[str]
already_running: List[str]
failed: List[StartFailure]

0
decnet/web/db/mysql/__init__.py Normal file
View File

98
decnet/web/db/mysql/database.py Normal file
View File

@@ -0,0 +1,98 @@
"""
MySQL async engine factory.
Builds a SQLAlchemy AsyncEngine against MySQL using the ``asyncmy`` driver.
Connection info is resolved (in order of precedence):
1. An explicit ``url`` argument passed to :func:`get_async_engine`
2. ``DECNET_DB_URL`` — full SQLAlchemy URL
3. Component env vars:
``DECNET_DB_HOST`` (default ``localhost``)
``DECNET_DB_PORT`` (default ``3306``)
``DECNET_DB_NAME`` (default ``decnet``)
``DECNET_DB_USER`` (default ``decnet``)
``DECNET_DB_PASSWORD`` (default empty — raises unless pytest is running)
"""
from __future__ import annotations
import os
from typing import Optional
from urllib.parse import quote_plus
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
DEFAULT_POOL_SIZE = int(os.environ.get("DECNET_DB_POOL_SIZE", "20"))
DEFAULT_MAX_OVERFLOW = int(os.environ.get("DECNET_DB_MAX_OVERFLOW", "40"))
DEFAULT_POOL_RECYCLE = int(os.environ.get("DECNET_DB_POOL_RECYCLE", "3600"))
DEFAULT_POOL_PRE_PING = os.environ.get("DECNET_DB_POOL_PRE_PING", "true").lower() == "true"
def build_mysql_url(
host: Optional[str] = None,
port: Optional[int] = None,
database: Optional[str] = None,
user: Optional[str] = None,
password: Optional[str] = None,
) -> str:
"""Compose an async SQLAlchemy URL for MySQL using the asyncmy driver.
Component args override env vars. Password is percent-encoded so special
characters (``@``, ``:``, ``/``…) don't break URL parsing.
"""
host = host or os.environ.get("DECNET_DB_HOST", "localhost")
port = port or int(os.environ.get("DECNET_DB_PORT", "3306"))
database = database or os.environ.get("DECNET_DB_NAME", "decnet")
user = user or os.environ.get("DECNET_DB_USER", "decnet")
if password is None:
password = os.environ.get("DECNET_DB_PASSWORD", "")
# Allow empty passwords during tests (pytest sets PYTEST_* env vars).
# Outside tests, an empty MySQL password is almost never intentional.
if not password and not any(k.startswith("PYTEST") for k in os.environ):
raise ValueError(
"DECNET_DB_PASSWORD is not set. Either export it, set DECNET_DB_URL, "
"or run under pytest for an empty-password default."
)
pw_enc = quote_plus(password)
user_enc = quote_plus(user)
return f"mysql+asyncmy://{user_enc}:{pw_enc}@{host}:{port}/{database}"
def resolve_url(url: Optional[str] = None) -> str:
"""Pick a connection URL: explicit arg → DECNET_DB_URL env → built from components."""
if url:
return url
env_url = os.environ.get("DECNET_DB_URL")
if env_url:
return env_url
return build_mysql_url()
def get_async_engine(
url: Optional[str] = None,
*,
pool_size: int = DEFAULT_POOL_SIZE,
max_overflow: int = DEFAULT_MAX_OVERFLOW,
pool_recycle: int = DEFAULT_POOL_RECYCLE,
pool_pre_ping: bool = DEFAULT_POOL_PRE_PING,
echo: bool = False,
) -> AsyncEngine:
"""Create an AsyncEngine for MySQL.
Defaults tuned for a dashboard workload: a modest pool, hourly recycle
to sidestep MySQL's idle-connection reaper, and pre-ping to fail fast
if a pooled connection has been killed server-side.
"""
dsn = resolve_url(url)
return create_async_engine(
dsn,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_recycle=pool_recycle,
pool_pre_ping=pool_pre_ping,
)

187
decnet/web/db/mysql/repository.py Normal file
View File

@@ -0,0 +1,187 @@
"""
MySQL implementation of :class:`BaseRepository`.
Inherits the portable SQLModel query code from :class:`SQLModelRepository`
and only overrides the two places where MySQL's SQL dialect differs from
SQLite's:
* :meth:`_migrate_attackers_table` — uses ``information_schema`` (MySQL
has no ``PRAGMA``).
* :meth:`get_log_histogram` — uses ``FROM_UNIXTIME`` /
``UNIX_TIMESTAMP`` + integer division for bucketing.
"""
from __future__ import annotations
from typing import List, Optional
from sqlalchemy import func, select, text, literal_column
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from sqlmodel.sql.expression import SelectOfScalar
from decnet.web.db.models import Log
from decnet.web.db.mysql.database import get_async_engine
from decnet.web.db.sqlmodel_repo import SQLModelRepository
class MySQLRepository(SQLModelRepository):
"""MySQL backend — uses ``asyncmy``."""
def __init__(self, url: Optional[str] = None, **engine_kwargs) -> None:
self.engine = get_async_engine(url=url, **engine_kwargs)
self.session_factory = async_sessionmaker(
self.engine, class_=AsyncSession, expire_on_commit=False
)
async def _migrate_attackers_table(self) -> None:
"""Drop the legacy (pre-UUID) ``attackers`` table if it exists without a ``uuid`` column.
Also adds the GeoIP columns (``country_code``, ``country_source``)
to existing tables that predate them. MySQL exposes column
metadata via ``information_schema.COLUMNS``; ``DATABASE()`` scopes
the lookup to the currently connected schema.
"""
async with self.engine.begin() as conn:
rows = (await conn.execute(text(
"SELECT COLUMN_NAME FROM information_schema.COLUMNS "
"WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'attackers'"
))).fetchall()
if not rows:
return # table absent; create_all() handles it.
if not any(r[0] == "uuid" for r in rows):
await conn.execute(text("DROP TABLE attackers"))
return
existing_cols = {r[0] for r in rows}
if "country_code" not in existing_cols:
await conn.execute(text(
"ALTER TABLE attackers "
"ADD COLUMN country_code VARCHAR(2) NULL, "
"ADD INDEX ix_attackers_country_code (country_code)"
))
if "country_source" not in existing_cols:
await conn.execute(text(
"ALTER TABLE attackers ADD COLUMN country_source VARCHAR(16) NULL"
))
async def _migrate_column_types(self) -> None:
"""Upgrade TEXT → MEDIUMTEXT for columns that accumulate large JSON blobs.
``create_all()`` never alters existing columns, so tables created before
``_BIG_TEXT`` was introduced keep their 64 KiB ``TEXT`` cap. This method
inspects ``information_schema`` and issues ``ALTER TABLE … MODIFY COLUMN``
for each offending column found.
"""
targets: dict[str, dict[str, str]] = {
"attackers": {
"commands": "MEDIUMTEXT NOT NULL DEFAULT '[]'",
"fingerprints": "MEDIUMTEXT NOT NULL DEFAULT '[]'",
"services": "MEDIUMTEXT NOT NULL DEFAULT '[]'",
"deckies": "MEDIUMTEXT NOT NULL DEFAULT '[]'",
},
"state": {
"value": "MEDIUMTEXT NOT NULL",
},
}
async with self.engine.begin() as conn:
rows = (await conn.execute(text(
"SELECT TABLE_NAME, COLUMN_NAME FROM information_schema.COLUMNS "
"WHERE TABLE_SCHEMA = DATABASE() "
" AND TABLE_NAME IN ('attackers', 'state') "
" AND COLUMN_NAME IN ('commands','fingerprints','services','deckies','value') "
" AND DATA_TYPE = 'text'"
))).fetchall()
for table_name, col_name in rows:
spec = targets.get(table_name, {}).get(col_name)
if spec:
await conn.execute(text(
f"ALTER TABLE `{table_name}` MODIFY COLUMN `{col_name}` {spec}"
))
async def _migrate_session_profile_table(self) -> None:
"""Add DEBT-036 keystroke-dynamics columns (start-of-action latency,
three-bucket pause histogram, top-bigrams JSON) to existing tables.
MySQL's ``ALTER TABLE ADD COLUMN`` fails if the column already
exists, so gate on ``information_schema.COLUMNS`` to stay
idempotent.
"""
async with self.engine.begin() as conn:
rows = (await conn.execute(text(
"SELECT COLUMN_NAME FROM information_schema.COLUMNS "
"WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'session_profile'"
))).fetchall()
if not rows:
return
existing_cols = {r[0] for r in rows}
additions = [
("kd_top_bigrams", "TEXT NULL"),
("kd_start_of_action_latency", "DOUBLE NULL"),
("kd_pause_hist_burst", "INT NULL"),
("kd_pause_hist_think", "INT NULL"),
("kd_pause_hist_distracted", "INT NULL"),
]
for col_name, col_spec in additions:
if col_name not in existing_cols:
await conn.execute(text(
f"ALTER TABLE session_profile ADD COLUMN {col_name} {col_spec}"
))
async def initialize(self) -> None:
"""Create tables and run all MySQL-specific migrations.
Uses a MySQL advisory lock to serialize DDL across concurrent
uvicorn workers — prevents the 'Table was skipped since its
definition is being modified by concurrent DDL' race.
"""
from sqlmodel import SQLModel
async with self.engine.connect() as lock_conn:
await lock_conn.execute(text("SELECT GET_LOCK('decnet_schema_init', 30)"))
try:
await self._migrate_attackers_table()
await self._migrate_session_profile_table()
await self._migrate_column_types()
async with self.engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
await self._ensure_admin_user()
finally:
await lock_conn.execute(text("SELECT RELEASE_LOCK('decnet_schema_init')"))
await lock_conn.close()
def _json_field_equals(self, key: str):
# MySQL 5.7+ exposes JSON_EXTRACT; quoted string result returned for
# TEXT-stored JSON, same behavior we rely on in SQLite.
return text(f"JSON_UNQUOTE(JSON_EXTRACT(fields, '$.{key}')) = :val")
async def get_log_histogram(
self,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
interval_minutes: int = 15,
) -> List[dict]:
bucket_seconds = max(interval_minutes, 1) * 60
# Truncate each timestamp to the start of its bucket:
# FROM_UNIXTIME( (UNIX_TIMESTAMP(timestamp) DIV N) * N )
# DIV is MySQL's integer division operator.
bucket_expr = literal_column(
f"FROM_UNIXTIME((UNIX_TIMESTAMP(timestamp) DIV {bucket_seconds}) * {bucket_seconds})"
).label("bucket_time")
statement: SelectOfScalar = select(bucket_expr, func.count().label("count")).select_from(Log)
statement = self._apply_filters(statement, search, start_time, end_time)
statement = statement.group_by(literal_column("bucket_time")).order_by(
literal_column("bucket_time")
)
async with self._session() as session:
results = await session.execute(statement)
# Normalize to ISO string for API parity with the SQLite backend
# (SQLite's datetime() returns a string already; FROM_UNIXTIME
# returns a datetime).
out: List[dict] = []
for r in results.all():
ts = r[0]
out.append({
"time": ts.isoformat(sep=" ") if hasattr(ts, "isoformat") else ts,
"count": r[1],
})
return out

1128
decnet/web/db/repository.py
View File

File diff suppressed because it is too large Load Diff

33
decnet/web/db/sqlite/database.py
View File

@@ -1,5 +1,7 @@
import os
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy import create_engine, Engine
from sqlalchemy import create_engine, Engine, event
from sqlmodel import SQLModel
from typing import AsyncGenerator
@@ -11,7 +13,34 @@ def get_async_engine(db_path: str) -> AsyncEngine:
prefix = "sqlite+aiosqlite:///"
if db_path.startswith(":memory:"):
prefix = "sqlite+aiosqlite://"
return create_async_engine(f"{prefix}{db_path}", echo=False, connect_args={"uri": True})
pool_size = int(os.environ.get("DECNET_DB_POOL_SIZE", "20"))
max_overflow = int(os.environ.get("DECNET_DB_MAX_OVERFLOW", "40"))
pool_recycle = int(os.environ.get("DECNET_DB_POOL_RECYCLE", "3600"))
# SQLite is a local file — dead-connection probes are pure overhead.
# Env var stays for network-mounted setups that still want it.
pool_pre_ping = os.environ.get("DECNET_DB_POOL_PRE_PING", "false").lower() == "true"
engine = create_async_engine(
f"{prefix}{db_path}",
echo=False,
pool_size=pool_size,
max_overflow=max_overflow,
pool_recycle=pool_recycle,
pool_pre_ping=pool_pre_ping,
connect_args={"uri": True, "timeout": 30},
)
@event.listens_for(engine.sync_engine, "connect")
def _set_sqlite_pragmas(dbapi_conn, _conn_record):
cursor = dbapi_conn.cursor()
cursor.execute("PRAGMA journal_mode=WAL")
cursor.execute("PRAGMA synchronous=NORMAL")
cursor.execute("PRAGMA busy_timeout=30000")
cursor.close()
return engine
def get_sync_engine(db_path: str) -> Engine:
prefix = "sqlite:///"

404
decnet/web/db/sqlite/repository.py
View File

@@ -1,23 +1,22 @@
import asyncio
import json
import uuid
from datetime import datetime
from typing import Any, Optional, List
from typing import List, Optional
from sqlalchemy import func, select, desc, asc, text, or_, update, literal_column
from sqlalchemy import func, select, text, literal_column
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from sqlmodel.sql.expression import SelectOfScalar
from decnet.config import load_state, _ROOT
from decnet.env import DECNET_ADMIN_USER, DECNET_ADMIN_PASSWORD
from decnet.web.auth import get_password_hash
from decnet.web.db.repository import BaseRepository
from decnet.web.db.models import User, Log, Bounty, State
from decnet.config import _ROOT
from decnet.web.db.models import Log
from decnet.web.db.sqlite.database import get_async_engine
from decnet.web.db.sqlmodel_repo import SQLModelRepository
class SQLiteRepository(BaseRepository):
"""SQLite implementation using SQLModel and SQLAlchemy Async."""
class SQLiteRepository(SQLModelRepository):
"""SQLite backend — uses ``aiosqlite``.
Overrides the two places where SQLite's SQL dialect differs from
MySQL/PostgreSQL: legacy-schema migration (via ``PRAGMA table_info``)
and the log-histogram bucket expression (via ``strftime`` + ``unixepoch``).
"""
def __init__(self, db_path: str = str(_ROOT / "decnet.db")) -> None:
self.db_path = db_path
@@ -26,173 +25,63 @@ class SQLiteRepository(BaseRepository):
self.engine, class_=AsyncSession, expire_on_commit=False
)
async def initialize(self) -> None:
"""Async warm-up / verification. Creates tables if they don't exist."""
from sqlmodel import SQLModel
async def _migrate_attackers_table(self) -> None:
"""Drop the old attackers table if it lacks the uuid column (pre-UUID schema).
Also adds the GeoIP columns (``country_code``, ``country_source``)
to existing tables that predate them. SQLite's
``ALTER TABLE ADD COLUMN`` is idempotent only if we gate on
``PRAGMA table_info`` first — re-adding raises.
"""
async with self.engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
async with self.session_factory() as session:
# Check if admin exists
result = await session.execute(
select(User).where(User.username == DECNET_ADMIN_USER)
)
if not result.scalar_one_or_none():
session.add(User(
uuid=str(uuid.uuid4()),
username=DECNET_ADMIN_USER,
password_hash=get_password_hash(DECNET_ADMIN_PASSWORD),
role="admin",
must_change_password=True,
rows = (await conn.execute(text("PRAGMA table_info(attackers)"))).fetchall()
if rows and not any(r[1] == "uuid" for r in rows):
await conn.execute(text("DROP TABLE attackers"))
return # create_all() rebuilds fresh — no need to patch columns.
if not rows:
return # table absent; create_all() handles it.
existing_cols = {r[1] for r in rows}
if "country_code" not in existing_cols:
await conn.execute(text(
"ALTER TABLE attackers ADD COLUMN country_code VARCHAR(2)"
))
await conn.execute(text(
"CREATE INDEX IF NOT EXISTS ix_attackers_country_code "
"ON attackers (country_code)"
))
if "country_source" not in existing_cols:
await conn.execute(text(
"ALTER TABLE attackers ADD COLUMN country_source VARCHAR(16)"
))
await session.commit()
async def reinitialize(self) -> None:
"""Initialize the database schema asynchronously (useful for tests)."""
from sqlmodel import SQLModel
async def _migrate_session_profile_table(self) -> None:
"""Add DEBT-036 keystroke-dynamics columns (start-of-action latency,
three-bucket pause histogram, top-bigrams JSON) to existing tables.
SQLite's ``ALTER TABLE ADD COLUMN`` fails if the column already
exists, so gate on ``PRAGMA table_info`` to stay idempotent.
"""
async with self.engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
rows = (await conn.execute(text("PRAGMA table_info(session_profile)"))).fetchall()
if not rows:
return # table absent; create_all() handles it.
existing_cols = {r[1] for r in rows}
additions = [
("kd_top_bigrams", "TEXT"),
("kd_start_of_action_latency", "REAL"),
("kd_pause_hist_burst", "INTEGER"),
("kd_pause_hist_think", "INTEGER"),
("kd_pause_hist_distracted", "INTEGER"),
]
for col_name, col_type in additions:
if col_name not in existing_cols:
await conn.execute(text(
f"ALTER TABLE session_profile ADD COLUMN {col_name} {col_type}"
))
async with self.session_factory() as session:
result = await session.execute(
select(User).where(User.username == DECNET_ADMIN_USER)
)
if not result.scalar_one_or_none():
session.add(User(
uuid=str(uuid.uuid4()),
username=DECNET_ADMIN_USER,
password_hash=get_password_hash(DECNET_ADMIN_PASSWORD),
role="admin",
must_change_password=True,
))
await session.commit()
# ------------------------------------------------------------------ logs
async def add_log(self, log_data: dict[str, Any]) -> None:
data = log_data.copy()
if "fields" in data and isinstance(data["fields"], dict):
data["fields"] = json.dumps(data["fields"])
if "timestamp" in data and isinstance(data["timestamp"], str):
try:
data["timestamp"] = datetime.fromisoformat(
data["timestamp"].replace("Z", "+00:00")
)
except ValueError:
pass
async with self.session_factory() as session:
session.add(Log(**data))
await session.commit()
def _apply_filters(
self,
statement: SelectOfScalar,
search: Optional[str],
start_time: Optional[str],
end_time: Optional[str],
) -> SelectOfScalar:
import re
import shlex
if start_time:
statement = statement.where(Log.timestamp >= start_time)
if end_time:
statement = statement.where(Log.timestamp <= end_time)
if search:
try:
tokens = shlex.split(search)
except ValueError:
tokens = search.split()
core_fields = {
"decky": Log.decky,
"service": Log.service,
"event": Log.event_type,
"attacker": Log.attacker_ip,
"attacker-ip": Log.attacker_ip,
"attacker_ip": Log.attacker_ip,
}
for token in tokens:
if ":" in token:
key, val = token.split(":", 1)
if key in core_fields:
statement = statement.where(core_fields[key] == val)
else:
key_safe = re.sub(r"[^a-zA-Z0-9_]", "", key)
if key_safe:
statement = statement.where(
text(f"json_extract(fields, '$.{key_safe}') = :val")
).params(val=val)
else:
lk = f"%{token}%"
statement = statement.where(
or_(
Log.raw_line.like(lk),
Log.decky.like(lk),
Log.service.like(lk),
Log.attacker_ip.like(lk),
)
)
return statement
async def get_logs(
self,
limit: int = 50,
offset: int = 0,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
) -> List[dict]:
statement = (
select(Log)
.order_by(desc(Log.timestamp))
.offset(offset)
.limit(limit)
)
statement = self._apply_filters(statement, search, start_time, end_time)
async with self.session_factory() as session:
results = await session.execute(statement)
return [log.model_dump(mode='json') for log in results.scalars().all()]
async def get_max_log_id(self) -> int:
async with self.session_factory() as session:
result = await session.execute(select(func.max(Log.id)))
val = result.scalar()
return val if val is not None else 0
async def get_logs_after_id(
self,
last_id: int,
limit: int = 50,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
) -> List[dict]:
statement = (
select(Log).where(Log.id > last_id).order_by(asc(Log.id)).limit(limit)
)
statement = self._apply_filters(statement, search, start_time, end_time)
async with self.session_factory() as session:
results = await session.execute(statement)
return [log.model_dump(mode='json') for log in results.scalars().all()]
async def get_total_logs(
self,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
) -> int:
statement = select(func.count()).select_from(Log)
statement = self._apply_filters(statement, search, start_time, end_time)
async with self.session_factory() as session:
result = await session.execute(statement)
return result.scalar() or 0
def _json_field_equals(self, key: str):
# SQLite stores JSON as text; json_extract is the canonical accessor.
return text(f"json_extract(fields, '$.{key}') = :val")
async def get_log_histogram(
self,
@@ -206,173 +95,12 @@ class SQLiteRepository(BaseRepository):
f"datetime((strftime('%s', timestamp) / {bucket_seconds}) * {bucket_seconds}, 'unixepoch')"
).label("bucket_time")
statement = select(bucket_expr, func.count().label("count")).select_from(Log)
statement: SelectOfScalar = select(bucket_expr, func.count().label("count")).select_from(Log)
statement = self._apply_filters(statement, search, start_time, end_time)
statement = statement.group_by(literal_column("bucket_time")).order_by(
literal_column("bucket_time")
)
async with self.session_factory() as session:
async with self._session() as session:
results = await session.execute(statement)
return [{"time": r[0], "count": r[1]} for r in results.all()]
async def get_stats_summary(self) -> dict[str, Any]:
async with self.session_factory() as session:
total_logs = (
await session.execute(select(func.count()).select_from(Log))
).scalar() or 0
unique_attackers = (
await session.execute(
select(func.count(func.distinct(Log.attacker_ip)))
)
).scalar() or 0
active_deckies = (
await session.execute(
select(func.count(func.distinct(Log.decky)))
)
).scalar() or 0
_state = await asyncio.to_thread(load_state)
deployed_deckies = len(_state[0].deckies) if _state else 0
return {
"total_logs": total_logs,
"unique_attackers": unique_attackers,
"active_deckies": active_deckies,
"deployed_deckies": deployed_deckies,
}
async def get_deckies(self) -> List[dict]:
_state = await asyncio.to_thread(load_state)
return [_d.model_dump() for _d in _state[0].deckies] if _state else []
# ------------------------------------------------------------------ users
async def get_user_by_username(self, username: str) -> Optional[dict]:
async with self.session_factory() as session:
result = await session.execute(
select(User).where(User.username == username)
)
user = result.scalar_one_or_none()
return user.model_dump() if user else None
async def get_user_by_uuid(self, uuid: str) -> Optional[dict]:
async with self.session_factory() as session:
result = await session.execute(
select(User).where(User.uuid == uuid)
)
user = result.scalar_one_or_none()
return user.model_dump() if user else None
async def create_user(self, user_data: dict[str, Any]) -> None:
async with self.session_factory() as session:
session.add(User(**user_data))
await session.commit()
async def update_user_password(
self, uuid: str, password_hash: str, must_change_password: bool = False
) -> None:
async with self.session_factory() as session:
await session.execute(
update(User)
.where(User.uuid == uuid)
.values(
password_hash=password_hash,
must_change_password=must_change_password,
)
)
await session.commit()
# ---------------------------------------------------------------- bounties
async def add_bounty(self, bounty_data: dict[str, Any]) -> None:
data = bounty_data.copy()
if "payload" in data and isinstance(data["payload"], dict):
data["payload"] = json.dumps(data["payload"])
async with self.session_factory() as session:
session.add(Bounty(**data))
await session.commit()
def _apply_bounty_filters(
self,
statement: SelectOfScalar,
bounty_type: Optional[str],
search: Optional[str]
) -> SelectOfScalar:
if bounty_type:
statement = statement.where(Bounty.bounty_type == bounty_type)
if search:
lk = f"%{search}%"
statement = statement.where(
or_(
Bounty.decky.like(lk),
Bounty.service.like(lk),
Bounty.attacker_ip.like(lk),
Bounty.payload.like(lk),
)
)
return statement
async def get_bounties(
self,
limit: int = 50,
offset: int = 0,
bounty_type: Optional[str] = None,
search: Optional[str] = None,
) -> List[dict]:
statement = (
select(Bounty)
.order_by(desc(Bounty.timestamp))
.offset(offset)
.limit(limit)
)
statement = self._apply_bounty_filters(statement, bounty_type, search)
async with self.session_factory() as session:
results = await session.execute(statement)
final = []
for item in results.scalars().all():
d = item.model_dump(mode='json')
try:
d["payload"] = json.loads(d["payload"])
except (json.JSONDecodeError, TypeError):
pass
final.append(d)
return final
async def get_total_bounties(
self, bounty_type: Optional[str] = None, search: Optional[str] = None
) -> int:
statement = select(func.count()).select_from(Bounty)
statement = self._apply_bounty_filters(statement, bounty_type, search)
async with self.session_factory() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def get_state(self, key: str) -> Optional[dict[str, Any]]:
async with self.session_factory() as session:
statement = select(State).where(State.key == key)
result = await session.execute(statement)
state = result.scalar_one_or_none()
if state:
return json.loads(state.value)
return None
async def set_state(self, key: str, value: Any) -> None: # noqa: ANN401
async with self.session_factory() as session:
# Check if exists
statement = select(State).where(State.key == key)
result = await session.execute(statement)
state = result.scalar_one_or_none()
value_json = json.dumps(value)
if state:
state.value = value_json
session.add(state)
else:
new_state = State(key=key, value=value_json)
session.add(new_state)
await session.commit()

166
decnet/web/db/sqlmodel_repo/__init__.py Normal file
View File

@@ -0,0 +1,166 @@
"""
Shared SQLModel-based repository implementation.
Contains all dialect-portable query code used by the SQLite and MySQL
backends. Dialect-specific behavior lives in subclasses:
* engine/session construction (``__init__``)
* ``_migrate_attackers_table`` (legacy schema check; DDL introspection
is not portable)
* ``get_log_histogram`` (date-bucket expression differs per dialect)
"""
from __future__ import annotations
import asyncio
import json
import orjson
import uuid
from typing import Any, Optional, List
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
from decnet.config import load_state
from decnet.env import DECNET_ADMIN_USER, DECNET_ADMIN_PASSWORD
from decnet.web.auth import get_password_hash
from decnet.web.db.repository import BaseRepository
from decnet.web.db.models import State, User
from decnet.web.db.sqlmodel_repo._helpers import ( # noqa: F401 (re-exported for tests/external)
_safe_session,
_detach_close,
_cleanup_tasks,
)
from decnet.web.db.sqlmodel_repo.attacker_intel import AttackerIntelMixin
from decnet.web.db.sqlmodel_repo.attackers import AttackersMixin
from decnet.web.db.sqlmodel_repo.auth import AuthMixin
from decnet.web.db.sqlmodel_repo.bounties import BountiesMixin
from decnet.web.db.sqlmodel_repo.campaigns import CampaignsMixin
from decnet.web.db.sqlmodel_repo.canary import CanaryMixin
from decnet.web.db.sqlmodel_repo.credentials import CredentialsMixin
from decnet.web.db.sqlmodel_repo.deckies import DeckiesMixin
from decnet.web.db.sqlmodel_repo.fleet import FleetMixin
from decnet.web.db.sqlmodel_repo.identities import IdentitiesMixin
from decnet.web.db.sqlmodel_repo.logs import LogsMixin
from decnet.web.db.sqlmodel_repo.orchestrator import OrchestratorMixin
from decnet.web.db.sqlmodel_repo.realism import RealismMixin
from decnet.web.db.sqlmodel_repo.swarm import SwarmMixin
from decnet.web.db.sqlmodel_repo.topology import TopologyMixin
from decnet.web.db.sqlmodel_repo.webhooks import WebhooksMixin
class SQLModelRepository(
AttackerIntelMixin,
AttackersMixin,
AuthMixin,
BountiesMixin,
CampaignsMixin,
CanaryMixin,
CredentialsMixin,
DeckiesMixin,
FleetMixin,
IdentitiesMixin,
LogsMixin,
OrchestratorMixin,
RealismMixin,
SwarmMixin,
TopologyMixin,
WebhooksMixin,
BaseRepository,
):
"""Concrete SQLModel/SQLAlchemy-async repository.
Subclasses provide ``self.engine`` (AsyncEngine) and ``self.session_factory``
in ``__init__``, and override the few dialect-specific helpers.
"""
engine: AsyncEngine
session_factory: async_sessionmaker[AsyncSession]
def _session(self):
"""Return a cancellation-safe session context manager."""
return _safe_session(self.session_factory)
# ------------------------------------------------------------ lifecycle
async def initialize(self) -> None:
"""Create tables if absent and seed the admin user."""
from sqlmodel import SQLModel
await self._migrate_attackers_table()
await self._migrate_session_profile_table()
async with self.engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
await self._ensure_admin_user()
async def reinitialize(self) -> None:
"""Re-create schema (for tests / reset flows). Does NOT drop existing tables."""
from sqlmodel import SQLModel
async with self.engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
await self._ensure_admin_user()
async def _ensure_admin_user(self) -> None:
async with self._session() as session:
result = await session.execute(
select(User).where(User.username == DECNET_ADMIN_USER)
)
existing = result.scalar_one_or_none()
if existing is None:
session.add(User(
uuid=str(uuid.uuid4()),
username=DECNET_ADMIN_USER,
password_hash=get_password_hash(DECNET_ADMIN_PASSWORD),
role="admin",
must_change_password=True,
))
await session.commit()
return
# Self-heal env drift: if admin never finalized their password,
# re-sync the hash from DECNET_ADMIN_PASSWORD. Otherwise leave
# the user's chosen password alone.
if existing.must_change_password:
existing.password_hash = get_password_hash(DECNET_ADMIN_PASSWORD)
session.add(existing)
await session.commit()
async def _migrate_attackers_table(self) -> None:
"""Legacy-schema cleanup. Override per dialect (DDL introspection is non-portable)."""
return None
async def _migrate_session_profile_table(self) -> None:
"""Add DEBT-036 keystroke-dynamics columns to existing session_profile
rows. Override per dialect — DDL introspection is non-portable."""
return None
async def get_deckies(self) -> List[dict]:
_state = await asyncio.to_thread(load_state)
return [_d.model_dump() for _d in _state[0].deckies] if _state else []
# --------------------------------------------------------------- users
async def get_state(self, key: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
statement = select(State).where(State.key == key)
result = await session.execute(statement)
state = result.scalar_one_or_none()
if state:
return json.loads(state.value)
return None
async def set_state(self, key: str, value: Any) -> None: # noqa: ANN401
async with self._session() as session:
statement = select(State).where(State.key == key)
result = await session.execute(statement)
state = result.scalar_one_or_none()
value_json = orjson.dumps(value).decode()
if state:
state.value = value_json
session.add(state)
else:
session.add(State(key=key, value=value_json))
await session.commit()

113
decnet/web/db/sqlmodel_repo/_helpers.py Normal file
View File

@@ -0,0 +1,113 @@
"""Module-level session helpers shared by every repository mixin.
``_safe_session`` and ``_detach_close`` make session cleanup robust under
client-cancellation. See ``_detach_close`` for the full rationale.
``_serialize_json_fields`` / ``_deserialize_json_fields`` live here
because they're used across multiple domain mixins (fleet, topology,
…); putting them in a single mixin would force the others to inherit
that mixin or import a free function — both worse than a shared helper.
"""
from __future__ import annotations
import asyncio
import json
from contextlib import asynccontextmanager
from typing import Any
import orjson
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from decnet.logging import get_logger
_log = get_logger("db.pool")
# Hold strong refs to in-flight cleanup tasks so they aren't GC'd mid-run.
_cleanup_tasks: set[asyncio.Task] = set()
def _detach_close(session: AsyncSession) -> None:
"""Hand session cleanup to a fresh task so the caller's cancellation
doesn't interrupt it.
``asyncio.shield`` doesn't help on the exception path: shield prevents
*other* tasks from cancelling the inner coroutine, but if the *current*
task is already cancelled, its next ``await`` re-raises
``CancelledError`` as soon as the inner coroutine yields. That's what
happens when uvicorn cancels a request mid-query — the rollback inside
``session.close()`` can't complete, and the aiomysql connection is
orphaned (pool logs "non-checked-in connection" on GC).
A fresh task isn't subject to the caller's pending cancellation, so
``close()`` (or the ``invalidate()`` fallback for a dead connection)
runs to completion and the pool reclaims the connection promptly.
Fire-and-forget on purpose: the caller is already unwinding and must
not wait on cleanup.
"""
async def _cleanup() -> None:
try:
await session.close()
except BaseException:
try:
session.sync_session.invalidate()
except BaseException:
_log.debug("detach-close: invalidate failed", exc_info=True)
try:
loop = asyncio.get_running_loop()
except RuntimeError:
# No running loop (shutdown path) — best-effort sync invalidate.
try:
session.sync_session.invalidate()
except BaseException:
_log.debug("detach-close: no-loop invalidate failed", exc_info=True)
return
task = loop.create_task(_cleanup())
_cleanup_tasks.add(task)
# Consume any exception to silence "Task exception was never retrieved".
task.add_done_callback(lambda t: (_cleanup_tasks.discard(t), t.exception()))
@asynccontextmanager
async def _safe_session(factory: async_sessionmaker[AsyncSession]):
"""Session context manager that keeps close() reliable under cancellation.
Success path: await close() inline so the caller observes cleanup
(commit visibility, connection release) before proceeding.
Exception path (includes CancelledError from client disconnects):
detach close() to a fresh task. The caller is unwinding and its
own cancellation would abort an inline close mid-rollback, leaving
the aiomysql connection orphaned.
"""
session = factory()
try:
yield session
except BaseException:
_detach_close(session)
raise
else:
await session.close()
def _serialize_json_fields(data: dict[str, Any], keys: tuple[str, ...]) -> dict[str, Any]:
"""Encode the named keys as JSON strings if they're not already."""
out = dict(data)
for k in keys:
v = out.get(k)
if v is not None and not isinstance(v, str):
out[k] = orjson.dumps(v).decode()
return out
def _deserialize_json_fields(d: dict[str, Any], keys: tuple[str, ...]) -> dict[str, Any]:
"""Decode the named JSON-string keys in place."""
for k in keys:
v = d.get(k)
if isinstance(v, str):
try:
d[k] = json.loads(v)
except (json.JSONDecodeError, TypeError):
pass
return d

102
decnet/web/db/sqlmodel_repo/attacker_intel.py Normal file
View File

@@ -0,0 +1,102 @@
"""Attacker-intel domain methods.
Owns reads/writes for ``AttackerIntel`` rows: per-attacker enrichment
data sourced from external providers (GreyNoise, AbuseIPDB, Feodo,
ThreatFox). Joined against ``Attacker`` for the unenriched-backlog
worker query.
"""
from __future__ import annotations
import json
import uuid as _uuid
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import desc, or_, select
from decnet.web.db.models import Attacker, AttackerIntel
class AttackerIntelMixin:
"""Mixin: methods composed onto ``SQLModelRepository``.
Expects ``self._session()`` from the base.
"""
async def upsert_attacker_intel(self, data: dict[str, Any]) -> str:
attacker_uuid_value = data["attacker_uuid"]
async with self._session() as session:
result = await session.execute(
select(AttackerIntel).where(
AttackerIntel.attacker_uuid == attacker_uuid_value,
)
)
existing = result.scalar_one_or_none()
if existing:
for k, v in data.items():
setattr(existing, k, v)
session.add(existing)
row_uuid = existing.uuid
else:
row_uuid = _uuid.uuid4().hex
session.add(AttackerIntel(uuid=row_uuid, **data))
await session.commit()
return row_uuid
async def get_attacker_intel_by_uuid(
self,
uuid: str,
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(AttackerIntel).where(AttackerIntel.attacker_uuid == uuid)
)
row = result.scalar_one_or_none()
if not row:
return None
d = row.model_dump(mode="json")
for key in (
"greynoise_raw",
"abuseipdb_raw",
"feodo_raw",
"threatfox_raw",
):
raw = d.get(key)
if isinstance(raw, str):
try:
d[key] = json.loads(raw)
except (json.JSONDecodeError, TypeError):
pass
return d
async def get_unenriched_attackers(
self, limit: int = 100,
) -> list[dict[str, Any]]:
"""``{"uuid", "ip"}`` pairs with no intel row OR a stale (expired) one.
Stale = ``expires_at < now``. Ordered by ``attackers.last_seen`` desc
so the worker prioritises recent activity on backfill. Both columns
are projected so the worker can write keyed on UUID and dispatch
provider calls keyed on IP without a second round-trip.
"""
now = datetime.now(timezone.utc)
async with self._session() as session:
stmt = (
select(Attacker.uuid, Attacker.ip)
.outerjoin(
AttackerIntel, AttackerIntel.attacker_uuid == Attacker.uuid,
)
.where(
or_(
AttackerIntel.uuid.is_(None),
AttackerIntel.expires_at < now,
)
)
.order_by(desc(Attacker.last_seen))
.limit(limit)
)
result = await session.execute(stmt)
return [
{"uuid": uuid_, "ip": ip}
for uuid_, ip in result.all()
]

32
decnet/web/db/sqlmodel_repo/attackers/__init__.py Normal file
View File

@@ -0,0 +1,32 @@
"""Attacker repository methods.
The full domain spans ~500 lines of methods across attacker rows,
behavior signals, session profiles, SMTP victim tracking, and
log-derived activity views. Each concern lives in its own submixin;
``AttackersMixin`` composes them.
``_deserialize_attacker`` lives on ``AttackersCoreMixin`` and is reached
from ``IdentitiesMixin.list_observations_for_identity`` via ``self.`` —
Python's MRO resolves it to the core mixin on the composed
``SQLModelRepository`` class.
"""
from __future__ import annotations
from decnet.web.db.sqlmodel_repo.attackers._core import AttackersCoreMixin
from decnet.web.db.sqlmodel_repo.attackers.activity import AttackerActivityMixin
from decnet.web.db.sqlmodel_repo.attackers.behavior import AttackerBehaviorMixin
from decnet.web.db.sqlmodel_repo.attackers.sessions import SessionProfilesMixin
from decnet.web.db.sqlmodel_repo.attackers.smtp import SmtpTargetsMixin
class AttackersMixin(
AttackerActivityMixin,
AttackerBehaviorMixin,
SessionProfilesMixin,
SmtpTargetsMixin,
AttackersCoreMixin,
):
"""Composed attackers mixin — see submixins for the actual methods."""
__all__ = ["AttackersMixin"]

95
decnet/web/db/sqlmodel_repo/attackers/_core.py Normal file
View File

@@ -0,0 +1,95 @@
"""Core ``Attacker`` row CRUD + the ``_deserialize_attacker`` helper.
The helper lives here because sibling submixins and ``IdentitiesMixin``
(``list_observations_for_identity``) both call it through ``self.`` —
MRO resolves them onto this mixin on the composed
``SQLModelRepository``.
"""
from __future__ import annotations
import json
import uuid as _uuid
from typing import Any, List, Optional
from sqlalchemy import desc, func, select
from decnet.web.db.models import Attacker
class AttackersCoreMixin:
@staticmethod
def _deserialize_attacker(d: dict[str, Any]) -> dict[str, Any]:
for key in ("services", "deckies", "fingerprints", "commands"):
if isinstance(d.get(key), str):
try:
d[key] = json.loads(d[key])
except (json.JSONDecodeError, TypeError):
pass
return d
async def upsert_attacker(self, data: dict[str, Any]) -> str:
async with self._session() as session:
result = await session.execute(
select(Attacker).where(Attacker.ip == data["ip"])
)
existing = result.scalar_one_or_none()
if existing:
for k, v in data.items():
setattr(existing, k, v)
session.add(existing)
row_uuid = existing.uuid
else:
row_uuid = str(_uuid.uuid4())
data = {**data, "uuid": row_uuid}
session.add(Attacker(**data))
await session.commit()
return row_uuid
async def get_attacker_by_uuid(self, uuid: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(Attacker).where(Attacker.uuid == uuid)
)
attacker = result.scalar_one_or_none()
if not attacker:
return None
return self._deserialize_attacker(attacker.model_dump(mode="json"))
async def get_attackers(
self,
limit: int = 50,
offset: int = 0,
search: Optional[str] = None,
sort_by: str = "recent",
service: Optional[str] = None,
) -> List[dict[str, Any]]:
order = {
"active": desc(Attacker.event_count),
"traversals": desc(Attacker.is_traversal),
}.get(sort_by, desc(Attacker.last_seen))
statement = select(Attacker).order_by(order).offset(offset).limit(limit)
if search:
statement = statement.where(Attacker.ip.like(f"%{search}%"))
if service:
statement = statement.where(Attacker.services.like(f'%"{service}"%'))
async with self._session() as session:
result = await session.execute(statement)
return [
self._deserialize_attacker(a.model_dump(mode="json"))
for a in result.scalars().all()
]
async def get_total_attackers(
self, search: Optional[str] = None, service: Optional[str] = None
) -> int:
statement = select(func.count()).select_from(Attacker)
if search:
statement = statement.where(Attacker.ip.like(f"%{search}%"))
if service:
statement = statement.where(Attacker.services.like(f'%"{service}"%'))
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0

207
decnet/web/db/sqlmodel_repo/attackers/activity.py Normal file
View File

@@ -0,0 +1,207 @@
"""Log-derived activity views: commands, service activity, IP leaks,
artifacts, stored mail, recorded sessions, transcripts.
These read from the ``logs`` and ``bounty`` tables joined against the
``Attacker`` row to scope by IP — no separate activity table.
"""
from __future__ import annotations
import json
from typing import Any, Optional
from sqlalchemy import desc, func, select
from decnet.web.db.models import Attacker, Bounty, Log
class AttackerActivityMixin:
async def get_attacker_commands(
self,
uuid: str,
limit: int = 50,
offset: int = 0,
service: Optional[str] = None,
) -> dict[str, Any]:
async with self._session() as session:
result = await session.execute(
select(Attacker.commands).where(Attacker.uuid == uuid)
)
raw = result.scalar_one_or_none()
if raw is None:
return {"total": 0, "data": []}
commands: list = json.loads(raw) if isinstance(raw, str) else raw
if service:
commands = [c for c in commands if c.get("service") == service]
total = len(commands)
page = commands[offset: offset + limit]
return {"total": total, "data": page}
async def get_attacker_service_activity(
self, attacker_uuid: str
) -> list[tuple[str, str]]:
"""Return distinct ``(service, event_type)`` pairs for an attacker.
Resolves IP then ``SELECT DISTINCT service, event_type FROM logs
WHERE attacker_ip = :ip`` — the result set is bounded by the
cardinality of services × event_types (tens, not thousands), so
this stays cheap even for attackers with long event streams.
Caller applies `event_kinds.bucket_services` to split into
scanned vs. interacted.
"""
async with self._session() as session:
ip_res = await session.execute(
select(Attacker.ip).where(Attacker.uuid == attacker_uuid)
)
ip = ip_res.scalar_one_or_none()
if not ip:
return []
rows = await session.execute(
select(Log.service, Log.event_type)
.where(Log.attacker_ip == ip)
.distinct()
)
return [(svc, evt) for svc, evt in rows.all()]
async def get_attacker_ip_leaks(
self, attacker_uuid: str, *, limit: int = 10,
) -> list[dict[str, Any]]:
"""Return ``bounty_type='ip_leak'`` rows for this attacker, newest
first, capped at ``limit``. Shape matches the XFF-mismatch
payload emitted by the ingester: keys include ``real_ip_claim``,
``source_header``, ``headers_seen``. Use
:meth:`count_attacker_ip_leaks` to get the unbounded total for
rotation detection."""
async with self._session() as session:
ip_res = await session.execute(
select(Attacker.ip).where(Attacker.uuid == attacker_uuid)
)
ip = ip_res.scalar_one_or_none()
if not ip:
return []
rows = await session.execute(
select(Bounty)
.where(Bounty.attacker_ip == ip)
.where(Bounty.bounty_type == "ip_leak")
.order_by(desc(Bounty.timestamp))
.limit(limit)
)
out: list[dict[str, Any]] = []
for row in rows.scalars().all():
rec = row.model_dump(mode="json")
# Bounty.payload is stored JSON-encoded; pre-decode for UX.
raw = rec.get("payload")
if isinstance(raw, str):
try:
rec["payload"] = json.loads(raw)
except (ValueError, TypeError):
rec["payload"] = {}
out.append(rec)
return out
async def count_attacker_ip_leaks(self, attacker_uuid: str) -> int:
"""Cheap COUNT(*) for XFF-rotation detection."""
async with self._session() as session:
ip_res = await session.execute(
select(Attacker.ip).where(Attacker.uuid == attacker_uuid)
)
ip = ip_res.scalar_one_or_none()
if not ip:
return 0
count_res = await session.execute(
select(func.count(Bounty.id))
.where(Bounty.attacker_ip == ip)
.where(Bounty.bounty_type == "ip_leak")
)
return int(count_res.scalar() or 0)
async def get_attacker_artifacts(self, uuid: str) -> list[dict[str, Any]]:
"""Return `file_captured` logs for the attacker identified by UUID.
Resolves the attacker's IP first, then queries the logs table on two
indexed columns (``attacker_ip`` and ``event_type``). No JSON extract
needed — the decky/stored_as are already decoded into ``fields`` by
the ingester and returned to the frontend for drawer rendering.
"""
async with self._session() as session:
ip_res = await session.execute(
select(Attacker.ip).where(Attacker.uuid == uuid)
)
ip = ip_res.scalar_one_or_none()
if not ip:
return []
rows = await session.execute(
select(Log)
.where(Log.attacker_ip == ip)
.where(Log.event_type == "file_captured")
.order_by(desc(Log.timestamp))
.limit(200)
)
return [r.model_dump(mode="json") for r in rows.scalars().all()]
async def get_attacker_stored_mail(self, uuid: str) -> list[dict[str, Any]]:
"""Return `message_stored` logs for an attacker, newest first.
Mirrors :meth:`get_attacker_artifacts` — the SMTP template emits one
`message_stored` row per accepted DATA body, with headers + sha256 +
attachment manifest already decoded into ``fields`` by the ingester.
Capped at 200 rows to match the artifact/transcript query shape.
"""
async with self._session() as session:
ip_res = await session.execute(
select(Attacker.ip).where(Attacker.uuid == uuid)
)
ip = ip_res.scalar_one_or_none()
if not ip:
return []
rows = await session.execute(
select(Log)
.where(Log.attacker_ip == ip)
.where(Log.event_type == "message_stored")
.order_by(desc(Log.timestamp))
.limit(200)
)
return [r.model_dump(mode="json") for r in rows.scalars().all()]
async def get_session_log(self, sid: str) -> Optional[dict[str, Any]]:
"""Look up the `session_recorded` Log row that owns a given sid.
sid is a v4 UUID embedded in the row's ``fields`` JSON blob. Matched
with LIKE on the textual sid substring — cheap given the bounded
cardinality of session_recorded rows vs. the full logs table.
"""
needle = f'"sid":"{sid}"'
async with self._session() as session:
rows = await session.execute(
select(Log)
.where(Log.event_type == "session_recorded")
.where(Log.fields.contains(needle))
.limit(1)
)
row = rows.scalars().first()
return row.model_dump(mode="json") if row else None
async def get_attacker_transcripts(self, uuid: str) -> list[dict[str, Any]]:
"""Return `session_recorded` logs for the attacker identified by UUID.
Mirror of :meth:`get_attacker_artifacts` — sessions ride in the same
Log table with event_type=session_recorded; the ingester decodes the
RFC 5424 SD fields (sid, service, decky, src_ip, duration_s, bytes,
truncated, shard_path) into the returned ``fields`` blob.
"""
async with self._session() as session:
ip_res = await session.execute(
select(Attacker.ip).where(Attacker.uuid == uuid)
)
ip = ip_res.scalar_one_or_none()
if not ip:
return []
rows = await session.execute(
select(Log)
.where(Log.attacker_ip == ip)
.where(Log.event_type == "session_recorded")
.order_by(desc(Log.timestamp))
.limit(200)
)
return [r.model_dump(mode="json") for r in rows.scalars().all()]

106
decnet/web/db/sqlmodel_repo/attackers/behavior.py Normal file
View File

@@ -0,0 +1,106 @@
"""Per-attacker behavior signals (TCP fingerprint, timing stats, phase
sequence, tool guesses, KEX order, SSH client banners)."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import select
from decnet.web.db.models import Attacker, AttackerBehavior
class AttackerBehaviorMixin:
async def upsert_attacker_behavior(
self,
attacker_uuid: str,
data: dict[str, Any],
) -> None:
async with self._session() as session:
result = await session.execute(
select(AttackerBehavior).where(
AttackerBehavior.attacker_uuid == attacker_uuid
)
)
existing = result.scalar_one_or_none()
payload = {**data, "updated_at": datetime.now(timezone.utc)}
if existing:
for k, v in payload.items():
setattr(existing, k, v)
session.add(existing)
else:
session.add(AttackerBehavior(attacker_uuid=attacker_uuid, **payload))
await session.commit()
async def get_attacker_behavior(
self,
attacker_uuid: str,
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(AttackerBehavior).where(
AttackerBehavior.attacker_uuid == attacker_uuid
)
)
row = result.scalar_one_or_none()
if not row:
return None
return self._deserialize_behavior(row.model_dump(mode="json"))
async def get_behaviors_for_ips(
self,
ips: set[str],
) -> dict[str, dict[str, Any]]:
if not ips:
return {}
async with self._session() as session:
result = await session.execute(
select(Attacker.ip, AttackerBehavior)
.join(AttackerBehavior, Attacker.uuid == AttackerBehavior.attacker_uuid)
.where(Attacker.ip.in_(ips))
)
out: dict[str, dict[str, Any]] = {}
for ip, row in result.all():
out[ip] = self._deserialize_behavior(row.model_dump(mode="json"))
return out
@staticmethod
def _deserialize_behavior(d: dict[str, Any]) -> dict[str, Any]:
for key in ("tcp_fingerprint", "timing_stats", "phase_sequence"):
if isinstance(d.get(key), str):
try:
d[key] = json.loads(d[key])
except (json.JSONDecodeError, TypeError):
pass
# Deserialize tool_guesses JSON array; normalise None → [].
raw = d.get("tool_guesses")
if isinstance(raw, str):
try:
parsed = json.loads(raw)
d["tool_guesses"] = parsed if isinstance(parsed, list) else [parsed]
except (json.JSONDecodeError, TypeError):
d["tool_guesses"] = []
elif raw is None:
d["tool_guesses"] = []
# Same list-or-None pattern for kex_order_raw.
raw_kex = d.get("kex_order_raw")
if isinstance(raw_kex, str):
try:
parsed_kex = json.loads(raw_kex)
d["kex_order_raw"] = parsed_kex if isinstance(parsed_kex, list) else [parsed_kex]
except (json.JSONDecodeError, TypeError):
d["kex_order_raw"] = []
elif raw_kex is None:
d["kex_order_raw"] = []
# Same list-or-None pattern for ssh_client_banners.
raw_banners = d.get("ssh_client_banners")
if isinstance(raw_banners, str):
try:
parsed_banners = json.loads(raw_banners)
d["ssh_client_banners"] = parsed_banners if isinstance(parsed_banners, list) else [parsed_banners]
except (json.JSONDecodeError, TypeError):
d["ssh_client_banners"] = []
elif raw_banners is None:
d["ssh_client_banners"] = []
return d

49
decnet/web/db/sqlmodel_repo/attackers/sessions.py Normal file
View File

@@ -0,0 +1,49 @@
"""Per-session profile rows (keystroke-dynamics features land here at
ingestion-time post-V2)."""
from __future__ import annotations
from typing import Any, Optional
from sqlalchemy import select
from decnet.web.db.models import SessionProfile
class SessionProfilesMixin:
async def upsert_session_profile(
self,
sid: str,
data: dict[str, Any],
) -> None:
"""
Write (or update) the session_profile row for *sid*.
Pre-v1, the typical call is the empty-write path at session close:
`upsert_session_profile(sid, {"log_id": <id>})` — all keystroke
feature columns stay NULL until the V2 ingestion job populates them.
"""
async with self._session() as session:
result = await session.execute(
select(SessionProfile).where(SessionProfile.sid == sid)
)
existing = result.scalar_one_or_none()
if existing:
for k, v in data.items():
setattr(existing, k, v)
session.add(existing)
else:
session.add(SessionProfile(sid=sid, **data))
await session.commit()
async def get_session_profile(
self,
sid: str,
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(SessionProfile).where(SessionProfile.sid == sid)
)
row = result.scalar_one_or_none()
if not row:
return None
return row.model_dump(mode="json")

69
decnet/web/db/sqlmodel_repo/attackers/smtp.py Normal file
View File

@@ -0,0 +1,69 @@
"""SMTP victim-domain tracking (per-attacker counters and
cross-attacker aggregate)."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from sqlalchemy import desc, func, select
from decnet.web.db.models import SmtpTarget
class SmtpTargetsMixin:
async def increment_smtp_target(self, attacker_uuid: str, domain: str) -> None:
"""Upsert an (attacker_uuid, domain) pair and bump count + last_seen.
Read-then-write under a single session — the UNIQUE constraint on
(attacker_uuid, domain) guards against duplicate rows if the race
ever materialises; we accept the ~1ms extra round-trip in exchange
for a single dialect-portable implementation.
"""
async with self._session() as session:
result = await session.execute(
select(SmtpTarget)
.where(SmtpTarget.attacker_uuid == attacker_uuid)
.where(SmtpTarget.domain == domain)
)
existing = result.scalar_one_or_none()
now = datetime.now(timezone.utc)
if existing:
existing.count += 1
existing.last_seen = now
session.add(existing)
else:
session.add(SmtpTarget(
attacker_uuid=attacker_uuid,
domain=domain,
first_seen=now,
last_seen=now,
count=1,
))
await session.commit()
async def list_smtp_targets(self, attacker_uuid: str) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(SmtpTarget)
.where(SmtpTarget.attacker_uuid == attacker_uuid)
.order_by(desc(SmtpTarget.last_seen))
)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def smtp_target_seen(self, domain: str) -> dict[str, Any]:
"""Aggregate rows for this domain across every attacker in the DB."""
async with self._session() as session:
result = await session.execute(
select(
func.coalesce(func.sum(SmtpTarget.count), 0),
func.min(SmtpTarget.first_seen),
func.max(SmtpTarget.last_seen),
).where(SmtpTarget.domain == domain)
)
total, first_seen, last_seen = result.one()
return {
"seen": int(total) > 0,
"count": int(total),
"first_seen": first_seen,
"last_seen": last_seen,
}

74
decnet/web/db/sqlmodel_repo/auth.py Normal file
View File

@@ -0,0 +1,74 @@
"""User CRUD."""
from __future__ import annotations
from typing import Any, Optional
from sqlalchemy import select, update
from decnet.web.db.models import User
class AuthMixin:
"""Mixin: composed onto ``SQLModelRepository``. Expects ``self._session()``.
``_ensure_admin_user`` stays in the package ``__init__`` so the
``DECNET_ADMIN_PASSWORD`` it reads remains addressable at the
``decnet.web.db.sqlmodel_repo`` module path (test monkeypatch surface).
"""
async def get_user_by_username(self, username: str) -> Optional[dict]:
async with self._session() as session:
result = await session.execute(
select(User).where(User.username == username)
)
user = result.scalar_one_or_none()
return user.model_dump() if user else None
async def get_user_by_uuid(self, uuid: str) -> Optional[dict]:
async with self._session() as session:
result = await session.execute(
select(User).where(User.uuid == uuid)
)
user = result.scalar_one_or_none()
return user.model_dump() if user else None
async def create_user(self, user_data: dict[str, Any]) -> None:
async with self._session() as session:
session.add(User(**user_data))
await session.commit()
async def update_user_password(
self, uuid: str, password_hash: str, must_change_password: bool = False
) -> None:
async with self._session() as session:
await session.execute(
update(User)
.where(User.uuid == uuid)
.values(
password_hash=password_hash,
must_change_password=must_change_password,
)
)
await session.commit()
async def list_users(self) -> list[dict]:
async with self._session() as session:
result = await session.execute(select(User))
return [u.model_dump() for u in result.scalars().all()]
async def delete_user(self, uuid: str) -> bool:
async with self._session() as session:
result = await session.execute(select(User).where(User.uuid == uuid))
user = result.scalar_one_or_none()
if not user:
return False
await session.delete(user)
await session.commit()
return True
async def update_user_role(self, uuid: str, role: str) -> None:
async with self._session() as session:
await session.execute(
update(User).where(User.uuid == uuid).values(role=role)
)
await session.commit()

139
decnet/web/db/sqlmodel_repo/bounties.py Normal file
View File

@@ -0,0 +1,139 @@
"""Bounty CRUD + the global purge helper that wipes logs/bounties/credentials/attackers together."""
from __future__ import annotations
import json
from collections import defaultdict
from typing import Any, List, Optional
import orjson
from sqlalchemy import asc, desc, func, or_, select, text
from sqlmodel.sql.expression import SelectOfScalar
from decnet.web.db.models import Bounty
class BountiesMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
async def purge_logs_and_bounties(self) -> dict[str, int]:
async with self._session() as session:
logs_deleted = (await session.execute(text("DELETE FROM logs"))).rowcount
bounties_deleted = (await session.execute(text("DELETE FROM bounty"))).rowcount
credentials_deleted = (
await session.execute(text("DELETE FROM credentials"))
).rowcount
# attacker_behavior has FK → attackers.uuid; delete children first.
await session.execute(text("DELETE FROM attacker_behavior"))
attackers_deleted = (await session.execute(text("DELETE FROM attackers"))).rowcount
await session.commit()
return {
"logs": logs_deleted,
"bounties": bounties_deleted,
"credentials": credentials_deleted,
"attackers": attackers_deleted,
}
async def add_bounty(self, bounty_data: dict[str, Any]) -> None:
data = bounty_data.copy()
if "payload" in data and isinstance(data["payload"], dict):
data["payload"] = orjson.dumps(data["payload"]).decode()
async with self._session() as session:
dup = await session.execute(
select(Bounty.id).where(
Bounty.bounty_type == data.get("bounty_type"),
Bounty.attacker_ip == data.get("attacker_ip"),
Bounty.payload == data.get("payload"),
).limit(1)
)
if dup.first() is not None:
return
session.add(Bounty(**data))
await session.commit()
def _apply_bounty_filters(
self,
statement: SelectOfScalar,
bounty_type: Optional[str],
search: Optional[str],
) -> SelectOfScalar:
if bounty_type:
statement = statement.where(Bounty.bounty_type == bounty_type)
if search:
lk = f"%{search}%"
statement = statement.where(
or_(
Bounty.decky.like(lk),
Bounty.service.like(lk),
Bounty.attacker_ip.like(lk),
Bounty.payload.like(lk),
)
)
return statement
async def get_bounties(
self,
limit: int = 50,
offset: int = 0,
bounty_type: Optional[str] = None,
search: Optional[str] = None,
) -> List[dict]:
statement = (
select(Bounty)
.order_by(desc(Bounty.timestamp))
.offset(offset)
.limit(limit)
)
statement = self._apply_bounty_filters(statement, bounty_type, search)
async with self._session() as session:
results = await session.execute(statement)
final = []
for item in results.scalars().all():
d = item.model_dump(mode="json")
try:
d["payload"] = json.loads(d["payload"])
except (json.JSONDecodeError, TypeError):
pass
final.append(d)
return final
async def get_total_bounties(
self, bounty_type: Optional[str] = None, search: Optional[str] = None
) -> int:
statement = select(func.count()).select_from(Bounty)
statement = self._apply_bounty_filters(statement, bounty_type, search)
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def get_all_bounties_by_ip(self) -> dict[str, List[dict[str, Any]]]:
async with self._session() as session:
result = await session.execute(
select(Bounty).order_by(asc(Bounty.timestamp))
)
grouped: dict[str, List[dict[str, Any]]] = defaultdict(list)
for item in result.scalars().all():
d = item.model_dump(mode="json")
try:
d["payload"] = json.loads(d["payload"])
except (json.JSONDecodeError, TypeError):
pass
grouped[item.attacker_ip].append(d)
return dict(grouped)
async def get_bounties_for_ips(self, ips: set[str]) -> dict[str, List[dict[str, Any]]]:
async with self._session() as session:
result = await session.execute(
select(Bounty).where(Bounty.attacker_ip.in_(ips)).order_by(asc(Bounty.timestamp))
)
grouped: dict[str, List[dict[str, Any]]] = defaultdict(list)
for item in result.scalars().all():
d = item.model_dump(mode="json")
try:
d["payload"] = json.loads(d["payload"])
except (json.JSONDecodeError, TypeError):
pass
grouped[item.attacker_ip].append(d)
return dict(grouped)

173
decnet/web/db/sqlmodel_repo/campaigns.py Normal file
View File

@@ -0,0 +1,173 @@
"""Campaign reads + writes.
Campaign = the second-tier clustering output that groups multiple
``AttackerIdentity`` rows into a coordinated activity cluster. The
campaign-clusterer worker drives the writes; the dashboard drives
the reads.
"""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import desc, func, select, update
from decnet.web.db.models import AttackerIdentity, Campaign
class CampaignsMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
async def get_campaign_by_uuid(self, uuid: str) -> Optional[dict[str, Any]]:
# Same chain-walk as get_identity_by_uuid; bounded against
# corrupted rings.
_MAX_MERGE_HOPS = 8
async with self._session() as session:
current_uuid = uuid
for _ in range(_MAX_MERGE_HOPS):
result = await session.execute(
select(Campaign).where(Campaign.uuid == current_uuid)
)
campaign = result.scalar_one_or_none()
if campaign is None:
return None
if campaign.merged_into_uuid is None:
return campaign.model_dump(mode="json")
current_uuid = campaign.merged_into_uuid
return campaign.model_dump(mode="json")
async def list_campaigns(
self, limit: int = 50, offset: int = 0,
) -> list[dict[str, Any]]:
statement = (
select(Campaign)
.where(Campaign.merged_into_uuid.is_(None))
.order_by(desc(Campaign.updated_at))
.offset(offset)
.limit(limit)
)
async with self._session() as session:
result = await session.execute(statement)
return [c.model_dump(mode="json") for c in result.scalars().all()]
async def count_campaigns(self) -> int:
statement = (
select(func.count())
.select_from(Campaign)
.where(Campaign.merged_into_uuid.is_(None))
)
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def list_identities_for_campaign(
self, campaign_uuid: str, limit: int = 50, offset: int = 0,
) -> list[dict[str, Any]]:
statement = (
select(AttackerIdentity)
.where(AttackerIdentity.campaign_id == campaign_uuid)
.order_by(desc(AttackerIdentity.updated_at))
.offset(offset)
.limit(limit)
)
async with self._session() as session:
result = await session.execute(statement)
return [i.model_dump(mode="json") for i in result.scalars().all()]
async def count_identities_for_campaign(self, campaign_uuid: str) -> int:
statement = (
select(func.count())
.select_from(AttackerIdentity)
.where(AttackerIdentity.campaign_id == campaign_uuid)
)
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def list_identities_for_clustering(
self, limit: Optional[int] = None,
) -> list[dict[str, Any]]:
# Project the columns the campaign clusterer's similarity
# graph reads. Narrow on purpose — future denormalised
# projections (commands_by_phase from log mining, decky-set
# aggregates) can land here without churning callers.
statement = select(
AttackerIdentity.uuid,
AttackerIdentity.campaign_id,
AttackerIdentity.merged_into_uuid,
AttackerIdentity.first_seen_at,
AttackerIdentity.last_seen_at,
AttackerIdentity.ja3_hashes,
AttackerIdentity.hassh_hashes,
AttackerIdentity.payload_simhashes,
AttackerIdentity.c2_endpoints,
).order_by(AttackerIdentity.created_at)
if limit is not None:
statement = statement.limit(limit)
async with self._session() as session:
result = await session.execute(statement)
return [
{
"uuid": row.uuid,
"campaign_id": row.campaign_id,
"merged_into_uuid": row.merged_into_uuid,
"first_seen_at": (
row.first_seen_at.isoformat()
if row.first_seen_at is not None
else None
),
"last_seen_at": (
row.last_seen_at.isoformat()
if row.last_seen_at is not None
else None
),
"ja3_hashes": row.ja3_hashes,
"hassh_hashes": row.hassh_hashes,
"payload_simhashes": row.payload_simhashes,
"c2_endpoints": row.c2_endpoints,
}
for row in result.all()
]
async def create_campaign(self, row: dict[str, Any]) -> str:
campaign = Campaign(**row)
async with self._session() as session:
session.add(campaign)
await session.commit()
return campaign.uuid
async def set_identity_campaign_id(
self, identity_uuid: str, campaign_uuid: Optional[str],
) -> None:
statement = (
update(AttackerIdentity)
.where(AttackerIdentity.uuid == identity_uuid)
.values(
campaign_id=campaign_uuid,
updated_at=datetime.now(timezone.utc),
)
)
async with self._session() as session:
await session.execute(statement)
await session.commit()
async def list_all_campaigns(self) -> list[dict[str, Any]]:
statement = select(Campaign).order_by(Campaign.created_at)
async with self._session() as session:
result = await session.execute(statement)
return [c.model_dump(mode="json") for c in result.scalars().all()]
async def update_campaign_merged_into(
self, campaign_uuid: str, winner_uuid: Optional[str],
) -> None:
statement = (
update(Campaign)
.where(Campaign.uuid == campaign_uuid)
.values(
merged_into_uuid=winner_uuid,
updated_at=datetime.now(timezone.utc),
)
)
async with self._session() as session:
await session.execute(statement)
await session.commit()

200
decnet/web/db/sqlmodel_repo/canary.py Normal file
View File

@@ -0,0 +1,200 @@
"""Canary blob/token CRUD + trigger ingestion."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import desc, func, select, update
from decnet.web.db.models import CanaryBlob, CanaryToken, CanaryTrigger
class CanaryMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
async def upsert_canary_blob(self, data: dict[str, Any]) -> dict[str, Any]:
sha = data.get("sha256")
if not sha:
raise ValueError("upsert_canary_blob: sha256 is required")
async with self._session() as session:
existing = await session.execute(
select(CanaryBlob).where(CanaryBlob.sha256 == sha)
)
row = existing.scalar_one_or_none()
if row:
return row.model_dump(mode="json")
row = CanaryBlob(**data)
session.add(row)
await session.commit()
await session.refresh(row)
return row.model_dump(mode="json")
async def get_canary_blob(self, uuid: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(CanaryBlob).where(CanaryBlob.uuid == uuid)
)
row = result.scalar_one_or_none()
return row.model_dump(mode="json") if row else None
async def get_canary_blob_by_sha256(
self, sha256: str
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(CanaryBlob).where(CanaryBlob.sha256 == sha256)
)
row = result.scalar_one_or_none()
return row.model_dump(mode="json") if row else None
async def list_canary_blobs(self) -> list[dict[str, Any]]:
# One round-trip: outer-join blobs -> tokens, group by blob, count
# live (non-revoked) references. Revoked tokens still occupy the
# blob conceptually until garbage-collected, so we count them too;
# the operator deletes blobs explicitly via the API.
async with self._session() as session:
stmt = (
select(CanaryBlob, func.count(CanaryToken.uuid))
.join(
CanaryToken,
CanaryToken.blob_uuid == CanaryBlob.uuid,
isouter=True,
)
.group_by(CanaryBlob.uuid)
.order_by(desc(CanaryBlob.uploaded_at))
)
result = await session.execute(stmt)
out: list[dict[str, Any]] = []
for blob, count in result.all():
d = blob.model_dump(mode="json")
d["token_count"] = int(count or 0)
out.append(d)
return out
async def delete_canary_blob(self, uuid: str) -> bool:
async with self._session() as session:
ref = await session.execute(
select(func.count(CanaryToken.uuid)).where(
CanaryToken.blob_uuid == uuid
)
)
if (ref.scalar_one() or 0) > 0:
return False
result = await session.execute(
select(CanaryBlob).where(CanaryBlob.uuid == uuid)
)
row = result.scalar_one_or_none()
if not row:
return False
await session.delete(row)
await session.commit()
return True
async def create_canary_token(self, data: dict[str, Any]) -> None:
async with self._session() as session:
session.add(CanaryToken(**data))
await session.commit()
async def get_canary_token(self, uuid: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(CanaryToken).where(CanaryToken.uuid == uuid)
)
row = result.scalar_one_or_none()
return row.model_dump(mode="json") if row else None
async def get_canary_token_by_slug(
self, callback_token: str
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(CanaryToken).where(
CanaryToken.callback_token == callback_token
)
)
row = result.scalar_one_or_none()
return row.model_dump(mode="json") if row else None
async def list_canary_tokens(
self,
*,
decky_name: Optional[str] = None,
state: Optional[str] = None,
kind: Optional[str] = None,
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = select(CanaryToken)
if decky_name is not None:
stmt = stmt.where(CanaryToken.decky_name == decky_name)
if state is not None:
stmt = stmt.where(CanaryToken.state == state)
if kind is not None:
stmt = stmt.where(CanaryToken.kind == kind)
stmt = stmt.order_by(desc(CanaryToken.placed_at))
result = await session.execute(stmt)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def update_canary_token_state(
self,
uuid: str,
state: str,
last_error: Optional[str] = None,
) -> bool:
async with self._session() as session:
result = await session.execute(
update(CanaryToken)
.where(CanaryToken.uuid == uuid)
.values(state=state, last_error=last_error)
)
await session.commit()
return result.rowcount > 0
async def record_canary_trigger(self, data: dict[str, Any]) -> str:
# Persist the trigger row + bump the token's counters in the same
# session so a subscriber that reads the token row right after
# receiving the bus event sees the updated count.
headers = data.get("raw_headers")
if isinstance(headers, dict):
data = {**data, "raw_headers": json.dumps(headers)}
async with self._session() as session:
row = CanaryTrigger(**data)
session.add(row)
ts = data.get("occurred_at") or datetime.now(timezone.utc)
await session.execute(
update(CanaryToken)
.where(CanaryToken.uuid == row.token_uuid)
.values(
last_triggered_at=ts,
trigger_count=CanaryToken.trigger_count + 1,
)
)
await session.commit()
await session.refresh(row)
return row.uuid
async def list_canary_triggers(
self, token_uuid: str, *, limit: int = 100, offset: int = 0,
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = (
select(CanaryTrigger)
.where(CanaryTrigger.token_uuid == token_uuid)
.order_by(desc(CanaryTrigger.occurred_at))
.limit(limit)
.offset(offset)
)
result = await session.execute(stmt)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def attribute_canary_trigger(
self, trigger_uuid: str, attacker_id: str,
) -> bool:
async with self._session() as session:
result = await session.execute(
update(CanaryTrigger)
.where(CanaryTrigger.uuid == trigger_uuid)
.values(attacker_id=attacker_id)
)
await session.commit()
return result.rowcount > 0

20
decnet/web/db/sqlmodel_repo/credentials/__init__.py Normal file
View File

@@ -0,0 +1,20 @@
"""Credential capture + credential-reuse correlation.
Capture (per-attempt rows) lives in ``_core.py``; the reuse correlator
(grouping rows that share a secret triple) lives in ``reuse.py``.
``CredentialsMixin`` composes the two.
"""
from __future__ import annotations
from decnet.web.db.sqlmodel_repo.credentials._core import CredentialsCoreMixin
from decnet.web.db.sqlmodel_repo.credentials.reuse import CredentialReuseMixin
class CredentialsMixin(
CredentialReuseMixin,
CredentialsCoreMixin,
):
"""Composed credentials mixin — see submixins for the actual methods."""
__all__ = ["CredentialsMixin"]

196
decnet/web/db/sqlmodel_repo/credentials/_core.py Normal file
View File

@@ -0,0 +1,196 @@
"""Credential capture: per-attempt rows in the ``Credential`` table."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, List, Optional
from sqlalchemy import desc, func, or_, select, update
from sqlmodel.sql.expression import SelectOfScalar
from decnet.web.db.models import Credential
class CredentialsCoreMixin:
async def upsert_credential(self, data: dict[str, Any]) -> int:
"""Upsert a credential attempt; returns the row id.
Dedup tuple: (attacker_ip, decky_name, service, secret_sha256,
principal_or_None). On match, ``attempt_count`` += 1 and
``last_seen`` advances; ``first_seen`` and ``fields`` are
preserved from the original sighting.
"""
payload = dict(data)
if "fields" in payload and isinstance(payload["fields"], dict):
# ensure_ascii=True keeps utf8mb4 columns safe even when
# service-specific keys carry non-ASCII bytes.
payload["fields"] = json.dumps(payload["fields"], ensure_ascii=True)
principal = payload.get("principal")
secret_kind = payload.get("secret_kind") or "plaintext"
async with self._session() as session:
stmt = select(Credential).where(
Credential.attacker_ip == payload["attacker_ip"],
Credential.decky_name == payload["decky_name"],
Credential.service == payload["service"],
Credential.secret_kind == secret_kind,
Credential.secret_sha256 == payload["secret_sha256"],
# NULL == NULL is False under SQL — branch the predicate.
(Credential.principal == principal) if principal is not None
else Credential.principal.is_(None),
)
existing = (await session.execute(stmt)).scalar_one_or_none()
now = datetime.now(timezone.utc)
if existing is not None:
existing.attempt_count = (existing.attempt_count or 1) + 1
existing.last_seen = now
if payload.get("outcome") is not None:
existing.outcome = payload["outcome"]
session.add(existing)
await session.commit()
return existing.id # type: ignore[return-value]
row = Credential(
attacker_ip=payload["attacker_ip"],
decky_name=payload["decky_name"],
service=payload["service"],
principal=principal,
secret_kind=secret_kind,
secret_sha256=payload["secret_sha256"],
secret_b64=payload.get("secret_b64"),
secret_printable=payload.get("secret_printable"),
outcome=payload.get("outcome"),
fields=payload.get("fields", "{}"),
first_seen=now,
last_seen=now,
attempt_count=1,
)
session.add(row)
await session.commit()
await session.refresh(row)
return row.id # type: ignore[return-value]
def _apply_credential_filters(
self,
statement: SelectOfScalar,
search: Optional[str],
service: Optional[str],
attacker_ip: Optional[str],
) -> SelectOfScalar:
if service:
statement = statement.where(Credential.service == service)
if attacker_ip:
statement = statement.where(Credential.attacker_ip == attacker_ip)
if search:
lk = f"%{search}%"
statement = statement.where(
or_(
Credential.decky_name.like(lk),
Credential.service.like(lk),
Credential.principal.like(lk),
Credential.secret_printable.like(lk),
)
)
return statement
async def get_credentials(
self,
limit: int = 50,
offset: int = 0,
search: Optional[str] = None,
service: Optional[str] = None,
attacker_ip: Optional[str] = None,
) -> List[dict[str, Any]]:
statement = (
select(Credential)
.order_by(desc(Credential.last_seen))
.offset(offset)
.limit(limit)
)
statement = self._apply_credential_filters(
statement, search, service, attacker_ip
)
async with self._session() as session:
result = await session.execute(statement)
out: List[dict[str, Any]] = []
for item in result.scalars().all():
d = item.model_dump(mode="json")
try:
d["fields"] = json.loads(d["fields"])
except (json.JSONDecodeError, TypeError):
pass
out.append(d)
return out
async def get_total_credentials(
self,
search: Optional[str] = None,
service: Optional[str] = None,
attacker_ip: Optional[str] = None,
) -> int:
statement = select(func.count()).select_from(Credential)
statement = self._apply_credential_filters(
statement, search, service, attacker_ip
)
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def get_credentials_for_attacker(
self, attacker_ip: str
) -> List[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(Credential)
.where(Credential.attacker_ip == attacker_ip)
.order_by(desc(Credential.last_seen))
)
out: List[dict[str, Any]] = []
for item in result.scalars().all():
d = item.model_dump(mode="json")
try:
d["fields"] = json.loads(d["fields"])
except (json.JSONDecodeError, TypeError):
pass
out.append(d)
return out
async def get_credential_attempts_for_secret(
self, secret_sha256: str
) -> List[dict[str, Any]]:
"""Every (attacker_ip, decky, service, principal) row sharing this
secret hash. Indexed lookup via ix_credentials_secret_service.
"""
async with self._session() as session:
result = await session.execute(
select(Credential)
.where(Credential.secret_sha256 == secret_sha256)
.order_by(desc(Credential.last_seen))
)
out: List[dict[str, Any]] = []
for item in result.scalars().all():
d = item.model_dump(mode="json")
try:
d["fields"] = json.loads(d["fields"])
except (json.JSONDecodeError, TypeError):
pass
out.append(d)
return out
async def update_credential_attacker_uuid(
self, attacker_ip: str, attacker_uuid: str
) -> int:
"""Backfill ``attacker_uuid`` on every Credential row matching the
given IP whose ``attacker_uuid`` is currently null. Run by the
profiler after it mints/updates an Attacker row.
"""
async with self._session() as session:
result = await session.execute(
update(Credential)
.where(
Credential.attacker_ip == attacker_ip,
Credential.attacker_uuid.is_(None),
)
.values(attacker_uuid=attacker_uuid)
)
await session.commit()
return int(result.rowcount or 0)

275
decnet/web/db/sqlmodel_repo/credentials/reuse.py Normal file
View File

@@ -0,0 +1,275 @@
"""Credential-reuse correlation: ``CredentialReuse`` finding rows that
group ``Credential`` attempts sharing the same (secret_sha256,
secret_kind, principal) triple."""
from __future__ import annotations
import json
import uuid as _uuid
from datetime import datetime, timezone
from typing import Any, List, Optional
from sqlalchemy import desc, func, select
from decnet.web.db.models import Credential, CredentialReuse
class CredentialReuseMixin:
@staticmethod
def _merge_unique(existing_json: str, value: Optional[str]) -> tuple[str, bool]:
"""Append ``value`` to a JSON list[str] column if not present.
Returns (new_json, changed). None values and duplicates are skipped.
"""
if value is None:
return existing_json, False
try:
current = json.loads(existing_json) if existing_json else []
if not isinstance(current, list):
current = []
except (json.JSONDecodeError, TypeError):
current = []
if value in current:
return existing_json, False
current.append(value)
return json.dumps(current, ensure_ascii=True), True
async def upsert_credential_reuse(
self,
*,
secret_sha256: str,
secret_kind: str,
principal: Optional[str],
attacker_uuid: Optional[str],
attacker_ip: str,
decky: str,
service: str,
attempt_count: int,
ts: Optional[datetime] = None,
) -> Optional[dict[str, Any]]:
"""Upsert a credential-reuse finding.
The row is keyed by ``(secret_sha256, secret_kind, principal_key)``
— ``principal_key`` is the canonicalised non-null form ("" when
principal is null) so the unique constraint behaves the same on
SQLite and MySQL.
Returns the row dict augmented with ``inserted: bool`` and
``changed: bool`` so the correlator can decide whether to publish
a bus event.
"""
principal_key = principal or ""
now = ts or datetime.now(timezone.utc)
async with self._session() as session:
existing = (await session.execute(
select(CredentialReuse).where(
CredentialReuse.secret_sha256 == secret_sha256,
CredentialReuse.secret_kind == secret_kind,
CredentialReuse.principal_key == principal_key,
)
)).scalar_one_or_none()
if existing is None:
row = CredentialReuse(
id=str(_uuid.uuid4()),
secret_sha256=secret_sha256,
secret_kind=secret_kind,
principal=principal,
principal_key=principal_key,
attacker_uuids=json.dumps(
[attacker_uuid] if attacker_uuid else [], ensure_ascii=True
),
attacker_ips=json.dumps([attacker_ip], ensure_ascii=True),
deckies=json.dumps([decky], ensure_ascii=True),
services=json.dumps([service], ensure_ascii=True),
target_count=1,
attempt_count=int(attempt_count),
confidence=1.0,
first_seen=now,
last_seen=now,
updated_at=now,
)
session.add(row)
await session.commit()
await session.refresh(row)
d = row.model_dump(mode="json")
d["inserted"] = True
d["changed"] = True
return d
changed = False
new_uuids, c1 = self._merge_unique(existing.attacker_uuids, attacker_uuid)
new_ips, c2 = self._merge_unique(existing.attacker_ips, attacker_ip)
new_deckies, c3 = self._merge_unique(existing.deckies, decky)
new_services, c4 = self._merge_unique(existing.services, service)
existing.attacker_uuids = new_uuids
existing.attacker_ips = new_ips
if c3 or c4:
existing.deckies = new_deckies
existing.services = new_services
# Recount target tuples from the underlying credentials
# table — a (decky, service) tuple only counts when both
# were observed together, which the JSON lists alone
# can't tell us.
stmt = (
select(func.count(func.distinct(
Credential.decky_name + ":" + Credential.service
)))
.where(
Credential.secret_sha256 == secret_sha256,
Credential.secret_kind == secret_kind,
(Credential.principal == principal) if principal is not None
else Credential.principal.is_(None),
)
)
target_count = (await session.execute(stmt)).scalar() or 0
existing.target_count = int(target_count)
existing.attempt_count = (existing.attempt_count or 0) + int(attempt_count)
existing.last_seen = now
existing.updated_at = now
if c1 or c2 or c3 or c4:
changed = True
session.add(existing)
await session.commit()
await session.refresh(existing)
d = existing.model_dump(mode="json")
d["inserted"] = False
d["changed"] = changed
return d
async def find_credential_reuse_candidates(
self, min_targets: int = 2
) -> List[dict[str, Any]]:
"""Find credential groups crossing the reuse threshold.
Returns one dict per qualifying ``(secret_sha256, secret_kind,
principal)`` group, with the keys plus a ``credentials`` list of
the underlying rows so the correlator can fold each into
``CredentialReuse`` via ``upsert_credential_reuse``.
"""
target_expr = func.count(
func.distinct(Credential.decky_name + ":" + Credential.service)
).label("target_count")
async with self._session() as session:
group_stmt = (
select(
Credential.secret_sha256,
Credential.secret_kind,
Credential.principal,
target_expr,
)
.group_by(
Credential.secret_sha256,
Credential.secret_kind,
Credential.principal,
)
.having(target_expr >= int(min_targets))
)
groups = (await session.execute(group_stmt)).all()
out: List[dict[str, Any]] = []
for sha, kind, principal, target_count in groups:
cred_stmt = select(Credential).where(
Credential.secret_sha256 == sha,
Credential.secret_kind == kind,
(Credential.principal == principal)
if principal is not None
else Credential.principal.is_(None),
)
rows = (await session.execute(cred_stmt)).scalars().all()
out.append({
"secret_sha256": sha,
"secret_kind": kind,
"principal": principal,
"target_count": int(target_count or 0),
"credentials": [r.model_dump(mode="json") for r in rows],
})
return out
async def list_credential_reuses(
self,
limit: int = 50,
offset: int = 0,
min_target_count: int = 2,
secret_kind: Optional[str] = None,
) -> tuple[int, List[dict[str, Any]]]:
async with self._session() as session:
base = select(CredentialReuse).where(
CredentialReuse.target_count >= min_target_count
)
if secret_kind:
base = base.where(CredentialReuse.secret_kind == secret_kind)
total_stmt = select(func.count()).select_from(base.subquery())
total = (await session.execute(total_stmt)).scalar() or 0
list_stmt = (
base.order_by(desc(CredentialReuse.target_count),
desc(CredentialReuse.last_seen))
.offset(offset).limit(limit)
)
rows = (await session.execute(list_stmt)).scalars().all()
out: List[dict[str, Any]] = []
for r in rows:
d = r.model_dump(mode="json")
for key in ("attacker_uuids", "attacker_ips", "deckies", "services"):
try:
d[key] = json.loads(d[key])
except (json.JSONDecodeError, TypeError):
d[key] = []
out.append(d)
await self._enrich_with_secret(session, out)
return int(total), out
async def get_credential_reuse_by_id(
self, reuse_id: str
) -> Optional[dict[str, Any]]:
async with self._session() as session:
row = (await session.execute(
select(CredentialReuse).where(CredentialReuse.id == reuse_id)
)).scalar_one_or_none()
if row is None:
return None
d = row.model_dump(mode="json")
for key in ("attacker_uuids", "attacker_ips", "deckies", "services"):
try:
d[key] = json.loads(d[key])
except (json.JSONDecodeError, TypeError):
d[key] = []
await self._enrich_with_secret(session, [d])
return d
@staticmethod
async def _enrich_with_secret(
session: Any, rows: List[dict[str, Any]]
) -> None:
"""Tack ``secret_printable`` + ``secret_b64`` onto each reuse row.
``CredentialReuse`` only stores the sha256+kind hash of the
secret — the actual printable/b64 representations live on the
underlying ``Credential`` rows. The dashboard wants to show the
secret in the drawer, so we lift one matching credential per
``(sha256, kind, principal)`` finding. One batched query for the
whole page; rows with no surviving credential (shouldn't happen
in practice) get nulls.
"""
if not rows:
return
sha_set = {r["secret_sha256"] for r in rows}
if not sha_set:
return
stmt = select(
Credential.secret_sha256,
Credential.secret_kind,
Credential.principal,
Credential.secret_printable,
Credential.secret_b64,
).where(Credential.secret_sha256.in_(sha_set))
secret_map: dict[
tuple[str, str, Optional[str]],
tuple[Optional[str], Optional[str]],
] = {}
for sha, kind, principal, printable, b64 in (
(await session.execute(stmt)).all()
):
secret_map.setdefault((sha, kind, principal), (printable, b64))
for r in rows:
key = (r["secret_sha256"], r["secret_kind"], r.get("principal"))
printable, b64 = secret_map.get(key, (None, None))
r["secret_printable"] = printable
r["secret_b64"] = b64

91
decnet/web/db/sqlmodel_repo/deckies.py Normal file
View File

@@ -0,0 +1,91 @@
"""Decky-shard CRUD (per-host shard registrations)."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, Optional
import orjson
from sqlalchemy import asc, select, text
from decnet.web.db.models import DeckyShard
class DeckiesMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
async def upsert_decky_shard(self, data: dict[str, Any]) -> None:
payload = {**data, "updated_at": datetime.now(timezone.utc)}
if isinstance(payload.get("services"), list):
payload["services"] = orjson.dumps(payload["services"]).decode()
async with self._session() as session:
result = await session.execute(
select(DeckyShard).where(DeckyShard.decky_name == payload["decky_name"])
)
existing = result.scalar_one_or_none()
if existing:
for k, v in payload.items():
setattr(existing, k, v)
session.add(existing)
else:
session.add(DeckyShard(**payload))
await session.commit()
async def list_decky_shards(
self, host_uuid: Optional[str] = None
) -> list[dict[str, Any]]:
statement = select(DeckyShard).order_by(asc(DeckyShard.decky_name))
if host_uuid:
statement = statement.where(DeckyShard.host_uuid == host_uuid)
async with self._session() as session:
result = await session.execute(statement)
out: list[dict[str, Any]] = []
for r in result.scalars().all():
d = r.model_dump(mode="json")
raw = d.get("services")
if isinstance(raw, str):
try:
d["services"] = json.loads(raw)
except (json.JSONDecodeError, TypeError):
d["services"] = []
# Flatten the stored DeckyConfig snapshot into the row so
# routers can hand it to DeckyShardView without re-parsing.
# Rows predating the migration have decky_config=NULL and
# fall through with the default (None/{}) view values.
cfg_raw = d.get("decky_config")
if isinstance(cfg_raw, str):
try:
cfg = json.loads(cfg_raw)
except (json.JSONDecodeError, TypeError):
cfg = {}
if isinstance(cfg, dict):
for k in ("hostname", "distro", "archetype",
"service_config", "mutate_interval",
"last_mutated"):
if k in cfg and d.get(k) is None:
d[k] = cfg[k]
# Keep decky_ip authoritative from the column (newer
# heartbeats overwrite it) but fall back to the
# snapshot if the column is still NULL.
if not d.get("decky_ip") and cfg.get("ip"):
d["decky_ip"] = cfg["ip"]
out.append(d)
return out
async def delete_decky_shards_for_host(self, host_uuid: str) -> int:
async with self._session() as session:
result = await session.execute(
text("DELETE FROM decky_shards WHERE host_uuid = :u"),
{"u": host_uuid},
)
await session.commit()
return result.rowcount or 0
async def delete_decky_shard(self, decky_name: str) -> bool:
async with self._session() as session:
result = await session.execute(
text("DELETE FROM decky_shards WHERE decky_name = :n"),
{"n": decky_name},
)
await session.commit()
return bool(result.rowcount)

152
decnet/web/db/sqlmodel_repo/fleet.py Normal file
View File

@@ -0,0 +1,152 @@
"""Fleet decky CRUD + cross-source running-decky aggregator."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
import orjson
from sqlalchemy import asc, select, text, update
from decnet.web.db.models import DeckyShard, FleetDecky, LOCAL_HOST_SENTINEL
from decnet.web.db.sqlmodel_repo._helpers import _deserialize_json_fields
class FleetMixin:
"""Mixin: composed onto ``SQLModelRepository``.
``list_running_deckies`` aggregates topology + fleet + swarm-shard
sources and stays here because the fleet entry is the canonical
shape; ``list_running_topology_deckies`` / ``list_running_fleet_deckies``
on ``self`` resolve through the composed class.
"""
async def upsert_fleet_decky(self, data: dict[str, Any]) -> None:
payload: dict[str, Any] = {
**data,
"updated_at": datetime.now(timezone.utc),
}
payload.setdefault("host_uuid", LOCAL_HOST_SENTINEL)
if payload.get("host_uuid") is None:
payload["host_uuid"] = LOCAL_HOST_SENTINEL
if isinstance(payload.get("services"), list):
payload["services"] = orjson.dumps(payload["services"]).decode()
if isinstance(payload.get("decky_config"), dict):
payload["decky_config"] = orjson.dumps(payload["decky_config"]).decode()
async with self._session() as session:
result = await session.execute(
select(FleetDecky).where(
FleetDecky.host_uuid == payload["host_uuid"],
FleetDecky.name == payload["name"],
)
)
existing = result.scalar_one_or_none()
if existing:
for k, v in payload.items():
setattr(existing, k, v)
session.add(existing)
else:
session.add(FleetDecky(**payload))
await session.commit()
async def delete_fleet_decky(self, *, host_uuid: str, name: str) -> None:
async with self._session() as session:
await session.execute(
text(
"DELETE FROM fleet_deckies "
"WHERE host_uuid = :h AND name = :n"
),
{"h": host_uuid, "n": name},
)
await session.commit()
async def list_fleet_deckies(
self, *, host_uuid: Optional[str] = None,
) -> list[dict[str, Any]]:
stmt = select(FleetDecky).order_by(asc(FleetDecky.name))
if host_uuid:
stmt = stmt.where(FleetDecky.host_uuid == host_uuid)
async with self._session() as session:
result = await session.execute(stmt)
return [
_deserialize_json_fields(
r.model_dump(mode="json"), ("services", "decky_config")
)
for r in result.scalars().all()
]
async def list_running_fleet_deckies(self) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(FleetDecky).where(FleetDecky.state == "running")
)
return [
_deserialize_json_fields(
r.model_dump(mode="json"), ("services", "decky_config")
)
for r in result.scalars().all()
]
async def update_fleet_decky_state(
self, *, host_uuid: str, name: str, state: str,
last_error: Optional[str] = None,
) -> None:
now = datetime.now(timezone.utc)
values: dict[str, Any] = {
"state": state,
"updated_at": now,
"last_seen": now,
}
if last_error is not None:
values["last_error"] = last_error
async with self._session() as session:
await session.execute(
update(FleetDecky)
.where(
FleetDecky.host_uuid == host_uuid,
FleetDecky.name == name,
)
.values(**values)
)
await session.commit()
async def list_running_deckies(self) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
# MazeNET — already shaped {uuid, name, ip, services}. We carry
# topology_id through so consumers (emailgen scheduler) can walk
# back to the parent topology row without a second round-trip;
# fleet/shard rows never have one, hence Optional.
for d in await self.list_running_topology_deckies():
out.append({
"uuid": d.get("uuid"),
"name": d.get("name"),
"ip": d.get("ip"),
"services": d.get("services") or [],
"topology_id": d.get("topology_id"),
"source": "topology",
})
# Fleet — column is `decky_ip`, PK is composite (host_uuid, name)
for d in await self.list_running_fleet_deckies():
out.append({
"uuid": f"{d.get('host_uuid')}:{d.get('name')}",
"name": d.get("name"),
"ip": d.get("decky_ip"),
"services": d.get("services") or [],
"source": "fleet",
})
# SWARM — DeckyShard rows in 'running' state on enrolled workers.
async with self._session() as session:
shard_rows = await session.execute(
select(DeckyShard).where(DeckyShard.state == "running")
)
for s in shard_rows.scalars().all():
d = _deserialize_json_fields(
s.model_dump(mode="json"), ("services", "decky_config")
)
out.append({
"uuid": f"{d.get('host_uuid')}:{d.get('decky_name')}",
"name": d.get("decky_name"),
"ip": d.get("decky_ip"),
"services": d.get("services") or [],
"source": "shard",
})
return out

185
decnet/web/db/sqlmodel_repo/identities.py Normal file
View File

@@ -0,0 +1,185 @@
"""AttackerIdentity reads + writes.
Identity = the clustering output that groups multiple ``Attacker`` rows
(usually different IPs from the same actor) into one logical actor.
The identity-clusterer worker drives the writes; the dashboard drives
the reads.
"""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import desc, func, select, update
from decnet.web.db.models import Attacker, AttackerIdentity
class IdentitiesMixin:
"""Mixin: composed onto ``SQLModelRepository``.
``self._deserialize_attacker`` resolves through ``AttackersMixin``
via MRO.
"""
async def get_identity_by_uuid(self, uuid: str) -> Optional[dict[str, Any]]:
# Follow merged_into_uuid up to the winner. Loop bounded by
# _MAX_MERGE_HOPS so a (hypothetically) corrupted ring can't
# spin the worker. Clusterer is responsible for never producing
# a cycle; this guard is belt-and-braces.
_MAX_MERGE_HOPS = 8
async with self._session() as session:
current_uuid = uuid
for _ in range(_MAX_MERGE_HOPS):
result = await session.execute(
select(AttackerIdentity).where(AttackerIdentity.uuid == current_uuid)
)
identity = result.scalar_one_or_none()
if identity is None:
return None
if identity.merged_into_uuid is None:
return identity.model_dump(mode="json")
current_uuid = identity.merged_into_uuid
# Hit the hop cap — surface what we have rather than recurse.
return identity.model_dump(mode="json")
async def list_identities(
self, limit: int = 50, offset: int = 0,
) -> list[dict[str, Any]]:
# Exclude merged-out rows so the list view is the de-duped truth.
# The history is still queryable per-uuid via get_identity_by_uuid
# and a future "merged into" endpoint when we need it.
statement = (
select(AttackerIdentity)
.where(AttackerIdentity.merged_into_uuid.is_(None))
.order_by(desc(AttackerIdentity.updated_at))
.offset(offset)
.limit(limit)
)
async with self._session() as session:
result = await session.execute(statement)
return [i.model_dump(mode="json") for i in result.scalars().all()]
async def count_identities(self) -> int:
statement = (
select(func.count())
.select_from(AttackerIdentity)
.where(AttackerIdentity.merged_into_uuid.is_(None))
)
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def list_observations_for_identity(
self, identity_uuid: str, limit: int = 50, offset: int = 0,
) -> list[dict[str, Any]]:
statement = (
select(Attacker)
.where(Attacker.identity_id == identity_uuid)
.order_by(desc(Attacker.last_seen))
.offset(offset)
.limit(limit)
)
async with self._session() as session:
result = await session.execute(statement)
return [
self._deserialize_attacker(a.model_dump(mode="json"))
for a in result.scalars().all()
]
async def count_observations_for_identity(self, identity_uuid: str) -> int:
statement = (
select(func.count())
.select_from(Attacker)
.where(Attacker.identity_id == identity_uuid)
)
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def list_attackers_for_clustering(
self, limit: Optional[int] = None,
) -> list[dict[str, Any]]:
# Project the columns the clusterer's similarity graph reads.
# Keep it narrow so future denormalised projections (payloads
# joined from logs, c2 endpoints aggregated from sessions) can
# land here without churning every caller. ``fingerprints`` is
# the raw JSON list — the clusterer parses for JA3 / HASSH.
statement = select(
Attacker.uuid, Attacker.asn, Attacker.identity_id, Attacker.fingerprints,
).order_by(Attacker.first_seen)
if limit is not None:
statement = statement.limit(limit)
async with self._session() as session:
result = await session.execute(statement)
return [
{
"uuid": row.uuid,
"asn": row.asn,
"identity_id": row.identity_id,
"fingerprints": row.fingerprints,
}
for row in result.all()
]
async def create_attacker_identity(self, row: dict[str, Any]) -> str:
identity = AttackerIdentity(**row)
async with self._session() as session:
session.add(identity)
await session.commit()
return identity.uuid
async def set_attacker_identity_id(
self, attacker_uuid: str, identity_uuid: str,
) -> None:
statement = (
update(Attacker)
.where(Attacker.uuid == attacker_uuid)
.values(identity_id=identity_uuid)
)
async with self._session() as session:
await session.execute(statement)
await session.commit()
async def list_all_identities(self) -> list[dict[str, Any]]:
statement = select(AttackerIdentity).order_by(AttackerIdentity.created_at)
async with self._session() as session:
result = await session.execute(statement)
return [i.model_dump(mode="json") for i in result.scalars().all()]
async def update_identity_merged_into(
self, identity_uuid: str, winner_uuid: Optional[str],
) -> None:
statement = (
update(AttackerIdentity)
.where(AttackerIdentity.uuid == identity_uuid)
.values(
merged_into_uuid=winner_uuid,
updated_at=datetime.now(timezone.utc),
)
)
async with self._session() as session:
await session.execute(statement)
await session.commit()
async def update_identity_fingerprints(
self,
identity_uuid: str,
*,
ja3_hashes: Optional[str] = None,
hassh_hashes: Optional[str] = None,
tls_cert_sha256: Optional[str] = None,
) -> None:
statement = (
update(AttackerIdentity)
.where(AttackerIdentity.uuid == identity_uuid)
.values(
ja3_hashes=ja3_hashes,
hassh_hashes=hassh_hashes,
tls_cert_sha256=tls_cert_sha256,
updated_at=datetime.now(timezone.utc),
)
)
async with self._session() as session:
await session.execute(statement)
await session.commit()

213
decnet/web/db/sqlmodel_repo/logs.py Normal file
View File

@@ -0,0 +1,213 @@
"""Log ingestion, query, and the stats summary endpoint.
``get_log_histogram`` is the per-dialect override point; the abstract
default raises NotImplementedError. ``get_stats_summary`` joins log
counts, topology-decky counts, and the on-disk fleet state into a
single dashboard payload.
"""
from __future__ import annotations
import asyncio
import re
import shlex
from datetime import datetime
from typing import Any, List, Optional
import orjson
from sqlalchemy import asc, desc, func, or_, select, text
from sqlmodel.sql.expression import SelectOfScalar
from decnet.config import load_state
from decnet.web.db.models import Log, TopologyDecky
class LogsMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
@staticmethod
def _normalize_log_row(log_data: dict[str, Any]) -> dict[str, Any]:
data = log_data.copy()
if "fields" in data and isinstance(data["fields"], dict):
data["fields"] = orjson.dumps(data["fields"]).decode()
if "timestamp" in data and isinstance(data["timestamp"], str):
try:
data["timestamp"] = datetime.fromisoformat(
data["timestamp"].replace("Z", "+00:00")
)
except ValueError:
pass
return data
async def add_log(self, log_data: dict[str, Any]) -> None:
data = self._normalize_log_row(log_data)
async with self._session() as session:
session.add(Log(**data))
await session.commit()
async def add_logs(self, log_entries: list[dict[str, Any]]) -> None:
"""Bulk insert — one session, one commit for the whole batch."""
if not log_entries:
return
_rows = [Log(**self._normalize_log_row(e)) for e in log_entries]
async with self._session() as session:
session.add_all(_rows)
await session.commit()
def _apply_filters(
self,
statement: SelectOfScalar,
search: Optional[str],
start_time: Optional[str],
end_time: Optional[str],
) -> SelectOfScalar:
if start_time:
statement = statement.where(Log.timestamp >= start_time)
if end_time:
statement = statement.where(Log.timestamp <= end_time)
if search:
try:
tokens = shlex.split(search)
except ValueError:
tokens = search.split()
core_fields = {
"decky": Log.decky,
"service": Log.service,
"event": Log.event_type,
"attacker": Log.attacker_ip,
"attacker-ip": Log.attacker_ip,
"attacker_ip": Log.attacker_ip,
}
for token in tokens:
if ":" in token:
key, val = token.split(":", 1)
if key in core_fields:
statement = statement.where(core_fields[key] == val)
else:
key_safe = re.sub(r"[^a-zA-Z0-9_]", "", key)
if key_safe:
statement = statement.where(
self._json_field_equals(key_safe)
).params(val=val)
else:
lk = f"%{token}%"
statement = statement.where(
or_(
Log.raw_line.like(lk),
Log.decky.like(lk),
Log.service.like(lk),
Log.attacker_ip.like(lk),
)
)
return statement
def _json_field_equals(self, key: str):
"""Return a text() predicate that matches rows where fields->key == :val.
Both SQLite and MySQL expose a ``JSON_EXTRACT`` function; MySQL also
exposes the same function under ``json_extract`` (case-insensitive).
The ``:val`` parameter is bound separately and must be supplied with
``.params(val=...)`` by the caller, which keeps us safe from injection.
"""
return text(f"JSON_EXTRACT(fields, '$.{key}') = :val")
async def get_logs(
self,
limit: int = 50,
offset: int = 0,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
) -> List[dict]:
statement = (
select(Log)
.order_by(desc(Log.timestamp))
.offset(offset)
.limit(limit)
)
statement = self._apply_filters(statement, search, start_time, end_time)
async with self._session() as session:
results = await session.execute(statement)
return [log.model_dump(mode="json") for log in results.scalars().all()]
async def get_max_log_id(self) -> int:
async with self._session() as session:
result = await session.execute(select(func.max(Log.id)))
val = result.scalar()
return val if val is not None else 0
async def get_logs_after_id(
self,
last_id: int,
limit: int = 50,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
) -> List[dict]:
statement = (
select(Log).where(Log.id > last_id).order_by(asc(Log.id)).limit(limit)
)
statement = self._apply_filters(statement, search, start_time, end_time)
async with self._session() as session:
results = await session.execute(statement)
return [log.model_dump(mode="json") for log in results.scalars().all()]
async def get_total_logs(
self,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
) -> int:
statement = select(func.count()).select_from(Log)
statement = self._apply_filters(statement, search, start_time, end_time)
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def get_log_histogram(
self,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
interval_minutes: int = 15,
) -> List[dict]:
"""Dialect-specific — override per backend."""
raise NotImplementedError
async def get_stats_summary(self) -> dict[str, Any]:
async with self._session() as session:
total_logs = (
await session.execute(select(func.count()).select_from(Log))
).scalar() or 0
unique_attackers = (
await session.execute(
select(func.count(func.distinct(Log.attacker_ip)))
)
).scalar() or 0
topo_total = (
await session.execute(select(func.count()).select_from(TopologyDecky))
).scalar() or 0
topo_running = (
await session.execute(
select(func.count())
.select_from(TopologyDecky)
.where(TopologyDecky.state == "running")
)
).scalar() or 0
_state = await asyncio.to_thread(load_state)
fleet_deckies = len(_state[0].deckies) if _state else 0
return {
"total_logs": total_logs,
"unique_attackers": unique_attackers,
# Fleet state file doesn't track per-decky runtime; treat all
# fleet rows as active and add MazeNET running rows on top.
"active_deckies": fleet_deckies + topo_running,
"deployed_deckies": fleet_deckies + topo_total,
}

191
decnet/web/db/sqlmodel_repo/orchestrator.py Normal file
View File

@@ -0,0 +1,191 @@
"""Orchestrator event log + email log + per-pool prune helpers."""
from __future__ import annotations
import json
from datetime import datetime
from typing import Any, Optional
from sqlalchemy import delete as sa_delete
from sqlalchemy import desc, func, or_, select
from decnet.web.db.models import OrchestratorEmail, OrchestratorEvent
class OrchestratorMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
async def record_orchestrator_event(self, data: dict[str, Any]) -> str:
payload = data.get("payload")
if isinstance(payload, (dict, list)):
data = {**data, "payload": json.dumps(payload)}
async with self._session() as session:
row = OrchestratorEvent(**data)
session.add(row)
await session.commit()
await session.refresh(row)
return row.uuid
async def list_orchestrator_events(
self,
limit: int = 100,
offset: int = 0,
*,
kind: Optional[str] = None,
since_ts: Optional[datetime] = None,
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = select(OrchestratorEvent)
if kind is not None:
stmt = stmt.where(OrchestratorEvent.kind == kind)
if since_ts is not None:
stmt = stmt.where(OrchestratorEvent.ts >= since_ts)
stmt = (
stmt.order_by(desc(OrchestratorEvent.ts))
.offset(offset)
.limit(limit)
)
result = await session.execute(stmt)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def count_orchestrator_events(
self, *, kind: Optional[str] = None,
) -> int:
stmt = select(func.count()).select_from(OrchestratorEvent)
if kind is not None:
stmt = stmt.where(OrchestratorEvent.kind == kind)
async with self._session() as session:
result = await session.execute(stmt)
return result.scalar() or 0
async def prune_orchestrator_events(self, per_dst_cap: int = 10000) -> int:
"""Trim per-dst rows to *per_dst_cap*, oldest-first. Returns deleted count."""
deleted = 0
async with self._session() as session:
dst_rows = await session.execute(
select(OrchestratorEvent.dst_decky_uuid).distinct()
)
for (dst,) in dst_rows.all():
keep = await session.execute(
select(OrchestratorEvent.uuid)
.where(OrchestratorEvent.dst_decky_uuid == dst)
.order_by(desc(OrchestratorEvent.ts))
.limit(per_dst_cap)
)
keep_uuids = [u for (u,) in keep.all()]
if not keep_uuids:
continue
stmt = sa_delete(OrchestratorEvent).where(
OrchestratorEvent.dst_decky_uuid == dst,
OrchestratorEvent.uuid.notin_(keep_uuids),
)
res = await session.execute(stmt)
deleted += res.rowcount or 0
await session.commit()
return deleted
async def record_orchestrator_email(self, data: dict[str, Any]) -> str:
payload = data.get("payload")
if isinstance(payload, (dict, list)):
data = {**data, "payload": json.dumps(payload)}
async with self._session() as session:
row = OrchestratorEmail(**data)
session.add(row)
await session.commit()
await session.refresh(row)
return row.uuid
async def list_orchestrator_emails(
self,
limit: int = 100,
offset: int = 0,
*,
mail_decky_uuid: Optional[str] = None,
thread_id: Optional[str] = None,
since_ts: Optional[datetime] = None,
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = select(OrchestratorEmail)
if mail_decky_uuid is not None:
stmt = stmt.where(
OrchestratorEmail.mail_decky_uuid == mail_decky_uuid
)
if thread_id is not None:
stmt = stmt.where(OrchestratorEmail.thread_id == thread_id)
if since_ts is not None:
stmt = stmt.where(OrchestratorEmail.ts >= since_ts)
stmt = (
stmt.order_by(desc(OrchestratorEmail.ts))
.offset(offset)
.limit(limit)
)
result = await session.execute(stmt)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def count_orchestrator_emails(
self,
*,
mail_decky_uuid: Optional[str] = None,
) -> int:
stmt = select(func.count()).select_from(OrchestratorEmail)
if mail_decky_uuid is not None:
stmt = stmt.where(OrchestratorEmail.mail_decky_uuid == mail_decky_uuid)
async with self._session() as session:
result = await session.execute(stmt)
return result.scalar() or 0
async def list_orchestrator_email_threads(
self,
mail_decky_uuid: str,
sender_email: str,
recipient_email: str,
*,
limit: int = 50,
) -> list[dict[str, Any]]:
# Most-recent row per (sender, recipient) pair under this mail decky.
# The scheduler only needs the latest message_id/subject/thread_id to
# construct a reply; older rows in the same thread aren't relevant
# for the "do we reply or start fresh" decision.
async with self._session() as session:
stmt = (
select(OrchestratorEmail)
.where(
OrchestratorEmail.mail_decky_uuid == mail_decky_uuid,
or_(
(OrchestratorEmail.sender_email == sender_email)
& (OrchestratorEmail.recipient_email == recipient_email),
(OrchestratorEmail.sender_email == recipient_email)
& (OrchestratorEmail.recipient_email == sender_email),
),
OrchestratorEmail.success.is_(True),
)
.order_by(desc(OrchestratorEmail.ts))
.limit(limit)
)
result = await session.execute(stmt)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def prune_orchestrator_emails(self, per_decky_cap: int = 10000) -> int:
"""Trim per-mail-decky rows to *per_decky_cap*, oldest-first."""
deleted = 0
async with self._session() as session:
decky_rows = await session.execute(
select(OrchestratorEmail.mail_decky_uuid).distinct()
)
for (mail_uuid,) in decky_rows.all():
keep = await session.execute(
select(OrchestratorEmail.uuid)
.where(OrchestratorEmail.mail_decky_uuid == mail_uuid)
.order_by(desc(OrchestratorEmail.ts))
.limit(per_decky_cap)
)
keep_uuids = [u for (u,) in keep.all()]
if not keep_uuids:
continue
stmt = sa_delete(OrchestratorEmail).where(
OrchestratorEmail.mail_decky_uuid == mail_uuid,
OrchestratorEmail.uuid.notin_(keep_uuids),
)
res = await session.execute(stmt)
deleted += res.rowcount or 0
await session.commit()
return deleted

157
decnet/web/db/sqlmodel_repo/realism.py Normal file
View File

@@ -0,0 +1,157 @@
"""Synthetic-file CRUD + realism config key/value store."""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import Any, Optional
from sqlalchemy import desc, func, select, update
from decnet.web.db.models import RealismConfig, SyntheticFile
from decnet.web.db.models.realism import SYNTHETIC_FILE_BODY_LIMIT
class RealismMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
async def record_synthetic_file(self, data: dict[str, Any]) -> str:
if "last_body" in data and data["last_body"] is not None:
data = {**data, "last_body": data["last_body"][:SYNTHETIC_FILE_BODY_LIMIT]}
async with self._session() as session:
row = SyntheticFile(**data)
session.add(row)
await session.commit()
await session.refresh(row)
return row.uuid
async def update_synthetic_file(
self, row_uuid: str, data: dict[str, Any],
) -> None:
if "last_body" in data and data["last_body"] is not None:
data = {**data, "last_body": data["last_body"][:SYNTHETIC_FILE_BODY_LIMIT]}
async with self._session() as session:
stmt = (
update(SyntheticFile)
.where(SyntheticFile.uuid == row_uuid)
.values(**data)
)
await session.execute(stmt)
await session.commit()
async def list_synthetic_files(
self,
*,
decky_uuid: Optional[str] = None,
persona: Optional[str] = None,
content_class: Optional[str] = None,
limit: int = 100,
offset: int = 0,
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = select(SyntheticFile)
if decky_uuid is not None:
stmt = stmt.where(SyntheticFile.decky_uuid == decky_uuid)
if persona is not None:
stmt = stmt.where(SyntheticFile.persona == persona)
if content_class is not None:
stmt = stmt.where(SyntheticFile.content_class == content_class)
stmt = (
stmt.order_by(desc(SyntheticFile.last_modified))
.offset(offset)
.limit(limit)
)
result = await session.execute(stmt)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def count_synthetic_files(
self,
*,
decky_uuid: Optional[str] = None,
persona: Optional[str] = None,
content_class: Optional[str] = None,
) -> int:
async with self._session() as session:
stmt = select(func.count(SyntheticFile.uuid))
if decky_uuid is not None:
stmt = stmt.where(SyntheticFile.decky_uuid == decky_uuid)
if persona is not None:
stmt = stmt.where(SyntheticFile.persona == persona)
if content_class is not None:
stmt = stmt.where(SyntheticFile.content_class == content_class)
result = await session.execute(stmt)
return int(result.scalar() or 0)
async def get_synthetic_file(
self, uuid: str,
) -> Optional[dict[str, Any]]:
async with self._session() as session:
stmt = select(SyntheticFile).where(SyntheticFile.uuid == uuid)
result = await session.execute(stmt)
row = result.scalars().first()
if row is None:
return None
return row.model_dump(mode="json")
async def get_realism_config(
self, key: str,
) -> Optional[dict[str, Any]]:
async with self._session() as session:
stmt = select(RealismConfig).where(RealismConfig.key == key)
result = await session.execute(stmt)
row = result.scalars().first()
if row is None:
return None
return row.model_dump(mode="json")
async def set_realism_config(
self, key: str, value: str,
) -> None:
"""Upsert one realism_config row. Last-write-wins."""
async with self._session() as session:
stmt = select(RealismConfig).where(RealismConfig.key == key)
result = await session.execute(stmt)
row = result.scalars().first()
if row is None:
session.add(RealismConfig(
key=key, value=value,
updated_at=datetime.now(timezone.utc),
))
else:
upd = (
update(RealismConfig)
.where(RealismConfig.uuid == row.uuid)
.values(value=value, updated_at=datetime.now(timezone.utc))
)
await session.execute(upd)
await session.commit()
async def pick_random_synthetic_file_for_edit(
self,
decky_uuid: str,
*,
max_age_days: int = 30,
) -> Optional[dict[str, Any]]:
# Editable classes: anything whose body is plain text we can
# mutate idempotently. Binary canary artifacts are out — they
# rotate via a fresh plant, not an edit.
editable = (
"note", "todo", "draft", "script", "log_cron", "log_daemon",
)
cutoff = datetime.now(timezone.utc) - timedelta(days=max_age_days)
async with self._session() as session:
stmt = (
select(SyntheticFile)
.where(
SyntheticFile.decky_uuid == decky_uuid,
SyntheticFile.content_class.in_(editable), # type: ignore[attr-defined]
SyntheticFile.last_modified >= cutoff,
)
# SQLite + MySQL both support func.random() / RAND() —
# SQLAlchemy's func.random() compiles per-dialect.
.order_by(func.random())
.limit(1)
)
result = await session.execute(stmt)
row = result.scalars().first()
if row is None:
return None
return row.model_dump(mode="json")

71
decnet/web/db/sqlmodel_repo/swarm.py Normal file
View File

@@ -0,0 +1,71 @@
"""Swarm host CRUD."""
from __future__ import annotations
from typing import Any, Optional
from sqlalchemy import asc, select, text, update
from decnet.web.db.models import SwarmHost
class SwarmMixin:
"""Mixin: composed onto ``SQLModelRepository``. Expects ``self._session()``."""
async def add_swarm_host(self, data: dict[str, Any]) -> None:
async with self._session() as session:
session.add(SwarmHost(**data))
await session.commit()
async def get_swarm_host_by_name(self, name: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(select(SwarmHost).where(SwarmHost.name == name))
row = result.scalar_one_or_none()
return row.model_dump(mode="json") if row else None
async def get_swarm_host_by_uuid(self, uuid: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(select(SwarmHost).where(SwarmHost.uuid == uuid))
row = result.scalar_one_or_none()
return row.model_dump(mode="json") if row else None
async def get_swarm_host_by_fingerprint(self, fingerprint: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(SwarmHost).where(SwarmHost.client_cert_fingerprint == fingerprint)
)
row = result.scalar_one_or_none()
return row.model_dump(mode="json") if row else None
async def list_swarm_hosts(self, status: Optional[str] = None) -> list[dict[str, Any]]:
statement = select(SwarmHost).order_by(asc(SwarmHost.name))
if status:
statement = statement.where(SwarmHost.status == status)
async with self._session() as session:
result = await session.execute(statement)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def update_swarm_host(self, uuid: str, fields: dict[str, Any]) -> None:
if not fields:
return
async with self._session() as session:
await session.execute(
update(SwarmHost).where(SwarmHost.uuid == uuid).values(**fields)
)
await session.commit()
async def delete_swarm_host(self, uuid: str) -> bool:
async with self._session() as session:
# Clean up child shards first (no ON DELETE CASCADE portable across dialects).
await session.execute(
text("DELETE FROM decky_shards WHERE host_uuid = :u"), {"u": uuid}
)
result = await session.execute(
select(SwarmHost).where(SwarmHost.uuid == uuid)
)
host = result.scalar_one_or_none()
if not host:
await session.commit()
return False
await session.delete(host)
await session.commit()
return True

32
decnet/web/db/sqlmodel_repo/topology/__init__.py Normal file
View File

@@ -0,0 +1,32 @@
"""MazeNET topology repository methods.
The full domain spans ~700 lines of methods across topologies, LANs,
deckies, edges, the status-event log, and the live reconciler mutation
queue. Each concern lives in its own submixin; ``TopologyMixin``
composes them.
The optimistic-locking helpers (``_assert_pending``,
``_check_and_bump_version``) live on ``TopologyCoreMixin`` and are
reached from sibling submixins via ``self.`` — Python's MRO resolves
them to the core mixin no matter which submixin holds the caller.
"""
from __future__ import annotations
from decnet.web.db.sqlmodel_repo.topology._core import TopologyCoreMixin
from decnet.web.db.sqlmodel_repo.topology.deckies import TopologyDeckiesMixin
from decnet.web.db.sqlmodel_repo.topology.edges import TopologyEdgesMixin
from decnet.web.db.sqlmodel_repo.topology.lans import LansMixin
from decnet.web.db.sqlmodel_repo.topology.mutations import TopologyMutationsMixin
class TopologyMixin(
TopologyDeckiesMixin,
TopologyEdgesMixin,
LansMixin,
TopologyMutationsMixin,
TopologyCoreMixin,
):
"""Composed topology mixin — see submixins for the actual methods."""
__all__ = ["TopologyMixin"]

250
decnet/web/db/sqlmodel_repo/topology/_core.py Normal file
View File

@@ -0,0 +1,250 @@
"""Topology table CRUD + the optimistic-locking helpers that the
sibling LAN / decky / edge / mutation mixins call through MRO."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import desc, func, select, text
from decnet.web.db.models import Topology, TopologyStatusEvent
from decnet.web.db.sqlmodel_repo._helpers import (
_deserialize_json_fields,
_serialize_json_fields,
)
class TopologyCoreMixin:
"""Topologies CRUD + ``_assert_pending`` / ``_check_and_bump_version``.
The two private helpers live here because every other topology
submixin (lans, deckies, edges, mutations) calls them through
``self.`` — MRO resolution lands them on this mixin no matter
which submixin holds the caller.
"""
async def create_topology(self, data: dict[str, Any]) -> str:
payload = _serialize_json_fields(data, ("config_snapshot",))
async with self._session() as session:
row = Topology(**payload)
session.add(row)
await session.commit()
await session.refresh(row)
return row.id
async def get_topology(self, topology_id: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(Topology).where(Topology.id == topology_id)
)
row = result.scalar_one_or_none()
if not row:
return None
d = row.model_dump(mode="json")
return _deserialize_json_fields(d, ("config_snapshot",))
async def list_topologies(
self,
status: Optional[str] = None,
limit: Optional[int] = None,
offset: Optional[int] = None,
) -> list[dict[str, Any]]:
statement = select(Topology).order_by(desc(Topology.created_at))
if status:
statement = statement.where(Topology.status == status)
if offset is not None:
statement = statement.offset(offset)
if limit is not None:
statement = statement.limit(limit)
async with self._session() as session:
result = await session.execute(statement)
return [
_deserialize_json_fields(
r.model_dump(mode="json"), ("config_snapshot",)
)
for r in result.scalars().all()
]
async def count_topologies(self, status: Optional[str] = None) -> int:
statement = select(func.count(Topology.id))
if status:
statement = statement.where(Topology.status == status)
async with self._session() as session:
result = await session.execute(statement)
return int(result.scalar_one() or 0)
async def update_topology_status(
self,
topology_id: str,
new_status: str,
reason: Optional[str] = None,
) -> None:
"""Update topology.status and append a TopologyStatusEvent atomically.
Transition legality is enforced in ``decnet.topology.status``; this
method trusts the caller.
"""
now = datetime.now(timezone.utc)
async with self._session() as session:
result = await session.execute(
select(Topology).where(Topology.id == topology_id)
)
topo = result.scalar_one_or_none()
if topo is None:
return
from_status = topo.status
topo.status = new_status
topo.status_changed_at = now
session.add(topo)
session.add(
TopologyStatusEvent(
topology_id=topology_id,
from_status=from_status,
to_status=new_status,
at=now,
reason=reason,
)
)
await session.commit()
async def set_topology_resync(self, topology_id: str, value: bool) -> None:
async with self._session() as session:
result = await session.execute(
select(Topology).where(Topology.id == topology_id)
)
topo = result.scalar_one_or_none()
if topo is None:
return
topo.needs_resync = bool(value)
session.add(topo)
await session.commit()
async def set_topology_email_personas(
self, topology_id: str, personas_json: str,
) -> bool:
"""Replace ``Topology.email_personas`` with the supplied JSON.
The string is stored as-is; validation/parsing is the caller's
job (and is repeated by the emailgen scheduler each tick anyway).
Returns True if a row was updated.
"""
async with self._session() as session:
result = await session.execute(
select(Topology).where(Topology.id == topology_id)
)
topo = result.scalar_one_or_none()
if topo is None:
return False
topo.email_personas = personas_json
session.add(topo)
await session.commit()
return True
async def list_topologies_needing_resync(self) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(Topology).where(Topology.needs_resync == True) # noqa: E712
)
return [
_deserialize_json_fields(
r.model_dump(mode="json"), ("config_snapshot",)
)
for r in result.scalars().all()
]
async def delete_topology_cascade(self, topology_id: str) -> bool:
"""Delete topology and all children. No portable ON DELETE CASCADE."""
async with self._session() as session:
params = {"t": topology_id}
await session.execute(
text("DELETE FROM topology_status_events WHERE topology_id = :t"),
params,
)
await session.execute(
text("DELETE FROM topology_edges WHERE topology_id = :t"),
params,
)
await session.execute(
text("DELETE FROM topology_deckies WHERE topology_id = :t"),
params,
)
await session.execute(
text("DELETE FROM lans WHERE topology_id = :t"),
params,
)
await session.execute(
text("DELETE FROM topology_mutations WHERE topology_id = :t"),
params,
)
result = await session.execute(
select(Topology).where(Topology.id == topology_id)
)
topo = result.scalar_one_or_none()
if not topo:
await session.commit()
return False
await session.delete(topo)
await session.commit()
return True
async def list_live_topology_ids(self) -> list[str]:
"""Return ids of topologies currently in ``active|degraded``."""
async with self._session() as session:
result = await session.execute(
select(Topology.id).where(
Topology.status.in_(["active", "degraded"])
)
)
return [r for r in result.scalars().all()]
# ─── concurrency / pending-state guards (used by sibling mixins) ──────
async def _assert_pending(self, session, topology_id: str) -> None:
"""Pre-deploy edits are pending-only. Raises TopologyNotEditable."""
from decnet.topology.status import TopologyNotEditable, TopologyStatus
result = await session.execute(
select(Topology).where(Topology.id == topology_id)
)
topo = result.scalar_one_or_none()
if topo is None:
raise ValueError(f"topology {topology_id!r} not found")
if topo.status != TopologyStatus.PENDING:
raise TopologyNotEditable(
status=topo.status,
reason="free-form edits are pending-only; use the "
"mutator (topology_mutations) after deploy",
)
async def _check_and_bump_version(
self,
session,
topology_id: str,
expected_version: Optional[int],
) -> None:
"""Optimistic-concurrency guard used by child-row mutators.
If ``expected_version`` is None, no check happens (backward-compat
for internal callers that don't need concurrency protection).
If supplied, loads the Topology row in the same session,
compares ``version == expected_version``, raises VersionConflict
on mismatch, otherwise bumps ``version += 1``. The caller must
commit the enclosing session.
"""
from decnet.topology.status import VersionConflict
if expected_version is None:
return
result = await session.execute(
select(Topology).where(Topology.id == topology_id)
)
topo = result.scalar_one_or_none()
if topo is None:
raise ValueError(f"topology {topology_id!r} not found")
if topo.version != expected_version:
raise VersionConflict(
current=topo.version, expected=expected_version
)
topo.version = topo.version + 1
session.add(topo)

125
decnet/web/db/sqlmodel_repo/topology/deckies.py Normal file
View File

@@ -0,0 +1,125 @@
"""Topology decky CRUD + the running-decky listing the fleet aggregator
calls through MRO."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import asc, select, text, update
from decnet.web.db.models import TopologyDecky
from decnet.web.db.sqlmodel_repo._helpers import (
_deserialize_json_fields,
_serialize_json_fields,
)
class TopologyDeckiesMixin:
"""``self._assert_pending`` / ``self._check_and_bump_version`` resolve
through ``TopologyCoreMixin`` via MRO."""
async def add_topology_decky(
self,
data: dict[str, Any],
*,
expected_version: Optional[int] = None,
) -> str:
payload = _serialize_json_fields(data, ("services", "decky_config"))
async with self._session() as session:
await self._check_and_bump_version(
session, data["topology_id"], expected_version
)
row = TopologyDecky(**payload)
session.add(row)
await session.commit()
await session.refresh(row)
return row.uuid
async def update_topology_decky(
self,
decky_uuid: str,
fields: dict[str, Any],
*,
expected_version: Optional[int] = None,
enforce_pending: bool = False,
) -> None:
if not fields:
return
payload = _serialize_json_fields(fields, ("services", "decky_config"))
payload.setdefault("updated_at", datetime.now(timezone.utc))
async with self._session() as session:
result = await session.execute(
select(TopologyDecky).where(TopologyDecky.uuid == decky_uuid)
)
d = result.scalar_one_or_none()
if d is None:
raise ValueError(f"decky {decky_uuid!r} not found")
if enforce_pending:
await self._assert_pending(session, d.topology_id)
if expected_version is not None:
await self._check_and_bump_version(
session, d.topology_id, expected_version
)
await session.execute(
update(TopologyDecky)
.where(TopologyDecky.uuid == decky_uuid)
.values(**payload)
)
await session.commit()
async def delete_topology_decky(
self,
decky_uuid: str,
*,
expected_version: Optional[int] = None,
) -> None:
"""Cascade-delete a decky + all its edges from a pending topology."""
async with self._session() as session:
result = await session.execute(
select(TopologyDecky).where(TopologyDecky.uuid == decky_uuid)
)
d = result.scalar_one_or_none()
if d is None:
return
await self._assert_pending(session, d.topology_id)
if expected_version is not None:
await self._check_and_bump_version(
session, d.topology_id, expected_version
)
await session.execute(
text("DELETE FROM topology_edges WHERE decky_uuid = :u"),
{"u": decky_uuid},
)
await session.execute(
text("DELETE FROM topology_deckies WHERE uuid = :u"),
{"u": decky_uuid},
)
await session.commit()
async def list_topology_deckies(
self, topology_id: str
) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(TopologyDecky)
.where(TopologyDecky.topology_id == topology_id)
.order_by(asc(TopologyDecky.name))
)
return [
_deserialize_json_fields(
r.model_dump(mode="json"), ("services", "decky_config")
)
for r in result.scalars().all()
]
async def list_running_topology_deckies(self) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(TopologyDecky).where(TopologyDecky.state == "running")
)
return [
_deserialize_json_fields(
r.model_dump(mode="json"), ("services", "decky_config")
)
for r in result.scalars().all()
]

74
decnet/web/db/sqlmodel_repo/topology/edges.py Normal file
View File

@@ -0,0 +1,74 @@
"""Topology edge CRUD + status-event log."""
from __future__ import annotations
from typing import Any, Optional
from sqlalchemy import desc, select, text
from decnet.web.db.models import TopologyEdge, TopologyStatusEvent
class TopologyEdgesMixin:
"""``self._assert_pending`` / ``self._check_and_bump_version`` resolve
through ``TopologyCoreMixin`` via MRO."""
async def add_topology_edge(
self,
data: dict[str, Any],
*,
expected_version: Optional[int] = None,
) -> str:
async with self._session() as session:
await self._check_and_bump_version(
session, data["topology_id"], expected_version
)
row = TopologyEdge(**data)
session.add(row)
await session.commit()
await session.refresh(row)
return row.id
async def delete_topology_edge(
self,
edge_id: str,
*,
expected_version: Optional[int] = None,
) -> None:
async with self._session() as session:
result = await session.execute(
select(TopologyEdge).where(TopologyEdge.id == edge_id)
)
edge = result.scalar_one_or_none()
if edge is None:
return
await self._assert_pending(session, edge.topology_id)
if expected_version is not None:
await self._check_and_bump_version(
session, edge.topology_id, expected_version
)
await session.execute(
text("DELETE FROM topology_edges WHERE id = :e"),
{"e": edge_id},
)
await session.commit()
async def list_topology_edges(
self, topology_id: str
) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(TopologyEdge).where(TopologyEdge.topology_id == topology_id)
)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def list_topology_status_events(
self, topology_id: str, limit: int = 100
) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(TopologyStatusEvent)
.where(TopologyStatusEvent.topology_id == topology_id)
.order_by(desc(TopologyStatusEvent.at))
.limit(limit)
)
return [r.model_dump(mode="json") for r in result.scalars().all()]

118
decnet/web/db/sqlmodel_repo/topology/lans.py Normal file
View File

@@ -0,0 +1,118 @@
"""LAN CRUD within a topology."""
from __future__ import annotations
from typing import Any, Optional
from sqlalchemy import asc, select, text, update
from decnet.web.db.models import LAN, TopologyEdge
class LansMixin:
"""``self._assert_pending`` / ``self._check_and_bump_version`` resolve
through ``TopologyCoreMixin`` via MRO."""
async def add_lan(
self,
data: dict[str, Any],
*,
expected_version: Optional[int] = None,
) -> str:
async with self._session() as session:
await self._check_and_bump_version(
session, data["topology_id"], expected_version
)
row = LAN(**data)
session.add(row)
await session.commit()
await session.refresh(row)
return row.id
async def update_lan(
self,
lan_id: str,
fields: dict[str, Any],
*,
expected_version: Optional[int] = None,
enforce_pending: bool = False,
) -> None:
if not fields:
return
async with self._session() as session:
result = await session.execute(
select(LAN).where(LAN.id == lan_id)
)
lan = result.scalar_one_or_none()
if lan is None:
raise ValueError(f"lan {lan_id!r} not found")
if enforce_pending:
await self._assert_pending(session, lan.topology_id)
if expected_version is not None:
await self._check_and_bump_version(
session, lan.topology_id, expected_version
)
await session.execute(
update(LAN).where(LAN.id == lan_id).values(**fields)
)
await session.commit()
async def delete_lan(
self,
lan_id: str,
*,
expected_version: Optional[int] = None,
) -> None:
"""Cascade-delete a LAN from a pending topology.
Rejects if any decky declares this LAN as its home (i.e. has a
non-bridge edge to it — the only LAN that decky lives in). The
caller must delete or reassign the home-deckies first.
"""
from decnet.topology.status import TopologyNotEditable # noqa: F401
async with self._session() as session:
result = await session.execute(select(LAN).where(LAN.id == lan_id))
lan = result.scalar_one_or_none()
if lan is None:
return
await self._assert_pending(session, lan.topology_id)
# Home-decky check: any decky whose only edge lands here?
edges_result = await session.execute(
select(TopologyEdge).where(TopologyEdge.lan_id == lan_id)
)
edges_here = edges_result.scalars().all()
decky_uuids_on_this_lan = {e.decky_uuid for e in edges_here}
for decky_uuid in decky_uuids_on_this_lan:
other = await session.execute(
select(TopologyEdge).where(
TopologyEdge.decky_uuid == decky_uuid,
TopologyEdge.lan_id != lan_id,
)
)
if other.scalars().first() is None:
raise ValueError(
f"cannot delete LAN {lan.name!r}: decky "
f"{decky_uuid} has no other LAN (would be orphaned)"
)
if expected_version is not None:
await self._check_and_bump_version(
session, lan.topology_id, expected_version
)
# Cascade edges → LAN.
await session.execute(
text("DELETE FROM topology_edges WHERE lan_id = :l"),
{"l": lan_id},
)
await session.execute(text("DELETE FROM lans WHERE id = :l"), {"l": lan_id})
await session.commit()
async def list_lans_for_topology(
self, topology_id: str
) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(LAN).where(LAN.topology_id == topology_id).order_by(asc(LAN.name))
)
return [r.model_dump(mode="json") for r in result.scalars().all()]

171
decnet/web/db/sqlmodel_repo/topology/mutations.py Normal file
View File

@@ -0,0 +1,171 @@
"""Live-reconciler mutation queue: enqueue + atomic claim + state writes."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
import orjson
from sqlalchemy import asc, desc, select, text
from decnet.web.db.models import TopologyMutation
class TopologyMutationsMixin:
"""``self._check_and_bump_version`` resolves through
``TopologyCoreMixin`` via MRO."""
async def enqueue_topology_mutation(
self,
topology_id: str,
op: str,
payload: dict[str, Any],
*,
expected_version: Optional[int] = None,
) -> str:
"""Append a pending mutation row and bump the topology version.
Intended for use while the topology is ``active|degraded``; the
reconciler picks these rows up on its next tick.
"""
async with self._session() as session:
await self._check_and_bump_version(
session, topology_id, expected_version
)
row = TopologyMutation(
topology_id=topology_id,
op=op,
payload=orjson.dumps(payload).decode(),
)
session.add(row)
await session.commit()
await session.refresh(row)
return row.id
async def claim_next_mutation(
self, topology_id: str
) -> Optional[dict[str, Any]]:
"""Atomically claim the oldest pending mutation for ``topology_id``.
Correctness-critical: this is ONE SQL statement. Splitting it
into SELECT-then-UPDATE would let two racing watch-loops both
see the same ``pending`` row and both transition it to
``applying`` — double-executing the op. With the single
``UPDATE ... WHERE id = (SELECT ... LIMIT 1) AND state='pending'``
pattern the loser's UPDATE matches zero rows and returns
``None`` — that is the expected, non-error outcome under
contention.
"""
async with self._session() as session:
now = datetime.now(timezone.utc).isoformat()
# Single-statement atomic claim. The inner SELECT picks the
# oldest pending row; the outer UPDATE re-checks state so a
# second racer that also saw that id finds state='applying'
# and matches zero rows.
# MySQL forbids referencing the UPDATE target inside a
# subquery (ERROR 1093). Wrapping the inner SELECT in a
# derived table forces materialisation and sidesteps the
# rule. SQLite accepts both forms, so this stays portable.
sql = text(
"""
UPDATE topology_mutations
SET state = 'applying'
WHERE id = (
SELECT id FROM (
SELECT id FROM topology_mutations
WHERE topology_id = :t AND state = 'pending'
ORDER BY requested_at ASC
LIMIT 1
) AS _next
)
AND state = 'pending'
"""
)
result = await session.execute(sql, {"t": topology_id})
if result.rowcount == 0:
await session.commit()
return None
# Re-read the row we just claimed. The post-UPDATE SELECT is
# safe: no racer can now transition an ``applying`` row back
# to ``pending``.
sel = await session.execute(
select(TopologyMutation)
.where(TopologyMutation.topology_id == topology_id)
.where(TopologyMutation.state == "applying")
.order_by(asc(TopologyMutation.requested_at))
.limit(1)
)
row = sel.scalar_one_or_none()
await session.commit()
_ = now
if row is None:
return None
return row.model_dump(mode="json")
async def mark_mutation_applied(self, mutation_id: str) -> None:
async with self._session() as session:
await session.execute(
text(
"UPDATE topology_mutations "
"SET state = 'applied', applied_at = :at "
"WHERE id = :i"
),
{
"at": datetime.now(timezone.utc).isoformat(),
"i": mutation_id,
},
)
await session.commit()
async def mark_mutation_failed(
self, mutation_id: str, reason: str
) -> None:
async with self._session() as session:
await session.execute(
text(
"UPDATE topology_mutations "
"SET state = 'failed', applied_at = :at, reason = :r "
"WHERE id = :i"
),
{
"at": datetime.now(timezone.utc).isoformat(),
"r": reason,
"i": mutation_id,
},
)
await session.commit()
async def list_topology_mutations(
self,
topology_id: str,
state: Optional[str] = None,
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = (
select(TopologyMutation)
.where(TopologyMutation.topology_id == topology_id)
.order_by(desc(TopologyMutation.requested_at))
)
if state is not None:
stmt = stmt.where(TopologyMutation.state == state)
result = await session.execute(stmt)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def has_pending_topology_mutation(self) -> bool:
"""Cheap watch-loop guard: any pending mutation on a live topology?
Uses the ``ix_topology_mutations_state_topology`` composite index
to keep the join cheap at scale. Returns False as soon as the
reconciler path should be skipped.
"""
async with self._session() as session:
result = await session.execute(
text(
"SELECT 1 FROM topology_mutations "
"WHERE state = 'pending' "
"AND topology_id IN ("
" SELECT id FROM topologies "
" WHERE status IN ('active', 'degraded')"
") LIMIT 1"
)
)
return result.first() is not None

133
decnet/web/db/sqlmodel_repo/webhooks.py Normal file
View File

@@ -0,0 +1,133 @@
"""Webhook subscription CRUD + delivery bookkeeping."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import select, update
from decnet.web.db.models import WebhookSubscription
class WebhooksMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
async def create_webhook_subscription(self, data: dict[str, Any]) -> None:
async with self._session() as session:
session.add(WebhookSubscription(**data))
await session.commit()
async def get_webhook_subscription(
self, uuid: str
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(WebhookSubscription).where(WebhookSubscription.uuid == uuid)
)
row = result.scalar_one_or_none()
return row.model_dump() if row else None
async def get_webhook_subscription_by_name(
self, name: str
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(WebhookSubscription).where(WebhookSubscription.name == name)
)
row = result.scalar_one_or_none()
return row.model_dump() if row else None
async def list_webhook_subscriptions(
self, enabled_only: bool = False
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = select(WebhookSubscription)
if enabled_only:
stmt = stmt.where(WebhookSubscription.enabled.is_(True))
stmt = stmt.order_by(WebhookSubscription.created_at)
result = await session.execute(stmt)
return [r.model_dump() for r in result.scalars().all()]
async def update_webhook_subscription(
self, uuid: str, patch: dict[str, Any]
) -> bool:
if not patch:
return True
patch = {**patch, "updated_at": datetime.now(timezone.utc)}
async with self._session() as session:
result = await session.execute(
update(WebhookSubscription)
.where(WebhookSubscription.uuid == uuid)
.values(**patch)
)
await session.commit()
return result.rowcount > 0
async def delete_webhook_subscription(self, uuid: str) -> bool:
async with self._session() as session:
result = await session.execute(
select(WebhookSubscription).where(WebhookSubscription.uuid == uuid)
)
row = result.scalar_one_or_none()
if not row:
return False
await session.delete(row)
await session.commit()
return True
async def record_webhook_success(
self, uuid: str, ts: datetime
) -> None:
async with self._session() as session:
await session.execute(
update(WebhookSubscription)
.where(WebhookSubscription.uuid == uuid)
.values(
consecutive_failures=0,
last_success_at=ts,
last_error=None,
updated_at=ts,
)
)
await session.commit()
async def record_webhook_failure(
self, uuid: str, ts: datetime, error: str
) -> int:
async with self._session() as session:
# Read current failure count, bump, write. Small race window on
# concurrent deliveries to the same subscription is acceptable —
# the counter informs the circuit-breaker heuristic, not a
# correctness invariant.
result = await session.execute(
select(WebhookSubscription.consecutive_failures).where(
WebhookSubscription.uuid == uuid
)
)
current = result.scalar_one_or_none() or 0
new_count = current + 1
await session.execute(
update(WebhookSubscription)
.where(WebhookSubscription.uuid == uuid)
.values(
consecutive_failures=new_count,
last_failure_at=ts,
last_error=error[:512] if error else None,
updated_at=ts,
)
)
await session.commit()
return new_count
async def trip_webhook_circuit(self, uuid: str, ts: datetime) -> None:
async with self._session() as session:
await session.execute(
update(WebhookSubscription)
.where(WebhookSubscription.uuid == uuid)
.values(
enabled=False,
auto_disabled_at=ts,
updated_at=ts,
)
)
await session.commit()

140
decnet/web/dependencies.py
View File

@@ -1,3 +1,5 @@
import asyncio
import time
from typing import Any, Optional
import jwt
@@ -23,6 +25,88 @@ repo = get_repo()
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/v1/auth/login")
# Per-request user lookup was the hidden tax behind every authed endpoint —
# SELECT users WHERE uuid=? ran once per call, serializing through aiosqlite.
# 10s TTL is well below JWT expiry and we invalidate on all user writes.
_USER_TTL = 10.0
_user_cache: dict[str, tuple[Optional[dict[str, Any]], float]] = {}
_user_cache_lock: Optional[asyncio.Lock] = None
# Username cache for the login hot path. Short TTL — the bcrypt verify
# still runs against the cached hash, so security is unchanged. The
# staleness window is: if a password is changed, the old password is
# usable for up to _USERNAME_TTL seconds until the cache expires (or
# invalidate_user_cache fires). We invalidate on every user write.
# Missing lookups are NOT cached to avoid locking out a just-created user.
_USERNAME_TTL = 5.0
_username_cache: dict[str, tuple[dict[str, Any], float]] = {}
_username_cache_lock: Optional[asyncio.Lock] = None
def _reset_user_cache() -> None:
global _user_cache, _user_cache_lock, _username_cache, _username_cache_lock
_user_cache = {}
_user_cache_lock = None
_username_cache = {}
_username_cache_lock = None
def invalidate_user_cache(user_uuid: Optional[str] = None) -> None:
"""Drop a single user (or all users) from the auth caches.
Callers: password change, role change, user create/delete.
The username cache is always cleared wholesale — we don't track
uuid→username and user writes are rare, so the cost is trivial.
"""
if user_uuid is None:
_user_cache.clear()
else:
_user_cache.pop(user_uuid, None)
_username_cache.clear()
async def get_user_by_username_cached(username: str) -> Optional[dict[str, Any]]:
"""Cached read of get_user_by_username for the login path.
Positive hits are cached for _USERNAME_TTL seconds. Misses bypass
the cache so a freshly-created user can log in immediately.
"""
global _username_cache_lock
entry = _username_cache.get(username)
now = time.monotonic()
if entry is not None and now - entry[1] < _USERNAME_TTL:
return entry[0]
if _username_cache_lock is None:
_username_cache_lock = asyncio.Lock()
async with _username_cache_lock:
entry = _username_cache.get(username)
now = time.monotonic()
if entry is not None and now - entry[1] < _USERNAME_TTL:
return entry[0]
user = await repo.get_user_by_username(username)
if user is not None:
_username_cache[username] = (user, time.monotonic())
return user
async def _get_user_cached(user_uuid: str) -> Optional[dict[str, Any]]:
global _user_cache_lock
entry = _user_cache.get(user_uuid)
now = time.monotonic()
if entry is not None and now - entry[1] < _USER_TTL:
return entry[0]
if _user_cache_lock is None:
_user_cache_lock = asyncio.Lock()
async with _user_cache_lock:
entry = _user_cache.get(user_uuid)
now = time.monotonic()
if entry is not None and now - entry[1] < _USER_TTL:
return entry[0]
user = await repo.get_user_by_uuid(user_uuid)
_user_cache[user_uuid] = (user, time.monotonic())
return user
async def get_stream_user(request: Request, token: Optional[str] = None) -> str:
"""Auth dependency for SSE endpoints — accepts Bearer header OR ?token= query param.
EventSource does not support custom headers, so the query-string fallback is intentional here only.
@@ -82,7 +166,7 @@ async def _decode_token(request: Request) -> str:
async def get_current_user(request: Request) -> str:
"""Auth dependency — enforces must_change_password."""
_user_uuid = await _decode_token(request)
_user = await repo.get_user_by_uuid(_user_uuid)
_user = await _get_user_cached(_user_uuid)
if _user and _user.get("must_change_password"):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
@@ -96,3 +180,57 @@ async def get_current_user_unchecked(request: Request) -> str:
Use only for endpoints that must remain reachable with the flag set (e.g. change-password).
"""
return await _decode_token(request)
# ---------------------------------------------------------------------------
# Role-based access control
# ---------------------------------------------------------------------------
def require_role(*allowed_roles: str):
"""Factory that returns a FastAPI dependency enforcing role membership.
Inlines JWT decode + user lookup + must_change_password + role check so the
user is only loaded from the DB once per request (not once in
``get_current_user`` and again here). Returns the full user dict so
endpoints can inspect ``user["uuid"]``, ``user["role"]``, etc.
"""
async def _check(request: Request) -> dict:
user_uuid = await _decode_token(request)
user = await _get_user_cached(user_uuid)
if not user:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Could not validate credentials",
headers={"WWW-Authenticate": "Bearer"},
)
if user.get("must_change_password"):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Password change required before accessing this resource",
)
if user["role"] not in allowed_roles:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Insufficient permissions",
)
return user
return _check
def require_stream_role(*allowed_roles: str):
"""Like ``require_role`` but for SSE endpoints that accept a query-param token."""
async def _check(request: Request, token: Optional[str] = None) -> dict:
user_uuid = await get_stream_user(request, token)
user = await _get_user_cached(user_uuid)
if not user or user["role"] not in allowed_roles:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Insufficient permissions",
)
return user
return _check
require_admin = require_role("admin")
require_viewer = require_role("viewer", "admin")
require_stream_viewer = require_stream_role("viewer", "admin")

1057
decnet/web/ingester.py
View File

File diff suppressed because it is too large Load Diff

88
decnet/web/limiter.py Normal file
View File

@@ -0,0 +1,88 @@
"""Rate-limiting infra for the dashboard API.
Uses slowapi (which wraps the `limits` library) with in-memory storage.
In-memory is intentional for v1:
- The dashboard API runs on a single process per host (the `decnet api`
worker). Swarm agents do not serve the dashboard; there is no need for
cross-process shared state.
- Adding Redis as a hard dependency of the master for this one feature
is disproportionate.
Trust boundary note: `get_remote_address` uses `request.client.host`,
i.e. the TCP peer's IP. We deliberately do NOT trust `X-Forwarded-For`
because it is trivially spoofable by any client. Operators running
DECNET behind a reverse proxy get one shared bucket for the whole proxy
— that is an accepted limitation recorded in the threat model
(see `development/THREAT_MODEL.md` §Dashboard↔API, DA-08). Revisit when
we introduce a verified-proxy config.
"""
from __future__ import annotations
import json
import os
from typing import Any, Awaitable, Callable
from fastapi import Request
from slowapi import Limiter
from slowapi.util import get_remote_address
def _limiter_enabled() -> bool:
"""``DECNET_LIMITER_ENABLED=false`` disables the limiter process-wide.
Intended for stress / load testing, where a single Locust host
represents thousands of virtual users but shares one source IP and
one admin username — the real-world limits (10/5min per IP, per
user) would otherwise cap every run at 10 successful logins. The
default is ``true``; nobody should ever ship a release with this
off.
"""
return os.environ.get("DECNET_LIMITER_ENABLED", "true").lower() != "false"
# Single process-wide limiter. Importing modules pull this instance to
# apply `@limiter.limit(...)` decorators on their routes. Default
# headers off: FastAPI response_model handlers return dicts, not
# Starlette Response objects, and slowapi's header injection only
# supports the latter. Legit clients can back off on their own from
# the 429 body; attackers ignore Retry-After anyway.
limiter: Limiter = Limiter(
key_func=get_remote_address,
storage_uri="memory://",
enabled=_limiter_enabled(),
)
def login_ip_key(request: Request) -> str:
"""Per-IP bucket key for the login endpoint.
Thin wrapper around slowapi's default so tests can monkey-patch this
module attribute without reaching into slowapi internals.
"""
return f"login-ip:{get_remote_address(request)}"
async def login_username_key(request: Request) -> str:
"""Per-username bucket key for the login endpoint.
Reads the request body to extract the claimed username. The body is
cached by Starlette, so FastAPI's subsequent Pydantic parsing still
sees the same bytes. Malformed bodies all collapse to a single
bucket — that is intentional; garbage traffic gets throttled as one
bad actor rather than offered an escape hatch.
"""
try:
body: bytes = await request.body()
data: Any = json.loads(body or b"{}")
username = data.get("username") if isinstance(data, dict) else None
if isinstance(username, str) and username:
return f"login-user:{username}"
except (json.JSONDecodeError, UnicodeDecodeError, ValueError):
pass
return "login-user:__unparseable__"
# Exported so tests can monkey-patch a synchronous counterpart if they
# need deterministic keys without parsing bodies.
LoginKeyFunc = Callable[[Request], Awaitable[str] | str]

127
decnet/web/router/__init__.py
View File

@@ -5,14 +5,65 @@ from .auth.api_change_pass import router as change_pass_router
from .logs.api_get_logs import router as logs_router
from .logs.api_get_histogram import router as histogram_router
from .bounty.api_get_bounties import router as bounty_router
from .credentials.api_get_credentials import router as credentials_router
from .credential_reuse.api_get_credential_reuse import router as credential_reuse_router
from .stats.api_get_stats import router as stats_router
from .fleet.api_get_deckies import router as get_deckies_router
from .fleet.api_mutate_decky import router as mutate_decky_router
from .fleet.api_mutate_interval import router as mutate_interval_router
from .fleet.api_deploy_deckies import router as deploy_deckies_router
from .stream.api_stream_events import router as stream_router
from .attackers.api_get_attackers import router as attackers_router
from .attackers.api_get_attacker_detail import router as attacker_detail_router
from .attackers.api_get_attacker_commands import router as attacker_commands_router
from .attackers.api_get_attacker_artifacts import router as attacker_artifacts_router
from .attackers.api_get_attacker_transcripts import router as attacker_transcripts_router
from .attackers.api_get_attacker_smtp_targets import router as attacker_smtp_targets_router
from .attackers.api_get_attacker_mail import router as attacker_mail_router
from .attackers.api_get_attacker_intel import router as attacker_intel_router
from .identities.api_list_identities import router as identities_list_router
from .identities.api_get_identity_detail import router as identity_detail_router
from .identities.api_list_identity_observations import router as identity_observations_router
from .identities.api_events import router as identity_events_router
from .campaigns.api_list_campaigns import router as campaigns_list_router
from .campaigns.api_get_campaign_detail import router as campaign_detail_router
from .campaigns.api_list_campaign_identities import router as campaign_identities_router
from .campaigns.api_events import router as campaign_events_router
from .orchestrator.api_list_events import router as orchestrator_list_router
from .orchestrator.api_events import router as orchestrator_events_router
from .realism.api_config import router as realism_config_router
from .realism.api_personas import router as realism_personas_router
from .realism.api_synthetic_files import router as realism_synthetic_files_router
from .transcripts import transcripts_router
from .config.api_get_config import router as config_get_router
from .config.api_update_config import router as config_update_router
from .config.api_manage_users import router as config_users_router
from .config.api_reinit import router as config_reinit_router
from .health.api_get_health import router as health_router
from .workers.api_list_workers import router as workers_list_router
from .workers.api_control_worker import router as workers_control_router
from .workers.api_start_worker import router as workers_start_router
from .workers.api_start_all_workers import router as workers_start_all_router
from .artifacts.api_get_artifact import router as artifacts_router
from .swarm_updates import swarm_updates_router
from .swarm_mgmt import swarm_mgmt_router
from .system import system_router
from .topology import topology_router
from .canary import canary_router
from .webhooks import webhooks_router
api_router = APIRouter()
api_router = APIRouter(
# Every route under /api/v1 is auth-guarded (either by an explicit
# require_* Depends or by the global auth middleware). Document 401/403
# here so the OpenAPI schema reflects reality for contract tests.
responses={
400: {"description": "Malformed request body"},
401: {"description": "Missing or invalid credentials"},
403: {"description": "Authenticated but not authorized"},
404: {"description": "Referenced resource does not exist"},
409: {"description": "Conflict with existing resource"},
},
)
# Authentication
api_router.include_router(login_router)
@@ -25,12 +76,86 @@ api_router.include_router(histogram_router)
# Bounty Vault
api_router.include_router(bounty_router)
# Credentials (deduped attacker auth attempts)
api_router.include_router(credentials_router)
# Credential reuse findings (cross-decky/cross-service same-secret hits)
api_router.include_router(credential_reuse_router)
# Fleet Management
api_router.include_router(get_deckies_router)
api_router.include_router(mutate_decky_router)
api_router.include_router(mutate_interval_router)
api_router.include_router(deploy_deckies_router)
# Attacker Profiles
api_router.include_router(attackers_router)
api_router.include_router(attacker_detail_router)
api_router.include_router(attacker_commands_router)
api_router.include_router(attacker_artifacts_router)
api_router.include_router(attacker_transcripts_router)
api_router.include_router(attacker_smtp_targets_router)
api_router.include_router(attacker_mail_router)
api_router.include_router(attacker_intel_router)
# Identity Resolution (read-only; populated by the clusterer worker —
# see development/IDENTITY_RESOLUTION.md). Empty until the clusterer
# ships; the API surface lands first so frontend + downstream work
# can target a stable shape.
api_router.include_router(identities_list_router)
api_router.include_router(identity_detail_router)
api_router.include_router(identity_observations_router)
api_router.include_router(identity_events_router)
api_router.include_router(campaigns_list_router)
api_router.include_router(campaign_detail_router)
api_router.include_router(campaign_identities_router)
api_router.include_router(campaign_events_router)
api_router.include_router(orchestrator_list_router)
api_router.include_router(orchestrator_events_router)
# Realism — global persona pool CRUD for the dashboard's
# "Persona Generation" page. The orchestrator reads from the same
# on-disk JSON file directly (see decnet.realism.personas_pool).
api_router.include_router(realism_personas_router)
api_router.include_router(realism_synthetic_files_router)
api_router.include_router(realism_config_router)
# Observability
api_router.include_router(stats_router)
api_router.include_router(stream_router)
api_router.include_router(health_router)
api_router.include_router(workers_list_router)
api_router.include_router(workers_control_router)
api_router.include_router(workers_start_router)
api_router.include_router(workers_start_all_router)
# Configuration
api_router.include_router(config_get_router)
api_router.include_router(config_update_router)
api_router.include_router(config_users_router)
api_router.include_router(config_reinit_router)
# Artifacts (captured attacker file drops)
api_router.include_router(artifacts_router)
# Transcripts (PTY session recordings, paged asciinema events)
api_router.include_router(transcripts_router)
# Remote Updates (dashboard → worker updater daemons)
api_router.include_router(swarm_updates_router)
# Swarm Management (dashboard: hosts, deckies, agent enrollment bundles)
api_router.include_router(swarm_mgmt_router)
# System info (deployment-mode auto-detection, etc.)
api_router.include_router(system_router)
# MazeNET Topologies (nested topology CRUD + mutation queue)
api_router.include_router(topology_router)
# Canary tokens — operator-facing CRUD (worker hosts the
# attacker-facing surface separately via `decnet canary`).
api_router.include_router(canary_router)
# External webhook subscriptions (SIEM/SOAR egress)
api_router.include_router(webhooks_router)

0
decnet/web/router/artifacts/__init__.py Normal file
View File

95
decnet/web/router/artifacts/api_get_artifact.py Normal file
View File

@@ -0,0 +1,95 @@
"""
Artifact download endpoint.
SSH deckies farm attacker file drops into a host-mounted quarantine:
/var/lib/decnet/artifacts/{decky}/ssh/{stored_as}
The capture event already flows through the normal log pipeline (one
RFC 5424 line per capture, see templates/ssh/emit_capture.py), so metadata
is served via /logs. This endpoint exists only to retrieve the raw bytes —
admin-gated because the payloads are attacker-controlled content.
"""
from __future__ import annotations
import os
import re
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi.responses import FileResponse
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import require_admin
router = APIRouter()
# Override via env for tests; the prod path matches the bind mount declared in
# decnet/services/ssh.py and decnet/services/smtp.py.
ARTIFACTS_ROOT = Path(os.environ.get("DECNET_ARTIFACTS_ROOT", "/var/lib/decnet/artifacts"))
# decky names come from the deployer — lowercase alnum plus hyphens.
_DECKY_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$")
# Services that own an artifacts subdir. Kept explicit so a caller can't
# pivot into arbitrary subpaths via the query string.
_ALLOWED_SERVICES = {"ssh", "smtp"}
# stored_as is assembled by the capturing template as:
# ${ts}_${sha:0:12}_${base}
# where ts is ISO-8601 UTC (e.g. 2026-04-18T02:22:56Z), sha is 12 hex chars,
# and base is the original filename's basename. Keep the filename charset
# tight but allow common punctuation dropped files actually use.
_STORED_AS_RE = re.compile(
r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z_[a-f0-9]{12}_[A-Za-z0-9._-]{1,255}$"
)
def _resolve_artifact_path(decky: str, stored_as: str, service: str) -> Path:
"""Validate inputs, resolve the on-disk path, and confirm it stays inside
the artifacts root. Raises HTTPException(400) on any violation."""
if service not in _ALLOWED_SERVICES:
raise HTTPException(status_code=400, detail="invalid service")
if not _DECKY_RE.fullmatch(decky):
raise HTTPException(status_code=400, detail="invalid decky name")
if not _STORED_AS_RE.fullmatch(stored_as):
raise HTTPException(status_code=400, detail="invalid stored_as")
root = ARTIFACTS_ROOT.resolve()
candidate = (root / decky / service / stored_as).resolve()
# defence-in-depth: even though the regexes reject `..`, make sure a
# symlink or weird filesystem state can't escape the root.
if root not in candidate.parents and candidate != root:
raise HTTPException(status_code=400, detail="path escapes artifacts root")
return candidate
@router.get(
"/artifacts/{decky}/{stored_as}",
tags=["Artifacts"],
responses={
400: {"description": "Invalid decky, service, or stored_as parameter"},
401: {"description": "Could not validate credentials"},
403: {"description": "Admin access required"},
404: {"description": "Artifact not found"},
},
)
@_traced("api.get_artifact")
async def get_artifact(
decky: str,
stored_as: str,
service: str = Query("ssh", pattern=r"^[a-z]{1,16}$"),
admin: dict = Depends(require_admin),
) -> FileResponse:
path = _resolve_artifact_path(decky, stored_as, service)
if not path.is_file():
raise HTTPException(status_code=404, detail="artifact not found")
return FileResponse(
path=str(path),
media_type="application/octet-stream",
filename=stored_as,
headers={
"Content-Disposition": f'attachment; filename="{stored_as}"',
"X-Content-Type-Options": "nosniff",
},
)

0
decnet/web/router/attackers/__init__.py Normal file
View File

34
decnet/web/router/attackers/api_get_attacker_artifacts.py Normal file
View File

@@ -0,0 +1,34 @@
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import require_viewer, repo
router = APIRouter()
@router.get(
"/attackers/{uuid}/artifacts",
tags=["Attacker Profiles"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
404: {"description": "Attacker not found"},
},
)
@_traced("api.get_attacker_artifacts")
async def get_attacker_artifacts(
uuid: str,
user: dict = Depends(require_viewer),
) -> dict[str, Any]:
"""List captured file-drop artifacts for an attacker (newest first).
Each entry is a `file_captured` log row — the frontend renders the
badge/drawer using the same `fields` payload as /logs.
"""
attacker = await repo.get_attacker_by_uuid(uuid)
if not attacker:
raise HTTPException(status_code=404, detail="Attacker not found")
rows = await repo.get_attacker_artifacts(uuid)
return {"total": len(rows), "data": rows}

42
decnet/web/router/attackers/api_get_attacker_commands.py Normal file
View File

@@ -0,0 +1,42 @@
from typing import Any, Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import require_viewer, repo
router = APIRouter()
@router.get(
"/attackers/{uuid}/commands",
tags=["Attacker Profiles"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
404: {"description": "Attacker not found"},
422: {"description": "Query parameter validation error (limit/offset out of range or invalid)"},
},
)
@_traced("api.get_attacker_commands")
async def get_attacker_commands(
uuid: str,
limit: int = Query(50, ge=1, le=200),
offset: int = Query(0, ge=0, le=2147483647),
service: Optional[str] = None,
user: dict = Depends(require_viewer),
) -> dict[str, Any]:
"""Retrieve paginated commands for an attacker profile."""
attacker = await repo.get_attacker_by_uuid(uuid)
if not attacker:
raise HTTPException(status_code=404, detail="Attacker not found")
def _norm(v: Optional[str]) -> Optional[str]:
if v in (None, "null", "NULL", "undefined", ""):
return None
return v
result = await repo.get_attacker_commands(
uuid=uuid, limit=limit, offset=offset, service=_norm(service),
)
return {"total": result["total"], "limit": limit, "offset": offset, "data": result["data"]}

44
decnet/web/router/attackers/api_get_attacker_detail.py Normal file
View File

@@ -0,0 +1,44 @@
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from decnet.correlation.event_kinds import bucket_services
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import require_viewer, repo
router = APIRouter()
@router.get(
"/attackers/{uuid}",
tags=["Attacker Profiles"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
404: {"description": "Attacker not found"},
},
)
@_traced("api.get_attacker_detail")
async def get_attacker_detail(
uuid: str,
user: dict = Depends(require_viewer),
) -> dict[str, Any]:
"""Retrieve a single attacker profile by UUID (with behavior block)."""
attacker = await repo.get_attacker_by_uuid(uuid)
if not attacker:
raise HTTPException(status_code=404, detail="Attacker not found")
attacker["behavior"] = await repo.get_attacker_behavior(uuid)
# Scanned vs. interacted-with — computed per-request from the log
# stream, not persisted. Cheap (DISTINCT bounded by service ×
# event_type cardinality), and changes to the classifier take effect
# immediately without a profiler re-tick.
pairs = await repo.get_attacker_service_activity(uuid)
attacker["service_activity"] = bucket_services(pairs)
# Attribution leaks — XFF / Forwarded / X-Real-IP mismatches captured
# by the HTTP bounty extractor. Cap the returned list at 10 so a
# rotation attack (100s of forged XFF values) doesn't flood the UI;
# `ip_leaks_total` carries the unbounded count so the UI can render
# a ROTATION DETECTED badge when the count crosses a threshold.
attacker["ip_leaks"] = await repo.get_attacker_ip_leaks(uuid, limit=10)
attacker["ip_leaks_total"] = await repo.count_attacker_ip_leaks(uuid)
return attacker

38
decnet/web/router/attackers/api_get_attacker_intel.py Normal file
View File

@@ -0,0 +1,38 @@
"""GET /api/v1/attackers/{uuid}/intel — latest threat-intel row for an attacker."""
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import repo, require_viewer
router = APIRouter()
@router.get(
"/attackers/{uuid}/intel",
tags=["Attacker Profiles"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
404: {"description": "No intel cached for this attacker"},
},
)
@_traced("api.get_attacker_intel")
async def get_attacker_intel(
uuid: str,
user: dict = Depends(require_viewer),
) -> dict[str, Any]:
"""Return the most recent cached threat-intel verdict for an attacker.
The row is populated out-of-band by the ``decnet enrich`` worker
(typically within seconds of first observation, sub-second when the
bus is healthy). 404 means either the worker has not run yet or the
UUID does not correspond to an attacker DECNET has seen.
"""
record = await repo.get_attacker_intel_by_uuid(uuid)
if not record:
raise HTTPException(
status_code=404, detail="No intel cached for this attacker",
)
return record

37
decnet/web/router/attackers/api_get_attacker_mail.py Normal file
View File

@@ -0,0 +1,37 @@
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import require_admin, repo
router = APIRouter()
@router.get(
"/attackers/{uuid}/mail",
tags=["Attacker Profiles"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Admin access required"},
404: {"description": "Attacker not found"},
},
)
@_traced("api.get_attacker_mail")
async def get_attacker_mail(
uuid: str,
admin: dict = Depends(require_admin),
) -> dict[str, Any]:
"""List stored messages this attacker relayed via the SMTP honeypots.
Each entry is a ``message_stored`` log row — headers + attachment
manifest live in ``fields``; the raw .eml bytes are fetched via
``/artifacts/{decky}/{stored_as}?service=smtp`` (also admin-gated).
Admin-only because message bodies are attacker-controlled content
and may include phishing kits / malware droppers.
"""
attacker = await repo.get_attacker_by_uuid(uuid)
if not attacker:
raise HTTPException(status_code=404, detail="Attacker not found")
rows = await repo.get_attacker_stored_mail(uuid)
return {"total": len(rows), "data": rows}

36
decnet/web/router/attackers/api_get_attacker_smtp_targets.py Normal file
View File

@@ -0,0 +1,36 @@
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import require_viewer, repo
router = APIRouter()
@router.get(
"/attackers/{uuid}/smtp-targets",
tags=["Attacker Profiles"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
404: {"description": "Attacker not found"},
},
)
@_traced("api.get_attacker_smtp_targets")
async def get_attacker_smtp_targets(
uuid: str,
user: dict = Depends(require_viewer),
) -> dict[str, Any]:
"""List victim domains this attacker targeted via the SMTP honeypots.
Rows are ordered by most-recent activity. Each row is one
(attacker, domain) pair with a running count + first/last seen — no
local-parts (user names) are ever stored, so this is safe to show
to any viewer role.
"""
attacker = await repo.get_attacker_by_uuid(uuid)
if not attacker:
raise HTTPException(status_code=404, detail="Attacker not found")
rows = await repo.list_smtp_targets(uuid)
return {"total": len(rows), "data": rows}

34
decnet/web/router/attackers/api_get_attacker_transcripts.py Normal file
View File

@@ -0,0 +1,34 @@
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import require_viewer, repo
router = APIRouter()
@router.get(
"/attackers/{uuid}/transcripts",
tags=["Attacker Profiles"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
404: {"description": "Attacker not found"},
},
)
@_traced("api.get_attacker_transcripts")
async def get_attacker_transcripts(
uuid: str,
user: dict = Depends(require_viewer),
) -> dict[str, Any]:
"""List PTY session recordings for an attacker (newest first).
Each entry is a `session_recorded` log row — the frontend lists them
in the AttackerDetail Sessions tab and opens SessionDrawer on click.
"""
attacker = await repo.get_attacker_by_uuid(uuid)
if not attacker:
raise HTTPException(status_code=404, detail="Attacker not found")
rows = await repo.get_attacker_transcripts(uuid)
return {"total": len(rows), "data": rows}

83
decnet/web/router/attackers/api_get_attackers.py Normal file
View File

@@ -0,0 +1,83 @@
import asyncio
import time
from typing import Any, Optional
from fastapi import APIRouter, Depends, Query
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import require_viewer, repo
from decnet.web.db.models import AttackersResponse
router = APIRouter()
# Same pattern as /logs — cache the unfiltered total count; filtered
# counts go straight to the DB.
_TOTAL_TTL = 2.0
_total_cache: tuple[Optional[int], float] = (None, 0.0)
_total_lock: Optional[asyncio.Lock] = None
def _reset_total_cache() -> None:
global _total_cache, _total_lock
_total_cache = (None, 0.0)
_total_lock = None
async def _get_total_attackers_cached() -> int:
global _total_cache, _total_lock
value, ts = _total_cache
now = time.monotonic()
if value is not None and now - ts < _TOTAL_TTL:
return value
if _total_lock is None:
_total_lock = asyncio.Lock()
async with _total_lock:
value, ts = _total_cache
now = time.monotonic()
if value is not None and now - ts < _TOTAL_TTL:
return value
value = await repo.get_total_attackers()
_total_cache = (value, time.monotonic())
return value
@router.get(
"/attackers",
response_model=AttackersResponse,
tags=["Attacker Profiles"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
422: {"description": "Validation error"},
},
)
@_traced("api.get_attackers")
async def get_attackers(
limit: int = Query(50, ge=1, le=1000),
offset: int = Query(0, ge=0, le=2147483647),
search: Optional[str] = None,
sort_by: str = Query("recent", pattern="^(recent|active|traversals)$"),
service: Optional[str] = None,
user: dict = Depends(require_viewer),
) -> dict[str, Any]:
"""Retrieve paginated attacker profiles."""
def _norm(v: Optional[str]) -> Optional[str]:
if v in (None, "null", "NULL", "undefined", ""):
return None
return v
s = _norm(search)
svc = _norm(service)
_data = await repo.get_attackers(limit=limit, offset=offset, search=s, sort_by=sort_by, service=svc)
if s is None and svc is None:
_total = await _get_total_attackers_cached()
else:
_total = await repo.get_total_attackers(search=s, service=svc)
# Bulk-join behavior rows for the IPs in this page to avoid N+1 queries.
_ips = {row["ip"] for row in _data if row.get("ip")}
_behaviors = await repo.get_behaviors_for_ips(_ips) if _ips else {}
for row in _data:
row["behavior"] = _behaviors.get(row.get("ip"))
return {"total": _total, "limit": limit, "offset": offset, "data": _data}

14
decnet/web/router/auth/api_change_pass.py
View File

@@ -2,9 +2,10 @@ from typing import Any, Optional
from fastapi import APIRouter, Depends, HTTPException, status
from decnet.web.auth import get_password_hash, verify_password
from decnet.web.dependencies import get_current_user_unchecked, repo
from decnet.web.db.models import ChangePasswordRequest
from decnet.telemetry import traced as _traced
from decnet.web.auth import ahash_password, averify_password
from decnet.web.dependencies import get_current_user_unchecked, invalidate_user_cache, repo
from decnet.web.db.models import ChangePasswordRequest, MessageResponse
router = APIRouter()
@@ -12,20 +13,23 @@ router = APIRouter()
@router.post(
"/auth/change-password",
tags=["Authentication"],
response_model=MessageResponse,
responses={
400: {"description": "Bad Request (e.g. malformed JSON)"},
401: {"description": "Could not validate credentials"},
422: {"description": "Validation error"}
},
)
@_traced("api.change_password")
async def change_password(request: ChangePasswordRequest, current_user: str = Depends(get_current_user_unchecked)) -> dict[str, str]:
_user: Optional[dict[str, Any]] = await repo.get_user_by_uuid(current_user)
if not _user or not verify_password(request.old_password, _user["password_hash"]):
if not _user or not await averify_password(request.old_password, _user["password_hash"]):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Incorrect old password",
)
_new_hash: str = get_password_hash(request.new_password)
_new_hash: str = await ahash_password(request.new_password)
await repo.update_user_password(current_user, _new_hash, must_change_password=False)
invalidate_user_cache(current_user)
return {"message": "Password updated successfully"}

33
decnet/web/router/auth/api_login.py
View File

@@ -1,19 +1,32 @@
from datetime import timedelta
from typing import Any, Optional
from fastapi import APIRouter, HTTPException, status
from fastapi import APIRouter, HTTPException, Request, status
from decnet.telemetry import traced as _traced
from decnet.web.auth import (
ACCESS_TOKEN_EXPIRE_MINUTES,
averify_password,
create_access_token,
verify_password,
)
from decnet.web.dependencies import repo
from decnet.web.dependencies import get_user_by_username_cached
from decnet.web.db.models import LoginRequest, Token
from decnet.web.limiter import limiter, login_ip_key, login_username_key
router = APIRouter()
# Two independent buckets, tripping either → 429:
#
# - per-IP (login_ip_key): catches a botnet hitting one account.
# - per-user (login_username_key): catches distributed credential
# stuffing against one account.
#
# Limits: 10 attempts per 5 minutes per bucket. Buckets are process-local
# (memory://); see decnet/web/limiter.py for the rationale. Buckets do
# NOT reset on successful login — a legitimate user tripping the limit
# via fat-fingering will need to wait the window out. 10 tries is
# generous; a rolling window naturally drains.
@router.post(
"/auth/login",
response_model=Token,
@@ -21,12 +34,16 @@ router = APIRouter()
responses={
400: {"description": "Bad Request (e.g. malformed JSON)"},
401: {"description": "Incorrect username or password"},
422: {"description": "Validation error"}
422: {"description": "Validation error"},
429: {"description": "Too many login attempts — retry after the window resets"},
},
)
async def login(request: LoginRequest) -> dict[str, Any]:
_user: Optional[dict[str, Any]] = await repo.get_user_by_username(request.username)
if not _user or not verify_password(request.password, _user["password_hash"]):
@limiter.limit("10/5 minutes", key_func=login_ip_key)
@limiter.limit("10/5 minutes", key_func=login_username_key)
@_traced("api.login")
async def login(request: Request, payload: LoginRequest) -> dict[str, Any]:
_user: Optional[dict[str, Any]] = await get_user_by_username_cached(payload.username)
if not _user or not await averify_password(payload.password, _user["password_hash"]):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Incorrect username or password",
@@ -40,6 +57,6 @@ async def login(request: LoginRequest) -> dict[str, Any]:
)
return {
"access_token": _access_token,
"token_type": "bearer", # nosec B105
"token_type": "bearer", # nosec B105 — OAuth2 token type, not a password
"must_change_password": bool(_user.get("must_change_password", False))
}

50
decnet/web/router/bounty/api_get_bounties.py
View File

@@ -1,21 +1,62 @@
import asyncio
import time
from typing import Any, Optional
from fastapi import APIRouter, Depends, Query
from decnet.web.dependencies import get_current_user, repo
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import require_viewer, repo
from decnet.web.db.models import BountyResponse
router = APIRouter()
# Cache the unfiltered default page — the UI/locust hit this constantly
# with no params. Filtered requests (bounty_type/search) bypass: rare
# and staleness matters for search.
_BOUNTY_TTL = 5.0
_DEFAULT_LIMIT = 50
_DEFAULT_OFFSET = 0
_bounty_cache: tuple[Optional[dict[str, Any]], float] = (None, 0.0)
_bounty_lock: Optional[asyncio.Lock] = None
def _reset_bounty_cache() -> None:
global _bounty_cache, _bounty_lock
_bounty_cache = (None, 0.0)
_bounty_lock = None
async def _get_bounty_default_cached() -> dict[str, Any]:
global _bounty_cache, _bounty_lock
value, ts = _bounty_cache
now = time.monotonic()
if value is not None and now - ts < _BOUNTY_TTL:
return value
if _bounty_lock is None:
_bounty_lock = asyncio.Lock()
async with _bounty_lock:
value, ts = _bounty_cache
now = time.monotonic()
if value is not None and now - ts < _BOUNTY_TTL:
return value
_data = await repo.get_bounties(
limit=_DEFAULT_LIMIT, offset=_DEFAULT_OFFSET, bounty_type=None, search=None,
)
_total = await repo.get_total_bounties(bounty_type=None, search=None)
value = {"total": _total, "limit": _DEFAULT_LIMIT, "offset": _DEFAULT_OFFSET, "data": _data}
_bounty_cache = (value, time.monotonic())
return value
@router.get("/bounty", response_model=BountyResponse, tags=["Bounty Vault"],
responses={401: {"description": "Could not validate credentials"}, 422: {"description": "Validation error"}},)
responses={401: {"description": "Could not validate credentials"}, 403: {"description": "Insufficient permissions"}, 422: {"description": "Validation error"}},)
@_traced("api.get_bounties")
async def get_bounties(
limit: int = Query(50, ge=1, le=1000),
offset: int = Query(0, ge=0, le=2147483647),
bounty_type: Optional[str] = None,
search: Optional[str] = None,
current_user: str = Depends(get_current_user)
user: dict = Depends(require_viewer)
) -> dict[str, Any]:
"""Retrieve collected bounties (harvested credentials, payloads, etc.)."""
def _norm(v: Optional[str]) -> Optional[str]:
@@ -26,6 +67,9 @@ async def get_bounties(
bt = _norm(bounty_type)
s = _norm(search)
if bt is None and s is None and limit == _DEFAULT_LIMIT and offset == _DEFAULT_OFFSET:
return await _get_bounty_default_cached()
_data = await repo.get_bounties(limit=limit, offset=offset, bounty_type=bt, search=s)
_total = await repo.get_total_bounties(bounty_type=bt, search=s)
return {

0
decnet/web/router/campaigns/__init__.py Normal file
View File

123
decnet/web/router/campaigns/api_events.py Normal file
View File

@@ -0,0 +1,123 @@
"""SSE stream of campaign events — one connection per viewer.
Subscribes to ``campaign.>`` on the bus for the duration of the
request and forwards each matching event as a Server-Sent Event.
Emits a one-shot snapshot on connect (current paginated campaign
list).
Mirror of :mod:`decnet.web.router.identities.api_events`. Auth: JWT
via ``?token=`` query param + ``require_stream_viewer`` role.
"""
from __future__ import annotations
import asyncio
from typing import AsyncGenerator
import orjson
from fastapi import APIRouter, Depends, Request
from fastapi.responses import StreamingResponse
from decnet.bus import topics as _topics
from decnet.bus.app import get_app_bus
from decnet.logging import get_logger
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import repo, require_stream_viewer
from decnet.web.sse_limits import sse_connection_slot
log = get_logger("api.campaigns.events")
router = APIRouter()
_KEEPALIVE_SECS = 15.0
_SNAPSHOT_LIMIT = 50
def _format_sse(event_name: str, data: dict) -> str:
return f"event: {event_name}\ndata: {orjson.dumps(data).decode()}\n\n"
@router.get(
"/campaigns/events",
tags=["Campaign Clustering"],
responses={
200: {
"content": {"text/event-stream": {}},
"description": "SSE stream of campaign-clustering events",
},
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
429: {"description": "Per-user SSE connection cap reached"},
},
)
@_traced("api.campaigns.events")
async def api_campaigns_events(
request: Request,
user: dict = Depends(require_stream_viewer),
) -> StreamingResponse:
# Event types: snapshot, formed, identity.assigned, merged, unmerged.
snapshot = await repo.list_campaigns(limit=_SNAPSHOT_LIMIT, offset=0)
async def generator() -> AsyncGenerator[str, None]:
async with sse_connection_slot(user["uuid"]):
yield ": keepalive\n\n"
yield _format_sse("snapshot", {"campaigns": snapshot})
bus = await get_app_bus()
if bus is None:
while not await request.is_disconnected():
try:
await asyncio.sleep(_KEEPALIVE_SECS)
except asyncio.CancelledError:
break
yield ": keepalive\n\n"
return
sub = bus.subscribe(f"{_topics.CAMPAIGN}.>")
try:
async with sub:
sub_iter = sub.__aiter__()
while True:
if await request.is_disconnected():
break
next_task = asyncio.ensure_future(sub_iter.__anext__())
try:
event = await asyncio.wait_for(
next_task, timeout=_KEEPALIVE_SECS,
)
except asyncio.TimeoutError:
next_task.cancel()
yield ": keepalive\n\n"
continue
except StopAsyncIteration:
break
yield _format_sse(
_sse_name_for(event.topic),
{
"topic": event.topic,
"type": event.type,
"ts": event.ts,
"payload": event.payload,
},
)
except asyncio.CancelledError:
pass
except Exception:
log.exception("campaign events stream crashed")
yield _format_sse("error", {"message": "Stream interrupted"})
return StreamingResponse(
generator(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"X-Accel-Buffering": "no",
},
)
def _sse_name_for(topic: str) -> str:
"""``campaign.formed`` → ``formed``;
``campaign.identity.assigned`` → ``identity.assigned``."""
if topic.startswith(f"{_topics.CAMPAIGN}."):
return topic[len(_topics.CAMPAIGN) + 1:]
return topic

40
decnet/web/router/campaigns/api_get_campaign_detail.py Normal file
View File

@@ -0,0 +1,40 @@
"""GET /api/v1/campaigns/{uuid} — single campaign row.
Soft-merge handling: if the requested UUID has merged_into_uuid set,
the repository follows the chain and returns the winner. Mirror of
:mod:`decnet.web.router.identities.api_get_identity_detail`.
"""
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import repo, require_viewer
router = APIRouter()
@router.get(
"/campaigns/{uuid}",
tags=["Campaign Clustering"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
404: {"description": "Campaign not found"},
},
)
@_traced("api.get_campaign_detail")
async def get_campaign_detail(
uuid: str,
user: dict = Depends(require_viewer),
) -> dict[str, Any]:
campaign = await repo.get_campaign_by_uuid(uuid)
if not campaign:
raise HTTPException(status_code=404, detail="Campaign not found")
# Cheap aggregate the CampaignDetail page surfaces — counted off
# the FK rather than the denormalized identity_count so the answer
# is always live.
campaign["identity_count_live"] = await repo.count_identities_for_campaign(
campaign["uuid"]
)
return campaign

41
decnet/web/router/campaigns/api_list_campaign_identities.py Normal file
View File

@@ -0,0 +1,41 @@
"""GET /api/v1/campaigns/{uuid}/identities — identities for a campaign.
Returns the ``AttackerIdentity`` rows whose ``campaign_id`` FK points
at this campaign. Mirror of
:mod:`decnet.web.router.identities.api_list_identity_observations`.
"""
from typing import Any
from fastapi import APIRouter, Depends, HTTPException, Query
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import repo, require_viewer
router = APIRouter()
@router.get(
"/campaigns/{uuid}/identities",
tags=["Campaign Clustering"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
404: {"description": "Campaign not found"},
},
)
@_traced("api.list_campaign_identities")
async def list_campaign_identities(
uuid: str,
limit: int = Query(50, ge=1, le=1000),
offset: int = Query(0, ge=0, le=2147483647),
user: dict = Depends(require_viewer),
) -> dict[str, Any]:
campaign = await repo.get_campaign_by_uuid(uuid)
if not campaign:
raise HTTPException(status_code=404, detail="Campaign not found")
canonical_uuid = campaign["uuid"]
data = await repo.list_identities_for_campaign(
canonical_uuid, limit=limit, offset=offset
)
total = await repo.count_identities_for_campaign(canonical_uuid)
return {"total": total, "limit": limit, "offset": offset, "data": data}

35
decnet/web/router/campaigns/api_list_campaigns.py Normal file
View File

@@ -0,0 +1,35 @@
"""GET /api/v1/campaigns — paginated list of campaigns.
Mirror of :mod:`decnet.web.router.identities.api_list_identities` for
the campaign layer. Returns an empty list while the campaign clusterer
hasn't run yet (the campaigns table ships empty).
"""
from typing import Any
from fastapi import APIRouter, Depends, Query
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import repo, require_viewer
router = APIRouter()
@router.get(
"/campaigns",
tags=["Campaign Clustering"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
422: {"description": "Validation error"},
},
)
@_traced("api.list_campaigns")
async def list_campaigns(
limit: int = Query(50, ge=1, le=1000),
offset: int = Query(0, ge=0, le=2147483647),
user: dict = Depends(require_viewer),
) -> dict[str, Any]:
"""Paginated campaign list, newest-updated first."""
data = await repo.list_campaigns(limit=limit, offset=offset)
total = await repo.count_campaigns()
return {"total": total, "limit": limit, "offset": offset, "data": data}

23
decnet/web/router/canary/__init__.py Normal file
View File

@@ -0,0 +1,23 @@
"""Canary tokens — operator-facing CRUD.
Mounted under ``/api/v1/canary``. Covers:
* ``POST /blobs`` — upload an artifact (multipart);
``GET /blobs``, ``DELETE /blobs/{id}`` — listing + cleanup
* ``POST /tokens`` — generate + plant a token on a target decky;
``GET /tokens``, ``GET /tokens/{id}``, ``DELETE /tokens/{id}``
— listing + detail + revoke
* ``GET /tokens/{id}/preview`` — instrumented bytes for sanity-check
* ``GET /tokens/{id}/triggers`` — paged callback log
The ``decnet canary`` worker runs the ATTACKER-facing surface (HTTP
slug + DNS); this module is the OPERATOR-facing surface only.
"""
from fastapi import APIRouter
from .api_blobs import router as blobs_router
from .api_tokens import router as tokens_router
canary_router = APIRouter(prefix="/canary")
canary_router.include_router(blobs_router)
canary_router.include_router(tokens_router)

172
decnet/web/router/canary/api_blobs.py Normal file
View File

@@ -0,0 +1,172 @@
"""Operator-uploaded canary blob CRUD.
Three endpoints:
* ``POST /blobs`` — multipart upload; sniffs MIME from the magic
bytes (no python-magic dependency), persists to disk under the
sha256 hash, returns the (possibly pre-existing) row.
* ``GET /blobs`` — list all blobs with their live token reference
count.
* ``DELETE /blobs/{uuid}`` — refcount-aware delete; returns 409 if
any token still references the blob.
Admin-gated: blobs are operator-supplied content that may carry
sensitive material (real-looking financial reports, etc.); listing
them and deleting them is an admin operation. Reading them via the
preview path is also admin-gated.
"""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from decnet.canary import storage
from decnet.logging import get_logger
from decnet.web.db.models import (
CanaryBlobResponse,
CanaryBlobsResponse,
MessageResponse,
)
from decnet.web.dependencies import repo, require_admin
log = get_logger("api.canary.blobs")
router = APIRouter(prefix="/blobs", tags=["Canary"])
# --- MIME sniffing (stdlib-only, replaces python-magic) -------------------
#
# The DOCX/XLSX/PDF/PNG/JPEG/GIF/HTML/JSON/YAML space covers everything
# our instrumenters know how to mutate. Anything else falls through to
# ``application/octet-stream`` and the API routes the token to the
# ``passthrough`` instrumenter.
_MAGIC_TABLE: tuple[tuple[bytes, str], ...] = (
(b"\x89PNG\r\n\x1a\n", "image/png"),
(b"\xff\xd8\xff", "image/jpeg"),
(b"GIF87a", "image/gif"),
(b"GIF89a", "image/gif"),
(b"%PDF-", "application/pdf"),
# OOXML (DOCX/XLSX) starts with PK\x03\x04 but so do plain zips.
# We disambiguate by Content_Types entry below.
(b"<!DOCTYPE", "text/html"),
(b"<html", "text/html"),
(b"<HTML", "text/html"),
(b"<?xml", "application/xml"),
)
def _sniff_mime(filename: str, head: bytes) -> str:
for marker, mime in _MAGIC_TABLE:
if head.startswith(marker):
return mime
if head[:4] == b"PK\x03\x04":
# OOXML alias detection: peek for the document-specific Override
# in [Content_Types].xml. We only need to look at the first
# block; the central directory comes later.
if b"wordprocessingml" in head:
return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
if b"spreadsheetml" in head:
return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
return "application/zip"
# Plaintext heuristic: if the head decodes as printable utf-8 we
# call it text/plain — that's good enough to route to the plain
# instrumenter, which also handles json/yaml/toml.
try:
head.decode("utf-8")
if all(b in (0x09, 0x0A, 0x0D) or b >= 0x20 for b in head[:128]):
lf = filename.lower()
if lf.endswith((".json",)):
return "application/json"
if lf.endswith((".yaml", ".yml")):
return "application/yaml"
if lf.endswith((".toml",)):
return "application/toml"
return "text/plain"
except UnicodeDecodeError:
pass
return "application/octet-stream"
def _row_to_response(row: dict[str, Any]) -> CanaryBlobResponse:
return CanaryBlobResponse(**row)
@router.post(
"",
response_model=CanaryBlobResponse,
status_code=201,
responses={
400: {"description": "Empty file or unreadable upload"},
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
},
)
async def api_upload_blob(
file: UploadFile = File(...),
admin: dict = Depends(require_admin),
) -> CanaryBlobResponse:
content = await file.read()
if not content:
raise HTTPException(status_code=400, detail="uploaded file is empty")
sniffed = _sniff_mime(file.filename or "", content[:1024])
sha, _path, size = storage.write_blob(content)
row = await repo.upsert_canary_blob({
"sha256": sha,
"filename": file.filename or "(unnamed)",
"content_type": sniffed,
"size_bytes": size,
"uploaded_by": admin.get("uuid", "unknown"),
"uploaded_at": datetime.now(timezone.utc),
})
row.setdefault("token_count", 0)
return _row_to_response(row)
@router.get(
"",
response_model=CanaryBlobsResponse,
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
},
)
async def api_list_blobs(
admin: dict = Depends(require_admin),
) -> CanaryBlobsResponse:
rows = await repo.list_canary_blobs()
return CanaryBlobsResponse(
blobs=[_row_to_response(r) for r in rows],
total=len(rows),
)
@router.delete(
"/{uuid}",
response_model=MessageResponse,
responses={
404: {"description": "Blob not found"},
409: {"description": "Blob still referenced by a token"},
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
},
)
async def api_delete_blob(
uuid: str,
admin: dict = Depends(require_admin),
) -> MessageResponse:
existing = await repo.get_canary_blob(uuid)
if existing is None:
raise HTTPException(status_code=404, detail="blob not found")
deleted = await repo.delete_canary_blob(uuid)
if not deleted:
raise HTTPException(
status_code=409,
detail="blob is still referenced by one or more tokens",
)
# DB row is gone; best-effort unlink the bytes on disk. A failure
# here leaves a recoverable orphan, never a dangling DB ref.
storage.unlink_blob(existing["sha256"])
return MessageResponse(message="ok")

318
decnet/web/router/canary/api_tokens.py Normal file
View File

@@ -0,0 +1,318 @@
"""Operator-facing canary token CRUD.
Every body-bearing route documents the 400 error per
:mod:`feedback_schemathesis_400`. Auth deps:
* writes (POST, DELETE) → :func:`require_admin`
* reads (GET, preview) → :func:`require_viewer`
The router resolves blobs / instrumenters / generators here, builds
the :class:`CanaryArtifact`, and hands it to the planter. The
worker is a separate process; it doesn't see this code path.
"""
from __future__ import annotations
from secrets import token_urlsafe
from typing import Any
from uuid import uuid4
from fastapi import APIRouter, Depends, HTTPException, Query, Response
from decnet.canary import (
CanaryContext,
get_generator,
get_instrumenter,
pick_instrumenter_for_mime,
storage,
)
from decnet.canary.base import InstrumenterRejectedError
from decnet.canary.factory import KNOWN_GENERATORS
from decnet.canary.paths import normalize_placement
from decnet.canary import planter
from decnet.logging import get_logger
from decnet.web.db.models import (
CanaryTokenCreateRequest,
CanaryTokenResponse,
CanaryTokensResponse,
CanaryTriggerResponse,
CanaryTriggersResponse,
MessageResponse,
)
from decnet.web.dependencies import repo, require_admin, require_viewer
log = get_logger("api.canary.tokens")
router = APIRouter(prefix="/tokens", tags=["Canary"])
def _http_base() -> str:
import os
return os.environ.get(
"DECNET_CANARY_HTTP_BASE", "http://localhost:8088",
).rstrip("/")
def _dns_zone() -> str:
import os
return os.environ.get("DECNET_CANARY_DNS_ZONE", "").strip(".").lower()
def _row_to_response(row: dict[str, Any]) -> CanaryTokenResponse:
return CanaryTokenResponse(**row)
def _trigger_row_to_response(row: dict[str, Any]) -> CanaryTriggerResponse:
# Decode raw_headers JSON for the response shape.
headers = row.get("raw_headers") or "{}"
try:
import json
decoded = json.loads(headers) if isinstance(headers, str) else headers
if not isinstance(decoded, dict):
decoded = {}
except (ValueError, TypeError):
decoded = {}
out = dict(row)
out["headers"] = decoded
out.pop("raw_headers", None)
return CanaryTriggerResponse(**out)
# ---------------------------------------------------------- create
@router.post(
"",
response_model=CanaryTokenResponse,
status_code=201,
responses={
400: {"description": "Invalid token request (missing/conflicting fields, bad path, instrumenter rejection)"},
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
404: {"description": "Referenced blob not found"},
},
)
async def api_create_token(
req: CanaryTokenCreateRequest,
admin: dict = Depends(require_admin),
) -> CanaryTokenResponse:
# Exactly one of blob_uuid / generator must be set.
if bool(req.blob_uuid) == bool(req.generator):
raise HTTPException(
status_code=400,
detail="provide exactly one of blob_uuid or generator",
)
try:
placement_path = normalize_placement(req.placement_path)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
slug = token_urlsafe(16)
ctx = CanaryContext(
callback_token=slug, http_base=_http_base(), dns_zone=_dns_zone(),
)
if req.generator:
if req.generator not in KNOWN_GENERATORS:
raise HTTPException(
status_code=400,
detail=f"unknown generator: {req.generator!r}",
)
generator = get_generator(req.generator)
artifact = generator.generate(ctx)
instrumenter_name = None
else:
# Upload-driven token.
blob = await repo.get_canary_blob(req.blob_uuid)
if blob is None:
raise HTTPException(status_code=404, detail="blob not found")
try:
blob_bytes = storage.read_blob(blob["sha256"])
except FileNotFoundError as e:
raise HTTPException(
status_code=410,
detail="blob bytes missing on disk; please re-upload",
) from e
instrumenter_name = pick_instrumenter_for_mime(blob["content_type"])
ins = get_instrumenter(instrumenter_name)
try:
artifact = ins.instrument(blob_bytes, ctx, target_path=placement_path)
except InstrumenterRejectedError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
artifact.path = placement_path
token_uuid = str(uuid4())
kind = req.kind
await repo.create_canary_token({
"uuid": token_uuid,
"kind": kind,
"decky_name": req.decky_name,
"blob_uuid": req.blob_uuid,
"instrumenter": instrumenter_name,
"generator": req.generator,
"placement_path": placement_path,
"callback_token": slug,
"secret_seed": slug,
"created_by": admin.get("uuid", "unknown"),
"state": "planted",
})
await planter.plant(req.decky_name, artifact, token_uuid=token_uuid, repo=repo)
row = await repo.get_canary_token(token_uuid)
return _row_to_response(row)
# ---------------------------------------------------------- list / detail
@router.get(
"",
response_model=CanaryTokensResponse,
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
},
)
async def api_list_tokens(
decky_name: str | None = Query(default=None),
state: str | None = Query(default=None),
kind: str | None = Query(default=None),
viewer: dict = Depends(require_viewer),
) -> CanaryTokensResponse:
rows = await repo.list_canary_tokens(
decky_name=decky_name, state=state, kind=kind,
)
return CanaryTokensResponse(
tokens=[_row_to_response(r) for r in rows],
total=len(rows),
)
@router.get(
"/{uuid}",
response_model=CanaryTokenResponse,
responses={
404: {"description": "Token not found"},
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
},
)
async def api_get_token(
uuid: str,
viewer: dict = Depends(require_viewer),
) -> CanaryTokenResponse:
row = await repo.get_canary_token(uuid)
if row is None:
raise HTTPException(status_code=404, detail="token not found")
return _row_to_response(row)
# ---------------------------------------------------------- preview
@router.get(
"/{uuid}/preview",
response_class=Response,
responses={
200: {"description": "Instrumented bytes (raw)"},
404: {"description": "Token not found"},
409: {"description": "Token has no preview-able bytes (passive aws_creds, etc.)"},
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
},
)
async def api_preview_token(
uuid: str,
admin: dict = Depends(require_admin),
) -> Response:
"""Return the instrumented bytes the planter dropped on the decky.
Re-derived deterministically from the row's ``secret_seed`` —
we don't store the rendered bytes server-side. Lets operators
diff-check what we wrote without ``docker exec``-ing into the
container.
"""
row = await repo.get_canary_token(uuid)
if row is None:
raise HTTPException(status_code=404, detail="token not found")
ctx = CanaryContext(
callback_token=row["callback_token"],
http_base=_http_base(),
dns_zone=_dns_zone(),
)
if row["generator"]:
artifact = get_generator(row["generator"]).generate(ctx)
elif row["blob_uuid"] and row["instrumenter"]:
blob = await repo.get_canary_blob(row["blob_uuid"])
if blob is None:
raise HTTPException(
status_code=409,
detail="blob has been deleted; preview unavailable",
)
try:
blob_bytes = storage.read_blob(blob["sha256"])
except FileNotFoundError as e:
raise HTTPException(
status_code=409,
detail="blob bytes missing on disk",
) from e
ins = get_instrumenter(row["instrumenter"])
try:
artifact = ins.instrument(
blob_bytes, ctx, target_path=row["placement_path"],
)
except InstrumenterRejectedError as e:
raise HTTPException(status_code=409, detail=str(e)) from e
else:
raise HTTPException(
status_code=409,
detail="token has neither generator nor instrumenter — nothing to preview",
)
return Response(content=artifact.content, media_type="application/octet-stream")
# ---------------------------------------------------------- triggers
@router.get(
"/{uuid}/triggers",
response_model=CanaryTriggersResponse,
responses={
404: {"description": "Token not found"},
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
},
)
async def api_list_triggers(
uuid: str,
limit: int = Query(default=100, ge=1, le=500),
offset: int = Query(default=0, ge=0),
viewer: dict = Depends(require_viewer),
) -> CanaryTriggersResponse:
row = await repo.get_canary_token(uuid)
if row is None:
raise HTTPException(status_code=404, detail="token not found")
rows = await repo.list_canary_triggers(uuid, limit=limit, offset=offset)
return CanaryTriggersResponse(
triggers=[_trigger_row_to_response(r) for r in rows],
total=len(rows),
)
# ---------------------------------------------------------- revoke
@router.delete(
"/{uuid}",
response_model=MessageResponse,
responses={
404: {"description": "Token not found"},
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
},
)
async def api_revoke_token(
uuid: str,
admin: dict = Depends(require_admin),
) -> MessageResponse:
row = await repo.get_canary_token(uuid)
if row is None:
raise HTTPException(status_code=404, detail="token not found")
await planter.revoke(
row["decky_name"], row["placement_path"],
token_uuid=uuid, repo=repo,
)
return MessageResponse(message="ok")

0
decnet/web/router/config/__init__.py Normal file
View File

124
decnet/web/router/config/api_get_config.py Normal file
View File

@@ -0,0 +1,124 @@
import asyncio
import time
from typing import Any, Optional
from fastapi import APIRouter, Depends
from decnet.env import DECNET_DEVELOPER
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import require_viewer, repo
from decnet.web.db.models import UserResponse
router = APIRouter()
_DEFAULT_DEPLOYMENT_LIMIT = 10
_DEFAULT_MUTATION_INTERVAL = "30m"
# Cache config_limits / config_globals reads — these change on rare admin
# writes but get polled constantly by the UI and locust.
_STATE_TTL = 5.0
_state_cache: dict[str, tuple[Optional[dict[str, Any]], float]] = {}
_state_locks: dict[str, asyncio.Lock] = {}
# Admin branch fetched repo.list_users() on every /config call — cache 5s,
# invalidate on user create/update/delete so the admin UI stays consistent.
_USERS_TTL = 5.0
_users_cache: tuple[Optional[list[dict[str, Any]]], float] = (None, 0.0)
_users_lock: Optional[asyncio.Lock] = None
def _reset_state_cache() -> None:
"""Reset cached config state — used by tests."""
global _users_cache, _users_lock
_state_cache.clear()
# Drop any locks bound to the previous event loop — reusing one from
# a dead loop deadlocks the next test.
_state_locks.clear()
_users_cache = (None, 0.0)
_users_lock = None
def invalidate_list_users_cache() -> None:
global _users_cache
_users_cache = (None, 0.0)
async def _get_list_users_cached() -> list[dict[str, Any]]:
global _users_cache, _users_lock
value, ts = _users_cache
now = time.monotonic()
if value is not None and now - ts < _USERS_TTL:
return value
if _users_lock is None:
_users_lock = asyncio.Lock()
async with _users_lock:
value, ts = _users_cache
now = time.monotonic()
if value is not None and now - ts < _USERS_TTL:
return value
value = await repo.list_users()
_users_cache = (value, time.monotonic())
return value
async def _get_state_cached(name: str) -> Optional[dict[str, Any]]:
entry = _state_cache.get(name)
now = time.monotonic()
if entry is not None and now - entry[1] < _STATE_TTL:
return entry[0]
lock = _state_locks.setdefault(name, asyncio.Lock())
async with lock:
entry = _state_cache.get(name)
now = time.monotonic()
if entry is not None and now - entry[1] < _STATE_TTL:
return entry[0]
value = await repo.get_state(name)
_state_cache[name] = (value, time.monotonic())
return value
@router.get(
"/config",
tags=["Configuration"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
},
)
@_traced("api.get_config")
async def api_get_config(user: dict = Depends(require_viewer)) -> dict:
limits_state = await _get_state_cached("config_limits")
globals_state = await _get_state_cached("config_globals")
deployment_limit = (
limits_state.get("deployment_limit", _DEFAULT_DEPLOYMENT_LIMIT)
if limits_state
else _DEFAULT_DEPLOYMENT_LIMIT
)
global_mutation_interval = (
globals_state.get("global_mutation_interval", _DEFAULT_MUTATION_INTERVAL)
if globals_state
else _DEFAULT_MUTATION_INTERVAL
)
base = {
"role": user["role"],
"deployment_limit": deployment_limit,
"global_mutation_interval": global_mutation_interval,
}
if user["role"] == "admin":
all_users = await _get_list_users_cached()
base["users"] = [
UserResponse(
uuid=u["uuid"],
username=u["username"],
role=u["role"],
must_change_password=u["must_change_password"],
).model_dump()
for u in all_users
]
if DECNET_DEVELOPER:
base["developer_mode"] = True
return base

144
decnet/web/router/config/api_manage_users.py Normal file
View File

@@ -0,0 +1,144 @@
import uuid as _uuid
from fastapi import APIRouter, Depends, HTTPException
from decnet.telemetry import traced as _traced
from decnet.web.auth import ahash_password
from decnet.web.dependencies import require_admin, invalidate_user_cache, repo
from decnet.web.router.config.api_get_config import invalidate_list_users_cache
from decnet.web.db.models import (
CreateUserRequest,
MessageResponse,
ResetUserPasswordRequest,
UpdateUserRoleRequest,
UserResponse,
)
router = APIRouter()
@router.post(
"/config/users",
tags=["Configuration"],
response_model=UserResponse,
responses={
400: {"description": "Bad Request (e.g. malformed JSON)"},
401: {"description": "Could not validate credentials"},
403: {"description": "Admin access required"},
409: {"description": "Username already exists"},
422: {"description": "Validation error"},
},
)
@_traced("api.create_user")
async def api_create_user(
req: CreateUserRequest,
admin: dict = Depends(require_admin),
) -> UserResponse:
existing = await repo.get_user_by_username(req.username)
if existing:
raise HTTPException(status_code=409, detail="Username already exists")
user_uuid = str(_uuid.uuid4())
await repo.create_user({
"uuid": user_uuid,
"username": req.username,
"password_hash": await ahash_password(req.password),
"role": req.role,
"must_change_password": True, # nosec B105 — not a password
})
invalidate_list_users_cache()
return UserResponse(
uuid=user_uuid,
username=req.username,
role=req.role,
must_change_password=True,
)
@router.delete(
"/config/users/{user_uuid}",
tags=["Configuration"],
response_model=MessageResponse,
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Admin access required / cannot delete self"},
404: {"description": "User not found"},
},
)
@_traced("api.delete_user")
async def api_delete_user(
user_uuid: str,
admin: dict = Depends(require_admin),
) -> dict[str, str]:
if user_uuid == admin["uuid"]:
raise HTTPException(status_code=403, detail="Cannot delete your own account")
deleted = await repo.delete_user(user_uuid)
if not deleted:
raise HTTPException(status_code=404, detail="User not found")
invalidate_user_cache(user_uuid)
invalidate_list_users_cache()
return {"message": "User deleted"}
@router.put(
"/config/users/{user_uuid}/role",
tags=["Configuration"],
response_model=MessageResponse,
responses={
400: {"description": "Bad Request (e.g. malformed JSON)"},
401: {"description": "Could not validate credentials"},
403: {"description": "Admin access required / cannot change own role"},
404: {"description": "User not found"},
422: {"description": "Validation error"},
},
)
@_traced("api.update_user_role")
async def api_update_user_role(
user_uuid: str,
req: UpdateUserRoleRequest,
admin: dict = Depends(require_admin),
) -> dict[str, str]:
if user_uuid == admin["uuid"]:
raise HTTPException(status_code=403, detail="Cannot change your own role")
target = await repo.get_user_by_uuid(user_uuid)
if not target:
raise HTTPException(status_code=404, detail="User not found")
await repo.update_user_role(user_uuid, req.role)
invalidate_user_cache(user_uuid)
invalidate_list_users_cache()
return {"message": "User role updated"}
@router.put(
"/config/users/{user_uuid}/reset-password",
tags=["Configuration"],
response_model=MessageResponse,
responses={
400: {"description": "Bad Request (e.g. malformed JSON)"},
401: {"description": "Could not validate credentials"},
403: {"description": "Admin access required"},
404: {"description": "User not found"},
422: {"description": "Validation error"},
},
)
@_traced("api.reset_user_password")
async def api_reset_user_password(
user_uuid: str,
req: ResetUserPasswordRequest,
admin: dict = Depends(require_admin),
) -> dict[str, str]:
target = await repo.get_user_by_uuid(user_uuid)
if not target:
raise HTTPException(status_code=404, detail="User not found")
await repo.update_user_password(
user_uuid,
await ahash_password(req.new_password),
must_change_password=True,
)
invalidate_user_cache(user_uuid)
invalidate_list_users_cache()
return {"message": "Password reset successfully"}

29
decnet/web/router/config/api_reinit.py Normal file
View File

@@ -0,0 +1,29 @@
from fastapi import APIRouter, Depends, HTTPException
from decnet.env import DECNET_DEVELOPER
from decnet.telemetry import traced as _traced
from decnet.web.db.models import PurgeResponse
from decnet.web.dependencies import require_admin, repo
router = APIRouter()
@router.delete(
"/config/reinit",
tags=["Configuration"],
response_model=PurgeResponse,
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Admin access required or developer mode not enabled"},
},
)
@_traced("api.reinit")
async def api_reinit(admin: dict = Depends(require_admin)) -> dict:
if not DECNET_DEVELOPER:
raise HTTPException(status_code=403, detail="Developer mode is not enabled")
counts = await repo.purge_logs_and_bounties()
return {
"message": "Data purged",
"deleted": counts,
}

50
decnet/web/router/config/api_update_config.py Normal file
View File

@@ -0,0 +1,50 @@
from fastapi import APIRouter, Depends
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import require_admin, repo
from decnet.web.db.models import DeploymentLimitRequest, GlobalMutationIntervalRequest, MessageResponse
router = APIRouter()
@router.put(
"/config/deployment-limit",
tags=["Configuration"],
response_model=MessageResponse,
responses={
400: {"description": "Bad Request (e.g. malformed JSON)"},
401: {"description": "Could not validate credentials"},
403: {"description": "Admin access required"},
422: {"description": "Validation error"},
},
)
@_traced("api.update_deployment_limit")
async def api_update_deployment_limit(
req: DeploymentLimitRequest,
admin: dict = Depends(require_admin),
) -> dict[str, str]:
await repo.set_state("config_limits", {"deployment_limit": req.deployment_limit})
return {"message": "Deployment limit updated"}
@router.put(
"/config/global-mutation-interval",
tags=["Configuration"],
response_model=MessageResponse,
responses={
400: {"description": "Bad Request (e.g. malformed JSON)"},
401: {"description": "Could not validate credentials"},
403: {"description": "Admin access required"},
422: {"description": "Validation error"},
},
)
@_traced("api.update_global_mutation_interval")
async def api_update_global_mutation_interval(
req: GlobalMutationIntervalRequest,
admin: dict = Depends(require_admin),
) -> dict[str, str]:
await repo.set_state(
"config_globals",
{"global_mutation_interval": req.global_mutation_interval},
)
return {"message": "Global mutation interval updated"}

0
decnet/web/router/credential_reuse/__init__.py Normal file
View File

74
decnet/web/router/credential_reuse/api_get_credential_reuse.py Normal file
View File

@@ -0,0 +1,74 @@
from typing import Any, Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import require_viewer, repo
from decnet.web.db.models import CredentialReuseResponse
router = APIRouter()
@router.get(
"/credential-reuse",
response_model=CredentialReuseResponse,
tags=["Credentials"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
422: {"description": "Validation error"},
},
)
@_traced("api.list_credential_reuse")
async def list_credential_reuse(
limit: int = Query(50, ge=1, le=1000),
offset: int = Query(0, ge=0, le=2147483647),
min_target_count: int = Query(2, ge=2, le=2147483647),
secret_kind: Optional[str] = None,
user: dict = Depends(require_viewer),
) -> dict[str, Any]:
"""Paged list of credential-reuse findings ordered by target_count desc.
Each row collapses every Credential capture sharing the same secret
+ principal across distinct (decky, service) pairs into a single
finding with the union of attacker UUIDs/IPs and reach.
"""
def _norm(v: Optional[str]) -> Optional[str]:
if v in (None, "null", "NULL", "undefined", ""):
return None
return v
kind = _norm(secret_kind)
total, data = await repo.list_credential_reuses(
limit=limit,
offset=offset,
min_target_count=min_target_count,
secret_kind=kind,
)
return {
"total": total,
"limit": limit,
"offset": offset,
"data": data,
}
@router.get(
"/credential-reuse/{reuse_id}",
tags=["Credentials"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
404: {"description": "CredentialReuse row not found"},
},
)
@_traced("api.get_credential_reuse")
async def get_credential_reuse(
reuse_id: str,
user: dict = Depends(require_viewer),
) -> dict[str, Any]:
"""One credential-reuse finding by UUID, or 404."""
row = await repo.get_credential_reuse_by_id(reuse_id)
if row is None:
raise HTTPException(status_code=404, detail="credential_reuse not found")
return row

0
decnet/web/router/credentials/__init__.py Normal file
View File

103
decnet/web/router/credentials/api_get_credentials.py Normal file
View File

@@ -0,0 +1,103 @@
import asyncio
import time
from typing import Any, Optional
from fastapi import APIRouter, Depends, Query
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import require_viewer, repo
from decnet.web.db.models import CredentialsResponse
router = APIRouter()
# Mirror the Bounty cache pattern: the dashboard hits the unfiltered
# default page constantly. Filtered requests bypass — staleness matters
# when an operator is searching for a specific principal/IP.
_CRED_TTL = 5.0
_DEFAULT_LIMIT = 50
_DEFAULT_OFFSET = 0
_cred_cache: tuple[Optional[dict[str, Any]], float] = (None, 0.0)
_cred_lock: Optional[asyncio.Lock] = None
def _reset_credentials_cache() -> None:
global _cred_cache, _cred_lock
_cred_cache = (None, 0.0)
_cred_lock = None
async def _get_credentials_default_cached() -> dict[str, Any]:
global _cred_cache, _cred_lock
value, ts = _cred_cache
now = time.monotonic()
if value is not None and now - ts < _CRED_TTL:
return value
if _cred_lock is None:
_cred_lock = asyncio.Lock()
async with _cred_lock:
value, ts = _cred_cache
now = time.monotonic()
if value is not None and now - ts < _CRED_TTL:
return value
_data = await repo.get_credentials(
limit=_DEFAULT_LIMIT, offset=_DEFAULT_OFFSET,
search=None, service=None, attacker_ip=None,
)
_total = await repo.get_total_credentials(
search=None, service=None, attacker_ip=None,
)
value = {"total": _total, "limit": _DEFAULT_LIMIT, "offset": _DEFAULT_OFFSET, "data": _data}
_cred_cache = (value, time.monotonic())
return value
@router.get(
"/credentials",
response_model=CredentialsResponse,
tags=["Credentials"],
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
422: {"description": "Validation error"},
},
)
@_traced("api.get_credentials")
async def get_credentials(
limit: int = Query(50, ge=1, le=1000),
offset: int = Query(0, ge=0, le=2147483647),
search: Optional[str] = None,
service: Optional[str] = None,
attacker_ip: Optional[str] = None,
user: dict = Depends(require_viewer),
) -> dict[str, Any]:
"""Retrieve captured credentials (deduped by attacker/decky/service/secret)."""
def _norm(v: Optional[str]) -> Optional[str]:
if v in (None, "null", "NULL", "undefined", ""):
return None
return v
s = _norm(search)
svc = _norm(service)
aip = _norm(attacker_ip)
if (
s is None
and svc is None
and aip is None
and limit == _DEFAULT_LIMIT
and offset == _DEFAULT_OFFSET
):
return await _get_credentials_default_cached()
_data = await repo.get_credentials(
limit=limit, offset=offset, search=s, service=svc, attacker_ip=aip,
)
_total = await repo.get_total_credentials(
search=s, service=svc, attacker_ip=aip,
)
return {
"total": _total,
"limit": limit,
"offset": offset,
"data": _data,
}

129
decnet/web/router/fleet/api_deploy_deckies.py
View File

@@ -1,14 +1,18 @@
import logging
import os
from fastapi import APIRouter, Depends, HTTPException
from decnet.config import DEFAULT_MUTATE_INTERVAL, DecnetConfig, _ROOT, log
from decnet.logging import get_logger
from decnet.telemetry import traced as _traced
from decnet.config import DEFAULT_MUTATE_INTERVAL, DecnetConfig, _ROOT
from decnet.engine import deploy as _deploy
from decnet.ini_loader import load_ini_from_string
from decnet.network import detect_interface, detect_subnet, get_host_ip
from decnet.web.dependencies import get_current_user, repo
from decnet.web.db.models import DeployIniRequest
from decnet.web.dependencies import require_admin, repo
from decnet.web.db.models import DeployIniRequest, DeployResponse
from decnet.web.router.swarm.api_deploy_swarm import dispatch_decnet_config
log = get_logger("api")
router = APIRouter()
@@ -16,15 +20,19 @@ router = APIRouter()
@router.post(
"/deckies/deploy",
tags=["Fleet Management"],
response_model=DeployResponse,
responses={
400: {"description": "Bad Request (e.g. malformed JSON)"},
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
409: {"description": "Configuration conflict (e.g. invalid IP allocation or network mismatch)"},
422: {"description": "Invalid INI config or schema validation error"},
500: {"description": "Deployment failed"}
500: {"description": "Deployment failed"},
502: {"description": "Partial swarm deploy failure — one or more worker hosts returned an error"},
}
)
async def api_deploy_deckies(req: DeployIniRequest, current_user: str = Depends(get_current_user)) -> dict[str, str]:
@_traced("api.deploy_deckies")
async def api_deploy_deckies(req: DeployIniRequest, admin: dict = Depends(require_admin)) -> dict[str, str]:
from decnet.fleet import build_deckies_from_ini
try:
@@ -38,16 +46,20 @@ async def api_deploy_deckies(req: DeployIniRequest, current_user: str = Depends(
state_dict = await repo.get_state("deployment")
ingest_log_file = os.environ.get("DECNET_INGEST_LOG_FILE")
config: DecnetConfig | None = None
if state_dict:
config = DecnetConfig(**state_dict["config"])
subnet_cidr = ini.subnet or config.subnet
gateway = ini.gateway or config.gateway
host_ip = get_host_ip(config.interface)
iface = config.interface
host_ip = get_host_ip(iface)
# Always sync config log_file with current API ingestion target
if ingest_log_file:
config.log_file = ingest_log_file
else:
# If no state exists, we need to infer network details from the INI or the host.
# No state yet — infer network details from the INI or the host. We
# defer instantiating DecnetConfig until after build_deckies_from_ini
# because DecnetConfig.deckies has min_length=1.
try:
iface = ini.interface or detect_interface()
subnet_cidr, gateway = ini.subnet, ini.gateway
@@ -62,16 +74,6 @@ async def api_deploy_deckies(req: DeployIniRequest, current_user: str = Depends(
detail=f"Network configuration conflict: {e}. "
"Add a [general] section with interface=, net=, and gw= to the INI."
)
config = DecnetConfig(
mode="unihost",
interface=iface,
subnet=subnet_cidr,
gateway=gateway,
deckies=[],
log_file=ingest_log_file,
ipvlan=False,
mutate_interval=ini.mutate_interval or DEFAULT_MUTATE_INTERVAL
)
try:
new_decky_configs = build_deckies_from_ini(
@@ -81,26 +83,99 @@ async def api_deploy_deckies(req: DeployIniRequest, current_user: str = Depends(
log.debug("deploy: build_deckies_from_ini rejected input: %s", e)
raise HTTPException(status_code=409, detail=str(e))
# Merge deckies
existing_deckies_map = {d.name: d for d in config.deckies}
for new_decky in new_decky_configs:
existing_deckies_map[new_decky.name] = new_decky
if config is None:
config = DecnetConfig(
mode="unihost",
interface=iface,
subnet=subnet_cidr,
gateway=gateway,
deckies=new_decky_configs,
log_file=ingest_log_file,
ipvlan=False,
mutate_interval=ini.mutate_interval or DEFAULT_MUTATE_INTERVAL,
)
config.deckies = list(existing_deckies_map.values())
# The INI is the source of truth for *which* deckies exist this deploy.
# The old "merge with prior state" behaviour meant submitting `[decky1]`
# after a 3-decky run silently redeployed decky2/decky3 too — and then
# collided on their stale IPs ("Address already in use"). Full replace
# matches what the operator sees in the submitted config.
config.deckies = list(new_decky_configs)
# We call deploy(config) which regenerates docker-compose and runs `up -d --remove-orphans`.
limits_state = await repo.get_state("config_limits")
deployment_limit = limits_state.get("deployment_limit", 10) if limits_state else 10
if len(config.deckies) > deployment_limit:
raise HTTPException(
status_code=409,
detail=f"Deployment would result in {len(config.deckies)} deckies, "
f"exceeding the configured limit of {deployment_limit}",
)
# Auto-mode: if we're a master with at least one enrolled/active SWARM
# host, shard the deckies across those workers instead of spawning docker
# containers on the master itself. Round-robin assignment over deckies
# that don't already carry a host_uuid (state from a prior swarm deploy
# keeps its original assignment).
swarm_hosts: list[dict] = []
if os.environ.get("DECNET_MODE", "master").lower() == "master":
swarm_hosts = [
h for h in await repo.list_swarm_hosts()
if h.get("status") in ("active", "enrolled") and h.get("address")
]
if swarm_hosts:
# Carry-over from a prior deployment may reference a host_uuid that's
# since been decommissioned / re-enrolled at a new uuid. Drop any
# assignment that isn't in the currently-reachable set, then round-
# robin-fill the blanks — otherwise dispatch 404s on a dead uuid.
live_uuids = {h["uuid"] for h in swarm_hosts}
for d in config.deckies:
if d.host_uuid and d.host_uuid not in live_uuids:
d.host_uuid = None
unassigned = [d for d in config.deckies if not d.host_uuid]
for i, d in enumerate(unassigned):
d.host_uuid = swarm_hosts[i % len(swarm_hosts)]["uuid"]
config = config.model_copy(update={"mode": "swarm"})
try:
result = await dispatch_decnet_config(config, repo, dry_run=False, no_cache=False)
except HTTPException:
raise
except Exception as e:
log.exception("swarm-auto deploy dispatch failed: %s", e)
raise HTTPException(status_code=500, detail="Swarm dispatch failed. Check server logs.")
await repo.set_state("deployment", {
"config": config.model_dump(),
"compose_path": state_dict["compose_path"] if state_dict else "",
})
failed = [r for r in result.results if not r.ok]
if failed:
detail = "; ".join(f"{r.host_name}: {r.detail}" for r in failed)
raise HTTPException(status_code=502, detail=f"Partial swarm deploy failure — {detail}")
return {
"message": f"Deckies deployed across {len(result.results)} swarm host(s)",
"mode": "swarm",
}
# Unihost path — docker-compose on the master itself.
# NB: the JSON state file (decnet-state.json) and fleet_deckies DB rows
# are both written *inside* _deploy(config) — engine.deployer is the
# single shared sink for every fleet-creation path (CLI deploy, this
# unihost API path, and per-worker SWARM agent deploys). Do not
# duplicate save_state / fleet upserts here.
try:
if os.environ.get("DECNET_CONTRACT_TEST") != "true":
_deploy(config)
# Persist new state to DB
new_state_payload = {
"config": config.model_dump(),
"compose_path": str(_ROOT / "docker-compose.yml") if not state_dict else state_dict["compose_path"]
}
await repo.set_state("deployment", new_state_payload)
except Exception as e:
logging.getLogger("decnet.web.api").exception("Deployment failed: %s", e)
log.exception("Deployment failed: %s", e)
raise HTTPException(status_code=500, detail="Deployment failed. Check server logs for details.")
return {"message": "Deckies deployed successfully"}
return {"message": "Deckies deployed successfully", "mode": "unihost"}

45
decnet/web/router/fleet/api_get_deckies.py
View File

@@ -1,13 +1,48 @@
from typing import Any
import asyncio
import time
from typing import Any, Optional
from fastapi import APIRouter, Depends
from decnet.web.dependencies import get_current_user, repo
from decnet.telemetry import traced as _traced
from decnet.web.dependencies import require_viewer, repo
router = APIRouter()
# /deckies is full fleet inventory — polled by the UI and under locust.
# Fleet state changes on deploy/teardown (seconds to minutes); a 5s window
# collapses the read storm into one DB hit.
_DECKIES_TTL = 5.0
_deckies_cache: tuple[Optional[list[dict[str, Any]]], float] = (None, 0.0)
_deckies_lock: Optional[asyncio.Lock] = None
def _reset_deckies_cache() -> None:
global _deckies_cache, _deckies_lock
_deckies_cache = (None, 0.0)
_deckies_lock = None
async def _get_deckies_cached() -> list[dict[str, Any]]:
global _deckies_cache, _deckies_lock
value, ts = _deckies_cache
now = time.monotonic()
if value is not None and now - ts < _DECKIES_TTL:
return value
if _deckies_lock is None:
_deckies_lock = asyncio.Lock()
async with _deckies_lock:
value, ts = _deckies_cache
now = time.monotonic()
if value is not None and now - ts < _DECKIES_TTL:
return value
value = await repo.get_deckies()
_deckies_cache = (value, time.monotonic())
return value
@router.get("/deckies", tags=["Fleet Management"],
responses={401: {"description": "Could not validate credentials"}, 422: {"description": "Validation error"}},)
async def get_deckies(current_user: str = Depends(get_current_user)) -> list[dict[str, Any]]:
return await repo.get_deckies()
responses={401: {"description": "Could not validate credentials"}, 403: {"description": "Insufficient permissions"}, 422: {"description": "Validation error"}},)
@_traced("api.get_deckies")
async def get_deckies(user: dict = Depends(require_viewer)) -> list[dict[str, Any]]:
return await _get_deckies_cached()

15
decnet/web/router/fleet/api_mutate_decky.py
View File

@@ -1,8 +1,10 @@
import os
from fastapi import APIRouter, Depends, HTTPException, Path
from decnet.telemetry import traced as _traced
from decnet.mutator import mutate_decky
from decnet.web.dependencies import get_current_user, repo
from decnet.web.db.models import MessageResponse
from decnet.web.dependencies import require_admin, repo
router = APIRouter()
@@ -10,11 +12,18 @@ router = APIRouter()
@router.post(
"/deckies/{decky_name}/mutate",
tags=["Fleet Management"],
responses={401: {"description": "Could not validate credentials"}, 404: {"description": "Decky not found"}}
response_model=MessageResponse,
responses={
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
404: {"description": "Decky not found"},
422: {"description": "Path parameter validation error (decky_name must match ^[a-z0-9\\-]{1,64}$)"},
}
)
@_traced("api.mutate_decky")
async def api_mutate_decky(
decky_name: str = Path(..., pattern=r"^[a-z0-9\-]{1,64}$"),
current_user: str = Depends(get_current_user),
admin: dict = Depends(require_admin),
) -> dict[str, str]:
if os.environ.get("DECNET_CONTRACT_TEST") == "true":
return {"message": f"Successfully mutated {decky_name} (Contract Test Mock)"}

10
decnet/web/router/fleet/api_mutate_interval.py
View File

@@ -1,8 +1,9 @@
from fastapi import APIRouter, Depends, HTTPException
from decnet.telemetry import traced as _traced
from decnet.config import DecnetConfig
from decnet.web.dependencies import get_current_user, repo
from decnet.web.db.models import MutateIntervalRequest
from decnet.web.dependencies import require_admin, repo
from decnet.web.db.models import MessageResponse, MutateIntervalRequest
router = APIRouter()
@@ -16,14 +17,17 @@ def _parse_duration(s: str) -> int:
@router.put("/deckies/{decky_name}/mutate-interval", tags=["Fleet Management"],
response_model=MessageResponse,
responses={
400: {"description": "Bad Request (e.g. malformed JSON)"},
401: {"description": "Could not validate credentials"},
403: {"description": "Insufficient permissions"},
404: {"description": "No active deployment or decky not found"},
422: {"description": "Validation error"}
},
)
async def api_update_mutate_interval(decky_name: str, req: MutateIntervalRequest, current_user: str = Depends(get_current_user)) -> dict[str, str]:
@_traced("api.update_mutate_interval")
async def api_update_mutate_interval(decky_name: str, req: MutateIntervalRequest, admin: dict = Depends(require_admin)) -> dict[str, str]:
state_dict = await repo.get_state("deployment")
if not state_dict:
raise HTTPException(status_code=404, detail="No active deployment")

0
decnet/web/router/health/__init__.py Normal file
View File

Some files were not shown because too many files have changed in this diff Show More