anti/DECNET
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: testing → main (reconcile 2-week divergence)

Browse Source
This commit is contained in:
anti
2026-04-28 18:36:00 -04:00
parent 499836c9e4
commit 862e4dbb31
1235 changed files with 160255 additions and 7996 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
decnet/web/db/factory.py
View File

@@ -1,18 +1,33 @@
"""
Repository factory — selects a :class:`BaseRepository` implementation based on
``DECNET_DB_TYPE`` (``sqlite`` or ``mysql``).
"""
from __future__ import annotations
import os
from typing import Any
from decnet.env import os
from decnet.web.db.repository import BaseRepository
def get_repository(**kwargs: Any) -> BaseRepository:
"""Factory function to instantiate the correct repository implementation based on environment."""
"""Instantiate the repository implementation selected by ``DECNET_DB_TYPE``.
Keyword arguments are forwarded to the concrete implementation:
* SQLite accepts ``db_path``.
* MySQL accepts ``url`` and engine tuning knobs (``pool_size``, …).
"""
db_type = os.environ.get("DECNET_DB_TYPE", "sqlite").lower()
if db_type == "sqlite":
from decnet.web.db.sqlite.repository import SQLiteRepository
return SQLiteRepository(**kwargs)
repo = SQLiteRepository(**kwargs)
elif db_type == "mysql":
# Placeholder for future implementation
# from decnet.web.db.mysql.repository import MySQLRepository
# return MySQLRepository()
raise NotImplementedError("MySQL support is planned but not yet implemented.")
from decnet.web.db.mysql.repository import MySQLRepository
repo = MySQLRepository(**kwargs)
else:
raise ValueError(f"Unsupported database type: {db_type}")
from decnet.telemetry import wrap_repository
return wrap_repository(repo)

95
decnet/web/db/models.py
View File

@@ -1,95 +0,0 @@
from datetime import datetime, timezone
from typing import Optional, Any, List, Annotated
from sqlmodel import SQLModel, Field
from pydantic import BaseModel, ConfigDict, Field as PydanticField, BeforeValidator
from decnet.models import IniContent
def _normalize_null(v: Any) -> Any:
if isinstance(v, str) and v.lower() in ("null", "undefined", ""):
return None
return v
NullableDatetime = Annotated[Optional[datetime], BeforeValidator(_normalize_null)]
NullableString = Annotated[Optional[str], BeforeValidator(_normalize_null)]
# --- Database Tables (SQLModel) ---
class User(SQLModel, table=True):
__tablename__ = "users"
uuid: str = Field(primary_key=True)
username: str = Field(index=True, unique=True)
password_hash: str
role: str = Field(default="viewer")
must_change_password: bool = Field(default=False)
class Log(SQLModel, table=True):
__tablename__ = "logs"
id: Optional[int] = Field(default=None, primary_key=True)
timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc), index=True)
decky: str = Field(index=True)
service: str = Field(index=True)
event_type: str = Field(index=True)
attacker_ip: str = Field(index=True)
raw_line: str
fields: str
msg: Optional[str] = None
class Bounty(SQLModel, table=True):
__tablename__ = "bounty"
id: Optional[int] = Field(default=None, primary_key=True)
timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc), index=True)
decky: str = Field(index=True)
service: str = Field(index=True)
attacker_ip: str = Field(index=True)
bounty_type: str = Field(index=True)
payload: str
class State(SQLModel, table=True):
__tablename__ = "state"
key: str = Field(primary_key=True)
value: str # Stores JSON serialized DecnetConfig or other state blobs
# --- API Request/Response Models (Pydantic) ---
class Token(BaseModel):
access_token: str
token_type: str
must_change_password: bool = False
class LoginRequest(BaseModel):
username: str
password: str = PydanticField(..., max_length=72)
class ChangePasswordRequest(BaseModel):
old_password: str = PydanticField(..., max_length=72)
new_password: str = PydanticField(..., max_length=72)
class LogsResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class BountyResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class StatsResponse(BaseModel):
total_logs: int
unique_attackers: int
active_deckies: int
deployed_deckies: int
class MutateIntervalRequest(BaseModel):
# Human-readable duration: <number><unit> where unit is m(inutes), d(ays), M(onths), y/Y(ears).
# Minimum granularity is 1 minute. Seconds are not accepted.
mutate_interval: Optional[str] = PydanticField(None, pattern=r"^[1-9]\d*[mdMyY]$")
class DeployIniRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
# This field now enforces strict INI structure during Pydantic initialization.
# The OpenAPI schema correctly shows it as a required string.
ini_content: IniContent = PydanticField(..., description="A valid INI formatted string")

319
decnet/web/db/models/__init__.py Normal file
View File

@@ -0,0 +1,319 @@
"""
Database tables (SQLModel) and HTTP request/response shapes (Pydantic).
Split into topical modules for readability, but every symbol is re-exported
from this package so ``from decnet.web.db.models import X`` keeps working
everywhere — no importer needs to know which submodule a class lives in.
"""
from ._base import (
NullableDatetime,
NullableString,
_BIG_TEXT,
_normalize_null,
)
from .common import (
MessageResponse,
)
from .canary import (
CanaryBlob,
CanaryBlobResponse,
CanaryBlobsResponse,
CanaryKind,
CanaryState,
CanaryToken,
CanaryTokenCreateRequest,
CanaryTokenResponse,
CanaryTokensResponse,
CanaryTrigger,
CanaryTriggerResponse,
CanaryTriggersResponse,
)
from .auth import (
AdminConfigResponse,
ChangePasswordRequest,
ConfigResponse,
CreateUserRequest,
DeploymentLimitRequest,
GlobalMutationIntervalRequest,
LoginRequest,
ResetUserPasswordRequest,
Token,
UpdateUserRoleRequest,
User,
UserResponse,
)
from .attackers import (
Attacker,
AttackerBehavior,
AttackerIdentity,
AttackersResponse,
SessionProfile,
SmtpTarget,
)
from .attacker_intel import (
AttackerIntel,
)
from .campaigns import (
Campaign,
CampaignsResponse,
)
from .deploy import (
DeployIniRequest,
DeployResponse,
MutateIntervalRequest,
PurgeResponse,
)
from .fleet import (
LOCAL_HOST_SENTINEL,
FleetDecky,
)
from .health import (
ComponentHealth,
HealthResponse,
)
from .orchestrator import (
OrchestratorEmail,
OrchestratorEmailsResponse,
OrchestratorEvent,
OrchestratorEventsResponse,
)
from .realism import (
RealismConfig,
SyntheticFile,
SyntheticFilesResponse,
)
from .logs import (
Bounty,
BountyResponse,
Credential,
CredentialReuse,
CredentialReuseResponse,
CredentialsResponse,
Log,
LogsResponse,
State,
StatsResponse,
)
from .swarm import (
DeckyShard,
DeckyShardView,
SwarmCheckResponse,
SwarmDeployRequest,
SwarmDeployResponse,
SwarmEnrolledBundle,
SwarmEnrollRequest,
SwarmHost,
SwarmHostHealth,
SwarmHostResult,
SwarmHostView,
SwarmTeardownRequest,
SwarmUpdaterBundle,
)
from .topology import (
LAN,
ArchetypeCatalogResponse,
ArchetypeEntry,
DeckyCreateRequest,
DeckyRow,
DeckyUpdateRequest,
DeployAcceptedResponse,
EdgeCreateRequest,
EdgeRow,
LANCreateRequest,
LANRow,
LANUpdateRequest,
MutationEnqueueRequest,
MutationEnqueueResponse,
MutationRow,
NextIPResponse,
NextSubnetResponse,
NotEditableResponse,
ReapReportResponse,
ServiceCatalogResponse,
Topology,
TopologyDecky,
TopologyDetail,
TopologyEdge,
TopologyGenerateRequest,
TopologyListResponse,
TopologyMutation,
TopologyStatusEvent,
TopologyStatusEventRow,
TopologySummary,
ValidationErrorResponse,
ValidationIssueResponse,
VersionConflictResponse,
)
from .updater import (
HostReleaseInfo,
HostReleasesResponse,
PushUpdateRequest,
PushUpdateResponse,
PushUpdateResult,
RollbackRequest,
RollbackResponse,
)
from .webhooks import (
SimpleEvent,
WebhookCreateRequest,
WebhookCreateResponse,
WebhookResponse,
WebhookSubscription,
WebhookTestResponse,
WebhookUpdateRequest,
)
from .workers import (
StartAllResponse,
StartFailure,
WorkerControlResponse,
WorkersResponse,
WorkerStatus,
)
__all__ = [
# _base
"NullableDatetime",
"NullableString",
"_BIG_TEXT",
"_normalize_null",
# common
"MessageResponse",
# canary
"CanaryBlob",
"CanaryBlobResponse",
"CanaryBlobsResponse",
"CanaryKind",
"CanaryState",
"CanaryToken",
"CanaryTokenCreateRequest",
"CanaryTokenResponse",
"CanaryTokensResponse",
"CanaryTrigger",
"CanaryTriggerResponse",
"CanaryTriggersResponse",
# auth
"AdminConfigResponse",
"ChangePasswordRequest",
"ConfigResponse",
"CreateUserRequest",
"DeploymentLimitRequest",
"GlobalMutationIntervalRequest",
"LoginRequest",
"ResetUserPasswordRequest",
"Token",
"UpdateUserRoleRequest",
"User",
"UserResponse",
# attackers
"Attacker",
"AttackerBehavior",
"AttackerIdentity",
"AttackerIntel",
"AttackersResponse",
"SessionProfile",
"SmtpTarget",
# campaigns
"Campaign",
"CampaignsResponse",
# deploy
"DeployIniRequest",
"DeployResponse",
"MutateIntervalRequest",
"PurgeResponse",
# fleet
"LOCAL_HOST_SENTINEL",
"FleetDecky",
# health
"ComponentHealth",
"HealthResponse",
# orchestrator
"OrchestratorEmail",
"OrchestratorEmailsResponse",
"OrchestratorEvent",
"OrchestratorEventsResponse",
# realism
"RealismConfig",
"SyntheticFile",
"SyntheticFilesResponse",
# logs
"Bounty",
"BountyResponse",
"Credential",
"CredentialReuse",
"CredentialReuseResponse",
"CredentialsResponse",
"Log",
"LogsResponse",
"State",
"StatsResponse",
# swarm
"DeckyShard",
"DeckyShardView",
"SwarmCheckResponse",
"SwarmDeployRequest",
"SwarmDeployResponse",
"SwarmEnrolledBundle",
"SwarmEnrollRequest",
"SwarmHost",
"SwarmHostHealth",
"SwarmHostResult",
"SwarmHostView",
"SwarmTeardownRequest",
"SwarmUpdaterBundle",
# topology
"LAN",
"ArchetypeCatalogResponse",
"ArchetypeEntry",
"DeckyCreateRequest",
"DeckyRow",
"DeckyUpdateRequest",
"DeployAcceptedResponse",
"EdgeCreateRequest",
"EdgeRow",
"LANCreateRequest",
"LANRow",
"LANUpdateRequest",
"MutationEnqueueRequest",
"MutationEnqueueResponse",
"MutationRow",
"NextIPResponse",
"NextSubnetResponse",
"NotEditableResponse",
"ReapReportResponse",
"ServiceCatalogResponse",
"Topology",
"TopologyDecky",
"TopologyDetail",
"TopologyEdge",
"TopologyGenerateRequest",
"TopologyListResponse",
"TopologyMutation",
"TopologyStatusEvent",
"TopologyStatusEventRow",
"TopologySummary",
"ValidationErrorResponse",
"ValidationIssueResponse",
"VersionConflictResponse",
# updater
"HostReleaseInfo",
"HostReleasesResponse",
"PushUpdateRequest",
"PushUpdateResponse",
"PushUpdateResult",
"RollbackRequest",
"RollbackResponse",
# webhooks
"SimpleEvent",
"WebhookCreateRequest",
"WebhookCreateResponse",
"WebhookResponse",
"WebhookSubscription",
"WebhookTestResponse",
"WebhookUpdateRequest",
# workers
"StartAllResponse",
"StartFailure",
"WorkerControlResponse",
"WorkersResponse",
"WorkerStatus",
]

23
decnet/web/db/models/_base.py Normal file
View File

@@ -0,0 +1,23 @@
"""Shared column/validator helpers used across model domain modules."""
from datetime import datetime
from typing import Annotated, Any, Optional
from pydantic import BeforeValidator
from sqlalchemy import Text
from sqlalchemy.dialects.mysql import MEDIUMTEXT
# Use on columns that accumulate over an attacker's lifetime (commands,
# fingerprints, state blobs). TEXT on MySQL caps at 64 KiB; MEDIUMTEXT
# stretches to 16 MiB. SQLite has no fixed-width text types so Text()
# stays unchanged there.
_BIG_TEXT = Text().with_variant(MEDIUMTEXT(), "mysql")
def _normalize_null(v: Any) -> Any:
if isinstance(v, str) and v.lower() in ("null", "undefined", ""):
return None
return v
NullableDatetime = Annotated[Optional[datetime], BeforeValidator(_normalize_null)]
NullableString = Annotated[Optional[str], BeforeValidator(_normalize_null)]

93
decnet/web/db/models/attacker_intel.py Normal file
View File

@@ -0,0 +1,93 @@
"""Threat-intel enrichment row — one per attacker IP, TTL-cached."""
from datetime import datetime, timezone
from typing import Optional
from sqlalchemy import Column
from sqlmodel import Field, SQLModel
from ._base import _BIG_TEXT
class AttackerIntel(SQLModel, table=True):
"""Aggregated threat-intel verdict for a single attacker IP.
Populated by the ``decnet enrich`` worker, which queries multiple
free-tier intel providers (GreyNoise Community, AbuseIPDB,
abuse.ch Feodo Tracker + ThreatFox) and writes one row per
attacker IP. The row is TTL-cached via ``expires_at`` so re-firings
inside the cache window short-circuit before any HTTP egress.
Per-provider columns are nullable until each provider has answered;
the enrichment pass writes whichever providers succeeded and leaves
the rest unchanged on a partial failure.
``schema_version`` is committed to storage from day one — federation
gossip in v2/v3 requires cross-operator compatibility, and
retrofitting a version column after rows exist is painful. Mirrors
the rationale on :class:`SessionProfile`.
"""
__tablename__ = "attacker_intel"
uuid: str = Field(primary_key=True) # uuid.uuid4().hex, generated by writer
# Canonical key. One intel row per attacker UUID; FK guarantees no orphan
# rows when an attacker is deleted, and UNIQUE keeps upserts honest.
attacker_uuid: str = Field(
foreign_key="attackers.uuid",
unique=True,
index=True,
)
# DENORMALISED — NOT a key. The IP the worker queried providers with at
# write time. Useful for SIEM payloads and audit lookups; updated on every
# upsert if the attacker rotates IPs. Never use this column as a lookup
# key; ``attacker_uuid`` is the only canonical identifier here.
attacker_ip: str = Field(index=True)
schema_version: int = Field(default=1)
# ── GreyNoise Community ─────────────────────────────────────────────
# classification ∈ {"benign", "malicious", "suspicious", "unknown"}
greynoise_classification: Optional[str] = Field(default=None, max_length=32)
greynoise_raw: str = Field(
default="{}",
sa_column=Column("greynoise_raw", _BIG_TEXT, nullable=False, default="{}"),
)
greynoise_queried_at: Optional[datetime] = Field(default=None)
# ── AbuseIPDB ────────────────────────────────────────────────────────
# 0..100 abuse confidence score
abuseipdb_score: Optional[int] = Field(default=None)
abuseipdb_raw: str = Field(
default="{}",
sa_column=Column("abuseipdb_raw", _BIG_TEXT, nullable=False, default="{}"),
)
abuseipdb_queried_at: Optional[datetime] = Field(default=None)
# ── abuse.ch Feodo Tracker ───────────────────────────────────────────
feodo_listed: Optional[bool] = Field(default=None)
feodo_raw: str = Field(
default="{}",
sa_column=Column("feodo_raw", _BIG_TEXT, nullable=False, default="{}"),
)
feodo_queried_at: Optional[datetime] = Field(default=None)
# ── abuse.ch ThreatFox ───────────────────────────────────────────────
threatfox_listed: Optional[bool] = Field(default=None)
threatfox_raw: str = Field(
default="{}",
sa_column=Column("threatfox_raw", _BIG_TEXT, nullable=False, default="{}"),
)
threatfox_queried_at: Optional[datetime] = Field(default=None)
# ── Aggregate verdict ────────────────────────────────────────────────
# Synthesised from per-provider columns. ∈ {"malicious", "suspicious",
# "benign", "unknown"}. Used by the dashboard and webhook consumers
# that don't want to reason over four provider columns.
aggregate_verdict: Optional[str] = Field(
default=None, max_length=32, index=True
)
# ── TTL bookkeeping ──────────────────────────────────────────────────
cached_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
expires_at: datetime = Field(index=True)

414
decnet/web/db/models/attackers.py Normal file
View File

@@ -0,0 +1,414 @@
"""Attacker core + per-attacker behavioral and per-session profile rows."""
from datetime import datetime, timezone
from typing import Any, List, Optional
from pydantic import BaseModel
from sqlalchemy import BINARY, Column, Text, UniqueConstraint
from sqlmodel import Field, SQLModel
from ._base import _BIG_TEXT
# ─── Keystroke-dynamics tuning constants ──────────────────────────────────────
#
# These are the semantic thresholds the session-profile ingester (DEBT-036)
# uses to bucket IATs and decide what "started a new action" means. Keeping
# them here (not inline in the ingester) so that:
# * the schema docstrings below can reference exact boundaries instead of
# copy-pasted magic numbers, and
# * a future calibration pass against real honeypot session data only has
# to touch one place.
# All values in seconds.
KD_PAUSE_BURST_MAX_S: float = 0.2 # IAT < this = muscle-memory digraph
KD_PAUSE_THINK_MAX_S: float = 1.5 # IAT < this = semantic / context-switch pause
# everything ≥ this lands in the distracted bucket
KD_START_OF_ACTION_IDLE_S: float = 2.0 # idle gap that counts as "new action"
# raised from 1s — 1s still catches a lot of
# mid-command hesitation, 2s is closer to
# empirical "meaningfully new action"
class Attacker(SQLModel, table=True):
"""
Per-IP **observation** row. Every distinct source IP we observe gets
one of these. The semantic role is "observation event," not "actor
identity" — an actor rotating across N IPs produces N rows here.
The deduped actor view lives in ``AttackerIdentity`` (one identity
per actor; many observations per identity); the per-operation view
lives in ``Campaign``. ``identity_id`` is set by the clusterer
worker once it resolves which observations are the same hands.
NULL while the clusterer hasn't run on this row yet.
See ``development/IDENTITY_RESOLUTION.md`` for the three-level
hierarchy rationale.
"""
__tablename__ = "attackers"
uuid: str = Field(primary_key=True)
ip: str = Field(index=True)
identity_id: Optional[str] = Field(
default=None,
foreign_key="attacker_identities.uuid",
index=True,
)
first_seen: datetime = Field(index=True)
last_seen: datetime = Field(index=True)
event_count: int = Field(default=0)
service_count: int = Field(default=0)
decky_count: int = Field(default=0)
# JSON blobs — these grow over the attacker's lifetime. Use MEDIUMTEXT on
# MySQL (16 MiB) for the fields that accumulate (fingerprints, commands,
# and the deckies/services lists that are unbounded in principle).
services: str = Field(
default="[]", sa_column=Column("services", _BIG_TEXT, nullable=False, default="[]")
) # JSON list[str]
deckies: str = Field(
default="[]", sa_column=Column("deckies", _BIG_TEXT, nullable=False, default="[]")
) # JSON list[str], first-contact ordered
traversal_path: Optional[str] = Field(
default=None, sa_column=Column("traversal_path", Text, nullable=True)
) # "decky-01 → decky-03 → decky-05"
is_traversal: bool = Field(default=False)
bounty_count: int = Field(default=0)
credential_count: int = Field(default=0)
fingerprints: str = Field(
default="[]", sa_column=Column("fingerprints", _BIG_TEXT, nullable=False, default="[]")
) # JSON list[dict] — bounty fingerprints
commands: str = Field(
default="[]", sa_column=Column("commands", _BIG_TEXT, nullable=False, default="[]")
) # JSON list[dict] — commands per service/decky
# GeoIP enrichment (populated by the profiler from decnet.geoip.enrich_ip).
# Nullable because private / loopback / IPv6 sources never resolve.
country_code: Optional[str] = Field(default=None, max_length=2, index=True)
country_source: Optional[str] = Field(default=None, max_length=16)
# ASN enrichment (populated by the profiler from decnet.asn.enrich_ip).
# Nullable for the same reasons as country_code, plus IPs not currently
# announced in the global BGP table (e.g. CGNAT, dark space).
asn: Optional[int] = Field(default=None, index=True)
as_name: Optional[str] = Field(default=None, max_length=128)
asn_source: Optional[str] = Field(default=None, max_length=16)
# Reverse-DNS (PTR) name, one-shot resolved by the profiler at first
# sighting. Nullable — many attackers run infra with no rDNS, and
# private/loopback addresses never resolve. 256 chars matches
# RFC 1035 max hostname length.
ptr_record: Optional[str] = Field(default=None, max_length=256)
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
class AttackerIdentity(SQLModel, table=True):
"""
Resolved actor identity — the dedup'd "same hands" row that one or
more ``Attacker`` observations FK into. Populated by the (future)
clusterer worker; NULL on every observation until it runs.
Why a separate table from ``Attacker``: an actor rotating across N
IPs produces N observation rows but only ONE identity row. The
identity is recovered from signals the attacker can't cheaply
rotate — JA3, HASSH, payload hashes, C2 callbacks, and (V2)
keystroke-rhythm SimHash. See ``development/IDENTITY_RESOLUTION.md``.
All clusterer-populated fields are nullable; the table ships empty
in the schema-only PR (commit 1) and stays empty until the
clusterer lands. Empty is valid.
``schema_version`` is non-negotiable from day one. Federation
gossip in V2 will share identity vectors across operators;
bumping feature definitions without a version field silently
poisons receivers.
"""
__tablename__ = "attacker_identities"
uuid: str = Field(primary_key=True)
schema_version: int = Field(default=1)
# Set by the campaign clusterer. The ``campaigns`` table now
# exists; this is a real FK. Nullable until the campaign clusterer
# has run on this identity row.
campaign_id: Optional[str] = Field(
default=None, foreign_key="campaigns.uuid", index=True
)
first_seen_at: Optional[datetime] = Field(default=None, index=True)
last_seen_at: Optional[datetime] = Field(default=None, index=True)
created_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
# Identity-cohesion score from the clusterer. Range [0, 1]; null
# until the clusterer writes. Higher = more confident the
# observations linked to this identity are the same hands.
confidence: Optional[float] = Field(default=None)
# Denormalized count of FK'd Attacker rows. Maintained by the
# clusterer when it links/unlinks. Cheap dashboard read.
observation_count: int = Field(default=0)
# Fingerprint summary columns. JSON-serialized list[str] in TEXT
# because: (a) federation gossip wants this exact shape on the
# wire, (b) MySQL can't index BLOB/TEXT without prefix lengths,
# (c) actors can present multiple JA3/HASSH values across tools
# so a scalar column is wrong.
ja3_hashes: Optional[str] = Field(
default=None, sa_column=Column("ja3_hashes", Text, nullable=True)
)
hassh_hashes: Optional[str] = Field(
default=None, sa_column=Column("hassh_hashes", Text, nullable=True)
)
# JSON list[str] — SHA-256 fingerprints of leaf certs presented by
# attacker-run TLS servers, captured by the active prober alongside
# JARM. Same federation-gossip rationale as ja3_hashes/hassh_hashes:
# a self-signed cert reused across C2 nodes is an instant cluster-link
# signal, and TEXT keeps MySQL indexable via prefix length.
tls_cert_sha256: Optional[str] = Field(
default=None, sa_column=Column("tls_cert_sha256", Text, nullable=True)
)
# Payload SimHash list — 64-bit ints serialized as hex strings.
# SimHashes are Hamming-comparable, which is the entire reason
# they're a list (not a set).
payload_simhashes: Optional[str] = Field(
default=None, sa_column=Column("payload_simhashes", Text, nullable=True)
)
c2_endpoints: Optional[str] = Field(
default=None, sa_column=Column("c2_endpoints", Text, nullable=True)
)
# V2 keystroke-dynamics hook. Same shape as
# SessionProfile.kd_digraph_simhash; this is the centroid (or
# majority vote) across the identity's sessions. BINARY(8) so
# MySQL can index without a prefix length, same as session_profile.
kd_digraph_simhash: Optional[bytes] = Field(
default=None,
sa_column=Column("kd_digraph_simhash", BINARY(8), nullable=True, index=True),
)
# Soft-merge audit trail. When the clusterer collapses two
# identities, the loser's row stays in place with this set to the
# winner's UUID — preserves the audit trail without orphaning FKs
# from any cached subscribers. Resolvers (e.g.
# GET /identities/{uuid}) follow the chain and surface the winner.
merged_into_uuid: Optional[str] = Field(
default=None, foreign_key="attacker_identities.uuid", index=True
)
# Operator-editable free-form notes — annotation surface for human
# analysts ("known APT-XX cluster," "matches MISP event 1234").
notes: Optional[str] = Field(
default=None, sa_column=Column("notes", Text, nullable=True)
)
class AttackerBehavior(SQLModel, table=True):
"""
Timing & behavioral profile for an attacker, joined to Attacker by uuid.
Kept in a separate table so the core Attacker row stays narrow and
behavior data can be updated independently (e.g. as the sniffer observes
more packets) without touching the event-count aggregates.
"""
__tablename__ = "attacker_behavior"
attacker_uuid: str = Field(primary_key=True, foreign_key="attackers.uuid")
# OS / TCP stack fingerprint (rolled up from sniffer events)
os_guess: Optional[str] = None
hop_distance: Optional[int] = None
tcp_fingerprint: str = Field(
default="{}",
sa_column=Column("tcp_fingerprint", Text, nullable=False, default="{}"),
) # JSON: window, wscale, mss, options_sig
# Raw SSH KEX algorithm preference strings observed across HASSH probes
# (one entry per hassh_fingerprint event). Keeping the raw ordered list
# enables post-hoc KEX-order fingerprinting beyond the HASSH hash.
kex_order_raw: Optional[str] = Field(
default=None,
sa_column=Column("kex_order_raw", Text, nullable=True),
) # JSON list[str] — kex_algorithms comma-separated strings
# Sniffer-observed SSH client identification strings (RFC 4253 §4.2),
# deduped in observation order. Captures the attacker's SSH client
# software (e.g. "SSH-2.0-OpenSSH_9.2p1", "SSH-2.0-libssh2_1.10.0").
ssh_client_banners: Optional[str] = Field(
default=None,
sa_column=Column("ssh_client_banners", Text, nullable=True),
) # JSON list[str]
retransmit_count: int = Field(default=0)
# Behavioral (derived by the profiler from log-event timing)
behavior_class: Optional[str] = None # beaconing | interactive | scanning | brute_force | slow_scan | mixed | unknown
beacon_interval_s: Optional[float] = None
beacon_jitter_pct: Optional[float] = None
tool_guesses: Optional[str] = None # JSON list[str] — all matched tools
timing_stats: str = Field(
default="{}",
sa_column=Column("timing_stats", Text, nullable=False, default="{}"),
) # JSON: mean/median/stdev/min/max IAT
phase_sequence: str = Field(
default="{}",
sa_column=Column("phase_sequence", Text, nullable=False, default="{}"),
) # JSON: recon_end/exfil_start/latency
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
class SessionProfile(SQLModel, table=True):
"""
Per-session keystroke-dynamics fingerprint.
One row per recorded interactive session. Pre-v1 the ingestion job
that populates these columns is not yet built (tracked as gap #2 in
SIGNAL_CAPTURE_AUDIT.md); the table ships empty so that:
* downstream correlation/federation work can target a stable schema, and
* `schema_version` is committed to storage from day one — federation
gossip in v2 requires cross-operator compatibility, and retrofitting
a version column after rows exist is painful.
All feature columns are nullable so the empty write path (one row per
closed session) is valid without the behavioral analyzer online yet.
"""
__tablename__ = "session_profile"
sid: str = Field(primary_key=True) # session UUID
log_id: Optional[int] = Field(
default=None, foreign_key="logs.id", index=True
)
schema_version: int = Field(default=1)
# ──────────────────────────────────────────────────────────────────────
# Keystroke-dynamics feature columns (kd_*).
#
# Intended use: session clustering and tooling attribution
# ("is this the same typist?" / "is this a known C2
# framework's paste cadence?").
# Explicitly NOT for: attribution to named individuals, access or
# admission decisions, any ML-driven identity lookup,
# or biometric-login-style user identification. Those
# framings push into legal/ethics territory we don't
# want this project walking into by accident.
# PII discipline: every kd_* column aggregates CHARACTERS and TIMING
# only — never raw input-stream content. Attacker
# passwords typed over SSH must not land here.
# Nulls semantic: a null means "ingester hasn't run on this session
# yet", not "zero events". Consumers should treat
# null as absent, not as a computed zero.
# ──────────────────────────────────────────────────────────────────────
# Inter-key interval timing moments (seconds).
kd_iki_mean: Optional[float] = None
kd_iki_stdev: Optional[float] = None
kd_iki_p50: Optional[float] = None
kd_iki_p95: Optional[float] = None
kd_enter_latency_p50: Optional[float] = None
kd_enter_latency_p95: Optional[float] = None
# Cadence ratios.
kd_burst_ratio: Optional[float] = None
kd_think_ratio: Optional[float] = None
# Control-character rates (events per keystroke).
kd_ctrl_backspace: Optional[float] = None
kd_ctrl_wkill: Optional[float] = None
kd_ctrl_ukill: Optional[float] = None
kd_ctrl_abort: Optional[float] = None
kd_ctrl_eof: Optional[float] = None
kd_arrow_rate: Optional[float] = None
kd_tab_rate: Optional[float] = None
# 8-byte SimHash over keystroke digraphs — Hamming-comparable across sessions.
# Fixed-width BINARY(8) rather than BLOB: MySQL can't index BLOB/TEXT
# columns without a prefix length, and SimHashes are always exactly 8
# bytes so a variable-length type gains nothing here.
#
# PII discipline: the simhash is computed over keystroke CHARACTERS
# (digraph bigrams), never over the raw content of the input stream —
# attacker passwords typed over SSH must never land in this column.
kd_digraph_simhash: Optional[bytes] = Field(
default=None,
sa_column=Column("kd_digraph_simhash", BINARY(8), nullable=True, index=True),
)
# Top-N most-common digraphs with their mean IAT, as JSON.
# Complements kd_digraph_simhash: the simhash answers "same typist?",
# this answers "same typist IN THE SAME MENTAL STATE?" (tired vs rested
# vs distracted shifts bigram-specific IATs measurably). Shape:
# [["th", 47, 0.082], ["in", 31, 0.091], ...] (bigram, count, mean_iat_s)
# Bounded by the ingester to N≤32 to cap row width.
#
# TODO(DEBT-036 upgrade path): JSON-in-TEXT is fine for v1's
# "surface the typist's top digraphs on the attacker page" use
# case, but every similarity query (e.g. "find sessions where the
# 'th' digraph mean IAT is within 20 ms of this one") has to pull
# the string, parse JSON, compare — O(sessions) with a constant
# overhead per row. If that query shape becomes hot, promote to a
# dedicated `session_bigram_stats(sid, bigram, count, mean_iat_s)`
# table with a (bigram, mean_iat_s) index, or a JSONB column on
# Postgres with a GIN index. Either is straightforward, neither
# changes the write-side ingester materially.
kd_top_bigrams: Optional[str] = Field(
default=None, sa_column=Column("kd_top_bigrams", Text, nullable=True),
)
# IAT of the first keystroke following an idle gap >
# KD_START_OF_ACTION_IDLE_S (or the session-start gap before the
# very first keystroke). Separates "initiating a command" from
# "executing a remembered one" — real humans have measurable
# start-of-action latency, bots don't. Median across all such
# initiations in the session, seconds.
#
# Prompt-agnostic on purpose: PS1 / multi-line prompts / sudo
# password prompts make prompt-anchored detection fragile. The
# idle-gap approach conflates post-prompt action-start with
# mid-session think-and-resume — acceptable for a single median
# field; if we later want to split them, feed the concurrent
# output-stream prompt-pattern into the ingester and fall back to
# time-only detection when it misses.
kd_start_of_action_latency: Optional[float] = None
# Three-bucket pause-length histogram, counts (not ratios — raw
# counts preserve the total-keystrokes denominator in the column
# itself). Bucket edges are the KD_PAUSE_* module constants:
# burst : IAT < KD_PAUSE_BURST_MAX_S (muscle-memory digraphs)
# think : KD_PAUSE_BURST_MAX_S ≤ IAT < KD_PAUSE_THINK_MAX_S
# (semantic boundary, context switch)
# distracted: IAT ≥ KD_PAUSE_THINK_MAX_S (went to look something
# up, got paged, reading another window)
# More discriminating than the flat burst_ratio / think_ratio pair:
# C2 operators concentrate in the burst bucket with a thin tail;
# opportunistic humans have a fat think bucket and a long
# distracted tail.
kd_pause_hist_burst: Optional[int] = None
kd_pause_hist_think: Optional[int] = None
kd_pause_hist_distracted: Optional[int] = None
# Longest IAT in the session, seconds. The distracted-bucket count
# alone can't tell "one 3-second pause" from "three 60-second
# pauses" — both contribute 1-3 to the distracted bucket but
# represent different behaviours (brief think vs actual
# disengagement). max_pause_gap carries that signal in one scalar.
kd_max_pause_gap: Optional[float] = None
# Derived totals.
total_keystrokes: Optional[int] = None
session_duration_s: Optional[float] = None
created_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc)
)
class SmtpTarget(SQLModel, table=True):
"""
Per-attacker list of victim domains observed via the SMTP honeypots.
Each row is one (attacker_uuid, domain) pair — an attacker who relays
mail to 500 addresses at acme.com collapses into a single row with
count=500. Only the *domain* is stored; local-parts (the bit before
`@`) are dropped at ingestion, so this table contains no PII beyond
the target organisation's identity.
Shape is designed for future V2 federation gossip: the
`smtp_target_seen(domain)` query returns aggregate counts with zero
cross-org attacker leakage — each operator can answer "have you seen
this domain being targeted?" without exposing *which* attackers did.
"""
__tablename__ = "smtp_targets"
id: Optional[int] = Field(default=None, primary_key=True)
attacker_uuid: str = Field(foreign_key="attackers.uuid", index=True)
domain: str = Field(index=True)
first_seen: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
last_seen: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
# Aggregate counter — one rcpt_to / message_accepted recipient bumps this.
count: int = Field(default=1)
__table_args__ = (
UniqueConstraint("attacker_uuid", "domain", name="uq_smtp_targets_attacker_domain"),
)
class AttackersResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]

73
decnet/web/db/models/auth.py Normal file
View File

@@ -0,0 +1,73 @@
"""Auth + user-management tables and DTOs."""
from typing import List, Literal
from pydantic import BaseModel, Field as PydanticField
from sqlmodel import Field, SQLModel
class User(SQLModel, table=True):
__tablename__ = "users"
uuid: str = Field(primary_key=True)
username: str = Field(index=True, unique=True)
password_hash: str
role: str = Field(default="viewer")
must_change_password: bool = Field(default=False)
# --- API Request/Response Models (Pydantic) ---
class Token(BaseModel):
access_token: str
token_type: str
must_change_password: bool = False
class LoginRequest(BaseModel):
username: str
password: str = PydanticField(..., max_length=72)
class ChangePasswordRequest(BaseModel):
old_password: str = PydanticField(..., max_length=72)
new_password: str = PydanticField(..., max_length=72)
# --- Configuration Models ---
class CreateUserRequest(BaseModel):
username: str = PydanticField(..., min_length=1, max_length=64)
password: str = PydanticField(..., min_length=8, max_length=72)
role: Literal["admin", "viewer"] = "viewer"
class UpdateUserRoleRequest(BaseModel):
role: Literal["admin", "viewer"]
class ResetUserPasswordRequest(BaseModel):
new_password: str = PydanticField(..., min_length=8, max_length=72)
class DeploymentLimitRequest(BaseModel):
deployment_limit: int = PydanticField(..., ge=1, le=500)
class GlobalMutationIntervalRequest(BaseModel):
global_mutation_interval: str = PydanticField(..., pattern=r"^[1-9]\d*[mdMyY]$")
class UserResponse(BaseModel):
uuid: str
username: str
role: str
must_change_password: bool
class ConfigResponse(BaseModel):
role: str
deployment_limit: int
global_mutation_interval: str
class AdminConfigResponse(ConfigResponse):
users: List[UserResponse]

83
decnet/web/db/models/campaigns.py Normal file
View File

@@ -0,0 +1,83 @@
"""Campaign — operation-level grouping of resolved attacker identities."""
from datetime import datetime, timezone
from typing import Any, List, Optional
from pydantic import BaseModel
from sqlalchemy import Column, Text
from sqlmodel import Field, SQLModel
class Campaign(SQLModel, table=True):
"""
Campaign — one operation, one or more identities.
Sits one level above ``AttackerIdentity``: an actor (identity) may
appear in multiple campaigns over time, and a campaign may have
several distinct identities cooperating (e.g. a night-shift and
day-shift operator on the same job — fixture F5 multi_operator).
Populated by the campaign clusterer worker (downstream of identity
resolution). Empty rows are valid; the table ships empty until the
clusterer lands. ``schema_version`` is non-negotiable from day one
for the same federation-gossip reason ``AttackerIdentity`` carries
one — bumping campaign-level feature definitions without a version
field silently poisons cross-operator gossip in V2.
See ``development/CAMPAIGN_CLUSTERING.md`` for the signal taxonomy
(phase-handoff, shared-infra, temporal overlap, cohort).
"""
__tablename__ = "campaigns"
uuid: str = Field(primary_key=True)
schema_version: int = Field(default=1)
first_seen_at: Optional[datetime] = Field(default=None, index=True)
last_seen_at: Optional[datetime] = Field(default=None, index=True)
created_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
# Campaign-cohesion score from the clusterer. Range [0, 1]; null
# until the clusterer writes. Higher = more confident the linked
# identities are part of the same operation.
confidence: Optional[float] = Field(default=None)
# Denormalized count of FK'd ``AttackerIdentity`` rows.
identity_count: int = Field(default=0)
# Aggregated fingerprint summary across member identities. Same
# JSON-serialized list[str] in TEXT shape as
# ``AttackerIdentity.{ja3,hassh,payload_simhashes,c2_endpoints}`` —
# federation gossip wants the same wire shape at every layer.
ja3_hashes: Optional[str] = Field(
default=None, sa_column=Column("ja3_hashes", Text, nullable=True)
)
hassh_hashes: Optional[str] = Field(
default=None, sa_column=Column("hassh_hashes", Text, nullable=True)
)
tls_cert_sha256: Optional[str] = Field(
default=None, sa_column=Column("tls_cert_sha256", Text, nullable=True)
)
payload_simhashes: Optional[str] = Field(
default=None, sa_column=Column("payload_simhashes", Text, nullable=True)
)
c2_endpoints: Optional[str] = Field(
default=None, sa_column=Column("c2_endpoints", Text, nullable=True)
)
# Soft-merge audit trail — same revocable-merge pattern as
# ``AttackerIdentity.merged_into_uuid``. When the clusterer
# collapses two campaigns, the loser's row stays in place with this
# set to the winner's UUID; resolvers follow the chain.
merged_into_uuid: Optional[str] = Field(
default=None, foreign_key="campaigns.uuid", index=True
)
# Operator-editable free-form notes — annotation surface for
# human analysts ("APT-XX Q2 campaign", "matches CTI report 5678").
notes: Optional[str] = Field(
default=None, sa_column=Column("notes", Text, nullable=True)
)
class CampaignsResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]

242
decnet/web/db/models/canary.py Normal file
View File

@@ -0,0 +1,242 @@
"""Canary token tables + CRUD DTOs.
Canary tokens are decoy artifacts (operator-uploaded honeydocs / synthesised
fake configs) planted inside a decky's filesystem. When an attacker exfils
the artifact and uses it, an HTTP slug or DNS subdomain encoded into the
file is hit; the ``decnet canary`` worker observes the callback and
publishes ``canary.{token_id}.triggered`` on the bus. The webhook fanout
+ correlator pick it up the same way they handle any other attacker
event — no canary-specific consumer wiring needed downstream.
Three tables:
* :class:`CanaryBlob` — operator-uploaded source artifact, deduped by
sha256. The original bytes live on disk under
``/var/lib/decnet/canary/blobs/{sha256}``; this row carries metadata
+ refcount-aware deletion.
* :class:`CanaryToken` — one planted artifact in one decky. Either
references a blob (``blob_id``) and an instrumenter, or is a wholly
synthesised fake (e.g. ``aws_creds`` / ``git_config`` from a
generator) and ``blob_id`` is NULL. ``callback_token`` is the short
random slug embedded into HTTP URLs and DNS labels — unique across
the fleet so the worker can resolve a hit to a row in one query.
* :class:`CanaryTrigger` — append-only log of every callback hit.
``attacker_id`` is back-filled by the correlator after it attributes
``src_ip`` to an existing :class:`Attacker`; NULL until then.
We follow the project convention from :mod:`webhooks` and
:mod:`orchestrator`: stringly-typed UUIDs (``str`` PKs via
``str(uuid4())``), no FK to the composite-PK fleet table, indexes on
the join keys. Pydantic request/response shapes live in this same
file (per :mod:`feedback_models_single_source`).
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, List, Literal, Optional
from uuid import uuid4
from pydantic import BaseModel, Field as PydanticField
from sqlalchemy import Column, Index, Text
from sqlmodel import Field, SQLModel
from ._base import _BIG_TEXT
# --- Enum-shaped string literals -------------------------------------------
CanaryKind = Literal["http", "dns", "aws_passive"]
"""Detection mechanism for a token.
* ``http`` — slug embedded in artifact; attacker fetches our HTTP endpoint.
* ``dns`` — subdomain embedded; attacker's resolver looks up our DNS server.
* ``aws_passive`` — fake AWS credentials with no callback wiring. Trips
zero alerts on its own; useful only as bait + as evidence the attacker
read the file when correlated with other timing signals.
"""
CanaryState = Literal["planted", "revoked", "failed"]
"""Lifecycle state of a token row.
* ``planted`` — file is in the decky and the slug/host is live.
* ``revoked`` — operator deleted the token; planter unlinked the file
(best-effort) and the slug/host stops resolving.
* ``failed`` — placement failed (docker exec error, instrumenter
rejected the blob, etc.); surfaced in the UI so the operator can
retry or pick a different kind.
"""
# --- DB tables -------------------------------------------------------------
class CanaryBlob(SQLModel, table=True):
"""Operator-uploaded source artifact, deduped by sha256.
The same bytes uploaded twice produce the same row (insert-or-get
semantics in the repository). We never store the bytes inline —
only the disk path derived from ``sha256``. Deletion is
refcount-aware: ``DELETE`` is rejected while at least one
:class:`CanaryToken` references the blob.
"""
__tablename__ = "canary_blobs"
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
sha256: str = Field(index=True, unique=True)
filename: str # original filename — UI display only, not used for path resolution
content_type: str # sniffed MIME (python-magic); drives instrumenter selection
size_bytes: int
uploaded_by: str = Field(index=True) # User.uuid
uploaded_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
class CanaryToken(SQLModel, table=True):
"""One canary artifact planted inside one decky."""
__tablename__ = "canary_tokens"
__table_args__ = (
Index("ix_canary_tokens_decky", "decky_name", "state"),
)
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
kind: str = Field(index=True) # CanaryKind literal at the API layer
decky_name: str = Field(index=True) # FleetDecky.name; no FK (composite PK)
blob_uuid: Optional[str] = Field(
default=None, foreign_key="canary_blobs.uuid", index=True,
)
# Which instrumenter mutated the blob (``docx``/``xlsx``/``pdf``/``html``/
# ``image``/``plain``/``passthrough``). NULL when the artifact came
# from a synthesizer (``git_config``/``env_file``/``ssh_key``/
# ``aws_creds``/``honeydoc``); ``generator`` carries that name instead.
instrumenter: Optional[str] = Field(default=None)
generator: Optional[str] = Field(default=None)
placement_path: str # absolute path inside the container
# Short random slug (e.g. 16 url-safe bytes). Embedded in HTTP URLs
# *and* DNS labels — same value, different envelope, so both
# detection paths resolve to the same token row.
callback_token: str = Field(unique=True, index=True)
# Stable secret used by re-instrumentation: same blob + same seed
# = same mutated bytes, so re-seeding produces the same on-disk
# artifact and the planter is naturally idempotent.
secret_seed: str
placed_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
last_triggered_at: Optional[datetime] = Field(default=None, index=True)
trigger_count: int = Field(default=0)
created_by: str = Field(index=True) # User.uuid; "system" for baseline-seeded tokens
state: str = Field(default="planted", index=True)
last_error: Optional[str] = Field(
default=None, sa_column=Column("last_error", Text, nullable=True),
)
class CanaryTrigger(SQLModel, table=True):
"""Append-only log of one callback hit."""
__tablename__ = "canary_triggers"
__table_args__ = (
Index("ix_canary_triggers_token_ts", "token_uuid", "occurred_at"),
Index("ix_canary_triggers_attacker", "attacker_id"),
)
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
token_uuid: str = Field(foreign_key="canary_tokens.uuid", index=True)
occurred_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
src_ip: str = Field(index=True)
user_agent: Optional[str] = None
request_path: Optional[str] = None # HTTP path including the slug
dns_qname: Optional[str] = None # DNS qname when the hit came over DNS
# JSON-encoded request headers (HTTP) or empty for DNS. Stored as
# TEXT for cross-dialect portability — same trick as
# :attr:`WebhookSubscription.topic_patterns`.
raw_headers: str = Field(
default="{}",
sa_column=Column("raw_headers", _BIG_TEXT, nullable=False, default="{}"),
)
# Set by the correlator once it attributes ``src_ip`` to an existing
# :class:`Attacker`. NULL until correlation runs (which happens on
# the bus event we publish, so latency is sub-second).
attacker_id: Optional[str] = Field(default=None, index=True)
def headers(self) -> dict[str, Any]:
"""Decode :attr:`raw_headers` JSON; ``{}`` on bad/empty input."""
try:
raw = json.loads(self.raw_headers or "{}")
except (ValueError, TypeError):
return {}
return raw if isinstance(raw, dict) else {}
# --- API request / response shapes -----------------------------------------
class CanaryBlobResponse(BaseModel):
uuid: str
sha256: str
filename: str
content_type: str
size_bytes: int
uploaded_by: str
uploaded_at: datetime
# Number of tokens currently referencing this blob. Surfaces in the
# UI so operators don't try to delete a blob that's still in use,
# and the API uses it to gate ``DELETE`` (returns 409).
token_count: int = 0
class CanaryTokenCreateRequest(BaseModel):
"""Generate + plant a new token.
Exactly one of ``blob_uuid`` (operator-supplied artifact) or
``generator`` (synthesised fake) must be set. Validated in the
router so the 400 carries a clear detail message.
"""
decky_name: str = PydanticField(..., min_length=1)
kind: CanaryKind
placement_path: str = PydanticField(..., min_length=1)
blob_uuid: Optional[str] = None
generator: Optional[str] = None # git_config | env_file | ssh_key | aws_creds | honeydoc
# Optional override for the path-mapping helper — useful when the
# operator wants a specific Windows-shaped path on a windows-persona
# decky. Defaults to placement_path verbatim.
persona_path_hint: Optional[str] = None
class CanaryTokenResponse(BaseModel):
uuid: str
kind: CanaryKind
decky_name: str
blob_uuid: Optional[str]
instrumenter: Optional[str]
generator: Optional[str]
placement_path: str
callback_token: str
placed_at: datetime
last_triggered_at: Optional[datetime]
trigger_count: int
created_by: str
state: CanaryState
last_error: Optional[str]
class CanaryTriggerResponse(BaseModel):
uuid: str
token_uuid: str
occurred_at: datetime
src_ip: str
user_agent: Optional[str]
request_path: Optional[str]
dns_qname: Optional[str]
headers: dict[str, Any] = PydanticField(default_factory=dict)
attacker_id: Optional[str]
class CanaryTokensResponse(BaseModel):
tokens: List[CanaryTokenResponse]
total: int
class CanaryTriggersResponse(BaseModel):
triggers: List[CanaryTriggerResponse]
total: int
class CanaryBlobsResponse(BaseModel):
blobs: List[CanaryBlobResponse]
total: int

15
decnet/web/db/models/common.py Normal file
View File

@@ -0,0 +1,15 @@
"""Generic response shapes used across multiple router domains."""
from __future__ import annotations
from pydantic import BaseModel
class MessageResponse(BaseModel):
"""Standard envelope for mutations whose only payload is a status message.
Pinning the wire shape at the decorator (``response_model=MessageResponse``)
prevents a handler that accidentally returns a richer dict — e.g. a user
row with ``password_hash`` — from leaking extra fields to the client.
"""
message: str

29
decnet/web/db/models/deploy.py Normal file
View File

@@ -0,0 +1,29 @@
"""Fleet deploy + mutate-interval request DTOs."""
from typing import Optional
from pydantic import BaseModel, ConfigDict, Field as PydanticField
from decnet.models import IniContent
class MutateIntervalRequest(BaseModel):
# Human-readable duration: <number><unit> where unit is m(inutes), d(ays), M(onths), y/Y(ears).
# Minimum granularity is 1 minute. Seconds are not accepted.
mutate_interval: Optional[str] = PydanticField(None, pattern=r"^[1-9]\d*[mdMyY]$")
class DeployIniRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
# This field now enforces strict INI structure during Pydantic initialization.
# The OpenAPI schema correctly shows it as a required string.
ini_content: IniContent = PydanticField(..., description="A valid INI formatted string")
class DeployResponse(BaseModel):
message: str
mode: str
class PurgeResponse(BaseModel):
message: str
deleted: dict[str, int]

72
decnet/web/db/models/fleet.py Normal file
View File

@@ -0,0 +1,72 @@
"""Fleet decky table — DB mirror of ``decnet-state.json``.
The legacy unihost / MACVLAN / IPVLAN deploy path persists fleet state to a
JSON file (``/var/lib/decnet/decnet-state.json``) via
:func:`decnet.config.save_state`. That file is consumed directly by
``decnet status``/``decnet teardown``, the sniffer, and the collector — all
host-local CLI / worker code that may run on a box without the API daemon.
The FleetDecky table is a *mirror* of that JSON state inside MySQL/SQLite so
DB-only consumers (the orchestrator, the web dashboard, the REST API) can
see fleet decoys without touching the filesystem.
Both writers — CLI ``decnet deploy`` (``engine.deployer.deploy``) and the
web/API deploy path (``web.router.fleet.api_deploy_deckies``) — write to
*both* surfaces. A reconciler (``decnet.fleet.reconciler``) handles drift.
Schema mirrors :class:`decnet.web.db.models.swarm.DeckyShard` field-for-field
so the dashboard can render fleet rows with the same card shape. The PK is
composite ``(host_uuid, name)`` to future-proof for multi-host motherships
(a master that runs its own local fleet AND swarm-shards onto workers). In
unihost mode ``host_uuid`` defaults to the sentinel
:data:`LOCAL_HOST_SENTINEL`; we deliberately do NOT FK to ``swarm_hosts``
because the local mothership is not enrolled as a swarm worker.
"""
from datetime import datetime, timezone
from typing import Optional
from sqlalchemy import Column, Text
from sqlmodel import Field, SQLModel
from ._base import _BIG_TEXT
LOCAL_HOST_SENTINEL = "local"
class FleetDecky(SQLModel, table=True):
"""A unihost / MACVLAN / IPVLAN decky deployed on the local mothership.
Disjoint from :class:`DeckyShard` (SWARM-only) and :class:`TopologyDecky`
(MazeNET-only). Composite PK lets multiple hosts coexist when a future
mothership runs both a local fleet and acts as a swarm master.
"""
__tablename__ = "fleet_deckies"
host_uuid: str = Field(
default=LOCAL_HOST_SENTINEL, primary_key=True, index=True,
)
name: str = Field(primary_key=True)
# JSON list of service names on this decky (snapshot of assignment).
services: str = Field(
sa_column=Column("services", _BIG_TEXT, nullable=False, default="[]")
)
# Full serialised DeckyConfig — lets the dashboard render the same rich
# card (hostname/distro/archetype/service_config/mutate_interval) without
# round-tripping to load_state() on every page render.
decky_config: Optional[str] = Field(
default=None, sa_column=Column("decky_config", _BIG_TEXT, nullable=True)
)
decky_ip: Optional[str] = Field(default=None)
# pending|running|failed|torn_down|degraded|tearing_down|teardown_failed
state: str = Field(default="pending", index=True)
last_error: Optional[str] = Field(
default=None, sa_column=Column("last_error", Text, nullable=True),
)
compose_hash: Optional[str] = Field(default=None)
# Last reconciler observation (docker inspect) — lets the dashboard show
# "stale" rows whose reconciler hasn't ticked.
last_seen: Optional[datetime] = Field(default=None)
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc)
)

14
decnet/web/db/models/health.py Normal file
View File

@@ -0,0 +1,14 @@
"""Health-endpoint DTOs."""
from typing import Literal, Optional
from pydantic import BaseModel
class ComponentHealth(BaseModel):
status: Literal["ok", "failing"]
detail: Optional[str] = None
class HealthResponse(BaseModel):
status: Literal["healthy", "degraded", "unhealthy"]
components: dict[str, ComponentHealth]

222
decnet/web/db/models/logs.py Normal file
View File

@@ -0,0 +1,222 @@
"""Log / Bounty / Credential / State tables + their list-response DTOs."""
from datetime import datetime, timezone
from typing import Any, List, Optional
from pydantic import BaseModel
from sqlalchemy import Column, Index, Text, UniqueConstraint
from sqlmodel import Field, SQLModel
from ._base import _BIG_TEXT
class Log(SQLModel, table=True):
__tablename__ = "logs"
id: Optional[int] = Field(default=None, primary_key=True)
timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc), index=True)
decky: str = Field(index=True)
service: str = Field(index=True)
event_type: str = Field(index=True)
attacker_ip: str = Field(index=True)
# Long-text columns — use TEXT so MySQL DDL doesn't truncate to VARCHAR(255).
# TEXT is equivalent to plain text in SQLite.
raw_line: str = Field(sa_column=Column("raw_line", Text, nullable=False))
fields: str = Field(sa_column=Column("fields", Text, nullable=False))
msg: Optional[str] = Field(default=None, sa_column=Column("msg", Text, nullable=True))
# OTEL trace context — bridges the collector→ingester trace to the SSE
# read path. Nullable so pre-existing rows and non-traced deployments
# are unaffected.
trace_id: Optional[str] = Field(default=None)
span_id: Optional[str] = Field(default=None)
class Bounty(SQLModel, table=True):
__tablename__ = "bounty"
id: Optional[int] = Field(default=None, primary_key=True)
timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc), index=True)
decky: str = Field(index=True)
service: str = Field(index=True)
attacker_ip: str = Field(index=True)
bounty_type: str = Field(index=True)
payload: str = Field(sa_column=Column("payload", Text, nullable=False))
class Credential(SQLModel, table=True):
"""One observed credential attempt against a decky service.
Forward-compatible across every auth-bearing service in the fleet:
SSH user+pass, Telnet user+pass, SMTP domain+pass, LDAP dn+pass,
Redis password-only, etc. The two universal lossless representations
(``secret_b64`` + ``secret_sha256``) hoist to indexed columns so
cross-service reuse queries don't scan opaque JSON.
Per-service identity (the human-meaningful "who's authenticating")
lives in ``principal`` — username for SSH, domain for SMTP, dn for
LDAP. Nullable for principal-less mechanisms (Redis AUTH, bearer
tokens). Fully service-specific keys ride in ``fields`` JSON.
Dedup contract: same (attacker_ip, decky, service, secret_sha256,
principal_or_empty) tuple → upsert, bumps ``attempt_count`` and
``last_seen``. Different secret or different principal → new row.
``attacker_uuid`` is backfilled by the profiler once an Attacker row
has been minted for the source IP. It is nullable on first write so
the credential ingest path stays decoupled from the profiler.
"""
__tablename__ = "credentials"
__table_args__ = (
Index("ix_credentials_secret_service", "secret_sha256", "service"),
Index("ix_credentials_principal_service", "principal", "service"),
)
id: Optional[int] = Field(default=None, primary_key=True)
# Keyed by attacker IP (not attackers.uuid) on the write path to
# avoid the chicken-and-egg of landing a credential before the
# profiler has minted the Attacker. The profiler backfills
# ``attacker_uuid`` once it knows the IP, so cross-IP reuse queries
# eventually have an indexed FK to traverse.
attacker_ip: str = Field(index=True)
attacker_uuid: Optional[str] = Field(
default=None, foreign_key="attackers.uuid", index=True
)
decky_name: str = Field(index=True)
service: str = Field(index=True)
principal: Optional[str] = Field(default=None, index=True, max_length=256)
# Discriminator for what `secret_b64` actually contains. Default
# ``"plaintext"`` — a recoverable password the attacker sent on the
# wire (SSH/Telnet/FTP/IMAP/POP3/SMTP/Redis/LDAP/MQTT). Other kinds:
# ``"postgres_md5_challenge"`` (md5(md5(pw+user)+salt) hex bytes
# the attacker sent in the Postgres password message — plaintext
# irrecoverable), ``"vnc_des_response"`` (16-byte DES-encrypted
# challenge response — same shape).
#
# Reuse semantics gracefully degrade: same secret_sha256 only
# correlates within a single ``secret_kind``. Cross-kind matches
# are meaningless because different challenges produce different
# bytes for the same plaintext password.
secret_kind: str = Field(default="plaintext", index=True, max_length=32)
# Universal lossless secret representations. For non-plaintext
# kinds, secret_b64 is base64 of the raw attacker-sent bytes (after
# hex-decode for protocols that ship the response as a hex string).
secret_sha256: str = Field(index=True, max_length=64)
secret_b64: Optional[str] = Field(default=None, max_length=2048)
# Best-effort printable form — non-printable bytes collapsed to '?'
# by either auth-helper.c (SSH/Telnet) or the ingester's legacy
# adapter (FTP/POP3/IMAP/SMTP). May be lossy on non-UTF8.
secret_printable: Optional[str] = Field(default=None, max_length=512)
outcome: Optional[str] = Field(default=None, max_length=16) # success|failure|observed
fields: str = Field(
sa_column=Column("fields", _BIG_TEXT, nullable=False, default="{}")
)
first_seen: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
last_seen: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
attempt_count: int = Field(default=1)
class CredentialReuse(SQLModel, table=True):
"""One observed credential reuse pattern across deckies and/or services.
A row here is a *finding* produced by the correlator: the same
``(secret_sha256, secret_kind, principal)`` tuple was observed
against ``target_count`` distinct decky×service pairs. Upserted on
that natural key — the row accumulates new deckies/services/IPs
over time as the credential is reused.
The ``confidence`` column is reserved for a future fuzzy-match pass
(credential variants, e.g. ``hunter2`` vs ``hunter22``); rows
written by the exact-secret correlator are always 1.0.
"""
__tablename__ = "credential_reuse"
__table_args__ = (
UniqueConstraint(
"secret_sha256", "secret_kind", "principal_key",
name="uq_credential_reuse_secret_principal",
),
)
id: str = Field(primary_key=True, max_length=36)
secret_sha256: str = Field(index=True, max_length=64)
secret_kind: str = Field(index=True, max_length=32)
# Optional human-readable principal (e.g. "root"). Nullable — for
# cross-principal reuse rows we leave this null, but we still need
# a unique constraint, so ``principal_key`` is the non-null
# canonicalised form ("" when principal is null) used in the
# uniqueness tuple. SQLite's NULLs-distinct-in-UNIQUE behaviour
# would otherwise let duplicate null-principal rows through.
principal: Optional[str] = Field(default=None, max_length=256)
principal_key: str = Field(default="", max_length=256)
attacker_uuids: str = Field(
default="[]",
sa_column=Column("attacker_uuids", _BIG_TEXT, nullable=False, default="[]"),
) # JSON list[str]
attacker_ips: str = Field(
default="[]",
sa_column=Column("attacker_ips", _BIG_TEXT, nullable=False, default="[]"),
) # JSON list[str]
deckies: str = Field(
default="[]",
sa_column=Column("deckies", _BIG_TEXT, nullable=False, default="[]"),
) # JSON list[str]
services: str = Field(
default="[]",
sa_column=Column("services", _BIG_TEXT, nullable=False, default="[]"),
) # JSON list[str]
# COUNT(DISTINCT decky||':'||service). The discriminative scalar
# for ranking and filtering — a credential seen on 12 targets is
# far more interesting than one seen on 2.
target_count: int = Field(default=0, index=True)
attempt_count: int = Field(default=0)
confidence: float = Field(default=1.0)
first_seen: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
last_seen: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
class CredentialReuseResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class State(SQLModel, table=True):
__tablename__ = "state"
key: str = Field(primary_key=True)
# JSON-serialized DecnetConfig or other state blobs — can be large as
# deckies/services accumulate. MEDIUMTEXT on MySQL (16 MiB ceiling).
value: str = Field(sa_column=Column("value", _BIG_TEXT, nullable=False))
class LogsResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class BountyResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class CredentialsResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class StatsResponse(BaseModel):
total_logs: int
unique_attackers: int
active_deckies: int
deployed_deckies: int

111
decnet/web/db/models/orchestrator.py Normal file
View File

@@ -0,0 +1,111 @@
"""Orchestrator-emitted activity events.
Purpose-built sibling to ``logs.Log`` so attacker-originated events stay
cleanly separable from synthetic life-injection events at query time.
The orchestrator worker is the sole writer.
"""
from datetime import datetime, timezone
from typing import Any, List, Optional
from uuid import uuid4
from pydantic import BaseModel
from sqlalchemy import Column, Index, Text
from sqlmodel import Field, SQLModel
class OrchestratorEvent(SQLModel, table=True):
"""One orchestrator-driven action against a decky.
``kind`` discriminates the two MVP flavours:
* ``"traffic"`` — a protocol-driven interaction (SSH command exec for
MVP). ``src_decky_uuid`` is the *logical* originator and may differ
from the actual TCP source for the duration of the MVP, where the
orchestrator process drives the connection from the host. ``v1``
will execute the connection from inside the source container.
* ``"file"`` — a filesystem touch via ``docker exec`` against the
destination decky. ``src_decky_uuid`` is null.
``payload`` is the per-action JSON envelope: command run, exit code,
stdout/stderr digest, file path, byte counts, etc. Schema is
deliberately loose — the worker can extend it without a migration.
"""
__tablename__ = "orchestrator_events"
__table_args__ = (
Index("ix_orchestrator_events_dst_ts", "dst_decky_uuid", "ts"),
)
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
ts: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
kind: str = Field(index=True, max_length=16) # traffic|file
protocol: str = Field(index=True, max_length=16) # ssh for MVP
action: str = Field(max_length=64) # exec:uptime|file:create|...
# No FK to topology_deckies: dst/src may be a TopologyDecky.uuid
# (MazeNET source), a "host_uuid:name" composite (fleet / SWARM shard
# sources), or — for retired deckies — a row that's already gone. The
# column is an opaque identifier matching whatever
# ``BaseRepository.list_running_deckies`` emits in its ``uuid`` field.
# Index is kept; the FK was misleading and broke fleet-source events.
src_decky_uuid: Optional[str] = Field(default=None, index=True)
dst_decky_uuid: str = Field(index=True)
success: bool = Field(default=False, index=True)
payload: str = Field(
sa_column=Column("payload", Text, nullable=False, default="{}")
)
class OrchestratorEventsResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class OrchestratorEmail(SQLModel, table=True):
"""One fake email generated by the ``decnet emailgen`` worker.
Sibling table to :class:`OrchestratorEvent` — kept disjoint because
email rows carry domain-specific fields (subject, message_id,
in_reply_to, language) that have no analogue in the SSH/file events
and would otherwise bloat ``OrchestratorEvent.payload``.
The mail decky's UUID lives in ``mail_decky_uuid`` (the host serving
the IMAP/POP3 mailbox). ``thread_id`` is a worker-side UUID used to
chain replies; ``in_reply_to`` is the parent email's RFC 2822
Message-ID header value (or ``None`` for thread roots).
``payload`` follows the same loose-JSON convention as
:class:`OrchestratorEvent`: ``bytes``, ``generation_ms``, ``model``,
``mannerisms_used``, etc. The worker can extend it without a
migration.
"""
__tablename__ = "orchestrator_emails"
__table_args__ = (
Index("ix_orchestrator_emails_mail_ts", "mail_decky_uuid", "ts"),
Index("ix_orchestrator_emails_thread", "thread_id"),
)
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
ts: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
mail_decky_uuid: str = Field(index=True)
thread_id: str = Field(index=True)
message_id: str = Field(max_length=255)
in_reply_to: Optional[str] = Field(default=None, max_length=255)
sender_email: str = Field(max_length=255, index=True)
recipient_email: str = Field(max_length=255, index=True)
subject: str = Field(max_length=512)
language: str = Field(max_length=8, default="en")
eml_path: str = Field(max_length=1024)
success: bool = Field(default=False, index=True)
payload: str = Field(
sa_column=Column("payload", Text, nullable=False, default="{}")
)
class OrchestratorEmailsResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]

107
decnet/web/db/models/realism.py Normal file
View File

@@ -0,0 +1,107 @@
"""Realism — synthetic-file state across orchestrator ticks.
The orchestrator's pre-realism file generator forgot every file the
moment it was planted: each tick wrote a brand-new ``notes-{ts}.txt``
with a literal unix-epoch suffix. No edits, no rotation, no diurnal
shape — three of the realism failures the migration is fixing.
:class:`SyntheticFile` is the per-(decky, path) memory that lets the
realism engine read back yesterday's ``TODO.md``, mutate it, write
back the new body, and let the dashboard inspect the lineage.
Pre-v1: schema lives directly in the SQLModel; no ``_migrate_*``
helper (per the project's "no new migrations pre-v1" rule —
``feedback_no_new_migrations_prev1.md``). Alembic lands at v1.
"""
from datetime import datetime, timezone
from typing import Any, List
from uuid import uuid4
from pydantic import BaseModel
from sqlalchemy import Column, Index, Text, UniqueConstraint
from sqlmodel import Field, SQLModel
SYNTHETIC_FILE_BODY_LIMIT = 65536
"""Cap on persisted ``synthetic_files.last_body`` bytes.
Enforced by the repo on both insert and update — callers may pass the
full body; the repo clips. Large blobs (DOCX/PDF, canary artifacts) are
wasted disk on the master side; the decky filesystem holds the canonical
bytes."""
class SyntheticFile(SQLModel, table=True):
"""One realism-planted file on one decky.
The unique key is ``(decky_uuid, path)`` — there's at most one
realism record per location, even if the planter has rotated the
file (rotation updates ``edit_count`` and ``last_modified``, not
a new row).
``last_body`` is capped — large blobs (DOCX/PDF, future canary
artifacts) are truncated at write time. The edit-in-place flow
(stage 3b) only needs the body when the content class supports
body-level mutation (``note``, ``todo``, ``draft``, ``script``),
so storing the canonical bytes for binary blobs would be wasted.
``content_hash`` is sha256 of the *body bytes only* — never of
metadata or wrapper headers — so a hash compare is a cheap
"did the body change?" check across edits.
"""
__tablename__ = "synthetic_files"
__table_args__ = (
UniqueConstraint(
"decky_uuid", "path", name="uq_synthetic_files_decky_path",
),
Index("ix_synthetic_files_decky_modified", "decky_uuid", "last_modified"),
)
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
decky_uuid: str = Field(index=True, max_length=64)
# Capped at 512 so the (decky_uuid, path) unique index fits MySQL's
# 3072-byte utf8mb4 limit: (64+512)*4 = 2304 bytes. Real realism +
# canary paths are well under (longest is
# ``/home/<persona>/Documents/Q3-Operations-Review.docx``, ~70 chars).
path: str = Field(max_length=512)
persona: str = Field(max_length=128) # EmailPersona.name
content_class: str = Field(max_length=32, index=True) # ContentClass enum value
created_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True,
)
last_modified: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc),
)
edit_count: int = Field(default=0)
content_hash: str = Field(max_length=64) # sha256 hex
last_body: str = Field(
sa_column=Column("last_body", Text, nullable=False, default="")
)
class SyntheticFilesResponse(BaseModel):
total: int
limit: int
offset: int
data: List[dict[str, Any]]
class RealismConfig(SQLModel, table=True):
"""Operator-tunable realism knobs.
Single-row-per-key schema: each row carries one piece of operator
config (today: ``key="weights"`` → JSON encoding the planner's
user/system/canary weights and canary probability). The planner
reads in-memory module globals; the orchestrator worker refreshes
those globals from this table on a periodic tick.
UUID PK + unique key per ``feedback_uuid_over_natural_keys.md``.
"""
__tablename__ = "realism_config"
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
key: str = Field(max_length=64, unique=True, index=True)
value: str = Field(
sa_column=Column("value", Text, nullable=False, default="{}"),
)
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc),
)

200
decnet/web/db/models/swarm.py Normal file
View File

@@ -0,0 +1,200 @@
"""Swarm host + decky shard tables and their HTTP DTOs."""
from datetime import datetime, timezone
from typing import Annotated, Any, Optional
from pydantic import BaseModel, Field as PydanticField
from sqlalchemy import Column, Text
from sqlmodel import Field, SQLModel
from decnet.models import DecnetConfig
from ._base import _BIG_TEXT
class SwarmHost(SQLModel, table=True):
"""A worker host enrolled into a DECNET swarm.
Rows exist only on the master. Populated by `decnet swarm enroll` and
read by the swarm controller when sharding deckies onto workers.
"""
__tablename__ = "swarm_hosts"
uuid: str = Field(primary_key=True)
name: str = Field(index=True, unique=True)
address: str # IP or hostname reachable by the master
agent_port: int = Field(default=8765)
status: str = Field(default="enrolled", index=True)
# ISO-8601 string of the last successful agent /health probe
last_heartbeat: Optional[datetime] = Field(default=None)
client_cert_fingerprint: str # SHA-256 hex of worker's issued client cert
# SHA-256 hex of the updater-identity cert, if the host was enrolled
# with ``--updater`` / ``issue_updater_bundle``. ``None`` for hosts
# that only have an agent identity.
updater_cert_fingerprint: Optional[str] = Field(default=None)
# Directory on the master where the per-worker cert bundle lives
cert_bundle_path: str
enrolled_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
notes: Optional[str] = Field(default=None, sa_column=Column("notes", Text, nullable=True))
# Per-host driver preference. True => deckies on this host run over IPvlan
# (L2) instead of macvlan — required when the host is a VirtualBox guest
# bridged over Wi-Fi, because Wi-Fi APs only allow one MAC per station
# and macvlan's per-container MACs rotate the VM's DHCP lease.
use_ipvlan: bool = Field(default=False)
class DeckyShard(SQLModel, table=True):
"""Mapping of a single decky to the worker host running it (swarm mode)."""
__tablename__ = "decky_shards"
decky_name: str = Field(primary_key=True)
host_uuid: str = Field(foreign_key="swarm_hosts.uuid", index=True)
# JSON list of service names running on this decky (snapshot of assignment).
services: str = Field(sa_column=Column("services", _BIG_TEXT, nullable=False, default="[]"))
# Full serialised DeckyConfig from the most recent dispatch or heartbeat.
# Lets the dashboard render the same rich card (hostname/distro/archetype/
# service_config/mutate_interval) that the local-fleet view uses, without
# needing a live round-trip to the worker for every page render.
decky_config: Optional[str] = Field(
default=None, sa_column=Column("decky_config", _BIG_TEXT, nullable=True)
)
decky_ip: Optional[str] = Field(default=None)
state: str = Field(default="pending", index=True) # pending|running|failed|torn_down|degraded|tearing_down|teardown_failed
last_error: Optional[str] = Field(default=None, sa_column=Column("last_error", Text, nullable=True))
compose_hash: Optional[str] = Field(default=None)
# Timestamp of the last heartbeat that echoed this shard; lets the UI
# show "stale" decks whose agent has gone silent.
last_seen: Optional[datetime] = Field(default=None)
updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
# --- Swarm API DTOs ---
# Request/response contracts for the master-side swarm controller
# (decnet/web/swarm_api.py). The underlying SQLModel tables — SwarmHost and
# DeckyShard — live above; these are the HTTP-facing shapes.
class SwarmEnrollRequest(BaseModel):
# x509 CommonName is capped at 64 bytes (RFC 5280 UB-common-name) — the
# cert issuer would reject anything longer with a ValueError.
# Pattern: ASCII hostname-safe characters only. The name is embedded
# both in the CN and as a SAN DNS entry; x509.DNSName only accepts
# A-label ASCII, so non-ASCII would blow up at issuance.
name: str = PydanticField(
..., min_length=1, max_length=64,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._\-]*$",
)
address: str = PydanticField(
..., min_length=1, max_length=253,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:\-]*$",
description="IP or DNS the master uses to reach the worker",
)
agent_port: int = PydanticField(default=8765, ge=1, le=65535)
sans: list[
Annotated[
str,
PydanticField(
min_length=1, max_length=253,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:\-]*$",
),
]
] = PydanticField(
default_factory=list,
description="Extra SANs (IPs / hostnames) to embed in the worker cert",
)
notes: Optional[str] = None
issue_updater_bundle: bool = PydanticField(
default=False,
description="If true, also issue an updater cert (CN=updater@<name>) for the remote self-updater",
)
class SwarmUpdaterBundle(BaseModel):
"""Subset of SwarmEnrolledBundle for the updater identity."""
fingerprint: str
updater_cert_pem: str
updater_key_pem: str
class SwarmEnrolledBundle(BaseModel):
"""Cert bundle returned to the operator — must be delivered to the worker."""
host_uuid: str
name: str
address: str
agent_port: int
fingerprint: str
ca_cert_pem: str
worker_cert_pem: str
worker_key_pem: str
updater: Optional[SwarmUpdaterBundle] = None
class SwarmHostView(BaseModel):
uuid: str
name: str
address: str
agent_port: int
status: str
last_heartbeat: Optional[datetime] = None
client_cert_fingerprint: str
updater_cert_fingerprint: Optional[str] = None
enrolled_at: datetime
notes: Optional[str] = None
use_ipvlan: bool = False
class DeckyShardView(BaseModel):
"""One decky → host mapping, enriched with the host's identity for display."""
decky_name: str
decky_ip: Optional[str] = None # resolved from the stored DecnetConfig at read time
host_uuid: str
host_name: str
host_address: str
host_status: str
services: list[str]
state: str
last_error: Optional[str] = None
compose_hash: Optional[str] = None
updated_at: datetime
# Enriched fields lifted from the stored DeckyConfig snapshot so the
# dashboard can render the same card shape as the local-fleet view.
hostname: Optional[str] = None
distro: Optional[str] = None
archetype: Optional[str] = None
service_config: dict[str, dict[str, Any]] = {}
mutate_interval: Optional[int] = None
last_mutated: float = 0.0
last_seen: Optional[datetime] = None
class SwarmDeployRequest(BaseModel):
config: DecnetConfig
dry_run: bool = False
no_cache: bool = False
class SwarmTeardownRequest(BaseModel):
host_uuid: Optional[str] = PydanticField(
default=None,
description="If set, tear down only this worker; otherwise tear down all hosts",
)
decky_id: Optional[str] = None
class SwarmHostResult(BaseModel):
host_uuid: str
host_name: str
ok: bool
detail: Any | None = None
class SwarmDeployResponse(BaseModel):
results: list[SwarmHostResult]
class SwarmHostHealth(BaseModel):
host_uuid: str
name: str
address: str
reachable: bool
detail: Any | None = None
class SwarmCheckResponse(BaseModel):
results: list[SwarmHostHealth]

442
decnet/web/db/models/topology.py Normal file
View File

@@ -0,0 +1,442 @@
"""MazeNET topology tables + the REST DTOs that wrap them."""
from datetime import datetime, timezone
from typing import Annotated, Any, Literal, Optional
from uuid import uuid4
from pydantic import BaseModel, BeforeValidator, ConfigDict, Field as PydanticField
from sqlalchemy import Column, Index, Text, UniqueConstraint
from sqlmodel import Field, SQLModel
from ._base import _BIG_TEXT
# --- MazeNET tables ---
# Nested deception topologies: an arbitrary-depth DAG of LANs connected by
# multi-homed "bridge" deckies. Purpose-built; disjoint from DeckyShard which
# remains SWARM-only.
class Topology(SQLModel, table=True):
__tablename__ = "topologies"
id: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
name: str = Field(index=True, unique=True)
mode: str = Field(default="unihost") # unihost|agent
# When ``mode == "agent"``, pins this topology to a specific enrolled
# worker. ``None`` for unihost topologies (master-local deploy).
target_host_uuid: Optional[str] = Field(
default=None, foreign_key="swarm_hosts.uuid", index=True
)
# Full TopologyConfig snapshot (including seed) used at generation time.
config_snapshot: str = Field(
sa_column=Column("config_snapshot", _BIG_TEXT, nullable=False, default="{}")
)
status: str = Field(
default="pending", index=True
) # pending|deploying|active|degraded|failed|tearing_down|torn_down
status_changed_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc)
)
created_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
# Optimistic-concurrency token. Bumped by repo methods that mutate
# the topology or any child row when an expected_version is supplied.
# Callers pass their last-seen version; mismatch raises VersionConflict.
version: int = Field(default=1, nullable=False)
# Set by the heartbeat handler when an agent's reported
# ``applied_version_hash`` diverges from what we expect it to be
# running. Drained by the mutator watch loop, which re-pushes via
# AgentClient and clears the flag. NULL for unihost topologies.
needs_resync: bool = Field(default=False, nullable=False)
# JSON-serialised list of EmailPersona dicts consumed by the
# ``decnet emailgen`` worker. Empty list = no fake mailbox owners
# configured for this topology, the worker skips it.
email_personas: str = Field(
sa_column=Column(
"email_personas", _BIG_TEXT, nullable=False, default="[]"
)
)
# ISO 639-1 language code applied to any persona that doesn't override
# ``language`` itself. English by default; ANTI's deployments default
# to "es" by editing this column.
language_default: str = Field(default="en", max_length=8)
class LAN(SQLModel, table=True):
__tablename__ = "lans"
__table_args__ = (UniqueConstraint("topology_id", "name", name="uq_lan_topology_name"),)
id: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
topology_id: str = Field(foreign_key="topologies.id", index=True)
name: str
# Populated after the Docker network is created; nullable before deploy.
docker_network_id: Optional[str] = Field(default=None)
subnet: str
is_dmz: bool = Field(default=False)
# Canvas layout coordinates (set by the web editor). Nullable so
# generator-emitted LANs don't need auto-layout at generation time.
x: Optional[float] = Field(default=None)
y: Optional[float] = Field(default=None)
class TopologyDecky(SQLModel, table=True):
"""A decky belonging to a MazeNET topology.
Disjoint from DeckyShard (which is SWARM-only). UUID PK; decky name is
unique only within a topology, so two topologies can both have a
``decky-01`` without colliding.
"""
__tablename__ = "topology_deckies"
__table_args__ = (
UniqueConstraint("topology_id", "name", name="uq_topology_decky_name"),
)
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
topology_id: str = Field(foreign_key="topologies.id", index=True)
name: str
# JSON list[str] of service names on this decky (snapshot of assignment).
services: str = Field(
sa_column=Column("services", _BIG_TEXT, nullable=False, default="[]")
)
# Full serialised DeckyConfig snapshot — lets the dashboard render the
# same card shape as DeckyShard without a live round-trip.
decky_config: Optional[str] = Field(
default=None, sa_column=Column("decky_config", _BIG_TEXT, nullable=True)
)
ip: Optional[str] = Field(default=None)
# Same vocabulary as DeckyShard.state to keep dashboard rendering uniform.
state: str = Field(
default="pending", index=True
) # pending|running|failed|torn_down|degraded|tearing_down|teardown_failed
last_error: Optional[str] = Field(
default=None, sa_column=Column("last_error", Text, nullable=True)
)
compose_hash: Optional[str] = Field(default=None)
last_seen: Optional[datetime] = Field(default=None)
updated_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc)
)
# Canvas layout coordinates (set by the web editor). Nullable so
# generator-emitted deckies don't need auto-layout at generation time.
x: Optional[float] = Field(default=None)
y: Optional[float] = Field(default=None)
class TopologyEdge(SQLModel, table=True):
"""Membership edge: a decky attached to a LAN.
A decky appearing in ≥2 edges is multi-homed (a bridge decky).
"""
__tablename__ = "topology_edges"
id: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
topology_id: str = Field(foreign_key="topologies.id", index=True)
decky_uuid: str = Field(foreign_key="topology_deckies.uuid", index=True)
lan_id: str = Field(foreign_key="lans.id", index=True)
is_bridge: bool = Field(default=False)
forwards_l3: bool = Field(default=False)
class TopologyStatusEvent(SQLModel, table=True):
"""Append-only audit log of topology status transitions."""
__tablename__ = "topology_status_events"
id: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
topology_id: str = Field(foreign_key="topologies.id", index=True)
from_status: str
to_status: str
at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
reason: Optional[str] = Field(
default=None, sa_column=Column("reason", Text, nullable=True)
)
class TopologyMutation(SQLModel, table=True):
"""Operator-requested live mutation for an active MazeNET topology.
Each row is one intent (add LAN, attach decky, etc.). The mutator's
reconciler claims ``pending`` rows atomically (see
``SQLModelRepository.claim_next_mutation``), applies them against
Docker, and writes ``applied`` or ``failed`` back. The ``(state,
topology_id)`` composite index keeps the watch-loop guard query
cheap even with years of mutation history.
"""
__tablename__ = "topology_mutations"
__table_args__ = (
Index(
"ix_topology_mutations_state_topology",
"state",
"topology_id",
),
)
id: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
topology_id: str = Field(foreign_key="topologies.id", index=True)
# add_lan|remove_lan|add_decky|attach_decky|detach_decky|
# remove_decky|update_decky|update_lan
op: str = Field(index=True)
# JSON-serialised op payload (keys depend on ``op``).
payload: str = Field(
sa_column=Column("payload", _BIG_TEXT, nullable=False, default="{}")
)
# pending|applying|applied|failed
state: str = Field(default="pending", index=True)
requested_at: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc), index=True
)
applied_at: Optional[datetime] = Field(default=None)
reason: Optional[str] = Field(
default=None, sa_column=Column("reason", Text, nullable=True)
)
# --- MazeNET Topology REST DTOs (phase 3) ---
# Request/response shapes for /api/v1/topologies. All write paths are
# admin-only; reads accept admin or viewer. Child CRUD is pending-only;
# mutations of active|degraded topologies go through the queue.
class TopologyGenerateRequest(BaseModel):
"""Body for POST /topologies — mirrors the `topology generate` CLI."""
name: str = PydanticField(..., min_length=1, max_length=64)
mode: str = PydanticField(default="unihost", pattern=r"^(unihost|agent)$")
target_host_uuid: Optional[str] = None
depth: int = PydanticField(..., ge=1, le=16)
branching_factor: int = PydanticField(..., ge=1, le=8)
deckies_per_lan_min: int = PydanticField(default=1, ge=0, le=32)
deckies_per_lan_max: int = PydanticField(default=3, ge=1, le=32)
bridge_forward_probability: float = PydanticField(default=1.0, ge=0.0, le=1.0)
cross_edge_probability: float = PydanticField(default=0.0, ge=0.0, le=1.0)
services_explicit: Optional[list[str]] = None
randomize_services: bool = True
seed: Optional[int] = PydanticField(default=None, ge=0)
class TopologySummary(BaseModel):
"""List-row shape for GET /topologies."""
model_config = ConfigDict(extra="ignore")
id: str
name: str
mode: str
target_host_uuid: Optional[str] = None
status: str
version: int
needs_resync: bool = False
created_at: datetime
status_changed_at: Optional[datetime] = None
class TopologyListResponse(BaseModel):
total: int
limit: Optional[int] = None
offset: Optional[int] = None
data: list[TopologySummary]
class LANRow(BaseModel):
model_config = ConfigDict(extra="ignore")
id: str
topology_id: str
name: str
subnet: str
is_dmz: bool = False
docker_network_id: Optional[str] = None
x: Optional[float] = None
y: Optional[float] = None
class DeckyRow(BaseModel):
model_config = ConfigDict(extra="ignore")
uuid: str
topology_id: str
name: str
services: list[str] = PydanticField(default_factory=list)
decky_config: Optional[dict[str, Any]] = None
ip: Optional[str] = None
state: str
last_error: Optional[str] = None
x: Optional[float] = None
y: Optional[float] = None
class EdgeRow(BaseModel):
model_config = ConfigDict(extra="ignore")
id: str
topology_id: str
decky_uuid: str
lan_id: str
is_bridge: bool = False
forwards_l3: bool = False
class TopologyDetail(BaseModel):
"""Hydrated topology — mirrors persistence.hydrate() output.
``topology`` uses :class:`TopologySummary` which already exposes
``target_host_uuid`` — agent-targeted topologies surface their
pinned host through that field.
"""
topology: TopologySummary
lans: list[LANRow]
deckies: list[DeckyRow]
edges: list[EdgeRow]
class TopologyStatusEventRow(BaseModel):
model_config = ConfigDict(extra="ignore")
id: str
topology_id: str
from_status: str
to_status: str
at: datetime
reason: Optional[str] = None
class LANCreateRequest(BaseModel):
name: str = PydanticField(..., min_length=1, max_length=64)
subnet: Optional[str] = None
is_dmz: bool = False
x: Optional[float] = None
y: Optional[float] = None
expected_version: Optional[int] = None
class LANUpdateRequest(BaseModel):
name: Optional[str] = None
subnet: Optional[str] = None
is_dmz: Optional[bool] = None
x: Optional[float] = None
y: Optional[float] = None
expected_version: Optional[int] = None
class DeckyCreateRequest(BaseModel):
name: str = PydanticField(..., min_length=1, max_length=64)
services: list[str] = PydanticField(default_factory=list)
decky_config: Optional[dict[str, Any]] = None
x: Optional[float] = None
y: Optional[float] = None
expected_version: Optional[int] = None
class DeckyUpdateRequest(BaseModel):
name: Optional[str] = None
services: Optional[list[str]] = None
decky_config: Optional[dict[str, Any]] = None
x: Optional[float] = None
y: Optional[float] = None
expected_version: Optional[int] = None
class EdgeCreateRequest(BaseModel):
decky_uuid: str
lan_id: str
is_bridge: bool = False
forwards_l3: bool = False
expected_version: Optional[int] = None
_MUTATION_OPS = Literal[
"add_lan",
"remove_lan",
"add_decky",
"attach_decky",
"detach_decky",
"remove_decky",
"update_decky",
"update_lan",
]
class MutationEnqueueRequest(BaseModel):
op: _MUTATION_OPS
payload: dict[str, Any] = PydanticField(default_factory=dict)
expected_version: Optional[int] = None
def _decode_json_payload(v: Any) -> Any:
"""Accept either a dict or a JSON-encoded string for mutation payloads."""
if isinstance(v, str):
import json as _json
return _json.loads(v) if v else {}
return v
_MutationPayload = Annotated[dict[str, Any], BeforeValidator(_decode_json_payload)]
class MutationRow(BaseModel):
model_config = ConfigDict(extra="ignore")
id: str
topology_id: str
op: str
payload: _MutationPayload = PydanticField(default_factory=dict)
state: str
requested_at: datetime
applied_at: Optional[datetime] = None
reason: Optional[str] = None
class MutationEnqueueResponse(BaseModel):
mutation_id: str
state: str = "pending"
class ValidationIssueResponse(BaseModel):
severity: str
code: str
message: str
target: dict[str, Any] = PydanticField(default_factory=dict)
class ValidationErrorResponse(BaseModel):
detail: str = "Topology validation failed"
issues: list[ValidationIssueResponse]
class VersionConflictResponse(BaseModel):
detail: str = "Topology version conflict"
current: int
expected: int
class NotEditableResponse(BaseModel):
detail: str = "Topology not editable"
status: str
reason: Optional[str] = None
class ServiceCatalogResponse(BaseModel):
services: list[str]
class ArchetypeEntry(BaseModel):
slug: str
display_name: str
description: str
services: list[str]
preferred_distros: list[str]
nmap_os: str
class ArchetypeCatalogResponse(BaseModel):
archetypes: list[ArchetypeEntry]
class NextIPResponse(BaseModel):
subnet: str
ip: str
class NextSubnetResponse(BaseModel):
subnet: str
class DeployAcceptedResponse(BaseModel):
topology_id: str
status: str
dry_run: bool = False
class ReapReportResponse(BaseModel):
live_prefixes: list[str]
orphan_prefixes: list[str]
containers_removed: list[str]
networks_removed: list[str]
errors: list[str]

73
decnet/web/db/models/updater.py Normal file
View File

@@ -0,0 +1,73 @@
"""Remote updates DTOs (master → worker /updater fan-out)."""
from typing import Any, Literal, Optional
from pydantic import BaseModel, Field as PydanticField
# --- Remote Updates (master → worker /updater) DTOs ---
# Powers the dashboard's Remote Updates page. The master dashboard calls
# these (auth-gated) endpoints; internally they fan out to each worker's
# updater daemon over mTLS via UpdaterClient.
class HostReleaseInfo(BaseModel):
host_uuid: str
host_name: str
address: str
reachable: bool
# These fields mirror the updater's /health payload when reachable; they
# are all Optional so an unreachable host still serializes cleanly.
agent_status: Optional[str] = None
current_sha: Optional[str] = None
previous_sha: Optional[str] = None
releases: list[dict[str, Any]] = PydanticField(default_factory=list)
detail: Optional[str] = None # populated when unreachable
class HostReleasesResponse(BaseModel):
hosts: list[HostReleaseInfo]
class PushUpdateRequest(BaseModel):
host_uuids: Optional[list[str]] = PydanticField(
default=None,
description="Target specific hosts; mutually exclusive with 'all'.",
)
all: bool = PydanticField(default=False, description="Target every non-decommissioned host with an updater bundle.")
include_self: bool = PydanticField(
default=False,
description="After a successful /update, also push /update-self to upgrade the updater itself.",
)
exclude: list[str] = PydanticField(
default_factory=list,
description="Additional tarball exclude globs (on top of the built-in defaults).",
)
class PushUpdateResult(BaseModel):
host_uuid: str
host_name: str
# updated = /update 200. rolled-back = /update 409 (auto-recovered).
# failed = transport error or non-200/409 response. self-updated = /update-self succeeded.
status: Literal["updated", "rolled-back", "failed", "self-updated", "self-failed"]
http_status: Optional[int] = None
sha: Optional[str] = None
detail: Optional[str] = None
stderr: Optional[str] = None
class PushUpdateResponse(BaseModel):
sha: str
tarball_bytes: int
results: list[PushUpdateResult]
class RollbackRequest(BaseModel):
host_uuid: str = PydanticField(..., description="Host to roll back to its previous release slot.")
class RollbackResponse(BaseModel):
host_uuid: str
host_name: str
status: Literal["rolled-back", "failed"]
http_status: Optional[int] = None
detail: Optional[str] = None

162
decnet/web/db/models/webhooks.py Normal file
View File

@@ -0,0 +1,162 @@
"""Webhook subscription table + CRUD DTOs.
Webhooks push DECNET bus events out to external SIEM / SOAR stacks
(Wazuh, Shuffle, TheHive, n8n, ...). Each subscription carries a set
of NATS-style topic patterns; the `decnet webhook` worker subscribes
to the union of patterns across all enabled subscriptions and POSTs
matching events to each matching URL with HMAC-SHA256 signing.
Simple mode (UI) exposes a friendly enum (`AttackerDetail`,
`DeckyStatus`, `SystemStatus`) that expands to patterns at save time.
Advanced mode lets an admin set raw patterns directly. Storage is
always the expanded list — the enum is sugar at the router layer.
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, List, Literal, Optional
from uuid import uuid4
from pydantic import BaseModel, Field as PydanticField, HttpUrl
from sqlmodel import Field, SQLModel
SimpleEvent = Literal["AttackerDetail", "DeckyStatus", "SystemStatus"]
class WebhookSubscription(SQLModel, table=True):
__tablename__ = "webhook_subscriptions"
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
name: str = Field(index=True, unique=True)
url: str
secret: str # HMAC-SHA256 key; plaintext pre-v1 (see DEBT-037 §7)
# JSON-encoded list[str] of NATS-style bus topic patterns.
# Storing as TEXT keeps the schema portable across SQLite and MySQL
# without pulling in dialect-specific JSON columns.
topic_patterns: str = Field(default="[]")
enabled: bool = Field(default=True, index=True)
consecutive_failures: int = Field(default=0)
last_success_at: Optional[datetime] = None
last_failure_at: Optional[datetime] = None
last_error: Optional[str] = None
# Set when the circuit breaker auto-disables the subscription after
# too many consecutive failures. NULL means "not tripped" — the
# subscription is either active (enabled=True) or admin-paused
# (enabled=False, auto_disabled_at=NULL). A non-NULL stamp with
# enabled=False means the worker tripped it; the operator clears
# the flag by re-enabling via PATCH.
auto_disabled_at: Optional[datetime] = None
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
def patterns(self) -> list[str]:
"""Decode `topic_patterns` to a list. Returns [] on bad/empty JSON."""
try:
raw = json.loads(self.topic_patterns or "[]")
except (ValueError, TypeError):
return []
return [p for p in raw if isinstance(p, str)]
# --- API Request / Response Models (Pydantic) ---
class WebhookCreateRequest(BaseModel):
name: str = PydanticField(..., min_length=1, max_length=64)
url: HttpUrl
# If secret is omitted, the router generates a secure random one and
# returns it exactly once on the create response. After that, callers
# can only rotate via PATCH.
secret: Optional[str] = PydanticField(None, min_length=16, max_length=256)
# At least one of simple_events / topic_patterns must be non-empty
# (validated in the router, not Pydantic, so the 400 carries a clear
# detail message).
simple_events: List[SimpleEvent] = PydanticField(default_factory=list)
topic_patterns: List[str] = PydanticField(default_factory=list)
enabled: bool = True
class WebhookUpdateRequest(BaseModel):
# Partial update — every field optional; the router diffs against the
# current row and only writes what changed.
name: Optional[str] = PydanticField(None, min_length=1, max_length=64)
url: Optional[HttpUrl] = None
secret: Optional[str] = PydanticField(None, min_length=16, max_length=256)
simple_events: Optional[List[SimpleEvent]] = None
topic_patterns: Optional[List[str]] = None
enabled: Optional[bool] = None
class WebhookResponse(BaseModel):
"""Public shape — deliberately omits `secret`.
The `warnings` field carries non-blocking advisories about the
subscription's configuration — e.g. an `http://` URL is fine but
surfaces a warning so the operator knows the event body is
plaintext on the wire. Empty list when nothing is worth flagging.
"""
uuid: str
name: str
url: str
topic_patterns: List[str]
enabled: bool
consecutive_failures: int
last_success_at: Optional[datetime] = None
last_failure_at: Optional[datetime] = None
last_error: Optional[str] = None
auto_disabled_at: Optional[datetime] = None
created_at: datetime
updated_at: datetime
warnings: List[str] = PydanticField(default_factory=list)
class WebhookCreateResponse(WebhookResponse):
"""Create-path response — carries the secret exactly once, for copy-out."""
secret: str
class WebhookTestResponse(BaseModel):
delivered: bool
status_code: Optional[int] = None
error: Optional[str] = None
def _compute_warnings(url: str) -> List[str]:
"""Non-blocking advisories about a subscription's configuration.
The HMAC signature detects tampering regardless of transport, but an
on-path attacker can still *read* the event body over plaintext HTTP.
We surface the warning and let the admin decide — matches DECNET's
operator-trust posture (see THREAT_MODEL WH-03).
"""
out: List[str] = []
lower = (url or "").lower()
if lower.startswith("http://"):
out.append(
"insecure_url: URL uses http://. Event bodies (including "
"payload fields) traverse the wire in plaintext; HMAC still "
"detects tampering but anyone on-path can read the event. "
"Use https:// in production."
)
return out
def _row_to_response_dict(row: dict[str, Any]) -> dict[str, Any]:
"""Normalize a DB row into the WebhookResponse dict shape.
Used by the CRUD router to decode `topic_patterns` JSON, drop the
`secret` column, and compute any configuration warnings.
"""
out = dict(row)
raw = out.pop("topic_patterns", "[]")
try:
out["topic_patterns"] = json.loads(raw or "[]")
except (ValueError, TypeError):
out["topic_patterns"] = []
out.pop("secret", None)
out["warnings"] = _compute_warnings(out.get("url", ""))
return out

50
decnet/web/db/models/workers.py Normal file
View File

@@ -0,0 +1,50 @@
"""Workers panel DTOs (bus-backed health + control)."""
from typing import Any, Dict, List, Literal, Optional
from pydantic import BaseModel, Field as PydanticField
# --- Workers panel (Config → Workers) ---
# Bus-backed health + control: workers heartbeat on ``system.<name>.health``
# and listen on ``system.<name>.control``. The API aggregates last-seen
# heartbeats via the worker registry; these are the HTTP-facing shapes.
class WorkerStatus(BaseModel):
name: str
# ``ok`` — heartbeat within 90s (3× 30s heartbeat interval)
# ``stale`` — worker was seen before but hasn't pulsed in 90s+
# ``unknown`` — we've never received a heartbeat from this name
status: Literal["ok", "stale", "unknown"]
last_heartbeat_ts: Optional[float] = None
seconds_since: Optional[float] = None
# Whatever the worker's ``extra()`` callback put in the heartbeat;
# opaque to the panel, displayed only if the UI knows the key.
extra: Dict[str, Any] = PydanticField(default_factory=dict)
# True iff a ``decnet-<name>.service`` unit file is present on the
# host. False flips the UI START button to disabled with a
# "Unit not installed" tooltip. Default True for backwards compat
# on clients that pre-date the field.
installed: bool = True
class WorkersResponse(BaseModel):
workers: List[WorkerStatus]
generated_at: float
bus_connected: bool
class WorkerControlResponse(BaseModel):
accepted: bool
worker: str
action: str
class StartFailure(BaseModel):
name: str
reason: str
class StartAllResponse(BaseModel):
started: List[str]
already_running: List[str]
failed: List[StartFailure]

0
decnet/web/db/mysql/__init__.py Normal file
View File

98
decnet/web/db/mysql/database.py Normal file
View File

@@ -0,0 +1,98 @@
"""
MySQL async engine factory.
Builds a SQLAlchemy AsyncEngine against MySQL using the ``asyncmy`` driver.
Connection info is resolved (in order of precedence):
1. An explicit ``url`` argument passed to :func:`get_async_engine`
2. ``DECNET_DB_URL`` — full SQLAlchemy URL
3. Component env vars:
``DECNET_DB_HOST`` (default ``localhost``)
``DECNET_DB_PORT`` (default ``3306``)
``DECNET_DB_NAME`` (default ``decnet``)
``DECNET_DB_USER`` (default ``decnet``)
``DECNET_DB_PASSWORD`` (default empty — raises unless pytest is running)
"""
from __future__ import annotations
import os
from typing import Optional
from urllib.parse import quote_plus
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
DEFAULT_POOL_SIZE = int(os.environ.get("DECNET_DB_POOL_SIZE", "20"))
DEFAULT_MAX_OVERFLOW = int(os.environ.get("DECNET_DB_MAX_OVERFLOW", "40"))
DEFAULT_POOL_RECYCLE = int(os.environ.get("DECNET_DB_POOL_RECYCLE", "3600"))
DEFAULT_POOL_PRE_PING = os.environ.get("DECNET_DB_POOL_PRE_PING", "true").lower() == "true"
def build_mysql_url(
host: Optional[str] = None,
port: Optional[int] = None,
database: Optional[str] = None,
user: Optional[str] = None,
password: Optional[str] = None,
) -> str:
"""Compose an async SQLAlchemy URL for MySQL using the asyncmy driver.
Component args override env vars. Password is percent-encoded so special
characters (``@``, ``:``, ``/``…) don't break URL parsing.
"""
host = host or os.environ.get("DECNET_DB_HOST", "localhost")
port = port or int(os.environ.get("DECNET_DB_PORT", "3306"))
database = database or os.environ.get("DECNET_DB_NAME", "decnet")
user = user or os.environ.get("DECNET_DB_USER", "decnet")
if password is None:
password = os.environ.get("DECNET_DB_PASSWORD", "")
# Allow empty passwords during tests (pytest sets PYTEST_* env vars).
# Outside tests, an empty MySQL password is almost never intentional.
if not password and not any(k.startswith("PYTEST") for k in os.environ):
raise ValueError(
"DECNET_DB_PASSWORD is not set. Either export it, set DECNET_DB_URL, "
"or run under pytest for an empty-password default."
)
pw_enc = quote_plus(password)
user_enc = quote_plus(user)
return f"mysql+asyncmy://{user_enc}:{pw_enc}@{host}:{port}/{database}"
def resolve_url(url: Optional[str] = None) -> str:
"""Pick a connection URL: explicit arg → DECNET_DB_URL env → built from components."""
if url:
return url
env_url = os.environ.get("DECNET_DB_URL")
if env_url:
return env_url
return build_mysql_url()
def get_async_engine(
url: Optional[str] = None,
*,
pool_size: int = DEFAULT_POOL_SIZE,
max_overflow: int = DEFAULT_MAX_OVERFLOW,
pool_recycle: int = DEFAULT_POOL_RECYCLE,
pool_pre_ping: bool = DEFAULT_POOL_PRE_PING,
echo: bool = False,
) -> AsyncEngine:
"""Create an AsyncEngine for MySQL.
Defaults tuned for a dashboard workload: a modest pool, hourly recycle
to sidestep MySQL's idle-connection reaper, and pre-ping to fail fast
if a pooled connection has been killed server-side.
"""
dsn = resolve_url(url)
return create_async_engine(
dsn,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_recycle=pool_recycle,
pool_pre_ping=pool_pre_ping,
)

187
decnet/web/db/mysql/repository.py Normal file
View File

@@ -0,0 +1,187 @@
"""
MySQL implementation of :class:`BaseRepository`.
Inherits the portable SQLModel query code from :class:`SQLModelRepository`
and only overrides the two places where MySQL's SQL dialect differs from
SQLite's:
* :meth:`_migrate_attackers_table` — uses ``information_schema`` (MySQL
has no ``PRAGMA``).
* :meth:`get_log_histogram` — uses ``FROM_UNIXTIME`` /
``UNIX_TIMESTAMP`` + integer division for bucketing.
"""
from __future__ import annotations
from typing import List, Optional
from sqlalchemy import func, select, text, literal_column
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from sqlmodel.sql.expression import SelectOfScalar
from decnet.web.db.models import Log
from decnet.web.db.mysql.database import get_async_engine
from decnet.web.db.sqlmodel_repo import SQLModelRepository
class MySQLRepository(SQLModelRepository):
"""MySQL backend — uses ``asyncmy``."""
def __init__(self, url: Optional[str] = None, **engine_kwargs) -> None:
self.engine = get_async_engine(url=url, **engine_kwargs)
self.session_factory = async_sessionmaker(
self.engine, class_=AsyncSession, expire_on_commit=False
)
async def _migrate_attackers_table(self) -> None:
"""Drop the legacy (pre-UUID) ``attackers`` table if it exists without a ``uuid`` column.
Also adds the GeoIP columns (``country_code``, ``country_source``)
to existing tables that predate them. MySQL exposes column
metadata via ``information_schema.COLUMNS``; ``DATABASE()`` scopes
the lookup to the currently connected schema.
"""
async with self.engine.begin() as conn:
rows = (await conn.execute(text(
"SELECT COLUMN_NAME FROM information_schema.COLUMNS "
"WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'attackers'"
))).fetchall()
if not rows:
return # table absent; create_all() handles it.
if not any(r[0] == "uuid" for r in rows):
await conn.execute(text("DROP TABLE attackers"))
return
existing_cols = {r[0] for r in rows}
if "country_code" not in existing_cols:
await conn.execute(text(
"ALTER TABLE attackers "
"ADD COLUMN country_code VARCHAR(2) NULL, "
"ADD INDEX ix_attackers_country_code (country_code)"
))
if "country_source" not in existing_cols:
await conn.execute(text(
"ALTER TABLE attackers ADD COLUMN country_source VARCHAR(16) NULL"
))
async def _migrate_column_types(self) -> None:
"""Upgrade TEXT → MEDIUMTEXT for columns that accumulate large JSON blobs.
``create_all()`` never alters existing columns, so tables created before
``_BIG_TEXT`` was introduced keep their 64 KiB ``TEXT`` cap. This method
inspects ``information_schema`` and issues ``ALTER TABLE … MODIFY COLUMN``
for each offending column found.
"""
targets: dict[str, dict[str, str]] = {
"attackers": {
"commands": "MEDIUMTEXT NOT NULL DEFAULT '[]'",
"fingerprints": "MEDIUMTEXT NOT NULL DEFAULT '[]'",
"services": "MEDIUMTEXT NOT NULL DEFAULT '[]'",
"deckies": "MEDIUMTEXT NOT NULL DEFAULT '[]'",
},
"state": {
"value": "MEDIUMTEXT NOT NULL",
},
}
async with self.engine.begin() as conn:
rows = (await conn.execute(text(
"SELECT TABLE_NAME, COLUMN_NAME FROM information_schema.COLUMNS "
"WHERE TABLE_SCHEMA = DATABASE() "
" AND TABLE_NAME IN ('attackers', 'state') "
" AND COLUMN_NAME IN ('commands','fingerprints','services','deckies','value') "
" AND DATA_TYPE = 'text'"
))).fetchall()
for table_name, col_name in rows:
spec = targets.get(table_name, {}).get(col_name)
if spec:
await conn.execute(text(
f"ALTER TABLE `{table_name}` MODIFY COLUMN `{col_name}` {spec}"
))
async def _migrate_session_profile_table(self) -> None:
"""Add DEBT-036 keystroke-dynamics columns (start-of-action latency,
three-bucket pause histogram, top-bigrams JSON) to existing tables.
MySQL's ``ALTER TABLE ADD COLUMN`` fails if the column already
exists, so gate on ``information_schema.COLUMNS`` to stay
idempotent.
"""
async with self.engine.begin() as conn:
rows = (await conn.execute(text(
"SELECT COLUMN_NAME FROM information_schema.COLUMNS "
"WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'session_profile'"
))).fetchall()
if not rows:
return
existing_cols = {r[0] for r in rows}
additions = [
("kd_top_bigrams", "TEXT NULL"),
("kd_start_of_action_latency", "DOUBLE NULL"),
("kd_pause_hist_burst", "INT NULL"),
("kd_pause_hist_think", "INT NULL"),
("kd_pause_hist_distracted", "INT NULL"),
]
for col_name, col_spec in additions:
if col_name not in existing_cols:
await conn.execute(text(
f"ALTER TABLE session_profile ADD COLUMN {col_name} {col_spec}"
))
async def initialize(self) -> None:
"""Create tables and run all MySQL-specific migrations.
Uses a MySQL advisory lock to serialize DDL across concurrent
uvicorn workers — prevents the 'Table was skipped since its
definition is being modified by concurrent DDL' race.
"""
from sqlmodel import SQLModel
async with self.engine.connect() as lock_conn:
await lock_conn.execute(text("SELECT GET_LOCK('decnet_schema_init', 30)"))
try:
await self._migrate_attackers_table()
await self._migrate_session_profile_table()
await self._migrate_column_types()
async with self.engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
await self._ensure_admin_user()
finally:
await lock_conn.execute(text("SELECT RELEASE_LOCK('decnet_schema_init')"))
await lock_conn.close()
def _json_field_equals(self, key: str):
# MySQL 5.7+ exposes JSON_EXTRACT; quoted string result returned for
# TEXT-stored JSON, same behavior we rely on in SQLite.
return text(f"JSON_UNQUOTE(JSON_EXTRACT(fields, '$.{key}')) = :val")
async def get_log_histogram(
self,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
interval_minutes: int = 15,
) -> List[dict]:
bucket_seconds = max(interval_minutes, 1) * 60
# Truncate each timestamp to the start of its bucket:
# FROM_UNIXTIME( (UNIX_TIMESTAMP(timestamp) DIV N) * N )
# DIV is MySQL's integer division operator.
bucket_expr = literal_column(
f"FROM_UNIXTIME((UNIX_TIMESTAMP(timestamp) DIV {bucket_seconds}) * {bucket_seconds})"
).label("bucket_time")
statement: SelectOfScalar = select(bucket_expr, func.count().label("count")).select_from(Log)
statement = self._apply_filters(statement, search, start_time, end_time)
statement = statement.group_by(literal_column("bucket_time")).order_by(
literal_column("bucket_time")
)
async with self._session() as session:
results = await session.execute(statement)
# Normalize to ISO string for API parity with the SQLite backend
# (SQLite's datetime() returns a string already; FROM_UNIXTIME
# returns a datetime).
out: List[dict] = []
for r in results.all():
ts = r[0]
out.append({
"time": ts.isoformat(sep=" ") if hasattr(ts, "isoformat") else ts,
"count": r[1],
})
return out

1128
decnet/web/db/repository.py
View File

File diff suppressed because it is too large Load Diff

33
decnet/web/db/sqlite/database.py
View File

@@ -1,5 +1,7 @@
import os
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy import create_engine, Engine
from sqlalchemy import create_engine, Engine, event
from sqlmodel import SQLModel
from typing import AsyncGenerator
@@ -11,7 +13,34 @@ def get_async_engine(db_path: str) -> AsyncEngine:
prefix = "sqlite+aiosqlite:///"
if db_path.startswith(":memory:"):
prefix = "sqlite+aiosqlite://"
return create_async_engine(f"{prefix}{db_path}", echo=False, connect_args={"uri": True})
pool_size = int(os.environ.get("DECNET_DB_POOL_SIZE", "20"))
max_overflow = int(os.environ.get("DECNET_DB_MAX_OVERFLOW", "40"))
pool_recycle = int(os.environ.get("DECNET_DB_POOL_RECYCLE", "3600"))
# SQLite is a local file — dead-connection probes are pure overhead.
# Env var stays for network-mounted setups that still want it.
pool_pre_ping = os.environ.get("DECNET_DB_POOL_PRE_PING", "false").lower() == "true"
engine = create_async_engine(
f"{prefix}{db_path}",
echo=False,
pool_size=pool_size,
max_overflow=max_overflow,
pool_recycle=pool_recycle,
pool_pre_ping=pool_pre_ping,
connect_args={"uri": True, "timeout": 30},
)
@event.listens_for(engine.sync_engine, "connect")
def _set_sqlite_pragmas(dbapi_conn, _conn_record):
cursor = dbapi_conn.cursor()
cursor.execute("PRAGMA journal_mode=WAL")
cursor.execute("PRAGMA synchronous=NORMAL")
cursor.execute("PRAGMA busy_timeout=30000")
cursor.close()
return engine
def get_sync_engine(db_path: str) -> Engine:
prefix = "sqlite:///"

404
decnet/web/db/sqlite/repository.py
View File

@@ -1,23 +1,22 @@
import asyncio
import json
import uuid
from datetime import datetime
from typing import Any, Optional, List
from typing import List, Optional
from sqlalchemy import func, select, desc, asc, text, or_, update, literal_column
from sqlalchemy import func, select, text, literal_column
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from sqlmodel.sql.expression import SelectOfScalar
from decnet.config import load_state, _ROOT
from decnet.env import DECNET_ADMIN_USER, DECNET_ADMIN_PASSWORD
from decnet.web.auth import get_password_hash
from decnet.web.db.repository import BaseRepository
from decnet.web.db.models import User, Log, Bounty, State
from decnet.config import _ROOT
from decnet.web.db.models import Log
from decnet.web.db.sqlite.database import get_async_engine
from decnet.web.db.sqlmodel_repo import SQLModelRepository
class SQLiteRepository(BaseRepository):
"""SQLite implementation using SQLModel and SQLAlchemy Async."""
class SQLiteRepository(SQLModelRepository):
"""SQLite backend — uses ``aiosqlite``.
Overrides the two places where SQLite's SQL dialect differs from
MySQL/PostgreSQL: legacy-schema migration (via ``PRAGMA table_info``)
and the log-histogram bucket expression (via ``strftime`` + ``unixepoch``).
"""
def __init__(self, db_path: str = str(_ROOT / "decnet.db")) -> None:
self.db_path = db_path
@@ -26,173 +25,63 @@ class SQLiteRepository(BaseRepository):
self.engine, class_=AsyncSession, expire_on_commit=False
)
async def initialize(self) -> None:
"""Async warm-up / verification. Creates tables if they don't exist."""
from sqlmodel import SQLModel
async def _migrate_attackers_table(self) -> None:
"""Drop the old attackers table if it lacks the uuid column (pre-UUID schema).
Also adds the GeoIP columns (``country_code``, ``country_source``)
to existing tables that predate them. SQLite's
``ALTER TABLE ADD COLUMN`` is idempotent only if we gate on
``PRAGMA table_info`` first — re-adding raises.
"""
async with self.engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
async with self.session_factory() as session:
# Check if admin exists
result = await session.execute(
select(User).where(User.username == DECNET_ADMIN_USER)
)
if not result.scalar_one_or_none():
session.add(User(
uuid=str(uuid.uuid4()),
username=DECNET_ADMIN_USER,
password_hash=get_password_hash(DECNET_ADMIN_PASSWORD),
role="admin",
must_change_password=True,
rows = (await conn.execute(text("PRAGMA table_info(attackers)"))).fetchall()
if rows and not any(r[1] == "uuid" for r in rows):
await conn.execute(text("DROP TABLE attackers"))
return # create_all() rebuilds fresh — no need to patch columns.
if not rows:
return # table absent; create_all() handles it.
existing_cols = {r[1] for r in rows}
if "country_code" not in existing_cols:
await conn.execute(text(
"ALTER TABLE attackers ADD COLUMN country_code VARCHAR(2)"
))
await conn.execute(text(
"CREATE INDEX IF NOT EXISTS ix_attackers_country_code "
"ON attackers (country_code)"
))
if "country_source" not in existing_cols:
await conn.execute(text(
"ALTER TABLE attackers ADD COLUMN country_source VARCHAR(16)"
))
await session.commit()
async def reinitialize(self) -> None:
"""Initialize the database schema asynchronously (useful for tests)."""
from sqlmodel import SQLModel
async def _migrate_session_profile_table(self) -> None:
"""Add DEBT-036 keystroke-dynamics columns (start-of-action latency,
three-bucket pause histogram, top-bigrams JSON) to existing tables.
SQLite's ``ALTER TABLE ADD COLUMN`` fails if the column already
exists, so gate on ``PRAGMA table_info`` to stay idempotent.
"""
async with self.engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
rows = (await conn.execute(text("PRAGMA table_info(session_profile)"))).fetchall()
if not rows:
return # table absent; create_all() handles it.
existing_cols = {r[1] for r in rows}
additions = [
("kd_top_bigrams", "TEXT"),
("kd_start_of_action_latency", "REAL"),
("kd_pause_hist_burst", "INTEGER"),
("kd_pause_hist_think", "INTEGER"),
("kd_pause_hist_distracted", "INTEGER"),
]
for col_name, col_type in additions:
if col_name not in existing_cols:
await conn.execute(text(
f"ALTER TABLE session_profile ADD COLUMN {col_name} {col_type}"
))
async with self.session_factory() as session:
result = await session.execute(
select(User).where(User.username == DECNET_ADMIN_USER)
)
if not result.scalar_one_or_none():
session.add(User(
uuid=str(uuid.uuid4()),
username=DECNET_ADMIN_USER,
password_hash=get_password_hash(DECNET_ADMIN_PASSWORD),
role="admin",
must_change_password=True,
))
await session.commit()
# ------------------------------------------------------------------ logs
async def add_log(self, log_data: dict[str, Any]) -> None:
data = log_data.copy()
if "fields" in data and isinstance(data["fields"], dict):
data["fields"] = json.dumps(data["fields"])
if "timestamp" in data and isinstance(data["timestamp"], str):
try:
data["timestamp"] = datetime.fromisoformat(
data["timestamp"].replace("Z", "+00:00")
)
except ValueError:
pass
async with self.session_factory() as session:
session.add(Log(**data))
await session.commit()
def _apply_filters(
self,
statement: SelectOfScalar,
search: Optional[str],
start_time: Optional[str],
end_time: Optional[str],
) -> SelectOfScalar:
import re
import shlex
if start_time:
statement = statement.where(Log.timestamp >= start_time)
if end_time:
statement = statement.where(Log.timestamp <= end_time)
if search:
try:
tokens = shlex.split(search)
except ValueError:
tokens = search.split()
core_fields = {
"decky": Log.decky,
"service": Log.service,
"event": Log.event_type,
"attacker": Log.attacker_ip,
"attacker-ip": Log.attacker_ip,
"attacker_ip": Log.attacker_ip,
}
for token in tokens:
if ":" in token:
key, val = token.split(":", 1)
if key in core_fields:
statement = statement.where(core_fields[key] == val)
else:
key_safe = re.sub(r"[^a-zA-Z0-9_]", "", key)
if key_safe:
statement = statement.where(
text(f"json_extract(fields, '$.{key_safe}') = :val")
).params(val=val)
else:
lk = f"%{token}%"
statement = statement.where(
or_(
Log.raw_line.like(lk),
Log.decky.like(lk),
Log.service.like(lk),
Log.attacker_ip.like(lk),
)
)
return statement
async def get_logs(
self,
limit: int = 50,
offset: int = 0,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
) -> List[dict]:
statement = (
select(Log)
.order_by(desc(Log.timestamp))
.offset(offset)
.limit(limit)
)
statement = self._apply_filters(statement, search, start_time, end_time)
async with self.session_factory() as session:
results = await session.execute(statement)
return [log.model_dump(mode='json') for log in results.scalars().all()]
async def get_max_log_id(self) -> int:
async with self.session_factory() as session:
result = await session.execute(select(func.max(Log.id)))
val = result.scalar()
return val if val is not None else 0
async def get_logs_after_id(
self,
last_id: int,
limit: int = 50,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
) -> List[dict]:
statement = (
select(Log).where(Log.id > last_id).order_by(asc(Log.id)).limit(limit)
)
statement = self._apply_filters(statement, search, start_time, end_time)
async with self.session_factory() as session:
results = await session.execute(statement)
return [log.model_dump(mode='json') for log in results.scalars().all()]
async def get_total_logs(
self,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
) -> int:
statement = select(func.count()).select_from(Log)
statement = self._apply_filters(statement, search, start_time, end_time)
async with self.session_factory() as session:
result = await session.execute(statement)
return result.scalar() or 0
def _json_field_equals(self, key: str):
# SQLite stores JSON as text; json_extract is the canonical accessor.
return text(f"json_extract(fields, '$.{key}') = :val")
async def get_log_histogram(
self,
@@ -206,173 +95,12 @@ class SQLiteRepository(BaseRepository):
f"datetime((strftime('%s', timestamp) / {bucket_seconds}) * {bucket_seconds}, 'unixepoch')"
).label("bucket_time")
statement = select(bucket_expr, func.count().label("count")).select_from(Log)
statement: SelectOfScalar = select(bucket_expr, func.count().label("count")).select_from(Log)
statement = self._apply_filters(statement, search, start_time, end_time)
statement = statement.group_by(literal_column("bucket_time")).order_by(
literal_column("bucket_time")
)
async with self.session_factory() as session:
async with self._session() as session:
results = await session.execute(statement)
return [{"time": r[0], "count": r[1]} for r in results.all()]
async def get_stats_summary(self) -> dict[str, Any]:
async with self.session_factory() as session:
total_logs = (
await session.execute(select(func.count()).select_from(Log))
).scalar() or 0
unique_attackers = (
await session.execute(
select(func.count(func.distinct(Log.attacker_ip)))
)
).scalar() or 0
active_deckies = (
await session.execute(
select(func.count(func.distinct(Log.decky)))
)
).scalar() or 0
_state = await asyncio.to_thread(load_state)
deployed_deckies = len(_state[0].deckies) if _state else 0
return {
"total_logs": total_logs,
"unique_attackers": unique_attackers,
"active_deckies": active_deckies,
"deployed_deckies": deployed_deckies,
}
async def get_deckies(self) -> List[dict]:
_state = await asyncio.to_thread(load_state)
return [_d.model_dump() for _d in _state[0].deckies] if _state else []
# ------------------------------------------------------------------ users
async def get_user_by_username(self, username: str) -> Optional[dict]:
async with self.session_factory() as session:
result = await session.execute(
select(User).where(User.username == username)
)
user = result.scalar_one_or_none()
return user.model_dump() if user else None
async def get_user_by_uuid(self, uuid: str) -> Optional[dict]:
async with self.session_factory() as session:
result = await session.execute(
select(User).where(User.uuid == uuid)
)
user = result.scalar_one_or_none()
return user.model_dump() if user else None
async def create_user(self, user_data: dict[str, Any]) -> None:
async with self.session_factory() as session:
session.add(User(**user_data))
await session.commit()
async def update_user_password(
self, uuid: str, password_hash: str, must_change_password: bool = False
) -> None:
async with self.session_factory() as session:
await session.execute(
update(User)
.where(User.uuid == uuid)
.values(
password_hash=password_hash,
must_change_password=must_change_password,
)
)
await session.commit()
# ---------------------------------------------------------------- bounties
async def add_bounty(self, bounty_data: dict[str, Any]) -> None:
data = bounty_data.copy()
if "payload" in data and isinstance(data["payload"], dict):
data["payload"] = json.dumps(data["payload"])
async with self.session_factory() as session:
session.add(Bounty(**data))
await session.commit()
def _apply_bounty_filters(
self,
statement: SelectOfScalar,
bounty_type: Optional[str],
search: Optional[str]
) -> SelectOfScalar:
if bounty_type:
statement = statement.where(Bounty.bounty_type == bounty_type)
if search:
lk = f"%{search}%"
statement = statement.where(
or_(
Bounty.decky.like(lk),
Bounty.service.like(lk),
Bounty.attacker_ip.like(lk),
Bounty.payload.like(lk),
)
)
return statement
async def get_bounties(
self,
limit: int = 50,
offset: int = 0,
bounty_type: Optional[str] = None,
search: Optional[str] = None,
) -> List[dict]:
statement = (
select(Bounty)
.order_by(desc(Bounty.timestamp))
.offset(offset)
.limit(limit)
)
statement = self._apply_bounty_filters(statement, bounty_type, search)
async with self.session_factory() as session:
results = await session.execute(statement)
final = []
for item in results.scalars().all():
d = item.model_dump(mode='json')
try:
d["payload"] = json.loads(d["payload"])
except (json.JSONDecodeError, TypeError):
pass
final.append(d)
return final
async def get_total_bounties(
self, bounty_type: Optional[str] = None, search: Optional[str] = None
) -> int:
statement = select(func.count()).select_from(Bounty)
statement = self._apply_bounty_filters(statement, bounty_type, search)
async with self.session_factory() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def get_state(self, key: str) -> Optional[dict[str, Any]]:
async with self.session_factory() as session:
statement = select(State).where(State.key == key)
result = await session.execute(statement)
state = result.scalar_one_or_none()
if state:
return json.loads(state.value)
return None
async def set_state(self, key: str, value: Any) -> None: # noqa: ANN401
async with self.session_factory() as session:
# Check if exists
statement = select(State).where(State.key == key)
result = await session.execute(statement)
state = result.scalar_one_or_none()
value_json = json.dumps(value)
if state:
state.value = value_json
session.add(state)
else:
new_state = State(key=key, value=value_json)
session.add(new_state)
await session.commit()

166
decnet/web/db/sqlmodel_repo/__init__.py Normal file
View File

@@ -0,0 +1,166 @@
"""
Shared SQLModel-based repository implementation.
Contains all dialect-portable query code used by the SQLite and MySQL
backends. Dialect-specific behavior lives in subclasses:
* engine/session construction (``__init__``)
* ``_migrate_attackers_table`` (legacy schema check; DDL introspection
is not portable)
* ``get_log_histogram`` (date-bucket expression differs per dialect)
"""
from __future__ import annotations
import asyncio
import json
import orjson
import uuid
from typing import Any, Optional, List
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
from decnet.config import load_state
from decnet.env import DECNET_ADMIN_USER, DECNET_ADMIN_PASSWORD
from decnet.web.auth import get_password_hash
from decnet.web.db.repository import BaseRepository
from decnet.web.db.models import State, User
from decnet.web.db.sqlmodel_repo._helpers import ( # noqa: F401 (re-exported for tests/external)
_safe_session,
_detach_close,
_cleanup_tasks,
)
from decnet.web.db.sqlmodel_repo.attacker_intel import AttackerIntelMixin
from decnet.web.db.sqlmodel_repo.attackers import AttackersMixin
from decnet.web.db.sqlmodel_repo.auth import AuthMixin
from decnet.web.db.sqlmodel_repo.bounties import BountiesMixin
from decnet.web.db.sqlmodel_repo.campaigns import CampaignsMixin
from decnet.web.db.sqlmodel_repo.canary import CanaryMixin
from decnet.web.db.sqlmodel_repo.credentials import CredentialsMixin
from decnet.web.db.sqlmodel_repo.deckies import DeckiesMixin
from decnet.web.db.sqlmodel_repo.fleet import FleetMixin
from decnet.web.db.sqlmodel_repo.identities import IdentitiesMixin
from decnet.web.db.sqlmodel_repo.logs import LogsMixin
from decnet.web.db.sqlmodel_repo.orchestrator import OrchestratorMixin
from decnet.web.db.sqlmodel_repo.realism import RealismMixin
from decnet.web.db.sqlmodel_repo.swarm import SwarmMixin
from decnet.web.db.sqlmodel_repo.topology import TopologyMixin
from decnet.web.db.sqlmodel_repo.webhooks import WebhooksMixin
class SQLModelRepository(
AttackerIntelMixin,
AttackersMixin,
AuthMixin,
BountiesMixin,
CampaignsMixin,
CanaryMixin,
CredentialsMixin,
DeckiesMixin,
FleetMixin,
IdentitiesMixin,
LogsMixin,
OrchestratorMixin,
RealismMixin,
SwarmMixin,
TopologyMixin,
WebhooksMixin,
BaseRepository,
):
"""Concrete SQLModel/SQLAlchemy-async repository.
Subclasses provide ``self.engine`` (AsyncEngine) and ``self.session_factory``
in ``__init__``, and override the few dialect-specific helpers.
"""
engine: AsyncEngine
session_factory: async_sessionmaker[AsyncSession]
def _session(self):
"""Return a cancellation-safe session context manager."""
return _safe_session(self.session_factory)
# ------------------------------------------------------------ lifecycle
async def initialize(self) -> None:
"""Create tables if absent and seed the admin user."""
from sqlmodel import SQLModel
await self._migrate_attackers_table()
await self._migrate_session_profile_table()
async with self.engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
await self._ensure_admin_user()
async def reinitialize(self) -> None:
"""Re-create schema (for tests / reset flows). Does NOT drop existing tables."""
from sqlmodel import SQLModel
async with self.engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
await self._ensure_admin_user()
async def _ensure_admin_user(self) -> None:
async with self._session() as session:
result = await session.execute(
select(User).where(User.username == DECNET_ADMIN_USER)
)
existing = result.scalar_one_or_none()
if existing is None:
session.add(User(
uuid=str(uuid.uuid4()),
username=DECNET_ADMIN_USER,
password_hash=get_password_hash(DECNET_ADMIN_PASSWORD),
role="admin",
must_change_password=True,
))
await session.commit()
return
# Self-heal env drift: if admin never finalized their password,
# re-sync the hash from DECNET_ADMIN_PASSWORD. Otherwise leave
# the user's chosen password alone.
if existing.must_change_password:
existing.password_hash = get_password_hash(DECNET_ADMIN_PASSWORD)
session.add(existing)
await session.commit()
async def _migrate_attackers_table(self) -> None:
"""Legacy-schema cleanup. Override per dialect (DDL introspection is non-portable)."""
return None
async def _migrate_session_profile_table(self) -> None:
"""Add DEBT-036 keystroke-dynamics columns to existing session_profile
rows. Override per dialect — DDL introspection is non-portable."""
return None
async def get_deckies(self) -> List[dict]:
_state = await asyncio.to_thread(load_state)
return [_d.model_dump() for _d in _state[0].deckies] if _state else []
# --------------------------------------------------------------- users
async def get_state(self, key: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
statement = select(State).where(State.key == key)
result = await session.execute(statement)
state = result.scalar_one_or_none()
if state:
return json.loads(state.value)
return None
async def set_state(self, key: str, value: Any) -> None: # noqa: ANN401
async with self._session() as session:
statement = select(State).where(State.key == key)
result = await session.execute(statement)
state = result.scalar_one_or_none()
value_json = orjson.dumps(value).decode()
if state:
state.value = value_json
session.add(state)
else:
session.add(State(key=key, value=value_json))
await session.commit()

113
decnet/web/db/sqlmodel_repo/_helpers.py Normal file
View File

@@ -0,0 +1,113 @@
"""Module-level session helpers shared by every repository mixin.
``_safe_session`` and ``_detach_close`` make session cleanup robust under
client-cancellation. See ``_detach_close`` for the full rationale.
``_serialize_json_fields`` / ``_deserialize_json_fields`` live here
because they're used across multiple domain mixins (fleet, topology,
…); putting them in a single mixin would force the others to inherit
that mixin or import a free function — both worse than a shared helper.
"""
from __future__ import annotations
import asyncio
import json
from contextlib import asynccontextmanager
from typing import Any
import orjson
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from decnet.logging import get_logger
_log = get_logger("db.pool")
# Hold strong refs to in-flight cleanup tasks so they aren't GC'd mid-run.
_cleanup_tasks: set[asyncio.Task] = set()
def _detach_close(session: AsyncSession) -> None:
"""Hand session cleanup to a fresh task so the caller's cancellation
doesn't interrupt it.
``asyncio.shield`` doesn't help on the exception path: shield prevents
*other* tasks from cancelling the inner coroutine, but if the *current*
task is already cancelled, its next ``await`` re-raises
``CancelledError`` as soon as the inner coroutine yields. That's what
happens when uvicorn cancels a request mid-query — the rollback inside
``session.close()`` can't complete, and the aiomysql connection is
orphaned (pool logs "non-checked-in connection" on GC).
A fresh task isn't subject to the caller's pending cancellation, so
``close()`` (or the ``invalidate()`` fallback for a dead connection)
runs to completion and the pool reclaims the connection promptly.
Fire-and-forget on purpose: the caller is already unwinding and must
not wait on cleanup.
"""
async def _cleanup() -> None:
try:
await session.close()
except BaseException:
try:
session.sync_session.invalidate()
except BaseException:
_log.debug("detach-close: invalidate failed", exc_info=True)
try:
loop = asyncio.get_running_loop()
except RuntimeError:
# No running loop (shutdown path) — best-effort sync invalidate.
try:
session.sync_session.invalidate()
except BaseException:
_log.debug("detach-close: no-loop invalidate failed", exc_info=True)
return
task = loop.create_task(_cleanup())
_cleanup_tasks.add(task)
# Consume any exception to silence "Task exception was never retrieved".
task.add_done_callback(lambda t: (_cleanup_tasks.discard(t), t.exception()))
@asynccontextmanager
async def _safe_session(factory: async_sessionmaker[AsyncSession]):
"""Session context manager that keeps close() reliable under cancellation.
Success path: await close() inline so the caller observes cleanup
(commit visibility, connection release) before proceeding.
Exception path (includes CancelledError from client disconnects):
detach close() to a fresh task. The caller is unwinding and its
own cancellation would abort an inline close mid-rollback, leaving
the aiomysql connection orphaned.
"""
session = factory()
try:
yield session
except BaseException:
_detach_close(session)
raise
else:
await session.close()
def _serialize_json_fields(data: dict[str, Any], keys: tuple[str, ...]) -> dict[str, Any]:
"""Encode the named keys as JSON strings if they're not already."""
out = dict(data)
for k in keys:
v = out.get(k)
if v is not None and not isinstance(v, str):
out[k] = orjson.dumps(v).decode()
return out
def _deserialize_json_fields(d: dict[str, Any], keys: tuple[str, ...]) -> dict[str, Any]:
"""Decode the named JSON-string keys in place."""
for k in keys:
v = d.get(k)
if isinstance(v, str):
try:
d[k] = json.loads(v)
except (json.JSONDecodeError, TypeError):
pass
return d

102
decnet/web/db/sqlmodel_repo/attacker_intel.py Normal file
View File

@@ -0,0 +1,102 @@
"""Attacker-intel domain methods.
Owns reads/writes for ``AttackerIntel`` rows: per-attacker enrichment
data sourced from external providers (GreyNoise, AbuseIPDB, Feodo,
ThreatFox). Joined against ``Attacker`` for the unenriched-backlog
worker query.
"""
from __future__ import annotations
import json
import uuid as _uuid
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import desc, or_, select
from decnet.web.db.models import Attacker, AttackerIntel
class AttackerIntelMixin:
"""Mixin: methods composed onto ``SQLModelRepository``.
Expects ``self._session()`` from the base.
"""
async def upsert_attacker_intel(self, data: dict[str, Any]) -> str:
attacker_uuid_value = data["attacker_uuid"]
async with self._session() as session:
result = await session.execute(
select(AttackerIntel).where(
AttackerIntel.attacker_uuid == attacker_uuid_value,
)
)
existing = result.scalar_one_or_none()
if existing:
for k, v in data.items():
setattr(existing, k, v)
session.add(existing)
row_uuid = existing.uuid
else:
row_uuid = _uuid.uuid4().hex
session.add(AttackerIntel(uuid=row_uuid, **data))
await session.commit()
return row_uuid
async def get_attacker_intel_by_uuid(
self,
uuid: str,
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(AttackerIntel).where(AttackerIntel.attacker_uuid == uuid)
)
row = result.scalar_one_or_none()
if not row:
return None
d = row.model_dump(mode="json")
for key in (
"greynoise_raw",
"abuseipdb_raw",
"feodo_raw",
"threatfox_raw",
):
raw = d.get(key)
if isinstance(raw, str):
try:
d[key] = json.loads(raw)
except (json.JSONDecodeError, TypeError):
pass
return d
async def get_unenriched_attackers(
self, limit: int = 100,
) -> list[dict[str, Any]]:
"""``{"uuid", "ip"}`` pairs with no intel row OR a stale (expired) one.
Stale = ``expires_at < now``. Ordered by ``attackers.last_seen`` desc
so the worker prioritises recent activity on backfill. Both columns
are projected so the worker can write keyed on UUID and dispatch
provider calls keyed on IP without a second round-trip.
"""
now = datetime.now(timezone.utc)
async with self._session() as session:
stmt = (
select(Attacker.uuid, Attacker.ip)
.outerjoin(
AttackerIntel, AttackerIntel.attacker_uuid == Attacker.uuid,
)
.where(
or_(
AttackerIntel.uuid.is_(None),
AttackerIntel.expires_at < now,
)
)
.order_by(desc(Attacker.last_seen))
.limit(limit)
)
result = await session.execute(stmt)
return [
{"uuid": uuid_, "ip": ip}
for uuid_, ip in result.all()
]

32
decnet/web/db/sqlmodel_repo/attackers/__init__.py Normal file
View File

@@ -0,0 +1,32 @@
"""Attacker repository methods.
The full domain spans ~500 lines of methods across attacker rows,
behavior signals, session profiles, SMTP victim tracking, and
log-derived activity views. Each concern lives in its own submixin;
``AttackersMixin`` composes them.
``_deserialize_attacker`` lives on ``AttackersCoreMixin`` and is reached
from ``IdentitiesMixin.list_observations_for_identity`` via ``self.`` —
Python's MRO resolves it to the core mixin on the composed
``SQLModelRepository`` class.
"""
from __future__ import annotations
from decnet.web.db.sqlmodel_repo.attackers._core import AttackersCoreMixin
from decnet.web.db.sqlmodel_repo.attackers.activity import AttackerActivityMixin
from decnet.web.db.sqlmodel_repo.attackers.behavior import AttackerBehaviorMixin
from decnet.web.db.sqlmodel_repo.attackers.sessions import SessionProfilesMixin
from decnet.web.db.sqlmodel_repo.attackers.smtp import SmtpTargetsMixin
class AttackersMixin(
AttackerActivityMixin,
AttackerBehaviorMixin,
SessionProfilesMixin,
SmtpTargetsMixin,
AttackersCoreMixin,
):
"""Composed attackers mixin — see submixins for the actual methods."""
__all__ = ["AttackersMixin"]

95
decnet/web/db/sqlmodel_repo/attackers/_core.py Normal file
View File

@@ -0,0 +1,95 @@
"""Core ``Attacker`` row CRUD + the ``_deserialize_attacker`` helper.
The helper lives here because sibling submixins and ``IdentitiesMixin``
(``list_observations_for_identity``) both call it through ``self.`` —
MRO resolves them onto this mixin on the composed
``SQLModelRepository``.
"""
from __future__ import annotations
import json
import uuid as _uuid
from typing import Any, List, Optional
from sqlalchemy import desc, func, select
from decnet.web.db.models import Attacker
class AttackersCoreMixin:
@staticmethod
def _deserialize_attacker(d: dict[str, Any]) -> dict[str, Any]:
for key in ("services", "deckies", "fingerprints", "commands"):
if isinstance(d.get(key), str):
try:
d[key] = json.loads(d[key])
except (json.JSONDecodeError, TypeError):
pass
return d
async def upsert_attacker(self, data: dict[str, Any]) -> str:
async with self._session() as session:
result = await session.execute(
select(Attacker).where(Attacker.ip == data["ip"])
)
existing = result.scalar_one_or_none()
if existing:
for k, v in data.items():
setattr(existing, k, v)
session.add(existing)
row_uuid = existing.uuid
else:
row_uuid = str(_uuid.uuid4())
data = {**data, "uuid": row_uuid}
session.add(Attacker(**data))
await session.commit()
return row_uuid
async def get_attacker_by_uuid(self, uuid: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(Attacker).where(Attacker.uuid == uuid)
)
attacker = result.scalar_one_or_none()
if not attacker:
return None
return self._deserialize_attacker(attacker.model_dump(mode="json"))
async def get_attackers(
self,
limit: int = 50,
offset: int = 0,
search: Optional[str] = None,
sort_by: str = "recent",
service: Optional[str] = None,
) -> List[dict[str, Any]]:
order = {
"active": desc(Attacker.event_count),
"traversals": desc(Attacker.is_traversal),
}.get(sort_by, desc(Attacker.last_seen))
statement = select(Attacker).order_by(order).offset(offset).limit(limit)
if search:
statement = statement.where(Attacker.ip.like(f"%{search}%"))
if service:
statement = statement.where(Attacker.services.like(f'%"{service}"%'))
async with self._session() as session:
result = await session.execute(statement)
return [
self._deserialize_attacker(a.model_dump(mode="json"))
for a in result.scalars().all()
]
async def get_total_attackers(
self, search: Optional[str] = None, service: Optional[str] = None
) -> int:
statement = select(func.count()).select_from(Attacker)
if search:
statement = statement.where(Attacker.ip.like(f"%{search}%"))
if service:
statement = statement.where(Attacker.services.like(f'%"{service}"%'))
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0

207
decnet/web/db/sqlmodel_repo/attackers/activity.py Normal file
View File

@@ -0,0 +1,207 @@
"""Log-derived activity views: commands, service activity, IP leaks,
artifacts, stored mail, recorded sessions, transcripts.
These read from the ``logs`` and ``bounty`` tables joined against the
``Attacker`` row to scope by IP — no separate activity table.
"""
from __future__ import annotations
import json
from typing import Any, Optional
from sqlalchemy import desc, func, select
from decnet.web.db.models import Attacker, Bounty, Log
class AttackerActivityMixin:
async def get_attacker_commands(
self,
uuid: str,
limit: int = 50,
offset: int = 0,
service: Optional[str] = None,
) -> dict[str, Any]:
async with self._session() as session:
result = await session.execute(
select(Attacker.commands).where(Attacker.uuid == uuid)
)
raw = result.scalar_one_or_none()
if raw is None:
return {"total": 0, "data": []}
commands: list = json.loads(raw) if isinstance(raw, str) else raw
if service:
commands = [c for c in commands if c.get("service") == service]
total = len(commands)
page = commands[offset: offset + limit]
return {"total": total, "data": page}
async def get_attacker_service_activity(
self, attacker_uuid: str
) -> list[tuple[str, str]]:
"""Return distinct ``(service, event_type)`` pairs for an attacker.
Resolves IP then ``SELECT DISTINCT service, event_type FROM logs
WHERE attacker_ip = :ip`` — the result set is bounded by the
cardinality of services × event_types (tens, not thousands), so
this stays cheap even for attackers with long event streams.
Caller applies `event_kinds.bucket_services` to split into
scanned vs. interacted.
"""
async with self._session() as session:
ip_res = await session.execute(
select(Attacker.ip).where(Attacker.uuid == attacker_uuid)
)
ip = ip_res.scalar_one_or_none()
if not ip:
return []
rows = await session.execute(
select(Log.service, Log.event_type)
.where(Log.attacker_ip == ip)
.distinct()
)
return [(svc, evt) for svc, evt in rows.all()]
async def get_attacker_ip_leaks(
self, attacker_uuid: str, *, limit: int = 10,
) -> list[dict[str, Any]]:
"""Return ``bounty_type='ip_leak'`` rows for this attacker, newest
first, capped at ``limit``. Shape matches the XFF-mismatch
payload emitted by the ingester: keys include ``real_ip_claim``,
``source_header``, ``headers_seen``. Use
:meth:`count_attacker_ip_leaks` to get the unbounded total for
rotation detection."""
async with self._session() as session:
ip_res = await session.execute(
select(Attacker.ip).where(Attacker.uuid == attacker_uuid)
)
ip = ip_res.scalar_one_or_none()
if not ip:
return []
rows = await session.execute(
select(Bounty)
.where(Bounty.attacker_ip == ip)
.where(Bounty.bounty_type == "ip_leak")
.order_by(desc(Bounty.timestamp))
.limit(limit)
)
out: list[dict[str, Any]] = []
for row in rows.scalars().all():
rec = row.model_dump(mode="json")
# Bounty.payload is stored JSON-encoded; pre-decode for UX.
raw = rec.get("payload")
if isinstance(raw, str):
try:
rec["payload"] = json.loads(raw)
except (ValueError, TypeError):
rec["payload"] = {}
out.append(rec)
return out
async def count_attacker_ip_leaks(self, attacker_uuid: str) -> int:
"""Cheap COUNT(*) for XFF-rotation detection."""
async with self._session() as session:
ip_res = await session.execute(
select(Attacker.ip).where(Attacker.uuid == attacker_uuid)
)
ip = ip_res.scalar_one_or_none()
if not ip:
return 0
count_res = await session.execute(
select(func.count(Bounty.id))
.where(Bounty.attacker_ip == ip)
.where(Bounty.bounty_type == "ip_leak")
)
return int(count_res.scalar() or 0)
async def get_attacker_artifacts(self, uuid: str) -> list[dict[str, Any]]:
"""Return `file_captured` logs for the attacker identified by UUID.
Resolves the attacker's IP first, then queries the logs table on two
indexed columns (``attacker_ip`` and ``event_type``). No JSON extract
needed — the decky/stored_as are already decoded into ``fields`` by
the ingester and returned to the frontend for drawer rendering.
"""
async with self._session() as session:
ip_res = await session.execute(
select(Attacker.ip).where(Attacker.uuid == uuid)
)
ip = ip_res.scalar_one_or_none()
if not ip:
return []
rows = await session.execute(
select(Log)
.where(Log.attacker_ip == ip)
.where(Log.event_type == "file_captured")
.order_by(desc(Log.timestamp))
.limit(200)
)
return [r.model_dump(mode="json") for r in rows.scalars().all()]
async def get_attacker_stored_mail(self, uuid: str) -> list[dict[str, Any]]:
"""Return `message_stored` logs for an attacker, newest first.
Mirrors :meth:`get_attacker_artifacts` — the SMTP template emits one
`message_stored` row per accepted DATA body, with headers + sha256 +
attachment manifest already decoded into ``fields`` by the ingester.
Capped at 200 rows to match the artifact/transcript query shape.
"""
async with self._session() as session:
ip_res = await session.execute(
select(Attacker.ip).where(Attacker.uuid == uuid)
)
ip = ip_res.scalar_one_or_none()
if not ip:
return []
rows = await session.execute(
select(Log)
.where(Log.attacker_ip == ip)
.where(Log.event_type == "message_stored")
.order_by(desc(Log.timestamp))
.limit(200)
)
return [r.model_dump(mode="json") for r in rows.scalars().all()]
async def get_session_log(self, sid: str) -> Optional[dict[str, Any]]:
"""Look up the `session_recorded` Log row that owns a given sid.
sid is a v4 UUID embedded in the row's ``fields`` JSON blob. Matched
with LIKE on the textual sid substring — cheap given the bounded
cardinality of session_recorded rows vs. the full logs table.
"""
needle = f'"sid":"{sid}"'
async with self._session() as session:
rows = await session.execute(
select(Log)
.where(Log.event_type == "session_recorded")
.where(Log.fields.contains(needle))
.limit(1)
)
row = rows.scalars().first()
return row.model_dump(mode="json") if row else None
async def get_attacker_transcripts(self, uuid: str) -> list[dict[str, Any]]:
"""Return `session_recorded` logs for the attacker identified by UUID.
Mirror of :meth:`get_attacker_artifacts` — sessions ride in the same
Log table with event_type=session_recorded; the ingester decodes the
RFC 5424 SD fields (sid, service, decky, src_ip, duration_s, bytes,
truncated, shard_path) into the returned ``fields`` blob.
"""
async with self._session() as session:
ip_res = await session.execute(
select(Attacker.ip).where(Attacker.uuid == uuid)
)
ip = ip_res.scalar_one_or_none()
if not ip:
return []
rows = await session.execute(
select(Log)
.where(Log.attacker_ip == ip)
.where(Log.event_type == "session_recorded")
.order_by(desc(Log.timestamp))
.limit(200)
)
return [r.model_dump(mode="json") for r in rows.scalars().all()]

106
decnet/web/db/sqlmodel_repo/attackers/behavior.py Normal file
View File

@@ -0,0 +1,106 @@
"""Per-attacker behavior signals (TCP fingerprint, timing stats, phase
sequence, tool guesses, KEX order, SSH client banners)."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import select
from decnet.web.db.models import Attacker, AttackerBehavior
class AttackerBehaviorMixin:
async def upsert_attacker_behavior(
self,
attacker_uuid: str,
data: dict[str, Any],
) -> None:
async with self._session() as session:
result = await session.execute(
select(AttackerBehavior).where(
AttackerBehavior.attacker_uuid == attacker_uuid
)
)
existing = result.scalar_one_or_none()
payload = {**data, "updated_at": datetime.now(timezone.utc)}
if existing:
for k, v in payload.items():
setattr(existing, k, v)
session.add(existing)
else:
session.add(AttackerBehavior(attacker_uuid=attacker_uuid, **payload))
await session.commit()
async def get_attacker_behavior(
self,
attacker_uuid: str,
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(AttackerBehavior).where(
AttackerBehavior.attacker_uuid == attacker_uuid
)
)
row = result.scalar_one_or_none()
if not row:
return None
return self._deserialize_behavior(row.model_dump(mode="json"))
async def get_behaviors_for_ips(
self,
ips: set[str],
) -> dict[str, dict[str, Any]]:
if not ips:
return {}
async with self._session() as session:
result = await session.execute(
select(Attacker.ip, AttackerBehavior)
.join(AttackerBehavior, Attacker.uuid == AttackerBehavior.attacker_uuid)
.where(Attacker.ip.in_(ips))
)
out: dict[str, dict[str, Any]] = {}
for ip, row in result.all():
out[ip] = self._deserialize_behavior(row.model_dump(mode="json"))
return out
@staticmethod
def _deserialize_behavior(d: dict[str, Any]) -> dict[str, Any]:
for key in ("tcp_fingerprint", "timing_stats", "phase_sequence"):
if isinstance(d.get(key), str):
try:
d[key] = json.loads(d[key])
except (json.JSONDecodeError, TypeError):
pass
# Deserialize tool_guesses JSON array; normalise None → [].
raw = d.get("tool_guesses")
if isinstance(raw, str):
try:
parsed = json.loads(raw)
d["tool_guesses"] = parsed if isinstance(parsed, list) else [parsed]
except (json.JSONDecodeError, TypeError):
d["tool_guesses"] = []
elif raw is None:
d["tool_guesses"] = []
# Same list-or-None pattern for kex_order_raw.
raw_kex = d.get("kex_order_raw")
if isinstance(raw_kex, str):
try:
parsed_kex = json.loads(raw_kex)
d["kex_order_raw"] = parsed_kex if isinstance(parsed_kex, list) else [parsed_kex]
except (json.JSONDecodeError, TypeError):
d["kex_order_raw"] = []
elif raw_kex is None:
d["kex_order_raw"] = []
# Same list-or-None pattern for ssh_client_banners.
raw_banners = d.get("ssh_client_banners")
if isinstance(raw_banners, str):
try:
parsed_banners = json.loads(raw_banners)
d["ssh_client_banners"] = parsed_banners if isinstance(parsed_banners, list) else [parsed_banners]
except (json.JSONDecodeError, TypeError):
d["ssh_client_banners"] = []
elif raw_banners is None:
d["ssh_client_banners"] = []
return d

49
decnet/web/db/sqlmodel_repo/attackers/sessions.py Normal file
View File

@@ -0,0 +1,49 @@
"""Per-session profile rows (keystroke-dynamics features land here at
ingestion-time post-V2)."""
from __future__ import annotations
from typing import Any, Optional
from sqlalchemy import select
from decnet.web.db.models import SessionProfile
class SessionProfilesMixin:
async def upsert_session_profile(
self,
sid: str,
data: dict[str, Any],
) -> None:
"""
Write (or update) the session_profile row for *sid*.
Pre-v1, the typical call is the empty-write path at session close:
`upsert_session_profile(sid, {"log_id": <id>})` — all keystroke
feature columns stay NULL until the V2 ingestion job populates them.
"""
async with self._session() as session:
result = await session.execute(
select(SessionProfile).where(SessionProfile.sid == sid)
)
existing = result.scalar_one_or_none()
if existing:
for k, v in data.items():
setattr(existing, k, v)
session.add(existing)
else:
session.add(SessionProfile(sid=sid, **data))
await session.commit()
async def get_session_profile(
self,
sid: str,
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(SessionProfile).where(SessionProfile.sid == sid)
)
row = result.scalar_one_or_none()
if not row:
return None
return row.model_dump(mode="json")

69
decnet/web/db/sqlmodel_repo/attackers/smtp.py Normal file
View File

@@ -0,0 +1,69 @@
"""SMTP victim-domain tracking (per-attacker counters and
cross-attacker aggregate)."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from sqlalchemy import desc, func, select
from decnet.web.db.models import SmtpTarget
class SmtpTargetsMixin:
async def increment_smtp_target(self, attacker_uuid: str, domain: str) -> None:
"""Upsert an (attacker_uuid, domain) pair and bump count + last_seen.
Read-then-write under a single session — the UNIQUE constraint on
(attacker_uuid, domain) guards against duplicate rows if the race
ever materialises; we accept the ~1ms extra round-trip in exchange
for a single dialect-portable implementation.
"""
async with self._session() as session:
result = await session.execute(
select(SmtpTarget)
.where(SmtpTarget.attacker_uuid == attacker_uuid)
.where(SmtpTarget.domain == domain)
)
existing = result.scalar_one_or_none()
now = datetime.now(timezone.utc)
if existing:
existing.count += 1
existing.last_seen = now
session.add(existing)
else:
session.add(SmtpTarget(
attacker_uuid=attacker_uuid,
domain=domain,
first_seen=now,
last_seen=now,
count=1,
))
await session.commit()
async def list_smtp_targets(self, attacker_uuid: str) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(SmtpTarget)
.where(SmtpTarget.attacker_uuid == attacker_uuid)
.order_by(desc(SmtpTarget.last_seen))
)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def smtp_target_seen(self, domain: str) -> dict[str, Any]:
"""Aggregate rows for this domain across every attacker in the DB."""
async with self._session() as session:
result = await session.execute(
select(
func.coalesce(func.sum(SmtpTarget.count), 0),
func.min(SmtpTarget.first_seen),
func.max(SmtpTarget.last_seen),
).where(SmtpTarget.domain == domain)
)
total, first_seen, last_seen = result.one()
return {
"seen": int(total) > 0,
"count": int(total),
"first_seen": first_seen,
"last_seen": last_seen,
}

74
decnet/web/db/sqlmodel_repo/auth.py Normal file
View File

@@ -0,0 +1,74 @@
"""User CRUD."""
from __future__ import annotations
from typing import Any, Optional
from sqlalchemy import select, update
from decnet.web.db.models import User
class AuthMixin:
"""Mixin: composed onto ``SQLModelRepository``. Expects ``self._session()``.
``_ensure_admin_user`` stays in the package ``__init__`` so the
``DECNET_ADMIN_PASSWORD`` it reads remains addressable at the
``decnet.web.db.sqlmodel_repo`` module path (test monkeypatch surface).
"""
async def get_user_by_username(self, username: str) -> Optional[dict]:
async with self._session() as session:
result = await session.execute(
select(User).where(User.username == username)
)
user = result.scalar_one_or_none()
return user.model_dump() if user else None
async def get_user_by_uuid(self, uuid: str) -> Optional[dict]:
async with self._session() as session:
result = await session.execute(
select(User).where(User.uuid == uuid)
)
user = result.scalar_one_or_none()
return user.model_dump() if user else None
async def create_user(self, user_data: dict[str, Any]) -> None:
async with self._session() as session:
session.add(User(**user_data))
await session.commit()
async def update_user_password(
self, uuid: str, password_hash: str, must_change_password: bool = False
) -> None:
async with self._session() as session:
await session.execute(
update(User)
.where(User.uuid == uuid)
.values(
password_hash=password_hash,
must_change_password=must_change_password,
)
)
await session.commit()
async def list_users(self) -> list[dict]:
async with self._session() as session:
result = await session.execute(select(User))
return [u.model_dump() for u in result.scalars().all()]
async def delete_user(self, uuid: str) -> bool:
async with self._session() as session:
result = await session.execute(select(User).where(User.uuid == uuid))
user = result.scalar_one_or_none()
if not user:
return False
await session.delete(user)
await session.commit()
return True
async def update_user_role(self, uuid: str, role: str) -> None:
async with self._session() as session:
await session.execute(
update(User).where(User.uuid == uuid).values(role=role)
)
await session.commit()

139
decnet/web/db/sqlmodel_repo/bounties.py Normal file
View File

@@ -0,0 +1,139 @@
"""Bounty CRUD + the global purge helper that wipes logs/bounties/credentials/attackers together."""
from __future__ import annotations
import json
from collections import defaultdict
from typing import Any, List, Optional
import orjson
from sqlalchemy import asc, desc, func, or_, select, text
from sqlmodel.sql.expression import SelectOfScalar
from decnet.web.db.models import Bounty
class BountiesMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
async def purge_logs_and_bounties(self) -> dict[str, int]:
async with self._session() as session:
logs_deleted = (await session.execute(text("DELETE FROM logs"))).rowcount
bounties_deleted = (await session.execute(text("DELETE FROM bounty"))).rowcount
credentials_deleted = (
await session.execute(text("DELETE FROM credentials"))
).rowcount
# attacker_behavior has FK → attackers.uuid; delete children first.
await session.execute(text("DELETE FROM attacker_behavior"))
attackers_deleted = (await session.execute(text("DELETE FROM attackers"))).rowcount
await session.commit()
return {
"logs": logs_deleted,
"bounties": bounties_deleted,
"credentials": credentials_deleted,
"attackers": attackers_deleted,
}
async def add_bounty(self, bounty_data: dict[str, Any]) -> None:
data = bounty_data.copy()
if "payload" in data and isinstance(data["payload"], dict):
data["payload"] = orjson.dumps(data["payload"]).decode()
async with self._session() as session:
dup = await session.execute(
select(Bounty.id).where(
Bounty.bounty_type == data.get("bounty_type"),
Bounty.attacker_ip == data.get("attacker_ip"),
Bounty.payload == data.get("payload"),
).limit(1)
)
if dup.first() is not None:
return
session.add(Bounty(**data))
await session.commit()
def _apply_bounty_filters(
self,
statement: SelectOfScalar,
bounty_type: Optional[str],
search: Optional[str],
) -> SelectOfScalar:
if bounty_type:
statement = statement.where(Bounty.bounty_type == bounty_type)
if search:
lk = f"%{search}%"
statement = statement.where(
or_(
Bounty.decky.like(lk),
Bounty.service.like(lk),
Bounty.attacker_ip.like(lk),
Bounty.payload.like(lk),
)
)
return statement
async def get_bounties(
self,
limit: int = 50,
offset: int = 0,
bounty_type: Optional[str] = None,
search: Optional[str] = None,
) -> List[dict]:
statement = (
select(Bounty)
.order_by(desc(Bounty.timestamp))
.offset(offset)
.limit(limit)
)
statement = self._apply_bounty_filters(statement, bounty_type, search)
async with self._session() as session:
results = await session.execute(statement)
final = []
for item in results.scalars().all():
d = item.model_dump(mode="json")
try:
d["payload"] = json.loads(d["payload"])
except (json.JSONDecodeError, TypeError):
pass
final.append(d)
return final
async def get_total_bounties(
self, bounty_type: Optional[str] = None, search: Optional[str] = None
) -> int:
statement = select(func.count()).select_from(Bounty)
statement = self._apply_bounty_filters(statement, bounty_type, search)
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def get_all_bounties_by_ip(self) -> dict[str, List[dict[str, Any]]]:
async with self._session() as session:
result = await session.execute(
select(Bounty).order_by(asc(Bounty.timestamp))
)
grouped: dict[str, List[dict[str, Any]]] = defaultdict(list)
for item in result.scalars().all():
d = item.model_dump(mode="json")
try:
d["payload"] = json.loads(d["payload"])
except (json.JSONDecodeError, TypeError):
pass
grouped[item.attacker_ip].append(d)
return dict(grouped)
async def get_bounties_for_ips(self, ips: set[str]) -> dict[str, List[dict[str, Any]]]:
async with self._session() as session:
result = await session.execute(
select(Bounty).where(Bounty.attacker_ip.in_(ips)).order_by(asc(Bounty.timestamp))
)
grouped: dict[str, List[dict[str, Any]]] = defaultdict(list)
for item in result.scalars().all():
d = item.model_dump(mode="json")
try:
d["payload"] = json.loads(d["payload"])
except (json.JSONDecodeError, TypeError):
pass
grouped[item.attacker_ip].append(d)
return dict(grouped)

173
decnet/web/db/sqlmodel_repo/campaigns.py Normal file
View File

@@ -0,0 +1,173 @@
"""Campaign reads + writes.
Campaign = the second-tier clustering output that groups multiple
``AttackerIdentity`` rows into a coordinated activity cluster. The
campaign-clusterer worker drives the writes; the dashboard drives
the reads.
"""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import desc, func, select, update
from decnet.web.db.models import AttackerIdentity, Campaign
class CampaignsMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
async def get_campaign_by_uuid(self, uuid: str) -> Optional[dict[str, Any]]:
# Same chain-walk as get_identity_by_uuid; bounded against
# corrupted rings.
_MAX_MERGE_HOPS = 8
async with self._session() as session:
current_uuid = uuid
for _ in range(_MAX_MERGE_HOPS):
result = await session.execute(
select(Campaign).where(Campaign.uuid == current_uuid)
)
campaign = result.scalar_one_or_none()
if campaign is None:
return None
if campaign.merged_into_uuid is None:
return campaign.model_dump(mode="json")
current_uuid = campaign.merged_into_uuid
return campaign.model_dump(mode="json")
async def list_campaigns(
self, limit: int = 50, offset: int = 0,
) -> list[dict[str, Any]]:
statement = (
select(Campaign)
.where(Campaign.merged_into_uuid.is_(None))
.order_by(desc(Campaign.updated_at))
.offset(offset)
.limit(limit)
)
async with self._session() as session:
result = await session.execute(statement)
return [c.model_dump(mode="json") for c in result.scalars().all()]
async def count_campaigns(self) -> int:
statement = (
select(func.count())
.select_from(Campaign)
.where(Campaign.merged_into_uuid.is_(None))
)
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def list_identities_for_campaign(
self, campaign_uuid: str, limit: int = 50, offset: int = 0,
) -> list[dict[str, Any]]:
statement = (
select(AttackerIdentity)
.where(AttackerIdentity.campaign_id == campaign_uuid)
.order_by(desc(AttackerIdentity.updated_at))
.offset(offset)
.limit(limit)
)
async with self._session() as session:
result = await session.execute(statement)
return [i.model_dump(mode="json") for i in result.scalars().all()]
async def count_identities_for_campaign(self, campaign_uuid: str) -> int:
statement = (
select(func.count())
.select_from(AttackerIdentity)
.where(AttackerIdentity.campaign_id == campaign_uuid)
)
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def list_identities_for_clustering(
self, limit: Optional[int] = None,
) -> list[dict[str, Any]]:
# Project the columns the campaign clusterer's similarity
# graph reads. Narrow on purpose — future denormalised
# projections (commands_by_phase from log mining, decky-set
# aggregates) can land here without churning callers.
statement = select(
AttackerIdentity.uuid,
AttackerIdentity.campaign_id,
AttackerIdentity.merged_into_uuid,
AttackerIdentity.first_seen_at,
AttackerIdentity.last_seen_at,
AttackerIdentity.ja3_hashes,
AttackerIdentity.hassh_hashes,
AttackerIdentity.payload_simhashes,
AttackerIdentity.c2_endpoints,
).order_by(AttackerIdentity.created_at)
if limit is not None:
statement = statement.limit(limit)
async with self._session() as session:
result = await session.execute(statement)
return [
{
"uuid": row.uuid,
"campaign_id": row.campaign_id,
"merged_into_uuid": row.merged_into_uuid,
"first_seen_at": (
row.first_seen_at.isoformat()
if row.first_seen_at is not None
else None
),
"last_seen_at": (
row.last_seen_at.isoformat()
if row.last_seen_at is not None
else None
),
"ja3_hashes": row.ja3_hashes,
"hassh_hashes": row.hassh_hashes,
"payload_simhashes": row.payload_simhashes,
"c2_endpoints": row.c2_endpoints,
}
for row in result.all()
]
async def create_campaign(self, row: dict[str, Any]) -> str:
campaign = Campaign(**row)
async with self._session() as session:
session.add(campaign)
await session.commit()
return campaign.uuid
async def set_identity_campaign_id(
self, identity_uuid: str, campaign_uuid: Optional[str],
) -> None:
statement = (
update(AttackerIdentity)
.where(AttackerIdentity.uuid == identity_uuid)
.values(
campaign_id=campaign_uuid,
updated_at=datetime.now(timezone.utc),
)
)
async with self._session() as session:
await session.execute(statement)
await session.commit()
async def list_all_campaigns(self) -> list[dict[str, Any]]:
statement = select(Campaign).order_by(Campaign.created_at)
async with self._session() as session:
result = await session.execute(statement)
return [c.model_dump(mode="json") for c in result.scalars().all()]
async def update_campaign_merged_into(
self, campaign_uuid: str, winner_uuid: Optional[str],
) -> None:
statement = (
update(Campaign)
.where(Campaign.uuid == campaign_uuid)
.values(
merged_into_uuid=winner_uuid,
updated_at=datetime.now(timezone.utc),
)
)
async with self._session() as session:
await session.execute(statement)
await session.commit()

200
decnet/web/db/sqlmodel_repo/canary.py Normal file
View File

@@ -0,0 +1,200 @@
"""Canary blob/token CRUD + trigger ingestion."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import desc, func, select, update
from decnet.web.db.models import CanaryBlob, CanaryToken, CanaryTrigger
class CanaryMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
async def upsert_canary_blob(self, data: dict[str, Any]) -> dict[str, Any]:
sha = data.get("sha256")
if not sha:
raise ValueError("upsert_canary_blob: sha256 is required")
async with self._session() as session:
existing = await session.execute(
select(CanaryBlob).where(CanaryBlob.sha256 == sha)
)
row = existing.scalar_one_or_none()
if row:
return row.model_dump(mode="json")
row = CanaryBlob(**data)
session.add(row)
await session.commit()
await session.refresh(row)
return row.model_dump(mode="json")
async def get_canary_blob(self, uuid: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(CanaryBlob).where(CanaryBlob.uuid == uuid)
)
row = result.scalar_one_or_none()
return row.model_dump(mode="json") if row else None
async def get_canary_blob_by_sha256(
self, sha256: str
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(CanaryBlob).where(CanaryBlob.sha256 == sha256)
)
row = result.scalar_one_or_none()
return row.model_dump(mode="json") if row else None
async def list_canary_blobs(self) -> list[dict[str, Any]]:
# One round-trip: outer-join blobs -> tokens, group by blob, count
# live (non-revoked) references. Revoked tokens still occupy the
# blob conceptually until garbage-collected, so we count them too;
# the operator deletes blobs explicitly via the API.
async with self._session() as session:
stmt = (
select(CanaryBlob, func.count(CanaryToken.uuid))
.join(
CanaryToken,
CanaryToken.blob_uuid == CanaryBlob.uuid,
isouter=True,
)
.group_by(CanaryBlob.uuid)
.order_by(desc(CanaryBlob.uploaded_at))
)
result = await session.execute(stmt)
out: list[dict[str, Any]] = []
for blob, count in result.all():
d = blob.model_dump(mode="json")
d["token_count"] = int(count or 0)
out.append(d)
return out
async def delete_canary_blob(self, uuid: str) -> bool:
async with self._session() as session:
ref = await session.execute(
select(func.count(CanaryToken.uuid)).where(
CanaryToken.blob_uuid == uuid
)
)
if (ref.scalar_one() or 0) > 0:
return False
result = await session.execute(
select(CanaryBlob).where(CanaryBlob.uuid == uuid)
)
row = result.scalar_one_or_none()
if not row:
return False
await session.delete(row)
await session.commit()
return True
async def create_canary_token(self, data: dict[str, Any]) -> None:
async with self._session() as session:
session.add(CanaryToken(**data))
await session.commit()
async def get_canary_token(self, uuid: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(CanaryToken).where(CanaryToken.uuid == uuid)
)
row = result.scalar_one_or_none()
return row.model_dump(mode="json") if row else None
async def get_canary_token_by_slug(
self, callback_token: str
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(CanaryToken).where(
CanaryToken.callback_token == callback_token
)
)
row = result.scalar_one_or_none()
return row.model_dump(mode="json") if row else None
async def list_canary_tokens(
self,
*,
decky_name: Optional[str] = None,
state: Optional[str] = None,
kind: Optional[str] = None,
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = select(CanaryToken)
if decky_name is not None:
stmt = stmt.where(CanaryToken.decky_name == decky_name)
if state is not None:
stmt = stmt.where(CanaryToken.state == state)
if kind is not None:
stmt = stmt.where(CanaryToken.kind == kind)
stmt = stmt.order_by(desc(CanaryToken.placed_at))
result = await session.execute(stmt)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def update_canary_token_state(
self,
uuid: str,
state: str,
last_error: Optional[str] = None,
) -> bool:
async with self._session() as session:
result = await session.execute(
update(CanaryToken)
.where(CanaryToken.uuid == uuid)
.values(state=state, last_error=last_error)
)
await session.commit()
return result.rowcount > 0
async def record_canary_trigger(self, data: dict[str, Any]) -> str:
# Persist the trigger row + bump the token's counters in the same
# session so a subscriber that reads the token row right after
# receiving the bus event sees the updated count.
headers = data.get("raw_headers")
if isinstance(headers, dict):
data = {**data, "raw_headers": json.dumps(headers)}
async with self._session() as session:
row = CanaryTrigger(**data)
session.add(row)
ts = data.get("occurred_at") or datetime.now(timezone.utc)
await session.execute(
update(CanaryToken)
.where(CanaryToken.uuid == row.token_uuid)
.values(
last_triggered_at=ts,
trigger_count=CanaryToken.trigger_count + 1,
)
)
await session.commit()
await session.refresh(row)
return row.uuid
async def list_canary_triggers(
self, token_uuid: str, *, limit: int = 100, offset: int = 0,
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = (
select(CanaryTrigger)
.where(CanaryTrigger.token_uuid == token_uuid)
.order_by(desc(CanaryTrigger.occurred_at))
.limit(limit)
.offset(offset)
)
result = await session.execute(stmt)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def attribute_canary_trigger(
self, trigger_uuid: str, attacker_id: str,
) -> bool:
async with self._session() as session:
result = await session.execute(
update(CanaryTrigger)
.where(CanaryTrigger.uuid == trigger_uuid)
.values(attacker_id=attacker_id)
)
await session.commit()
return result.rowcount > 0

20
decnet/web/db/sqlmodel_repo/credentials/__init__.py Normal file
View File

@@ -0,0 +1,20 @@
"""Credential capture + credential-reuse correlation.
Capture (per-attempt rows) lives in ``_core.py``; the reuse correlator
(grouping rows that share a secret triple) lives in ``reuse.py``.
``CredentialsMixin`` composes the two.
"""
from __future__ import annotations
from decnet.web.db.sqlmodel_repo.credentials._core import CredentialsCoreMixin
from decnet.web.db.sqlmodel_repo.credentials.reuse import CredentialReuseMixin
class CredentialsMixin(
CredentialReuseMixin,
CredentialsCoreMixin,
):
"""Composed credentials mixin — see submixins for the actual methods."""
__all__ = ["CredentialsMixin"]

196
decnet/web/db/sqlmodel_repo/credentials/_core.py Normal file
View File

@@ -0,0 +1,196 @@
"""Credential capture: per-attempt rows in the ``Credential`` table."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, List, Optional
from sqlalchemy import desc, func, or_, select, update
from sqlmodel.sql.expression import SelectOfScalar
from decnet.web.db.models import Credential
class CredentialsCoreMixin:
async def upsert_credential(self, data: dict[str, Any]) -> int:
"""Upsert a credential attempt; returns the row id.
Dedup tuple: (attacker_ip, decky_name, service, secret_sha256,
principal_or_None). On match, ``attempt_count`` += 1 and
``last_seen`` advances; ``first_seen`` and ``fields`` are
preserved from the original sighting.
"""
payload = dict(data)
if "fields" in payload and isinstance(payload["fields"], dict):
# ensure_ascii=True keeps utf8mb4 columns safe even when
# service-specific keys carry non-ASCII bytes.
payload["fields"] = json.dumps(payload["fields"], ensure_ascii=True)
principal = payload.get("principal")
secret_kind = payload.get("secret_kind") or "plaintext"
async with self._session() as session:
stmt = select(Credential).where(
Credential.attacker_ip == payload["attacker_ip"],
Credential.decky_name == payload["decky_name"],
Credential.service == payload["service"],
Credential.secret_kind == secret_kind,
Credential.secret_sha256 == payload["secret_sha256"],
# NULL == NULL is False under SQL — branch the predicate.
(Credential.principal == principal) if principal is not None
else Credential.principal.is_(None),
)
existing = (await session.execute(stmt)).scalar_one_or_none()
now = datetime.now(timezone.utc)
if existing is not None:
existing.attempt_count = (existing.attempt_count or 1) + 1
existing.last_seen = now
if payload.get("outcome") is not None:
existing.outcome = payload["outcome"]
session.add(existing)
await session.commit()
return existing.id # type: ignore[return-value]
row = Credential(
attacker_ip=payload["attacker_ip"],
decky_name=payload["decky_name"],
service=payload["service"],
principal=principal,
secret_kind=secret_kind,
secret_sha256=payload["secret_sha256"],
secret_b64=payload.get("secret_b64"),
secret_printable=payload.get("secret_printable"),
outcome=payload.get("outcome"),
fields=payload.get("fields", "{}"),
first_seen=now,
last_seen=now,
attempt_count=1,
)
session.add(row)
await session.commit()
await session.refresh(row)
return row.id # type: ignore[return-value]
def _apply_credential_filters(
self,
statement: SelectOfScalar,
search: Optional[str],
service: Optional[str],
attacker_ip: Optional[str],
) -> SelectOfScalar:
if service:
statement = statement.where(Credential.service == service)
if attacker_ip:
statement = statement.where(Credential.attacker_ip == attacker_ip)
if search:
lk = f"%{search}%"
statement = statement.where(
or_(
Credential.decky_name.like(lk),
Credential.service.like(lk),
Credential.principal.like(lk),
Credential.secret_printable.like(lk),
)
)
return statement
async def get_credentials(
self,
limit: int = 50,
offset: int = 0,
search: Optional[str] = None,
service: Optional[str] = None,
attacker_ip: Optional[str] = None,
) -> List[dict[str, Any]]:
statement = (
select(Credential)
.order_by(desc(Credential.last_seen))
.offset(offset)
.limit(limit)
)
statement = self._apply_credential_filters(
statement, search, service, attacker_ip
)
async with self._session() as session:
result = await session.execute(statement)
out: List[dict[str, Any]] = []
for item in result.scalars().all():
d = item.model_dump(mode="json")
try:
d["fields"] = json.loads(d["fields"])
except (json.JSONDecodeError, TypeError):
pass
out.append(d)
return out
async def get_total_credentials(
self,
search: Optional[str] = None,
service: Optional[str] = None,
attacker_ip: Optional[str] = None,
) -> int:
statement = select(func.count()).select_from(Credential)
statement = self._apply_credential_filters(
statement, search, service, attacker_ip
)
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def get_credentials_for_attacker(
self, attacker_ip: str
) -> List[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(Credential)
.where(Credential.attacker_ip == attacker_ip)
.order_by(desc(Credential.last_seen))
)
out: List[dict[str, Any]] = []
for item in result.scalars().all():
d = item.model_dump(mode="json")
try:
d["fields"] = json.loads(d["fields"])
except (json.JSONDecodeError, TypeError):
pass
out.append(d)
return out
async def get_credential_attempts_for_secret(
self, secret_sha256: str
) -> List[dict[str, Any]]:
"""Every (attacker_ip, decky, service, principal) row sharing this
secret hash. Indexed lookup via ix_credentials_secret_service.
"""
async with self._session() as session:
result = await session.execute(
select(Credential)
.where(Credential.secret_sha256 == secret_sha256)
.order_by(desc(Credential.last_seen))
)
out: List[dict[str, Any]] = []
for item in result.scalars().all():
d = item.model_dump(mode="json")
try:
d["fields"] = json.loads(d["fields"])
except (json.JSONDecodeError, TypeError):
pass
out.append(d)
return out
async def update_credential_attacker_uuid(
self, attacker_ip: str, attacker_uuid: str
) -> int:
"""Backfill ``attacker_uuid`` on every Credential row matching the
given IP whose ``attacker_uuid`` is currently null. Run by the
profiler after it mints/updates an Attacker row.
"""
async with self._session() as session:
result = await session.execute(
update(Credential)
.where(
Credential.attacker_ip == attacker_ip,
Credential.attacker_uuid.is_(None),
)
.values(attacker_uuid=attacker_uuid)
)
await session.commit()
return int(result.rowcount or 0)

275
decnet/web/db/sqlmodel_repo/credentials/reuse.py Normal file
View File

@@ -0,0 +1,275 @@
"""Credential-reuse correlation: ``CredentialReuse`` finding rows that
group ``Credential`` attempts sharing the same (secret_sha256,
secret_kind, principal) triple."""
from __future__ import annotations
import json
import uuid as _uuid
from datetime import datetime, timezone
from typing import Any, List, Optional
from sqlalchemy import desc, func, select
from decnet.web.db.models import Credential, CredentialReuse
class CredentialReuseMixin:
@staticmethod
def _merge_unique(existing_json: str, value: Optional[str]) -> tuple[str, bool]:
"""Append ``value`` to a JSON list[str] column if not present.
Returns (new_json, changed). None values and duplicates are skipped.
"""
if value is None:
return existing_json, False
try:
current = json.loads(existing_json) if existing_json else []
if not isinstance(current, list):
current = []
except (json.JSONDecodeError, TypeError):
current = []
if value in current:
return existing_json, False
current.append(value)
return json.dumps(current, ensure_ascii=True), True
async def upsert_credential_reuse(
self,
*,
secret_sha256: str,
secret_kind: str,
principal: Optional[str],
attacker_uuid: Optional[str],
attacker_ip: str,
decky: str,
service: str,
attempt_count: int,
ts: Optional[datetime] = None,
) -> Optional[dict[str, Any]]:
"""Upsert a credential-reuse finding.
The row is keyed by ``(secret_sha256, secret_kind, principal_key)``
— ``principal_key`` is the canonicalised non-null form ("" when
principal is null) so the unique constraint behaves the same on
SQLite and MySQL.
Returns the row dict augmented with ``inserted: bool`` and
``changed: bool`` so the correlator can decide whether to publish
a bus event.
"""
principal_key = principal or ""
now = ts or datetime.now(timezone.utc)
async with self._session() as session:
existing = (await session.execute(
select(CredentialReuse).where(
CredentialReuse.secret_sha256 == secret_sha256,
CredentialReuse.secret_kind == secret_kind,
CredentialReuse.principal_key == principal_key,
)
)).scalar_one_or_none()
if existing is None:
row = CredentialReuse(
id=str(_uuid.uuid4()),
secret_sha256=secret_sha256,
secret_kind=secret_kind,
principal=principal,
principal_key=principal_key,
attacker_uuids=json.dumps(
[attacker_uuid] if attacker_uuid else [], ensure_ascii=True
),
attacker_ips=json.dumps([attacker_ip], ensure_ascii=True),
deckies=json.dumps([decky], ensure_ascii=True),
services=json.dumps([service], ensure_ascii=True),
target_count=1,
attempt_count=int(attempt_count),
confidence=1.0,
first_seen=now,
last_seen=now,
updated_at=now,
)
session.add(row)
await session.commit()
await session.refresh(row)
d = row.model_dump(mode="json")
d["inserted"] = True
d["changed"] = True
return d
changed = False
new_uuids, c1 = self._merge_unique(existing.attacker_uuids, attacker_uuid)
new_ips, c2 = self._merge_unique(existing.attacker_ips, attacker_ip)
new_deckies, c3 = self._merge_unique(existing.deckies, decky)
new_services, c4 = self._merge_unique(existing.services, service)
existing.attacker_uuids = new_uuids
existing.attacker_ips = new_ips
if c3 or c4:
existing.deckies = new_deckies
existing.services = new_services
# Recount target tuples from the underlying credentials
# table — a (decky, service) tuple only counts when both
# were observed together, which the JSON lists alone
# can't tell us.
stmt = (
select(func.count(func.distinct(
Credential.decky_name + ":" + Credential.service
)))
.where(
Credential.secret_sha256 == secret_sha256,
Credential.secret_kind == secret_kind,
(Credential.principal == principal) if principal is not None
else Credential.principal.is_(None),
)
)
target_count = (await session.execute(stmt)).scalar() or 0
existing.target_count = int(target_count)
existing.attempt_count = (existing.attempt_count or 0) + int(attempt_count)
existing.last_seen = now
existing.updated_at = now
if c1 or c2 or c3 or c4:
changed = True
session.add(existing)
await session.commit()
await session.refresh(existing)
d = existing.model_dump(mode="json")
d["inserted"] = False
d["changed"] = changed
return d
async def find_credential_reuse_candidates(
self, min_targets: int = 2
) -> List[dict[str, Any]]:
"""Find credential groups crossing the reuse threshold.
Returns one dict per qualifying ``(secret_sha256, secret_kind,
principal)`` group, with the keys plus a ``credentials`` list of
the underlying rows so the correlator can fold each into
``CredentialReuse`` via ``upsert_credential_reuse``.
"""
target_expr = func.count(
func.distinct(Credential.decky_name + ":" + Credential.service)
).label("target_count")
async with self._session() as session:
group_stmt = (
select(
Credential.secret_sha256,
Credential.secret_kind,
Credential.principal,
target_expr,
)
.group_by(
Credential.secret_sha256,
Credential.secret_kind,
Credential.principal,
)
.having(target_expr >= int(min_targets))
)
groups = (await session.execute(group_stmt)).all()
out: List[dict[str, Any]] = []
for sha, kind, principal, target_count in groups:
cred_stmt = select(Credential).where(
Credential.secret_sha256 == sha,
Credential.secret_kind == kind,
(Credential.principal == principal)
if principal is not None
else Credential.principal.is_(None),
)
rows = (await session.execute(cred_stmt)).scalars().all()
out.append({
"secret_sha256": sha,
"secret_kind": kind,
"principal": principal,
"target_count": int(target_count or 0),
"credentials": [r.model_dump(mode="json") for r in rows],
})
return out
async def list_credential_reuses(
self,
limit: int = 50,
offset: int = 0,
min_target_count: int = 2,
secret_kind: Optional[str] = None,
) -> tuple[int, List[dict[str, Any]]]:
async with self._session() as session:
base = select(CredentialReuse).where(
CredentialReuse.target_count >= min_target_count
)
if secret_kind:
base = base.where(CredentialReuse.secret_kind == secret_kind)
total_stmt = select(func.count()).select_from(base.subquery())
total = (await session.execute(total_stmt)).scalar() or 0
list_stmt = (
base.order_by(desc(CredentialReuse.target_count),
desc(CredentialReuse.last_seen))
.offset(offset).limit(limit)
)
rows = (await session.execute(list_stmt)).scalars().all()
out: List[dict[str, Any]] = []
for r in rows:
d = r.model_dump(mode="json")
for key in ("attacker_uuids", "attacker_ips", "deckies", "services"):
try:
d[key] = json.loads(d[key])
except (json.JSONDecodeError, TypeError):
d[key] = []
out.append(d)
await self._enrich_with_secret(session, out)
return int(total), out
async def get_credential_reuse_by_id(
self, reuse_id: str
) -> Optional[dict[str, Any]]:
async with self._session() as session:
row = (await session.execute(
select(CredentialReuse).where(CredentialReuse.id == reuse_id)
)).scalar_one_or_none()
if row is None:
return None
d = row.model_dump(mode="json")
for key in ("attacker_uuids", "attacker_ips", "deckies", "services"):
try:
d[key] = json.loads(d[key])
except (json.JSONDecodeError, TypeError):
d[key] = []
await self._enrich_with_secret(session, [d])
return d
@staticmethod
async def _enrich_with_secret(
session: Any, rows: List[dict[str, Any]]
) -> None:
"""Tack ``secret_printable`` + ``secret_b64`` onto each reuse row.
``CredentialReuse`` only stores the sha256+kind hash of the
secret — the actual printable/b64 representations live on the
underlying ``Credential`` rows. The dashboard wants to show the
secret in the drawer, so we lift one matching credential per
``(sha256, kind, principal)`` finding. One batched query for the
whole page; rows with no surviving credential (shouldn't happen
in practice) get nulls.
"""
if not rows:
return
sha_set = {r["secret_sha256"] for r in rows}
if not sha_set:
return
stmt = select(
Credential.secret_sha256,
Credential.secret_kind,
Credential.principal,
Credential.secret_printable,
Credential.secret_b64,
).where(Credential.secret_sha256.in_(sha_set))
secret_map: dict[
tuple[str, str, Optional[str]],
tuple[Optional[str], Optional[str]],
] = {}
for sha, kind, principal, printable, b64 in (
(await session.execute(stmt)).all()
):
secret_map.setdefault((sha, kind, principal), (printable, b64))
for r in rows:
key = (r["secret_sha256"], r["secret_kind"], r.get("principal"))
printable, b64 = secret_map.get(key, (None, None))
r["secret_printable"] = printable
r["secret_b64"] = b64

91
decnet/web/db/sqlmodel_repo/deckies.py Normal file
View File

@@ -0,0 +1,91 @@
"""Decky-shard CRUD (per-host shard registrations)."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, Optional
import orjson
from sqlalchemy import asc, select, text
from decnet.web.db.models import DeckyShard
class DeckiesMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
async def upsert_decky_shard(self, data: dict[str, Any]) -> None:
payload = {**data, "updated_at": datetime.now(timezone.utc)}
if isinstance(payload.get("services"), list):
payload["services"] = orjson.dumps(payload["services"]).decode()
async with self._session() as session:
result = await session.execute(
select(DeckyShard).where(DeckyShard.decky_name == payload["decky_name"])
)
existing = result.scalar_one_or_none()
if existing:
for k, v in payload.items():
setattr(existing, k, v)
session.add(existing)
else:
session.add(DeckyShard(**payload))
await session.commit()
async def list_decky_shards(
self, host_uuid: Optional[str] = None
) -> list[dict[str, Any]]:
statement = select(DeckyShard).order_by(asc(DeckyShard.decky_name))
if host_uuid:
statement = statement.where(DeckyShard.host_uuid == host_uuid)
async with self._session() as session:
result = await session.execute(statement)
out: list[dict[str, Any]] = []
for r in result.scalars().all():
d = r.model_dump(mode="json")
raw = d.get("services")
if isinstance(raw, str):
try:
d["services"] = json.loads(raw)
except (json.JSONDecodeError, TypeError):
d["services"] = []
# Flatten the stored DeckyConfig snapshot into the row so
# routers can hand it to DeckyShardView without re-parsing.
# Rows predating the migration have decky_config=NULL and
# fall through with the default (None/{}) view values.
cfg_raw = d.get("decky_config")
if isinstance(cfg_raw, str):
try:
cfg = json.loads(cfg_raw)
except (json.JSONDecodeError, TypeError):
cfg = {}
if isinstance(cfg, dict):
for k in ("hostname", "distro", "archetype",
"service_config", "mutate_interval",
"last_mutated"):
if k in cfg and d.get(k) is None:
d[k] = cfg[k]
# Keep decky_ip authoritative from the column (newer
# heartbeats overwrite it) but fall back to the
# snapshot if the column is still NULL.
if not d.get("decky_ip") and cfg.get("ip"):
d["decky_ip"] = cfg["ip"]
out.append(d)
return out
async def delete_decky_shards_for_host(self, host_uuid: str) -> int:
async with self._session() as session:
result = await session.execute(
text("DELETE FROM decky_shards WHERE host_uuid = :u"),
{"u": host_uuid},
)
await session.commit()
return result.rowcount or 0
async def delete_decky_shard(self, decky_name: str) -> bool:
async with self._session() as session:
result = await session.execute(
text("DELETE FROM decky_shards WHERE decky_name = :n"),
{"n": decky_name},
)
await session.commit()
return bool(result.rowcount)

152
decnet/web/db/sqlmodel_repo/fleet.py Normal file
View File

@@ -0,0 +1,152 @@
"""Fleet decky CRUD + cross-source running-decky aggregator."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
import orjson
from sqlalchemy import asc, select, text, update
from decnet.web.db.models import DeckyShard, FleetDecky, LOCAL_HOST_SENTINEL
from decnet.web.db.sqlmodel_repo._helpers import _deserialize_json_fields
class FleetMixin:
"""Mixin: composed onto ``SQLModelRepository``.
``list_running_deckies`` aggregates topology + fleet + swarm-shard
sources and stays here because the fleet entry is the canonical
shape; ``list_running_topology_deckies`` / ``list_running_fleet_deckies``
on ``self`` resolve through the composed class.
"""
async def upsert_fleet_decky(self, data: dict[str, Any]) -> None:
payload: dict[str, Any] = {
**data,
"updated_at": datetime.now(timezone.utc),
}
payload.setdefault("host_uuid", LOCAL_HOST_SENTINEL)
if payload.get("host_uuid") is None:
payload["host_uuid"] = LOCAL_HOST_SENTINEL
if isinstance(payload.get("services"), list):
payload["services"] = orjson.dumps(payload["services"]).decode()
if isinstance(payload.get("decky_config"), dict):
payload["decky_config"] = orjson.dumps(payload["decky_config"]).decode()
async with self._session() as session:
result = await session.execute(
select(FleetDecky).where(
FleetDecky.host_uuid == payload["host_uuid"],
FleetDecky.name == payload["name"],
)
)
existing = result.scalar_one_or_none()
if existing:
for k, v in payload.items():
setattr(existing, k, v)
session.add(existing)
else:
session.add(FleetDecky(**payload))
await session.commit()
async def delete_fleet_decky(self, *, host_uuid: str, name: str) -> None:
async with self._session() as session:
await session.execute(
text(
"DELETE FROM fleet_deckies "
"WHERE host_uuid = :h AND name = :n"
),
{"h": host_uuid, "n": name},
)
await session.commit()
async def list_fleet_deckies(
self, *, host_uuid: Optional[str] = None,
) -> list[dict[str, Any]]:
stmt = select(FleetDecky).order_by(asc(FleetDecky.name))
if host_uuid:
stmt = stmt.where(FleetDecky.host_uuid == host_uuid)
async with self._session() as session:
result = await session.execute(stmt)
return [
_deserialize_json_fields(
r.model_dump(mode="json"), ("services", "decky_config")
)
for r in result.scalars().all()
]
async def list_running_fleet_deckies(self) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(FleetDecky).where(FleetDecky.state == "running")
)
return [
_deserialize_json_fields(
r.model_dump(mode="json"), ("services", "decky_config")
)
for r in result.scalars().all()
]
async def update_fleet_decky_state(
self, *, host_uuid: str, name: str, state: str,
last_error: Optional[str] = None,
) -> None:
now = datetime.now(timezone.utc)
values: dict[str, Any] = {
"state": state,
"updated_at": now,
"last_seen": now,
}
if last_error is not None:
values["last_error"] = last_error
async with self._session() as session:
await session.execute(
update(FleetDecky)
.where(
FleetDecky.host_uuid == host_uuid,
FleetDecky.name == name,
)
.values(**values)
)
await session.commit()
async def list_running_deckies(self) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
# MazeNET — already shaped {uuid, name, ip, services}. We carry
# topology_id through so consumers (emailgen scheduler) can walk
# back to the parent topology row without a second round-trip;
# fleet/shard rows never have one, hence Optional.
for d in await self.list_running_topology_deckies():
out.append({
"uuid": d.get("uuid"),
"name": d.get("name"),
"ip": d.get("ip"),
"services": d.get("services") or [],
"topology_id": d.get("topology_id"),
"source": "topology",
})
# Fleet — column is `decky_ip`, PK is composite (host_uuid, name)
for d in await self.list_running_fleet_deckies():
out.append({
"uuid": f"{d.get('host_uuid')}:{d.get('name')}",
"name": d.get("name"),
"ip": d.get("decky_ip"),
"services": d.get("services") or [],
"source": "fleet",
})
# SWARM — DeckyShard rows in 'running' state on enrolled workers.
async with self._session() as session:
shard_rows = await session.execute(
select(DeckyShard).where(DeckyShard.state == "running")
)
for s in shard_rows.scalars().all():
d = _deserialize_json_fields(
s.model_dump(mode="json"), ("services", "decky_config")
)
out.append({
"uuid": f"{d.get('host_uuid')}:{d.get('decky_name')}",
"name": d.get("decky_name"),
"ip": d.get("decky_ip"),
"services": d.get("services") or [],
"source": "shard",
})
return out

185
decnet/web/db/sqlmodel_repo/identities.py Normal file
View File

@@ -0,0 +1,185 @@
"""AttackerIdentity reads + writes.
Identity = the clustering output that groups multiple ``Attacker`` rows
(usually different IPs from the same actor) into one logical actor.
The identity-clusterer worker drives the writes; the dashboard drives
the reads.
"""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import desc, func, select, update
from decnet.web.db.models import Attacker, AttackerIdentity
class IdentitiesMixin:
"""Mixin: composed onto ``SQLModelRepository``.
``self._deserialize_attacker`` resolves through ``AttackersMixin``
via MRO.
"""
async def get_identity_by_uuid(self, uuid: str) -> Optional[dict[str, Any]]:
# Follow merged_into_uuid up to the winner. Loop bounded by
# _MAX_MERGE_HOPS so a (hypothetically) corrupted ring can't
# spin the worker. Clusterer is responsible for never producing
# a cycle; this guard is belt-and-braces.
_MAX_MERGE_HOPS = 8
async with self._session() as session:
current_uuid = uuid
for _ in range(_MAX_MERGE_HOPS):
result = await session.execute(
select(AttackerIdentity).where(AttackerIdentity.uuid == current_uuid)
)
identity = result.scalar_one_or_none()
if identity is None:
return None
if identity.merged_into_uuid is None:
return identity.model_dump(mode="json")
current_uuid = identity.merged_into_uuid
# Hit the hop cap — surface what we have rather than recurse.
return identity.model_dump(mode="json")
async def list_identities(
self, limit: int = 50, offset: int = 0,
) -> list[dict[str, Any]]:
# Exclude merged-out rows so the list view is the de-duped truth.
# The history is still queryable per-uuid via get_identity_by_uuid
# and a future "merged into" endpoint when we need it.
statement = (
select(AttackerIdentity)
.where(AttackerIdentity.merged_into_uuid.is_(None))
.order_by(desc(AttackerIdentity.updated_at))
.offset(offset)
.limit(limit)
)
async with self._session() as session:
result = await session.execute(statement)
return [i.model_dump(mode="json") for i in result.scalars().all()]
async def count_identities(self) -> int:
statement = (
select(func.count())
.select_from(AttackerIdentity)
.where(AttackerIdentity.merged_into_uuid.is_(None))
)
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def list_observations_for_identity(
self, identity_uuid: str, limit: int = 50, offset: int = 0,
) -> list[dict[str, Any]]:
statement = (
select(Attacker)
.where(Attacker.identity_id == identity_uuid)
.order_by(desc(Attacker.last_seen))
.offset(offset)
.limit(limit)
)
async with self._session() as session:
result = await session.execute(statement)
return [
self._deserialize_attacker(a.model_dump(mode="json"))
for a in result.scalars().all()
]
async def count_observations_for_identity(self, identity_uuid: str) -> int:
statement = (
select(func.count())
.select_from(Attacker)
.where(Attacker.identity_id == identity_uuid)
)
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def list_attackers_for_clustering(
self, limit: Optional[int] = None,
) -> list[dict[str, Any]]:
# Project the columns the clusterer's similarity graph reads.
# Keep it narrow so future denormalised projections (payloads
# joined from logs, c2 endpoints aggregated from sessions) can
# land here without churning every caller. ``fingerprints`` is
# the raw JSON list — the clusterer parses for JA3 / HASSH.
statement = select(
Attacker.uuid, Attacker.asn, Attacker.identity_id, Attacker.fingerprints,
).order_by(Attacker.first_seen)
if limit is not None:
statement = statement.limit(limit)
async with self._session() as session:
result = await session.execute(statement)
return [
{
"uuid": row.uuid,
"asn": row.asn,
"identity_id": row.identity_id,
"fingerprints": row.fingerprints,
}
for row in result.all()
]
async def create_attacker_identity(self, row: dict[str, Any]) -> str:
identity = AttackerIdentity(**row)
async with self._session() as session:
session.add(identity)
await session.commit()
return identity.uuid
async def set_attacker_identity_id(
self, attacker_uuid: str, identity_uuid: str,
) -> None:
statement = (
update(Attacker)
.where(Attacker.uuid == attacker_uuid)
.values(identity_id=identity_uuid)
)
async with self._session() as session:
await session.execute(statement)
await session.commit()
async def list_all_identities(self) -> list[dict[str, Any]]:
statement = select(AttackerIdentity).order_by(AttackerIdentity.created_at)
async with self._session() as session:
result = await session.execute(statement)
return [i.model_dump(mode="json") for i in result.scalars().all()]
async def update_identity_merged_into(
self, identity_uuid: str, winner_uuid: Optional[str],
) -> None:
statement = (
update(AttackerIdentity)
.where(AttackerIdentity.uuid == identity_uuid)
.values(
merged_into_uuid=winner_uuid,
updated_at=datetime.now(timezone.utc),
)
)
async with self._session() as session:
await session.execute(statement)
await session.commit()
async def update_identity_fingerprints(
self,
identity_uuid: str,
*,
ja3_hashes: Optional[str] = None,
hassh_hashes: Optional[str] = None,
tls_cert_sha256: Optional[str] = None,
) -> None:
statement = (
update(AttackerIdentity)
.where(AttackerIdentity.uuid == identity_uuid)
.values(
ja3_hashes=ja3_hashes,
hassh_hashes=hassh_hashes,
tls_cert_sha256=tls_cert_sha256,
updated_at=datetime.now(timezone.utc),
)
)
async with self._session() as session:
await session.execute(statement)
await session.commit()

213
decnet/web/db/sqlmodel_repo/logs.py Normal file
View File

@@ -0,0 +1,213 @@
"""Log ingestion, query, and the stats summary endpoint.
``get_log_histogram`` is the per-dialect override point; the abstract
default raises NotImplementedError. ``get_stats_summary`` joins log
counts, topology-decky counts, and the on-disk fleet state into a
single dashboard payload.
"""
from __future__ import annotations
import asyncio
import re
import shlex
from datetime import datetime
from typing import Any, List, Optional
import orjson
from sqlalchemy import asc, desc, func, or_, select, text
from sqlmodel.sql.expression import SelectOfScalar
from decnet.config import load_state
from decnet.web.db.models import Log, TopologyDecky
class LogsMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
@staticmethod
def _normalize_log_row(log_data: dict[str, Any]) -> dict[str, Any]:
data = log_data.copy()
if "fields" in data and isinstance(data["fields"], dict):
data["fields"] = orjson.dumps(data["fields"]).decode()
if "timestamp" in data and isinstance(data["timestamp"], str):
try:
data["timestamp"] = datetime.fromisoformat(
data["timestamp"].replace("Z", "+00:00")
)
except ValueError:
pass
return data
async def add_log(self, log_data: dict[str, Any]) -> None:
data = self._normalize_log_row(log_data)
async with self._session() as session:
session.add(Log(**data))
await session.commit()
async def add_logs(self, log_entries: list[dict[str, Any]]) -> None:
"""Bulk insert — one session, one commit for the whole batch."""
if not log_entries:
return
_rows = [Log(**self._normalize_log_row(e)) for e in log_entries]
async with self._session() as session:
session.add_all(_rows)
await session.commit()
def _apply_filters(
self,
statement: SelectOfScalar,
search: Optional[str],
start_time: Optional[str],
end_time: Optional[str],
) -> SelectOfScalar:
if start_time:
statement = statement.where(Log.timestamp >= start_time)
if end_time:
statement = statement.where(Log.timestamp <= end_time)
if search:
try:
tokens = shlex.split(search)
except ValueError:
tokens = search.split()
core_fields = {
"decky": Log.decky,
"service": Log.service,
"event": Log.event_type,
"attacker": Log.attacker_ip,
"attacker-ip": Log.attacker_ip,
"attacker_ip": Log.attacker_ip,
}
for token in tokens:
if ":" in token:
key, val = token.split(":", 1)
if key in core_fields:
statement = statement.where(core_fields[key] == val)
else:
key_safe = re.sub(r"[^a-zA-Z0-9_]", "", key)
if key_safe:
statement = statement.where(
self._json_field_equals(key_safe)
).params(val=val)
else:
lk = f"%{token}%"
statement = statement.where(
or_(
Log.raw_line.like(lk),
Log.decky.like(lk),
Log.service.like(lk),
Log.attacker_ip.like(lk),
)
)
return statement
def _json_field_equals(self, key: str):
"""Return a text() predicate that matches rows where fields->key == :val.
Both SQLite and MySQL expose a ``JSON_EXTRACT`` function; MySQL also
exposes the same function under ``json_extract`` (case-insensitive).
The ``:val`` parameter is bound separately and must be supplied with
``.params(val=...)`` by the caller, which keeps us safe from injection.
"""
return text(f"JSON_EXTRACT(fields, '$.{key}') = :val")
async def get_logs(
self,
limit: int = 50,
offset: int = 0,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
) -> List[dict]:
statement = (
select(Log)
.order_by(desc(Log.timestamp))
.offset(offset)
.limit(limit)
)
statement = self._apply_filters(statement, search, start_time, end_time)
async with self._session() as session:
results = await session.execute(statement)
return [log.model_dump(mode="json") for log in results.scalars().all()]
async def get_max_log_id(self) -> int:
async with self._session() as session:
result = await session.execute(select(func.max(Log.id)))
val = result.scalar()
return val if val is not None else 0
async def get_logs_after_id(
self,
last_id: int,
limit: int = 50,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
) -> List[dict]:
statement = (
select(Log).where(Log.id > last_id).order_by(asc(Log.id)).limit(limit)
)
statement = self._apply_filters(statement, search, start_time, end_time)
async with self._session() as session:
results = await session.execute(statement)
return [log.model_dump(mode="json") for log in results.scalars().all()]
async def get_total_logs(
self,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
) -> int:
statement = select(func.count()).select_from(Log)
statement = self._apply_filters(statement, search, start_time, end_time)
async with self._session() as session:
result = await session.execute(statement)
return result.scalar() or 0
async def get_log_histogram(
self,
search: Optional[str] = None,
start_time: Optional[str] = None,
end_time: Optional[str] = None,
interval_minutes: int = 15,
) -> List[dict]:
"""Dialect-specific — override per backend."""
raise NotImplementedError
async def get_stats_summary(self) -> dict[str, Any]:
async with self._session() as session:
total_logs = (
await session.execute(select(func.count()).select_from(Log))
).scalar() or 0
unique_attackers = (
await session.execute(
select(func.count(func.distinct(Log.attacker_ip)))
)
).scalar() or 0
topo_total = (
await session.execute(select(func.count()).select_from(TopologyDecky))
).scalar() or 0
topo_running = (
await session.execute(
select(func.count())
.select_from(TopologyDecky)
.where(TopologyDecky.state == "running")
)
).scalar() or 0
_state = await asyncio.to_thread(load_state)
fleet_deckies = len(_state[0].deckies) if _state else 0
return {
"total_logs": total_logs,
"unique_attackers": unique_attackers,
# Fleet state file doesn't track per-decky runtime; treat all
# fleet rows as active and add MazeNET running rows on top.
"active_deckies": fleet_deckies + topo_running,
"deployed_deckies": fleet_deckies + topo_total,
}

191
decnet/web/db/sqlmodel_repo/orchestrator.py Normal file
View File

@@ -0,0 +1,191 @@
"""Orchestrator event log + email log + per-pool prune helpers."""
from __future__ import annotations
import json
from datetime import datetime
from typing import Any, Optional
from sqlalchemy import delete as sa_delete
from sqlalchemy import desc, func, or_, select
from decnet.web.db.models import OrchestratorEmail, OrchestratorEvent
class OrchestratorMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
async def record_orchestrator_event(self, data: dict[str, Any]) -> str:
payload = data.get("payload")
if isinstance(payload, (dict, list)):
data = {**data, "payload": json.dumps(payload)}
async with self._session() as session:
row = OrchestratorEvent(**data)
session.add(row)
await session.commit()
await session.refresh(row)
return row.uuid
async def list_orchestrator_events(
self,
limit: int = 100,
offset: int = 0,
*,
kind: Optional[str] = None,
since_ts: Optional[datetime] = None,
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = select(OrchestratorEvent)
if kind is not None:
stmt = stmt.where(OrchestratorEvent.kind == kind)
if since_ts is not None:
stmt = stmt.where(OrchestratorEvent.ts >= since_ts)
stmt = (
stmt.order_by(desc(OrchestratorEvent.ts))
.offset(offset)
.limit(limit)
)
result = await session.execute(stmt)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def count_orchestrator_events(
self, *, kind: Optional[str] = None,
) -> int:
stmt = select(func.count()).select_from(OrchestratorEvent)
if kind is not None:
stmt = stmt.where(OrchestratorEvent.kind == kind)
async with self._session() as session:
result = await session.execute(stmt)
return result.scalar() or 0
async def prune_orchestrator_events(self, per_dst_cap: int = 10000) -> int:
"""Trim per-dst rows to *per_dst_cap*, oldest-first. Returns deleted count."""
deleted = 0
async with self._session() as session:
dst_rows = await session.execute(
select(OrchestratorEvent.dst_decky_uuid).distinct()
)
for (dst,) in dst_rows.all():
keep = await session.execute(
select(OrchestratorEvent.uuid)
.where(OrchestratorEvent.dst_decky_uuid == dst)
.order_by(desc(OrchestratorEvent.ts))
.limit(per_dst_cap)
)
keep_uuids = [u for (u,) in keep.all()]
if not keep_uuids:
continue
stmt = sa_delete(OrchestratorEvent).where(
OrchestratorEvent.dst_decky_uuid == dst,
OrchestratorEvent.uuid.notin_(keep_uuids),
)
res = await session.execute(stmt)
deleted += res.rowcount or 0
await session.commit()
return deleted
async def record_orchestrator_email(self, data: dict[str, Any]) -> str:
payload = data.get("payload")
if isinstance(payload, (dict, list)):
data = {**data, "payload": json.dumps(payload)}
async with self._session() as session:
row = OrchestratorEmail(**data)
session.add(row)
await session.commit()
await session.refresh(row)
return row.uuid
async def list_orchestrator_emails(
self,
limit: int = 100,
offset: int = 0,
*,
mail_decky_uuid: Optional[str] = None,
thread_id: Optional[str] = None,
since_ts: Optional[datetime] = None,
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = select(OrchestratorEmail)
if mail_decky_uuid is not None:
stmt = stmt.where(
OrchestratorEmail.mail_decky_uuid == mail_decky_uuid
)
if thread_id is not None:
stmt = stmt.where(OrchestratorEmail.thread_id == thread_id)
if since_ts is not None:
stmt = stmt.where(OrchestratorEmail.ts >= since_ts)
stmt = (
stmt.order_by(desc(OrchestratorEmail.ts))
.offset(offset)
.limit(limit)
)
result = await session.execute(stmt)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def count_orchestrator_emails(
self,
*,
mail_decky_uuid: Optional[str] = None,
) -> int:
stmt = select(func.count()).select_from(OrchestratorEmail)
if mail_decky_uuid is not None:
stmt = stmt.where(OrchestratorEmail.mail_decky_uuid == mail_decky_uuid)
async with self._session() as session:
result = await session.execute(stmt)
return result.scalar() or 0
async def list_orchestrator_email_threads(
self,
mail_decky_uuid: str,
sender_email: str,
recipient_email: str,
*,
limit: int = 50,
) -> list[dict[str, Any]]:
# Most-recent row per (sender, recipient) pair under this mail decky.
# The scheduler only needs the latest message_id/subject/thread_id to
# construct a reply; older rows in the same thread aren't relevant
# for the "do we reply or start fresh" decision.
async with self._session() as session:
stmt = (
select(OrchestratorEmail)
.where(
OrchestratorEmail.mail_decky_uuid == mail_decky_uuid,
or_(
(OrchestratorEmail.sender_email == sender_email)
& (OrchestratorEmail.recipient_email == recipient_email),
(OrchestratorEmail.sender_email == recipient_email)
& (OrchestratorEmail.recipient_email == sender_email),
),
OrchestratorEmail.success.is_(True),
)
.order_by(desc(OrchestratorEmail.ts))
.limit(limit)
)
result = await session.execute(stmt)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def prune_orchestrator_emails(self, per_decky_cap: int = 10000) -> int:
"""Trim per-mail-decky rows to *per_decky_cap*, oldest-first."""
deleted = 0
async with self._session() as session:
decky_rows = await session.execute(
select(OrchestratorEmail.mail_decky_uuid).distinct()
)
for (mail_uuid,) in decky_rows.all():
keep = await session.execute(
select(OrchestratorEmail.uuid)
.where(OrchestratorEmail.mail_decky_uuid == mail_uuid)
.order_by(desc(OrchestratorEmail.ts))
.limit(per_decky_cap)
)
keep_uuids = [u for (u,) in keep.all()]
if not keep_uuids:
continue
stmt = sa_delete(OrchestratorEmail).where(
OrchestratorEmail.mail_decky_uuid == mail_uuid,
OrchestratorEmail.uuid.notin_(keep_uuids),
)
res = await session.execute(stmt)
deleted += res.rowcount or 0
await session.commit()
return deleted

157
decnet/web/db/sqlmodel_repo/realism.py Normal file
View File

@@ -0,0 +1,157 @@
"""Synthetic-file CRUD + realism config key/value store."""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import Any, Optional
from sqlalchemy import desc, func, select, update
from decnet.web.db.models import RealismConfig, SyntheticFile
from decnet.web.db.models.realism import SYNTHETIC_FILE_BODY_LIMIT
class RealismMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
async def record_synthetic_file(self, data: dict[str, Any]) -> str:
if "last_body" in data and data["last_body"] is not None:
data = {**data, "last_body": data["last_body"][:SYNTHETIC_FILE_BODY_LIMIT]}
async with self._session() as session:
row = SyntheticFile(**data)
session.add(row)
await session.commit()
await session.refresh(row)
return row.uuid
async def update_synthetic_file(
self, row_uuid: str, data: dict[str, Any],
) -> None:
if "last_body" in data and data["last_body"] is not None:
data = {**data, "last_body": data["last_body"][:SYNTHETIC_FILE_BODY_LIMIT]}
async with self._session() as session:
stmt = (
update(SyntheticFile)
.where(SyntheticFile.uuid == row_uuid)
.values(**data)
)
await session.execute(stmt)
await session.commit()
async def list_synthetic_files(
self,
*,
decky_uuid: Optional[str] = None,
persona: Optional[str] = None,
content_class: Optional[str] = None,
limit: int = 100,
offset: int = 0,
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = select(SyntheticFile)
if decky_uuid is not None:
stmt = stmt.where(SyntheticFile.decky_uuid == decky_uuid)
if persona is not None:
stmt = stmt.where(SyntheticFile.persona == persona)
if content_class is not None:
stmt = stmt.where(SyntheticFile.content_class == content_class)
stmt = (
stmt.order_by(desc(SyntheticFile.last_modified))
.offset(offset)
.limit(limit)
)
result = await session.execute(stmt)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def count_synthetic_files(
self,
*,
decky_uuid: Optional[str] = None,
persona: Optional[str] = None,
content_class: Optional[str] = None,
) -> int:
async with self._session() as session:
stmt = select(func.count(SyntheticFile.uuid))
if decky_uuid is not None:
stmt = stmt.where(SyntheticFile.decky_uuid == decky_uuid)
if persona is not None:
stmt = stmt.where(SyntheticFile.persona == persona)
if content_class is not None:
stmt = stmt.where(SyntheticFile.content_class == content_class)
result = await session.execute(stmt)
return int(result.scalar() or 0)
async def get_synthetic_file(
self, uuid: str,
) -> Optional[dict[str, Any]]:
async with self._session() as session:
stmt = select(SyntheticFile).where(SyntheticFile.uuid == uuid)
result = await session.execute(stmt)
row = result.scalars().first()
if row is None:
return None
return row.model_dump(mode="json")
async def get_realism_config(
self, key: str,
) -> Optional[dict[str, Any]]:
async with self._session() as session:
stmt = select(RealismConfig).where(RealismConfig.key == key)
result = await session.execute(stmt)
row = result.scalars().first()
if row is None:
return None
return row.model_dump(mode="json")
async def set_realism_config(
self, key: str, value: str,
) -> None:
"""Upsert one realism_config row. Last-write-wins."""
async with self._session() as session:
stmt = select(RealismConfig).where(RealismConfig.key == key)
result = await session.execute(stmt)
row = result.scalars().first()
if row is None:
session.add(RealismConfig(
key=key, value=value,
updated_at=datetime.now(timezone.utc),
))
else:
upd = (
update(RealismConfig)
.where(RealismConfig.uuid == row.uuid)
.values(value=value, updated_at=datetime.now(timezone.utc))
)
await session.execute(upd)
await session.commit()
async def pick_random_synthetic_file_for_edit(
self,
decky_uuid: str,
*,
max_age_days: int = 30,
) -> Optional[dict[str, Any]]:
# Editable classes: anything whose body is plain text we can
# mutate idempotently. Binary canary artifacts are out — they
# rotate via a fresh plant, not an edit.
editable = (
"note", "todo", "draft", "script", "log_cron", "log_daemon",
)
cutoff = datetime.now(timezone.utc) - timedelta(days=max_age_days)
async with self._session() as session:
stmt = (
select(SyntheticFile)
.where(
SyntheticFile.decky_uuid == decky_uuid,
SyntheticFile.content_class.in_(editable), # type: ignore[attr-defined]
SyntheticFile.last_modified >= cutoff,
)
# SQLite + MySQL both support func.random() / RAND() —
# SQLAlchemy's func.random() compiles per-dialect.
.order_by(func.random())
.limit(1)
)
result = await session.execute(stmt)
row = result.scalars().first()
if row is None:
return None
return row.model_dump(mode="json")

71
decnet/web/db/sqlmodel_repo/swarm.py Normal file
View File

@@ -0,0 +1,71 @@
"""Swarm host CRUD."""
from __future__ import annotations
from typing import Any, Optional
from sqlalchemy import asc, select, text, update
from decnet.web.db.models import SwarmHost
class SwarmMixin:
"""Mixin: composed onto ``SQLModelRepository``. Expects ``self._session()``."""
async def add_swarm_host(self, data: dict[str, Any]) -> None:
async with self._session() as session:
session.add(SwarmHost(**data))
await session.commit()
async def get_swarm_host_by_name(self, name: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(select(SwarmHost).where(SwarmHost.name == name))
row = result.scalar_one_or_none()
return row.model_dump(mode="json") if row else None
async def get_swarm_host_by_uuid(self, uuid: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(select(SwarmHost).where(SwarmHost.uuid == uuid))
row = result.scalar_one_or_none()
return row.model_dump(mode="json") if row else None
async def get_swarm_host_by_fingerprint(self, fingerprint: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(SwarmHost).where(SwarmHost.client_cert_fingerprint == fingerprint)
)
row = result.scalar_one_or_none()
return row.model_dump(mode="json") if row else None
async def list_swarm_hosts(self, status: Optional[str] = None) -> list[dict[str, Any]]:
statement = select(SwarmHost).order_by(asc(SwarmHost.name))
if status:
statement = statement.where(SwarmHost.status == status)
async with self._session() as session:
result = await session.execute(statement)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def update_swarm_host(self, uuid: str, fields: dict[str, Any]) -> None:
if not fields:
return
async with self._session() as session:
await session.execute(
update(SwarmHost).where(SwarmHost.uuid == uuid).values(**fields)
)
await session.commit()
async def delete_swarm_host(self, uuid: str) -> bool:
async with self._session() as session:
# Clean up child shards first (no ON DELETE CASCADE portable across dialects).
await session.execute(
text("DELETE FROM decky_shards WHERE host_uuid = :u"), {"u": uuid}
)
result = await session.execute(
select(SwarmHost).where(SwarmHost.uuid == uuid)
)
host = result.scalar_one_or_none()
if not host:
await session.commit()
return False
await session.delete(host)
await session.commit()
return True

32
decnet/web/db/sqlmodel_repo/topology/__init__.py Normal file
View File

@@ -0,0 +1,32 @@
"""MazeNET topology repository methods.
The full domain spans ~700 lines of methods across topologies, LANs,
deckies, edges, the status-event log, and the live reconciler mutation
queue. Each concern lives in its own submixin; ``TopologyMixin``
composes them.
The optimistic-locking helpers (``_assert_pending``,
``_check_and_bump_version``) live on ``TopologyCoreMixin`` and are
reached from sibling submixins via ``self.`` — Python's MRO resolves
them to the core mixin no matter which submixin holds the caller.
"""
from __future__ import annotations
from decnet.web.db.sqlmodel_repo.topology._core import TopologyCoreMixin
from decnet.web.db.sqlmodel_repo.topology.deckies import TopologyDeckiesMixin
from decnet.web.db.sqlmodel_repo.topology.edges import TopologyEdgesMixin
from decnet.web.db.sqlmodel_repo.topology.lans import LansMixin
from decnet.web.db.sqlmodel_repo.topology.mutations import TopologyMutationsMixin
class TopologyMixin(
TopologyDeckiesMixin,
TopologyEdgesMixin,
LansMixin,
TopologyMutationsMixin,
TopologyCoreMixin,
):
"""Composed topology mixin — see submixins for the actual methods."""
__all__ = ["TopologyMixin"]

250
decnet/web/db/sqlmodel_repo/topology/_core.py Normal file
View File

@@ -0,0 +1,250 @@
"""Topology table CRUD + the optimistic-locking helpers that the
sibling LAN / decky / edge / mutation mixins call through MRO."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import desc, func, select, text
from decnet.web.db.models import Topology, TopologyStatusEvent
from decnet.web.db.sqlmodel_repo._helpers import (
_deserialize_json_fields,
_serialize_json_fields,
)
class TopologyCoreMixin:
"""Topologies CRUD + ``_assert_pending`` / ``_check_and_bump_version``.
The two private helpers live here because every other topology
submixin (lans, deckies, edges, mutations) calls them through
``self.`` — MRO resolution lands them on this mixin no matter
which submixin holds the caller.
"""
async def create_topology(self, data: dict[str, Any]) -> str:
payload = _serialize_json_fields(data, ("config_snapshot",))
async with self._session() as session:
row = Topology(**payload)
session.add(row)
await session.commit()
await session.refresh(row)
return row.id
async def get_topology(self, topology_id: str) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(Topology).where(Topology.id == topology_id)
)
row = result.scalar_one_or_none()
if not row:
return None
d = row.model_dump(mode="json")
return _deserialize_json_fields(d, ("config_snapshot",))
async def list_topologies(
self,
status: Optional[str] = None,
limit: Optional[int] = None,
offset: Optional[int] = None,
) -> list[dict[str, Any]]:
statement = select(Topology).order_by(desc(Topology.created_at))
if status:
statement = statement.where(Topology.status == status)
if offset is not None:
statement = statement.offset(offset)
if limit is not None:
statement = statement.limit(limit)
async with self._session() as session:
result = await session.execute(statement)
return [
_deserialize_json_fields(
r.model_dump(mode="json"), ("config_snapshot",)
)
for r in result.scalars().all()
]
async def count_topologies(self, status: Optional[str] = None) -> int:
statement = select(func.count(Topology.id))
if status:
statement = statement.where(Topology.status == status)
async with self._session() as session:
result = await session.execute(statement)
return int(result.scalar_one() or 0)
async def update_topology_status(
self,
topology_id: str,
new_status: str,
reason: Optional[str] = None,
) -> None:
"""Update topology.status and append a TopologyStatusEvent atomically.
Transition legality is enforced in ``decnet.topology.status``; this
method trusts the caller.
"""
now = datetime.now(timezone.utc)
async with self._session() as session:
result = await session.execute(
select(Topology).where(Topology.id == topology_id)
)
topo = result.scalar_one_or_none()
if topo is None:
return
from_status = topo.status
topo.status = new_status
topo.status_changed_at = now
session.add(topo)
session.add(
TopologyStatusEvent(
topology_id=topology_id,
from_status=from_status,
to_status=new_status,
at=now,
reason=reason,
)
)
await session.commit()
async def set_topology_resync(self, topology_id: str, value: bool) -> None:
async with self._session() as session:
result = await session.execute(
select(Topology).where(Topology.id == topology_id)
)
topo = result.scalar_one_or_none()
if topo is None:
return
topo.needs_resync = bool(value)
session.add(topo)
await session.commit()
async def set_topology_email_personas(
self, topology_id: str, personas_json: str,
) -> bool:
"""Replace ``Topology.email_personas`` with the supplied JSON.
The string is stored as-is; validation/parsing is the caller's
job (and is repeated by the emailgen scheduler each tick anyway).
Returns True if a row was updated.
"""
async with self._session() as session:
result = await session.execute(
select(Topology).where(Topology.id == topology_id)
)
topo = result.scalar_one_or_none()
if topo is None:
return False
topo.email_personas = personas_json
session.add(topo)
await session.commit()
return True
async def list_topologies_needing_resync(self) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(Topology).where(Topology.needs_resync == True) # noqa: E712
)
return [
_deserialize_json_fields(
r.model_dump(mode="json"), ("config_snapshot",)
)
for r in result.scalars().all()
]
async def delete_topology_cascade(self, topology_id: str) -> bool:
"""Delete topology and all children. No portable ON DELETE CASCADE."""
async with self._session() as session:
params = {"t": topology_id}
await session.execute(
text("DELETE FROM topology_status_events WHERE topology_id = :t"),
params,
)
await session.execute(
text("DELETE FROM topology_edges WHERE topology_id = :t"),
params,
)
await session.execute(
text("DELETE FROM topology_deckies WHERE topology_id = :t"),
params,
)
await session.execute(
text("DELETE FROM lans WHERE topology_id = :t"),
params,
)
await session.execute(
text("DELETE FROM topology_mutations WHERE topology_id = :t"),
params,
)
result = await session.execute(
select(Topology).where(Topology.id == topology_id)
)
topo = result.scalar_one_or_none()
if not topo:
await session.commit()
return False
await session.delete(topo)
await session.commit()
return True
async def list_live_topology_ids(self) -> list[str]:
"""Return ids of topologies currently in ``active|degraded``."""
async with self._session() as session:
result = await session.execute(
select(Topology.id).where(
Topology.status.in_(["active", "degraded"])
)
)
return [r for r in result.scalars().all()]
# ─── concurrency / pending-state guards (used by sibling mixins) ──────
async def _assert_pending(self, session, topology_id: str) -> None:
"""Pre-deploy edits are pending-only. Raises TopologyNotEditable."""
from decnet.topology.status import TopologyNotEditable, TopologyStatus
result = await session.execute(
select(Topology).where(Topology.id == topology_id)
)
topo = result.scalar_one_or_none()
if topo is None:
raise ValueError(f"topology {topology_id!r} not found")
if topo.status != TopologyStatus.PENDING:
raise TopologyNotEditable(
status=topo.status,
reason="free-form edits are pending-only; use the "
"mutator (topology_mutations) after deploy",
)
async def _check_and_bump_version(
self,
session,
topology_id: str,
expected_version: Optional[int],
) -> None:
"""Optimistic-concurrency guard used by child-row mutators.
If ``expected_version`` is None, no check happens (backward-compat
for internal callers that don't need concurrency protection).
If supplied, loads the Topology row in the same session,
compares ``version == expected_version``, raises VersionConflict
on mismatch, otherwise bumps ``version += 1``. The caller must
commit the enclosing session.
"""
from decnet.topology.status import VersionConflict
if expected_version is None:
return
result = await session.execute(
select(Topology).where(Topology.id == topology_id)
)
topo = result.scalar_one_or_none()
if topo is None:
raise ValueError(f"topology {topology_id!r} not found")
if topo.version != expected_version:
raise VersionConflict(
current=topo.version, expected=expected_version
)
topo.version = topo.version + 1
session.add(topo)

125
decnet/web/db/sqlmodel_repo/topology/deckies.py Normal file
View File

@@ -0,0 +1,125 @@
"""Topology decky CRUD + the running-decky listing the fleet aggregator
calls through MRO."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import asc, select, text, update
from decnet.web.db.models import TopologyDecky
from decnet.web.db.sqlmodel_repo._helpers import (
_deserialize_json_fields,
_serialize_json_fields,
)
class TopologyDeckiesMixin:
"""``self._assert_pending`` / ``self._check_and_bump_version`` resolve
through ``TopologyCoreMixin`` via MRO."""
async def add_topology_decky(
self,
data: dict[str, Any],
*,
expected_version: Optional[int] = None,
) -> str:
payload = _serialize_json_fields(data, ("services", "decky_config"))
async with self._session() as session:
await self._check_and_bump_version(
session, data["topology_id"], expected_version
)
row = TopologyDecky(**payload)
session.add(row)
await session.commit()
await session.refresh(row)
return row.uuid
async def update_topology_decky(
self,
decky_uuid: str,
fields: dict[str, Any],
*,
expected_version: Optional[int] = None,
enforce_pending: bool = False,
) -> None:
if not fields:
return
payload = _serialize_json_fields(fields, ("services", "decky_config"))
payload.setdefault("updated_at", datetime.now(timezone.utc))
async with self._session() as session:
result = await session.execute(
select(TopologyDecky).where(TopologyDecky.uuid == decky_uuid)
)
d = result.scalar_one_or_none()
if d is None:
raise ValueError(f"decky {decky_uuid!r} not found")
if enforce_pending:
await self._assert_pending(session, d.topology_id)
if expected_version is not None:
await self._check_and_bump_version(
session, d.topology_id, expected_version
)
await session.execute(
update(TopologyDecky)
.where(TopologyDecky.uuid == decky_uuid)
.values(**payload)
)
await session.commit()
async def delete_topology_decky(
self,
decky_uuid: str,
*,
expected_version: Optional[int] = None,
) -> None:
"""Cascade-delete a decky + all its edges from a pending topology."""
async with self._session() as session:
result = await session.execute(
select(TopologyDecky).where(TopologyDecky.uuid == decky_uuid)
)
d = result.scalar_one_or_none()
if d is None:
return
await self._assert_pending(session, d.topology_id)
if expected_version is not None:
await self._check_and_bump_version(
session, d.topology_id, expected_version
)
await session.execute(
text("DELETE FROM topology_edges WHERE decky_uuid = :u"),
{"u": decky_uuid},
)
await session.execute(
text("DELETE FROM topology_deckies WHERE uuid = :u"),
{"u": decky_uuid},
)
await session.commit()
async def list_topology_deckies(
self, topology_id: str
) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(TopologyDecky)
.where(TopologyDecky.topology_id == topology_id)
.order_by(asc(TopologyDecky.name))
)
return [
_deserialize_json_fields(
r.model_dump(mode="json"), ("services", "decky_config")
)
for r in result.scalars().all()
]
async def list_running_topology_deckies(self) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(TopologyDecky).where(TopologyDecky.state == "running")
)
return [
_deserialize_json_fields(
r.model_dump(mode="json"), ("services", "decky_config")
)
for r in result.scalars().all()
]

74
decnet/web/db/sqlmodel_repo/topology/edges.py Normal file
View File

@@ -0,0 +1,74 @@
"""Topology edge CRUD + status-event log."""
from __future__ import annotations
from typing import Any, Optional
from sqlalchemy import desc, select, text
from decnet.web.db.models import TopologyEdge, TopologyStatusEvent
class TopologyEdgesMixin:
"""``self._assert_pending`` / ``self._check_and_bump_version`` resolve
through ``TopologyCoreMixin`` via MRO."""
async def add_topology_edge(
self,
data: dict[str, Any],
*,
expected_version: Optional[int] = None,
) -> str:
async with self._session() as session:
await self._check_and_bump_version(
session, data["topology_id"], expected_version
)
row = TopologyEdge(**data)
session.add(row)
await session.commit()
await session.refresh(row)
return row.id
async def delete_topology_edge(
self,
edge_id: str,
*,
expected_version: Optional[int] = None,
) -> None:
async with self._session() as session:
result = await session.execute(
select(TopologyEdge).where(TopologyEdge.id == edge_id)
)
edge = result.scalar_one_or_none()
if edge is None:
return
await self._assert_pending(session, edge.topology_id)
if expected_version is not None:
await self._check_and_bump_version(
session, edge.topology_id, expected_version
)
await session.execute(
text("DELETE FROM topology_edges WHERE id = :e"),
{"e": edge_id},
)
await session.commit()
async def list_topology_edges(
self, topology_id: str
) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(TopologyEdge).where(TopologyEdge.topology_id == topology_id)
)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def list_topology_status_events(
self, topology_id: str, limit: int = 100
) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(TopologyStatusEvent)
.where(TopologyStatusEvent.topology_id == topology_id)
.order_by(desc(TopologyStatusEvent.at))
.limit(limit)
)
return [r.model_dump(mode="json") for r in result.scalars().all()]

118
decnet/web/db/sqlmodel_repo/topology/lans.py Normal file
View File

@@ -0,0 +1,118 @@
"""LAN CRUD within a topology."""
from __future__ import annotations
from typing import Any, Optional
from sqlalchemy import asc, select, text, update
from decnet.web.db.models import LAN, TopologyEdge
class LansMixin:
"""``self._assert_pending`` / ``self._check_and_bump_version`` resolve
through ``TopologyCoreMixin`` via MRO."""
async def add_lan(
self,
data: dict[str, Any],
*,
expected_version: Optional[int] = None,
) -> str:
async with self._session() as session:
await self._check_and_bump_version(
session, data["topology_id"], expected_version
)
row = LAN(**data)
session.add(row)
await session.commit()
await session.refresh(row)
return row.id
async def update_lan(
self,
lan_id: str,
fields: dict[str, Any],
*,
expected_version: Optional[int] = None,
enforce_pending: bool = False,
) -> None:
if not fields:
return
async with self._session() as session:
result = await session.execute(
select(LAN).where(LAN.id == lan_id)
)
lan = result.scalar_one_or_none()
if lan is None:
raise ValueError(f"lan {lan_id!r} not found")
if enforce_pending:
await self._assert_pending(session, lan.topology_id)
if expected_version is not None:
await self._check_and_bump_version(
session, lan.topology_id, expected_version
)
await session.execute(
update(LAN).where(LAN.id == lan_id).values(**fields)
)
await session.commit()
async def delete_lan(
self,
lan_id: str,
*,
expected_version: Optional[int] = None,
) -> None:
"""Cascade-delete a LAN from a pending topology.
Rejects if any decky declares this LAN as its home (i.e. has a
non-bridge edge to it — the only LAN that decky lives in). The
caller must delete or reassign the home-deckies first.
"""
from decnet.topology.status import TopologyNotEditable # noqa: F401
async with self._session() as session:
result = await session.execute(select(LAN).where(LAN.id == lan_id))
lan = result.scalar_one_or_none()
if lan is None:
return
await self._assert_pending(session, lan.topology_id)
# Home-decky check: any decky whose only edge lands here?
edges_result = await session.execute(
select(TopologyEdge).where(TopologyEdge.lan_id == lan_id)
)
edges_here = edges_result.scalars().all()
decky_uuids_on_this_lan = {e.decky_uuid for e in edges_here}
for decky_uuid in decky_uuids_on_this_lan:
other = await session.execute(
select(TopologyEdge).where(
TopologyEdge.decky_uuid == decky_uuid,
TopologyEdge.lan_id != lan_id,
)
)
if other.scalars().first() is None:
raise ValueError(
f"cannot delete LAN {lan.name!r}: decky "
f"{decky_uuid} has no other LAN (would be orphaned)"
)
if expected_version is not None:
await self._check_and_bump_version(
session, lan.topology_id, expected_version
)
# Cascade edges → LAN.
await session.execute(
text("DELETE FROM topology_edges WHERE lan_id = :l"),
{"l": lan_id},
)
await session.execute(text("DELETE FROM lans WHERE id = :l"), {"l": lan_id})
await session.commit()
async def list_lans_for_topology(
self, topology_id: str
) -> list[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(LAN).where(LAN.topology_id == topology_id).order_by(asc(LAN.name))
)
return [r.model_dump(mode="json") for r in result.scalars().all()]

171
decnet/web/db/sqlmodel_repo/topology/mutations.py Normal file
View File

@@ -0,0 +1,171 @@
"""Live-reconciler mutation queue: enqueue + atomic claim + state writes."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
import orjson
from sqlalchemy import asc, desc, select, text
from decnet.web.db.models import TopologyMutation
class TopologyMutationsMixin:
"""``self._check_and_bump_version`` resolves through
``TopologyCoreMixin`` via MRO."""
async def enqueue_topology_mutation(
self,
topology_id: str,
op: str,
payload: dict[str, Any],
*,
expected_version: Optional[int] = None,
) -> str:
"""Append a pending mutation row and bump the topology version.
Intended for use while the topology is ``active|degraded``; the
reconciler picks these rows up on its next tick.
"""
async with self._session() as session:
await self._check_and_bump_version(
session, topology_id, expected_version
)
row = TopologyMutation(
topology_id=topology_id,
op=op,
payload=orjson.dumps(payload).decode(),
)
session.add(row)
await session.commit()
await session.refresh(row)
return row.id
async def claim_next_mutation(
self, topology_id: str
) -> Optional[dict[str, Any]]:
"""Atomically claim the oldest pending mutation for ``topology_id``.
Correctness-critical: this is ONE SQL statement. Splitting it
into SELECT-then-UPDATE would let two racing watch-loops both
see the same ``pending`` row and both transition it to
``applying`` — double-executing the op. With the single
``UPDATE ... WHERE id = (SELECT ... LIMIT 1) AND state='pending'``
pattern the loser's UPDATE matches zero rows and returns
``None`` — that is the expected, non-error outcome under
contention.
"""
async with self._session() as session:
now = datetime.now(timezone.utc).isoformat()
# Single-statement atomic claim. The inner SELECT picks the
# oldest pending row; the outer UPDATE re-checks state so a
# second racer that also saw that id finds state='applying'
# and matches zero rows.
# MySQL forbids referencing the UPDATE target inside a
# subquery (ERROR 1093). Wrapping the inner SELECT in a
# derived table forces materialisation and sidesteps the
# rule. SQLite accepts both forms, so this stays portable.
sql = text(
"""
UPDATE topology_mutations
SET state = 'applying'
WHERE id = (
SELECT id FROM (
SELECT id FROM topology_mutations
WHERE topology_id = :t AND state = 'pending'
ORDER BY requested_at ASC
LIMIT 1
) AS _next
)
AND state = 'pending'
"""
)
result = await session.execute(sql, {"t": topology_id})
if result.rowcount == 0:
await session.commit()
return None
# Re-read the row we just claimed. The post-UPDATE SELECT is
# safe: no racer can now transition an ``applying`` row back
# to ``pending``.
sel = await session.execute(
select(TopologyMutation)
.where(TopologyMutation.topology_id == topology_id)
.where(TopologyMutation.state == "applying")
.order_by(asc(TopologyMutation.requested_at))
.limit(1)
)
row = sel.scalar_one_or_none()
await session.commit()
_ = now
if row is None:
return None
return row.model_dump(mode="json")
async def mark_mutation_applied(self, mutation_id: str) -> None:
async with self._session() as session:
await session.execute(
text(
"UPDATE topology_mutations "
"SET state = 'applied', applied_at = :at "
"WHERE id = :i"
),
{
"at": datetime.now(timezone.utc).isoformat(),
"i": mutation_id,
},
)
await session.commit()
async def mark_mutation_failed(
self, mutation_id: str, reason: str
) -> None:
async with self._session() as session:
await session.execute(
text(
"UPDATE topology_mutations "
"SET state = 'failed', applied_at = :at, reason = :r "
"WHERE id = :i"
),
{
"at": datetime.now(timezone.utc).isoformat(),
"r": reason,
"i": mutation_id,
},
)
await session.commit()
async def list_topology_mutations(
self,
topology_id: str,
state: Optional[str] = None,
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = (
select(TopologyMutation)
.where(TopologyMutation.topology_id == topology_id)
.order_by(desc(TopologyMutation.requested_at))
)
if state is not None:
stmt = stmt.where(TopologyMutation.state == state)
result = await session.execute(stmt)
return [r.model_dump(mode="json") for r in result.scalars().all()]
async def has_pending_topology_mutation(self) -> bool:
"""Cheap watch-loop guard: any pending mutation on a live topology?
Uses the ``ix_topology_mutations_state_topology`` composite index
to keep the join cheap at scale. Returns False as soon as the
reconciler path should be skipped.
"""
async with self._session() as session:
result = await session.execute(
text(
"SELECT 1 FROM topology_mutations "
"WHERE state = 'pending' "
"AND topology_id IN ("
" SELECT id FROM topologies "
" WHERE status IN ('active', 'degraded')"
") LIMIT 1"
)
)
return result.first() is not None

133
decnet/web/db/sqlmodel_repo/webhooks.py Normal file
View File

@@ -0,0 +1,133 @@
"""Webhook subscription CRUD + delivery bookkeeping."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import select, update
from decnet.web.db.models import WebhookSubscription
class WebhooksMixin:
"""Mixin: composed onto ``SQLModelRepository``."""
async def create_webhook_subscription(self, data: dict[str, Any]) -> None:
async with self._session() as session:
session.add(WebhookSubscription(**data))
await session.commit()
async def get_webhook_subscription(
self, uuid: str
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(WebhookSubscription).where(WebhookSubscription.uuid == uuid)
)
row = result.scalar_one_or_none()
return row.model_dump() if row else None
async def get_webhook_subscription_by_name(
self, name: str
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(WebhookSubscription).where(WebhookSubscription.name == name)
)
row = result.scalar_one_or_none()
return row.model_dump() if row else None
async def list_webhook_subscriptions(
self, enabled_only: bool = False
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = select(WebhookSubscription)
if enabled_only:
stmt = stmt.where(WebhookSubscription.enabled.is_(True))
stmt = stmt.order_by(WebhookSubscription.created_at)
result = await session.execute(stmt)
return [r.model_dump() for r in result.scalars().all()]
async def update_webhook_subscription(
self, uuid: str, patch: dict[str, Any]
) -> bool:
if not patch:
return True
patch = {**patch, "updated_at": datetime.now(timezone.utc)}
async with self._session() as session:
result = await session.execute(
update(WebhookSubscription)
.where(WebhookSubscription.uuid == uuid)
.values(**patch)
)
await session.commit()
return result.rowcount > 0
async def delete_webhook_subscription(self, uuid: str) -> bool:
async with self._session() as session:
result = await session.execute(
select(WebhookSubscription).where(WebhookSubscription.uuid == uuid)
)
row = result.scalar_one_or_none()
if not row:
return False
await session.delete(row)
await session.commit()
return True
async def record_webhook_success(
self, uuid: str, ts: datetime
) -> None:
async with self._session() as session:
await session.execute(
update(WebhookSubscription)
.where(WebhookSubscription.uuid == uuid)
.values(
consecutive_failures=0,
last_success_at=ts,
last_error=None,
updated_at=ts,
)
)
await session.commit()
async def record_webhook_failure(
self, uuid: str, ts: datetime, error: str
) -> int:
async with self._session() as session:
# Read current failure count, bump, write. Small race window on
# concurrent deliveries to the same subscription is acceptable —
# the counter informs the circuit-breaker heuristic, not a
# correctness invariant.
result = await session.execute(
select(WebhookSubscription.consecutive_failures).where(
WebhookSubscription.uuid == uuid
)
)
current = result.scalar_one_or_none() or 0
new_count = current + 1
await session.execute(
update(WebhookSubscription)
.where(WebhookSubscription.uuid == uuid)
.values(
consecutive_failures=new_count,
last_failure_at=ts,
last_error=error[:512] if error else None,
updated_at=ts,
)
)
await session.commit()
return new_count
async def trip_webhook_circuit(self, uuid: str, ts: datetime) -> None:
async with self._session() as session:
await session.execute(
update(WebhookSubscription)
.where(WebhookSubscription.uuid == uuid)
.values(
enabled=False,
auto_disabled_at=ts,
updated_at=ts,
)
)
await session.commit()