anti/DECNET
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(webhooks): subscription CRUD + HMAC-signed delivery client

Browse Source 
Introduces the webhook egress foundation — a new WebhookSubscription
table, admin-gated CRUD under /api/v1/webhooks, and the shared
delivery client that both the test-ping route and the upcoming worker
will use. No worker yet; this commit is API + model + client only.

Simple-mode enum (AttackerDetail / DeckyStatus / SystemStatus) expands
to bus-topic patterns at the router layer; storage is always the raw
pattern list. Advanced mode lets admins supply raw NATS-style patterns
directly. Filter-at-subscribe: the worker (next commit) will subscribe
to the union of patterns across enabled subscriptions.

Delivery client handles HMAC-SHA256 signing (X-DECNET-Signature),
retry on 429/5xx/network errors with jittered backoff, no-retry on
4xx. Secrets never leave the server on GET/LIST — only the create
response carries the secret for copy-out.

CRUD routes publish WEBHOOK_SUBSCRIPTIONS_CHANGED on the bus after
every mutation so the (future) worker can hot-reload.

Opens DEBT-037 for the deferred items (circuit breaker, dead-letter,
batch delivery, payload templates, secret-at-rest).
This commit is contained in:
anti
2026-04-24 15:30:05 -04:00
parent 162f7c1194
commit b70845a85d
17 changed files with 1222 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
decnet/web/db/models/__init__.py
View File

@@ -112,6 +112,15 @@ from .updater import (
RollbackRequest,
RollbackResponse,
)
from .webhooks import (
SimpleEvent,
WebhookCreateRequest,
WebhookCreateResponse,
WebhookResponse,
WebhookSubscription,
WebhookTestResponse,
WebhookUpdateRequest,
)
from .workers import (
StartAllResponse,
StartFailure,
@@ -218,6 +227,14 @@ __all__ = [
"PushUpdateResult",
"RollbackRequest",
"RollbackResponse",
# webhooks
"SimpleEvent",
"WebhookCreateRequest",
"WebhookCreateResponse",
"WebhookResponse",
"WebhookSubscription",
"WebhookTestResponse",
"WebhookUpdateRequest",
# workers
"StartAllResponse",
"StartFailure",

126
decnet/web/db/models/webhooks.py Normal file
View File

@@ -0,0 +1,126 @@
"""Webhook subscription table + CRUD DTOs.
Webhooks push DECNET bus events out to external SIEM / SOAR stacks
(Wazuh, Shuffle, TheHive, n8n, ...). Each subscription carries a set
of NATS-style topic patterns; the `decnet webhook` worker subscribes
to the union of patterns across all enabled subscriptions and POSTs
matching events to each matching URL with HMAC-SHA256 signing.
Simple mode (UI) exposes a friendly enum (`AttackerDetail`,
`DeckyStatus`, `SystemStatus`) that expands to patterns at save time.
Advanced mode lets an admin set raw patterns directly. Storage is
always the expanded list — the enum is sugar at the router layer.
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any, List, Literal, Optional
from uuid import uuid4
from pydantic import BaseModel, Field as PydanticField, HttpUrl
from sqlmodel import Field, SQLModel
SimpleEvent = Literal["AttackerDetail", "DeckyStatus", "SystemStatus"]
class WebhookSubscription(SQLModel, table=True):
__tablename__ = "webhook_subscriptions"
uuid: str = Field(default_factory=lambda: str(uuid4()), primary_key=True)
name: str = Field(index=True, unique=True)
url: str
secret: str # HMAC-SHA256 key; plaintext pre-v1 (see DEBT-037 §7)
# JSON-encoded list[str] of NATS-style bus topic patterns.
# Storing as TEXT keeps the schema portable across SQLite and MySQL
# without pulling in dialect-specific JSON columns.
topic_patterns: str = Field(default="[]")
enabled: bool = Field(default=True, index=True)
consecutive_failures: int = Field(default=0)
last_success_at: Optional[datetime] = None
last_failure_at: Optional[datetime] = None
last_error: Optional[str] = None
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
def patterns(self) -> list[str]:
"""Decode `topic_patterns` to a list. Returns [] on bad/empty JSON."""
try:
raw = json.loads(self.topic_patterns or "[]")
except (ValueError, TypeError):
return []
return [p for p in raw if isinstance(p, str)]
# --- API Request / Response Models (Pydantic) ---
class WebhookCreateRequest(BaseModel):
name: str = PydanticField(..., min_length=1, max_length=64)
url: HttpUrl
# If secret is omitted, the router generates a secure random one and
# returns it exactly once on the create response. After that, callers
# can only rotate via PATCH.
secret: Optional[str] = PydanticField(None, min_length=16, max_length=256)
# At least one of simple_events / topic_patterns must be non-empty
# (validated in the router, not Pydantic, so the 400 carries a clear
# detail message).
simple_events: List[SimpleEvent] = PydanticField(default_factory=list)
topic_patterns: List[str] = PydanticField(default_factory=list)
enabled: bool = True
class WebhookUpdateRequest(BaseModel):
# Partial update — every field optional; the router diffs against the
# current row and only writes what changed.
name: Optional[str] = PydanticField(None, min_length=1, max_length=64)
url: Optional[HttpUrl] = None
secret: Optional[str] = PydanticField(None, min_length=16, max_length=256)
simple_events: Optional[List[SimpleEvent]] = None
topic_patterns: Optional[List[str]] = None
enabled: Optional[bool] = None
class WebhookResponse(BaseModel):
"""Public shape — deliberately omits `secret`."""
uuid: str
name: str
url: str
topic_patterns: List[str]
enabled: bool
consecutive_failures: int
last_success_at: Optional[datetime] = None
last_failure_at: Optional[datetime] = None
last_error: Optional[str] = None
created_at: datetime
updated_at: datetime
class WebhookCreateResponse(WebhookResponse):
"""Create-path response — carries the secret exactly once, for copy-out."""
secret: str
class WebhookTestResponse(BaseModel):
delivered: bool
status_code: Optional[int] = None
error: Optional[str] = None
def _row_to_response_dict(row: dict[str, Any]) -> dict[str, Any]:
"""Normalize a DB row into the WebhookResponse dict shape.
Used by the CRUD router to decode `topic_patterns` JSON and drop the
`secret` column before returning to the client.
"""
out = dict(row)
raw = out.pop("topic_patterns", "[]")
try:
out["topic_patterns"] = json.loads(raw or "[]")
except (ValueError, TypeError):
out["topic_patterns"] = []
out.pop("secret", None)
return out

37
decnet/web/db/repository.py
View File

@@ -431,3 +431,40 @@ class BaseRepository(ABC):
async def list_live_topology_ids(self) -> list[str]:
return []
# --------------------------------------------------------- webhooks
# Webhook subscriptions — external SIEM / SOAR egress configuration.
# Default NotImplementedError keeps non-default backends honest; the
# SQLModel-backed SQLite and MySQL repos override everything below.
async def create_webhook_subscription(self, data: dict[str, Any]) -> None:
raise NotImplementedError
async def get_webhook_subscription(self, uuid: str) -> Optional[dict[str, Any]]:
raise NotImplementedError
async def get_webhook_subscription_by_name(
self, name: str
) -> Optional[dict[str, Any]]:
raise NotImplementedError
async def list_webhook_subscriptions(
self, enabled_only: bool = False
) -> list[dict[str, Any]]:
raise NotImplementedError
async def update_webhook_subscription(
self, uuid: str, patch: dict[str, Any]
) -> bool:
raise NotImplementedError
async def delete_webhook_subscription(self, uuid: str) -> bool:
raise NotImplementedError
async def record_webhook_success(self, uuid: str, ts: Any) -> None:
raise NotImplementedError
async def record_webhook_failure(
self, uuid: str, ts: Any, error: str
) -> None:
raise NotImplementedError

108
decnet/web/db/sqlmodel_repo.py
View File

@@ -44,6 +44,7 @@ from decnet.web.db.models import (
TopologyEdge,
TopologyStatusEvent,
TopologyMutation,
WebhookSubscription,
)
@@ -1744,3 +1745,110 @@ class SQLModelRepository(BaseRepository):
)
)
return [r for r in result.scalars().all()]
# --------------------------------------------------------- webhooks
async def create_webhook_subscription(self, data: dict[str, Any]) -> None:
async with self._session() as session:
session.add(WebhookSubscription(**data))
await session.commit()
async def get_webhook_subscription(
self, uuid: str
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(WebhookSubscription).where(WebhookSubscription.uuid == uuid)
)
row = result.scalar_one_or_none()
return row.model_dump() if row else None
async def get_webhook_subscription_by_name(
self, name: str
) -> Optional[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(WebhookSubscription).where(WebhookSubscription.name == name)
)
row = result.scalar_one_or_none()
return row.model_dump() if row else None
async def list_webhook_subscriptions(
self, enabled_only: bool = False
) -> list[dict[str, Any]]:
async with self._session() as session:
stmt = select(WebhookSubscription)
if enabled_only:
stmt = stmt.where(WebhookSubscription.enabled.is_(True))
stmt = stmt.order_by(WebhookSubscription.created_at)
result = await session.execute(stmt)
return [r.model_dump() for r in result.scalars().all()]
async def update_webhook_subscription(
self, uuid: str, patch: dict[str, Any]
) -> bool:
if not patch:
return True
patch = {**patch, "updated_at": datetime.now(timezone.utc)}
async with self._session() as session:
result = await session.execute(
update(WebhookSubscription)
.where(WebhookSubscription.uuid == uuid)
.values(**patch)
)
await session.commit()
return result.rowcount > 0
async def delete_webhook_subscription(self, uuid: str) -> bool:
async with self._session() as session:
result = await session.execute(
select(WebhookSubscription).where(WebhookSubscription.uuid == uuid)
)
row = result.scalar_one_or_none()
if not row:
return False
await session.delete(row)
await session.commit()
return True
async def record_webhook_success(
self, uuid: str, ts: datetime
) -> None:
async with self._session() as session:
await session.execute(
update(WebhookSubscription)
.where(WebhookSubscription.uuid == uuid)
.values(
consecutive_failures=0,
last_success_at=ts,
last_error=None,
updated_at=ts,
)
)
await session.commit()
async def record_webhook_failure(
self, uuid: str, ts: datetime, error: str
) -> None:
async with self._session() as session:
# Read current failure count, bump, write. Small race window on
# concurrent deliveries to the same subscription is acceptable —
# the counter informs the circuit-breaker heuristic (DEBT-037),
# not a correctness invariant.
result = await session.execute(
select(WebhookSubscription.consecutive_failures).where(
WebhookSubscription.uuid == uuid
)
)
current = result.scalar_one_or_none() or 0
await session.execute(
update(WebhookSubscription)
.where(WebhookSubscription.uuid == uuid)
.values(
consecutive_failures=current + 1,
last_failure_at=ts,
last_error=error[:512] if error else None,
updated_at=ts,
)
)
await session.commit()

4
decnet/web/router/__init__.py
View File

@@ -33,6 +33,7 @@ from .swarm_updates import swarm_updates_router
from .swarm_mgmt import swarm_mgmt_router
from .system import system_router
from .topology import topology_router
from .webhooks import webhooks_router
api_router = APIRouter(
# Every route under /api/v1 is auth-guarded (either by an explicit
@@ -105,3 +106,6 @@ api_router.include_router(system_router)
# MazeNET Topologies (nested topology CRUD + mutation queue)
api_router.include_router(topology_router)
# External webhook subscriptions (SIEM/SOAR egress)
api_router.include_router(webhooks_router)

18
decnet/web/router/webhooks/__init__.py Normal file
View File

@@ -0,0 +1,18 @@
"""Webhook subscription CRUD.
Admin-gated management of external-egress webhook subscriptions. The
actual delivery happens in the `decnet webhook` worker, which watches
the DB + bus and POSTs matching events out. This module is the API
surface operators use to configure destinations.
Mounted under `/api/v1/webhooks` by the main api router.
"""
from fastapi import APIRouter
from .api_manage_webhooks import router as manage_webhooks_router
from .api_test_webhook import router as test_webhook_router
webhooks_router = APIRouter(prefix="/webhooks")
webhooks_router.include_router(manage_webhooks_router)
webhooks_router.include_router(test_webhook_router)

222
decnet/web/router/webhooks/api_manage_webhooks.py Normal file
View File

@@ -0,0 +1,222 @@
"""Webhook subscription CRUD — admin-gated."""
from __future__ import annotations
import json
import secrets
from datetime import datetime, timezone
from typing import Any
from fastapi import APIRouter, Depends, HTTPException
from decnet.bus import topics as _topics
from decnet.bus.app import get_app_bus
from decnet.logging import get_logger
from decnet.telemetry import traced as _traced
from decnet.web.db.models import (
MessageResponse,
WebhookCreateRequest,
WebhookCreateResponse,
WebhookResponse,
WebhookUpdateRequest,
)
from decnet.web.db.models.webhooks import _row_to_response_dict
from decnet.web.dependencies import repo, require_admin
from decnet.webhook.enums import merge_patterns
log = get_logger("api.webhooks")
router = APIRouter()
async def _notify_subscriptions_changed() -> None:
"""Publish `system.webhook.subscriptions_changed` on the bus.
Fire-and-forget per the bus contract — a dropped signal is recoverable
because the webhook worker also reloads on a slow timer as a fallback.
"""
try:
bus = await get_app_bus()
if bus is None:
return
await bus.publish(
_topics.WEBHOOK_SUBSCRIPTIONS_CHANGED,
{},
event_type="changed",
)
except Exception as e: # noqa: BLE001 — bus failures must not break CRUD
log.warning("webhook subscriptions-changed publish failed: %s", e)
def _row_to_response(row: dict[str, Any]) -> WebhookResponse:
return WebhookResponse(**_row_to_response_dict(row))
@router.post(
"/",
tags=["Webhooks"],
response_model=WebhookCreateResponse,
status_code=201,
responses={
400: {"description": "At least one of simple_events / topic_patterns required"},
409: {"description": "Name already in use"},
},
)
@_traced("api.webhook.create")
async def api_create_webhook(
req: WebhookCreateRequest,
admin: dict = Depends(require_admin),
) -> WebhookCreateResponse:
patterns = merge_patterns(req.simple_events, req.topic_patterns)
if not patterns:
raise HTTPException(
status_code=400,
detail="Provide at least one simple_events entry or topic_patterns pattern.",
)
existing = await repo.get_webhook_subscription_by_name(req.name)
if existing:
raise HTTPException(status_code=409, detail="Webhook name already exists")
# Auto-generate a URL-safe secret if the caller didn't provide one.
# 32 bytes of os-entropy is the same ballpark as a CSRF token.
secret = req.secret or secrets.token_urlsafe(32)
now = datetime.now(timezone.utc)
data = {
"name": req.name,
"url": str(req.url),
"secret": secret,
"topic_patterns": json.dumps(patterns),
"enabled": req.enabled,
"consecutive_failures": 0,
"created_at": now,
"updated_at": now,
}
await repo.create_webhook_subscription(data)
row = await repo.get_webhook_subscription_by_name(req.name)
if row is None:
# Should never happen — the create just committed. Treat as 500
# rather than silently masking a storage bug.
raise HTTPException(status_code=500, detail="Webhook created but not retrievable")
await _notify_subscriptions_changed()
return WebhookCreateResponse(
**_row_to_response_dict(row),
secret=secret,
)
@router.get(
"/",
tags=["Webhooks"],
response_model=list[WebhookResponse],
)
@_traced("api.webhook.list")
async def api_list_webhooks(
admin: dict = Depends(require_admin),
) -> list[WebhookResponse]:
rows = await repo.list_webhook_subscriptions()
return [_row_to_response(r) for r in rows]
@router.get(
"/{uuid}",
tags=["Webhooks"],
response_model=WebhookResponse,
responses={404: {"description": "Webhook not found"}},
)
@_traced("api.webhook.get")
async def api_get_webhook(
uuid: str,
admin: dict = Depends(require_admin),
) -> WebhookResponse:
row = await repo.get_webhook_subscription(uuid)
if not row:
raise HTTPException(status_code=404, detail="Webhook not found")
return _row_to_response(row)
@router.patch(
"/{uuid}",
tags=["Webhooks"],
response_model=WebhookResponse,
responses={
400: {"description": "Empty or invalid patch"},
404: {"description": "Webhook not found"},
409: {"description": "Name already in use"},
},
)
@_traced("api.webhook.update")
async def api_update_webhook(
uuid: str,
req: WebhookUpdateRequest,
admin: dict = Depends(require_admin),
) -> WebhookResponse:
current = await repo.get_webhook_subscription(uuid)
if not current:
raise HTTPException(status_code=404, detail="Webhook not found")
patch: dict[str, Any] = {}
if req.name is not None and req.name != current["name"]:
clash = await repo.get_webhook_subscription_by_name(req.name)
if clash and clash["uuid"] != uuid:
raise HTTPException(status_code=409, detail="Webhook name already exists")
patch["name"] = req.name
if req.url is not None:
patch["url"] = str(req.url)
if req.secret is not None:
patch["secret"] = req.secret
if req.enabled is not None:
patch["enabled"] = req.enabled
if req.simple_events is not None or req.topic_patterns is not None:
# Re-merge using whatever the caller supplied; a caller that wants
# to clear all patterns must explicitly pass both as empty lists.
simple = req.simple_events if req.simple_events is not None else []
raw = req.topic_patterns if req.topic_patterns is not None else []
patterns = merge_patterns(simple, raw)
if not patterns:
raise HTTPException(
status_code=400,
detail="Cannot clear all patterns; disable the webhook instead.",
)
patch["topic_patterns"] = json.dumps(patterns)
if not patch:
# No-op patch — return the current row untouched.
return _row_to_response(current)
updated = await repo.update_webhook_subscription(uuid, patch)
if not updated:
raise HTTPException(status_code=404, detail="Webhook not found")
await _notify_subscriptions_changed()
row = await repo.get_webhook_subscription(uuid)
if row is None:
raise HTTPException(status_code=404, detail="Webhook not found")
return _row_to_response(row)
@router.delete(
"/{uuid}",
tags=["Webhooks"],
response_model=MessageResponse,
responses={404: {"description": "Webhook not found"}},
)
@_traced("api.webhook.delete")
async def api_delete_webhook(
uuid: str,
admin: dict = Depends(require_admin),
) -> dict[str, str]:
deleted = await repo.delete_webhook_subscription(uuid)
if not deleted:
raise HTTPException(status_code=404, detail="Webhook not found")
await _notify_subscriptions_changed()
return {"message": "Webhook deleted"}

60
decnet/web/router/webhooks/api_test_webhook.py Normal file
View File

@@ -0,0 +1,60 @@
"""POST /webhooks/{uuid}/test — fire a synthetic ping to verify plumbing.
This hits the same delivery path the worker uses, so a 200 here proves
the destination URL, HMAC secret, and network egress all work without
waiting for a real bus event.
"""
from __future__ import annotations
from datetime import datetime, timezone
from uuid import uuid4
from fastapi import APIRouter, Depends, HTTPException
from decnet.logging import get_logger
from decnet.telemetry import traced as _traced
from decnet.web.db.models import WebhookTestResponse
from decnet.web.dependencies import repo, require_admin
from decnet.webhook.client import deliver, SyntheticEvent
log = get_logger("api.webhooks.test")
router = APIRouter()
@router.post(
"/{uuid}/test",
tags=["Webhooks"],
response_model=WebhookTestResponse,
responses={
404: {"description": "Webhook not found"},
},
)
@_traced("api.webhook.test")
async def api_test_webhook(
uuid: str,
admin: dict = Depends(require_admin),
) -> WebhookTestResponse:
sub = await repo.get_webhook_subscription(uuid)
if not sub:
raise HTTPException(status_code=404, detail="Webhook not found")
event = SyntheticEvent(
topic="test.ping",
type="test",
ts=datetime.now(timezone.utc).isoformat(),
id=str(uuid4()),
payload={
"message": "Synthetic test event from DECNET",
"triggered_by": admin.get("username", "unknown"),
},
)
# Single attempt — no retries on manual tests. The operator wants a
# fast signal about the current state of the receiver, not a
# retry-and-wait behavior.
result = await deliver(sub, event, retry_schedule=[])
return WebhookTestResponse(
delivered=result.ok,
status_code=result.status_code,
error=result.error,
)