feat: Phase 1 — JA3/JA3S sniffer, Attacker model, profile worker
Add passive TLS fingerprinting via a sniffer container on the MACVLAN interface, plus the Attacker table and periodic rebuild worker that correlates per-IP profiles from Log + Bounty + CorrelationEngine. - templates/sniffer/: Scapy sniffer with pure-Python TLS parser; emits tls_client_hello / tls_session RFC 5424 lines with ja3, ja3s, sni, alpn, raw_ciphers, raw_extensions; GREASE filtered per RFC 8701 - decnet/services/sniffer.py: service plugin (no ports, NET_RAW/NET_ADMIN) - decnet/web/db/models.py: Attacker SQLModel table + AttackersResponse - decnet/web/db/repository.py: 5 new abstract methods - decnet/web/db/sqlite/repository.py: implement all 5 (upsert, pagination, sort by recent/active/traversals, bounty grouping) - decnet/web/attacker_worker.py: 30s periodic rebuild via CorrelationEngine; extracts commands from log fields, merges fingerprint bounties - decnet/web/api.py: wire attacker_profile_worker into lifespan - decnet/web/ingester.py: extract JA3 bounty (fingerprint_type=ja3) - development/DEVELOPMENT.md: full attacker intelligence collection roadmap - pyproject.toml: scapy>=2.6.1 added to dev deps - tests: test_sniffer_ja3.py (40+ vectors), test_attacker_worker.py, test_base_repo.py / test_web_api.py updated for new surface
This commit is contained in:
@@ -14,16 +14,18 @@ from decnet.logging import get_logger
|
||||
from decnet.web.dependencies import repo
|
||||
from decnet.collector import log_collector_worker
|
||||
from decnet.web.ingester import log_ingestion_worker
|
||||
from decnet.web.attacker_worker import attacker_profile_worker
|
||||
from decnet.web.router import api_router
|
||||
|
||||
log = get_logger("api")
|
||||
ingestion_task: Optional[asyncio.Task[Any]] = None
|
||||
collector_task: Optional[asyncio.Task[Any]] = None
|
||||
attacker_task: Optional[asyncio.Task[Any]] = None
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
global ingestion_task, collector_task
|
||||
global ingestion_task, collector_task, attacker_task
|
||||
|
||||
log.info("API startup initialising database")
|
||||
for attempt in range(1, 6):
|
||||
@@ -51,13 +53,18 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
log.debug("API startup collector worker started log_file=%s", _log_file)
|
||||
elif not _log_file:
|
||||
log.warning("DECNET_INGEST_LOG_FILE not set — Docker log collection disabled.")
|
||||
|
||||
# Start attacker profile rebuild worker
|
||||
if attacker_task is None or attacker_task.done():
|
||||
attacker_task = asyncio.create_task(attacker_profile_worker(repo))
|
||||
log.debug("API startup attacker profile worker started")
|
||||
else:
|
||||
log.info("Contract Test Mode: skipping background worker startup")
|
||||
|
||||
yield
|
||||
|
||||
log.info("API shutdown cancelling background tasks")
|
||||
for task in (ingestion_task, collector_task):
|
||||
for task in (ingestion_task, collector_task, attacker_task):
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
try:
|
||||
|
||||
176
decnet/web/attacker_worker.py
Normal file
176
decnet/web/attacker_worker.py
Normal file
@@ -0,0 +1,176 @@
|
||||
"""
|
||||
Attacker profile builder — background worker.
|
||||
|
||||
Periodically rebuilds the `attackers` table by:
|
||||
1. Feeding all stored Log.raw_line values through the CorrelationEngine
|
||||
(which parses RFC 5424 and tracks per-IP event histories + traversals).
|
||||
2. Merging with the Bounty table (fingerprints, credentials).
|
||||
3. Extracting commands executed per IP from the structured log fields.
|
||||
4. Upserting one Attacker record per observed IP.
|
||||
|
||||
Runs every _REBUILD_INTERVAL seconds. Full rebuild each cycle — simple and
|
||||
correct at honeypot log volumes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from decnet.correlation.engine import CorrelationEngine
|
||||
from decnet.correlation.parser import LogEvent
|
||||
from decnet.logging import get_logger
|
||||
from decnet.web.db.repository import BaseRepository
|
||||
|
||||
logger = get_logger("attacker_worker")
|
||||
|
||||
_REBUILD_INTERVAL = 30 # seconds
|
||||
|
||||
# Event types that indicate active command/query execution (not just connection/scan)
|
||||
_COMMAND_EVENT_TYPES = frozenset({
|
||||
"command", "exec", "query", "input", "shell_input",
|
||||
"execute", "run", "sql_query", "redis_command",
|
||||
})
|
||||
|
||||
# Fields that carry the executed command/query text
|
||||
_COMMAND_FIELDS = ("command", "query", "input", "line", "sql", "cmd")
|
||||
|
||||
|
||||
async def attacker_profile_worker(repo: BaseRepository) -> None:
|
||||
"""Periodically rebuilds the Attacker table. Designed to run as an asyncio Task."""
|
||||
logger.info("attacker profile worker started interval=%ds", _REBUILD_INTERVAL)
|
||||
while True:
|
||||
await asyncio.sleep(_REBUILD_INTERVAL)
|
||||
try:
|
||||
await _rebuild(repo)
|
||||
except Exception as exc:
|
||||
logger.error("attacker worker: rebuild failed: %s", exc)
|
||||
|
||||
|
||||
async def _rebuild(repo: BaseRepository) -> None:
|
||||
all_logs = await repo.get_all_logs_raw()
|
||||
if not all_logs:
|
||||
return
|
||||
|
||||
# Feed raw RFC 5424 lines into the CorrelationEngine
|
||||
engine = CorrelationEngine()
|
||||
for row in all_logs:
|
||||
engine.ingest(row["raw_line"])
|
||||
|
||||
if not engine._events:
|
||||
return
|
||||
|
||||
traversal_map = {t.attacker_ip: t for t in engine.traversals(min_deckies=2)}
|
||||
all_bounties = await repo.get_all_bounties_by_ip()
|
||||
|
||||
count = 0
|
||||
for ip, events in engine._events.items():
|
||||
traversal = traversal_map.get(ip)
|
||||
bounties = all_bounties.get(ip, [])
|
||||
commands = _extract_commands(all_logs, ip)
|
||||
|
||||
record = _build_record(ip, events, traversal, bounties, commands)
|
||||
await repo.upsert_attacker(record)
|
||||
count += 1
|
||||
|
||||
logger.debug("attacker worker: rebuilt %d profiles", count)
|
||||
|
||||
|
||||
def _build_record(
|
||||
ip: str,
|
||||
events: list[LogEvent],
|
||||
traversal: Any,
|
||||
bounties: list[dict[str, Any]],
|
||||
commands: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
services = sorted({e.service for e in events})
|
||||
deckies = (
|
||||
traversal.deckies
|
||||
if traversal
|
||||
else _first_contact_deckies(events)
|
||||
)
|
||||
fingerprints = [b for b in bounties if b.get("bounty_type") == "fingerprint"]
|
||||
credential_count = sum(1 for b in bounties if b.get("bounty_type") == "credential")
|
||||
|
||||
return {
|
||||
"ip": ip,
|
||||
"first_seen": min(e.timestamp for e in events),
|
||||
"last_seen": max(e.timestamp for e in events),
|
||||
"event_count": len(events),
|
||||
"service_count": len(services),
|
||||
"decky_count": len({e.decky for e in events}),
|
||||
"services": json.dumps(services),
|
||||
"deckies": json.dumps(deckies),
|
||||
"traversal_path": traversal.path if traversal else None,
|
||||
"is_traversal": traversal is not None,
|
||||
"bounty_count": len(bounties),
|
||||
"credential_count": credential_count,
|
||||
"fingerprints": json.dumps(fingerprints),
|
||||
"commands": json.dumps(commands),
|
||||
"updated_at": datetime.now(timezone.utc),
|
||||
}
|
||||
|
||||
|
||||
def _first_contact_deckies(events: list[LogEvent]) -> list[str]:
|
||||
"""Return unique deckies in first-contact order (for non-traversal attackers)."""
|
||||
seen: list[str] = []
|
||||
for e in sorted(events, key=lambda x: x.timestamp):
|
||||
if e.decky not in seen:
|
||||
seen.append(e.decky)
|
||||
return seen
|
||||
|
||||
|
||||
def _extract_commands(
|
||||
all_logs: list[dict[str, Any]], ip: str
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Extract executed commands for a given attacker IP from raw log rows.
|
||||
|
||||
Looks for rows where:
|
||||
- attacker_ip matches
|
||||
- event_type is a known command-execution type
|
||||
- fields JSON contains a command-like key
|
||||
|
||||
Returns a list of {service, decky, command, timestamp} dicts.
|
||||
"""
|
||||
commands: list[dict[str, Any]] = []
|
||||
for row in all_logs:
|
||||
if row.get("attacker_ip") != ip:
|
||||
continue
|
||||
if row.get("event_type") not in _COMMAND_EVENT_TYPES:
|
||||
continue
|
||||
|
||||
raw_fields = row.get("fields")
|
||||
if not raw_fields:
|
||||
continue
|
||||
|
||||
# fields is stored as a JSON string in the DB row
|
||||
if isinstance(raw_fields, str):
|
||||
try:
|
||||
fields = json.loads(raw_fields)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
else:
|
||||
fields = raw_fields
|
||||
|
||||
cmd_text: str | None = None
|
||||
for key in _COMMAND_FIELDS:
|
||||
val = fields.get(key)
|
||||
if val:
|
||||
cmd_text = str(val)
|
||||
break
|
||||
|
||||
if not cmd_text:
|
||||
continue
|
||||
|
||||
ts = row.get("timestamp")
|
||||
commands.append({
|
||||
"service": row.get("service", ""),
|
||||
"decky": row.get("decky", ""),
|
||||
"command": cmd_text,
|
||||
"timestamp": ts.isoformat() if isinstance(ts, datetime) else str(ts),
|
||||
})
|
||||
|
||||
return commands
|
||||
@@ -50,6 +50,27 @@ class State(SQLModel, table=True):
|
||||
key: str = Field(primary_key=True)
|
||||
value: str # Stores JSON serialized DecnetConfig or other state blobs
|
||||
|
||||
|
||||
class Attacker(SQLModel, table=True):
|
||||
__tablename__ = "attackers"
|
||||
ip: str = Field(primary_key=True)
|
||||
first_seen: datetime = Field(index=True)
|
||||
last_seen: datetime = Field(index=True)
|
||||
event_count: int = Field(default=0)
|
||||
service_count: int = Field(default=0)
|
||||
decky_count: int = Field(default=0)
|
||||
services: str = Field(default="[]") # JSON list[str]
|
||||
deckies: str = Field(default="[]") # JSON list[str], first-contact ordered
|
||||
traversal_path: Optional[str] = None # "decky-01 → decky-03 → decky-05"
|
||||
is_traversal: bool = Field(default=False)
|
||||
bounty_count: int = Field(default=0)
|
||||
credential_count: int = Field(default=0)
|
||||
fingerprints: str = Field(default="[]") # JSON list[dict] — bounty fingerprints
|
||||
commands: str = Field(default="[]") # JSON list[dict] — commands per service/decky
|
||||
updated_at: datetime = Field(
|
||||
default_factory=lambda: datetime.now(timezone.utc), index=True
|
||||
)
|
||||
|
||||
# --- API Request/Response Models (Pydantic) ---
|
||||
|
||||
class Token(BaseModel):
|
||||
@@ -77,6 +98,12 @@ class BountyResponse(BaseModel):
|
||||
offset: int
|
||||
data: List[dict[str, Any]]
|
||||
|
||||
class AttackersResponse(BaseModel):
|
||||
total: int
|
||||
limit: int
|
||||
offset: int
|
||||
data: List[dict[str, Any]]
|
||||
|
||||
class StatsResponse(BaseModel):
|
||||
total_logs: int
|
||||
unique_attackers: int
|
||||
|
||||
@@ -90,3 +90,34 @@ class BaseRepository(ABC):
|
||||
async def set_state(self, key: str, value: Any) -> None:
|
||||
"""Store a specific state entry by key."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_all_logs_raw(self) -> list[dict[str, Any]]:
|
||||
"""Retrieve all log rows with fields needed by the attacker profile worker."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_all_bounties_by_ip(self) -> dict[str, list[dict[str, Any]]]:
|
||||
"""Retrieve all bounty rows grouped by attacker_ip."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def upsert_attacker(self, data: dict[str, Any]) -> None:
|
||||
"""Insert or replace an attacker profile record."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_attackers(
|
||||
self,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
search: Optional[str] = None,
|
||||
sort_by: str = "recent",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Retrieve paginated attacker profile records."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_total_attackers(self, search: Optional[str] = None) -> int:
|
||||
"""Retrieve the total count of attacker profile records, optionally filtered."""
|
||||
pass
|
||||
|
||||
@@ -12,7 +12,7 @@ from decnet.config import load_state, _ROOT
|
||||
from decnet.env import DECNET_ADMIN_USER, DECNET_ADMIN_PASSWORD
|
||||
from decnet.web.auth import get_password_hash
|
||||
from decnet.web.db.repository import BaseRepository
|
||||
from decnet.web.db.models import User, Log, Bounty, State
|
||||
from decnet.web.db.models import User, Log, Bounty, State, Attacker
|
||||
from decnet.web.db.sqlite.database import get_async_engine
|
||||
|
||||
|
||||
@@ -371,3 +371,92 @@ class SQLiteRepository(BaseRepository):
|
||||
session.add(new_state)
|
||||
|
||||
await session.commit()
|
||||
|
||||
# --------------------------------------------------------------- attackers
|
||||
|
||||
async def get_all_logs_raw(self) -> List[dict[str, Any]]:
|
||||
async with self.session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(
|
||||
Log.id,
|
||||
Log.raw_line,
|
||||
Log.attacker_ip,
|
||||
Log.service,
|
||||
Log.event_type,
|
||||
Log.decky,
|
||||
Log.timestamp,
|
||||
Log.fields,
|
||||
)
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": r.id,
|
||||
"raw_line": r.raw_line,
|
||||
"attacker_ip": r.attacker_ip,
|
||||
"service": r.service,
|
||||
"event_type": r.event_type,
|
||||
"decky": r.decky,
|
||||
"timestamp": r.timestamp,
|
||||
"fields": r.fields,
|
||||
}
|
||||
for r in result.all()
|
||||
]
|
||||
|
||||
async def get_all_bounties_by_ip(self) -> dict[str, List[dict[str, Any]]]:
|
||||
from collections import defaultdict
|
||||
async with self.session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(Bounty).order_by(asc(Bounty.timestamp))
|
||||
)
|
||||
grouped: dict[str, List[dict[str, Any]]] = defaultdict(list)
|
||||
for item in result.scalars().all():
|
||||
d = item.model_dump(mode="json")
|
||||
try:
|
||||
d["payload"] = json.loads(d["payload"])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
grouped[item.attacker_ip].append(d)
|
||||
return dict(grouped)
|
||||
|
||||
async def upsert_attacker(self, data: dict[str, Any]) -> None:
|
||||
async with self.session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(Attacker).where(Attacker.ip == data["ip"])
|
||||
)
|
||||
existing = result.scalar_one_or_none()
|
||||
if existing:
|
||||
for k, v in data.items():
|
||||
setattr(existing, k, v)
|
||||
session.add(existing)
|
||||
else:
|
||||
session.add(Attacker(**data))
|
||||
await session.commit()
|
||||
|
||||
async def get_attackers(
|
||||
self,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
search: Optional[str] = None,
|
||||
sort_by: str = "recent",
|
||||
) -> List[dict[str, Any]]:
|
||||
order = {
|
||||
"active": desc(Attacker.event_count),
|
||||
"traversals": desc(Attacker.is_traversal),
|
||||
}.get(sort_by, desc(Attacker.last_seen))
|
||||
|
||||
statement = select(Attacker).order_by(order).offset(offset).limit(limit)
|
||||
if search:
|
||||
statement = statement.where(Attacker.ip.like(f"%{search}%"))
|
||||
|
||||
async with self.session_factory() as session:
|
||||
result = await session.execute(statement)
|
||||
return [a.model_dump(mode="json") for a in result.scalars().all()]
|
||||
|
||||
async def get_total_attackers(self, search: Optional[str] = None) -> int:
|
||||
statement = select(func.count()).select_from(Attacker)
|
||||
if search:
|
||||
statement = statement.where(Attacker.ip.like(f"%{search}%"))
|
||||
|
||||
async with self.session_factory() as session:
|
||||
result = await session.execute(statement)
|
||||
return result.scalar() or 0
|
||||
|
||||
@@ -130,3 +130,24 @@ async def _extract_bounty(repo: BaseRepository, log_data: dict[str, Any]) -> Non
|
||||
|
||||
# 4. SSH client banner fingerprint (deferred — requires asyncssh server)
|
||||
# Fires on: service=ssh, event_type=client_banner, fields.client_banner
|
||||
|
||||
# 5. JA3/JA3S TLS fingerprint from sniffer container
|
||||
_ja3 = _fields.get("ja3")
|
||||
if _ja3 and log_data.get("service") == "sniffer":
|
||||
await repo.add_bounty({
|
||||
"decky": log_data.get("decky"),
|
||||
"service": "sniffer",
|
||||
"attacker_ip": log_data.get("attacker_ip"),
|
||||
"bounty_type": "fingerprint",
|
||||
"payload": {
|
||||
"fingerprint_type": "ja3",
|
||||
"ja3": _ja3,
|
||||
"ja3s": _fields.get("ja3s"),
|
||||
"tls_version": _fields.get("tls_version"),
|
||||
"sni": _fields.get("sni") or None,
|
||||
"alpn": _fields.get("alpn") or None,
|
||||
"dst_port": _fields.get("dst_port"),
|
||||
"raw_ciphers": _fields.get("raw_ciphers"),
|
||||
"raw_extensions": _fields.get("raw_extensions"),
|
||||
},
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user