refactor(intel): re-key attacker_intel on attacker_uuid (closes DEBT-041)
The threat-intel surface was IP-keyed on day one as an expedient — the
worker is woken by IP-bearing bus events. ANTI's call: don't carry that
debt. NO IPs as primary keys anywhere on the attacker-intel surface.
Schema:
- attacker_uuid is now the canonical key — UNIQUE + FK to attackers.uuid.
- attacker_ip stays as a denormalised, indexed, NON-UNIQUE value column.
Updated on every upsert; useful for SIEM payloads and audit lookups,
but explicitly NOT a key. Model docstring says so.
- Pre-v1, no Alembic migration needed. SQLModel.metadata.create_all()
builds the new shape on fresh DBs.
Repo:
- upsert_attacker_intel now keys on attacker_uuid.
- get_attacker_intel_by_ip → get_attacker_intel_by_uuid.
- get_unenriched_attacker_ips → get_unenriched_attackers, returning
[{uuid, ip}] tuples so the worker writes by UUID and dispatches
provider calls by IP without a second round-trip.
Worker:
- _enrich_one(uuid, ip, ...) — UUID lands on the row, IP rides for
provider egress.
- attacker.intel.enriched bus payload gains attacker_uuid alongside
attacker_ip — webhook → SIEM consumers benefit; no removal.
API:
- GET /api/v1/attackers/{ip}/intel deleted outright (rip-and-replace,
never deployed beyond dev).
- GET /api/v1/attackers/{uuid}/intel is the only public route, matching
every other /attackers/* route.
Frontend:
- <IntelPanel uuid={id!} /> uses the URL param directly, fetches in
parallel with the rest of AttackerDetail rather than waiting on
attacker.ip.
Tests: re-keyed in place, 39 passed (same coverage as before the
refactor). Provider-impl tests untouched.
DEBT-041: closed in DEBT.md (entry preserved as historical rationale,
summary table flipped to ✅, remaining-open list shortened by one).
This commit is contained in:
@@ -60,11 +60,17 @@ def _aggregate(verdicts: list[Optional[str]]) -> Optional[str]:
|
|||||||
|
|
||||||
|
|
||||||
async def _enrich_one(
|
async def _enrich_one(
|
||||||
|
attacker_uuid: str,
|
||||||
ip: str,
|
ip: str,
|
||||||
providers: list[IntelProvider],
|
providers: list[IntelProvider],
|
||||||
ttl_hours: int,
|
ttl_hours: int,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Fan out across providers for a single IP and assemble the row update."""
|
"""Fan out across providers for a single attacker and assemble the row.
|
||||||
|
|
||||||
|
Keyed on ``attacker_uuid`` for the eventual upsert; the IP is the wire
|
||||||
|
value the providers see and is denormalised onto the row for SIEM /
|
||||||
|
audit consumers.
|
||||||
|
"""
|
||||||
results: list[IntelResult] = await asyncio.gather(
|
results: list[IntelResult] = await asyncio.gather(
|
||||||
*(p.lookup(ip) for p in providers),
|
*(p.lookup(ip) for p in providers),
|
||||||
return_exceptions=False, # providers contractually never raise
|
return_exceptions=False, # providers contractually never raise
|
||||||
@@ -72,6 +78,7 @@ async def _enrich_one(
|
|||||||
|
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
row: dict[str, Any] = {
|
row: dict[str, Any] = {
|
||||||
|
"attacker_uuid": attacker_uuid,
|
||||||
"attacker_ip": ip,
|
"attacker_ip": ip,
|
||||||
"cached_at": now,
|
"cached_at": now,
|
||||||
"expires_at": now + timedelta(hours=ttl_hours),
|
"expires_at": now + timedelta(hours=ttl_hours),
|
||||||
@@ -144,7 +151,7 @@ async def run_intel_loop(
|
|||||||
try:
|
try:
|
||||||
while not shutdown.is_set():
|
while not shutdown.is_set():
|
||||||
try:
|
try:
|
||||||
pending = await repo.get_unenriched_attacker_ips(
|
pending = await repo.get_unenriched_attackers(
|
||||||
limit=backfill_batch,
|
limit=backfill_batch,
|
||||||
)
|
)
|
||||||
except Exception: # noqa: BLE001
|
except Exception: # noqa: BLE001
|
||||||
@@ -152,16 +159,21 @@ async def run_intel_loop(
|
|||||||
pending = []
|
pending = []
|
||||||
|
|
||||||
if pending and providers:
|
if pending and providers:
|
||||||
for ip in pending:
|
for entry in pending:
|
||||||
if shutdown.is_set():
|
if shutdown.is_set():
|
||||||
break
|
break
|
||||||
|
attacker_uuid = entry["uuid"]
|
||||||
|
ip = entry["ip"]
|
||||||
try:
|
try:
|
||||||
row = await _enrich_one(ip, providers, ttl_hours)
|
row = await _enrich_one(
|
||||||
|
attacker_uuid, ip, providers, ttl_hours,
|
||||||
|
)
|
||||||
await repo.upsert_attacker_intel(row)
|
await repo.upsert_attacker_intel(row)
|
||||||
await publish_safely(
|
await publish_safely(
|
||||||
bus,
|
bus,
|
||||||
_topics.attacker(_topics.ATTACKER_INTEL_ENRICHED),
|
_topics.attacker(_topics.ATTACKER_INTEL_ENRICHED),
|
||||||
{
|
{
|
||||||
|
"attacker_uuid": attacker_uuid,
|
||||||
"attacker_ip": ip,
|
"attacker_ip": ip,
|
||||||
"aggregate_verdict": row.get("aggregate_verdict"),
|
"aggregate_verdict": row.get("aggregate_verdict"),
|
||||||
"providers": [p.name for p in providers],
|
"providers": [p.name for p in providers],
|
||||||
@@ -170,7 +182,8 @@ async def run_intel_loop(
|
|||||||
)
|
)
|
||||||
except Exception: # noqa: BLE001
|
except Exception: # noqa: BLE001
|
||||||
log.exception(
|
log.exception(
|
||||||
"intel worker: enrichment failed for ip=%s", ip,
|
"intel worker: enrichment failed for uuid=%s ip=%s",
|
||||||
|
attacker_uuid, ip,
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -30,8 +30,18 @@ class AttackerIntel(SQLModel, table=True):
|
|||||||
__tablename__ = "attacker_intel"
|
__tablename__ = "attacker_intel"
|
||||||
|
|
||||||
uuid: str = Field(primary_key=True) # uuid.uuid4().hex, generated by writer
|
uuid: str = Field(primary_key=True) # uuid.uuid4().hex, generated by writer
|
||||||
attacker_uuid: Optional[str] = Field(default=None, index=True)
|
# Canonical key. One intel row per attacker UUID; FK guarantees no orphan
|
||||||
attacker_ip: str = Field(index=True, unique=True)
|
# rows when an attacker is deleted, and UNIQUE keeps upserts honest.
|
||||||
|
attacker_uuid: str = Field(
|
||||||
|
foreign_key="attackers.uuid",
|
||||||
|
unique=True,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
# DENORMALISED — NOT a key. The IP the worker queried providers with at
|
||||||
|
# write time. Useful for SIEM payloads and audit lookups; updated on every
|
||||||
|
# upsert if the attacker rotates IPs. Never use this column as a lookup
|
||||||
|
# key; ``attacker_uuid`` is the only canonical identifier here.
|
||||||
|
attacker_ip: str = Field(index=True)
|
||||||
schema_version: int = Field(default=1)
|
schema_version: int = Field(default=1)
|
||||||
|
|
||||||
# ── GreyNoise Community ─────────────────────────────────────────────
|
# ── GreyNoise Community ─────────────────────────────────────────────
|
||||||
|
|||||||
@@ -280,23 +280,31 @@ class BaseRepository(ABC):
|
|||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def upsert_attacker_intel(self, data: dict[str, Any]) -> str:
|
async def upsert_attacker_intel(self, data: dict[str, Any]) -> str:
|
||||||
"""Insert or update the threat-intel row for an attacker IP.
|
"""Insert or update the threat-intel row for an attacker UUID.
|
||||||
|
|
||||||
``data`` MUST include ``attacker_ip`` and ``expires_at``. Returns
|
``data`` MUST include ``attacker_uuid``, ``attacker_ip`` and
|
||||||
the row UUID. Used by the ``decnet enrich`` worker.
|
``expires_at``. Returns the row UUID. Keyed on ``attacker_uuid``
|
||||||
|
(UNIQUE + FK to ``attackers.uuid``); ``attacker_ip`` is denormalised
|
||||||
|
— it gets overwritten on every upsert if the attacker rotates IPs.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def get_attacker_intel_by_ip(self, ip: str) -> Optional[dict[str, Any]]:
|
async def get_attacker_intel_by_uuid(self, uuid: str) -> Optional[dict[str, Any]]:
|
||||||
"""Return the threat-intel row for ``ip`` or ``None`` if missing."""
|
"""Return the threat-intel row for ``uuid`` or ``None`` if missing."""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def get_unenriched_attacker_ips(self, limit: int = 100) -> list[str]:
|
async def get_unenriched_attackers(
|
||||||
"""List attacker IPs with no intel row OR whose row is past expires_at.
|
self, limit: int = 100,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""List ``{"uuid", "ip"}`` pairs for attackers with no intel row OR
|
||||||
|
whose row is past ``expires_at``.
|
||||||
|
|
||||||
Used by the enrich worker to backfill on startup and on each wake.
|
Used by the enrich worker to backfill on startup and on each wake.
|
||||||
|
Returns both fields so the worker can write keyed on UUID without
|
||||||
|
a second per-attacker DB round-trip to resolve the IP for outbound
|
||||||
|
provider calls.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|||||||
@@ -1197,10 +1197,12 @@ class SQLModelRepository(BaseRepository):
|
|||||||
return row.model_dump(mode="json")
|
return row.model_dump(mode="json")
|
||||||
|
|
||||||
async def upsert_attacker_intel(self, data: dict[str, Any]) -> str:
|
async def upsert_attacker_intel(self, data: dict[str, Any]) -> str:
|
||||||
ip = data["attacker_ip"]
|
attacker_uuid_value = data["attacker_uuid"]
|
||||||
async with self._session() as session:
|
async with self._session() as session:
|
||||||
result = await session.execute(
|
result = await session.execute(
|
||||||
select(AttackerIntel).where(AttackerIntel.attacker_ip == ip)
|
select(AttackerIntel).where(
|
||||||
|
AttackerIntel.attacker_uuid == attacker_uuid_value,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
existing = result.scalar_one_or_none()
|
existing = result.scalar_one_or_none()
|
||||||
if existing:
|
if existing:
|
||||||
@@ -1214,13 +1216,13 @@ class SQLModelRepository(BaseRepository):
|
|||||||
await session.commit()
|
await session.commit()
|
||||||
return row_uuid
|
return row_uuid
|
||||||
|
|
||||||
async def get_attacker_intel_by_ip(
|
async def get_attacker_intel_by_uuid(
|
||||||
self,
|
self,
|
||||||
ip: str,
|
uuid: str,
|
||||||
) -> Optional[dict[str, Any]]:
|
) -> Optional[dict[str, Any]]:
|
||||||
async with self._session() as session:
|
async with self._session() as session:
|
||||||
result = await session.execute(
|
result = await session.execute(
|
||||||
select(AttackerIntel).where(AttackerIntel.attacker_ip == ip)
|
select(AttackerIntel).where(AttackerIntel.attacker_uuid == uuid)
|
||||||
)
|
)
|
||||||
row = result.scalar_one_or_none()
|
row = result.scalar_one_or_none()
|
||||||
if not row:
|
if not row:
|
||||||
@@ -1240,17 +1242,23 @@ class SQLModelRepository(BaseRepository):
|
|||||||
pass
|
pass
|
||||||
return d
|
return d
|
||||||
|
|
||||||
async def get_unenriched_attacker_ips(self, limit: int = 100) -> list[str]:
|
async def get_unenriched_attackers(
|
||||||
"""IPs in ``attackers`` with no intel row OR a stale (expired) one.
|
self, limit: int = 100,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""``{"uuid", "ip"}`` pairs with no intel row OR a stale (expired) one.
|
||||||
|
|
||||||
Stale = ``expires_at < now``. Ordered by ``attackers.last_seen`` desc
|
Stale = ``expires_at < now``. Ordered by ``attackers.last_seen`` desc
|
||||||
so the worker prioritises recent activity on backfill.
|
so the worker prioritises recent activity on backfill. Both columns
|
||||||
|
are projected so the worker can write keyed on UUID and dispatch
|
||||||
|
provider calls keyed on IP without a second round-trip.
|
||||||
"""
|
"""
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
async with self._session() as session:
|
async with self._session() as session:
|
||||||
stmt = (
|
stmt = (
|
||||||
select(Attacker.ip)
|
select(Attacker.uuid, Attacker.ip)
|
||||||
.outerjoin(AttackerIntel, AttackerIntel.attacker_ip == Attacker.ip)
|
.outerjoin(
|
||||||
|
AttackerIntel, AttackerIntel.attacker_uuid == Attacker.uuid,
|
||||||
|
)
|
||||||
.where(
|
.where(
|
||||||
or_(
|
or_(
|
||||||
AttackerIntel.uuid.is_(None),
|
AttackerIntel.uuid.is_(None),
|
||||||
@@ -1261,7 +1269,10 @@ class SQLModelRepository(BaseRepository):
|
|||||||
.limit(limit)
|
.limit(limit)
|
||||||
)
|
)
|
||||||
result = await session.execute(stmt)
|
result = await session.execute(stmt)
|
||||||
return [row for row in result.scalars().all()]
|
return [
|
||||||
|
{"uuid": uuid_, "ip": ip}
|
||||||
|
for uuid_, ip in result.all()
|
||||||
|
]
|
||||||
|
|
||||||
async def increment_smtp_target(self, attacker_uuid: str, domain: str) -> None:
|
async def increment_smtp_target(self, attacker_uuid: str, domain: str) -> None:
|
||||||
"""Upsert an (attacker_uuid, domain) pair and bump count + last_seen.
|
"""Upsert an (attacker_uuid, domain) pair and bump count + last_seen.
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""GET /api/v1/attackers/{ip}/intel — latest threat-intel row for an IP."""
|
"""GET /api/v1/attackers/{uuid}/intel — latest threat-intel row for an attacker."""
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
@@ -10,27 +10,29 @@ router = APIRouter()
|
|||||||
|
|
||||||
|
|
||||||
@router.get(
|
@router.get(
|
||||||
"/attackers/{ip}/intel",
|
"/attackers/{uuid}/intel",
|
||||||
tags=["Attacker Profiles"],
|
tags=["Attacker Profiles"],
|
||||||
responses={
|
responses={
|
||||||
401: {"description": "Could not validate credentials"},
|
401: {"description": "Could not validate credentials"},
|
||||||
403: {"description": "Insufficient permissions"},
|
403: {"description": "Insufficient permissions"},
|
||||||
404: {"description": "No intel cached for this IP"},
|
404: {"description": "No intel cached for this attacker"},
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@_traced("api.get_attacker_intel")
|
@_traced("api.get_attacker_intel")
|
||||||
async def get_attacker_intel(
|
async def get_attacker_intel(
|
||||||
ip: str,
|
uuid: str,
|
||||||
user: dict = Depends(require_viewer),
|
user: dict = Depends(require_viewer),
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Return the most recent cached threat-intel verdict for ``ip``.
|
"""Return the most recent cached threat-intel verdict for an attacker.
|
||||||
|
|
||||||
The row is populated out-of-band by the ``decnet enrich`` worker
|
The row is populated out-of-band by the ``decnet enrich`` worker
|
||||||
(typically within seconds of first observation, sub-second when the
|
(typically within seconds of first observation, sub-second when the
|
||||||
bus is healthy). 404 means either the worker has not run yet or the
|
bus is healthy). 404 means either the worker has not run yet or the
|
||||||
IP has never been observed by DECNET.
|
UUID does not correspond to an attacker DECNET has seen.
|
||||||
"""
|
"""
|
||||||
record = await repo.get_attacker_intel_by_ip(ip)
|
record = await repo.get_attacker_intel_by_uuid(uuid)
|
||||||
if not record:
|
if not record:
|
||||||
raise HTTPException(status_code=404, detail="No intel cached for this IP")
|
raise HTTPException(
|
||||||
|
status_code=404, detail="No intel cached for this attacker",
|
||||||
|
)
|
||||||
return record
|
return record
|
||||||
|
|||||||
@@ -956,6 +956,7 @@ const LeakedIPsRow: React.FC<LeakedIPsRowProps> = ({ leaks, total }) => {
|
|||||||
// fields plus null gaps where a provider hasn't answered yet. We treat
|
// fields plus null gaps where a provider hasn't answered yet. We treat
|
||||||
// every column as optional on the wire.
|
// every column as optional on the wire.
|
||||||
type IntelRow = {
|
type IntelRow = {
|
||||||
|
attacker_uuid: string;
|
||||||
attacker_ip: string;
|
attacker_ip: string;
|
||||||
schema_version?: number;
|
schema_version?: number;
|
||||||
aggregate_verdict?: 'malicious' | 'suspicious' | 'benign' | 'unknown' | null;
|
aggregate_verdict?: 'malicious' | 'suspicious' | 'benign' | 'unknown' | null;
|
||||||
@@ -1013,7 +1014,7 @@ const ProviderRow: React.FC<{
|
|||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|
||||||
const IntelPanel: React.FC<{ ip: string }> = ({ ip }) => {
|
const IntelPanel: React.FC<{ uuid: string }> = ({ uuid }) => {
|
||||||
const [intel, setIntel] = useState<IntelRow | null>(null);
|
const [intel, setIntel] = useState<IntelRow | null>(null);
|
||||||
const [state, setState] = useState<'loading' | 'absent' | 'ok' | 'error'>('loading');
|
const [state, setState] = useState<'loading' | 'absent' | 'ok' | 'error'>('loading');
|
||||||
|
|
||||||
@@ -1022,7 +1023,7 @@ const IntelPanel: React.FC<{ ip: string }> = ({ ip }) => {
|
|||||||
const load = async () => {
|
const load = async () => {
|
||||||
setState('loading');
|
setState('loading');
|
||||||
try {
|
try {
|
||||||
const res = await api.get(`/attackers/${encodeURIComponent(ip)}/intel`);
|
const res = await api.get(`/attackers/${encodeURIComponent(uuid)}/intel`);
|
||||||
if (!cancelled) {
|
if (!cancelled) {
|
||||||
setIntel(res.data);
|
setIntel(res.data);
|
||||||
setState('ok');
|
setState('ok');
|
||||||
@@ -1039,7 +1040,7 @@ const IntelPanel: React.FC<{ ip: string }> = ({ ip }) => {
|
|||||||
};
|
};
|
||||||
load();
|
load();
|
||||||
return () => { cancelled = true; };
|
return () => { cancelled = true; };
|
||||||
}, [ip]);
|
}, [uuid]);
|
||||||
|
|
||||||
if (state === 'loading') {
|
if (state === 'loading') {
|
||||||
return (
|
return (
|
||||||
@@ -1756,13 +1757,13 @@ const AttackerDetail: React.FC = () => {
|
|||||||
);
|
);
|
||||||
})()}
|
})()}
|
||||||
|
|
||||||
{/* Threat-Intel Enrichment — keyed by attacker.ip (see DEBT-041) */}
|
{/* Threat-Intel Enrichment — UUID-keyed, fetches in parallel with the parent. */}
|
||||||
<Section
|
<Section
|
||||||
title={<><Globe size={14} style={{ verticalAlign: 'middle', marginRight: '6px' }} />THREAT INTEL</>}
|
title={<><Globe size={14} style={{ verticalAlign: 'middle', marginRight: '6px' }} />THREAT INTEL</>}
|
||||||
open={openSections.intel}
|
open={openSections.intel}
|
||||||
onToggle={() => toggle('intel')}
|
onToggle={() => toggle('intel')}
|
||||||
>
|
>
|
||||||
<IntelPanel ip={attacker.ip} />
|
<IntelPanel uuid={id!} />
|
||||||
</Section>
|
</Section>
|
||||||
|
|
||||||
{/* Captured Artifacts */}
|
{/* Captured Artifacts */}
|
||||||
|
|||||||
@@ -409,7 +409,9 @@ Shared prep landed in commit 1: `_sync_ntlmssp_sources()` in `decnet/engine/depl
|
|||||||
- Full `TS_INFO_PACKET` (basic-RDP plaintext password) — see scope-down note in commit 2. Re-open as a follow-up DEBT if attacker telemetry actually shows traffic on `PROTOCOL_RDP` without NLA.
|
- Full `TS_INFO_PACKET` (basic-RDP plaintext password) — see scope-down note in commit 2. Re-open as a follow-up DEBT if attacker telemetry actually shows traffic on `PROTOCOL_RDP` without NLA.
|
||||||
- Pubkey / Kerberos auth paths — out of scope; mirrors DEBT-038's deferral on the SSH side.
|
- Pubkey / Kerberos auth paths — out of scope; mirrors DEBT-038's deferral on the SSH side.
|
||||||
|
|
||||||
### DEBT-041 — Intel API + UI keyed by attacker.ip, not attacker.uuid
|
### ~~DEBT-041 — Intel API + UI keyed by attacker.ip, not attacker.uuid~~ ✅ RESOLVED
|
||||||
|
Closed by re-key commit on `dev`. `attacker_intel.attacker_uuid` is now the canonical key (UNIQUE + FK to `attackers.uuid`); `attacker_ip` stays as a denormalised value column (indexed, not unique). `GET /api/v1/attackers/{uuid}/intel` is the only public route — the IP-keyed alias was deleted, not deprecated. Bus event `attacker.intel.enriched` payload gains `attacker_uuid` alongside `attacker_ip` for SIEM consumers. `<IntelPanel uuid={...} />` swaps to UUID. The ticket text below is preserved as the original rationale.
|
||||||
|
|
||||||
**Files:** `decnet/web/router/attackers/api_get_attacker_intel.py`, `decnet/web/db/sqlmodel_repo.py:upsert_attacker_intel`, `decnet/web/db/models/attacker_intel.py`, `decnet_web/src/components/AttackerDetail.tsx` (`<IntelPanel ip={attacker.ip} />`).
|
**Files:** `decnet/web/router/attackers/api_get_attacker_intel.py`, `decnet/web/db/sqlmodel_repo.py:upsert_attacker_intel`, `decnet/web/db/models/attacker_intel.py`, `decnet_web/src/components/AttackerDetail.tsx` (`<IntelPanel ip={attacker.ip} />`).
|
||||||
|
|
||||||
The threat-intel enrichment surface (DEBT-N/A: `feat(intel)` series) keys every public surface — `GET /api/v1/attackers/{ip}/intel`, the row's `attacker_ip` UNIQUE, and the React `<IntelPanel ip=...>` — on the attacker's IP rather than the canonical `attacker.uuid` we use for every other attacker-detail route. The decision was deliberate in v1: the enricher is woken by `attacker.observed` / `attacker.scored` events whose payload is naturally IP-keyed, the row models a *one-row-per-IP* TTL cache, and standing up a parallel UUID lookup endpoint would have added a join hop with no consumer.
|
The threat-intel enrichment surface (DEBT-N/A: `feat(intel)` series) keys every public surface — `GET /api/v1/attackers/{ip}/intel`, the row's `attacker_ip` UNIQUE, and the React `<IntelPanel ip=...>` — on the attacker's IP rather than the canonical `attacker.uuid` we use for every other attacker-detail route. The decision was deliberate in v1: the enricher is woken by `attacker.observed` / `attacker.scored` events whose payload is naturally IP-keyed, the row models a *one-row-per-IP* TTL cache, and standing up a parallel UUID lookup endpoint would have added a join hop with no consumer.
|
||||||
@@ -516,7 +518,7 @@ The prober already computes JARM (`worker.py:286`), HASSH (`worker.py:334`), and
|
|||||||
| DEBT-038 | 🟡 Medium | Honeypot / SSH cred capture | open (document-only) |
|
| DEBT-038 | 🟡 Medium | Honeypot / SSH cred capture | open (document-only) |
|
||||||
| ~~DEBT-039~~ | ✅ | Honeypot / Cred emitters | resolved |
|
| ~~DEBT-039~~ | ✅ | Honeypot / Cred emitters | resolved |
|
||||||
| ~~DEBT-040~~ | ✅ | Honeypot / RDP+SMB cred framers | resolved |
|
| ~~DEBT-040~~ | ✅ | Honeypot / RDP+SMB cred framers | resolved |
|
||||||
| DEBT-041 | 🟡 Medium | API / UI / Threat-intel keying | open |
|
| ~~DEBT-041~~ | ✅ | API / UI / Threat-intel keying | resolved |
|
||||||
|
|
||||||
**Remaining open:** DEBT-011 (Alembic), DEBT-023 (image pinning), DEBT-026 (modular mailboxes), DEBT-027 (Dynamic bait store), DEBT-028 (deploy endpoint tests), DEBT-032 (fingerprint rotation detection), DEBT-033 (transcript shard rotation), DEBT-035 (artifacts uid/gid alignment), DEBT-036 (session-profile ingester), DEBT-037 (webhook delivery hardening), DEBT-038 (SSH PAM cred-capture limitations — document-only), DEBT-041 (intel API/UI keyed by IP, not UUID).
|
**Remaining open:** DEBT-011 (Alembic), DEBT-023 (image pinning), DEBT-026 (modular mailboxes), DEBT-027 (Dynamic bait store), DEBT-028 (deploy endpoint tests), DEBT-032 (fingerprint rotation detection), DEBT-033 (transcript shard rotation), DEBT-035 (artifacts uid/gid alignment), DEBT-036 (session-profile ingester), DEBT-037 (webhook delivery hardening), DEBT-038 (SSH PAM cred-capture limitations — document-only).
|
||||||
**Estimated remaining effort:** ~21 hours. DEBT-030 Phase B (optimistic staged-buffer editor) is a follow-up, not debt.
|
**Estimated remaining effort:** ~21 hours. DEBT-030 Phase B (optimistic staged-buffer editor) is a follow-up, not debt.
|
||||||
|
|||||||
@@ -2,11 +2,12 @@
|
|||||||
Round-trip tests for the ``attacker_intel`` table and its repo helpers.
|
Round-trip tests for the ``attacker_intel`` table and its repo helpers.
|
||||||
|
|
||||||
Covers:
|
Covers:
|
||||||
* empty-write upsert path
|
* empty-write upsert path (attacker_uuid as canonical key)
|
||||||
* per-provider partial update
|
* per-provider partial update preserves untouched columns
|
||||||
* JSON-blob deserialization on read
|
* JSON-blob deserialization on read
|
||||||
* TTL bookkeeping (cached_at + expires_at) round-trips intact
|
* TTL bookkeeping (cached_at + expires_at) round-trips intact
|
||||||
* ``get_unenriched_attacker_ips`` selects fresh + stale, skips cached
|
* ``get_unenriched_attackers`` returns ``{"uuid", "ip"}`` pairs and
|
||||||
|
selects fresh + stale rows while skipping cached ones
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -24,9 +25,20 @@ async def repo(tmp_path):
|
|||||||
return r
|
return r
|
||||||
|
|
||||||
|
|
||||||
def _intel_payload(ip: str, *, ttl_hours: int = 24, **overrides) -> dict:
|
async def _seed_attacker(repo, ip: str) -> str:
|
||||||
|
"""Seed an attackers row and return its UUID (the FK target)."""
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
return await repo.upsert_attacker(
|
||||||
|
{"ip": ip, "first_seen": now, "last_seen": now, "event_count": 1}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _intel_payload(
|
||||||
|
*, attacker_uuid: str, ip: str, ttl_hours: int = 24, **overrides
|
||||||
|
) -> dict:
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
base = {
|
base = {
|
||||||
|
"attacker_uuid": attacker_uuid,
|
||||||
"attacker_ip": ip,
|
"attacker_ip": ip,
|
||||||
"cached_at": now,
|
"cached_at": now,
|
||||||
"expires_at": now + timedelta(hours=ttl_hours),
|
"expires_at": now + timedelta(hours=ttl_hours),
|
||||||
@@ -37,11 +49,15 @@ def _intel_payload(ip: str, *, ttl_hours: int = 24, **overrides) -> dict:
|
|||||||
|
|
||||||
@pytest.mark.anyio
|
@pytest.mark.anyio
|
||||||
async def test_empty_upsert_writes_minimal_row(repo):
|
async def test_empty_upsert_writes_minimal_row(repo):
|
||||||
row_uuid = await repo.upsert_attacker_intel(_intel_payload("1.2.3.4"))
|
a_uuid = await _seed_attacker(repo, "1.2.3.4")
|
||||||
|
row_uuid = await repo.upsert_attacker_intel(
|
||||||
|
_intel_payload(attacker_uuid=a_uuid, ip="1.2.3.4")
|
||||||
|
)
|
||||||
assert row_uuid
|
assert row_uuid
|
||||||
|
|
||||||
row = await repo.get_attacker_intel_by_ip("1.2.3.4")
|
row = await repo.get_attacker_intel_by_uuid(a_uuid)
|
||||||
assert row is not None
|
assert row is not None
|
||||||
|
assert row["attacker_uuid"] == a_uuid
|
||||||
assert row["attacker_ip"] == "1.2.3.4"
|
assert row["attacker_ip"] == "1.2.3.4"
|
||||||
assert row["uuid"] == row_uuid
|
assert row["uuid"] == row_uuid
|
||||||
assert row["schema_version"] == 1
|
assert row["schema_version"] == 1
|
||||||
@@ -55,10 +71,11 @@ async def test_empty_upsert_writes_minimal_row(repo):
|
|||||||
|
|
||||||
@pytest.mark.anyio
|
@pytest.mark.anyio
|
||||||
async def test_partial_provider_update_preserves_others(repo):
|
async def test_partial_provider_update_preserves_others(repo):
|
||||||
|
a_uuid = await _seed_attacker(repo, "9.9.9.9")
|
||||||
# First pass: GreyNoise responds, others lag.
|
# First pass: GreyNoise responds, others lag.
|
||||||
first_uuid = await repo.upsert_attacker_intel(
|
first_uuid = await repo.upsert_attacker_intel(
|
||||||
_intel_payload(
|
_intel_payload(
|
||||||
"9.9.9.9",
|
attacker_uuid=a_uuid, ip="9.9.9.9",
|
||||||
greynoise_classification="malicious",
|
greynoise_classification="malicious",
|
||||||
greynoise_raw='{"classification":"malicious"}',
|
greynoise_raw='{"classification":"malicious"}',
|
||||||
greynoise_queried_at=datetime.now(timezone.utc),
|
greynoise_queried_at=datetime.now(timezone.utc),
|
||||||
@@ -68,15 +85,15 @@ async def test_partial_provider_update_preserves_others(repo):
|
|||||||
# columns — the worker passes only the new fields.
|
# columns — the worker passes only the new fields.
|
||||||
second_uuid = await repo.upsert_attacker_intel(
|
second_uuid = await repo.upsert_attacker_intel(
|
||||||
_intel_payload(
|
_intel_payload(
|
||||||
"9.9.9.9",
|
attacker_uuid=a_uuid, ip="9.9.9.9",
|
||||||
abuseipdb_score=85,
|
abuseipdb_score=85,
|
||||||
abuseipdb_raw='{"abuseConfidenceScore":85}',
|
abuseipdb_raw='{"abuseConfidenceScore":85}',
|
||||||
abuseipdb_queried_at=datetime.now(timezone.utc),
|
abuseipdb_queried_at=datetime.now(timezone.utc),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
assert first_uuid == second_uuid # same row
|
assert first_uuid == second_uuid # same row keyed on attacker_uuid
|
||||||
|
|
||||||
row = await repo.get_attacker_intel_by_ip("9.9.9.9")
|
row = await repo.get_attacker_intel_by_uuid(a_uuid)
|
||||||
assert row["greynoise_classification"] == "malicious"
|
assert row["greynoise_classification"] == "malicious"
|
||||||
assert row["greynoise_raw"] == {"classification": "malicious"}
|
assert row["greynoise_raw"] == {"classification": "malicious"}
|
||||||
assert row["abuseipdb_score"] == 85
|
assert row["abuseipdb_score"] == 85
|
||||||
@@ -85,29 +102,28 @@ async def test_partial_provider_update_preserves_others(repo):
|
|||||||
|
|
||||||
@pytest.mark.anyio
|
@pytest.mark.anyio
|
||||||
async def test_get_missing_returns_none(repo):
|
async def test_get_missing_returns_none(repo):
|
||||||
assert await repo.get_attacker_intel_by_ip("0.0.0.0") is None
|
assert await repo.get_attacker_intel_by_uuid("nonexistent-uuid") is None
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.anyio
|
@pytest.mark.anyio
|
||||||
async def test_unenriched_selects_fresh_and_stale_ips(repo):
|
async def test_unenriched_returns_uuid_ip_pairs(repo):
|
||||||
# Seed three attackers via upsert_attacker.
|
fresh_uuid = await _seed_attacker(repo, "10.0.0.1")
|
||||||
now = datetime.now(timezone.utc)
|
stale_uuid = await _seed_attacker(repo, "10.0.0.2")
|
||||||
for ip in ("10.0.0.1", "10.0.0.2", "10.0.0.3"):
|
new_uuid = await _seed_attacker(repo, "10.0.0.3")
|
||||||
await repo.upsert_attacker(
|
|
||||||
{
|
|
||||||
"ip": ip,
|
|
||||||
"first_seen": now,
|
|
||||||
"last_seen": now,
|
|
||||||
"event_count": 1,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
# 10.0.0.1 has fresh intel (not due for refresh).
|
# 10.0.0.1 has fresh intel (not due for refresh).
|
||||||
await repo.upsert_attacker_intel(_intel_payload("10.0.0.1", ttl_hours=24))
|
await repo.upsert_attacker_intel(
|
||||||
|
_intel_payload(attacker_uuid=fresh_uuid, ip="10.0.0.1", ttl_hours=24)
|
||||||
|
)
|
||||||
# 10.0.0.2 has stale intel (already expired).
|
# 10.0.0.2 has stale intel (already expired).
|
||||||
await repo.upsert_attacker_intel(_intel_payload("10.0.0.2", ttl_hours=-1))
|
await repo.upsert_attacker_intel(
|
||||||
|
_intel_payload(attacker_uuid=stale_uuid, ip="10.0.0.2", ttl_hours=-1)
|
||||||
|
)
|
||||||
# 10.0.0.3 has no intel row at all.
|
# 10.0.0.3 has no intel row at all.
|
||||||
|
|
||||||
pending = await repo.get_unenriched_attacker_ips(limit=10)
|
pending = await repo.get_unenriched_attackers(limit=10)
|
||||||
assert "10.0.0.1" not in pending # fresh, skipped
|
by_uuid = {entry["uuid"]: entry["ip"] for entry in pending}
|
||||||
assert "10.0.0.2" in pending # stale, queue it
|
|
||||||
assert "10.0.0.3" in pending # never enriched
|
assert fresh_uuid not in by_uuid # fresh, skipped
|
||||||
|
assert by_uuid.get(stale_uuid) == "10.0.0.2" # stale, queue it
|
||||||
|
assert by_uuid.get(new_uuid) == "10.0.0.3" # never enriched
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ Covers — without any real provider impls — that the loop:
|
|||||||
* fans out across fake providers and writes the aggregate row
|
* fans out across fake providers and writes the aggregate row
|
||||||
* aggregate_verdict picks the strongest provider verdict
|
* aggregate_verdict picks the strongest provider verdict
|
||||||
* a provider returning ``error`` is logged but does not poison the row
|
* a provider returning ``error`` is logged but does not poison the row
|
||||||
* gates IPs through ``get_unenriched_attacker_ips`` (TTL respected)
|
* gates attackers through ``get_unenriched_attackers`` (TTL respected)
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -92,12 +92,17 @@ async def test_loop_exits_on_shutdown_signal(repo):
|
|||||||
await asyncio.wait_for(task, timeout=2.0)
|
await asyncio.wait_for(task, timeout=2.0)
|
||||||
|
|
||||||
|
|
||||||
|
async def _seed_attacker(repo, ip: str) -> str:
|
||||||
|
"""Seed an attackers row and return its UUID."""
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
return await repo.upsert_attacker(
|
||||||
|
{"ip": ip, "first_seen": now, "last_seen": now, "event_count": 1}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.anyio
|
@pytest.mark.anyio
|
||||||
async def test_no_providers_skips_enrichment(repo):
|
async def test_no_providers_skips_enrichment(repo):
|
||||||
now = datetime.now(timezone.utc)
|
a_uuid = await _seed_attacker(repo, "1.1.1.1")
|
||||||
await repo.upsert_attacker(
|
|
||||||
{"ip": "1.1.1.1", "first_seen": now, "last_seen": now, "event_count": 1}
|
|
||||||
)
|
|
||||||
shutdown = asyncio.Event()
|
shutdown = asyncio.Event()
|
||||||
task = asyncio.create_task(
|
task = asyncio.create_task(
|
||||||
run_intel_loop(
|
run_intel_loop(
|
||||||
@@ -110,16 +115,13 @@ async def test_no_providers_skips_enrichment(repo):
|
|||||||
await asyncio.sleep(0.15)
|
await asyncio.sleep(0.15)
|
||||||
shutdown.set()
|
shutdown.set()
|
||||||
await asyncio.wait_for(task, timeout=2.0)
|
await asyncio.wait_for(task, timeout=2.0)
|
||||||
# No row written for 1.1.1.1.
|
# No row written for the seeded attacker.
|
||||||
assert await repo.get_attacker_intel_by_ip("1.1.1.1") is None
|
assert await repo.get_attacker_intel_by_uuid(a_uuid) is None
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.anyio
|
@pytest.mark.anyio
|
||||||
async def test_fan_out_writes_aggregate_row(repo):
|
async def test_fan_out_writes_aggregate_row(repo):
|
||||||
now = datetime.now(timezone.utc)
|
a_uuid = await _seed_attacker(repo, "2.2.2.2")
|
||||||
await repo.upsert_attacker(
|
|
||||||
{"ip": "2.2.2.2", "first_seen": now, "last_seen": now, "event_count": 1}
|
|
||||||
)
|
|
||||||
|
|
||||||
gn = _FakeProvider(
|
gn = _FakeProvider(
|
||||||
"greynoise",
|
"greynoise",
|
||||||
@@ -154,23 +156,22 @@ async def test_fan_out_writes_aggregate_row(repo):
|
|||||||
shutdown.set()
|
shutdown.set()
|
||||||
await asyncio.wait_for(task, timeout=2.0)
|
await asyncio.wait_for(task, timeout=2.0)
|
||||||
|
|
||||||
row = await repo.get_attacker_intel_by_ip("2.2.2.2")
|
row = await repo.get_attacker_intel_by_uuid(a_uuid)
|
||||||
assert row is not None
|
assert row is not None
|
||||||
|
assert row["attacker_uuid"] == a_uuid
|
||||||
|
assert row["attacker_ip"] == "2.2.2.2"
|
||||||
assert row["greynoise_classification"] == "benign"
|
assert row["greynoise_classification"] == "benign"
|
||||||
assert row["abuseipdb_score"] == 90
|
assert row["abuseipdb_score"] == 90
|
||||||
# Strongest verdict wins.
|
# Strongest verdict wins.
|
||||||
assert row["aggregate_verdict"] == "malicious"
|
assert row["aggregate_verdict"] == "malicious"
|
||||||
# Both providers were queried.
|
# Both providers were queried by IP.
|
||||||
assert gn.calls == ["2.2.2.2"]
|
assert gn.calls == ["2.2.2.2"]
|
||||||
assert aip.calls == ["2.2.2.2"]
|
assert aip.calls == ["2.2.2.2"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.anyio
|
@pytest.mark.anyio
|
||||||
async def test_provider_error_does_not_poison_row(repo):
|
async def test_provider_error_does_not_poison_row(repo):
|
||||||
now = datetime.now(timezone.utc)
|
a_uuid = await _seed_attacker(repo, "3.3.3.3")
|
||||||
await repo.upsert_attacker(
|
|
||||||
{"ip": "3.3.3.3", "first_seen": now, "last_seen": now, "event_count": 1}
|
|
||||||
)
|
|
||||||
|
|
||||||
good = _FakeProvider(
|
good = _FakeProvider(
|
||||||
"greynoise",
|
"greynoise",
|
||||||
@@ -196,7 +197,7 @@ async def test_provider_error_does_not_poison_row(repo):
|
|||||||
shutdown.set()
|
shutdown.set()
|
||||||
await asyncio.wait_for(task, timeout=2.0)
|
await asyncio.wait_for(task, timeout=2.0)
|
||||||
|
|
||||||
row = await repo.get_attacker_intel_by_ip("3.3.3.3")
|
row = await repo.get_attacker_intel_by_uuid(a_uuid)
|
||||||
assert row is not None
|
assert row is not None
|
||||||
assert row["greynoise_classification"] == "benign"
|
assert row["greynoise_classification"] == "benign"
|
||||||
# Broken provider's columns stay null; row is still written.
|
# Broken provider's columns stay null; row is still written.
|
||||||
@@ -226,10 +227,7 @@ async def test_intel_enriched_event_published_to_bus(repo, monkeypatch):
|
|||||||
sub = shared_bus.subscribe(attacker(ATTACKER_INTEL_ENRICHED))
|
sub = shared_bus.subscribe(attacker(ATTACKER_INTEL_ENRICHED))
|
||||||
await sub.__aenter__()
|
await sub.__aenter__()
|
||||||
|
|
||||||
now = datetime.now(timezone.utc)
|
a_uuid = await _seed_attacker(repo, "4.4.4.4")
|
||||||
await repo.upsert_attacker(
|
|
||||||
{"ip": "4.4.4.4", "first_seen": now, "last_seen": now, "event_count": 1}
|
|
||||||
)
|
|
||||||
|
|
||||||
provider = _FakeProvider(
|
provider = _FakeProvider(
|
||||||
"greynoise",
|
"greynoise",
|
||||||
@@ -258,6 +256,7 @@ async def test_intel_enriched_event_published_to_bus(repo, monkeypatch):
|
|||||||
await sub.__aexit__(None, None, None)
|
await sub.__aexit__(None, None, None)
|
||||||
|
|
||||||
payload = event.payload
|
payload = event.payload
|
||||||
|
assert payload["attacker_uuid"] == a_uuid
|
||||||
assert payload["attacker_ip"] == "4.4.4.4"
|
assert payload["attacker_ip"] == "4.4.4.4"
|
||||||
assert payload["aggregate_verdict"] == "malicious"
|
assert payload["aggregate_verdict"] == "malicious"
|
||||||
assert payload["providers"] == ["greynoise"]
|
assert payload["providers"] == ["greynoise"]
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""Tests for GET /api/v1/attackers/{ip}/intel."""
|
"""Tests for GET /api/v1/attackers/{uuid}/intel."""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from unittest.mock import AsyncMock, patch
|
from unittest.mock import AsyncMock, patch
|
||||||
@@ -14,6 +14,7 @@ async def test_returns_cached_intel_row():
|
|||||||
)
|
)
|
||||||
|
|
||||||
fake_row = {
|
fake_row = {
|
||||||
|
"attacker_uuid": "att-uuid-xyz",
|
||||||
"attacker_ip": "1.2.3.4",
|
"attacker_ip": "1.2.3.4",
|
||||||
"aggregate_verdict": "malicious",
|
"aggregate_verdict": "malicious",
|
||||||
"greynoise_classification": "malicious",
|
"greynoise_classification": "malicious",
|
||||||
@@ -24,11 +25,12 @@ async def test_returns_cached_intel_row():
|
|||||||
with patch(
|
with patch(
|
||||||
"decnet.web.router.attackers.api_get_attacker_intel.repo"
|
"decnet.web.router.attackers.api_get_attacker_intel.repo"
|
||||||
) as mock_repo:
|
) as mock_repo:
|
||||||
mock_repo.get_attacker_intel_by_ip = AsyncMock(return_value=fake_row)
|
mock_repo.get_attacker_intel_by_uuid = AsyncMock(return_value=fake_row)
|
||||||
result = await get_attacker_intel(
|
result = await get_attacker_intel(
|
||||||
ip="1.2.3.4",
|
uuid="att-uuid-xyz",
|
||||||
user={"uuid": "viewer", "role": "viewer"},
|
user={"uuid": "viewer", "role": "viewer"},
|
||||||
)
|
)
|
||||||
|
assert result["attacker_uuid"] == "att-uuid-xyz"
|
||||||
assert result["aggregate_verdict"] == "malicious"
|
assert result["aggregate_verdict"] == "malicious"
|
||||||
assert result["abuseipdb_score"] == 92
|
assert result["abuseipdb_score"] == 92
|
||||||
|
|
||||||
@@ -42,10 +44,10 @@ async def test_404_when_no_row_cached():
|
|||||||
with patch(
|
with patch(
|
||||||
"decnet.web.router.attackers.api_get_attacker_intel.repo"
|
"decnet.web.router.attackers.api_get_attacker_intel.repo"
|
||||||
) as mock_repo:
|
) as mock_repo:
|
||||||
mock_repo.get_attacker_intel_by_ip = AsyncMock(return_value=None)
|
mock_repo.get_attacker_intel_by_uuid = AsyncMock(return_value=None)
|
||||||
with pytest.raises(HTTPException) as excinfo:
|
with pytest.raises(HTTPException) as excinfo:
|
||||||
await get_attacker_intel(
|
await get_attacker_intel(
|
||||||
ip="0.0.0.0",
|
uuid="missing-uuid",
|
||||||
user={"uuid": "viewer", "role": "viewer"},
|
user={"uuid": "viewer", "role": "viewer"},
|
||||||
)
|
)
|
||||||
assert excinfo.value.status_code == 404
|
assert excinfo.value.status_code == 404
|
||||||
|
|||||||
Reference in New Issue
Block a user