anti/DECNET
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(intel): migrate AttackerIntel JSON-string columns to native SQLAlchemy JSON

Browse Source 
Five list columns (greynoise_tags, abuseipdb_categories, threatfox_threat_types,
threatfox_ioc_types, threatfox_malware_families) and four dict columns
(*_raw) are now Column(JSON) with list/dict type annotations and
default_factory=list/dict. Providers return native Python objects; the
application-layer json.dumps/json.loads round-trip and _decode_json_list
helpers are gone. to_intel_event_payload() reads columns directly.

Also caps pytest xdist at -n 4 and excludes tests/api from norecursedirs
to prevent schemathesis workers from OOM-killing the dev loop.
This commit is contained in:
anti
2026-05-10 09:17:15 -04:00
parent de3634d739
commit 9a7b03700c
16 changed files with 90 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
decnet/intel/abuseipdb.py
View File

@@ -17,7 +17,6 @@ later if operators report drift.
"""
from __future__ import annotations
import json
import os
from datetime import datetime, timezone
from typing import Optional
@@ -110,8 +109,8 @@ class AbuseIPDBProvider(IntelProvider):
verdict=verdict,
column_updates={
"abuseipdb_score": score,
"abuseipdb_categories": json.dumps(sorted(categories)),
"abuseipdb_raw": json.dumps(data),
"abuseipdb_categories": sorted(categories),
"abuseipdb_raw": data,
"abuseipdb_queried_at": datetime.now(timezone.utc),
},
)

5
decnet/intel/feodo.py
View File

@@ -13,7 +13,6 @@ of attacker IPs map to a single network round-trip per refresh window.
"""
from __future__ import annotations
import json
import time
from datetime import datetime, timezone
from typing import Any, Optional
@@ -94,7 +93,7 @@ class FeodoProvider(IntelProvider):
column_updates={
"feodo_listed": False,
"feodo_malware_family": None,
"feodo_raw": "{}",
"feodo_raw": {},
"feodo_queried_at": datetime.now(timezone.utc),
},
)
@@ -108,7 +107,7 @@ class FeodoProvider(IntelProvider):
column_updates={
"feodo_listed": True,
"feodo_malware_family": family,
"feodo_raw": json.dumps(entry),
"feodo_raw": entry,
"feodo_queried_at": datetime.now(timezone.utc),
},
)

9
decnet/intel/greynoise.py
View File

@@ -25,7 +25,6 @@ Status code semantics:
"""
from __future__ import annotations
import json
import os
from datetime import datetime, timezone
from typing import Optional
@@ -72,8 +71,8 @@ class GreyNoiseProvider(IntelProvider):
column_updates={
"greynoise_classification": "unknown",
"greynoise_name": None,
"greynoise_tags": "[]",
"greynoise_raw": json.dumps({"message": "not seen"}),
"greynoise_tags": [],
"greynoise_raw": {"message": "not seen"},
"greynoise_queried_at": datetime.now(timezone.utc),
},
)
@@ -107,8 +106,8 @@ class GreyNoiseProvider(IntelProvider):
column_updates={
"greynoise_classification": classification,
"greynoise_name": name,
"greynoise_tags": json.dumps(tags),
"greynoise_raw": json.dumps(data),
"greynoise_tags": tags,
"greynoise_raw": data,
"greynoise_queried_at": datetime.now(timezone.utc),
},
)

17
decnet/intel/threatfox.py
View File

@@ -12,7 +12,6 @@ caps requests/min — the provider works either way.
"""
from __future__ import annotations
import json
import os
from datetime import datetime, timezone
from typing import Optional
@@ -71,10 +70,10 @@ class ThreatFoxProvider(IntelProvider):
verdict=None, # absence is not a benign signal
column_updates={
"threatfox_listed": False,
"threatfox_threat_types": "[]",
"threatfox_ioc_types": "[]",
"threatfox_malware_families": "[]",
"threatfox_raw": "{}",
"threatfox_threat_types": [],
"threatfox_ioc_types": [],
"threatfox_malware_families": [],
"threatfox_raw": {},
"threatfox_queried_at": datetime.now(timezone.utc),
},
)
@@ -113,10 +112,10 @@ class ThreatFoxProvider(IntelProvider):
verdict="malicious" if listed else None,
column_updates={
"threatfox_listed": listed,
"threatfox_threat_types": json.dumps(sorted(threat_types)),
"threatfox_ioc_types": json.dumps(sorted(ioc_types)),
"threatfox_malware_families": json.dumps(sorted(families)),
"threatfox_raw": json.dumps(data),
"threatfox_threat_types": sorted(threat_types),
"threatfox_ioc_types": sorted(ioc_types),
"threatfox_malware_families": sorted(families),
"threatfox_raw": data,
"threatfox_queried_at": datetime.now(timezone.utc),
},
)

36
decnet/intel/worker.py
View File

@@ -20,7 +20,6 @@ from __future__ import annotations
import asyncio
import contextlib
import json
from datetime import datetime, timedelta, timezone
from typing import Any, Optional
@@ -60,18 +59,6 @@ def _aggregate(verdicts: list[Optional[str]]) -> Optional[str]:
return None
def _decode_json_list(value: Any) -> list[Any]:
if isinstance(value, list):
return value
if isinstance(value, str) and value:
try:
decoded = json.loads(value)
except (json.JSONDecodeError, TypeError):
return []
return decoded if isinstance(decoded, list) else []
return []
def _build_intel_event_payload(
attacker_uuid: str,
ip: str,
@@ -80,11 +67,6 @@ def _build_intel_event_payload(
) -> dict[str, Any]:
"""Project the AttackerIntel row into the bus event the TTP worker
consumes as ``source_kind="intel"``.
The TTP worker forwards the payload verbatim to the IntelLifter.
Per-provider taxonomy fields (categories, tags, threat_types) are
decoded back to native lists here so the lifter does not have to
care that the storage layer JSON-encodes them.
"""
return {
"attacker_uuid": attacker_uuid,
@@ -93,27 +75,19 @@ def _build_intel_event_payload(
"providers": [p.name for p in providers],
# AbuseIPDB
"abuseipdb_score": row.get("abuseipdb_score"),
"abuseipdb_categories": _decode_json_list(
row.get("abuseipdb_categories"),
),
"abuseipdb_categories": row.get("abuseipdb_categories") or [],
# GreyNoise
"greynoise_classification": row.get("greynoise_classification"),
"greynoise_name": row.get("greynoise_name"),
"greynoise_tags": _decode_json_list(row.get("greynoise_tags")),
"greynoise_tags": row.get("greynoise_tags") or [],
# Feodo
"feodo_listed": row.get("feodo_listed"),
"feodo_malware_family": row.get("feodo_malware_family"),
# ThreatFox
"threatfox_listed": row.get("threatfox_listed"),
"threatfox_threat_types": _decode_json_list(
row.get("threatfox_threat_types"),
),
"threatfox_ioc_types": _decode_json_list(
row.get("threatfox_ioc_types"),
),
"threatfox_malware_families": _decode_json_list(
row.get("threatfox_malware_families"),
),
"threatfox_threat_types": row.get("threatfox_threat_types") or [],
"threatfox_ioc_types": row.get("threatfox_ioc_types") or [],
"threatfox_malware_families": row.get("threatfox_malware_families") or [],
}

101
decnet/web/db/models/attacker_intel.py
View File

@@ -1,25 +1,10 @@
"""Threat-intel enrichment row — one per attacker IP, TTL-cached."""
import json as _json
from datetime import datetime, timezone
from typing import Any, Optional
from sqlalchemy import Column
from sqlalchemy import JSON, Column
from sqlmodel import Field, SQLModel
from ._base import _BIG_TEXT
def _decode_json_list(value: Any) -> list[Any]:
if isinstance(value, list):
return value
if isinstance(value, str) and value:
try:
decoded = _json.loads(value)
except (_json.JSONDecodeError, TypeError):
return []
return decoded if isinstance(decoded, list) else []
return []
class AttackerIntel(SQLModel, table=True):
"""Aggregated threat-intel verdict for a single attacker IP.
@@ -63,37 +48,35 @@ class AttackerIntel(SQLModel, table=True):
# empty unless an operator wires a non-Community provider that does.
greynoise_classification: Optional[str] = Field(default=None, max_length=32)
greynoise_name: Optional[str] = Field(default=None, max_length=128)
greynoise_tags: str = Field(
default="[]",
sa_column=Column("greynoise_tags", _BIG_TEXT, nullable=False, default="[]"),
) # JSON list[str] — behavioral / actor tags
greynoise_raw: str = Field(
default="{}",
sa_column=Column("greynoise_raw", _BIG_TEXT, nullable=False, default="{}"),
greynoise_tags: list[str] = Field(
default_factory=list,
sa_column=Column("greynoise_tags", JSON, nullable=False),
)
greynoise_raw: dict[str, Any] = Field(
default_factory=dict,
sa_column=Column("greynoise_raw", JSON, nullable=False),
)
greynoise_queried_at: Optional[datetime] = Field(default=None)
# ── AbuseIPDB ────────────────────────────────────────────────────────
# 0..100 abuse confidence score
abuseipdb_score: Optional[int] = Field(default=None)
abuseipdb_categories: str = Field(
default="[]",
sa_column=Column(
"abuseipdb_categories", _BIG_TEXT, nullable=False, default="[]",
),
) # JSON list[int] — flattened set of categories across recent reports
abuseipdb_raw: str = Field(
default="{}",
sa_column=Column("abuseipdb_raw", _BIG_TEXT, nullable=False, default="{}"),
abuseipdb_categories: list[int] = Field(
default_factory=list,
sa_column=Column("abuseipdb_categories", JSON, nullable=False),
)
abuseipdb_raw: dict[str, Any] = Field(
default_factory=dict,
sa_column=Column("abuseipdb_raw", JSON, nullable=False),
)
abuseipdb_queried_at: Optional[datetime] = Field(default=None)
# ── abuse.ch Feodo Tracker ───────────────────────────────────────────
feodo_listed: Optional[bool] = Field(default=None)
feodo_malware_family: Optional[str] = Field(default=None, max_length=64)
feodo_raw: str = Field(
default="{}",
sa_column=Column("feodo_raw", _BIG_TEXT, nullable=False, default="{}"),
feodo_raw: dict[str, Any] = Field(
default_factory=dict,
sa_column=Column("feodo_raw", JSON, nullable=False),
)
feodo_queried_at: Optional[datetime] = Field(default=None)
@@ -105,27 +88,21 @@ class AttackerIntel(SQLModel, table=True):
# IntelLifter keys ATT&CK techniques on ``threat_type``, the canonical
# taxonomy field per ThreatFox's API.
threatfox_listed: Optional[bool] = Field(default=None)
threatfox_threat_types: str = Field(
default="[]",
sa_column=Column(
"threatfox_threat_types", _BIG_TEXT, nullable=False, default="[]",
),
) # JSON list[str]
threatfox_ioc_types: str = Field(
default="[]",
sa_column=Column(
"threatfox_ioc_types", _BIG_TEXT, nullable=False, default="[]",
),
) # JSON list[str]
threatfox_malware_families: str = Field(
default="[]",
sa_column=Column(
"threatfox_malware_families", _BIG_TEXT, nullable=False, default="[]",
),
) # JSON list[str]
threatfox_raw: str = Field(
default="{}",
sa_column=Column("threatfox_raw", _BIG_TEXT, nullable=False, default="{}"),
threatfox_threat_types: list[str] = Field(
default_factory=list,
sa_column=Column("threatfox_threat_types", JSON, nullable=False),
)
threatfox_ioc_types: list[str] = Field(
default_factory=list,
sa_column=Column("threatfox_ioc_types", JSON, nullable=False),
)
threatfox_malware_families: list[str] = Field(
default_factory=list,
sa_column=Column("threatfox_malware_families", JSON, nullable=False),
)
threatfox_raw: dict[str, Any] = Field(
default_factory=dict,
sa_column=Column("threatfox_raw", JSON, nullable=False),
)
threatfox_queried_at: Optional[datetime] = Field(default=None)
@@ -166,21 +143,19 @@ class AttackerIntel(SQLModel, table=True):
"aggregate_verdict": self.aggregate_verdict,
# AbuseIPDB
"abuseipdb_score": self.abuseipdb_score,
"abuseipdb_categories": _decode_json_list(self.abuseipdb_categories),
"abuseipdb_categories": self.abuseipdb_categories,
# GreyNoise
"greynoise_classification": self.greynoise_classification,
"greynoise_name": self.greynoise_name,
"greynoise_tags": _decode_json_list(self.greynoise_tags),
"greynoise_tags": self.greynoise_tags,
# Feodo
"feodo_listed": self.feodo_listed,
"feodo_malware_family": self.feodo_malware_family,
# ThreatFox
"threatfox_listed": self.threatfox_listed,
"threatfox_threat_types": _decode_json_list(self.threatfox_threat_types),
"threatfox_ioc_types": _decode_json_list(self.threatfox_ioc_types),
"threatfox_malware_families": _decode_json_list(
self.threatfox_malware_families
),
"threatfox_threat_types": self.threatfox_threat_types,
"threatfox_ioc_types": self.threatfox_ioc_types,
"threatfox_malware_families": self.threatfox_malware_families,
}
if providers is not None:
d["providers"] = providers

24
decnet/web/db/sqlmodel_repo/attacker_intel.py
View File

@@ -7,7 +7,6 @@ worker query.
"""
from __future__ import annotations
import json
import uuid as _uuid
from datetime import datetime, timezone
from typing import Any, Optional
@@ -67,28 +66,7 @@ class AttackerIntelMixin(_MixinBase):
row = result.scalar_one_or_none()
if not row:
return None
d = row.model_dump(mode="json")
# Two passes: ``*_raw`` columns hold provider response blobs
# (objects); the per-provider taxonomy columns hold JSON
# arrays the IntelLifter consumes as native lists.
for key in (
"greynoise_raw",
"abuseipdb_raw",
"feodo_raw",
"threatfox_raw",
"greynoise_tags",
"abuseipdb_categories",
"threatfox_threat_types",
"threatfox_ioc_types",
"threatfox_malware_families",
):
raw = d.get(key)
if isinstance(raw, str):
try:
d[key] = json.loads(raw)
except (json.JSONDecodeError, TypeError):
pass
return d
return row.model_dump(mode="json")
async def get_unenriched_attackers(
self, limit: int = 100,

14
decnet/web/db/sqlmodel_repo/attackers/_core.py
View File

@@ -114,22 +114,10 @@ class AttackersCoreMixin(_MixinBase):
async with self._session() as session:
rows = (await session.execute(stmt)).all()
_intel_raw_keys = ("greynoise_raw", "abuseipdb_raw", "feodo_raw", "threatfox_raw")
result = []
for attacker, intel in rows:
d = self._deserialize_attacker(attacker.model_dump(mode="json"))
if intel is not None:
intel_d = intel.model_dump(mode="json")
for key in _intel_raw_keys:
raw = intel_d.get(key)
if isinstance(raw, str):
try:
intel_d[key] = json.loads(raw)
except (json.JSONDecodeError, TypeError):
pass
d["threat_intel"] = intel_d
else:
d["threat_intel"] = None
d["threat_intel"] = intel.model_dump(mode="json") if intel is not None else None
result.append(d)
return result