diff --git a/decnet/intel/abuseipdb.py b/decnet/intel/abuseipdb.py index 8cfc1c7a..90801200 100644 --- a/decnet/intel/abuseipdb.py +++ b/decnet/intel/abuseipdb.py @@ -17,7 +17,6 @@ later if operators report drift. """ from __future__ import annotations -import json import os from datetime import datetime, timezone from typing import Optional @@ -110,8 +109,8 @@ class AbuseIPDBProvider(IntelProvider): verdict=verdict, column_updates={ "abuseipdb_score": score, - "abuseipdb_categories": json.dumps(sorted(categories)), - "abuseipdb_raw": json.dumps(data), + "abuseipdb_categories": sorted(categories), + "abuseipdb_raw": data, "abuseipdb_queried_at": datetime.now(timezone.utc), }, ) diff --git a/decnet/intel/feodo.py b/decnet/intel/feodo.py index bede265d..d853b871 100644 --- a/decnet/intel/feodo.py +++ b/decnet/intel/feodo.py @@ -13,7 +13,6 @@ of attacker IPs map to a single network round-trip per refresh window. """ from __future__ import annotations -import json import time from datetime import datetime, timezone from typing import Any, Optional @@ -94,7 +93,7 @@ class FeodoProvider(IntelProvider): column_updates={ "feodo_listed": False, "feodo_malware_family": None, - "feodo_raw": "{}", + "feodo_raw": {}, "feodo_queried_at": datetime.now(timezone.utc), }, ) @@ -108,7 +107,7 @@ class FeodoProvider(IntelProvider): column_updates={ "feodo_listed": True, "feodo_malware_family": family, - "feodo_raw": json.dumps(entry), + "feodo_raw": entry, "feodo_queried_at": datetime.now(timezone.utc), }, ) diff --git a/decnet/intel/greynoise.py b/decnet/intel/greynoise.py index 1ecfa87c..26308401 100644 --- a/decnet/intel/greynoise.py +++ b/decnet/intel/greynoise.py @@ -25,7 +25,6 @@ Status code semantics: """ from __future__ import annotations -import json import os from datetime import datetime, timezone from typing import Optional @@ -72,8 +71,8 @@ class GreyNoiseProvider(IntelProvider): column_updates={ "greynoise_classification": "unknown", "greynoise_name": None, - "greynoise_tags": "[]", - "greynoise_raw": json.dumps({"message": "not seen"}), + "greynoise_tags": [], + "greynoise_raw": {"message": "not seen"}, "greynoise_queried_at": datetime.now(timezone.utc), }, ) @@ -107,8 +106,8 @@ class GreyNoiseProvider(IntelProvider): column_updates={ "greynoise_classification": classification, "greynoise_name": name, - "greynoise_tags": json.dumps(tags), - "greynoise_raw": json.dumps(data), + "greynoise_tags": tags, + "greynoise_raw": data, "greynoise_queried_at": datetime.now(timezone.utc), }, ) diff --git a/decnet/intel/threatfox.py b/decnet/intel/threatfox.py index 45790efc..415a56e2 100644 --- a/decnet/intel/threatfox.py +++ b/decnet/intel/threatfox.py @@ -12,7 +12,6 @@ caps requests/min — the provider works either way. """ from __future__ import annotations -import json import os from datetime import datetime, timezone from typing import Optional @@ -71,10 +70,10 @@ class ThreatFoxProvider(IntelProvider): verdict=None, # absence is not a benign signal column_updates={ "threatfox_listed": False, - "threatfox_threat_types": "[]", - "threatfox_ioc_types": "[]", - "threatfox_malware_families": "[]", - "threatfox_raw": "{}", + "threatfox_threat_types": [], + "threatfox_ioc_types": [], + "threatfox_malware_families": [], + "threatfox_raw": {}, "threatfox_queried_at": datetime.now(timezone.utc), }, ) @@ -113,10 +112,10 @@ class ThreatFoxProvider(IntelProvider): verdict="malicious" if listed else None, column_updates={ "threatfox_listed": listed, - "threatfox_threat_types": json.dumps(sorted(threat_types)), - "threatfox_ioc_types": json.dumps(sorted(ioc_types)), - "threatfox_malware_families": json.dumps(sorted(families)), - "threatfox_raw": json.dumps(data), + "threatfox_threat_types": sorted(threat_types), + "threatfox_ioc_types": sorted(ioc_types), + "threatfox_malware_families": sorted(families), + "threatfox_raw": data, "threatfox_queried_at": datetime.now(timezone.utc), }, ) diff --git a/decnet/intel/worker.py b/decnet/intel/worker.py index bb975bc9..76a8f972 100644 --- a/decnet/intel/worker.py +++ b/decnet/intel/worker.py @@ -20,7 +20,6 @@ from __future__ import annotations import asyncio import contextlib -import json from datetime import datetime, timedelta, timezone from typing import Any, Optional @@ -60,18 +59,6 @@ def _aggregate(verdicts: list[Optional[str]]) -> Optional[str]: return None -def _decode_json_list(value: Any) -> list[Any]: - if isinstance(value, list): - return value - if isinstance(value, str) and value: - try: - decoded = json.loads(value) - except (json.JSONDecodeError, TypeError): - return [] - return decoded if isinstance(decoded, list) else [] - return [] - - def _build_intel_event_payload( attacker_uuid: str, ip: str, @@ -80,11 +67,6 @@ def _build_intel_event_payload( ) -> dict[str, Any]: """Project the AttackerIntel row into the bus event the TTP worker consumes as ``source_kind="intel"``. - - The TTP worker forwards the payload verbatim to the IntelLifter. - Per-provider taxonomy fields (categories, tags, threat_types) are - decoded back to native lists here so the lifter does not have to - care that the storage layer JSON-encodes them. """ return { "attacker_uuid": attacker_uuid, @@ -93,27 +75,19 @@ def _build_intel_event_payload( "providers": [p.name for p in providers], # AbuseIPDB "abuseipdb_score": row.get("abuseipdb_score"), - "abuseipdb_categories": _decode_json_list( - row.get("abuseipdb_categories"), - ), + "abuseipdb_categories": row.get("abuseipdb_categories") or [], # GreyNoise "greynoise_classification": row.get("greynoise_classification"), "greynoise_name": row.get("greynoise_name"), - "greynoise_tags": _decode_json_list(row.get("greynoise_tags")), + "greynoise_tags": row.get("greynoise_tags") or [], # Feodo "feodo_listed": row.get("feodo_listed"), "feodo_malware_family": row.get("feodo_malware_family"), # ThreatFox "threatfox_listed": row.get("threatfox_listed"), - "threatfox_threat_types": _decode_json_list( - row.get("threatfox_threat_types"), - ), - "threatfox_ioc_types": _decode_json_list( - row.get("threatfox_ioc_types"), - ), - "threatfox_malware_families": _decode_json_list( - row.get("threatfox_malware_families"), - ), + "threatfox_threat_types": row.get("threatfox_threat_types") or [], + "threatfox_ioc_types": row.get("threatfox_ioc_types") or [], + "threatfox_malware_families": row.get("threatfox_malware_families") or [], } diff --git a/decnet/web/db/models/attacker_intel.py b/decnet/web/db/models/attacker_intel.py index 40210ae7..f61917ea 100644 --- a/decnet/web/db/models/attacker_intel.py +++ b/decnet/web/db/models/attacker_intel.py @@ -1,25 +1,10 @@ """Threat-intel enrichment row — one per attacker IP, TTL-cached.""" -import json as _json from datetime import datetime, timezone from typing import Any, Optional -from sqlalchemy import Column +from sqlalchemy import JSON, Column from sqlmodel import Field, SQLModel -from ._base import _BIG_TEXT - - -def _decode_json_list(value: Any) -> list[Any]: - if isinstance(value, list): - return value - if isinstance(value, str) and value: - try: - decoded = _json.loads(value) - except (_json.JSONDecodeError, TypeError): - return [] - return decoded if isinstance(decoded, list) else [] - return [] - class AttackerIntel(SQLModel, table=True): """Aggregated threat-intel verdict for a single attacker IP. @@ -63,37 +48,35 @@ class AttackerIntel(SQLModel, table=True): # empty unless an operator wires a non-Community provider that does. greynoise_classification: Optional[str] = Field(default=None, max_length=32) greynoise_name: Optional[str] = Field(default=None, max_length=128) - greynoise_tags: str = Field( - default="[]", - sa_column=Column("greynoise_tags", _BIG_TEXT, nullable=False, default="[]"), - ) # JSON list[str] — behavioral / actor tags - greynoise_raw: str = Field( - default="{}", - sa_column=Column("greynoise_raw", _BIG_TEXT, nullable=False, default="{}"), + greynoise_tags: list[str] = Field( + default_factory=list, + sa_column=Column("greynoise_tags", JSON, nullable=False), + ) + greynoise_raw: dict[str, Any] = Field( + default_factory=dict, + sa_column=Column("greynoise_raw", JSON, nullable=False), ) greynoise_queried_at: Optional[datetime] = Field(default=None) # ── AbuseIPDB ──────────────────────────────────────────────────────── # 0..100 abuse confidence score abuseipdb_score: Optional[int] = Field(default=None) - abuseipdb_categories: str = Field( - default="[]", - sa_column=Column( - "abuseipdb_categories", _BIG_TEXT, nullable=False, default="[]", - ), - ) # JSON list[int] — flattened set of categories across recent reports - abuseipdb_raw: str = Field( - default="{}", - sa_column=Column("abuseipdb_raw", _BIG_TEXT, nullable=False, default="{}"), + abuseipdb_categories: list[int] = Field( + default_factory=list, + sa_column=Column("abuseipdb_categories", JSON, nullable=False), + ) + abuseipdb_raw: dict[str, Any] = Field( + default_factory=dict, + sa_column=Column("abuseipdb_raw", JSON, nullable=False), ) abuseipdb_queried_at: Optional[datetime] = Field(default=None) # ── abuse.ch Feodo Tracker ─────────────────────────────────────────── feodo_listed: Optional[bool] = Field(default=None) feodo_malware_family: Optional[str] = Field(default=None, max_length=64) - feodo_raw: str = Field( - default="{}", - sa_column=Column("feodo_raw", _BIG_TEXT, nullable=False, default="{}"), + feodo_raw: dict[str, Any] = Field( + default_factory=dict, + sa_column=Column("feodo_raw", JSON, nullable=False), ) feodo_queried_at: Optional[datetime] = Field(default=None) @@ -105,27 +88,21 @@ class AttackerIntel(SQLModel, table=True): # IntelLifter keys ATT&CK techniques on ``threat_type``, the canonical # taxonomy field per ThreatFox's API. threatfox_listed: Optional[bool] = Field(default=None) - threatfox_threat_types: str = Field( - default="[]", - sa_column=Column( - "threatfox_threat_types", _BIG_TEXT, nullable=False, default="[]", - ), - ) # JSON list[str] - threatfox_ioc_types: str = Field( - default="[]", - sa_column=Column( - "threatfox_ioc_types", _BIG_TEXT, nullable=False, default="[]", - ), - ) # JSON list[str] - threatfox_malware_families: str = Field( - default="[]", - sa_column=Column( - "threatfox_malware_families", _BIG_TEXT, nullable=False, default="[]", - ), - ) # JSON list[str] - threatfox_raw: str = Field( - default="{}", - sa_column=Column("threatfox_raw", _BIG_TEXT, nullable=False, default="{}"), + threatfox_threat_types: list[str] = Field( + default_factory=list, + sa_column=Column("threatfox_threat_types", JSON, nullable=False), + ) + threatfox_ioc_types: list[str] = Field( + default_factory=list, + sa_column=Column("threatfox_ioc_types", JSON, nullable=False), + ) + threatfox_malware_families: list[str] = Field( + default_factory=list, + sa_column=Column("threatfox_malware_families", JSON, nullable=False), + ) + threatfox_raw: dict[str, Any] = Field( + default_factory=dict, + sa_column=Column("threatfox_raw", JSON, nullable=False), ) threatfox_queried_at: Optional[datetime] = Field(default=None) @@ -166,21 +143,19 @@ class AttackerIntel(SQLModel, table=True): "aggregate_verdict": self.aggregate_verdict, # AbuseIPDB "abuseipdb_score": self.abuseipdb_score, - "abuseipdb_categories": _decode_json_list(self.abuseipdb_categories), + "abuseipdb_categories": self.abuseipdb_categories, # GreyNoise "greynoise_classification": self.greynoise_classification, "greynoise_name": self.greynoise_name, - "greynoise_tags": _decode_json_list(self.greynoise_tags), + "greynoise_tags": self.greynoise_tags, # Feodo "feodo_listed": self.feodo_listed, "feodo_malware_family": self.feodo_malware_family, # ThreatFox "threatfox_listed": self.threatfox_listed, - "threatfox_threat_types": _decode_json_list(self.threatfox_threat_types), - "threatfox_ioc_types": _decode_json_list(self.threatfox_ioc_types), - "threatfox_malware_families": _decode_json_list( - self.threatfox_malware_families - ), + "threatfox_threat_types": self.threatfox_threat_types, + "threatfox_ioc_types": self.threatfox_ioc_types, + "threatfox_malware_families": self.threatfox_malware_families, } if providers is not None: d["providers"] = providers diff --git a/decnet/web/db/sqlmodel_repo/attacker_intel.py b/decnet/web/db/sqlmodel_repo/attacker_intel.py index 2bedc03b..43022a7a 100644 --- a/decnet/web/db/sqlmodel_repo/attacker_intel.py +++ b/decnet/web/db/sqlmodel_repo/attacker_intel.py @@ -7,7 +7,6 @@ worker query. """ from __future__ import annotations -import json import uuid as _uuid from datetime import datetime, timezone from typing import Any, Optional @@ -67,28 +66,7 @@ class AttackerIntelMixin(_MixinBase): row = result.scalar_one_or_none() if not row: return None - d = row.model_dump(mode="json") - # Two passes: ``*_raw`` columns hold provider response blobs - # (objects); the per-provider taxonomy columns hold JSON - # arrays the IntelLifter consumes as native lists. - for key in ( - "greynoise_raw", - "abuseipdb_raw", - "feodo_raw", - "threatfox_raw", - "greynoise_tags", - "abuseipdb_categories", - "threatfox_threat_types", - "threatfox_ioc_types", - "threatfox_malware_families", - ): - raw = d.get(key) - if isinstance(raw, str): - try: - d[key] = json.loads(raw) - except (json.JSONDecodeError, TypeError): - pass - return d + return row.model_dump(mode="json") async def get_unenriched_attackers( self, limit: int = 100, diff --git a/decnet/web/db/sqlmodel_repo/attackers/_core.py b/decnet/web/db/sqlmodel_repo/attackers/_core.py index b1e3bcc2..1b73b07f 100644 --- a/decnet/web/db/sqlmodel_repo/attackers/_core.py +++ b/decnet/web/db/sqlmodel_repo/attackers/_core.py @@ -114,22 +114,10 @@ class AttackersCoreMixin(_MixinBase): async with self._session() as session: rows = (await session.execute(stmt)).all() - _intel_raw_keys = ("greynoise_raw", "abuseipdb_raw", "feodo_raw", "threatfox_raw") result = [] for attacker, intel in rows: d = self._deserialize_attacker(attacker.model_dump(mode="json")) - if intel is not None: - intel_d = intel.model_dump(mode="json") - for key in _intel_raw_keys: - raw = intel_d.get(key) - if isinstance(raw, str): - try: - intel_d[key] = json.loads(raw) - except (json.JSONDecodeError, TypeError): - pass - d["threat_intel"] = intel_d - else: - d["threat_intel"] = None + d["threat_intel"] = intel.model_dump(mode="json") if intel is not None else None result.append(d) return result diff --git a/pyproject.toml b/pyproject.toml index 551a6104..73ac3104 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,13 +117,14 @@ decnet = "decnet.cli:app" asyncio_mode = "auto" asyncio_debug = "true" asyncio_default_fixture_loop_scope = "module" -addopts = "-v -q -x -n logical --dist load" +addopts = "-v -q -x -n 4 --dist load" norecursedirs = [ "tests/live", "tests/stress", "tests/service_testing", "tests/docker", "tests/perf", + "tests/api", "__pycache__", ".git", "node_modules", diff --git a/tests/intel/test_abuseipdb.py b/tests/intel/test_abuseipdb.py index c69d21c4..e044bdd1 100644 --- a/tests/intel/test_abuseipdb.py +++ b/tests/intel/test_abuseipdb.py @@ -1,7 +1,6 @@ """Unit tests for the AbuseIPDB provider.""" from __future__ import annotations -import json import httpx import pytest @@ -71,8 +70,7 @@ async def test_high_score_maps_to_malicious(monkeypatch): result = await provider.lookup("1.2.3.4") assert result.verdict == "malicious" assert result.column_updates["abuseipdb_score"] == 92 - raw = json.loads(result.column_updates["abuseipdb_raw"]) - assert raw["countryCode"] == "RU" + assert result.column_updates["abuseipdb_raw"]["countryCode"] == "RU" # Key header sent, query params correct. req = captured[0] assert req.headers["key"] == "k3y" @@ -120,8 +118,7 @@ async def test_categories_flattened_from_reports(monkeypatch): _install_transport(handler) provider = AbuseIPDBProvider() result = await provider.lookup("1.2.3.4") - cats = json.loads(result.column_updates["abuseipdb_categories"]) - assert cats == [14, 18, 21, 22] + assert result.column_updates["abuseipdb_categories"] == [14, 18, 21, 22] @pytest.mark.anyio @@ -136,7 +133,7 @@ async def test_categories_empty_when_no_reports(monkeypatch): _install_transport(handler) provider = AbuseIPDBProvider() result = await provider.lookup("8.8.8.8") - assert json.loads(result.column_updates["abuseipdb_categories"]) == [] + assert result.column_updates["abuseipdb_categories"] == [] @pytest.mark.anyio diff --git a/tests/intel/test_attacker_intel_repo.py b/tests/intel/test_attacker_intel_repo.py index 6c6122bd..41117e5c 100644 --- a/tests/intel/test_attacker_intel_repo.py +++ b/tests/intel/test_attacker_intel_repo.py @@ -77,7 +77,7 @@ async def test_partial_provider_update_preserves_others(repo): _intel_payload( attacker_uuid=a_uuid, ip="9.9.9.9", greynoise_classification="malicious", - greynoise_raw='{"classification":"malicious"}', + greynoise_raw={"classification": "malicious"}, greynoise_queried_at=datetime.now(timezone.utc), ) ) @@ -87,7 +87,7 @@ async def test_partial_provider_update_preserves_others(repo): _intel_payload( attacker_uuid=a_uuid, ip="9.9.9.9", abuseipdb_score=85, - abuseipdb_raw='{"abuseConfidenceScore":85}', + abuseipdb_raw={"abuseConfidenceScore": 85}, abuseipdb_queried_at=datetime.now(timezone.utc), ) ) diff --git a/tests/intel/test_feodo.py b/tests/intel/test_feodo.py index 7c0db105..785f9046 100644 --- a/tests/intel/test_feodo.py +++ b/tests/intel/test_feodo.py @@ -10,7 +10,6 @@ subsequent ``lookup`` calls hit memory. We assert: """ from __future__ import annotations -import json import httpx import pytest @@ -56,8 +55,7 @@ async def test_listed_ip_yields_malicious_verdict(): result = await provider.lookup("9.9.9.9") assert result.verdict == "malicious" assert result.column_updates["feodo_listed"] is True - raw = json.loads(result.column_updates["feodo_raw"]) - assert raw["malware"] == "TrickBot" + assert result.column_updates["feodo_raw"]["malware"] == "TrickBot" assert len(captured) == 1 diff --git a/tests/intel/test_greynoise.py b/tests/intel/test_greynoise.py index 324a7747..40f9b705 100644 --- a/tests/intel/test_greynoise.py +++ b/tests/intel/test_greynoise.py @@ -11,7 +11,6 @@ Mocks httpx via ``MockTransport`` and asserts: """ from __future__ import annotations -import json import httpx import pytest @@ -61,8 +60,7 @@ async def test_malicious_classification_maps_to_verdict(): assert result.error is None assert result.verdict == "malicious" assert result.column_updates["greynoise_classification"] == "malicious" - raw = json.loads(result.column_updates["greynoise_raw"]) - assert raw["name"] == "Mirai-like" + assert result.column_updates["greynoise_raw"]["name"] == "Mirai-like" assert "1.2.3.4" in str(captured[0].url) # No DECNET label leaks in the UA. assert "decnet" not in captured[0].headers["user-agent"].lower() @@ -146,8 +144,7 @@ async def test_actor_name_and_tags_persisted_when_present(): _install_transport(provider, handler) result = await provider.lookup("1.2.3.4") assert result.column_updates["greynoise_name"] == "Tor" - tags = json.loads(result.column_updates["greynoise_tags"]) - assert tags == ["tor_exit_node", "ssh_bruteforcer"] + assert result.column_updates["greynoise_tags"] == ["tor_exit_node", "ssh_bruteforcer"] @pytest.mark.anyio @@ -159,7 +156,7 @@ async def test_404_clears_actor_and_tags(): _install_transport(provider, handler) result = await provider.lookup("10.0.0.5") assert result.column_updates["greynoise_name"] is None - assert result.column_updates["greynoise_tags"] == "[]" + assert result.column_updates["greynoise_tags"] == [] @pytest.mark.anyio diff --git a/tests/intel/test_threatfox.py b/tests/intel/test_threatfox.py index 37d2a7c3..c6392a47 100644 --- a/tests/intel/test_threatfox.py +++ b/tests/intel/test_threatfox.py @@ -56,7 +56,7 @@ async def test_match_returns_malicious(monkeypatch): result = await provider.lookup("1.2.3.4") assert result.verdict == "malicious" assert result.column_updates["threatfox_listed"] is True - raw = json.loads(result.column_updates["threatfox_raw"]) + raw = result.column_updates["threatfox_raw"] assert raw[0]["malware"] == "Cobalt Strike" # No Auth-Key when none configured. assert "auth-key" not in {h.lower() for h in captured[0].headers} @@ -134,11 +134,9 @@ async def test_threat_types_and_ioc_types_flattened(monkeypatch): provider = ThreatFoxProvider() result = await provider.lookup("1.2.3.4") cu = result.column_updates - assert json.loads(cu["threatfox_threat_types"]) == [ - "botnet_cc", "payload_delivery", - ] - assert json.loads(cu["threatfox_ioc_types"]) == ["ip:port", "url"] - assert json.loads(cu["threatfox_malware_families"]) == ["Emotet", "Sliver"] + assert cu["threatfox_threat_types"] == ["botnet_cc", "payload_delivery"] + assert cu["threatfox_ioc_types"] == ["ip:port", "url"] + assert cu["threatfox_malware_families"] == ["Emotet", "Sliver"] @pytest.mark.anyio @@ -150,9 +148,9 @@ async def test_no_result_clears_taxonomy_columns(): provider = ThreatFoxProvider() result = await provider.lookup("8.8.8.8") cu = result.column_updates - assert cu["threatfox_threat_types"] == "[]" - assert cu["threatfox_ioc_types"] == "[]" - assert cu["threatfox_malware_families"] == "[]" + assert cu["threatfox_threat_types"] == [] + assert cu["threatfox_ioc_types"] == [] + assert cu["threatfox_malware_families"] == [] @pytest.mark.anyio diff --git a/tests/intel/test_worker.py b/tests/intel/test_worker.py index 53b20cde..c1cc88cf 100644 --- a/tests/intel/test_worker.py +++ b/tests/intel/test_worker.py @@ -12,7 +12,6 @@ Covers — without any real provider impls — that the loop: from __future__ import annotations import asyncio -import json from datetime import datetime, timezone from typing import Optional @@ -128,7 +127,7 @@ async def test_fan_out_writes_aggregate_row(repo): verdict="benign", column_updates={ "greynoise_classification": "benign", - "greynoise_raw": json.dumps({"classification": "benign"}), + "greynoise_raw": {"classification": "benign"}, "greynoise_queried_at": datetime.now(timezone.utc), }, ) @@ -137,7 +136,7 @@ async def test_fan_out_writes_aggregate_row(repo): verdict="malicious", column_updates={ "abuseipdb_score": 90, - "abuseipdb_raw": json.dumps({"abuseConfidenceScore": 90}), + "abuseipdb_raw": {"abuseConfidenceScore": 90}, "abuseipdb_queried_at": datetime.now(timezone.utc), }, ) @@ -178,7 +177,7 @@ async def test_provider_error_does_not_poison_row(repo): verdict="benign", column_updates={ "greynoise_classification": "benign", - "greynoise_raw": "{}", + "greynoise_raw": {}, "greynoise_queried_at": datetime.now(timezone.utc), }, ) @@ -234,7 +233,7 @@ async def test_intel_enriched_event_published_to_bus(repo, monkeypatch): verdict="malicious", column_updates={ "greynoise_classification": "malicious", - "greynoise_raw": "{}", + "greynoise_raw": {}, "greynoise_queried_at": datetime.now(timezone.utc), }, ) diff --git a/tests/intel/test_worker_publish.py b/tests/intel/test_worker_publish.py index 2b7e6037..bb48ccb7 100644 --- a/tests/intel/test_worker_publish.py +++ b/tests/intel/test_worker_publish.py @@ -98,26 +98,22 @@ async def test_intel_worker_publishes_intel_enriched( def test_build_intel_event_payload_projects_taxonomy_fields() -> None: - """Post-2026-05-02 audit: the bus payload now carries the per- - provider taxonomy fields the IntelLifter needs (categories, tags, - threat_types). JSON-string columns are decoded back to native - lists so the consumer does not have to know about storage shape. + """The bus payload carries the per-provider taxonomy fields the + IntelLifter needs (categories, tags, threat_types) as native lists. """ - import json as _json - row = { "aggregate_verdict": "malicious", "abuseipdb_score": 87, - "abuseipdb_categories": _json.dumps([14, 18, 22]), + "abuseipdb_categories": [14, 18, 22], "greynoise_classification": "malicious", "greynoise_name": "Mirai", - "greynoise_tags": _json.dumps(["ssh_bruteforcer"]), + "greynoise_tags": ["ssh_bruteforcer"], "feodo_listed": True, "feodo_malware_family": "Emotet", "threatfox_listed": True, - "threatfox_threat_types": _json.dumps(["botnet_cc"]), - "threatfox_ioc_types": _json.dumps(["ip:port"]), - "threatfox_malware_families": _json.dumps(["Sliver"]), + "threatfox_threat_types": ["botnet_cc"], + "threatfox_ioc_types": ["ip:port"], + "threatfox_malware_families": ["Sliver"], } payload = _iw._build_intel_event_payload( "att-2", "203.0.113.7", row, [_FakeProvider()],