feat(intel): persist per-provider taxonomy on AttackerIntel for TTP dispatch
The 2026-05-02 ship-time audit of the R0054-R0058 intel rule pack found that AbuseIPDB / GreyNoise / ThreatFox stored only the aggregate verdict (score / classification / listed-bool) plus the raw response blob. The TTP IntelLifter expects per-provider taxonomy fields (categories, tags, threat_types) that were never populated, so R0054 / R0055 / R0057 emitted zero tags in production despite passing unit tests. Add typed columns: abuseipdb_categories, greynoise_tags, greynoise_name, feodo_malware_family, threatfox_threat_types, threatfox_ioc_types, threatfox_malware_families. Each provider now parses the relevant taxonomy out of the upstream response and writes it through column_updates. JSON-list columns ride as TEXT with default "[]" to keep the SQLite/MySQL backend split honest, deserialised back to native lists by the repo on read.
This commit is contained in:
@@ -95,6 +95,50 @@ async def test_low_score_maps_to_benign(monkeypatch):
|
||||
assert result.column_updates["abuseipdb_score"] == 0
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_categories_flattened_from_reports(monkeypatch):
|
||||
"""Post-2026-05-02 audit: provider must extract the union of
|
||||
``data.reports[*].categories`` so the IntelLifter can dispatch
|
||||
ATT&CK techniques. Sorted for deterministic test + bus diff."""
|
||||
monkeypatch.setenv("DECNET_ABUSEIPDB_API_KEY", "k3y")
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"data": {
|
||||
"abuseConfidenceScore": 80,
|
||||
"reports": [
|
||||
{"categories": [18, 22]},
|
||||
{"categories": [22, 14]},
|
||||
{"categories": []},
|
||||
{"not_a_dict": True},
|
||||
{"categories": [21]},
|
||||
],
|
||||
}},
|
||||
)
|
||||
|
||||
_install_transport(handler)
|
||||
provider = AbuseIPDBProvider()
|
||||
result = await provider.lookup("1.2.3.4")
|
||||
cats = json.loads(result.column_updates["abuseipdb_categories"])
|
||||
assert cats == [14, 18, 21, 22]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_categories_empty_when_no_reports(monkeypatch):
|
||||
monkeypatch.setenv("DECNET_ABUSEIPDB_API_KEY", "k3y")
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200, json={"data": {"abuseConfidenceScore": 5}},
|
||||
)
|
||||
|
||||
_install_transport(handler)
|
||||
provider = AbuseIPDBProvider()
|
||||
result = await provider.lookup("8.8.8.8")
|
||||
assert json.loads(result.column_updates["abuseipdb_categories"]) == []
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_429_returns_error(monkeypatch):
|
||||
monkeypatch.setenv("DECNET_ABUSEIPDB_API_KEY", "k3y")
|
||||
|
||||
@@ -87,6 +87,31 @@ async def test_unlisted_ip_returns_no_verdict():
|
||||
assert result.column_updates["feodo_listed"] is False
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_listed_ip_persists_malware_family():
|
||||
"""Post-2026-05-02 audit: IntelLifter reads
|
||||
``feodo_malware_family`` for evidence; persist it as a typed
|
||||
column rather than only inside ``feodo_raw``."""
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=_FEED)
|
||||
|
||||
_install_transport(handler)
|
||||
provider = FeodoProvider(refresh_interval_s=999.0)
|
||||
result = await provider.lookup("9.9.9.9")
|
||||
assert result.column_updates["feodo_malware_family"] == "TrickBot"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_unlisted_ip_clears_family():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=_FEED)
|
||||
|
||||
_install_transport(handler)
|
||||
provider = FeodoProvider(refresh_interval_s=999.0)
|
||||
result = await provider.lookup("1.2.3.4")
|
||||
assert result.column_updates["feodo_malware_family"] is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_feed_failure_reports_error():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
|
||||
@@ -123,6 +123,45 @@ async def test_429_returns_error_no_writes():
|
||||
assert result.column_updates == {}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_actor_name_and_tags_persisted_when_present():
|
||||
"""Post-2026-05-02 audit: ``name`` (actor label) and any ``tags``
|
||||
list returned by the upstream survive into ``column_updates``.
|
||||
|
||||
The Community endpoint does not return ``tags`` in practice; the
|
||||
test seeds the field anyway so non-Community provider plans that
|
||||
do (paid / Enterprise) work without further code changes.
|
||||
"""
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"classification": "malicious",
|
||||
"name": "Tor",
|
||||
"tags": ["tor_exit_node", "ssh_bruteforcer"],
|
||||
},
|
||||
)
|
||||
|
||||
provider = GreyNoiseProvider()
|
||||
_install_transport(provider, handler)
|
||||
result = await provider.lookup("1.2.3.4")
|
||||
assert result.column_updates["greynoise_name"] == "Tor"
|
||||
tags = json.loads(result.column_updates["greynoise_tags"])
|
||||
assert tags == ["tor_exit_node", "ssh_bruteforcer"]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_404_clears_actor_and_tags():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(404, json={"message": "not seen"})
|
||||
|
||||
provider = GreyNoiseProvider()
|
||||
_install_transport(provider, handler)
|
||||
result = await provider.lookup("10.0.0.5")
|
||||
assert result.column_updates["greynoise_name"] is None
|
||||
assert result.column_updates["greynoise_tags"] == "[]"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_network_failure_becomes_error():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
|
||||
@@ -100,6 +100,61 @@ async def test_unexpected_status_is_error():
|
||||
assert result.column_updates == {}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_threat_types_and_ioc_types_flattened(monkeypatch):
|
||||
"""Post-2026-05-02 audit: provider must extract the union of
|
||||
``threat_type`` / ``ioc_type`` / ``malware`` across all matches.
|
||||
The IntelLifter dispatches ATT&CK on threat_type."""
|
||||
monkeypatch.delenv("DECNET_THREATFOX_API_KEY", raising=False)
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"query_status": "ok", "data": [
|
||||
{
|
||||
"ioc_type": "ip:port",
|
||||
"threat_type": "botnet_cc",
|
||||
"malware": "Sliver",
|
||||
},
|
||||
{
|
||||
"ioc_type": "url",
|
||||
"threat_type": "payload_delivery",
|
||||
"malware_printable": "Emotet",
|
||||
},
|
||||
{
|
||||
"ioc_type": "ip:port", # duplicate, dedup'd
|
||||
"threat_type": "botnet_cc", # duplicate
|
||||
"malware": "Sliver", # duplicate
|
||||
},
|
||||
"not a dict — silently skipped",
|
||||
]},
|
||||
)
|
||||
|
||||
_install_transport(handler)
|
||||
provider = ThreatFoxProvider()
|
||||
result = await provider.lookup("1.2.3.4")
|
||||
cu = result.column_updates
|
||||
assert json.loads(cu["threatfox_threat_types"]) == [
|
||||
"botnet_cc", "payload_delivery",
|
||||
]
|
||||
assert json.loads(cu["threatfox_ioc_types"]) == ["ip:port", "url"]
|
||||
assert json.loads(cu["threatfox_malware_families"]) == ["Emotet", "Sliver"]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_no_result_clears_taxonomy_columns():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json={"query_status": "no_result"})
|
||||
|
||||
_install_transport(handler)
|
||||
provider = ThreatFoxProvider()
|
||||
result = await provider.lookup("8.8.8.8")
|
||||
cu = result.column_updates
|
||||
assert cu["threatfox_threat_types"] == "[]"
|
||||
assert cu["threatfox_ioc_types"] == "[]"
|
||||
assert cu["threatfox_malware_families"] == "[]"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_http_error_surfaces():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
|
||||
Reference in New Issue
Block a user