feat(stix_export): wire fingerprint bounties through all endpoints + tests

Remaining files from the fingerprint-bounties + characterizes-SRO commit:
misp_export, repository, bounties mixin, all 4 router endpoints, and test suite
updates. Prerequisite: previous commit added _extract_fingerprint_bounty_data
and the stix_export changes.
This commit is contained in:
2026-05-09 09:14:48 -04:00
parent 51d0fc7b6c
commit 4c6b12dcf8
13 changed files with 181 additions and 4 deletions

View File

@@ -48,6 +48,7 @@ def build_attacker_misp_event(
smtp_targets: list[dict[str, Any]],
commands: list[str] | None = None,
observations: list[dict[str, Any]] | None = None,
fingerprint_bounties: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
"""Return a MISP event dict for *attacker*.
@@ -65,6 +66,7 @@ def build_attacker_misp_event(
smtp_targets=smtp_targets,
commands=commands,
observations=observations,
fingerprint_bounties=fingerprint_bounties,
)
return _parse_bundle(bundle)
@@ -73,6 +75,7 @@ def build_fleet_misp_collection(
rows: list[dict[str, Any]],
ttp_by_attacker: dict[str, list[dict[str, Any]]],
observations_by_attacker: dict[str, list[dict[str, Any]]] | None = None,
fingerprint_bounties_by_ip: dict[str, list[dict[str, Any]]] | None = None,
) -> dict[str, Any]:
"""Return a MISP collection dict with one event per attacker in *rows*.
@@ -84,6 +87,7 @@ def build_fleet_misp_collection(
"""
events: list[dict[str, Any]] = []
obs_map = observations_by_attacker or {}
fp_map = fingerprint_bounties_by_ip or {}
for row in rows:
raw_cmds = row.get("commands") or []
if isinstance(raw_cmds, str):
@@ -107,6 +111,7 @@ def build_fleet_misp_collection(
smtp_targets=[],
commands=cmds,
observations=obs_map.get(row["uuid"]),
fingerprint_bounties=fp_map.get(row.get("ip", ""), []),
)
event = _parse_bundle(bundle)
if event:

View File

@@ -394,6 +394,27 @@ class BaseRepository(ABC):
"""
raise NotImplementedError
async def get_fingerprint_bounties_by_ip(
self, ip: str
) -> list[dict[str, Any]]:
"""Return all fingerprint bounties for *ip* with parsed payload dicts.
Filters to ``bounty_type='fingerprint'``. Each returned dict has the
standard Bounty columns plus a decoded ``payload`` dict.
"""
raise NotImplementedError
async def get_all_fingerprint_bounties_for_export(
self,
) -> dict[str, list[dict[str, Any]]]:
"""Return ``{attacker_ip: [bounty_dict, ...]}`` for all fingerprint
bounties in the database.
Used by fleet exports to attach per-attacker fingerprint bounties
(JARM hashes, HTTP quirks) without N+1 queries.
"""
raise NotImplementedError
async def upsert_observed_attachment(
self,
*,

View File

@@ -141,6 +141,40 @@ class BountiesMixin(_MixinBase):
grouped[item.attacker_ip].append(d)
return dict(grouped)
async def get_fingerprint_bounties_by_ip(self, ip: str) -> List[dict[str, Any]]:
async with self._session() as session:
result = await session.execute(
select(Bounty)
.where(Bounty.attacker_ip == ip, Bounty.bounty_type == "fingerprint")
.order_by(asc(Bounty.timestamp))
)
out: List[dict[str, Any]] = []
for item in result.scalars().all():
d = item.model_dump(mode="json")
try:
d["payload"] = json.loads(d["payload"])
except (json.JSONDecodeError, TypeError):
pass
out.append(d)
return out
async def get_all_fingerprint_bounties_for_export(self) -> dict[str, List[dict[str, Any]]]:
async with self._session() as session:
result = await session.execute(
select(Bounty)
.where(Bounty.bounty_type == "fingerprint")
.order_by(asc(Bounty.timestamp))
)
grouped: dict[str, List[dict[str, Any]]] = defaultdict(list)
for item in result.scalars().all():
d = item.model_dump(mode="json")
try:
d["payload"] = json.loads(d["payload"])
except (json.JSONDecodeError, TypeError):
pass
grouped[item.attacker_ip].append(d)
return dict(grouped)
async def count_probe_relays(self, attacker_ip: str, decky: str) -> int:
"""Return how many probe_relay bounties exist for this (attacker_ip, decky) pair."""
async with self._session() as session:

View File

@@ -66,6 +66,7 @@ async def api_export_attacker_misp(
repo.list_smtp_targets(uuid),
repo.list_attacker_commands_deduped(uuid),
repo.list_observations_by_attacker(uuid),
repo.get_fingerprint_bounties_by_ip(attacker["ip"]),
)
behavior = cast(dict[str, Any] | None, results[0])
identity = cast(dict[str, Any] | None, results[1])
@@ -76,6 +77,7 @@ async def api_export_attacker_misp(
smtp_targets = cast(list[dict[str, Any]], results[6])
commands = cast(list[str], results[7])
observations = cast(list[dict[str, Any]], results[8])
fingerprint_bounties = cast(list[dict[str, Any]], results[9])
event = build_attacker_misp_event(
attacker=attacker,
@@ -91,6 +93,7 @@ async def api_export_attacker_misp(
smtp_targets=smtp_targets,
commands=commands,
observations=observations,
fingerprint_bounties=fingerprint_bounties,
)
return Response(
content=json.dumps(event, default=str),

View File

@@ -70,6 +70,7 @@ async def api_export_attacker_stix(
repo.list_smtp_targets(uuid),
repo.list_attacker_commands_deduped(uuid),
repo.list_observations_by_attacker(uuid),
repo.get_fingerprint_bounties_by_ip(attacker["ip"]),
)
behavior = cast(dict[str, Any] | None, results[0])
identity = cast(dict[str, Any] | None, results[1])
@@ -80,6 +81,7 @@ async def api_export_attacker_stix(
smtp_targets = cast(list[dict[str, Any]], results[6])
commands = cast(list[str], results[7])
observations = cast(list[dict[str, Any]], results[8])
fingerprint_bounties = cast(list[dict[str, Any]], results[9])
bundle = build_attacker_bundle(
attacker=attacker,
@@ -95,6 +97,7 @@ async def api_export_attacker_stix(
smtp_targets=smtp_targets,
commands=commands,
observations=observations,
fingerprint_bounties=fingerprint_bounties,
)
return Response(
content=bundle.serialize(pretty=True, indent=2),

View File

@@ -43,15 +43,17 @@ async def api_export_attackers_misp(
user: dict[str, Any] = Depends(require_viewer),
) -> Response:
"""Download a MISP collection JSON covering every observed attacker."""
rows, ttp_by_attacker, obs_by_attacker = await asyncio.gather(
rows, ttp_by_attacker, obs_by_attacker, fp_by_ip = await asyncio.gather(
repo.get_all_attackers_for_export(),
repo.get_all_ttp_rollups_for_export(),
repo.get_all_observations_for_export(),
repo.get_all_fingerprint_bounties_for_export(),
)
collection = build_fleet_misp_collection(
rows=rows,
ttp_by_attacker=ttp_by_attacker,
observations_by_attacker=obs_by_attacker,
fingerprint_bounties_by_ip=fp_by_ip,
)
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
return Response(

View File

@@ -42,11 +42,12 @@ async def api_export_attackers_stix(
user: dict[str, Any] = Depends(require_viewer),
) -> Response:
"""Download a STIX 2.1 bundle covering every observed attacker."""
rows, ttp_by_attacker, obs_by_attacker = await _gather_fleet_data()
rows, ttp_by_attacker, obs_by_attacker, fp_by_ip = await _gather_fleet_data()
bundle = build_fleet_bundle(
rows=rows,
ttp_by_attacker=ttp_by_attacker,
observations_by_attacker=obs_by_attacker,
fingerprint_bounties_by_ip=fp_by_ip,
)
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
return Response(
@@ -64,11 +65,13 @@ async def _gather_fleet_data() -> tuple[
list[dict[str, Any]],
dict[str, list[dict[str, Any]]],
dict[str, list[dict[str, Any]]],
dict[str, list[dict[str, Any]]],
]:
import asyncio
rows, ttp_by_attacker, obs_by_attacker = await asyncio.gather(
rows, ttp_by_attacker, obs_by_attacker, fp_by_ip = await asyncio.gather(
repo.get_all_attackers_for_export(),
repo.get_all_ttp_rollups_for_export(),
repo.get_all_observations_for_export(),
repo.get_all_fingerprint_bounties_for_export(),
)
return rows, ttp_by_attacker, obs_by_attacker
return rows, ttp_by_attacker, obs_by_attacker, fp_by_ip

View File

@@ -115,6 +115,10 @@ class DummyRepo(BaseRepository):
await super().list_observations_by_attacker(attacker_uuid); return []
async def get_all_observations_for_export(self):
await super().get_all_observations_for_export(); return {}
async def get_fingerprint_bounties_by_ip(self, ip):
await super().get_fingerprint_bounties_by_ip(ip); return []
async def get_all_fingerprint_bounties_for_export(self):
await super().get_all_fingerprint_bounties_for_export(); return {}
async def get_attacker_uuid_by_ip(self, ip):
await super().get_attacker_uuid_by_ip(ip); return None
# TTP rollup surface (TTP_TAGGING.md)
@@ -261,6 +265,10 @@ async def test_base_repo_coverage():
await dr.list_observations_by_attacker("a")
with pytest.raises(NotImplementedError):
await dr.get_all_observations_for_export()
with pytest.raises(NotImplementedError):
await dr.get_fingerprint_bounties_by_ip("1.1.1.1")
with pytest.raises(NotImplementedError):
await dr.get_all_fingerprint_bounties_for_export()
with pytest.raises(NotImplementedError):
await dr.get_attacker_uuid_by_ip("1.1.1.1")
with pytest.raises(NotImplementedError):

View File

@@ -239,6 +239,100 @@ def test_kd_digraph_simhash_base64_input():
assert profile.kd_digraph_simhash == raw.hex()
def _fp_bounties() -> list[dict]:
return [
{
"payload": {
"fingerprint_type": "jarm",
"hash": "2ad2ad16d2ad2ad00042d42d000000f93d17e5fba64fc1c6f4cb080b9a5cf1e",
"target_ip": "1.2.3.4",
"target_port": "443",
}
},
{
"payload": {
"fingerprint_type": "http_quirks",
"order_hash": "abc123",
"order": ["Host", "User-Agent", "Accept"],
"casing_hash": "def456",
"casing_category": "title_case",
"stable_count": 3,
"tool_guess": "curl",
}
},
{
"payload": {
"fingerprint_type": "jarm",
"hash": "2ad2ad16d2ad2ad00042d42d000000f93d17e5fba64fc1c6f4cb080b9a5cf1e",
}
},
]
def test_fingerprint_bounties_jarm_in_protocol_fingerprints():
"""JARM hashes from bounties appear deduplicated in protocol_fingerprints."""
bundle = build_attacker_bundle(
attacker=_attacker(),
behavior=None, identity=None, intel=None,
technique_rollup=[], raw_tags=[], artifacts=[],
smtp_targets=[], observations=None,
fingerprint_bounties=_fp_bounties(),
)
ta = _get_ta(bundle)
assert ta.extensions, "expected extension block with fingerprint bounties"
ext = ta.extensions[ACTOR_FINGERPRINT_EXT_ID]
fp = ext.protocol_fingerprints
assert "jarm_hashes" in fp
assert len(fp["jarm_hashes"]) == 1, "duplicate JARM hash must be collapsed"
assert "http_quirks" in fp
assert fp["http_quirks"][0]["tool_guess"] == "curl"
assert fp["http_quirks"][0]["order"] == ["Host", "User-Agent", "Accept"]
def test_fingerprint_bounties_empty_produces_no_extension():
"""Empty fingerprint bounties with no other signal → no extension block."""
bundle = build_attacker_bundle(
attacker=_attacker(),
behavior=None, identity=None, intel=None,
technique_rollup=[], raw_tags=[], artifacts=[],
smtp_targets=[], observations=None,
fingerprint_bounties=[],
)
ta = _get_ta(bundle)
assert not getattr(ta, "extensions", None)
def test_behave_profile_has_characterizes_relationship():
"""When behave_profile is present the bundle contains a 'characterizes' Relationship."""
bundle = build_attacker_bundle(
attacker=_attacker(),
behavior=None, identity=None, intel=None,
technique_rollup=[], raw_tags=[], artifacts=[],
smtp_targets=[], observations=_obs(),
)
ta = _get_ta(bundle)
profile = next(o for o in bundle.objects if o.type == "x-decnet-behave-profile")
rels = [o for o in bundle.objects if o.type == "relationship"
and o.relationship_type == "characterizes"]
assert len(rels) == 1
rel = rels[0]
assert rel.source_ref == profile.id
assert rel.target_ref == ta.id
def test_no_behave_profile_no_characterizes_relationship():
"""Skinny attacker with no observations → no 'characterizes' relationship."""
bundle = build_attacker_bundle(
attacker=_attacker(),
behavior=None, identity=None, intel=None,
technique_rollup=[], raw_tags=[], artifacts=[],
smtp_targets=[], observations=None,
)
rels = [o for o in bundle.objects if o.type == "relationship"
and o.relationship_type == "characterizes"]
assert len(rels) == 0
def test_inter_decnet_round_trip():
"""Primary fidelity: stix2.parse restores typed objects, not bare dicts."""
ident = _identity()

View File

@@ -113,6 +113,7 @@ def _mock_repo(*, attacker=None, intel=None, rollup=None, tags=None,
m.list_smtp_targets = AsyncMock(return_value=smtp or [])
m.list_attacker_commands_deduped = AsyncMock(return_value=commands or [])
m.list_observations_by_attacker = AsyncMock(return_value=[])
m.get_fingerprint_bounties_by_ip = AsyncMock(return_value=[])
return m

View File

@@ -129,6 +129,7 @@ def _mock_repo(*, attacker=None, identity=None, intel=None,
m.list_smtp_targets = AsyncMock(return_value=smtp or [])
m.list_attacker_commands_deduped = AsyncMock(return_value=commands or [])
m.list_observations_by_attacker = AsyncMock(return_value=[])
m.get_fingerprint_bounties_by_ip = AsyncMock(return_value=[])
return m

View File

@@ -52,6 +52,7 @@ def _mock_repo(*, rows=None, ttp_by_attacker=None):
m.get_all_attackers_for_export = AsyncMock(return_value=rows or [])
m.get_all_ttp_rollups_for_export = AsyncMock(return_value=ttp_by_attacker or {})
m.get_all_observations_for_export = AsyncMock(return_value={})
m.get_all_fingerprint_bounties_for_export = AsyncMock(return_value={})
return m

View File

@@ -53,6 +53,7 @@ def _mock_repo(*, rows=None, ttp_by_attacker=None):
m.get_all_attackers_for_export = AsyncMock(return_value=rows or [])
m.get_all_ttp_rollups_for_export = AsyncMock(return_value=ttp_by_attacker or {})
m.get_all_observations_for_export = AsyncMock(return_value={})
m.get_all_fingerprint_bounties_for_export = AsyncMock(return_value={})
return m