feat(stix_export): wire fingerprint bounties through all endpoints + tests
Remaining files from the fingerprint-bounties + characterizes-SRO commit: misp_export, repository, bounties mixin, all 4 router endpoints, and test suite updates. Prerequisite: previous commit added _extract_fingerprint_bounty_data and the stix_export changes.
This commit is contained in:
@@ -48,6 +48,7 @@ def build_attacker_misp_event(
|
||||
smtp_targets: list[dict[str, Any]],
|
||||
commands: list[str] | None = None,
|
||||
observations: list[dict[str, Any]] | None = None,
|
||||
fingerprint_bounties: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return a MISP event dict for *attacker*.
|
||||
|
||||
@@ -65,6 +66,7 @@ def build_attacker_misp_event(
|
||||
smtp_targets=smtp_targets,
|
||||
commands=commands,
|
||||
observations=observations,
|
||||
fingerprint_bounties=fingerprint_bounties,
|
||||
)
|
||||
return _parse_bundle(bundle)
|
||||
|
||||
@@ -73,6 +75,7 @@ def build_fleet_misp_collection(
|
||||
rows: list[dict[str, Any]],
|
||||
ttp_by_attacker: dict[str, list[dict[str, Any]]],
|
||||
observations_by_attacker: dict[str, list[dict[str, Any]]] | None = None,
|
||||
fingerprint_bounties_by_ip: dict[str, list[dict[str, Any]]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return a MISP collection dict with one event per attacker in *rows*.
|
||||
|
||||
@@ -84,6 +87,7 @@ def build_fleet_misp_collection(
|
||||
"""
|
||||
events: list[dict[str, Any]] = []
|
||||
obs_map = observations_by_attacker or {}
|
||||
fp_map = fingerprint_bounties_by_ip or {}
|
||||
for row in rows:
|
||||
raw_cmds = row.get("commands") or []
|
||||
if isinstance(raw_cmds, str):
|
||||
@@ -107,6 +111,7 @@ def build_fleet_misp_collection(
|
||||
smtp_targets=[],
|
||||
commands=cmds,
|
||||
observations=obs_map.get(row["uuid"]),
|
||||
fingerprint_bounties=fp_map.get(row.get("ip", ""), []),
|
||||
)
|
||||
event = _parse_bundle(bundle)
|
||||
if event:
|
||||
|
||||
@@ -394,6 +394,27 @@ class BaseRepository(ABC):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_fingerprint_bounties_by_ip(
|
||||
self, ip: str
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return all fingerprint bounties for *ip* with parsed payload dicts.
|
||||
|
||||
Filters to ``bounty_type='fingerprint'``. Each returned dict has the
|
||||
standard Bounty columns plus a decoded ``payload`` dict.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_all_fingerprint_bounties_for_export(
|
||||
self,
|
||||
) -> dict[str, list[dict[str, Any]]]:
|
||||
"""Return ``{attacker_ip: [bounty_dict, ...]}`` for all fingerprint
|
||||
bounties in the database.
|
||||
|
||||
Used by fleet exports to attach per-attacker fingerprint bounties
|
||||
(JARM hashes, HTTP quirks) without N+1 queries.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def upsert_observed_attachment(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -141,6 +141,40 @@ class BountiesMixin(_MixinBase):
|
||||
grouped[item.attacker_ip].append(d)
|
||||
return dict(grouped)
|
||||
|
||||
async def get_fingerprint_bounties_by_ip(self, ip: str) -> List[dict[str, Any]]:
|
||||
async with self._session() as session:
|
||||
result = await session.execute(
|
||||
select(Bounty)
|
||||
.where(Bounty.attacker_ip == ip, Bounty.bounty_type == "fingerprint")
|
||||
.order_by(asc(Bounty.timestamp))
|
||||
)
|
||||
out: List[dict[str, Any]] = []
|
||||
for item in result.scalars().all():
|
||||
d = item.model_dump(mode="json")
|
||||
try:
|
||||
d["payload"] = json.loads(d["payload"])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
out.append(d)
|
||||
return out
|
||||
|
||||
async def get_all_fingerprint_bounties_for_export(self) -> dict[str, List[dict[str, Any]]]:
|
||||
async with self._session() as session:
|
||||
result = await session.execute(
|
||||
select(Bounty)
|
||||
.where(Bounty.bounty_type == "fingerprint")
|
||||
.order_by(asc(Bounty.timestamp))
|
||||
)
|
||||
grouped: dict[str, List[dict[str, Any]]] = defaultdict(list)
|
||||
for item in result.scalars().all():
|
||||
d = item.model_dump(mode="json")
|
||||
try:
|
||||
d["payload"] = json.loads(d["payload"])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
grouped[item.attacker_ip].append(d)
|
||||
return dict(grouped)
|
||||
|
||||
async def count_probe_relays(self, attacker_ip: str, decky: str) -> int:
|
||||
"""Return how many probe_relay bounties exist for this (attacker_ip, decky) pair."""
|
||||
async with self._session() as session:
|
||||
|
||||
@@ -66,6 +66,7 @@ async def api_export_attacker_misp(
|
||||
repo.list_smtp_targets(uuid),
|
||||
repo.list_attacker_commands_deduped(uuid),
|
||||
repo.list_observations_by_attacker(uuid),
|
||||
repo.get_fingerprint_bounties_by_ip(attacker["ip"]),
|
||||
)
|
||||
behavior = cast(dict[str, Any] | None, results[0])
|
||||
identity = cast(dict[str, Any] | None, results[1])
|
||||
@@ -76,6 +77,7 @@ async def api_export_attacker_misp(
|
||||
smtp_targets = cast(list[dict[str, Any]], results[6])
|
||||
commands = cast(list[str], results[7])
|
||||
observations = cast(list[dict[str, Any]], results[8])
|
||||
fingerprint_bounties = cast(list[dict[str, Any]], results[9])
|
||||
|
||||
event = build_attacker_misp_event(
|
||||
attacker=attacker,
|
||||
@@ -91,6 +93,7 @@ async def api_export_attacker_misp(
|
||||
smtp_targets=smtp_targets,
|
||||
commands=commands,
|
||||
observations=observations,
|
||||
fingerprint_bounties=fingerprint_bounties,
|
||||
)
|
||||
return Response(
|
||||
content=json.dumps(event, default=str),
|
||||
|
||||
@@ -70,6 +70,7 @@ async def api_export_attacker_stix(
|
||||
repo.list_smtp_targets(uuid),
|
||||
repo.list_attacker_commands_deduped(uuid),
|
||||
repo.list_observations_by_attacker(uuid),
|
||||
repo.get_fingerprint_bounties_by_ip(attacker["ip"]),
|
||||
)
|
||||
behavior = cast(dict[str, Any] | None, results[0])
|
||||
identity = cast(dict[str, Any] | None, results[1])
|
||||
@@ -80,6 +81,7 @@ async def api_export_attacker_stix(
|
||||
smtp_targets = cast(list[dict[str, Any]], results[6])
|
||||
commands = cast(list[str], results[7])
|
||||
observations = cast(list[dict[str, Any]], results[8])
|
||||
fingerprint_bounties = cast(list[dict[str, Any]], results[9])
|
||||
|
||||
bundle = build_attacker_bundle(
|
||||
attacker=attacker,
|
||||
@@ -95,6 +97,7 @@ async def api_export_attacker_stix(
|
||||
smtp_targets=smtp_targets,
|
||||
commands=commands,
|
||||
observations=observations,
|
||||
fingerprint_bounties=fingerprint_bounties,
|
||||
)
|
||||
return Response(
|
||||
content=bundle.serialize(pretty=True, indent=2),
|
||||
|
||||
@@ -43,15 +43,17 @@ async def api_export_attackers_misp(
|
||||
user: dict[str, Any] = Depends(require_viewer),
|
||||
) -> Response:
|
||||
"""Download a MISP collection JSON covering every observed attacker."""
|
||||
rows, ttp_by_attacker, obs_by_attacker = await asyncio.gather(
|
||||
rows, ttp_by_attacker, obs_by_attacker, fp_by_ip = await asyncio.gather(
|
||||
repo.get_all_attackers_for_export(),
|
||||
repo.get_all_ttp_rollups_for_export(),
|
||||
repo.get_all_observations_for_export(),
|
||||
repo.get_all_fingerprint_bounties_for_export(),
|
||||
)
|
||||
collection = build_fleet_misp_collection(
|
||||
rows=rows,
|
||||
ttp_by_attacker=ttp_by_attacker,
|
||||
observations_by_attacker=obs_by_attacker,
|
||||
fingerprint_bounties_by_ip=fp_by_ip,
|
||||
)
|
||||
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
return Response(
|
||||
|
||||
@@ -42,11 +42,12 @@ async def api_export_attackers_stix(
|
||||
user: dict[str, Any] = Depends(require_viewer),
|
||||
) -> Response:
|
||||
"""Download a STIX 2.1 bundle covering every observed attacker."""
|
||||
rows, ttp_by_attacker, obs_by_attacker = await _gather_fleet_data()
|
||||
rows, ttp_by_attacker, obs_by_attacker, fp_by_ip = await _gather_fleet_data()
|
||||
bundle = build_fleet_bundle(
|
||||
rows=rows,
|
||||
ttp_by_attacker=ttp_by_attacker,
|
||||
observations_by_attacker=obs_by_attacker,
|
||||
fingerprint_bounties_by_ip=fp_by_ip,
|
||||
)
|
||||
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
return Response(
|
||||
@@ -64,11 +65,13 @@ async def _gather_fleet_data() -> tuple[
|
||||
list[dict[str, Any]],
|
||||
dict[str, list[dict[str, Any]]],
|
||||
dict[str, list[dict[str, Any]]],
|
||||
dict[str, list[dict[str, Any]]],
|
||||
]:
|
||||
import asyncio
|
||||
rows, ttp_by_attacker, obs_by_attacker = await asyncio.gather(
|
||||
rows, ttp_by_attacker, obs_by_attacker, fp_by_ip = await asyncio.gather(
|
||||
repo.get_all_attackers_for_export(),
|
||||
repo.get_all_ttp_rollups_for_export(),
|
||||
repo.get_all_observations_for_export(),
|
||||
repo.get_all_fingerprint_bounties_for_export(),
|
||||
)
|
||||
return rows, ttp_by_attacker, obs_by_attacker
|
||||
return rows, ttp_by_attacker, obs_by_attacker, fp_by_ip
|
||||
|
||||
@@ -115,6 +115,10 @@ class DummyRepo(BaseRepository):
|
||||
await super().list_observations_by_attacker(attacker_uuid); return []
|
||||
async def get_all_observations_for_export(self):
|
||||
await super().get_all_observations_for_export(); return {}
|
||||
async def get_fingerprint_bounties_by_ip(self, ip):
|
||||
await super().get_fingerprint_bounties_by_ip(ip); return []
|
||||
async def get_all_fingerprint_bounties_for_export(self):
|
||||
await super().get_all_fingerprint_bounties_for_export(); return {}
|
||||
async def get_attacker_uuid_by_ip(self, ip):
|
||||
await super().get_attacker_uuid_by_ip(ip); return None
|
||||
# TTP rollup surface (TTP_TAGGING.md)
|
||||
@@ -261,6 +265,10 @@ async def test_base_repo_coverage():
|
||||
await dr.list_observations_by_attacker("a")
|
||||
with pytest.raises(NotImplementedError):
|
||||
await dr.get_all_observations_for_export()
|
||||
with pytest.raises(NotImplementedError):
|
||||
await dr.get_fingerprint_bounties_by_ip("1.1.1.1")
|
||||
with pytest.raises(NotImplementedError):
|
||||
await dr.get_all_fingerprint_bounties_for_export()
|
||||
with pytest.raises(NotImplementedError):
|
||||
await dr.get_attacker_uuid_by_ip("1.1.1.1")
|
||||
with pytest.raises(NotImplementedError):
|
||||
|
||||
@@ -239,6 +239,100 @@ def test_kd_digraph_simhash_base64_input():
|
||||
assert profile.kd_digraph_simhash == raw.hex()
|
||||
|
||||
|
||||
def _fp_bounties() -> list[dict]:
|
||||
return [
|
||||
{
|
||||
"payload": {
|
||||
"fingerprint_type": "jarm",
|
||||
"hash": "2ad2ad16d2ad2ad00042d42d000000f93d17e5fba64fc1c6f4cb080b9a5cf1e",
|
||||
"target_ip": "1.2.3.4",
|
||||
"target_port": "443",
|
||||
}
|
||||
},
|
||||
{
|
||||
"payload": {
|
||||
"fingerprint_type": "http_quirks",
|
||||
"order_hash": "abc123",
|
||||
"order": ["Host", "User-Agent", "Accept"],
|
||||
"casing_hash": "def456",
|
||||
"casing_category": "title_case",
|
||||
"stable_count": 3,
|
||||
"tool_guess": "curl",
|
||||
}
|
||||
},
|
||||
{
|
||||
"payload": {
|
||||
"fingerprint_type": "jarm",
|
||||
"hash": "2ad2ad16d2ad2ad00042d42d000000f93d17e5fba64fc1c6f4cb080b9a5cf1e",
|
||||
}
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_fingerprint_bounties_jarm_in_protocol_fingerprints():
|
||||
"""JARM hashes from bounties appear deduplicated in protocol_fingerprints."""
|
||||
bundle = build_attacker_bundle(
|
||||
attacker=_attacker(),
|
||||
behavior=None, identity=None, intel=None,
|
||||
technique_rollup=[], raw_tags=[], artifacts=[],
|
||||
smtp_targets=[], observations=None,
|
||||
fingerprint_bounties=_fp_bounties(),
|
||||
)
|
||||
ta = _get_ta(bundle)
|
||||
assert ta.extensions, "expected extension block with fingerprint bounties"
|
||||
ext = ta.extensions[ACTOR_FINGERPRINT_EXT_ID]
|
||||
fp = ext.protocol_fingerprints
|
||||
assert "jarm_hashes" in fp
|
||||
assert len(fp["jarm_hashes"]) == 1, "duplicate JARM hash must be collapsed"
|
||||
assert "http_quirks" in fp
|
||||
assert fp["http_quirks"][0]["tool_guess"] == "curl"
|
||||
assert fp["http_quirks"][0]["order"] == ["Host", "User-Agent", "Accept"]
|
||||
|
||||
|
||||
def test_fingerprint_bounties_empty_produces_no_extension():
|
||||
"""Empty fingerprint bounties with no other signal → no extension block."""
|
||||
bundle = build_attacker_bundle(
|
||||
attacker=_attacker(),
|
||||
behavior=None, identity=None, intel=None,
|
||||
technique_rollup=[], raw_tags=[], artifacts=[],
|
||||
smtp_targets=[], observations=None,
|
||||
fingerprint_bounties=[],
|
||||
)
|
||||
ta = _get_ta(bundle)
|
||||
assert not getattr(ta, "extensions", None)
|
||||
|
||||
|
||||
def test_behave_profile_has_characterizes_relationship():
|
||||
"""When behave_profile is present the bundle contains a 'characterizes' Relationship."""
|
||||
bundle = build_attacker_bundle(
|
||||
attacker=_attacker(),
|
||||
behavior=None, identity=None, intel=None,
|
||||
technique_rollup=[], raw_tags=[], artifacts=[],
|
||||
smtp_targets=[], observations=_obs(),
|
||||
)
|
||||
ta = _get_ta(bundle)
|
||||
profile = next(o for o in bundle.objects if o.type == "x-decnet-behave-profile")
|
||||
rels = [o for o in bundle.objects if o.type == "relationship"
|
||||
and o.relationship_type == "characterizes"]
|
||||
assert len(rels) == 1
|
||||
rel = rels[0]
|
||||
assert rel.source_ref == profile.id
|
||||
assert rel.target_ref == ta.id
|
||||
|
||||
|
||||
def test_no_behave_profile_no_characterizes_relationship():
|
||||
"""Skinny attacker with no observations → no 'characterizes' relationship."""
|
||||
bundle = build_attacker_bundle(
|
||||
attacker=_attacker(),
|
||||
behavior=None, identity=None, intel=None,
|
||||
technique_rollup=[], raw_tags=[], artifacts=[],
|
||||
smtp_targets=[], observations=None,
|
||||
)
|
||||
rels = [o for o in bundle.objects if o.type == "relationship"
|
||||
and o.relationship_type == "characterizes"]
|
||||
assert len(rels) == 0
|
||||
|
||||
|
||||
def test_inter_decnet_round_trip():
|
||||
"""Primary fidelity: stix2.parse restores typed objects, not bare dicts."""
|
||||
ident = _identity()
|
||||
|
||||
@@ -113,6 +113,7 @@ def _mock_repo(*, attacker=None, intel=None, rollup=None, tags=None,
|
||||
m.list_smtp_targets = AsyncMock(return_value=smtp or [])
|
||||
m.list_attacker_commands_deduped = AsyncMock(return_value=commands or [])
|
||||
m.list_observations_by_attacker = AsyncMock(return_value=[])
|
||||
m.get_fingerprint_bounties_by_ip = AsyncMock(return_value=[])
|
||||
return m
|
||||
|
||||
|
||||
|
||||
@@ -129,6 +129,7 @@ def _mock_repo(*, attacker=None, identity=None, intel=None,
|
||||
m.list_smtp_targets = AsyncMock(return_value=smtp or [])
|
||||
m.list_attacker_commands_deduped = AsyncMock(return_value=commands or [])
|
||||
m.list_observations_by_attacker = AsyncMock(return_value=[])
|
||||
m.get_fingerprint_bounties_by_ip = AsyncMock(return_value=[])
|
||||
return m
|
||||
|
||||
|
||||
|
||||
@@ -52,6 +52,7 @@ def _mock_repo(*, rows=None, ttp_by_attacker=None):
|
||||
m.get_all_attackers_for_export = AsyncMock(return_value=rows or [])
|
||||
m.get_all_ttp_rollups_for_export = AsyncMock(return_value=ttp_by_attacker or {})
|
||||
m.get_all_observations_for_export = AsyncMock(return_value={})
|
||||
m.get_all_fingerprint_bounties_for_export = AsyncMock(return_value={})
|
||||
return m
|
||||
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ def _mock_repo(*, rows=None, ttp_by_attacker=None):
|
||||
m.get_all_attackers_for_export = AsyncMock(return_value=rows or [])
|
||||
m.get_all_ttp_rollups_for_export = AsyncMock(return_value=ttp_by_attacker or {})
|
||||
m.get_all_observations_for_export = AsyncMock(return_value={})
|
||||
m.get_all_fingerprint_bounties_for_export = AsyncMock(return_value={})
|
||||
return m
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user