diff --git a/decnet/ttp/misp_export.py b/decnet/ttp/misp_export.py index 6863fbfa..3daa6b69 100644 --- a/decnet/ttp/misp_export.py +++ b/decnet/ttp/misp_export.py @@ -48,6 +48,7 @@ def build_attacker_misp_event( smtp_targets: list[dict[str, Any]], commands: list[str] | None = None, observations: list[dict[str, Any]] | None = None, + fingerprint_bounties: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: """Return a MISP event dict for *attacker*. @@ -65,6 +66,7 @@ def build_attacker_misp_event( smtp_targets=smtp_targets, commands=commands, observations=observations, + fingerprint_bounties=fingerprint_bounties, ) return _parse_bundle(bundle) @@ -73,6 +75,7 @@ def build_fleet_misp_collection( rows: list[dict[str, Any]], ttp_by_attacker: dict[str, list[dict[str, Any]]], observations_by_attacker: dict[str, list[dict[str, Any]]] | None = None, + fingerprint_bounties_by_ip: dict[str, list[dict[str, Any]]] | None = None, ) -> dict[str, Any]: """Return a MISP collection dict with one event per attacker in *rows*. @@ -84,6 +87,7 @@ def build_fleet_misp_collection( """ events: list[dict[str, Any]] = [] obs_map = observations_by_attacker or {} + fp_map = fingerprint_bounties_by_ip or {} for row in rows: raw_cmds = row.get("commands") or [] if isinstance(raw_cmds, str): @@ -107,6 +111,7 @@ def build_fleet_misp_collection( smtp_targets=[], commands=cmds, observations=obs_map.get(row["uuid"]), + fingerprint_bounties=fp_map.get(row.get("ip", ""), []), ) event = _parse_bundle(bundle) if event: diff --git a/decnet/web/db/repository.py b/decnet/web/db/repository.py index 6d003f13..59fd3d4e 100644 --- a/decnet/web/db/repository.py +++ b/decnet/web/db/repository.py @@ -394,6 +394,27 @@ class BaseRepository(ABC): """ raise NotImplementedError + async def get_fingerprint_bounties_by_ip( + self, ip: str + ) -> list[dict[str, Any]]: + """Return all fingerprint bounties for *ip* with parsed payload dicts. + + Filters to ``bounty_type='fingerprint'``. Each returned dict has the + standard Bounty columns plus a decoded ``payload`` dict. + """ + raise NotImplementedError + + async def get_all_fingerprint_bounties_for_export( + self, + ) -> dict[str, list[dict[str, Any]]]: + """Return ``{attacker_ip: [bounty_dict, ...]}`` for all fingerprint + bounties in the database. + + Used by fleet exports to attach per-attacker fingerprint bounties + (JARM hashes, HTTP quirks) without N+1 queries. + """ + raise NotImplementedError + async def upsert_observed_attachment( self, *, diff --git a/decnet/web/db/sqlmodel_repo/bounties.py b/decnet/web/db/sqlmodel_repo/bounties.py index 94c7ac56..e7910f26 100644 --- a/decnet/web/db/sqlmodel_repo/bounties.py +++ b/decnet/web/db/sqlmodel_repo/bounties.py @@ -141,6 +141,40 @@ class BountiesMixin(_MixinBase): grouped[item.attacker_ip].append(d) return dict(grouped) + async def get_fingerprint_bounties_by_ip(self, ip: str) -> List[dict[str, Any]]: + async with self._session() as session: + result = await session.execute( + select(Bounty) + .where(Bounty.attacker_ip == ip, Bounty.bounty_type == "fingerprint") + .order_by(asc(Bounty.timestamp)) + ) + out: List[dict[str, Any]] = [] + for item in result.scalars().all(): + d = item.model_dump(mode="json") + try: + d["payload"] = json.loads(d["payload"]) + except (json.JSONDecodeError, TypeError): + pass + out.append(d) + return out + + async def get_all_fingerprint_bounties_for_export(self) -> dict[str, List[dict[str, Any]]]: + async with self._session() as session: + result = await session.execute( + select(Bounty) + .where(Bounty.bounty_type == "fingerprint") + .order_by(asc(Bounty.timestamp)) + ) + grouped: dict[str, List[dict[str, Any]]] = defaultdict(list) + for item in result.scalars().all(): + d = item.model_dump(mode="json") + try: + d["payload"] = json.loads(d["payload"]) + except (json.JSONDecodeError, TypeError): + pass + grouped[item.attacker_ip].append(d) + return dict(grouped) + async def count_probe_relays(self, attacker_ip: str, decky: str) -> int: """Return how many probe_relay bounties exist for this (attacker_ip, decky) pair.""" async with self._session() as session: diff --git a/decnet/web/router/attackers/api_export_attacker_misp.py b/decnet/web/router/attackers/api_export_attacker_misp.py index beb9e9ff..7697986a 100644 --- a/decnet/web/router/attackers/api_export_attacker_misp.py +++ b/decnet/web/router/attackers/api_export_attacker_misp.py @@ -66,6 +66,7 @@ async def api_export_attacker_misp( repo.list_smtp_targets(uuid), repo.list_attacker_commands_deduped(uuid), repo.list_observations_by_attacker(uuid), + repo.get_fingerprint_bounties_by_ip(attacker["ip"]), ) behavior = cast(dict[str, Any] | None, results[0]) identity = cast(dict[str, Any] | None, results[1]) @@ -76,6 +77,7 @@ async def api_export_attacker_misp( smtp_targets = cast(list[dict[str, Any]], results[6]) commands = cast(list[str], results[7]) observations = cast(list[dict[str, Any]], results[8]) + fingerprint_bounties = cast(list[dict[str, Any]], results[9]) event = build_attacker_misp_event( attacker=attacker, @@ -91,6 +93,7 @@ async def api_export_attacker_misp( smtp_targets=smtp_targets, commands=commands, observations=observations, + fingerprint_bounties=fingerprint_bounties, ) return Response( content=json.dumps(event, default=str), diff --git a/decnet/web/router/attackers/api_export_attacker_stix.py b/decnet/web/router/attackers/api_export_attacker_stix.py index 70670aaf..0c97472e 100644 --- a/decnet/web/router/attackers/api_export_attacker_stix.py +++ b/decnet/web/router/attackers/api_export_attacker_stix.py @@ -70,6 +70,7 @@ async def api_export_attacker_stix( repo.list_smtp_targets(uuid), repo.list_attacker_commands_deduped(uuid), repo.list_observations_by_attacker(uuid), + repo.get_fingerprint_bounties_by_ip(attacker["ip"]), ) behavior = cast(dict[str, Any] | None, results[0]) identity = cast(dict[str, Any] | None, results[1]) @@ -80,6 +81,7 @@ async def api_export_attacker_stix( smtp_targets = cast(list[dict[str, Any]], results[6]) commands = cast(list[str], results[7]) observations = cast(list[dict[str, Any]], results[8]) + fingerprint_bounties = cast(list[dict[str, Any]], results[9]) bundle = build_attacker_bundle( attacker=attacker, @@ -95,6 +97,7 @@ async def api_export_attacker_stix( smtp_targets=smtp_targets, commands=commands, observations=observations, + fingerprint_bounties=fingerprint_bounties, ) return Response( content=bundle.serialize(pretty=True, indent=2), diff --git a/decnet/web/router/attackers/api_export_attackers_misp.py b/decnet/web/router/attackers/api_export_attackers_misp.py index 1d644793..22df69f2 100644 --- a/decnet/web/router/attackers/api_export_attackers_misp.py +++ b/decnet/web/router/attackers/api_export_attackers_misp.py @@ -43,15 +43,17 @@ async def api_export_attackers_misp( user: dict[str, Any] = Depends(require_viewer), ) -> Response: """Download a MISP collection JSON covering every observed attacker.""" - rows, ttp_by_attacker, obs_by_attacker = await asyncio.gather( + rows, ttp_by_attacker, obs_by_attacker, fp_by_ip = await asyncio.gather( repo.get_all_attackers_for_export(), repo.get_all_ttp_rollups_for_export(), repo.get_all_observations_for_export(), + repo.get_all_fingerprint_bounties_for_export(), ) collection = build_fleet_misp_collection( rows=rows, ttp_by_attacker=ttp_by_attacker, observations_by_attacker=obs_by_attacker, + fingerprint_bounties_by_ip=fp_by_ip, ) ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") return Response( diff --git a/decnet/web/router/attackers/api_export_attackers_stix.py b/decnet/web/router/attackers/api_export_attackers_stix.py index 01b60cce..af4aa9a9 100644 --- a/decnet/web/router/attackers/api_export_attackers_stix.py +++ b/decnet/web/router/attackers/api_export_attackers_stix.py @@ -42,11 +42,12 @@ async def api_export_attackers_stix( user: dict[str, Any] = Depends(require_viewer), ) -> Response: """Download a STIX 2.1 bundle covering every observed attacker.""" - rows, ttp_by_attacker, obs_by_attacker = await _gather_fleet_data() + rows, ttp_by_attacker, obs_by_attacker, fp_by_ip = await _gather_fleet_data() bundle = build_fleet_bundle( rows=rows, ttp_by_attacker=ttp_by_attacker, observations_by_attacker=obs_by_attacker, + fingerprint_bounties_by_ip=fp_by_ip, ) ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") return Response( @@ -64,11 +65,13 @@ async def _gather_fleet_data() -> tuple[ list[dict[str, Any]], dict[str, list[dict[str, Any]]], dict[str, list[dict[str, Any]]], + dict[str, list[dict[str, Any]]], ]: import asyncio - rows, ttp_by_attacker, obs_by_attacker = await asyncio.gather( + rows, ttp_by_attacker, obs_by_attacker, fp_by_ip = await asyncio.gather( repo.get_all_attackers_for_export(), repo.get_all_ttp_rollups_for_export(), repo.get_all_observations_for_export(), + repo.get_all_fingerprint_bounties_for_export(), ) - return rows, ttp_by_attacker, obs_by_attacker + return rows, ttp_by_attacker, obs_by_attacker, fp_by_ip diff --git a/tests/db/test_base_repo.py b/tests/db/test_base_repo.py index 12735941..5ff04cc7 100644 --- a/tests/db/test_base_repo.py +++ b/tests/db/test_base_repo.py @@ -115,6 +115,10 @@ class DummyRepo(BaseRepository): await super().list_observations_by_attacker(attacker_uuid); return [] async def get_all_observations_for_export(self): await super().get_all_observations_for_export(); return {} + async def get_fingerprint_bounties_by_ip(self, ip): + await super().get_fingerprint_bounties_by_ip(ip); return [] + async def get_all_fingerprint_bounties_for_export(self): + await super().get_all_fingerprint_bounties_for_export(); return {} async def get_attacker_uuid_by_ip(self, ip): await super().get_attacker_uuid_by_ip(ip); return None # TTP rollup surface (TTP_TAGGING.md) @@ -261,6 +265,10 @@ async def test_base_repo_coverage(): await dr.list_observations_by_attacker("a") with pytest.raises(NotImplementedError): await dr.get_all_observations_for_export() + with pytest.raises(NotImplementedError): + await dr.get_fingerprint_bounties_by_ip("1.1.1.1") + with pytest.raises(NotImplementedError): + await dr.get_all_fingerprint_bounties_for_export() with pytest.raises(NotImplementedError): await dr.get_attacker_uuid_by_ip("1.1.1.1") with pytest.raises(NotImplementedError): diff --git a/tests/ttp/test_stix_export_threat_actor_extensions.py b/tests/ttp/test_stix_export_threat_actor_extensions.py index 0455cb90..eaa3104d 100644 --- a/tests/ttp/test_stix_export_threat_actor_extensions.py +++ b/tests/ttp/test_stix_export_threat_actor_extensions.py @@ -239,6 +239,100 @@ def test_kd_digraph_simhash_base64_input(): assert profile.kd_digraph_simhash == raw.hex() +def _fp_bounties() -> list[dict]: + return [ + { + "payload": { + "fingerprint_type": "jarm", + "hash": "2ad2ad16d2ad2ad00042d42d000000f93d17e5fba64fc1c6f4cb080b9a5cf1e", + "target_ip": "1.2.3.4", + "target_port": "443", + } + }, + { + "payload": { + "fingerprint_type": "http_quirks", + "order_hash": "abc123", + "order": ["Host", "User-Agent", "Accept"], + "casing_hash": "def456", + "casing_category": "title_case", + "stable_count": 3, + "tool_guess": "curl", + } + }, + { + "payload": { + "fingerprint_type": "jarm", + "hash": "2ad2ad16d2ad2ad00042d42d000000f93d17e5fba64fc1c6f4cb080b9a5cf1e", + } + }, + ] + + +def test_fingerprint_bounties_jarm_in_protocol_fingerprints(): + """JARM hashes from bounties appear deduplicated in protocol_fingerprints.""" + bundle = build_attacker_bundle( + attacker=_attacker(), + behavior=None, identity=None, intel=None, + technique_rollup=[], raw_tags=[], artifacts=[], + smtp_targets=[], observations=None, + fingerprint_bounties=_fp_bounties(), + ) + ta = _get_ta(bundle) + assert ta.extensions, "expected extension block with fingerprint bounties" + ext = ta.extensions[ACTOR_FINGERPRINT_EXT_ID] + fp = ext.protocol_fingerprints + assert "jarm_hashes" in fp + assert len(fp["jarm_hashes"]) == 1, "duplicate JARM hash must be collapsed" + assert "http_quirks" in fp + assert fp["http_quirks"][0]["tool_guess"] == "curl" + assert fp["http_quirks"][0]["order"] == ["Host", "User-Agent", "Accept"] + + +def test_fingerprint_bounties_empty_produces_no_extension(): + """Empty fingerprint bounties with no other signal → no extension block.""" + bundle = build_attacker_bundle( + attacker=_attacker(), + behavior=None, identity=None, intel=None, + technique_rollup=[], raw_tags=[], artifacts=[], + smtp_targets=[], observations=None, + fingerprint_bounties=[], + ) + ta = _get_ta(bundle) + assert not getattr(ta, "extensions", None) + + +def test_behave_profile_has_characterizes_relationship(): + """When behave_profile is present the bundle contains a 'characterizes' Relationship.""" + bundle = build_attacker_bundle( + attacker=_attacker(), + behavior=None, identity=None, intel=None, + technique_rollup=[], raw_tags=[], artifacts=[], + smtp_targets=[], observations=_obs(), + ) + ta = _get_ta(bundle) + profile = next(o for o in bundle.objects if o.type == "x-decnet-behave-profile") + rels = [o for o in bundle.objects if o.type == "relationship" + and o.relationship_type == "characterizes"] + assert len(rels) == 1 + rel = rels[0] + assert rel.source_ref == profile.id + assert rel.target_ref == ta.id + + +def test_no_behave_profile_no_characterizes_relationship(): + """Skinny attacker with no observations → no 'characterizes' relationship.""" + bundle = build_attacker_bundle( + attacker=_attacker(), + behavior=None, identity=None, intel=None, + technique_rollup=[], raw_tags=[], artifacts=[], + smtp_targets=[], observations=None, + ) + rels = [o for o in bundle.objects if o.type == "relationship" + and o.relationship_type == "characterizes"] + assert len(rels) == 0 + + def test_inter_decnet_round_trip(): """Primary fidelity: stix2.parse restores typed objects, not bare dicts.""" ident = _identity() diff --git a/tests/web/test_api_export_attacker_misp.py b/tests/web/test_api_export_attacker_misp.py index 21f7dc8a..e6e79ffd 100644 --- a/tests/web/test_api_export_attacker_misp.py +++ b/tests/web/test_api_export_attacker_misp.py @@ -113,6 +113,7 @@ def _mock_repo(*, attacker=None, intel=None, rollup=None, tags=None, m.list_smtp_targets = AsyncMock(return_value=smtp or []) m.list_attacker_commands_deduped = AsyncMock(return_value=commands or []) m.list_observations_by_attacker = AsyncMock(return_value=[]) + m.get_fingerprint_bounties_by_ip = AsyncMock(return_value=[]) return m diff --git a/tests/web/test_api_export_attacker_stix.py b/tests/web/test_api_export_attacker_stix.py index 5763fc81..ca028cfa 100644 --- a/tests/web/test_api_export_attacker_stix.py +++ b/tests/web/test_api_export_attacker_stix.py @@ -129,6 +129,7 @@ def _mock_repo(*, attacker=None, identity=None, intel=None, m.list_smtp_targets = AsyncMock(return_value=smtp or []) m.list_attacker_commands_deduped = AsyncMock(return_value=commands or []) m.list_observations_by_attacker = AsyncMock(return_value=[]) + m.get_fingerprint_bounties_by_ip = AsyncMock(return_value=[]) return m diff --git a/tests/web/test_api_export_attackers_misp.py b/tests/web/test_api_export_attackers_misp.py index 6a16dc77..ceac6309 100644 --- a/tests/web/test_api_export_attackers_misp.py +++ b/tests/web/test_api_export_attackers_misp.py @@ -52,6 +52,7 @@ def _mock_repo(*, rows=None, ttp_by_attacker=None): m.get_all_attackers_for_export = AsyncMock(return_value=rows or []) m.get_all_ttp_rollups_for_export = AsyncMock(return_value=ttp_by_attacker or {}) m.get_all_observations_for_export = AsyncMock(return_value={}) + m.get_all_fingerprint_bounties_for_export = AsyncMock(return_value={}) return m diff --git a/tests/web/test_api_export_attackers_stix.py b/tests/web/test_api_export_attackers_stix.py index 7e708a07..a50ba4ea 100644 --- a/tests/web/test_api_export_attackers_stix.py +++ b/tests/web/test_api_export_attackers_stix.py @@ -53,6 +53,7 @@ def _mock_repo(*, rows=None, ttp_by_attacker=None): m.get_all_attackers_for_export = AsyncMock(return_value=rows or []) m.get_all_ttp_rollups_for_export = AsyncMock(return_value=ttp_by_attacker or {}) m.get_all_observations_for_export = AsyncMock(return_value={}) + m.get_all_fingerprint_bounties_for_export = AsyncMock(return_value={}) return m