feat(prober-cert): roll up fingerprints onto AttackerIdentity
Brings the federation-gossip columns on AttackerIdentity to life —
ja3_hashes, hassh_hashes, and the new tls_cert_sha256 — by projecting
the union of every member observation's fingerprints JSON onto the
identity at clusterer create / link / merge time.
- decnet/profiler/identity_rollup.py: pure extract_fp_summaries()
reads the production bounty shape (payload.fingerprint_type +
payload.{ja3,hash,cert_sha256}) and returns deduped+sorted JSON
list[str] per family, or None when a family has no signal so the
column stays NULL instead of '[]'.
- BaseRepository.update_identity_fingerprints + SQLModel impl: one
idempotent write that overwrites the three summary columns and
bumps updated_at.
- ConnectedComponentsClusterer: after every per-component
reconciliation (fresh-create OR existing-merge+link), recomputes
and writes the rollup for the target identity. Wrapped in a
best-effort helper so a write failure logs but never breaks the
tick.
- Tests: extract_fp_summaries unit (dedup, sort determinism,
unknown types ignored, malformed JSON, nested-stringified
payloads, non-string values); end-to-end clusterer ticks
populate the columns on create + on later observation links;
no-fingerprint clusters keep the columns NULL.
This commit is contained in:
@@ -140,14 +140,41 @@ async def repo(tmp_path):
|
||||
|
||||
async def _seed_attacker(
|
||||
repo, ip: str, *,
|
||||
ja3: str | None = None, hassh: str | None = None, asn: int | None = None,
|
||||
ja3: str | None = None,
|
||||
hassh: str | None = None,
|
||||
asn: int | None = None,
|
||||
cert_sha256: str | None = None,
|
||||
) -> str:
|
||||
now = datetime.now(timezone.utc)
|
||||
fingerprints = []
|
||||
# Two-shape fingerprint payload:
|
||||
# - the "kind" entries feed the clusterer's from_attacker_row
|
||||
# (test-fixture shape, line ~115 of connected_components.py)
|
||||
# - the "bounty_type/payload" entries feed identity_rollup's
|
||||
# extract_fp_summaries (production shape, written by the
|
||||
# profiler from real bounty rows). Both shapes coexist in
|
||||
# the same JSON list so the same seed exercises clustering
|
||||
# AND the identity-column rollup.
|
||||
fingerprints: list[dict] = []
|
||||
if ja3:
|
||||
fingerprints.append({"kind": "ja3", "hash": ja3})
|
||||
fingerprints.append({
|
||||
"bounty_type": "fingerprint",
|
||||
"payload": {"fingerprint_type": "ja3", "ja3": ja3},
|
||||
})
|
||||
if hassh:
|
||||
fingerprints.append({"kind": "hassh", "hash": hassh})
|
||||
fingerprints.append({
|
||||
"bounty_type": "fingerprint",
|
||||
"payload": {"fingerprint_type": "hassh_server", "hash": hassh},
|
||||
})
|
||||
if cert_sha256:
|
||||
fingerprints.append({
|
||||
"bounty_type": "fingerprint",
|
||||
"payload": {
|
||||
"fingerprint_type": "tls_certificate",
|
||||
"cert_sha256": cert_sha256,
|
||||
},
|
||||
})
|
||||
return await repo.upsert_attacker({
|
||||
"ip": ip,
|
||||
"first_seen": now,
|
||||
@@ -377,6 +404,70 @@ async def test_tick_links_new_observation_to_existing_identity(repo):
|
||||
assert d in linked_uuids
|
||||
|
||||
|
||||
# ─── identity fingerprint rollup ───────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_tick_rolls_up_fingerprint_columns_on_create(repo):
|
||||
"""A fresh-component tick must populate ja3_hashes / hassh_hashes /
|
||||
tls_cert_sha256 on the newly-minted identity row, deduplicated and
|
||||
sorted across all member observations."""
|
||||
await _seed_attacker(
|
||||
repo, "1.1.1.1", ja3="ja3-x", hassh="hassh-y", cert_sha256="ab" * 32,
|
||||
)
|
||||
await _seed_attacker(
|
||||
repo, "2.2.2.2", ja3="ja3-x", hassh="hassh-y", cert_sha256="cd" * 32,
|
||||
)
|
||||
c = ConnectedComponentsClusterer()
|
||||
result = await c.tick(repo)
|
||||
assert len(result.identities_formed) == 1
|
||||
identity_uuid = result.identities_formed[0]["identity_uuid"]
|
||||
|
||||
rows = {i["uuid"]: i for i in await repo.list_all_identities()}
|
||||
identity = rows[identity_uuid]
|
||||
assert json.loads(identity["ja3_hashes"]) == ["ja3-x"]
|
||||
assert json.loads(identity["hassh_hashes"]) == ["hassh-y"]
|
||||
assert json.loads(identity["tls_cert_sha256"]) == sorted(["ab" * 32, "cd" * 32])
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_tick_rolls_up_fingerprints_on_link(repo):
|
||||
"""When a new observation links into an existing identity, the
|
||||
rollup must reflect any new cert SHA-256 it brings."""
|
||||
await _seed_attacker(
|
||||
repo, "1.1.1.1", ja3="ja3-x", cert_sha256="ab" * 32,
|
||||
)
|
||||
c = ConnectedComponentsClusterer()
|
||||
first = await c.tick(repo)
|
||||
identity_uuid = first.identities_formed[0]["identity_uuid"]
|
||||
|
||||
# New observation, same JA3, fresh cert.
|
||||
await _seed_attacker(
|
||||
repo, "2.2.2.2", ja3="ja3-x", cert_sha256="cd" * 32,
|
||||
)
|
||||
await c.tick(repo)
|
||||
|
||||
rows = {i["uuid"]: i for i in await repo.list_all_identities()}
|
||||
identity = rows[identity_uuid]
|
||||
assert json.loads(identity["tls_cert_sha256"]) == sorted(["ab" * 32, "cd" * 32])
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_tick_leaves_columns_null_when_no_fingerprints(repo):
|
||||
"""Two attackers with NO fingerprint signal cluster as separate
|
||||
singletons; their identity rows must keep all rollup columns NULL
|
||||
(not "[]" — NULL distinguishes 'no signal yet' from 'known empty')."""
|
||||
await _seed_attacker(repo, "1.1.1.1")
|
||||
await _seed_attacker(repo, "2.2.2.2")
|
||||
c = ConnectedComponentsClusterer()
|
||||
await c.tick(repo)
|
||||
|
||||
for identity in await repo.list_all_identities():
|
||||
assert identity["ja3_hashes"] is None
|
||||
assert identity["hassh_hashes"] is None
|
||||
assert identity["tls_cert_sha256"] is None
|
||||
|
||||
|
||||
# ─── fixture-bound assertions (in-memory) ──────────────────────────────────
|
||||
|
||||
|
||||
|
||||
141
tests/profiler/test_identity_rollup.py
Normal file
141
tests/profiler/test_identity_rollup.py
Normal file
@@ -0,0 +1,141 @@
|
||||
"""Tests for ``decnet.profiler.identity_rollup.extract_fp_summaries``.
|
||||
|
||||
Pure unit tests against the production bounty shape that
|
||||
``decnet.profiler.worker._build_record`` writes into
|
||||
``Attacker.fingerprints`` — a list of ``{bounty_type, payload, ...}``
|
||||
dicts where the meaningful data lives under ``payload.fingerprint_type``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from decnet.profiler.identity_rollup import extract_fp_summaries
|
||||
|
||||
|
||||
def _bounty(fp_type: str, **payload_extras) -> dict:
|
||||
"""Build a bounty dict shaped the way the profiler writes it."""
|
||||
return {
|
||||
"bounty_type": "fingerprint",
|
||||
"payload": {"fingerprint_type": fp_type, **payload_extras},
|
||||
}
|
||||
|
||||
|
||||
def _row_with(*entries) -> dict:
|
||||
return {"fingerprints": json.dumps(list(entries))}
|
||||
|
||||
|
||||
class TestExtractFpSummaries:
|
||||
|
||||
def test_empty_input_returns_all_none(self):
|
||||
result = extract_fp_summaries([])
|
||||
assert result == {
|
||||
"ja3_hashes": None,
|
||||
"hassh_hashes": None,
|
||||
"tls_cert_sha256": None,
|
||||
}
|
||||
|
||||
def test_single_row_single_cert(self):
|
||||
row = _row_with(_bounty("tls_certificate", cert_sha256="ab" * 32))
|
||||
result = extract_fp_summaries([row])
|
||||
assert result["ja3_hashes"] is None
|
||||
assert result["hassh_hashes"] is None
|
||||
assert json.loads(result["tls_cert_sha256"]) == ["ab" * 32]
|
||||
|
||||
def test_dedupe_across_rows(self):
|
||||
sha = "ab" * 32
|
||||
a = _row_with(_bounty("tls_certificate", cert_sha256=sha))
|
||||
b = _row_with(_bounty("tls_certificate", cert_sha256=sha))
|
||||
result = extract_fp_summaries([a, b])
|
||||
assert json.loads(result["tls_cert_sha256"]) == [sha]
|
||||
|
||||
def test_sorted_output_is_deterministic(self):
|
||||
a = _row_with(
|
||||
_bounty("tls_certificate", cert_sha256="ff" * 32),
|
||||
_bounty("tls_certificate", cert_sha256="11" * 32),
|
||||
_bounty("tls_certificate", cert_sha256="aa" * 32),
|
||||
)
|
||||
result = extract_fp_summaries([a])
|
||||
# Same input twice must produce byte-identical output.
|
||||
assert result == extract_fp_summaries([a])
|
||||
assert json.loads(result["tls_cert_sha256"]) == sorted(
|
||||
["ff" * 32, "11" * 32, "aa" * 32]
|
||||
)
|
||||
|
||||
def test_all_three_families_at_once(self):
|
||||
row = _row_with(
|
||||
_bounty("ja3", ja3="ja3-abc"),
|
||||
_bounty("hassh_server", hash="hassh-def"),
|
||||
_bounty("tls_certificate", cert_sha256="ab" * 32),
|
||||
)
|
||||
result = extract_fp_summaries([row])
|
||||
assert json.loads(result["ja3_hashes"]) == ["ja3-abc"]
|
||||
assert json.loads(result["hassh_hashes"]) == ["hassh-def"]
|
||||
assert json.loads(result["tls_cert_sha256"]) == ["ab" * 32]
|
||||
|
||||
def test_unknown_fingerprint_type_ignored(self):
|
||||
# tcpfp / ja4l / http_quirks have no rollup column yet; they
|
||||
# must not pollute the three families that do.
|
||||
row = _row_with(
|
||||
_bounty("tcpfp", hash="tcpfp-x"),
|
||||
_bounty("ja4l", ja4l="ja4l-y"),
|
||||
_bounty("http_quirks", quirks="..."),
|
||||
)
|
||||
result = extract_fp_summaries([row])
|
||||
assert result["ja3_hashes"] is None
|
||||
assert result["hassh_hashes"] is None
|
||||
assert result["tls_cert_sha256"] is None
|
||||
|
||||
def test_missing_payload_key_skipped(self):
|
||||
# tls_certificate bounty shaped like a sniffer-only payload
|
||||
# (no cert_sha256). Must not crash, must not record an entry.
|
||||
row = _row_with({
|
||||
"bounty_type": "fingerprint",
|
||||
"payload": {"fingerprint_type": "tls_certificate", "subject_cn": "x"},
|
||||
})
|
||||
result = extract_fp_summaries([row])
|
||||
assert result["tls_cert_sha256"] is None
|
||||
|
||||
def test_malformed_fingerprints_json_returns_all_none(self):
|
||||
result = extract_fp_summaries([{"fingerprints": "not json"}])
|
||||
assert all(v is None for v in result.values())
|
||||
|
||||
def test_missing_fingerprints_field_returns_all_none(self):
|
||||
result = extract_fp_summaries([{"some_other_field": True}])
|
||||
assert all(v is None for v in result.values())
|
||||
|
||||
def test_payload_as_string_is_json_decoded(self):
|
||||
# Defensive: some legacy storage may have nested-stringified payloads.
|
||||
row = {
|
||||
"fingerprints": json.dumps([{
|
||||
"bounty_type": "fingerprint",
|
||||
"payload": json.dumps({
|
||||
"fingerprint_type": "tls_certificate",
|
||||
"cert_sha256": "cd" * 32,
|
||||
}),
|
||||
}]),
|
||||
}
|
||||
result = extract_fp_summaries([row])
|
||||
assert json.loads(result["tls_cert_sha256"]) == ["cd" * 32]
|
||||
|
||||
def test_non_string_hash_values_skipped(self):
|
||||
row = _row_with({
|
||||
"bounty_type": "fingerprint",
|
||||
"payload": {"fingerprint_type": "tls_certificate", "cert_sha256": 12345},
|
||||
})
|
||||
result = extract_fp_summaries([row])
|
||||
assert result["tls_cert_sha256"] is None
|
||||
|
||||
def test_dedup_across_many_rows_with_overlap(self):
|
||||
rows = [
|
||||
_row_with(_bounty("ja3", ja3="ja3-shared")),
|
||||
_row_with(
|
||||
_bounty("ja3", ja3="ja3-shared"),
|
||||
_bounty("ja3", ja3="ja3-second"),
|
||||
),
|
||||
_row_with(_bounty("ja3", ja3="ja3-third")),
|
||||
]
|
||||
result = extract_fp_summaries(rows)
|
||||
assert json.loads(result["ja3_hashes"]) == sorted(
|
||||
["ja3-shared", "ja3-second", "ja3-third"]
|
||||
)
|
||||
Reference in New Issue
Block a user