feat(creds): future-proof Credential storage model
Replaces the opaque Bounty.bounty_type='credential' path with a
dedicated `credentials` table whose schema is forward-compatible
across every auth-bearing service in the fleet. Hoisted indexed
columns (secret_sha256, principal, service, attacker_ip) carry the
universal reuse-analytics signal; service-specific JSON keys ride
in `fields`. Cross-service reuse queries become an indexed lookup
on secret_sha256 instead of JSON_EXTRACT scans.
Schema decisions baked in (per ANTI):
- New `Credential` table, not extension to Bounty
- Hoisted `principal` column for cross-service principal-reuse
- Standardized JSON keys: every payload carries secret_b64 +
secret_printable + principal universally; service-specific extras
(user, domain, dn, mech, …) ride alongside
The auth-helper SD-block emits the new shape natively. The ingester
forks at _extract_bounty:
- Native shape (SSH/Telnet, future emitters): secret_b64 present →
direct upsert_credential
- Legacy shape (FTP/POP3/IMAP/SMTP today): username + password →
adapter synthesizes secret_{b64,sha256,printable} on the fly,
upserts into the same Credential table. Tracked as DEBT-039;
one-shot bridge until those service templates migrate.
Defense-in-depth across five layers (input validation):
- C helper: bytes outside [0x20, 0x7f) collapse to '?', RFC 5424
escape rules for \\, ", ]; b64 preserves exact bytes
- Ingester native branch: rejects malformed secret_b64 (regex), drops
the credential row but keeps the underlying Log
- Ingester legacy adapter: same printable-ASCII filter as the C
code; sha256 + b64 over the original utf-8 bytes (lossless, even
when secret_printable is sanitized)
- DB column caps with truncation warning; sha256 always over the
full pre-truncation bytes so reuse queries match across truncation
- JSON serialized with ensure_ascii=True so utf8mb4 columns stay
safe even with non-ASCII service-specific keys
Bounty.bounty_type='credential' is no longer written. Pre-v1: no
historical backfill; existing rows stay untouched but unused.
595 tests pass; new tests cover the model + repo (upsert dedup,
null-principal independence, cross-service reuse, filters), both
ingester branches, b64 validation, sanitization preserving the
fingerprinting signal in b64.
This commit is contained in:
@@ -17,46 +17,117 @@ import pytest
|
||||
|
||||
class TestExtractBounty:
|
||||
@pytest.mark.asyncio
|
||||
async def test_credential_extraction(self):
|
||||
async def test_credential_legacy_adapter(self):
|
||||
"""FTP/POP3/IMAP/SMTP shape (username + password) → upsert_credential."""
|
||||
from decnet.web.ingester import _extract_bounty
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.add_bounty = AsyncMock()
|
||||
mock_repo.upsert_credential = AsyncMock()
|
||||
log_data: dict = {
|
||||
"decky": "decky-01",
|
||||
"service": "ssh",
|
||||
"service": "ftp",
|
||||
"attacker_ip": "10.0.0.5",
|
||||
"fields": {"username": "admin", "password": "hunter2"},
|
||||
}
|
||||
await _extract_bounty(mock_repo, log_data)
|
||||
mock_repo.add_bounty.assert_awaited_once()
|
||||
bounty = mock_repo.add_bounty.call_args[0][0]
|
||||
assert bounty["bounty_type"] == "credential"
|
||||
assert bounty["payload"]["username"] == "admin"
|
||||
assert bounty["payload"]["password"] == "hunter2"
|
||||
mock_repo.upsert_credential.assert_awaited_once()
|
||||
cred = mock_repo.upsert_credential.call_args[0][0]
|
||||
assert cred["service"] == "ftp"
|
||||
assert cred["principal"] == "admin"
|
||||
assert cred["secret_printable"] == "hunter2"
|
||||
# b64 + sha256 computed over the original utf-8 bytes.
|
||||
import base64, hashlib
|
||||
assert cred["secret_b64"] == base64.b64encode(b"hunter2").decode()
|
||||
assert cred["secret_sha256"] == hashlib.sha256(b"hunter2").hexdigest()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_credential_native_shape(self):
|
||||
"""SSH/Telnet auth-helper shape (secret_b64) → upsert_credential."""
|
||||
from decnet.web.ingester import _extract_bounty
|
||||
import base64, hashlib
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.upsert_credential = AsyncMock()
|
||||
log_data: dict = {
|
||||
"decky": "decky-01",
|
||||
"service": "ssh",
|
||||
"attacker_ip": "10.0.0.5",
|
||||
"fields": {
|
||||
"username": "root",
|
||||
"principal": "root",
|
||||
"secret_printable": "hunter2",
|
||||
"secret_b64": base64.b64encode(b"hunter2").decode(),
|
||||
},
|
||||
}
|
||||
await _extract_bounty(mock_repo, log_data)
|
||||
mock_repo.upsert_credential.assert_awaited_once()
|
||||
cred = mock_repo.upsert_credential.call_args[0][0]
|
||||
assert cred["service"] == "ssh"
|
||||
assert cred["principal"] == "root"
|
||||
assert cred["secret_sha256"] == hashlib.sha256(b"hunter2").hexdigest()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_credential_native_invalid_b64_dropped(self):
|
||||
"""Malformed secret_b64 → row dropped with a warning, no upsert."""
|
||||
from decnet.web.ingester import _extract_bounty
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.upsert_credential = AsyncMock()
|
||||
log_data: dict = {
|
||||
"decky": "decky-01",
|
||||
"service": "ssh",
|
||||
"attacker_ip": "10.0.0.5",
|
||||
"fields": {"secret_b64": "not!base64!!"},
|
||||
}
|
||||
await _extract_bounty(mock_repo, log_data)
|
||||
mock_repo.upsert_credential.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_credential_legacy_sanitizes_nonprintable(self):
|
||||
"""Non-printable bytes in legacy password collapse to '?' in
|
||||
secret_printable; b64 + sha256 reflect the ORIGINAL bytes."""
|
||||
from decnet.web.ingester import _extract_bounty
|
||||
import base64, hashlib
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.upsert_credential = AsyncMock()
|
||||
# ANSI escape + NUL byte in the password.
|
||||
bad_pw = "\x1b[31mbad\x00trail"
|
||||
log_data: dict = {
|
||||
"decky": "decky-01",
|
||||
"service": "ftp",
|
||||
"attacker_ip": "10.0.0.5",
|
||||
"fields": {"username": "user", "password": bad_pw},
|
||||
}
|
||||
await _extract_bounty(mock_repo, log_data)
|
||||
cred = mock_repo.upsert_credential.call_args[0][0]
|
||||
# No 0x1b, no NUL — collapsed to '?'.
|
||||
assert "\x1b" not in cred["secret_printable"]
|
||||
assert "\x00" not in cred["secret_printable"]
|
||||
# Original bytes survive in b64 + sha256.
|
||||
raw = bad_pw.encode("utf-8")
|
||||
assert base64.b64decode(cred["secret_b64"]) == raw
|
||||
assert cred["secret_sha256"] == hashlib.sha256(raw).hexdigest()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_fields_skips(self):
|
||||
from decnet.web.ingester import _extract_bounty
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.add_bounty = AsyncMock()
|
||||
mock_repo.upsert_credential = AsyncMock()
|
||||
await _extract_bounty(mock_repo, {"decky": "x"})
|
||||
mock_repo.add_bounty.assert_not_awaited()
|
||||
mock_repo.upsert_credential.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fields_not_dict_skips(self):
|
||||
from decnet.web.ingester import _extract_bounty
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.add_bounty = AsyncMock()
|
||||
mock_repo.upsert_credential = AsyncMock()
|
||||
await _extract_bounty(mock_repo, {"fields": "not-a-dict"})
|
||||
mock_repo.add_bounty.assert_not_awaited()
|
||||
mock_repo.upsert_credential.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_password_skips(self):
|
||||
from decnet.web.ingester import _extract_bounty
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.add_bounty = AsyncMock()
|
||||
mock_repo.upsert_credential = AsyncMock()
|
||||
await _extract_bounty(mock_repo, {"fields": {"username": "admin"}})
|
||||
mock_repo.add_bounty.assert_not_awaited()
|
||||
mock_repo.upsert_credential.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_username_skips(self):
|
||||
|
||||
Reference in New Issue
Block a user