merge: testing → main (reconcile 2-week divergence)
This commit is contained in:
0
tests/intel/__init__.py
Normal file
0
tests/intel/__init__.py
Normal file
109
tests/intel/test_abuseipdb.py
Normal file
109
tests/intel/test_abuseipdb.py
Normal file
@@ -0,0 +1,109 @@
|
||||
"""Unit tests for the AbuseIPDB provider."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from decnet.intel.abuseipdb import AbuseIPDBProvider, _score_to_verdict
|
||||
|
||||
|
||||
def _install_transport(handler) -> list[httpx.Request]:
|
||||
captured: list[httpx.Request] = []
|
||||
|
||||
async def _wrapped(request: httpx.Request) -> httpx.Response:
|
||||
captured.append(request)
|
||||
return await handler(request)
|
||||
|
||||
transport = httpx.MockTransport(_wrapped)
|
||||
from decnet.intel import abuseipdb as mod
|
||||
|
||||
def _factory():
|
||||
return httpx.AsyncClient(
|
||||
transport=transport,
|
||||
headers={"User-Agent": "curl/7.88.1"},
|
||||
)
|
||||
|
||||
mod.stealth_client = _factory # type: ignore[assignment]
|
||||
return captured
|
||||
|
||||
|
||||
def test_score_thresholds():
|
||||
assert _score_to_verdict(0) == "benign"
|
||||
assert _score_to_verdict(24) == "benign"
|
||||
assert _score_to_verdict(25) == "suspicious"
|
||||
assert _score_to_verdict(74) == "suspicious"
|
||||
assert _score_to_verdict(75) == "malicious"
|
||||
assert _score_to_verdict(100) == "malicious"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_missing_api_key_returns_error_no_egress(monkeypatch):
|
||||
monkeypatch.delenv("DECNET_ABUSEIPDB_API_KEY", raising=False)
|
||||
captured = _install_transport(
|
||||
lambda r: (_ for _ in ()).throw(AssertionError("must not egress"))
|
||||
)
|
||||
provider = AbuseIPDBProvider()
|
||||
result = await provider.lookup("1.2.3.4")
|
||||
assert result.error == "DECNET_ABUSEIPDB_API_KEY not configured"
|
||||
assert result.column_updates == {}
|
||||
assert captured == [] # no request made
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_high_score_maps_to_malicious(monkeypatch):
|
||||
monkeypatch.setenv("DECNET_ABUSEIPDB_API_KEY", "k3y")
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"data": {
|
||||
"ipAddress": "1.2.3.4",
|
||||
"abuseConfidenceScore": 92,
|
||||
"totalReports": 41,
|
||||
"countryCode": "RU",
|
||||
}},
|
||||
)
|
||||
|
||||
captured = _install_transport(handler)
|
||||
provider = AbuseIPDBProvider()
|
||||
result = await provider.lookup("1.2.3.4")
|
||||
assert result.verdict == "malicious"
|
||||
assert result.column_updates["abuseipdb_score"] == 92
|
||||
raw = json.loads(result.column_updates["abuseipdb_raw"])
|
||||
assert raw["countryCode"] == "RU"
|
||||
# Key header sent, query params correct.
|
||||
req = captured[0]
|
||||
assert req.headers["key"] == "k3y"
|
||||
assert "ipAddress=1.2.3.4" in str(req.url)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_low_score_maps_to_benign(monkeypatch):
|
||||
monkeypatch.setenv("DECNET_ABUSEIPDB_API_KEY", "k3y")
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200, json={"data": {"abuseConfidenceScore": 0}},
|
||||
)
|
||||
|
||||
_install_transport(handler)
|
||||
provider = AbuseIPDBProvider()
|
||||
result = await provider.lookup("8.8.8.8")
|
||||
assert result.verdict == "benign"
|
||||
assert result.column_updates["abuseipdb_score"] == 0
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_429_returns_error(monkeypatch):
|
||||
monkeypatch.setenv("DECNET_ABUSEIPDB_API_KEY", "k3y")
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(429)
|
||||
|
||||
_install_transport(handler)
|
||||
provider = AbuseIPDBProvider()
|
||||
result = await provider.lookup("1.1.1.1")
|
||||
assert result.error == "HTTP 429"
|
||||
assert result.column_updates == {}
|
||||
129
tests/intel/test_attacker_intel_repo.py
Normal file
129
tests/intel/test_attacker_intel_repo.py
Normal file
@@ -0,0 +1,129 @@
|
||||
"""
|
||||
Round-trip tests for the ``attacker_intel`` table and its repo helpers.
|
||||
|
||||
Covers:
|
||||
* empty-write upsert path (attacker_uuid as canonical key)
|
||||
* per-provider partial update preserves untouched columns
|
||||
* JSON-blob deserialization on read
|
||||
* TTL bookkeeping (cached_at + expires_at) round-trips intact
|
||||
* ``get_unenriched_attackers`` returns ``{"uuid", "ip"}`` pairs and
|
||||
selects fresh + stale rows while skipping cached ones
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from decnet.web.db.factory import get_repository
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def repo(tmp_path):
|
||||
r = get_repository(db_path=str(tmp_path / "attacker_intel.db"))
|
||||
await r.initialize()
|
||||
return r
|
||||
|
||||
|
||||
async def _seed_attacker(repo, ip: str) -> str:
|
||||
"""Seed an attackers row and return its UUID (the FK target)."""
|
||||
now = datetime.now(timezone.utc)
|
||||
return await repo.upsert_attacker(
|
||||
{"ip": ip, "first_seen": now, "last_seen": now, "event_count": 1}
|
||||
)
|
||||
|
||||
|
||||
def _intel_payload(
|
||||
*, attacker_uuid: str, ip: str, ttl_hours: int = 24, **overrides
|
||||
) -> dict:
|
||||
now = datetime.now(timezone.utc)
|
||||
base = {
|
||||
"attacker_uuid": attacker_uuid,
|
||||
"attacker_ip": ip,
|
||||
"cached_at": now,
|
||||
"expires_at": now + timedelta(hours=ttl_hours),
|
||||
}
|
||||
base.update(overrides)
|
||||
return base
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_empty_upsert_writes_minimal_row(repo):
|
||||
a_uuid = await _seed_attacker(repo, "1.2.3.4")
|
||||
row_uuid = await repo.upsert_attacker_intel(
|
||||
_intel_payload(attacker_uuid=a_uuid, ip="1.2.3.4")
|
||||
)
|
||||
assert row_uuid
|
||||
|
||||
row = await repo.get_attacker_intel_by_uuid(a_uuid)
|
||||
assert row is not None
|
||||
assert row["attacker_uuid"] == a_uuid
|
||||
assert row["attacker_ip"] == "1.2.3.4"
|
||||
assert row["uuid"] == row_uuid
|
||||
assert row["schema_version"] == 1
|
||||
# All per-provider verdicts default to None.
|
||||
assert row["greynoise_classification"] is None
|
||||
assert row["abuseipdb_score"] is None
|
||||
assert row["feodo_listed"] is None
|
||||
assert row["threatfox_listed"] is None
|
||||
assert row["aggregate_verdict"] is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_partial_provider_update_preserves_others(repo):
|
||||
a_uuid = await _seed_attacker(repo, "9.9.9.9")
|
||||
# First pass: GreyNoise responds, others lag.
|
||||
first_uuid = await repo.upsert_attacker_intel(
|
||||
_intel_payload(
|
||||
attacker_uuid=a_uuid, ip="9.9.9.9",
|
||||
greynoise_classification="malicious",
|
||||
greynoise_raw='{"classification":"malicious"}',
|
||||
greynoise_queried_at=datetime.now(timezone.utc),
|
||||
)
|
||||
)
|
||||
# Second pass: AbuseIPDB lands. Re-upsert MUST NOT clobber GreyNoise
|
||||
# columns — the worker passes only the new fields.
|
||||
second_uuid = await repo.upsert_attacker_intel(
|
||||
_intel_payload(
|
||||
attacker_uuid=a_uuid, ip="9.9.9.9",
|
||||
abuseipdb_score=85,
|
||||
abuseipdb_raw='{"abuseConfidenceScore":85}',
|
||||
abuseipdb_queried_at=datetime.now(timezone.utc),
|
||||
)
|
||||
)
|
||||
assert first_uuid == second_uuid # same row keyed on attacker_uuid
|
||||
|
||||
row = await repo.get_attacker_intel_by_uuid(a_uuid)
|
||||
assert row["greynoise_classification"] == "malicious"
|
||||
assert row["greynoise_raw"] == {"classification": "malicious"}
|
||||
assert row["abuseipdb_score"] == 85
|
||||
assert row["abuseipdb_raw"] == {"abuseConfidenceScore": 85}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_get_missing_returns_none(repo):
|
||||
assert await repo.get_attacker_intel_by_uuid("nonexistent-uuid") is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_unenriched_returns_uuid_ip_pairs(repo):
|
||||
fresh_uuid = await _seed_attacker(repo, "10.0.0.1")
|
||||
stale_uuid = await _seed_attacker(repo, "10.0.0.2")
|
||||
new_uuid = await _seed_attacker(repo, "10.0.0.3")
|
||||
|
||||
# 10.0.0.1 has fresh intel (not due for refresh).
|
||||
await repo.upsert_attacker_intel(
|
||||
_intel_payload(attacker_uuid=fresh_uuid, ip="10.0.0.1", ttl_hours=24)
|
||||
)
|
||||
# 10.0.0.2 has stale intel (already expired).
|
||||
await repo.upsert_attacker_intel(
|
||||
_intel_payload(attacker_uuid=stale_uuid, ip="10.0.0.2", ttl_hours=-1)
|
||||
)
|
||||
# 10.0.0.3 has no intel row at all.
|
||||
|
||||
pending = await repo.get_unenriched_attackers(limit=10)
|
||||
by_uuid = {entry["uuid"]: entry["ip"] for entry in pending}
|
||||
|
||||
assert fresh_uuid not in by_uuid # fresh, skipped
|
||||
assert by_uuid.get(stale_uuid) == "10.0.0.2" # stale, queue it
|
||||
assert by_uuid.get(new_uuid) == "10.0.0.3" # never enriched
|
||||
57
tests/intel/test_factory.py
Normal file
57
tests/intel/test_factory.py
Normal file
@@ -0,0 +1,57 @@
|
||||
"""Tests for the intel provider factory.
|
||||
|
||||
The factory returns a **list** of configured providers (not a singleton
|
||||
like :mod:`decnet.geoip.factory`). Coverage:
|
||||
|
||||
* disabled master switch returns ``[]``
|
||||
* empty provider list returns ``[]``
|
||||
* unknown provider name raises ``ValueError`` (typo guard)
|
||||
* trimming + case-insensitivity of the providers env var
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from decnet.intel.factory import get_intel_providers
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_env(monkeypatch):
|
||||
# Disable real providers — concrete impls land in later commits, but
|
||||
# the factory tests should pass against whatever subset exists today
|
||||
# via empty/unknown lists.
|
||||
for key in (
|
||||
"DECNET_INTEL_ENABLED",
|
||||
"DECNET_INTEL_PROVIDERS",
|
||||
"DECNET_GREYNOISE_API_KEY",
|
||||
"DECNET_ABUSEIPDB_API_KEY",
|
||||
"DECNET_THREATFOX_API_KEY",
|
||||
):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
|
||||
def test_disabled_returns_empty(monkeypatch):
|
||||
monkeypatch.setenv("DECNET_INTEL_ENABLED", "false")
|
||||
monkeypatch.setenv("DECNET_INTEL_PROVIDERS", "greynoise")
|
||||
assert get_intel_providers() == []
|
||||
|
||||
|
||||
def test_empty_provider_list_returns_empty(monkeypatch):
|
||||
monkeypatch.setenv("DECNET_INTEL_PROVIDERS", "")
|
||||
assert get_intel_providers() == []
|
||||
|
||||
|
||||
def test_unknown_provider_name_raises(monkeypatch):
|
||||
monkeypatch.setenv("DECNET_INTEL_PROVIDERS", "definitely-not-real")
|
||||
with pytest.raises(ValueError, match="Unknown intel provider"):
|
||||
get_intel_providers()
|
||||
|
||||
|
||||
def test_whitespace_and_case_normalised(monkeypatch):
|
||||
# The factory imports concrete provider modules lazily; this test only
|
||||
# asserts that case+whitespace normalization doesn't trip the lookup.
|
||||
# We use an unknown name (which would also be unknown if not lowercased)
|
||||
# to exercise the path without requiring provider impls to exist yet.
|
||||
monkeypatch.setenv("DECNET_INTEL_PROVIDERS", " Mystery , ")
|
||||
with pytest.raises(ValueError, match="mystery"):
|
||||
get_intel_providers()
|
||||
99
tests/intel/test_feodo.py
Normal file
99
tests/intel/test_feodo.py
Normal file
@@ -0,0 +1,99 @@
|
||||
"""Unit tests for the abuse.ch Feodo Tracker provider.
|
||||
|
||||
Bulk-feed semantics: one HTTP fetch loads the in-memory set, all
|
||||
subsequent ``lookup`` calls hit memory. We assert:
|
||||
|
||||
* a fresh provider triggers exactly one refresh, then answers from cache
|
||||
* a listed IP returns verdict='malicious' with the upstream record
|
||||
* an unlisted IP returns verdict=None (absence ≠ benign)
|
||||
* a feed fetch failure is reported as an error, not silently swallowed
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from decnet.intel.feodo import FeodoProvider
|
||||
|
||||
|
||||
def _install_transport(handler) -> list[httpx.Request]:
|
||||
captured: list[httpx.Request] = []
|
||||
|
||||
async def _wrapped(request: httpx.Request) -> httpx.Response:
|
||||
captured.append(request)
|
||||
return await handler(request)
|
||||
|
||||
transport = httpx.MockTransport(_wrapped)
|
||||
from decnet.intel import feodo as mod
|
||||
|
||||
def _factory(*, timeout: float = 20.0):
|
||||
return httpx.AsyncClient(
|
||||
transport=transport,
|
||||
headers={"User-Agent": "curl/7.88.1"},
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
mod.stealth_client = _factory # type: ignore[assignment]
|
||||
return captured
|
||||
|
||||
|
||||
_FEED = [
|
||||
{"ip_address": "9.9.9.9", "port": 443, "malware": "TrickBot"},
|
||||
{"ip_address": "10.10.10.10", "port": 80, "malware": "Emotet"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_listed_ip_yields_malicious_verdict():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=_FEED)
|
||||
|
||||
captured = _install_transport(handler)
|
||||
provider = FeodoProvider(refresh_interval_s=999.0)
|
||||
|
||||
result = await provider.lookup("9.9.9.9")
|
||||
assert result.verdict == "malicious"
|
||||
assert result.column_updates["feodo_listed"] is True
|
||||
raw = json.loads(result.column_updates["feodo_raw"])
|
||||
assert raw["malware"] == "TrickBot"
|
||||
assert len(captured) == 1
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_subsequent_lookups_dont_refetch():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=_FEED)
|
||||
|
||||
captured = _install_transport(handler)
|
||||
provider = FeodoProvider(refresh_interval_s=999.0)
|
||||
|
||||
await provider.lookup("9.9.9.9")
|
||||
await provider.lookup("10.10.10.10")
|
||||
await provider.lookup("not-listed.example")
|
||||
assert len(captured) == 1 # one refresh, three answers
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_unlisted_ip_returns_no_verdict():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=_FEED)
|
||||
|
||||
_install_transport(handler)
|
||||
provider = FeodoProvider(refresh_interval_s=999.0)
|
||||
result = await provider.lookup("1.2.3.4")
|
||||
assert result.verdict is None
|
||||
assert result.column_updates["feodo_listed"] is False
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_feed_failure_reports_error():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(503)
|
||||
|
||||
_install_transport(handler)
|
||||
provider = FeodoProvider(refresh_interval_s=999.0)
|
||||
result = await provider.lookup("1.2.3.4")
|
||||
assert result.error == "HTTP 503"
|
||||
assert result.column_updates == {}
|
||||
136
tests/intel/test_greynoise.py
Normal file
136
tests/intel/test_greynoise.py
Normal file
@@ -0,0 +1,136 @@
|
||||
"""Unit tests for the GreyNoise Community provider.
|
||||
|
||||
Mocks httpx via ``MockTransport`` and asserts:
|
||||
|
||||
* request URL + headers (API key when present, none when absent)
|
||||
* malicious / benign / suspicious classification → verdict mapping
|
||||
* 404 → verdict='unknown' with no error (cache the absence)
|
||||
* non-200/404 → error populated, no column writes
|
||||
* network exception → error populated
|
||||
* the row never advertises DECNET in the egress UA
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from decnet.intel.greynoise import GreyNoiseProvider
|
||||
|
||||
|
||||
def _install_transport(provider: GreyNoiseProvider, handler) -> list[httpx.Request]:
|
||||
"""Patch ``stealth_client`` so it returns a client wired to ``handler``."""
|
||||
captured: list[httpx.Request] = []
|
||||
|
||||
async def _wrapped(request: httpx.Request) -> httpx.Response:
|
||||
captured.append(request)
|
||||
return await handler(request)
|
||||
|
||||
transport = httpx.MockTransport(_wrapped)
|
||||
|
||||
from decnet.intel import greynoise as gn_mod
|
||||
|
||||
def _factory():
|
||||
return httpx.AsyncClient(
|
||||
transport=transport,
|
||||
headers={"User-Agent": "curl/7.88.1"},
|
||||
)
|
||||
|
||||
gn_mod.stealth_client = _factory # type: ignore[assignment]
|
||||
return captured
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_malicious_classification_maps_to_verdict():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"ip": "1.2.3.4",
|
||||
"noise": True,
|
||||
"classification": "malicious",
|
||||
"name": "Mirai-like",
|
||||
},
|
||||
)
|
||||
|
||||
provider = GreyNoiseProvider()
|
||||
captured = _install_transport(provider, handler)
|
||||
|
||||
result = await provider.lookup("1.2.3.4")
|
||||
assert result.error is None
|
||||
assert result.verdict == "malicious"
|
||||
assert result.column_updates["greynoise_classification"] == "malicious"
|
||||
raw = json.loads(result.column_updates["greynoise_raw"])
|
||||
assert raw["name"] == "Mirai-like"
|
||||
assert "1.2.3.4" in str(captured[0].url)
|
||||
# No DECNET label leaks in the UA.
|
||||
assert "decnet" not in captured[0].headers["user-agent"].lower()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_api_key_is_sent_when_configured(monkeypatch):
|
||||
monkeypatch.setenv("DECNET_GREYNOISE_API_KEY", "k3y-abc")
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json={"classification": "benign"})
|
||||
|
||||
provider = GreyNoiseProvider()
|
||||
captured = _install_transport(provider, handler)
|
||||
|
||||
await provider.lookup("8.8.8.8")
|
||||
assert captured[0].headers.get("key") == "k3y-abc"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_no_api_key_means_no_header(monkeypatch):
|
||||
monkeypatch.delenv("DECNET_GREYNOISE_API_KEY", raising=False)
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json={"classification": "benign"})
|
||||
|
||||
provider = GreyNoiseProvider()
|
||||
captured = _install_transport(provider, handler)
|
||||
|
||||
await provider.lookup("8.8.8.8")
|
||||
assert "key" not in captured[0].headers
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_404_caches_unknown_without_error():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(404, json={"message": "IP not observed"})
|
||||
|
||||
provider = GreyNoiseProvider()
|
||||
_install_transport(provider, handler)
|
||||
|
||||
result = await provider.lookup("10.0.0.5")
|
||||
assert result.error is None
|
||||
assert result.verdict == "unknown"
|
||||
assert result.column_updates["greynoise_classification"] == "unknown"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_429_returns_error_no_writes():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(429)
|
||||
|
||||
provider = GreyNoiseProvider()
|
||||
_install_transport(provider, handler)
|
||||
|
||||
result = await provider.lookup("1.1.1.1")
|
||||
assert result.error == "HTTP 429"
|
||||
assert result.column_updates == {}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_network_failure_becomes_error():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ConnectError("upstream unreachable")
|
||||
|
||||
provider = GreyNoiseProvider()
|
||||
_install_transport(provider, handler)
|
||||
|
||||
result = await provider.lookup("1.1.1.1")
|
||||
assert result.error and result.error.startswith("network:")
|
||||
assert result.column_updates == {}
|
||||
65
tests/intel/test_stealth_http.py
Normal file
65
tests/intel/test_stealth_http.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""Stealth-egress HTTP client must NOT advertise DECNET.
|
||||
|
||||
Captures the request that the client emits (using httpx's MockTransport)
|
||||
and asserts the User-Agent never contains a DECNET marker. This is the
|
||||
most important contract on the file — every threat-intel egress path
|
||||
inherits it.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from decnet.net.http import DEFAULT_STEALTH_USER_AGENT, stealth_client
|
||||
|
||||
|
||||
_FORBIDDEN_TOKENS = ("decnet", "honeypot", "decoy", "deck")
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_default_user_agent_is_curl_shaped():
|
||||
captured: list[httpx.Request] = []
|
||||
|
||||
async def _handler(request: httpx.Request) -> httpx.Response:
|
||||
captured.append(request)
|
||||
return httpx.Response(200, json={"ok": True})
|
||||
|
||||
transport = httpx.MockTransport(_handler)
|
||||
async with stealth_client() as base:
|
||||
# Swap transport to keep test offline.
|
||||
base._transport = transport # noqa: SLF001 — internal field, deliberate
|
||||
await base.get("https://api.example.test/check")
|
||||
|
||||
ua = captured[0].headers.get("user-agent", "")
|
||||
assert ua == DEFAULT_STEALTH_USER_AGENT
|
||||
lower = ua.lower()
|
||||
for token in _FORBIDDEN_TOKENS:
|
||||
assert token not in lower, f"stealth UA leaked {token!r}: {ua!r}"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_custom_user_agent_override_takes_effect():
|
||||
captured: list[httpx.Request] = []
|
||||
|
||||
async def _handler(request: httpx.Request) -> httpx.Response:
|
||||
captured.append(request)
|
||||
return httpx.Response(200)
|
||||
|
||||
transport = httpx.MockTransport(_handler)
|
||||
async with stealth_client(user_agent="Mozilla/5.0 (test)") as base:
|
||||
base._transport = transport # noqa: SLF001
|
||||
await base.get("https://api.example.test/")
|
||||
|
||||
assert captured[0].headers["user-agent"] == "Mozilla/5.0 (test)"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_redirects_do_not_follow_by_default():
|
||||
async def _handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(302, headers={"Location": "https://elsewhere/"})
|
||||
|
||||
transport = httpx.MockTransport(_handler)
|
||||
async with stealth_client() as base:
|
||||
base._transport = transport # noqa: SLF001
|
||||
resp = await base.get("https://api.example.test/")
|
||||
assert resp.status_code == 302
|
||||
111
tests/intel/test_threatfox.py
Normal file
111
tests/intel/test_threatfox.py
Normal file
@@ -0,0 +1,111 @@
|
||||
"""Unit tests for the abuse.ch ThreatFox provider."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from decnet.intel.threatfox import ThreatFoxProvider
|
||||
|
||||
|
||||
def _install_transport(handler) -> list[httpx.Request]:
|
||||
captured: list[httpx.Request] = []
|
||||
|
||||
async def _wrapped(request: httpx.Request) -> httpx.Response:
|
||||
captured.append(request)
|
||||
return await handler(request)
|
||||
|
||||
transport = httpx.MockTransport(_wrapped)
|
||||
from decnet.intel import threatfox as mod
|
||||
|
||||
def _factory():
|
||||
return httpx.AsyncClient(
|
||||
transport=transport,
|
||||
headers={"User-Agent": "curl/7.88.1"},
|
||||
)
|
||||
|
||||
mod.stealth_client = _factory # type: ignore[assignment]
|
||||
return captured
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_match_returns_malicious(monkeypatch):
|
||||
monkeypatch.delenv("DECNET_THREATFOX_API_KEY", raising=False)
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
body = json.loads(request.content.decode())
|
||||
assert body == {"query": "search_ioc", "search_term": "1.2.3.4"}
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"query_status": "ok",
|
||||
"data": [
|
||||
{
|
||||
"ioc": "1.2.3.4",
|
||||
"ioc_type": "ip:port",
|
||||
"malware": "Cobalt Strike",
|
||||
"confidence_level": 80,
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
captured = _install_transport(handler)
|
||||
provider = ThreatFoxProvider()
|
||||
result = await provider.lookup("1.2.3.4")
|
||||
assert result.verdict == "malicious"
|
||||
assert result.column_updates["threatfox_listed"] is True
|
||||
raw = json.loads(result.column_updates["threatfox_raw"])
|
||||
assert raw[0]["malware"] == "Cobalt Strike"
|
||||
# No Auth-Key when none configured.
|
||||
assert "auth-key" not in {h.lower() for h in captured[0].headers}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_auth_key_sent_when_configured(monkeypatch):
|
||||
monkeypatch.setenv("DECNET_THREATFOX_API_KEY", "tfx-key")
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json={"query_status": "no_result"})
|
||||
|
||||
captured = _install_transport(handler)
|
||||
provider = ThreatFoxProvider()
|
||||
await provider.lookup("8.8.8.8")
|
||||
assert captured[0].headers["auth-key"] == "tfx-key"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_no_result_caches_unlisted():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json={"query_status": "no_result"})
|
||||
|
||||
_install_transport(handler)
|
||||
provider = ThreatFoxProvider()
|
||||
result = await provider.lookup("8.8.8.8")
|
||||
assert result.verdict is None
|
||||
assert result.column_updates["threatfox_listed"] is False
|
||||
assert result.error is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_unexpected_status_is_error():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json={"query_status": "illegal_search"})
|
||||
|
||||
_install_transport(handler)
|
||||
provider = ThreatFoxProvider()
|
||||
result = await provider.lookup("oops")
|
||||
assert result.error and "illegal_search" in result.error
|
||||
assert result.column_updates == {}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_http_error_surfaces():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(502)
|
||||
|
||||
_install_transport(handler)
|
||||
provider = ThreatFoxProvider()
|
||||
result = await provider.lookup("1.1.1.1")
|
||||
assert result.error == "HTTP 502"
|
||||
262
tests/intel/test_worker.py
Normal file
262
tests/intel/test_worker.py
Normal file
@@ -0,0 +1,262 @@
|
||||
"""End-to-end tests for the intel worker shell.
|
||||
|
||||
Covers — without any real provider impls — that the loop:
|
||||
|
||||
* exits cleanly on shutdown signal (and via cancel)
|
||||
* does nothing when no providers are configured
|
||||
* fans out across fake providers and writes the aggregate row
|
||||
* aggregate_verdict picks the strongest provider verdict
|
||||
* a provider returning ``error`` is logged but does not poison the row
|
||||
* gates attackers through ``get_unenriched_attackers`` (TTL respected)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
|
||||
from decnet.intel.base import IntelProvider, IntelResult
|
||||
from decnet.intel.worker import run_intel_loop, _aggregate
|
||||
from decnet.web.db.factory import get_repository
|
||||
|
||||
|
||||
class _FakeProvider(IntelProvider):
|
||||
"""Test double — instantly returns a canned :class:`IntelResult`."""
|
||||
|
||||
concurrency = 1
|
||||
min_dispatch_interval_s = 0.0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
*,
|
||||
verdict: Optional[str] = None,
|
||||
error: Optional[str] = None,
|
||||
column_updates: Optional[dict] = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.name = name
|
||||
self._verdict = verdict
|
||||
self._error = error
|
||||
self._cols = column_updates or {}
|
||||
self.calls: list[str] = []
|
||||
|
||||
async def lookup(self, ip: str) -> IntelResult:
|
||||
self.calls.append(ip)
|
||||
return IntelResult(
|
||||
provider=self.name,
|
||||
verdict=self._verdict,
|
||||
error=self._error,
|
||||
column_updates=self._cols,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def repo(tmp_path):
|
||||
r = get_repository(db_path=str(tmp_path / "intel_worker.db"))
|
||||
await r.initialize()
|
||||
return r
|
||||
|
||||
|
||||
# Disable bus connection in tests — workers under test should run in
|
||||
# poll-only mode without hitting a real Unix socket.
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_bus(monkeypatch):
|
||||
monkeypatch.setenv("DECNET_BUS_ENABLED", "false")
|
||||
|
||||
|
||||
def test_aggregate_picks_strongest_verdict():
|
||||
assert _aggregate(["benign", "malicious", None]) == "malicious"
|
||||
assert _aggregate(["benign", "suspicious"]) == "suspicious"
|
||||
assert _aggregate(["benign", None]) == "benign"
|
||||
assert _aggregate([None, None]) is None
|
||||
assert _aggregate([]) is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_loop_exits_on_shutdown_signal(repo):
|
||||
shutdown = asyncio.Event()
|
||||
task = asyncio.create_task(
|
||||
run_intel_loop(
|
||||
repo,
|
||||
poll_interval_secs=0.05,
|
||||
providers=[],
|
||||
shutdown=shutdown,
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(0.1)
|
||||
shutdown.set()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
|
||||
async def _seed_attacker(repo, ip: str) -> str:
|
||||
"""Seed an attackers row and return its UUID."""
|
||||
now = datetime.now(timezone.utc)
|
||||
return await repo.upsert_attacker(
|
||||
{"ip": ip, "first_seen": now, "last_seen": now, "event_count": 1}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_no_providers_skips_enrichment(repo):
|
||||
a_uuid = await _seed_attacker(repo, "1.1.1.1")
|
||||
shutdown = asyncio.Event()
|
||||
task = asyncio.create_task(
|
||||
run_intel_loop(
|
||||
repo,
|
||||
poll_interval_secs=0.05,
|
||||
providers=[],
|
||||
shutdown=shutdown,
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(0.15)
|
||||
shutdown.set()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
# No row written for the seeded attacker.
|
||||
assert await repo.get_attacker_intel_by_uuid(a_uuid) is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_fan_out_writes_aggregate_row(repo):
|
||||
a_uuid = await _seed_attacker(repo, "2.2.2.2")
|
||||
|
||||
gn = _FakeProvider(
|
||||
"greynoise",
|
||||
verdict="benign",
|
||||
column_updates={
|
||||
"greynoise_classification": "benign",
|
||||
"greynoise_raw": json.dumps({"classification": "benign"}),
|
||||
"greynoise_queried_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
aip = _FakeProvider(
|
||||
"abuseipdb",
|
||||
verdict="malicious",
|
||||
column_updates={
|
||||
"abuseipdb_score": 90,
|
||||
"abuseipdb_raw": json.dumps({"abuseConfidenceScore": 90}),
|
||||
"abuseipdb_queried_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
|
||||
shutdown = asyncio.Event()
|
||||
task = asyncio.create_task(
|
||||
run_intel_loop(
|
||||
repo,
|
||||
poll_interval_secs=0.05,
|
||||
providers=[gn, aip],
|
||||
shutdown=shutdown,
|
||||
)
|
||||
)
|
||||
# One tick is enough — both providers respond instantly.
|
||||
await asyncio.sleep(0.15)
|
||||
shutdown.set()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
row = await repo.get_attacker_intel_by_uuid(a_uuid)
|
||||
assert row is not None
|
||||
assert row["attacker_uuid"] == a_uuid
|
||||
assert row["attacker_ip"] == "2.2.2.2"
|
||||
assert row["greynoise_classification"] == "benign"
|
||||
assert row["abuseipdb_score"] == 90
|
||||
# Strongest verdict wins.
|
||||
assert row["aggregate_verdict"] == "malicious"
|
||||
# Both providers were queried by IP.
|
||||
assert gn.calls == ["2.2.2.2"]
|
||||
assert aip.calls == ["2.2.2.2"]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_provider_error_does_not_poison_row(repo):
|
||||
a_uuid = await _seed_attacker(repo, "3.3.3.3")
|
||||
|
||||
good = _FakeProvider(
|
||||
"greynoise",
|
||||
verdict="benign",
|
||||
column_updates={
|
||||
"greynoise_classification": "benign",
|
||||
"greynoise_raw": "{}",
|
||||
"greynoise_queried_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
broken = _FakeProvider("abuseipdb", error="HTTP 500")
|
||||
|
||||
shutdown = asyncio.Event()
|
||||
task = asyncio.create_task(
|
||||
run_intel_loop(
|
||||
repo,
|
||||
poll_interval_secs=0.05,
|
||||
providers=[good, broken],
|
||||
shutdown=shutdown,
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(0.15)
|
||||
shutdown.set()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
row = await repo.get_attacker_intel_by_uuid(a_uuid)
|
||||
assert row is not None
|
||||
assert row["greynoise_classification"] == "benign"
|
||||
# Broken provider's columns stay null; row is still written.
|
||||
assert row["abuseipdb_score"] is None
|
||||
# Aggregate reflects only the providers that responded.
|
||||
assert row["aggregate_verdict"] == "benign"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_intel_enriched_event_published_to_bus(repo, monkeypatch):
|
||||
"""End-to-end: worker dispatches providers + publishes the event."""
|
||||
from decnet.bus.fake import FakeBus
|
||||
from decnet.bus.topics import ATTACKER_INTEL_ENRICHED, attacker
|
||||
|
||||
# Re-enable bus path; swap factory for a shared FakeBus instance the
|
||||
# test can also subscribe to.
|
||||
monkeypatch.setenv("DECNET_BUS_ENABLED", "true")
|
||||
monkeypatch.setenv("DECNET_BUS_TYPE", "fake")
|
||||
shared_bus = FakeBus()
|
||||
|
||||
from decnet.intel import worker as worker_mod
|
||||
monkeypatch.setattr(
|
||||
worker_mod, "get_bus", lambda **_: shared_bus,
|
||||
)
|
||||
|
||||
# Subscribe before the worker starts so we don't race the publish.
|
||||
sub = shared_bus.subscribe(attacker(ATTACKER_INTEL_ENRICHED))
|
||||
await sub.__aenter__()
|
||||
|
||||
a_uuid = await _seed_attacker(repo, "4.4.4.4")
|
||||
|
||||
provider = _FakeProvider(
|
||||
"greynoise",
|
||||
verdict="malicious",
|
||||
column_updates={
|
||||
"greynoise_classification": "malicious",
|
||||
"greynoise_raw": "{}",
|
||||
"greynoise_queried_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
|
||||
shutdown = asyncio.Event()
|
||||
task = asyncio.create_task(
|
||||
run_intel_loop(
|
||||
repo,
|
||||
poll_interval_secs=0.05,
|
||||
providers=[provider],
|
||||
shutdown=shutdown,
|
||||
)
|
||||
)
|
||||
try:
|
||||
event = await asyncio.wait_for(sub.__anext__(), timeout=2.0)
|
||||
finally:
|
||||
shutdown.set()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
await sub.__aexit__(None, None, None)
|
||||
|
||||
payload = event.payload
|
||||
assert payload["attacker_uuid"] == a_uuid
|
||||
assert payload["attacker_ip"] == "4.4.4.4"
|
||||
assert payload["aggregate_verdict"] == "malicious"
|
||||
assert payload["providers"] == ["greynoise"]
|
||||
Reference in New Issue
Block a user