merge: testing → main (reconcile 2-week divergence)
This commit is contained in:
10
decnet/intel/__init__.py
Normal file
10
decnet/intel/__init__.py
Normal file
@@ -0,0 +1,10 @@
|
||||
"""Threat-intel enrichment subsystem — out-of-band lookups for attacker IPs.
|
||||
|
||||
Sibling to :mod:`decnet.geoip` and :mod:`decnet.asn`, but runs as a
|
||||
separate worker (``decnet enrich``) rather than inline in the profiler:
|
||||
3rd-party HTTP latency and free-tier rate limits should not block the
|
||||
profiler tick.
|
||||
|
||||
Public surface: :func:`decnet.intel.factory.get_intel_providers` and the
|
||||
:class:`decnet.intel.base.IntelProvider` ABC.
|
||||
"""
|
||||
104
decnet/intel/abuseipdb.py
Normal file
104
decnet/intel/abuseipdb.py
Normal file
@@ -0,0 +1,104 @@
|
||||
"""AbuseIPDB provider.
|
||||
|
||||
Endpoint: ``GET https://api.abuseipdb.com/api/v2/check``
|
||||
|
||||
Free tier: 1000 lookups/day. Always requires an API key passed in the
|
||||
``Key`` header — the provider self-disables (returns an error) when no
|
||||
key is configured rather than burning quota at the free public IP.
|
||||
|
||||
Verdict mapping is tier-based on the ``abuseConfidenceScore`` (0–100):
|
||||
|
||||
* ``>= 75`` — ``malicious``
|
||||
* ``25..74`` — ``suspicious``
|
||||
* ``< 25`` — ``benign``
|
||||
|
||||
This matches AbuseIPDB's own UI thresholds reasonably closely; tune
|
||||
later if operators report drift.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from decnet.intel.base import IntelProvider, IntelResult
|
||||
from decnet.logging import get_logger
|
||||
from decnet.net.http import stealth_client
|
||||
|
||||
log = get_logger("intel.abuseipdb")
|
||||
|
||||
_ENDPOINT = "https://api.abuseipdb.com/api/v2/check"
|
||||
_DEFAULT_MAX_AGE_DAYS = 30
|
||||
|
||||
|
||||
def _score_to_verdict(score: int) -> str:
|
||||
if score >= 75:
|
||||
return "malicious"
|
||||
if score >= 25:
|
||||
return "suspicious"
|
||||
return "benign"
|
||||
|
||||
|
||||
class AbuseIPDBProvider(IntelProvider):
|
||||
name = "abuseipdb"
|
||||
concurrency = 4
|
||||
# 1000/day = avg 1 every ~86s. We don't enforce the daily cap here —
|
||||
# operators who burn it through the worker will see HTTP 429 and the
|
||||
# row gets retried after the TTL window.
|
||||
min_dispatch_interval_s = 0.5
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
api_key: Optional[str] = None,
|
||||
max_age_days: int = _DEFAULT_MAX_AGE_DAYS,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._api_key = api_key or os.environ.get(
|
||||
"DECNET_ABUSEIPDB_API_KEY"
|
||||
) or None
|
||||
self._max_age_days = max_age_days
|
||||
|
||||
async def lookup(self, ip: str) -> IntelResult:
|
||||
if not self._api_key:
|
||||
return IntelResult(
|
||||
provider=self.name,
|
||||
error="DECNET_ABUSEIPDB_API_KEY not configured",
|
||||
)
|
||||
params = {
|
||||
"ipAddress": ip,
|
||||
"maxAgeInDays": str(self._max_age_days),
|
||||
}
|
||||
headers = {
|
||||
"Key": self._api_key,
|
||||
"Accept": "application/json",
|
||||
}
|
||||
try:
|
||||
async with stealth_client() as client:
|
||||
resp = await client.get(_ENDPOINT, headers=headers, params=params)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return IntelResult(provider=self.name, error=f"network: {exc}")
|
||||
|
||||
if resp.status_code != 200:
|
||||
return IntelResult(
|
||||
provider=self.name,
|
||||
error=f"HTTP {resp.status_code}",
|
||||
)
|
||||
try:
|
||||
payload = resp.json()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return IntelResult(provider=self.name, error=f"parse: {exc}")
|
||||
|
||||
data = payload.get("data") or {}
|
||||
score = int(data.get("abuseConfidenceScore") or 0)
|
||||
verdict = _score_to_verdict(score)
|
||||
return IntelResult(
|
||||
provider=self.name,
|
||||
verdict=verdict,
|
||||
column_updates={
|
||||
"abuseipdb_score": score,
|
||||
"abuseipdb_raw": json.dumps(data),
|
||||
"abuseipdb_queried_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
80
decnet/intel/base.py
Normal file
80
decnet/intel/base.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""Threat-intel provider protocol.
|
||||
|
||||
Each concrete provider (:mod:`decnet.intel.greynoise`,
|
||||
:mod:`decnet.intel.abuseipdb`, :mod:`decnet.intel.feodo`,
|
||||
:mod:`decnet.intel.threatfox`) implements this. Callers must obtain
|
||||
providers via :func:`decnet.intel.factory.get_intel_providers` — never
|
||||
instantiate a concrete provider class directly.
|
||||
|
||||
Unlike :mod:`decnet.geoip` (which returns a single ``Provider``), the
|
||||
intel subsystem returns a **list** of providers — enrichment fans out
|
||||
across all of them per IP, and partial successes are stored row-wise.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class IntelResult:
|
||||
"""Per-provider enrichment outcome.
|
||||
|
||||
The worker maps these into the per-provider columns on
|
||||
``attacker_intel`` (e.g. ``greynoise_classification`` /
|
||||
``greynoise_raw`` / ``greynoise_queried_at``).
|
||||
|
||||
``column_updates`` carries the dialect-portable column→value map the
|
||||
repository ``upsert_attacker_intel`` will apply. ``raw`` is the
|
||||
serialized provider response (already JSON-encoded by the provider so
|
||||
the worker doesn't need to know the wire shape).
|
||||
"""
|
||||
|
||||
provider: str
|
||||
"""Short tag — matches the column prefix in ``attacker_intel``
|
||||
(``greynoise``, ``abuseipdb``, ``feodo``, ``threatfox``)."""
|
||||
|
||||
column_updates: dict[str, Any] = field(default_factory=dict)
|
||||
"""Columns to write on the ``attacker_intel`` row."""
|
||||
|
||||
verdict: Optional[str] = None
|
||||
"""Provider-local verdict label, e.g. ``"malicious"`` / ``"benign"``.
|
||||
Used by the worker to compute ``aggregate_verdict``. ``None`` =
|
||||
"no opinion" (e.g. IP not present in a blocklist)."""
|
||||
|
||||
error: Optional[str] = None
|
||||
"""Populated when the provider call failed. The worker logs it and
|
||||
leaves the row unchanged for this provider so a partial-success
|
||||
enrichment doesn't clobber a previous good answer."""
|
||||
|
||||
|
||||
class IntelProvider(ABC):
|
||||
"""Abstract threat-intel provider."""
|
||||
|
||||
#: Short tag — matches ``IntelResult.provider`` and the column prefix
|
||||
#: on ``attacker_intel``.
|
||||
name: str
|
||||
|
||||
#: Per-provider in-flight cap. Free tiers are surprisingly tight
|
||||
#: (GreyNoise community ~50/min); 4 is a safe default but providers
|
||||
#: can override.
|
||||
concurrency: int = 4
|
||||
|
||||
#: Minimum seconds between dispatches. Token-bucket-lite — see
|
||||
#: :class:`decnet.intel.worker.RateLimitedDispatcher`.
|
||||
min_dispatch_interval_s: float = 0.0
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._semaphore = asyncio.Semaphore(self.concurrency)
|
||||
|
||||
@abstractmethod
|
||||
async def lookup(self, ip: str) -> IntelResult:
|
||||
"""Query the provider for ``ip`` and return the result.
|
||||
|
||||
MUST NOT raise — capture errors in ``IntelResult.error`` so a
|
||||
single provider's outage doesn't break the worker pass for an
|
||||
entire IP. Implementations should also respect
|
||||
``self._semaphore`` to bound in-flight calls.
|
||||
"""
|
||||
73
decnet/intel/factory.py
Normal file
73
decnet/intel/factory.py
Normal file
@@ -0,0 +1,73 @@
|
||||
"""Threat-intel provider factory.
|
||||
|
||||
Returns the **list** of configured :class:`IntelProvider` instances —
|
||||
diverges from :mod:`decnet.geoip.factory` (which returns a single
|
||||
provider) because intel enrichment fans out across every enabled
|
||||
provider per IP, with partial-success handling per row.
|
||||
|
||||
Configuration knobs (env-overridable; INI-driven defaults via
|
||||
``decnet/config_ini.py``):
|
||||
|
||||
* ``DECNET_INTEL_ENABLED`` — master kill-switch (default ``true``).
|
||||
* ``DECNET_INTEL_PROVIDERS`` — comma-separated list. Default
|
||||
``"greynoise,abuseipdb,feodo,threatfox"``.
|
||||
|
||||
Per-provider keys (``DECNET_GREYNOISE_API_KEY``,
|
||||
``DECNET_ABUSEIPDB_API_KEY``, ``DECNET_THREATFOX_API_KEY``) are read by
|
||||
each concrete provider; the factory just instantiates and returns.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import List
|
||||
|
||||
from decnet.intel.base import IntelProvider
|
||||
|
||||
_KNOWN_PROVIDERS = ("greynoise", "abuseipdb", "feodo", "threatfox")
|
||||
|
||||
|
||||
def _enabled() -> bool:
|
||||
return os.environ.get("DECNET_INTEL_ENABLED", "true").lower() != "false"
|
||||
|
||||
|
||||
def _provider_list() -> list[str]:
|
||||
raw = os.environ.get(
|
||||
"DECNET_INTEL_PROVIDERS", ",".join(_KNOWN_PROVIDERS),
|
||||
)
|
||||
return [p.strip().lower() for p in raw.split(",") if p.strip()]
|
||||
|
||||
|
||||
def get_intel_providers() -> List[IntelProvider]:
|
||||
"""Return the configured threat-intel providers.
|
||||
|
||||
Returns ``[]`` when intel is disabled or the configured list is
|
||||
empty — the worker treats that as "stay running but never make a
|
||||
call," which is the right behavior for an operator who wants the
|
||||
table maintained but no egress.
|
||||
|
||||
Unknown provider names raise :class:`ValueError` so a typo in
|
||||
``decnet.ini`` surfaces immediately rather than silently dropping a
|
||||
provider.
|
||||
"""
|
||||
if not _enabled():
|
||||
return []
|
||||
|
||||
providers: List[IntelProvider] = []
|
||||
for name in _provider_list():
|
||||
if name == "greynoise":
|
||||
from decnet.intel.greynoise import GreyNoiseProvider
|
||||
providers.append(GreyNoiseProvider())
|
||||
elif name == "abuseipdb":
|
||||
from decnet.intel.abuseipdb import AbuseIPDBProvider
|
||||
providers.append(AbuseIPDBProvider())
|
||||
elif name == "feodo":
|
||||
from decnet.intel.feodo import FeodoProvider
|
||||
providers.append(FeodoProvider())
|
||||
elif name == "threatfox":
|
||||
from decnet.intel.threatfox import ThreatFoxProvider
|
||||
providers.append(ThreatFoxProvider())
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown intel provider: {name!r}. Known: {_KNOWN_PROVIDERS}"
|
||||
)
|
||||
return providers
|
||||
108
decnet/intel/feodo.py
Normal file
108
decnet/intel/feodo.py
Normal file
@@ -0,0 +1,108 @@
|
||||
"""abuse.ch Feodo Tracker provider — bulk JSON botnet C2 feed.
|
||||
|
||||
Endpoint: ``GET https://feodotracker.abuse.ch/downloads/ipblocklist.json``
|
||||
|
||||
This is the only provider in the v1 set that uses a *bulk* feed instead
|
||||
of a per-IP query: the upstream is a list of every botnet C2 IP abuse.ch
|
||||
has seen recently (Emotet, TrickBot, Dridex, etc.), refreshed every few
|
||||
minutes. We fetch the full list once per ``refresh_interval_s`` and
|
||||
answer ``lookup(ip)`` calls from the in-process set.
|
||||
|
||||
This makes Feodo Tracker effectively free at the call-site: thousands
|
||||
of attacker IPs map to a single network round-trip per refresh window.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Optional
|
||||
|
||||
from decnet.intel.base import IntelProvider, IntelResult
|
||||
from decnet.logging import get_logger
|
||||
from decnet.net.http import stealth_client
|
||||
|
||||
log = get_logger("intel.feodo")
|
||||
|
||||
_ENDPOINT = "https://feodotracker.abuse.ch/downloads/ipblocklist.json"
|
||||
_DEFAULT_REFRESH_S = 3600.0
|
||||
|
||||
|
||||
class FeodoProvider(IntelProvider):
|
||||
name = "feodo"
|
||||
concurrency = 1 # only one concurrent refresh; lookups are pure set ops
|
||||
min_dispatch_interval_s = 0.0
|
||||
|
||||
def __init__(self, *, refresh_interval_s: float = _DEFAULT_REFRESH_S) -> None:
|
||||
super().__init__()
|
||||
self._refresh_interval_s = refresh_interval_s
|
||||
# ip → upstream record dict, keyed by ``ip_address``.
|
||||
self._index: dict[str, dict[str, Any]] = {}
|
||||
self._loaded_at: float = 0.0
|
||||
self._last_error: Optional[str] = None
|
||||
|
||||
async def _refresh(self) -> Optional[str]:
|
||||
"""Refetch the bulk feed. Returns an error string or ``None``."""
|
||||
try:
|
||||
async with stealth_client(timeout=20.0) as client:
|
||||
resp = await client.get(_ENDPOINT)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return f"network: {exc}"
|
||||
if resp.status_code != 200:
|
||||
return f"HTTP {resp.status_code}"
|
||||
try:
|
||||
payload = resp.json()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return f"parse: {exc}"
|
||||
if not isinstance(payload, list):
|
||||
return "feed: not a list"
|
||||
|
||||
new_index: dict[str, dict[str, Any]] = {}
|
||||
for entry in payload:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
ip = entry.get("ip_address")
|
||||
if isinstance(ip, str):
|
||||
new_index[ip] = entry
|
||||
self._index = new_index
|
||||
self._loaded_at = time.monotonic()
|
||||
self._last_error = None
|
||||
log.info("feodo: refreshed bulk feed entries=%d", len(new_index))
|
||||
return None
|
||||
|
||||
async def _ensure_fresh(self) -> None:
|
||||
if (
|
||||
not self._index
|
||||
or (time.monotonic() - self._loaded_at) >= self._refresh_interval_s
|
||||
):
|
||||
err = await self._refresh()
|
||||
if err:
|
||||
self._last_error = err
|
||||
|
||||
async def lookup(self, ip: str) -> IntelResult:
|
||||
await self._ensure_fresh()
|
||||
if not self._index and self._last_error:
|
||||
return IntelResult(provider=self.name, error=self._last_error)
|
||||
|
||||
entry = self._index.get(ip)
|
||||
if entry is None:
|
||||
# Not on the C2 list — explicit benign-ish signal. Cache it
|
||||
# so we don't keep checking the same set on every wake.
|
||||
return IntelResult(
|
||||
provider=self.name,
|
||||
verdict=None, # absence ≠ "benign", let other providers speak
|
||||
column_updates={
|
||||
"feodo_listed": False,
|
||||
"feodo_raw": "{}",
|
||||
"feodo_queried_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
return IntelResult(
|
||||
provider=self.name,
|
||||
verdict="malicious",
|
||||
column_updates={
|
||||
"feodo_listed": True,
|
||||
"feodo_raw": json.dumps(entry),
|
||||
"feodo_queried_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
107
decnet/intel/greynoise.py
Normal file
107
decnet/intel/greynoise.py
Normal file
@@ -0,0 +1,107 @@
|
||||
"""GreyNoise Community API provider.
|
||||
|
||||
Endpoint: ``GET https://api.greynoise.io/v3/community/<ip>``
|
||||
|
||||
The Community endpoint requires no API key for low-volume use; an
|
||||
optional ``DECNET_GREYNOISE_API_KEY`` lifts the rate limit. We always
|
||||
send the key when present.
|
||||
|
||||
Response shape (relevant fields)::
|
||||
|
||||
{
|
||||
"ip": "1.2.3.4",
|
||||
"noise": true, // observed scanning the public internet
|
||||
"riot": false, // member of the "Rule It Out" benign set
|
||||
"classification": "benign | malicious | unknown",
|
||||
"name": "Censys", // tool/operator label, when known
|
||||
"link": "https://...",
|
||||
"last_seen": "2026-04-25"
|
||||
}
|
||||
|
||||
Status code semantics:
|
||||
* 200 — IP found, JSON body as above
|
||||
* 404 — IP not observed by GreyNoise (treat as ``"unknown"``, not error)
|
||||
* 429 — rate-limited (treat as transient error)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from decnet.intel.base import IntelProvider, IntelResult
|
||||
from decnet.logging import get_logger
|
||||
from decnet.net.http import stealth_client
|
||||
|
||||
log = get_logger("intel.greynoise")
|
||||
|
||||
_ENDPOINT = "https://api.greynoise.io/v3/community/{ip}"
|
||||
|
||||
|
||||
class GreyNoiseProvider(IntelProvider):
|
||||
name = "greynoise"
|
||||
concurrency = 4
|
||||
# Community tier is ~50/min; ~1.5s between dispatches keeps us well
|
||||
# under that without serialising entirely.
|
||||
min_dispatch_interval_s = 1.5
|
||||
|
||||
def __init__(self, *, api_key: Optional[str] = None) -> None:
|
||||
super().__init__()
|
||||
self._api_key = api_key or os.environ.get(
|
||||
"DECNET_GREYNOISE_API_KEY"
|
||||
) or None
|
||||
|
||||
async def lookup(self, ip: str) -> IntelResult:
|
||||
url = _ENDPOINT.format(ip=ip)
|
||||
headers = {"Accept": "application/json"}
|
||||
if self._api_key:
|
||||
headers["key"] = self._api_key
|
||||
try:
|
||||
async with stealth_client() as client:
|
||||
resp = await client.get(url, headers=headers)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return IntelResult(provider=self.name, error=f"network: {exc}")
|
||||
|
||||
if resp.status_code == 404:
|
||||
# IP not in GreyNoise's view of the internet — record the row
|
||||
# so we don't keep re-querying within the TTL window.
|
||||
return IntelResult(
|
||||
provider=self.name,
|
||||
verdict="unknown",
|
||||
column_updates={
|
||||
"greynoise_classification": "unknown",
|
||||
"greynoise_raw": json.dumps({"message": "not seen"}),
|
||||
"greynoise_queried_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return IntelResult(
|
||||
provider=self.name,
|
||||
error=f"HTTP {resp.status_code}",
|
||||
)
|
||||
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return IntelResult(provider=self.name, error=f"parse: {exc}")
|
||||
|
||||
classification = (data.get("classification") or "unknown").lower()
|
||||
verdict = _CLASSIFICATION_TO_VERDICT.get(classification, "unknown")
|
||||
return IntelResult(
|
||||
provider=self.name,
|
||||
verdict=verdict,
|
||||
column_updates={
|
||||
"greynoise_classification": classification,
|
||||
"greynoise_raw": json.dumps(data),
|
||||
"greynoise_queried_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_CLASSIFICATION_TO_VERDICT = {
|
||||
"malicious": "malicious",
|
||||
"suspicious": "suspicious",
|
||||
"benign": "benign",
|
||||
"unknown": "unknown",
|
||||
}
|
||||
94
decnet/intel/threatfox.py
Normal file
94
decnet/intel/threatfox.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""abuse.ch ThreatFox provider — per-IOC query API.
|
||||
|
||||
Endpoint: ``POST https://threatfox-api.abuse.ch/api/v1/``
|
||||
|
||||
ThreatFox returns IOC matches across many types (URL, domain, IP, hash).
|
||||
We send ``{"query": "search_ioc", "search_term": "<ip>"}`` and treat any
|
||||
non-empty ``data`` array as a malicious match.
|
||||
|
||||
API key handling: ThreatFox accepts an optional ``Auth-Key`` header for
|
||||
higher rate limits. Without a key the public endpoint still answers but
|
||||
caps requests/min — the provider works either way.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from decnet.intel.base import IntelProvider, IntelResult
|
||||
from decnet.logging import get_logger
|
||||
from decnet.net.http import stealth_client
|
||||
|
||||
log = get_logger("intel.threatfox")
|
||||
|
||||
_ENDPOINT = "https://threatfox-api.abuse.ch/api/v1/"
|
||||
|
||||
|
||||
class ThreatFoxProvider(IntelProvider):
|
||||
name = "threatfox"
|
||||
concurrency = 4
|
||||
min_dispatch_interval_s = 0.5
|
||||
|
||||
def __init__(self, *, api_key: Optional[str] = None) -> None:
|
||||
super().__init__()
|
||||
self._api_key = api_key or os.environ.get(
|
||||
"DECNET_THREATFOX_API_KEY"
|
||||
) or None
|
||||
|
||||
async def lookup(self, ip: str) -> IntelResult:
|
||||
body = {"query": "search_ioc", "search_term": ip}
|
||||
headers = {"Accept": "application/json"}
|
||||
if self._api_key:
|
||||
headers["Auth-Key"] = self._api_key
|
||||
|
||||
try:
|
||||
async with stealth_client() as client:
|
||||
resp = await client.post(
|
||||
_ENDPOINT, headers=headers, json=body,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return IntelResult(provider=self.name, error=f"network: {exc}")
|
||||
|
||||
if resp.status_code != 200:
|
||||
return IntelResult(
|
||||
provider=self.name, error=f"HTTP {resp.status_code}",
|
||||
)
|
||||
try:
|
||||
payload = resp.json()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return IntelResult(provider=self.name, error=f"parse: {exc}")
|
||||
|
||||
status = payload.get("query_status")
|
||||
# ThreatFox returns query_status="no_result" when the IOC isn't
|
||||
# tracked, and query_status="ok" with a non-empty data list when
|
||||
# it is. Anything else (illegal_search, etc.) is a contract
|
||||
# violation we surface as an error.
|
||||
if status == "no_result":
|
||||
return IntelResult(
|
||||
provider=self.name,
|
||||
verdict=None, # absence is not a benign signal
|
||||
column_updates={
|
||||
"threatfox_listed": False,
|
||||
"threatfox_raw": "{}",
|
||||
"threatfox_queried_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
if status != "ok":
|
||||
return IntelResult(
|
||||
provider=self.name,
|
||||
error=f"query_status={status!r}",
|
||||
)
|
||||
|
||||
data = payload.get("data") or []
|
||||
listed = bool(data)
|
||||
return IntelResult(
|
||||
provider=self.name,
|
||||
verdict="malicious" if listed else None,
|
||||
column_updates={
|
||||
"threatfox_listed": listed,
|
||||
"threatfox_raw": json.dumps(data),
|
||||
"threatfox_queried_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
233
decnet/intel/worker.py
Normal file
233
decnet/intel/worker.py
Normal file
@@ -0,0 +1,233 @@
|
||||
"""Long-running threat-intel enrichment worker.
|
||||
|
||||
Fans out per attacker IP across the configured intel providers
|
||||
(GreyNoise / AbuseIPDB / abuse.ch Feodo + ThreatFox), writes the
|
||||
combined verdict to ``attacker_intel``, and publishes
|
||||
``attacker.intel.enriched`` for downstream consumers (SIEM webhooks,
|
||||
dashboard).
|
||||
|
||||
Mirrors :mod:`decnet.correlation.reuse_worker` — bus-woken on
|
||||
``attacker.scored`` and ``attacker.observed`` for sub-second latency,
|
||||
falls back to a slow tick (default 60s) when the bus is unavailable so
|
||||
operators with bus disabled still get periodic backfills.
|
||||
|
||||
A single worker instance handles all providers; provider-level
|
||||
concurrency is bounded by the per-provider semaphore on each
|
||||
:class:`~decnet.intel.base.IntelProvider`. The worker itself does not
|
||||
hold a global lock — each IP runs through its providers concurrently.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Optional
|
||||
|
||||
from decnet.bus import topics as _topics
|
||||
from decnet.bus.base import BaseBus
|
||||
from decnet.bus.factory import get_bus
|
||||
from decnet.bus.publish import (
|
||||
publish_safely,
|
||||
run_control_listener_signal as _run_control_listener_signal,
|
||||
run_health_heartbeat as _run_health_heartbeat,
|
||||
)
|
||||
from decnet.intel.base import IntelProvider, IntelResult
|
||||
from decnet.intel.factory import get_intel_providers
|
||||
from decnet.logging import get_logger
|
||||
from decnet.web.db.repository import BaseRepository
|
||||
|
||||
log = get_logger("intel.worker")
|
||||
|
||||
_DEFAULT_POLL_SECS = 60.0
|
||||
_DEFAULT_TTL_HOURS = 24
|
||||
_BACKFILL_BATCH = 50
|
||||
|
||||
# Aggregate-verdict precedence: most-confident first. Any provider
|
||||
# returning the higher tier wins regardless of how many lower-tier
|
||||
# verdicts exist alongside it.
|
||||
_VERDICT_PRECEDENCE = ("malicious", "suspicious", "benign", "unknown")
|
||||
|
||||
|
||||
def _aggregate(verdicts: list[Optional[str]]) -> Optional[str]:
|
||||
"""Pick the strongest provider verdict, or ``None`` if all silent."""
|
||||
seen = {v for v in verdicts if v}
|
||||
if not seen:
|
||||
return None
|
||||
for tier in _VERDICT_PRECEDENCE:
|
||||
if tier in seen:
|
||||
return tier
|
||||
return None
|
||||
|
||||
|
||||
async def _enrich_one(
|
||||
attacker_uuid: str,
|
||||
ip: str,
|
||||
providers: list[IntelProvider],
|
||||
ttl_hours: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Fan out across providers for a single attacker and assemble the row.
|
||||
|
||||
Keyed on ``attacker_uuid`` for the eventual upsert; the IP is the wire
|
||||
value the providers see and is denormalised onto the row for SIEM /
|
||||
audit consumers.
|
||||
"""
|
||||
results: list[IntelResult] = await asyncio.gather(
|
||||
*(p.lookup(ip) for p in providers),
|
||||
return_exceptions=False, # providers contractually never raise
|
||||
)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
row: dict[str, Any] = {
|
||||
"attacker_uuid": attacker_uuid,
|
||||
"attacker_ip": ip,
|
||||
"cached_at": now,
|
||||
"expires_at": now + timedelta(hours=ttl_hours),
|
||||
}
|
||||
verdicts: list[Optional[str]] = []
|
||||
for result in results:
|
||||
if result.error:
|
||||
log.warning(
|
||||
"intel: provider %s failed for ip=%s: %s",
|
||||
result.provider, ip, result.error,
|
||||
)
|
||||
continue
|
||||
row.update(result.column_updates)
|
||||
verdicts.append(result.verdict)
|
||||
row["aggregate_verdict"] = _aggregate(verdicts)
|
||||
return row
|
||||
|
||||
|
||||
async def run_intel_loop(
|
||||
repo: BaseRepository,
|
||||
*,
|
||||
poll_interval_secs: float = _DEFAULT_POLL_SECS,
|
||||
ttl_hours: int = _DEFAULT_TTL_HOURS,
|
||||
backfill_batch: int = _BACKFILL_BATCH,
|
||||
providers: Optional[list[IntelProvider]] = None,
|
||||
shutdown: Optional[asyncio.Event] = None,
|
||||
) -> None:
|
||||
"""Run the intel-enrichment loop until cancelled.
|
||||
|
||||
*providers* defaults to :func:`get_intel_providers` — tests pass a
|
||||
list of fakes. *shutdown* is an optional external stop signal; the
|
||||
loop also exits cleanly on ``CancelledError`` and ``KeyboardInterrupt``.
|
||||
"""
|
||||
if providers is None:
|
||||
providers = get_intel_providers()
|
||||
log.info(
|
||||
"intel worker started providers=%s poll=%ss ttl=%sh",
|
||||
[p.name for p in providers], poll_interval_secs, ttl_hours,
|
||||
)
|
||||
|
||||
bus: Optional[BaseBus] = None
|
||||
wake = asyncio.Event()
|
||||
wake_tasks: list[asyncio.Task] = []
|
||||
heartbeat_task: Optional[asyncio.Task] = None
|
||||
try:
|
||||
candidate = get_bus(client_name="enrich")
|
||||
await candidate.connect()
|
||||
bus = candidate
|
||||
wake_tasks.append(asyncio.create_task(
|
||||
_wake_on(bus, wake, _topics.attacker(_topics.ATTACKER_OBSERVED)),
|
||||
))
|
||||
wake_tasks.append(asyncio.create_task(
|
||||
_wake_on(bus, wake, _topics.attacker(_topics.ATTACKER_SCORED)),
|
||||
))
|
||||
heartbeat_task = asyncio.create_task(
|
||||
_run_health_heartbeat(bus, "enrich"),
|
||||
)
|
||||
wake_tasks.append(asyncio.create_task(
|
||||
_run_control_listener_signal(bus, "enrich"),
|
||||
))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning(
|
||||
"intel worker: bus unavailable, running in poll-only mode: %s",
|
||||
exc,
|
||||
)
|
||||
|
||||
if shutdown is None:
|
||||
shutdown = asyncio.Event()
|
||||
|
||||
try:
|
||||
while not shutdown.is_set():
|
||||
try:
|
||||
pending = await repo.get_unenriched_attackers(
|
||||
limit=backfill_batch,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception("intel worker: backfill query failed")
|
||||
pending = []
|
||||
|
||||
if pending and providers:
|
||||
for entry in pending:
|
||||
if shutdown.is_set():
|
||||
break
|
||||
attacker_uuid = entry["uuid"]
|
||||
ip = entry["ip"]
|
||||
try:
|
||||
row = await _enrich_one(
|
||||
attacker_uuid, ip, providers, ttl_hours,
|
||||
)
|
||||
await repo.upsert_attacker_intel(row)
|
||||
await publish_safely(
|
||||
bus,
|
||||
_topics.attacker(_topics.ATTACKER_INTEL_ENRICHED),
|
||||
{
|
||||
"attacker_uuid": attacker_uuid,
|
||||
"attacker_ip": ip,
|
||||
"aggregate_verdict": row.get("aggregate_verdict"),
|
||||
"providers": [p.name for p in providers],
|
||||
},
|
||||
event_type=_topics.ATTACKER_INTEL_ENRICHED,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
log.exception(
|
||||
"intel worker: enrichment failed for uuid=%s ip=%s",
|
||||
attacker_uuid, ip,
|
||||
)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
wake.wait(), timeout=float(poll_interval_secs),
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
wake.clear()
|
||||
except (asyncio.CancelledError, KeyboardInterrupt):
|
||||
log.info("intel worker stopped")
|
||||
finally:
|
||||
for t in wake_tasks:
|
||||
t.cancel()
|
||||
if heartbeat_task is not None:
|
||||
heartbeat_task.cancel()
|
||||
for t in (*wake_tasks, heartbeat_task):
|
||||
if t is None:
|
||||
continue
|
||||
with contextlib.suppress(asyncio.CancelledError, Exception):
|
||||
await t
|
||||
if bus is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await bus.close()
|
||||
|
||||
|
||||
async def _wake_on(bus: BaseBus, wake: asyncio.Event, pattern: str) -> None:
|
||||
"""Flip *wake* every time *pattern* fires on the bus.
|
||||
|
||||
Survives transient subscriber errors by logging and exiting; the
|
||||
poll-interval fallback keeps the loop alive in poll-only mode.
|
||||
"""
|
||||
try:
|
||||
sub = bus.subscribe(pattern)
|
||||
async with sub:
|
||||
async for _event in sub:
|
||||
wake.set()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning(
|
||||
"intel worker: subscriber for %s died (%s); falling back to poll",
|
||||
pattern, exc,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["run_intel_loop"]
|
||||
Reference in New Issue
Block a user