Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d7a2b5b9cf | |||
| c918538f35 | |||
| beaa604811 | |||
| 7b0ff127c3 | |||
| 419172ecfb | |||
| fcc9a9aad1 | |||
| 1a765854ec | |||
| a5e11f7d86 | |||
| 74096b6df0 |
39
CHANGELOG.md
39
CHANGELOG.md
@@ -5,6 +5,43 @@ All notable changes to DECNET are documented here.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [1.2.0] - 2026-06-18
|
||||
|
||||
Prefork worker consolidation — share the import floor across *separate* processes
|
||||
(own GIL, full isolation) via copy-on-write, for the heavy/isolation-critical
|
||||
workers the in-process supervisor can't co-host.
|
||||
|
||||
### Added
|
||||
- `decnet.prefork` — prefork supervisor primitive: a master imports the base
|
||||
floor once, then forks one child per worker (own process/GIL, CoW-shared
|
||||
floor), reaps and restarts with backoff, and shuts down gracefully. CoW
|
||||
viability measured on CPython 3.14 (idle child ~1 MB private, ~71 MB shared;
|
||||
`gc.freeze()` unnecessary thanks to PEP 683 immortal objects).
|
||||
- `decnet fleet <name>` — prefork master that imports the shared base floor once
|
||||
then forks one child per worker. First fleet `heavy` = profiler + ttp (DB-only,
|
||||
process-isolated heavy tier); systemd unit `decnet-fleet-heavy.service`
|
||||
Conflicts= the units it replaces and carries no extra privilege.
|
||||
Verified live: fleet footprint ≈412 MB Pss (master 67 + profiler 81 + ttp 264)
|
||||
vs 661 MB standalone — profiler's RSS collapsed 353→110 MB (base floor now
|
||||
CoW-shared). ttp barely moved: its bulk is the privately-parsed ATT&CK bundle,
|
||||
which it alone consumes — so master-warming it was confirmed pointless and
|
||||
dropped. Lesson: prefork pays for base-floor-bound workers, not state-bound ones.
|
||||
|
||||
### Changed
|
||||
- MITRE ATT&CK Enterprise bundle pinned 19.0 → **19.1**. The bundle and its
|
||||
LICENSE now resolve from `decnet/data/` (hash-pinned in `attack_version.py`,
|
||||
fetched on demand via `python -m decnet.ttp.attack_stix fetch`, gitignored —
|
||||
not committed).
|
||||
|
||||
### Removed
|
||||
- Per-worker systemd unit templates superseded by consolidation:
|
||||
`decnet-{reconciler,enrich,orchestrator,mutator}` (→ `supervise batch`),
|
||||
`decnet-{clusterer,campaign-clusterer,attribution,reuse-correlator}`
|
||||
(→ `supervise cpu`), and `decnet-{profiler,ttp}` (→ `fleet heavy`).
|
||||
`decnet.target` now pulls in the 3 consolidated units. The underlying CLI
|
||||
commands remain for manual/standalone runs; a worker can be re-extracted to its
|
||||
own unit by editing the group/fleet spec.
|
||||
|
||||
## [1.1.1] - 2026-06-18
|
||||
|
||||
### Fixed
|
||||
@@ -66,5 +103,7 @@ own unit.
|
||||
|
||||
Initial 1.0 release. See tag `v1.0.0`.
|
||||
|
||||
[1.2.0]: https://git.resacachile.cl/anti/DECNET/compare/v1.1.1...v1.2.0
|
||||
[1.1.1]: https://git.resacachile.cl/anti/DECNET/compare/v1.1.0...v1.1.1
|
||||
[1.1.0]: https://git.resacachile.cl/anti/DECNET/compare/v1.0.0...v1.1.0
|
||||
[1.0.0]: https://git.resacachile.cl/anti/DECNET/releases/tag/v1.0.0
|
||||
|
||||
@@ -26,6 +26,7 @@ from . import (
|
||||
canary,
|
||||
db,
|
||||
deploy,
|
||||
fleet,
|
||||
forwarder,
|
||||
geoip,
|
||||
init,
|
||||
@@ -62,7 +63,7 @@ for _mod in (
|
||||
swarm,
|
||||
deploy, lifecycle, workers, inventory,
|
||||
web, profiler, orchestrator, realism, reconciler, sniffer, db,
|
||||
topology, bus, geoip, init, webhook, canary, ttp, supervise,
|
||||
topology, bus, geoip, init, webhook, canary, ttp, supervise, fleet,
|
||||
):
|
||||
_mod.register(app)
|
||||
|
||||
|
||||
94
decnet/cli/fleet.py
Normal file
94
decnet/cli/fleet.py
Normal file
@@ -0,0 +1,94 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""``decnet fleet <name>`` — prefork supervisor (DECNET 1.2).
|
||||
|
||||
Imports the shared base floor ONCE in the master, then forks one child process
|
||||
per worker (see :mod:`decnet.prefork`). Children share the floor via copy-on-write
|
||||
(measured ~71 MB shared / ~1 MB private per idle child on CPython 3.14) while
|
||||
keeping their OWN process and GIL — unlike ``decnet supervise``, which co-hosts
|
||||
workers as asyncio tasks in one shared-GIL process.
|
||||
|
||||
Use ``fleet`` for workers that must stay process-isolated (heavy resident state,
|
||||
sustained CPU) but shouldn't each re-import the world; use ``supervise`` for cheap
|
||||
co-resident IO workers.
|
||||
|
||||
CONSOLIDATION COSTS (same shape as ``supervise``):
|
||||
* Forked children inherit the master's privileges — a fleet's systemd unit
|
||||
carries the UNION of its members' caps. So group by privilege profile, not
|
||||
convenience. The ``heavy`` fleet is DB-only (no docker socket, no raw net).
|
||||
* To share via CoW the master pre-imports each worker's module BEFORE forking,
|
||||
so its RSS is large — but that RSS is the shared floor, not per-child cost.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import typer
|
||||
|
||||
from . import utils as _utils
|
||||
from .utils import console, log
|
||||
|
||||
_FLEETS = ("heavy",)
|
||||
|
||||
|
||||
def _build_fleet(name: str) -> dict:
|
||||
"""Return ``{worker_name: entry_thunk}`` for *name*.
|
||||
|
||||
Imports happen here, in the MASTER, before :func:`run_fleet` forks — that is
|
||||
what lets children share the imported code/objects via copy-on-write. Each
|
||||
thunk blocks running one worker; ``repo`` is initialized inside the child
|
||||
(post-fork) so every child opens its own pool, never a fork-inherited one.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
if name == "heavy":
|
||||
from decnet.profiler import attacker_profile_worker
|
||||
from decnet.ttp.worker import run_ttp_worker_loop
|
||||
from decnet.web.dependencies import repo
|
||||
|
||||
# Importing the worker modules here (in the master) is what lets children
|
||||
# share their code via CoW. Heavy per-worker runtime state (ATT&CK bundle,
|
||||
# ML) still loads lazily in each child — warming it in the master to share
|
||||
# it too is a future optimization, gated on a live RSS measurement showing
|
||||
# the big object graph actually CoW-shares rather than refcount-dirtying.
|
||||
def _profiler() -> None:
|
||||
async def _go() -> None:
|
||||
await repo.initialize()
|
||||
await attacker_profile_worker(repo, interval=60)
|
||||
asyncio.run(_go())
|
||||
|
||||
def _ttp() -> None:
|
||||
async def _go() -> None:
|
||||
await repo.initialize()
|
||||
await run_ttp_worker_loop(repo, poll_interval_secs=60.0)
|
||||
asyncio.run(_go())
|
||||
|
||||
return {"profiler": _profiler, "ttp": _ttp}
|
||||
|
||||
raise ValueError(f"unknown fleet: {name}")
|
||||
|
||||
|
||||
def register(app: typer.Typer) -> None:
|
||||
@app.command(name="fleet")
|
||||
def fleet_cmd(
|
||||
name: str = typer.Argument(
|
||||
..., help=f"Worker fleet to fork. One of: {', '.join(_FLEETS)}"
|
||||
),
|
||||
daemon: bool = typer.Option(
|
||||
False, "--daemon", "-d", help="Detach to background as a daemon process"
|
||||
),
|
||||
) -> None:
|
||||
"""Prefork a worker fleet: shared base floor (CoW), one child process per worker."""
|
||||
from decnet.prefork import run_fleet
|
||||
|
||||
if name not in _FLEETS:
|
||||
console.print(
|
||||
f"[red]unknown fleet {name!r}; known fleets: {', '.join(_FLEETS)}[/]"
|
||||
)
|
||||
raise typer.Exit(2)
|
||||
|
||||
if daemon:
|
||||
log.info("fleet %s daemonizing", name)
|
||||
_utils._daemonize()
|
||||
|
||||
log.info("fleet %s starting", name)
|
||||
console.print(f"[bold cyan]Fleet starting[/] {name} (prefork)")
|
||||
specs = _build_fleet(name)
|
||||
run_fleet(specs)
|
||||
7
decnet/data/.gitignore
vendored
Normal file
7
decnet/data/.gitignore
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
# MITRE ATT&CK STIX bundle + license live here but are NOT committed:
|
||||
# ~51MB and MITRE-licensed (fetched on demand, hash-pinned in attack_version.py).
|
||||
#
|
||||
# Populate locally / in CI with:
|
||||
# DECNET_ATTACK_CACHE_DIR=decnet/data python -m decnet.ttp.attack_stix fetch
|
||||
/enterprise-attack-*.json
|
||||
/LICENSE.txt
|
||||
140
decnet/prefork.py
Normal file
140
decnet/prefork.py
Normal file
@@ -0,0 +1,140 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Prefork supervisor — import the base floor ONCE in a master, then fork one
|
||||
child process per worker. Children share the ~70 MB import floor via
|
||||
copy-on-write.
|
||||
|
||||
Measured on CPython 3.14 (development/cow_probe.py): an idle forked child keeps
|
||||
~71 MB shared and dirties only ~1 MB private; a working child dirties ~26 MB
|
||||
(its own heap, not the floor). PEP 683 immortal objects keep module/code pages
|
||||
clean, so the classic refcount-dirties-CoW problem does not bite and gc.freeze()
|
||||
is unnecessary on 3.14.
|
||||
|
||||
Contrast with :mod:`decnet.supervisor` (asyncio tasks in ONE process, shared
|
||||
GIL): use that for cheap co-resident IO workers. Use prefork for workers that
|
||||
must keep their OWN process / GIL — CPU-heavy or isolation-critical — but
|
||||
shouldn't each re-import the world.
|
||||
|
||||
Each worker spec is a zero-arg callable that BLOCKS running the worker (e.g.
|
||||
``lambda: asyncio.run(profiler_worker(repo))``). It executes in the forked
|
||||
child; the master only forks, reaps, and restarts.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
log = logging.getLogger("decnet.prefork")
|
||||
|
||||
WorkerEntry = Callable[[], None]
|
||||
|
||||
|
||||
def run_fleet(
|
||||
specs: dict[str, WorkerEntry],
|
||||
*,
|
||||
max_backoff: float = 30.0,
|
||||
poll_interval: float = 0.2,
|
||||
stop_after: float | None = None,
|
||||
) -> None:
|
||||
"""Fork one child per worker and supervise them until SIGTERM/SIGINT.
|
||||
|
||||
A dead child is re-forked after exponential backoff (in-process
|
||||
``Restart=on-failure``). Backoff is tracked per worker and scheduled
|
||||
non-blockingly, so one worker's restart delay never stalls reaping of
|
||||
another. On shutdown, children get SIGTERM, then SIGKILL after a grace
|
||||
period.
|
||||
|
||||
``stop_after`` (seconds) is a test hook: cleanly shut the fleet down after
|
||||
that long instead of waiting for a signal.
|
||||
"""
|
||||
if not specs:
|
||||
return
|
||||
|
||||
children: dict[int, str] = {} # pid -> name
|
||||
backoff: dict[str, float] = {n: 1.0 for n in specs}
|
||||
due: dict[str, float] = {} # name -> earliest restart time
|
||||
stopping = {"flag": False}
|
||||
|
||||
def _request_stop(_signum: int, _frame: object) -> None:
|
||||
stopping["flag"] = True
|
||||
|
||||
signal.signal(signal.SIGTERM, _request_stop)
|
||||
signal.signal(signal.SIGINT, _request_stop)
|
||||
|
||||
def spawn(name: str) -> None:
|
||||
pid = os.fork()
|
||||
if pid == 0: # ---- child ----
|
||||
# Restore default signal handling so the worker's own asyncio
|
||||
# handlers (or KeyboardInterrupt) work as if launched standalone.
|
||||
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
||||
signal.signal(signal.SIGINT, signal.SIG_DFL)
|
||||
try:
|
||||
specs[name]()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
except BaseException: # noqa: BLE001 — last-resort child logging
|
||||
log.exception("prefork: worker %s raised", name)
|
||||
os._exit(1)
|
||||
os._exit(0)
|
||||
children[pid] = name # ---- parent ----
|
||||
log.info("prefork: spawned %s pid=%d", name, pid)
|
||||
|
||||
log.info("prefork: master pid=%d forking %d workers: %s",
|
||||
os.getpid(), len(specs), ", ".join(specs))
|
||||
for name in specs:
|
||||
spawn(name)
|
||||
|
||||
deadline = (time.monotonic() + stop_after) if stop_after is not None else None
|
||||
while not stopping["flag"]:
|
||||
if deadline is not None and time.monotonic() >= deadline:
|
||||
break
|
||||
now = time.monotonic()
|
||||
# Restart any workers whose backoff has elapsed.
|
||||
for name in [n for n, t in due.items() if now >= t]:
|
||||
del due[name]
|
||||
spawn(name)
|
||||
# Reap without blocking so concurrent crashes are all handled.
|
||||
try:
|
||||
pid, status = os.waitpid(-1, os.WNOHANG)
|
||||
except ChildProcessError:
|
||||
pid = 0
|
||||
if pid == 0:
|
||||
time.sleep(poll_interval)
|
||||
continue
|
||||
name = children.pop(pid, None)
|
||||
if name is None:
|
||||
continue
|
||||
code = os.waitstatus_to_exitcode(status)
|
||||
log.warning("prefork: %s (pid=%d) exited code=%d; restart in %.0fs",
|
||||
name, pid, code, backoff[name])
|
||||
due[name] = time.monotonic() + backoff[name]
|
||||
backoff[name] = min(backoff[name] * 2.0, max_backoff)
|
||||
|
||||
_shutdown(children)
|
||||
|
||||
|
||||
def _shutdown(children: dict[int, str], *, grace: float = 15.0) -> None:
|
||||
"""SIGTERM all children, reap within ``grace``, SIGKILL stragglers."""
|
||||
for pid in list(children):
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
children.pop(pid, None)
|
||||
deadline = time.monotonic() + grace
|
||||
while children and time.monotonic() < deadline:
|
||||
try:
|
||||
pid, _ = os.waitpid(-1, os.WNOHANG)
|
||||
except ChildProcessError:
|
||||
break
|
||||
if pid:
|
||||
children.pop(pid, None)
|
||||
else:
|
||||
time.sleep(0.1)
|
||||
for pid in list(children):
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
log.info("prefork: fleet shut down")
|
||||
@@ -16,12 +16,12 @@ from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
ATTACK_BUNDLE_VERSION: Final[str] = "19.0"
|
||||
ATTACK_BUNDLE_VERSION: Final[str] = "19.1"
|
||||
|
||||
# sha256 of the canonical MITRE-published enterprise-attack-19.0.json
|
||||
# sha256 of the canonical MITRE-published enterprise-attack-19.1.json
|
||||
# from https://github.com/mitre-attack/attack-stix-data.
|
||||
ATTACK_BUNDLE_SHA256: Final[str] = (
|
||||
"df520ea0775a57db7bff760145b02fed89290802913e056b7ed5970b02f3626a"
|
||||
"bdf1ce86a4e604214c5076d37ae4dcb322678afc528df8492e6fdc1b554f5da3"
|
||||
)
|
||||
|
||||
# Raw download URL for the pinned version.
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
[Unit]
|
||||
Description=DECNET Attribution Engine v0 (per-(identity, primitive) state machine)
|
||||
Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#attribution
|
||||
After=network-online.target decnet-bus.service
|
||||
Wants=network-online.target decnet-bus.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ user }}
|
||||
Group={{ group }}
|
||||
WorkingDirectory={{ install_dir }}
|
||||
EnvironmentFile=-{{ install_dir }}/.env.local
|
||||
Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.attribution.log
|
||||
# Subscribes to attacker.observation.> and, for each event, ensures a
|
||||
# stub AttackerIdentity row, runs the per-ValueKind merger over the
|
||||
# full identity-keyed observation series, upserts the derived state in
|
||||
# attribution_state, and publishes attribution.profile.state_changed
|
||||
# only on transition. Periodic tick (default 60s) fires
|
||||
# attribution.profile.multi_actor_suspected when >= 2 primitives flag
|
||||
# the same identity. Closes DEBT-051.
|
||||
ExecStart={{ venv_dir }}/bin/decnet attribution
|
||||
StandardOutput=append:/var/log/decnet/decnet.attribution.log
|
||||
StandardError=append:/var/log/decnet/decnet.attribution.log
|
||||
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
|
||||
# Security Hardening
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=full
|
||||
ProtectHome=read-only
|
||||
PrivateTmp=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
ReadWritePaths={{ install_dir }} /var/log/decnet
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=15
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,52 +0,0 @@
|
||||
[Unit]
|
||||
Description=DECNET Campaign Clusterer (identities → campaigns / operations)
|
||||
Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#campaign-clusterer
|
||||
After=network-online.target decnet-bus.service decnet-clusterer.service
|
||||
Wants=network-online.target decnet-bus.service decnet-clusterer.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ user }}
|
||||
Group={{ group }}
|
||||
WorkingDirectory={{ install_dir }}
|
||||
EnvironmentFile=-{{ install_dir }}/.env.local
|
||||
Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.campaign-clusterer.log
|
||||
# Subscribes to identity.>; falls back to a 60s slow-tick poll when
|
||||
# the bus is idle or unavailable. Reads AttackerIdentity rows,
|
||||
# projects them into the campaign-level similarity graph
|
||||
# (phase-handoff / shared-infra / temporal overlap / cohort), runs
|
||||
# union-find, writes campaigns rows + sets
|
||||
# attacker_identities.campaign_id, and publishes campaign.formed /
|
||||
# campaign.identity.assigned / campaign.merged / campaign.unmerged
|
||||
# plus the cross-family identity.campaign.assigned for identity-side
|
||||
# subscribers.
|
||||
#
|
||||
# Master-only: gated via MASTER_ONLY_COMMANDS in decnet/cli/gating.py.
|
||||
# Sits one layer above decnet-clusterer (the After=/Wants= ensures the
|
||||
# identity layer is up first; the campaign clusterer then wakes on
|
||||
# identity.> events fired by it).
|
||||
ExecStart={{ venv_dir }}/bin/decnet campaign-clusterer
|
||||
StandardOutput=append:/var/log/decnet/decnet.campaign-clusterer.log
|
||||
StandardError=append:/var/log/decnet/decnet.campaign-clusterer.log
|
||||
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
|
||||
# Security Hardening
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=full
|
||||
ProtectHome=read-only
|
||||
PrivateTmp=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
ReadWritePaths={{ install_dir }} /var/log/decnet
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=15
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,47 +0,0 @@
|
||||
[Unit]
|
||||
Description=DECNET Identity Clusterer (per-IP observations → identities)
|
||||
Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#identity-clusterer
|
||||
After=network-online.target decnet-bus.service
|
||||
Wants=network-online.target decnet-bus.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ user }}
|
||||
Group={{ group }}
|
||||
WorkingDirectory={{ install_dir }}
|
||||
EnvironmentFile=-{{ install_dir }}/.env.local
|
||||
Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.clusterer.log
|
||||
# Subscribes to attacker.observed and attacker.scored; falls back to a
|
||||
# 60s slow-tick poll when the bus is idle or unavailable. Reads
|
||||
# Attacker rows, projects per-IP observations into the similarity
|
||||
# graph (JA3 / HASSH / payload-hash / C2-endpoint), runs union-find,
|
||||
# writes attacker_identities rows + sets attackers.identity_id, and
|
||||
# publishes identity.formed / identity.observation.linked /
|
||||
# identity.merged / identity.unmerged.
|
||||
#
|
||||
# Master-only: gated via MASTER_ONLY_COMMANDS in decnet/cli/gating.py.
|
||||
ExecStart={{ venv_dir }}/bin/decnet clusterer
|
||||
StandardOutput=append:/var/log/decnet/decnet.clusterer.log
|
||||
StandardError=append:/var/log/decnet/decnet.clusterer.log
|
||||
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
|
||||
# Security Hardening
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=full
|
||||
ProtectHome=read-only
|
||||
PrivateTmp=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
ReadWritePaths={{ install_dir }} /var/log/decnet
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=15
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,47 +0,0 @@
|
||||
[Unit]
|
||||
Description=DECNET Threat-Intel Enrichment (GreyNoise + AbuseIPDB + abuse.ch)
|
||||
Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#intel-enrichment
|
||||
After=network-online.target decnet-bus.service
|
||||
Wants=network-online.target decnet-bus.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ user }}
|
||||
Group={{ group }}
|
||||
WorkingDirectory={{ install_dir }}
|
||||
EnvironmentFile=-{{ install_dir }}/.env.local
|
||||
Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.enrich.log
|
||||
# Subscribes to attacker.observed and attacker.scored; falls back to a 60s
|
||||
# slow-tick poll when the bus is idle or unavailable. Per attacker IP fans
|
||||
# out across the configured intel providers, writes the aggregate verdict
|
||||
# to attacker_intel, and publishes attacker.intel.enriched.
|
||||
#
|
||||
# Free-tier API keys are read from .env.local:
|
||||
# DECNET_GREYNOISE_API_KEY= (optional, lifts rate limit)
|
||||
# DECNET_ABUSEIPDB_API_KEY= (required for AbuseIPDB lookups)
|
||||
# DECNET_THREATFOX_API_KEY= (optional, lifts rate limit)
|
||||
ExecStart={{ venv_dir }}/bin/decnet enrich
|
||||
StandardOutput=append:/var/log/decnet/decnet.enrich.log
|
||||
StandardError=append:/var/log/decnet/decnet.enrich.log
|
||||
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
|
||||
# Security Hardening
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=full
|
||||
ProtectHome=read-only
|
||||
PrivateTmp=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
ReadWritePaths={{ install_dir }} /var/log/decnet
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=15
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
48
deploy/decnet-fleet-heavy.service.j2
Normal file
48
deploy/decnet-fleet-heavy.service.j2
Normal file
@@ -0,0 +1,48 @@
|
||||
[Unit]
|
||||
Description=DECNET Heavy Fleet (prefork master forking profiler + ttp as CoW-sharing children)
|
||||
Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#fleet
|
||||
After=network-online.target decnet-bus.service
|
||||
Wants=network-online.target decnet-bus.service
|
||||
# Replaces the individual decnet-profiler / decnet-ttp units. Do NOT enable
|
||||
# those alongside this one.
|
||||
Conflicts=decnet-profiler.service decnet-ttp.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ user }}
|
||||
Group={{ group }}
|
||||
WorkingDirectory={{ install_dir }}
|
||||
EnvironmentFile=-{{ install_dir }}/.env.local
|
||||
Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.fleet-heavy.log
|
||||
ExecStart={{ venv_dir }}/bin/decnet fleet heavy
|
||||
StandardOutput=append:/var/log/decnet/decnet.fleet-heavy.log
|
||||
StandardError=append:/var/log/decnet/decnet.fleet-heavy.log
|
||||
|
||||
# Prefork master imports the shared base floor once, then forks one child per
|
||||
# worker; children share the floor via copy-on-write. Both members are DB-only
|
||||
# (no docker socket, no raw sockets) so this unit carries NO extra privilege —
|
||||
# the prefork privilege-union cost is nil for this fleet by construction.
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
|
||||
# Security Hardening
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=full
|
||||
# Dev installs under /home need ProtectHome=read-only: the ttp child reads
|
||||
# ./rules/ttp/ from the project root (read-only suffices — YAML reads only).
|
||||
ProtectHome=read-only
|
||||
PrivateTmp=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
ReadWritePaths={{ install_dir }} /var/log/decnet
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
# Master forwards SIGTERM to children and reaps; give it room for both to drain.
|
||||
TimeoutStopSec=25
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,41 +0,0 @@
|
||||
[Unit]
|
||||
Description=DECNET Mutator (runtime fleet mutation watch loop)
|
||||
Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#mutator
|
||||
After=network-online.target docker.service decnet-bus.service
|
||||
Wants=network-online.target decnet-bus.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ user }}
|
||||
Group={{ group }}
|
||||
# Mutator recomposes decky services via docker compose.
|
||||
SupplementaryGroups=docker
|
||||
WorkingDirectory={{ install_dir }}
|
||||
EnvironmentFile=-{{ install_dir }}/.env.local
|
||||
Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.mutator.log
|
||||
ExecStart={{ venv_dir }}/bin/decnet mutate --watch
|
||||
StandardOutput=append:/var/log/decnet/decnet.mutator.log
|
||||
StandardError=append:/var/log/decnet/decnet.mutator.log
|
||||
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
|
||||
# Security Hardening
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=full
|
||||
ProtectHome=read-only
|
||||
PrivateTmp=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
ReadWritePaths={{ install_dir }} /var/log/decnet
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=15
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,50 +0,0 @@
|
||||
[Unit]
|
||||
Description=DECNET Orchestrator (synthetic life-injection — inter-decky traffic, file plants, email drops)
|
||||
Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#orchestrator
|
||||
After=network-online.target decnet-bus.service
|
||||
Wants=network-online.target decnet-bus.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ user }}
|
||||
Group={{ group }}
|
||||
WorkingDirectory={{ install_dir }}
|
||||
EnvironmentFile=-{{ install_dir }}/.env.local
|
||||
Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.orchestrator.log
|
||||
# Realism content engine — LLM + persona-pool config used by the
|
||||
# email + (post-stage-6) file-class enrichment paths. See
|
||||
# decnet/realism/llm/factory.py and decnet/realism/personas_pool.py.
|
||||
Environment=DECNET_REALISM_LLM=ollama
|
||||
Environment=DECNET_REALISM_MODEL=llama3.1
|
||||
Environment=DECNET_REALISM_TIMEOUT=60
|
||||
Environment=DECNET_REALISM_PERSONAS=/etc/decnet/email_personas.json
|
||||
ExecStart={{ venv_dir }}/bin/decnet orchestrate
|
||||
StandardOutput=append:/var/log/decnet/decnet.orchestrator.log
|
||||
StandardError=append:/var/log/decnet/decnet.orchestrator.log
|
||||
|
||||
# The orchestrator drives `docker exec` against decky containers, so it
|
||||
# needs membership in the docker group. It does NOT bind to the network,
|
||||
# launch new containers, or write outside its own logs and install dir.
|
||||
SupplementaryGroups=docker
|
||||
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
|
||||
# Security Hardening
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=full
|
||||
ProtectHome=read-only
|
||||
PrivateTmp=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
ReadWritePaths={{ install_dir }} /var/log/decnet
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=15
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,38 +0,0 @@
|
||||
[Unit]
|
||||
Description=DECNET Profiler (attacker profiling and scoring)
|
||||
Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#profiler
|
||||
After=network-online.target decnet-bus.service
|
||||
Wants=network-online.target decnet-bus.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ user }}
|
||||
Group={{ group }}
|
||||
WorkingDirectory={{ install_dir }}
|
||||
EnvironmentFile=-{{ install_dir }}/.env.local
|
||||
Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.profiler.log
|
||||
ExecStart={{ venv_dir }}/bin/decnet profiler
|
||||
StandardOutput=append:/var/log/decnet/decnet.profiler.log
|
||||
StandardError=append:/var/log/decnet/decnet.profiler.log
|
||||
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
|
||||
# Security Hardening
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=full
|
||||
ProtectHome=read-only
|
||||
PrivateTmp=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
ReadWritePaths={{ install_dir }} /var/log/decnet
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=15
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,47 +0,0 @@
|
||||
[Unit]
|
||||
Description=DECNET Fleet Reconciler (converges decnet-state.json ↔ fleet_deckies DB ↔ docker)
|
||||
Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#reconciler
|
||||
After=network-online.target decnet-bus.service
|
||||
Wants=network-online.target decnet-bus.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ user }}
|
||||
Group={{ group }}
|
||||
WorkingDirectory={{ install_dir }}
|
||||
EnvironmentFile=-{{ install_dir }}/.env.local
|
||||
Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.reconciler.log
|
||||
ExecStart={{ venv_dir }}/bin/decnet reconcile
|
||||
StandardOutput=append:/var/log/decnet/decnet.reconciler.log
|
||||
StandardError=append:/var/log/decnet/decnet.reconciler.log
|
||||
|
||||
# The reconciler queries the docker daemon (via `docker.from_env()`) to
|
||||
# observe per-container state. Membership in the docker group lets it
|
||||
# read /var/run/docker.sock without root. It does NOT exec into
|
||||
# containers, bind to the network, or spawn new containers.
|
||||
SupplementaryGroups=docker
|
||||
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
|
||||
# Security Hardening
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=full
|
||||
ProtectHome=read-only
|
||||
PrivateTmp=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
# Read-only access to /var/lib/decnet so we can read decnet-state.json.
|
||||
# Read-write access only to install_dir + log dir.
|
||||
ReadOnlyPaths=/var/lib/decnet
|
||||
ReadWritePaths={{ install_dir }} /var/log/decnet
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=15
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,41 +0,0 @@
|
||||
[Unit]
|
||||
Description=DECNET Credential-Reuse Correlator (cross-target secret-reuse detection)
|
||||
Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#reuse-correlator
|
||||
After=network-online.target decnet-bus.service
|
||||
Wants=network-online.target decnet-bus.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ user }}
|
||||
Group={{ group }}
|
||||
WorkingDirectory={{ install_dir }}
|
||||
EnvironmentFile=-{{ install_dir }}/.env.local
|
||||
Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.reuse-correlator.log
|
||||
# Subscribes to credential.captured and attacker.observed; falls back to
|
||||
# a 60s slow-tick poll when the bus is idle or unavailable. Publishes
|
||||
# credential.reuse.detected once per new/grown finding.
|
||||
ExecStart={{ venv_dir }}/bin/decnet reuse-correlate
|
||||
StandardOutput=append:/var/log/decnet/decnet.reuse-correlator.log
|
||||
StandardError=append:/var/log/decnet/decnet.reuse-correlator.log
|
||||
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
|
||||
# Security Hardening
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=full
|
||||
ProtectHome=read-only
|
||||
PrivateTmp=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
ReadWritePaths={{ install_dir }} /var/log/decnet
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=15
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,57 +0,0 @@
|
||||
[Unit]
|
||||
Description=DECNET TTP Tagger (MITRE ATT&CK technique tagging)
|
||||
Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#ttp-tagger
|
||||
After=network-online.target decnet-bus.service decnet-clusterer.service decnet-enrich.service decnet-reuse-correlator.service
|
||||
Wants=network-online.target decnet-bus.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ user }}
|
||||
Group={{ group }}
|
||||
WorkingDirectory={{ install_dir }}
|
||||
EnvironmentFile=-{{ install_dir }}/.env.local
|
||||
Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.ttp.log
|
||||
# Subscribes to attacker.session.ended (primary), attacker.observed,
|
||||
# attacker.intel.enriched, identity.formed, identity.merged,
|
||||
# credential.reuse.detected, email.received, and canary.> ; falls back
|
||||
# to a 60s slow-tick poll when the bus is idle or unavailable. Each
|
||||
# event is dispatched through the CompositeTagger (RuleEngine +
|
||||
# Behavioral / Intel / Email / CanaryFingerprint / Identity /
|
||||
# Credential lifters), persisted via the idempotent INSERT OR IGNORE
|
||||
# repo write, and ttp.tagged + ttp.rule.fired.<technique_id> are
|
||||
# published only when the insert returned a non-zero rowcount
|
||||
# (loop-prevention invariant — see TTP_TAGGING.md §"Bus topics").
|
||||
#
|
||||
# Master-only: gated via MASTER_ONLY_COMMANDS in decnet/cli/gating.py.
|
||||
# Sits one layer above the identity / intel / reuse-correlator
|
||||
# workers — the After= dependencies ensure their bus topics are live
|
||||
# before the TTP worker subscribes.
|
||||
ExecStart={{ venv_dir }}/bin/decnet ttp
|
||||
StandardOutput=append:/var/log/decnet/decnet.ttp.log
|
||||
StandardError=append:/var/log/decnet/decnet.ttp.log
|
||||
|
||||
CapabilityBoundingSet=
|
||||
AmbientCapabilities=
|
||||
|
||||
# Security Hardening
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=full
|
||||
# Dev installs under /home need ProtectHome=read-only (the worker
|
||||
# reads ./rules/ttp/ from the project root, which lives under /home
|
||||
# on dev boxes — read-only suffices because the FilesystemRuleStore
|
||||
# only reads YAMLs, never writes).
|
||||
ProtectHome=read-only
|
||||
PrivateTmp=yes
|
||||
ProtectKernelTunables=yes
|
||||
ProtectKernelModules=yes
|
||||
ProtectControlGroups=yes
|
||||
RestrictSUIDSGID=yes
|
||||
LockPersonality=yes
|
||||
ReadWritePaths={{ install_dir }} /var/log/decnet
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=15
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -5,23 +5,21 @@ Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers
|
||||
# heartbeats to it), then the API + data-plane workers. systemd resolves the
|
||||
# actual ordering via each unit's own After=/Wants= on decnet-bus.service —
|
||||
# this target is a convenience grouping, not an ordering primitive.
|
||||
# Consolidated since 1.1/1.2: the batch (reconcile/enrich/orchestrate/mutate)
|
||||
# and cpu (clusterer/campaign-clusterer/attribution/reuse-correlate) supervisor
|
||||
# groups and the heavy (profiler/ttp) prefork fleet replace their per-worker
|
||||
# units. The standalone workers below kept their own units.
|
||||
Wants=decnet-bus.service \
|
||||
decnet-api.service \
|
||||
decnet-web.service \
|
||||
decnet-collector.service \
|
||||
decnet-profiler.service \
|
||||
decnet-sniffer.service \
|
||||
decnet-prober.service \
|
||||
decnet-mutator.service \
|
||||
decnet-reconciler.service \
|
||||
decnet-reuse-correlator.service \
|
||||
decnet-enrich.service \
|
||||
decnet-clusterer.service \
|
||||
decnet-campaign-clusterer.service \
|
||||
decnet-ttp.service \
|
||||
decnet-webhook.service \
|
||||
decnet-canary.service \
|
||||
decnet-orchestrator.service
|
||||
decnet-supervise-batch.service \
|
||||
decnet-supervise-cpu.service \
|
||||
decnet-fleet-heavy.service
|
||||
After=decnet-bus.service
|
||||
|
||||
[Install]
|
||||
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "decnet"
|
||||
version = "1.1.1"
|
||||
version = "1.2.0"
|
||||
description = "Deception network: deploy honeypot deckies that appear as real LAN hosts"
|
||||
readme = "README.md"
|
||||
authors = [{ name = "Samuel Paschuan", email = "samuel.paschuan@xmartlab.com" }]
|
||||
|
||||
35
tests/cli/test_fleet.py
Normal file
35
tests/cli/test_fleet.py
Normal file
@@ -0,0 +1,35 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""CLI surface for ``decnet fleet`` (DECNET 1.2 prefork). The fork/restart
|
||||
mechanism itself is covered by tests/test_prefork.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from decnet.cli import app
|
||||
from decnet.cli.fleet import _FLEETS, _build_fleet
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
def test_fleet_is_registered():
|
||||
result = runner.invoke(app, ["fleet", "--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "fleet" in result.stdout.lower()
|
||||
|
||||
|
||||
def test_unknown_fleet_exits_2():
|
||||
result = runner.invoke(app, ["fleet", "not-a-fleet"])
|
||||
assert result.exit_code == 2
|
||||
assert "unknown fleet" in result.stdout
|
||||
|
||||
|
||||
def test_heavy_fleet_builds_expected_workers():
|
||||
# _build_fleet imports worker modules + builds thunks but runs nothing
|
||||
# (no fork, no repo.initialize) — safe to call in-process.
|
||||
specs = _build_fleet("heavy")
|
||||
assert set(specs) == {"profiler", "ttp"}
|
||||
assert all(callable(t) for t in specs.values())
|
||||
|
||||
|
||||
def test_heavy_is_known():
|
||||
assert "heavy" in _FLEETS
|
||||
@@ -1,152 +0,0 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Smoke tests for the orchestrator systemd unit + decnet.target wiring.
|
||||
|
||||
These don't exercise systemd (the test host wouldn't have it); they
|
||||
just assert the static contents of ``deploy/decnet-orchestrator.service.j2``
|
||||
and ``deploy/decnet.target`` match what ``decnet init`` will install.
|
||||
|
||||
Anti-regressions for two specific failure modes:
|
||||
|
||||
1. After the realism migration (stage 5), ``decnet-emailgen.service``
|
||||
is gone — the orchestrator covers the email branch. A regression
|
||||
that re-introduces the emailgen unit file or the ``decnet.target``
|
||||
entry would only surface on a fresh host install; cheap to catch
|
||||
here.
|
||||
2. The orchestrator unit must ship the ``DECNET_REALISM_*`` env block
|
||||
so the LLM enrichment + persona-pool path are configurable per
|
||||
host without editing the .j2.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
DEPLOY = REPO / "deploy"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unit_text() -> str:
|
||||
return (DEPLOY / "decnet-orchestrator.service.j2").read_text()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def target_text() -> str:
|
||||
return (DEPLOY / "decnet.target").read_text()
|
||||
|
||||
|
||||
# ── orchestrator unit ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_orchestrator_unit_exists():
|
||||
assert (DEPLOY / "decnet-orchestrator.service.j2").exists()
|
||||
|
||||
|
||||
def test_orchestrator_unit_uses_orchestrate_subcommand(unit_text):
|
||||
assert "decnet orchestrate" in unit_text
|
||||
|
||||
|
||||
def test_orchestrator_unit_has_docker_supplementary_group(unit_text):
|
||||
"""SSHDriver shells `docker exec` against decky containers — without
|
||||
this group the worker can't reach the docker socket."""
|
||||
assert "SupplementaryGroups=docker" in unit_text
|
||||
|
||||
|
||||
def test_orchestrator_unit_orders_after_bus(unit_text):
|
||||
"""Bus must be up first so heartbeats publish from the start."""
|
||||
assert "After=network-online.target decnet-bus.service" in unit_text
|
||||
assert "Wants=network-online.target decnet-bus.service" in unit_text
|
||||
|
||||
|
||||
def test_orchestrator_unit_has_security_hardening(unit_text):
|
||||
for directive in (
|
||||
"NoNewPrivileges=yes",
|
||||
"ProtectSystem=full",
|
||||
"ProtectHome=read-only",
|
||||
"PrivateTmp=yes",
|
||||
"ProtectKernelTunables=yes",
|
||||
"ProtectKernelModules=yes",
|
||||
"ProtectControlGroups=yes",
|
||||
"RestrictSUIDSGID=yes",
|
||||
"LockPersonality=yes",
|
||||
):
|
||||
assert directive in unit_text, f"missing {directive}"
|
||||
|
||||
|
||||
def test_orchestrator_unit_writes_to_log_dir(unit_text):
|
||||
assert "/var/log/decnet/decnet.orchestrator.log" in unit_text
|
||||
assert "ReadWritePaths={{ install_dir }} /var/log/decnet" in unit_text
|
||||
|
||||
|
||||
def test_orchestrator_unit_restart_on_failure(unit_text):
|
||||
assert "Restart=on-failure" in unit_text
|
||||
|
||||
|
||||
def test_orchestrator_unit_carries_realism_env_block(unit_text):
|
||||
"""Stage 5 + 6 contract: the orchestrator's LLM enrichment and
|
||||
persona-pool path are configured per host via DECNET_REALISM_*
|
||||
env vars. Shipping them in the .j2 means an operator who never
|
||||
drops a .env.local still gets sane defaults."""
|
||||
for var in (
|
||||
"DECNET_REALISM_LLM",
|
||||
"DECNET_REALISM_MODEL",
|
||||
"DECNET_REALISM_TIMEOUT",
|
||||
"DECNET_REALISM_PERSONAS",
|
||||
):
|
||||
assert var in unit_text, f"missing {var} in unit"
|
||||
|
||||
|
||||
def test_orchestrator_unit_does_not_carry_legacy_emailgen_envs(unit_text):
|
||||
"""Pre-v1 clean break per the realism migration: the
|
||||
DECNET_EMAILGEN_* env vars are no longer read. Carrying them in
|
||||
the unit would mislead operators into thinking they still apply."""
|
||||
for legacy in (
|
||||
"DECNET_EMAILGEN_LLM",
|
||||
"DECNET_EMAILGEN_MODEL",
|
||||
"DECNET_EMAILGEN_TIMEOUT",
|
||||
"DECNET_EMAILGEN_PERSONAS",
|
||||
):
|
||||
assert legacy not in unit_text, (
|
||||
f"legacy env {legacy} still referenced; clean-break broken"
|
||||
)
|
||||
|
||||
|
||||
# ── decnet.target ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_target_wants_orchestrator(target_text):
|
||||
assert "decnet-orchestrator.service" in target_text
|
||||
|
||||
|
||||
def test_target_does_not_want_emailgen(target_text):
|
||||
"""Stage 5 of the realism migration deleted decnet-emailgen.service.
|
||||
A fresh `decnet init` against a target file that still mentions it
|
||||
fails systemctl start with `Unit decnet-emailgen.service could not
|
||||
be found`, blocking the whole target. Anti-regression."""
|
||||
assert "decnet-emailgen.service" not in target_text
|
||||
|
||||
|
||||
def test_target_wants_canary(target_text):
|
||||
"""Canary worker is a peer of orchestrator; both are part of the
|
||||
realism + callback story. Bundle check."""
|
||||
assert "decnet-canary.service" in target_text
|
||||
|
||||
|
||||
def test_target_orders_after_bus(target_text):
|
||||
"""Whole target depends on the bus being up."""
|
||||
assert "After=decnet-bus.service" in target_text
|
||||
|
||||
|
||||
# ── unit file no longer exists ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_emailgen_unit_template_is_gone():
|
||||
"""The pre-collapse ``deploy/decnet-emailgen.service.j2`` must stay
|
||||
deleted. A future commit that re-creates it (e.g. by accident
|
||||
during a partial revert) would break the realism migration's
|
||||
service-collapse contract."""
|
||||
assert not (DEPLOY / "decnet-emailgen.service.j2").exists(), (
|
||||
"decnet-emailgen.service.j2 reappeared — service collapse undone?"
|
||||
)
|
||||
55
tests/prefork_driver.py
Normal file
55
tests/prefork_driver.py
Normal file
@@ -0,0 +1,55 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Standalone driver for the prefork supervisor — runnable directly OR via
|
||||
tests/test_prefork.py (which execs it in a subprocess so no fork happens inside
|
||||
the pytest/xdist worker).
|
||||
|
||||
python tests/prefork_driver.py <out_dir>
|
||||
|
||||
Forks two fake workers under decnet.prefork.run_fleet:
|
||||
* "tick" — append a line every 0.2s forever (proves a worker runs & stays up)
|
||||
* "crasher" — write a marker then exit(1) (proves restart-on-crash)
|
||||
Runs for ~2s via stop_after, then shuts the fleet down. Writes results into
|
||||
<out_dir>; the caller asserts on them.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
# Running this file as a script puts its own dir (tests/) on sys.path[0], which
|
||||
# shadows the stdlib `logging` via tests/logging/. Drop it before importing
|
||||
# decnet (still importable — it's installed in the venv).
|
||||
if sys.path and os.path.basename(sys.path[0]) == "tests":
|
||||
sys.path.pop(0)
|
||||
|
||||
from decnet.prefork import run_fleet # noqa: E402
|
||||
|
||||
|
||||
def main(out: str) -> None:
|
||||
tick_log = os.path.join(out, "tick.log")
|
||||
crash_log = os.path.join(out, "crash.log")
|
||||
|
||||
def tick() -> None:
|
||||
while True:
|
||||
with open(tick_log, "a") as f:
|
||||
f.write("t\n")
|
||||
time.sleep(0.2)
|
||||
|
||||
def crasher() -> None:
|
||||
with open(crash_log, "a") as f:
|
||||
f.write("c\n")
|
||||
time.sleep(0.15)
|
||||
os._exit(1)
|
||||
|
||||
# Fast backoff so we observe multiple restarts inside the short window.
|
||||
run_fleet(
|
||||
{"tick": tick, "crasher": crasher},
|
||||
max_backoff=0.2,
|
||||
poll_interval=0.05,
|
||||
stop_after=2.0,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1] if len(sys.argv) > 1 else ".")
|
||||
40
tests/test_prefork.py
Normal file
40
tests/test_prefork.py
Normal file
@@ -0,0 +1,40 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Prefork supervisor behaviour, exercised via a subprocess driver so no fork
|
||||
happens inside the pytest/xdist worker (which would be unsafe).
|
||||
|
||||
Proves: workers fork and run, a crashing worker is restarted with backoff, and
|
||||
the fleet shuts down cleanly (stop_after returns, no orphaned children).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def test_prefork_runs_and_restarts(tmp_path: pathlib.Path):
|
||||
driver = pathlib.Path(__file__).parent / "prefork_driver.py"
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(driver), str(tmp_path)],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
assert proc.returncode == 0, f"driver failed:\n{proc.stderr}"
|
||||
|
||||
tick = (tmp_path / "tick.log").read_text().splitlines()
|
||||
crash = (tmp_path / "crash.log").read_text().splitlines()
|
||||
|
||||
# tick ran continuously for ~2s at 0.2s cadence → several lines.
|
||||
assert len(tick) >= 5, f"tick worker did not stay up: {len(tick)} lines"
|
||||
# crasher died fast and was restarted repeatedly → many markers.
|
||||
assert len(crash) >= 3, f"crasher was not restarted: {len(crash)} markers"
|
||||
|
||||
|
||||
def test_empty_fleet_returns(tmp_path: pathlib.Path):
|
||||
# run_fleet([]) must be a no-op, not hang.
|
||||
code = (
|
||||
"from decnet.prefork import run_fleet; run_fleet({}, stop_after=5)"
|
||||
)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "-c", code], capture_output=True, text=True, timeout=15
|
||||
)
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
@@ -20,7 +20,7 @@ from decnet.clustering import ukc
|
||||
from decnet.ttp import attack_stix
|
||||
from decnet.ttp.impl import intel_lifter
|
||||
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json"
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
@@ -19,7 +19,7 @@ from decnet.ttp import attack_stix
|
||||
from decnet.ttp.attack_catalog import technique_name
|
||||
|
||||
_RULES_DIR = Path(__file__).resolve().parents[2] / "rules" / "ttp"
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json"
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
|
||||
@@ -19,7 +19,7 @@ from decnet.ttp.attack_version import (
|
||||
ATTACK_LICENSE_SHA256,
|
||||
)
|
||||
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json"
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
@@ -15,7 +15,7 @@ import pytest
|
||||
|
||||
from decnet.ttp import attack_stix
|
||||
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json"
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
@@ -24,7 +24,7 @@ from decnet.ttp.impl._emit import emit_tags
|
||||
from decnet.ttp.impl.rule_engine import CompiledRule
|
||||
from decnet.ttp.store.base import RuleState
|
||||
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json"
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
@@ -19,7 +19,7 @@ from decnet.web.router.ttp.api_get_groups_for_technique import (
|
||||
api_groups_for_technique,
|
||||
)
|
||||
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json"
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
@@ -31,7 +31,7 @@ from decnet.ttp.data.intel_loader import (
|
||||
load_provider_mapping,
|
||||
)
|
||||
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json"
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json"
|
||||
_DATA_DIR = Path(__file__).resolve().parents[2] / "decnet" / "ttp" / "data" / "intel"
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from decnet.web.router.attackers.api_export_attacker_misp import (
|
||||
api_export_attacker_misp,
|
||||
)
|
||||
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json"
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json"
|
||||
_FAKE_USER: dict = {"uuid": "test-user", "role": "viewer"}
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"""Tests for GET /api/v1/attackers/{uuid}/export/stix.
|
||||
|
||||
Tests call the handler directly (no TestClient). The attack_stix bundle
|
||||
is pinned to the repo's enterprise-attack-19.0.json so Sighting and
|
||||
is pinned to the repo's decnet/data/enterprise-attack-19.1.json so Sighting and
|
||||
Relationship target_refs are real MITRE STIX IDs.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
@@ -20,7 +20,7 @@ from decnet.web.router.attackers.api_export_attacker_stix import (
|
||||
api_export_attacker_stix,
|
||||
)
|
||||
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json"
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json"
|
||||
_FAKE_USER: dict = {"uuid": "test-user", "role": "viewer"}
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from decnet.web.router.attackers.api_export_attackers_misp import (
|
||||
api_export_attackers_misp,
|
||||
)
|
||||
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json"
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json"
|
||||
_FAKE_USER: dict = {"uuid": "test-user", "role": "viewer"}
|
||||
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from decnet.web.router.attackers.api_export_attackers_stix import (
|
||||
api_export_attackers_stix,
|
||||
)
|
||||
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json"
|
||||
_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json"
|
||||
_FAKE_USER: dict = {"uuid": "test-user", "role": "viewer"}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user