From 74096b6df016d8b24b66280525195b5b6ac23e79 Mon Sep 17 00:00:00 2001 From: anti Date: Thu, 18 Jun 2026 18:36:38 -0400 Subject: [PATCH 1/8] feat(1.2): prefork supervisor primitive + tests (C, CoW gate passed) CoW measurement on CPython 3.14: forked idle child keeps ~71MB shared, dirties ~1MB private; working child ~26MB. PEP 683 immortal objects keep code/module pages clean so gc.freeze() is unnecessary (freeze==nofreeze). prefork.run_fleet: master imports the base floor once, forks one child per worker (own process/GIL, CoW-shared floor), reaps + restarts with backoff, graceful SIGTERM->SIGKILL shutdown. Not yet wired to a command (that lands when 1.2 picks the target worker set). --- decnet/prefork.py | 140 ++++++++++++++++++++++++++++++++++++++++ tests/prefork_driver.py | 55 ++++++++++++++++ tests/test_prefork.py | 40 ++++++++++++ 3 files changed, 235 insertions(+) create mode 100644 decnet/prefork.py create mode 100644 tests/prefork_driver.py create mode 100644 tests/test_prefork.py diff --git a/decnet/prefork.py b/decnet/prefork.py new file mode 100644 index 00000000..50ed22fa --- /dev/null +++ b/decnet/prefork.py @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Prefork supervisor — import the base floor ONCE in a master, then fork one +child process per worker. Children share the ~70 MB import floor via +copy-on-write. + +Measured on CPython 3.14 (development/cow_probe.py): an idle forked child keeps +~71 MB shared and dirties only ~1 MB private; a working child dirties ~26 MB +(its own heap, not the floor). PEP 683 immortal objects keep module/code pages +clean, so the classic refcount-dirties-CoW problem does not bite and gc.freeze() +is unnecessary on 3.14. + +Contrast with :mod:`decnet.supervisor` (asyncio tasks in ONE process, shared +GIL): use that for cheap co-resident IO workers. Use prefork for workers that +must keep their OWN process / GIL — CPU-heavy or isolation-critical — but +shouldn't each re-import the world. + +Each worker spec is a zero-arg callable that BLOCKS running the worker (e.g. +``lambda: asyncio.run(profiler_worker(repo))``). It executes in the forked +child; the master only forks, reaps, and restarts. +""" +from __future__ import annotations + +import logging +import os +import signal +import time +from collections.abc import Callable + +log = logging.getLogger("decnet.prefork") + +WorkerEntry = Callable[[], None] + + +def run_fleet( + specs: dict[str, WorkerEntry], + *, + max_backoff: float = 30.0, + poll_interval: float = 0.2, + stop_after: float | None = None, +) -> None: + """Fork one child per worker and supervise them until SIGTERM/SIGINT. + + A dead child is re-forked after exponential backoff (in-process + ``Restart=on-failure``). Backoff is tracked per worker and scheduled + non-blockingly, so one worker's restart delay never stalls reaping of + another. On shutdown, children get SIGTERM, then SIGKILL after a grace + period. + + ``stop_after`` (seconds) is a test hook: cleanly shut the fleet down after + that long instead of waiting for a signal. + """ + if not specs: + return + + children: dict[int, str] = {} # pid -> name + backoff: dict[str, float] = {n: 1.0 for n in specs} + due: dict[str, float] = {} # name -> earliest restart time + stopping = {"flag": False} + + def _request_stop(_signum: int, _frame: object) -> None: + stopping["flag"] = True + + signal.signal(signal.SIGTERM, _request_stop) + signal.signal(signal.SIGINT, _request_stop) + + def spawn(name: str) -> None: + pid = os.fork() + if pid == 0: # ---- child ---- + # Restore default signal handling so the worker's own asyncio + # handlers (or KeyboardInterrupt) work as if launched standalone. + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGINT, signal.SIG_DFL) + try: + specs[name]() + except KeyboardInterrupt: + pass + except BaseException: # noqa: BLE001 — last-resort child logging + log.exception("prefork: worker %s raised", name) + os._exit(1) + os._exit(0) + children[pid] = name # ---- parent ---- + log.info("prefork: spawned %s pid=%d", name, pid) + + log.info("prefork: master pid=%d forking %d workers: %s", + os.getpid(), len(specs), ", ".join(specs)) + for name in specs: + spawn(name) + + deadline = (time.monotonic() + stop_after) if stop_after is not None else None + while not stopping["flag"]: + if deadline is not None and time.monotonic() >= deadline: + break + now = time.monotonic() + # Restart any workers whose backoff has elapsed. + for name in [n for n, t in due.items() if now >= t]: + del due[name] + spawn(name) + # Reap without blocking so concurrent crashes are all handled. + try: + pid, status = os.waitpid(-1, os.WNOHANG) + except ChildProcessError: + pid = 0 + if pid == 0: + time.sleep(poll_interval) + continue + name = children.pop(pid, None) + if name is None: + continue + code = os.waitstatus_to_exitcode(status) + log.warning("prefork: %s (pid=%d) exited code=%d; restart in %.0fs", + name, pid, code, backoff[name]) + due[name] = time.monotonic() + backoff[name] + backoff[name] = min(backoff[name] * 2.0, max_backoff) + + _shutdown(children) + + +def _shutdown(children: dict[int, str], *, grace: float = 15.0) -> None: + """SIGTERM all children, reap within ``grace``, SIGKILL stragglers.""" + for pid in list(children): + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + children.pop(pid, None) + deadline = time.monotonic() + grace + while children and time.monotonic() < deadline: + try: + pid, _ = os.waitpid(-1, os.WNOHANG) + except ChildProcessError: + break + if pid: + children.pop(pid, None) + else: + time.sleep(0.1) + for pid in list(children): + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + log.info("prefork: fleet shut down") diff --git a/tests/prefork_driver.py b/tests/prefork_driver.py new file mode 100644 index 00000000..d1fa3cfa --- /dev/null +++ b/tests/prefork_driver.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Standalone driver for the prefork supervisor — runnable directly OR via +tests/test_prefork.py (which execs it in a subprocess so no fork happens inside +the pytest/xdist worker). + + python tests/prefork_driver.py + +Forks two fake workers under decnet.prefork.run_fleet: + * "tick" — append a line every 0.2s forever (proves a worker runs & stays up) + * "crasher" — write a marker then exit(1) (proves restart-on-crash) +Runs for ~2s via stop_after, then shuts the fleet down. Writes results into +; the caller asserts on them. +""" +from __future__ import annotations + +import os +import sys +import time + +# Running this file as a script puts its own dir (tests/) on sys.path[0], which +# shadows the stdlib `logging` via tests/logging/. Drop it before importing +# decnet (still importable — it's installed in the venv). +if sys.path and os.path.basename(sys.path[0]) == "tests": + sys.path.pop(0) + +from decnet.prefork import run_fleet # noqa: E402 + + +def main(out: str) -> None: + tick_log = os.path.join(out, "tick.log") + crash_log = os.path.join(out, "crash.log") + + def tick() -> None: + while True: + with open(tick_log, "a") as f: + f.write("t\n") + time.sleep(0.2) + + def crasher() -> None: + with open(crash_log, "a") as f: + f.write("c\n") + time.sleep(0.15) + os._exit(1) + + # Fast backoff so we observe multiple restarts inside the short window. + run_fleet( + {"tick": tick, "crasher": crasher}, + max_backoff=0.2, + poll_interval=0.05, + stop_after=2.0, + ) + + +if __name__ == "__main__": + main(sys.argv[1] if len(sys.argv) > 1 else ".") diff --git a/tests/test_prefork.py b/tests/test_prefork.py new file mode 100644 index 00000000..0c120f22 --- /dev/null +++ b/tests/test_prefork.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Prefork supervisor behaviour, exercised via a subprocess driver so no fork +happens inside the pytest/xdist worker (which would be unsafe). + +Proves: workers fork and run, a crashing worker is restarted with backoff, and +the fleet shuts down cleanly (stop_after returns, no orphaned children). +""" +from __future__ import annotations + +import pathlib +import subprocess +import sys + + +def test_prefork_runs_and_restarts(tmp_path: pathlib.Path): + driver = pathlib.Path(__file__).parent / "prefork_driver.py" + proc = subprocess.run( + [sys.executable, str(driver), str(tmp_path)], + capture_output=True, text=True, timeout=30, + ) + assert proc.returncode == 0, f"driver failed:\n{proc.stderr}" + + tick = (tmp_path / "tick.log").read_text().splitlines() + crash = (tmp_path / "crash.log").read_text().splitlines() + + # tick ran continuously for ~2s at 0.2s cadence → several lines. + assert len(tick) >= 5, f"tick worker did not stay up: {len(tick)} lines" + # crasher died fast and was restarted repeatedly → many markers. + assert len(crash) >= 3, f"crasher was not restarted: {len(crash)} markers" + + +def test_empty_fleet_returns(tmp_path: pathlib.Path): + # run_fleet([]) must be a no-op, not hang. + code = ( + "from decnet.prefork import run_fleet; run_fleet({}, stop_after=5)" + ) + proc = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, timeout=15 + ) + assert proc.returncode == 0, proc.stderr From a5e11f7d86ddb97494f4dbb0d5576afc80cef011 Mon Sep 17 00:00:00 2001 From: anti Date: Thu, 18 Jun 2026 18:53:35 -0400 Subject: [PATCH 2/8] chore(1.2): open 1.2.0 dev cycle (version 1.2.0.dev0, CHANGELOG Unreleased) --- CHANGELOG.md | 14 ++++++++++++++ pyproject.toml | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49fb736a..eae81ddf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ All notable changes to DECNET are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] (1.2.0) + +Prefork worker consolidation — share the import floor across *separate* processes +(own GIL, full isolation) via copy-on-write, for the heavy/isolation-critical +workers the in-process supervisor can't co-host. + +### Added +- `decnet.prefork` — prefork supervisor primitive: a master imports the base + floor once, then forks one child per worker (own process/GIL, CoW-shared + floor), reaps and restarts with backoff, and shuts down gracefully. CoW + viability measured on CPython 3.14 (idle child ~1 MB private, ~71 MB shared; + `gc.freeze()` unnecessary thanks to PEP 683 immortal objects). Not yet wired to + a command — the target worker set lands next. + ## [1.1.1] - 2026-06-18 ### Fixed diff --git a/pyproject.toml b/pyproject.toml index d252a35b..0a4526a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "decnet" -version = "1.1.1" +version = "1.2.0.dev0" description = "Deception network: deploy honeypot deckies that appear as real LAN hosts" readme = "README.md" authors = [{ name = "Samuel Paschuan", email = "samuel.paschuan@xmartlab.com" }] From 1a765854ec0f166eb4a4fa4b4c067ab6342beca0 Mon Sep 17 00:00:00 2001 From: anti Date: Thu, 18 Jun 2026 19:02:58 -0400 Subject: [PATCH 3/8] fix(1.2): relocate ATT&CK bundle to decnet/data/, bump 19.0 -> 19.1 Bundle pointer moved from repo root to decnet/data/ (with LICENSE.txt), gitignored + fetched on demand (51MB, MITRE-licensed). Version pin bumped 19.0->19.1 with the new sha256; license unchanged. All _REPO_BUNDLE test constants repointed. Fixes test-web failures after the repo-root bundle was deleted. --- CHANGELOG.md | 6 ++++++ decnet/data/.gitignore | 7 +++++++ decnet/ttp/attack_version.py | 6 +++--- tests/ttp/test_attack_bundle_validation.py | 2 +- tests/ttp/test_attack_catalog.py | 2 +- tests/ttp/test_attack_license.py | 2 +- tests/ttp/test_attack_url.py | 2 +- tests/ttp/test_emit_attaches_mitre_url.py | 2 +- tests/ttp/test_groups_for_technique.py | 2 +- tests/ttp/test_intel_mappings.py | 2 +- tests/web/test_api_export_attacker_misp.py | 2 +- tests/web/test_api_export_attacker_stix.py | 4 ++-- tests/web/test_api_export_attackers_misp.py | 2 +- tests/web/test_api_export_attackers_stix.py | 2 +- 14 files changed, 28 insertions(+), 15 deletions(-) create mode 100644 decnet/data/.gitignore diff --git a/CHANGELOG.md b/CHANGELOG.md index eae81ddf..47ff23b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,12 @@ workers the in-process supervisor can't co-host. `gc.freeze()` unnecessary thanks to PEP 683 immortal objects). Not yet wired to a command — the target worker set lands next. +### Changed +- MITRE ATT&CK Enterprise bundle pinned 19.0 → **19.1**. The bundle and its + LICENSE now resolve from `decnet/data/` (hash-pinned in `attack_version.py`, + fetched on demand via `python -m decnet.ttp.attack_stix fetch`, gitignored — + not committed). + ## [1.1.1] - 2026-06-18 ### Fixed diff --git a/decnet/data/.gitignore b/decnet/data/.gitignore new file mode 100644 index 00000000..0da15350 --- /dev/null +++ b/decnet/data/.gitignore @@ -0,0 +1,7 @@ +# MITRE ATT&CK STIX bundle + license live here but are NOT committed: +# ~51MB and MITRE-licensed (fetched on demand, hash-pinned in attack_version.py). +# +# Populate locally / in CI with: +# DECNET_ATTACK_CACHE_DIR=decnet/data python -m decnet.ttp.attack_stix fetch +/enterprise-attack-*.json +/LICENSE.txt diff --git a/decnet/ttp/attack_version.py b/decnet/ttp/attack_version.py index 323f595a..c9854d94 100644 --- a/decnet/ttp/attack_version.py +++ b/decnet/ttp/attack_version.py @@ -16,12 +16,12 @@ from __future__ import annotations from typing import Final -ATTACK_BUNDLE_VERSION: Final[str] = "19.0" +ATTACK_BUNDLE_VERSION: Final[str] = "19.1" -# sha256 of the canonical MITRE-published enterprise-attack-19.0.json +# sha256 of the canonical MITRE-published enterprise-attack-19.1.json # from https://github.com/mitre-attack/attack-stix-data. ATTACK_BUNDLE_SHA256: Final[str] = ( - "df520ea0775a57db7bff760145b02fed89290802913e056b7ed5970b02f3626a" + "bdf1ce86a4e604214c5076d37ae4dcb322678afc528df8492e6fdc1b554f5da3" ) # Raw download URL for the pinned version. diff --git a/tests/ttp/test_attack_bundle_validation.py b/tests/ttp/test_attack_bundle_validation.py index 93cdb1ae..7bdf02e5 100644 --- a/tests/ttp/test_attack_bundle_validation.py +++ b/tests/ttp/test_attack_bundle_validation.py @@ -20,7 +20,7 @@ from decnet.clustering import ukc from decnet.ttp import attack_stix from decnet.ttp.impl import intel_lifter -_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json" +_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json" @pytest.fixture(autouse=True) diff --git a/tests/ttp/test_attack_catalog.py b/tests/ttp/test_attack_catalog.py index 5f48cbcb..557db736 100644 --- a/tests/ttp/test_attack_catalog.py +++ b/tests/ttp/test_attack_catalog.py @@ -19,7 +19,7 @@ from decnet.ttp import attack_stix from decnet.ttp.attack_catalog import technique_name _RULES_DIR = Path(__file__).resolve().parents[2] / "rules" / "ttp" -_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json" +_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json" @pytest.fixture(scope="module", autouse=True) diff --git a/tests/ttp/test_attack_license.py b/tests/ttp/test_attack_license.py index 881b1747..9cbc2804 100644 --- a/tests/ttp/test_attack_license.py +++ b/tests/ttp/test_attack_license.py @@ -19,7 +19,7 @@ from decnet.ttp.attack_version import ( ATTACK_LICENSE_SHA256, ) -_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json" +_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json" @pytest.fixture(autouse=True) diff --git a/tests/ttp/test_attack_url.py b/tests/ttp/test_attack_url.py index 0e25ee99..5553de3a 100644 --- a/tests/ttp/test_attack_url.py +++ b/tests/ttp/test_attack_url.py @@ -15,7 +15,7 @@ import pytest from decnet.ttp import attack_stix -_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json" +_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json" @pytest.fixture(autouse=True) diff --git a/tests/ttp/test_emit_attaches_mitre_url.py b/tests/ttp/test_emit_attaches_mitre_url.py index 50866fd4..6e69667b 100644 --- a/tests/ttp/test_emit_attaches_mitre_url.py +++ b/tests/ttp/test_emit_attaches_mitre_url.py @@ -24,7 +24,7 @@ from decnet.ttp.impl._emit import emit_tags from decnet.ttp.impl.rule_engine import CompiledRule from decnet.ttp.store.base import RuleState -_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json" +_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json" @pytest.fixture(autouse=True) diff --git a/tests/ttp/test_groups_for_technique.py b/tests/ttp/test_groups_for_technique.py index 4012fe5c..f6414a87 100644 --- a/tests/ttp/test_groups_for_technique.py +++ b/tests/ttp/test_groups_for_technique.py @@ -19,7 +19,7 @@ from decnet.web.router.ttp.api_get_groups_for_technique import ( api_groups_for_technique, ) -_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json" +_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json" @pytest.fixture(autouse=True) diff --git a/tests/ttp/test_intel_mappings.py b/tests/ttp/test_intel_mappings.py index 8f61b0ed..d4f7df14 100644 --- a/tests/ttp/test_intel_mappings.py +++ b/tests/ttp/test_intel_mappings.py @@ -31,7 +31,7 @@ from decnet.ttp.data.intel_loader import ( load_provider_mapping, ) -_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json" +_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json" _DATA_DIR = Path(__file__).resolve().parents[2] / "decnet" / "ttp" / "data" / "intel" diff --git a/tests/web/test_api_export_attacker_misp.py b/tests/web/test_api_export_attacker_misp.py index 3180d212..3118a764 100644 --- a/tests/web/test_api_export_attacker_misp.py +++ b/tests/web/test_api_export_attacker_misp.py @@ -14,7 +14,7 @@ from decnet.web.router.attackers.api_export_attacker_misp import ( api_export_attacker_misp, ) -_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json" +_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json" _FAKE_USER: dict = {"uuid": "test-user", "role": "viewer"} diff --git a/tests/web/test_api_export_attacker_stix.py b/tests/web/test_api_export_attacker_stix.py index d68816a2..50f87efa 100644 --- a/tests/web/test_api_export_attacker_stix.py +++ b/tests/web/test_api_export_attacker_stix.py @@ -2,7 +2,7 @@ """Tests for GET /api/v1/attackers/{uuid}/export/stix. Tests call the handler directly (no TestClient). The attack_stix bundle -is pinned to the repo's enterprise-attack-19.0.json so Sighting and +is pinned to the repo's decnet/data/enterprise-attack-19.1.json so Sighting and Relationship target_refs are real MITRE STIX IDs. """ from __future__ import annotations @@ -20,7 +20,7 @@ from decnet.web.router.attackers.api_export_attacker_stix import ( api_export_attacker_stix, ) -_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json" +_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json" _FAKE_USER: dict = {"uuid": "test-user", "role": "viewer"} diff --git a/tests/web/test_api_export_attackers_misp.py b/tests/web/test_api_export_attackers_misp.py index 7f0677ed..7a1c928c 100644 --- a/tests/web/test_api_export_attackers_misp.py +++ b/tests/web/test_api_export_attackers_misp.py @@ -14,7 +14,7 @@ from decnet.web.router.attackers.api_export_attackers_misp import ( api_export_attackers_misp, ) -_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json" +_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json" _FAKE_USER: dict = {"uuid": "test-user", "role": "viewer"} diff --git a/tests/web/test_api_export_attackers_stix.py b/tests/web/test_api_export_attackers_stix.py index 2341a908..5c03d7f1 100644 --- a/tests/web/test_api_export_attackers_stix.py +++ b/tests/web/test_api_export_attackers_stix.py @@ -15,7 +15,7 @@ from decnet.web.router.attackers.api_export_attackers_stix import ( api_export_attackers_stix, ) -_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "enterprise-attack-19.0.json" +_REPO_BUNDLE = Path(__file__).resolve().parents[2] / "decnet" / "data" / "enterprise-attack-19.1.json" _FAKE_USER: dict = {"uuid": "test-user", "role": "viewer"} From fcc9a9aad198f3c159e8863ebad9152eede9b4b8 Mon Sep 17 00:00:00 2001 From: anti Date: Thu, 18 Jun 2026 19:32:27 -0400 Subject: [PATCH 4/8] =?UTF-8?q?feat(1.2):=20decnet=20fleet=20=E2=80=94=20p?= =?UTF-8?q?refork=20master=20for=20the=20heavy=20worker=20tier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the prefork primitive into a CLI command. 'decnet fleet heavy' imports the shared base floor once in the master, then forks profiler + ttp as CoW-sharing child processes (own process/GIL, full isolation, shared ~71MB floor). DB-only tier => systemd unit carries no extra privilege (prefork's privilege-union cost is nil for this fleet). Unit Conflicts= the profiler/ttp units it replaces. Heavy per-worker state (ATT&CK/ML) still loads per-child; warming it in the master to share is deferred until a live RSS measurement shows the big object graph CoW-shares rather than refcount-dirties. --- decnet/cli/__init__.py | 3 +- decnet/cli/fleet.py | 94 ++++++++++++++++++++++++++++ deploy/decnet-fleet-heavy.service.j2 | 48 ++++++++++++++ tests/cli/test_fleet.py | 35 +++++++++++ 4 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 decnet/cli/fleet.py create mode 100644 deploy/decnet-fleet-heavy.service.j2 create mode 100644 tests/cli/test_fleet.py diff --git a/decnet/cli/__init__.py b/decnet/cli/__init__.py index 03461e5a..63de545c 100644 --- a/decnet/cli/__init__.py +++ b/decnet/cli/__init__.py @@ -26,6 +26,7 @@ from . import ( canary, db, deploy, + fleet, forwarder, geoip, init, @@ -62,7 +63,7 @@ for _mod in ( swarm, deploy, lifecycle, workers, inventory, web, profiler, orchestrator, realism, reconciler, sniffer, db, - topology, bus, geoip, init, webhook, canary, ttp, supervise, + topology, bus, geoip, init, webhook, canary, ttp, supervise, fleet, ): _mod.register(app) diff --git a/decnet/cli/fleet.py b/decnet/cli/fleet.py new file mode 100644 index 00000000..3b720719 --- /dev/null +++ b/decnet/cli/fleet.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""``decnet fleet `` — prefork supervisor (DECNET 1.2). + +Imports the shared base floor ONCE in the master, then forks one child process +per worker (see :mod:`decnet.prefork`). Children share the floor via copy-on-write +(measured ~71 MB shared / ~1 MB private per idle child on CPython 3.14) while +keeping their OWN process and GIL — unlike ``decnet supervise``, which co-hosts +workers as asyncio tasks in one shared-GIL process. + +Use ``fleet`` for workers that must stay process-isolated (heavy resident state, +sustained CPU) but shouldn't each re-import the world; use ``supervise`` for cheap +co-resident IO workers. + +CONSOLIDATION COSTS (same shape as ``supervise``): + * Forked children inherit the master's privileges — a fleet's systemd unit + carries the UNION of its members' caps. So group by privilege profile, not + convenience. The ``heavy`` fleet is DB-only (no docker socket, no raw net). + * To share via CoW the master pre-imports each worker's module BEFORE forking, + so its RSS is large — but that RSS is the shared floor, not per-child cost. +""" +from __future__ import annotations + +import typer + +from . import utils as _utils +from .utils import console, log + +_FLEETS = ("heavy",) + + +def _build_fleet(name: str) -> dict: + """Return ``{worker_name: entry_thunk}`` for *name*. + + Imports happen here, in the MASTER, before :func:`run_fleet` forks — that is + what lets children share the imported code/objects via copy-on-write. Each + thunk blocks running one worker; ``repo`` is initialized inside the child + (post-fork) so every child opens its own pool, never a fork-inherited one. + """ + import asyncio + + if name == "heavy": + from decnet.profiler import attacker_profile_worker + from decnet.ttp.worker import run_ttp_worker_loop + from decnet.web.dependencies import repo + + # Importing the worker modules here (in the master) is what lets children + # share their code via CoW. Heavy per-worker runtime state (ATT&CK bundle, + # ML) still loads lazily in each child — warming it in the master to share + # it too is a future optimization, gated on a live RSS measurement showing + # the big object graph actually CoW-shares rather than refcount-dirtying. + def _profiler() -> None: + async def _go() -> None: + await repo.initialize() + await attacker_profile_worker(repo, interval=60) + asyncio.run(_go()) + + def _ttp() -> None: + async def _go() -> None: + await repo.initialize() + await run_ttp_worker_loop(repo, poll_interval_secs=60.0) + asyncio.run(_go()) + + return {"profiler": _profiler, "ttp": _ttp} + + raise ValueError(f"unknown fleet: {name}") + + +def register(app: typer.Typer) -> None: + @app.command(name="fleet") + def fleet_cmd( + name: str = typer.Argument( + ..., help=f"Worker fleet to fork. One of: {', '.join(_FLEETS)}" + ), + daemon: bool = typer.Option( + False, "--daemon", "-d", help="Detach to background as a daemon process" + ), + ) -> None: + """Prefork a worker fleet: shared base floor (CoW), one child process per worker.""" + from decnet.prefork import run_fleet + + if name not in _FLEETS: + console.print( + f"[red]unknown fleet {name!r}; known fleets: {', '.join(_FLEETS)}[/]" + ) + raise typer.Exit(2) + + if daemon: + log.info("fleet %s daemonizing", name) + _utils._daemonize() + + log.info("fleet %s starting", name) + console.print(f"[bold cyan]Fleet starting[/] {name} (prefork)") + specs = _build_fleet(name) + run_fleet(specs) diff --git a/deploy/decnet-fleet-heavy.service.j2 b/deploy/decnet-fleet-heavy.service.j2 new file mode 100644 index 00000000..607f65a5 --- /dev/null +++ b/deploy/decnet-fleet-heavy.service.j2 @@ -0,0 +1,48 @@ +[Unit] +Description=DECNET Heavy Fleet (prefork master forking profiler + ttp as CoW-sharing children) +Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#fleet +After=network-online.target decnet-bus.service +Wants=network-online.target decnet-bus.service +# Replaces the individual decnet-profiler / decnet-ttp units. Do NOT enable +# those alongside this one. +Conflicts=decnet-profiler.service decnet-ttp.service + +[Service] +Type=simple +User={{ user }} +Group={{ group }} +WorkingDirectory={{ install_dir }} +EnvironmentFile=-{{ install_dir }}/.env.local +Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.fleet-heavy.log +ExecStart={{ venv_dir }}/bin/decnet fleet heavy +StandardOutput=append:/var/log/decnet/decnet.fleet-heavy.log +StandardError=append:/var/log/decnet/decnet.fleet-heavy.log + +# Prefork master imports the shared base floor once, then forks one child per +# worker; children share the floor via copy-on-write. Both members are DB-only +# (no docker socket, no raw sockets) so this unit carries NO extra privilege — +# the prefork privilege-union cost is nil for this fleet by construction. +CapabilityBoundingSet= +AmbientCapabilities= + +# Security Hardening +NoNewPrivileges=yes +ProtectSystem=full +# Dev installs under /home need ProtectHome=read-only: the ttp child reads +# ./rules/ttp/ from the project root (read-only suffices — YAML reads only). +ProtectHome=read-only +PrivateTmp=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectControlGroups=yes +RestrictSUIDSGID=yes +LockPersonality=yes +ReadWritePaths={{ install_dir }} /var/log/decnet + +Restart=on-failure +RestartSec=5 +# Master forwards SIGTERM to children and reaps; give it room for both to drain. +TimeoutStopSec=25 + +[Install] +WantedBy=multi-user.target diff --git a/tests/cli/test_fleet.py b/tests/cli/test_fleet.py new file mode 100644 index 00000000..baadf85b --- /dev/null +++ b/tests/cli/test_fleet.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""CLI surface for ``decnet fleet`` (DECNET 1.2 prefork). The fork/restart +mechanism itself is covered by tests/test_prefork.py.""" +from __future__ import annotations + +from typer.testing import CliRunner + +from decnet.cli import app +from decnet.cli.fleet import _FLEETS, _build_fleet + +runner = CliRunner() + + +def test_fleet_is_registered(): + result = runner.invoke(app, ["fleet", "--help"]) + assert result.exit_code == 0 + assert "fleet" in result.stdout.lower() + + +def test_unknown_fleet_exits_2(): + result = runner.invoke(app, ["fleet", "not-a-fleet"]) + assert result.exit_code == 2 + assert "unknown fleet" in result.stdout + + +def test_heavy_fleet_builds_expected_workers(): + # _build_fleet imports worker modules + builds thunks but runs nothing + # (no fork, no repo.initialize) — safe to call in-process. + specs = _build_fleet("heavy") + assert set(specs) == {"profiler", "ttp"} + assert all(callable(t) for t in specs.values()) + + +def test_heavy_is_known(): + assert "heavy" in _FLEETS From 419172ecfb78afc4d1259ac0805a7da401115ef9 Mon Sep 17 00:00:00 2001 From: anti Date: Thu, 18 Jun 2026 19:32:38 -0400 Subject: [PATCH 5/8] =?UTF-8?q?docs(1.2):=20changelog=20=E2=80=94=20decnet?= =?UTF-8?q?=20fleet=20prefork=20command?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47ff23b8..335e250d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,12 @@ workers the in-process supervisor can't co-host. floor once, then forks one child per worker (own process/GIL, CoW-shared floor), reaps and restarts with backoff, and shuts down gracefully. CoW viability measured on CPython 3.14 (idle child ~1 MB private, ~71 MB shared; - `gc.freeze()` unnecessary thanks to PEP 683 immortal objects). Not yet wired to - a command — the target worker set lands next. + `gc.freeze()` unnecessary thanks to PEP 683 immortal objects). +- `decnet fleet ` — prefork master that imports the shared base floor once + then forks one child per worker. First fleet `heavy` = profiler + ttp (DB-only, + process-isolated heavy tier); systemd unit `decnet-fleet-heavy.service` + Conflicts= the units it replaces and carries no extra privilege. Live RSS + delta + heavy-state warming pending a controlled swap. ### Changed - MITRE ATT&CK Enterprise bundle pinned 19.0 → **19.1**. The bundle and its From 7b0ff127c3ff7fca0af0d21b1da8c356779c47aa Mon Sep 17 00:00:00 2001 From: anti Date: Thu, 18 Jun 2026 19:39:15 -0400 Subject: [PATCH 6/8] =?UTF-8?q?docs(1.2):=20heavy=20fleet=20verified=20liv?= =?UTF-8?q?e=20=E2=80=94=20~412MB=20Pss=20vs=20661MB;=20prefork=20helps=20?= =?UTF-8?q?base-floor-bound=20workers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 335e250d..54c35ebe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,8 +20,12 @@ workers the in-process supervisor can't co-host. - `decnet fleet ` — prefork master that imports the shared base floor once then forks one child per worker. First fleet `heavy` = profiler + ttp (DB-only, process-isolated heavy tier); systemd unit `decnet-fleet-heavy.service` - Conflicts= the units it replaces and carries no extra privilege. Live RSS - delta + heavy-state warming pending a controlled swap. + Conflicts= the units it replaces and carries no extra privilege. + Verified live: fleet footprint ≈412 MB Pss (master 67 + profiler 81 + ttp 264) + vs 661 MB standalone — profiler's RSS collapsed 353→110 MB (base floor now + CoW-shared). ttp barely moved: its bulk is the privately-parsed ATT&CK bundle, + which it alone consumes — so master-warming it was confirmed pointless and + dropped. Lesson: prefork pays for base-floor-bound workers, not state-bound ones. ### Changed - MITRE ATT&CK Enterprise bundle pinned 19.0 → **19.1**. The bundle and its From beaa6048111025a33ff073640a12e1ee34406bdc Mon Sep 17 00:00:00 2001 From: anti Date: Thu, 18 Jun 2026 19:42:51 -0400 Subject: [PATCH 7/8] chore(1.2): remove per-worker unit templates superseded by consolidation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The batch/cpu supervisor groups + heavy fleet replace 10 per-worker units (reconciler/enrich/orchestrator/mutator/clusterer/campaign-clusterer/ attribution/reuse-correlator/profiler/ttp). Removed their deploy/*.service.j2 templates and rewired decnet.target to the 3 consolidated units. Dropped test_orchestrator_unit.py (tested a removed unit). CLI commands (decnet ttp, mutate, …) stay for manual runs; new units' Conflicts= still name the old units defensively for hosts mid-migration. --- deploy/decnet-attribution.service.j2 | 45 ------ deploy/decnet-campaign-clusterer.service.j2 | 52 ------- deploy/decnet-clusterer.service.j2 | 47 ------ deploy/decnet-enrich.service.j2 | 47 ------ deploy/decnet-mutator.service.j2 | 41 ------ deploy/decnet-orchestrator.service.j2 | 50 ------- deploy/decnet-profiler.service.j2 | 38 ----- deploy/decnet-reconciler.service.j2 | 47 ------ deploy/decnet-reuse-correlator.service.j2 | 41 ------ deploy/decnet-ttp.service.j2 | 57 -------- deploy/decnet.target | 16 +-- tests/deploy/test_orchestrator_unit.py | 152 -------------------- 12 files changed, 7 insertions(+), 626 deletions(-) delete mode 100644 deploy/decnet-attribution.service.j2 delete mode 100644 deploy/decnet-campaign-clusterer.service.j2 delete mode 100644 deploy/decnet-clusterer.service.j2 delete mode 100644 deploy/decnet-enrich.service.j2 delete mode 100644 deploy/decnet-mutator.service.j2 delete mode 100644 deploy/decnet-orchestrator.service.j2 delete mode 100644 deploy/decnet-profiler.service.j2 delete mode 100644 deploy/decnet-reconciler.service.j2 delete mode 100644 deploy/decnet-reuse-correlator.service.j2 delete mode 100644 deploy/decnet-ttp.service.j2 delete mode 100644 tests/deploy/test_orchestrator_unit.py diff --git a/deploy/decnet-attribution.service.j2 b/deploy/decnet-attribution.service.j2 deleted file mode 100644 index 22afbd97..00000000 --- a/deploy/decnet-attribution.service.j2 +++ /dev/null @@ -1,45 +0,0 @@ -[Unit] -Description=DECNET Attribution Engine v0 (per-(identity, primitive) state machine) -Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#attribution -After=network-online.target decnet-bus.service -Wants=network-online.target decnet-bus.service - -[Service] -Type=simple -User={{ user }} -Group={{ group }} -WorkingDirectory={{ install_dir }} -EnvironmentFile=-{{ install_dir }}/.env.local -Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.attribution.log -# Subscribes to attacker.observation.> and, for each event, ensures a -# stub AttackerIdentity row, runs the per-ValueKind merger over the -# full identity-keyed observation series, upserts the derived state in -# attribution_state, and publishes attribution.profile.state_changed -# only on transition. Periodic tick (default 60s) fires -# attribution.profile.multi_actor_suspected when >= 2 primitives flag -# the same identity. Closes DEBT-051. -ExecStart={{ venv_dir }}/bin/decnet attribution -StandardOutput=append:/var/log/decnet/decnet.attribution.log -StandardError=append:/var/log/decnet/decnet.attribution.log - -CapabilityBoundingSet= -AmbientCapabilities= - -# Security Hardening -NoNewPrivileges=yes -ProtectSystem=full -ProtectHome=read-only -PrivateTmp=yes -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectControlGroups=yes -RestrictSUIDSGID=yes -LockPersonality=yes -ReadWritePaths={{ install_dir }} /var/log/decnet - -Restart=on-failure -RestartSec=5 -TimeoutStopSec=15 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/decnet-campaign-clusterer.service.j2 b/deploy/decnet-campaign-clusterer.service.j2 deleted file mode 100644 index 8bc128d8..00000000 --- a/deploy/decnet-campaign-clusterer.service.j2 +++ /dev/null @@ -1,52 +0,0 @@ -[Unit] -Description=DECNET Campaign Clusterer (identities → campaigns / operations) -Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#campaign-clusterer -After=network-online.target decnet-bus.service decnet-clusterer.service -Wants=network-online.target decnet-bus.service decnet-clusterer.service - -[Service] -Type=simple -User={{ user }} -Group={{ group }} -WorkingDirectory={{ install_dir }} -EnvironmentFile=-{{ install_dir }}/.env.local -Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.campaign-clusterer.log -# Subscribes to identity.>; falls back to a 60s slow-tick poll when -# the bus is idle or unavailable. Reads AttackerIdentity rows, -# projects them into the campaign-level similarity graph -# (phase-handoff / shared-infra / temporal overlap / cohort), runs -# union-find, writes campaigns rows + sets -# attacker_identities.campaign_id, and publishes campaign.formed / -# campaign.identity.assigned / campaign.merged / campaign.unmerged -# plus the cross-family identity.campaign.assigned for identity-side -# subscribers. -# -# Master-only: gated via MASTER_ONLY_COMMANDS in decnet/cli/gating.py. -# Sits one layer above decnet-clusterer (the After=/Wants= ensures the -# identity layer is up first; the campaign clusterer then wakes on -# identity.> events fired by it). -ExecStart={{ venv_dir }}/bin/decnet campaign-clusterer -StandardOutput=append:/var/log/decnet/decnet.campaign-clusterer.log -StandardError=append:/var/log/decnet/decnet.campaign-clusterer.log - -CapabilityBoundingSet= -AmbientCapabilities= - -# Security Hardening -NoNewPrivileges=yes -ProtectSystem=full -ProtectHome=read-only -PrivateTmp=yes -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectControlGroups=yes -RestrictSUIDSGID=yes -LockPersonality=yes -ReadWritePaths={{ install_dir }} /var/log/decnet - -Restart=on-failure -RestartSec=5 -TimeoutStopSec=15 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/decnet-clusterer.service.j2 b/deploy/decnet-clusterer.service.j2 deleted file mode 100644 index 22e8d990..00000000 --- a/deploy/decnet-clusterer.service.j2 +++ /dev/null @@ -1,47 +0,0 @@ -[Unit] -Description=DECNET Identity Clusterer (per-IP observations → identities) -Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#identity-clusterer -After=network-online.target decnet-bus.service -Wants=network-online.target decnet-bus.service - -[Service] -Type=simple -User={{ user }} -Group={{ group }} -WorkingDirectory={{ install_dir }} -EnvironmentFile=-{{ install_dir }}/.env.local -Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.clusterer.log -# Subscribes to attacker.observed and attacker.scored; falls back to a -# 60s slow-tick poll when the bus is idle or unavailable. Reads -# Attacker rows, projects per-IP observations into the similarity -# graph (JA3 / HASSH / payload-hash / C2-endpoint), runs union-find, -# writes attacker_identities rows + sets attackers.identity_id, and -# publishes identity.formed / identity.observation.linked / -# identity.merged / identity.unmerged. -# -# Master-only: gated via MASTER_ONLY_COMMANDS in decnet/cli/gating.py. -ExecStart={{ venv_dir }}/bin/decnet clusterer -StandardOutput=append:/var/log/decnet/decnet.clusterer.log -StandardError=append:/var/log/decnet/decnet.clusterer.log - -CapabilityBoundingSet= -AmbientCapabilities= - -# Security Hardening -NoNewPrivileges=yes -ProtectSystem=full -ProtectHome=read-only -PrivateTmp=yes -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectControlGroups=yes -RestrictSUIDSGID=yes -LockPersonality=yes -ReadWritePaths={{ install_dir }} /var/log/decnet - -Restart=on-failure -RestartSec=5 -TimeoutStopSec=15 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/decnet-enrich.service.j2 b/deploy/decnet-enrich.service.j2 deleted file mode 100644 index 344c1e4c..00000000 --- a/deploy/decnet-enrich.service.j2 +++ /dev/null @@ -1,47 +0,0 @@ -[Unit] -Description=DECNET Threat-Intel Enrichment (GreyNoise + AbuseIPDB + abuse.ch) -Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#intel-enrichment -After=network-online.target decnet-bus.service -Wants=network-online.target decnet-bus.service - -[Service] -Type=simple -User={{ user }} -Group={{ group }} -WorkingDirectory={{ install_dir }} -EnvironmentFile=-{{ install_dir }}/.env.local -Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.enrich.log -# Subscribes to attacker.observed and attacker.scored; falls back to a 60s -# slow-tick poll when the bus is idle or unavailable. Per attacker IP fans -# out across the configured intel providers, writes the aggregate verdict -# to attacker_intel, and publishes attacker.intel.enriched. -# -# Free-tier API keys are read from .env.local: -# DECNET_GREYNOISE_API_KEY= (optional, lifts rate limit) -# DECNET_ABUSEIPDB_API_KEY= (required for AbuseIPDB lookups) -# DECNET_THREATFOX_API_KEY= (optional, lifts rate limit) -ExecStart={{ venv_dir }}/bin/decnet enrich -StandardOutput=append:/var/log/decnet/decnet.enrich.log -StandardError=append:/var/log/decnet/decnet.enrich.log - -CapabilityBoundingSet= -AmbientCapabilities= - -# Security Hardening -NoNewPrivileges=yes -ProtectSystem=full -ProtectHome=read-only -PrivateTmp=yes -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectControlGroups=yes -RestrictSUIDSGID=yes -LockPersonality=yes -ReadWritePaths={{ install_dir }} /var/log/decnet - -Restart=on-failure -RestartSec=5 -TimeoutStopSec=15 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/decnet-mutator.service.j2 b/deploy/decnet-mutator.service.j2 deleted file mode 100644 index 8353cee8..00000000 --- a/deploy/decnet-mutator.service.j2 +++ /dev/null @@ -1,41 +0,0 @@ -[Unit] -Description=DECNET Mutator (runtime fleet mutation watch loop) -Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#mutator -After=network-online.target docker.service decnet-bus.service -Wants=network-online.target decnet-bus.service -Requires=docker.service - -[Service] -Type=simple -User={{ user }} -Group={{ group }} -# Mutator recomposes decky services via docker compose. -SupplementaryGroups=docker -WorkingDirectory={{ install_dir }} -EnvironmentFile=-{{ install_dir }}/.env.local -Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.mutator.log -ExecStart={{ venv_dir }}/bin/decnet mutate --watch -StandardOutput=append:/var/log/decnet/decnet.mutator.log -StandardError=append:/var/log/decnet/decnet.mutator.log - -CapabilityBoundingSet= -AmbientCapabilities= - -# Security Hardening -NoNewPrivileges=yes -ProtectSystem=full -ProtectHome=read-only -PrivateTmp=yes -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectControlGroups=yes -RestrictSUIDSGID=yes -LockPersonality=yes -ReadWritePaths={{ install_dir }} /var/log/decnet - -Restart=on-failure -RestartSec=5 -TimeoutStopSec=15 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/decnet-orchestrator.service.j2 b/deploy/decnet-orchestrator.service.j2 deleted file mode 100644 index 4553bf07..00000000 --- a/deploy/decnet-orchestrator.service.j2 +++ /dev/null @@ -1,50 +0,0 @@ -[Unit] -Description=DECNET Orchestrator (synthetic life-injection — inter-decky traffic, file plants, email drops) -Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#orchestrator -After=network-online.target decnet-bus.service -Wants=network-online.target decnet-bus.service - -[Service] -Type=simple -User={{ user }} -Group={{ group }} -WorkingDirectory={{ install_dir }} -EnvironmentFile=-{{ install_dir }}/.env.local -Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.orchestrator.log -# Realism content engine — LLM + persona-pool config used by the -# email + (post-stage-6) file-class enrichment paths. See -# decnet/realism/llm/factory.py and decnet/realism/personas_pool.py. -Environment=DECNET_REALISM_LLM=ollama -Environment=DECNET_REALISM_MODEL=llama3.1 -Environment=DECNET_REALISM_TIMEOUT=60 -Environment=DECNET_REALISM_PERSONAS=/etc/decnet/email_personas.json -ExecStart={{ venv_dir }}/bin/decnet orchestrate -StandardOutput=append:/var/log/decnet/decnet.orchestrator.log -StandardError=append:/var/log/decnet/decnet.orchestrator.log - -# The orchestrator drives `docker exec` against decky containers, so it -# needs membership in the docker group. It does NOT bind to the network, -# launch new containers, or write outside its own logs and install dir. -SupplementaryGroups=docker - -CapabilityBoundingSet= -AmbientCapabilities= - -# Security Hardening -NoNewPrivileges=yes -ProtectSystem=full -ProtectHome=read-only -PrivateTmp=yes -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectControlGroups=yes -RestrictSUIDSGID=yes -LockPersonality=yes -ReadWritePaths={{ install_dir }} /var/log/decnet - -Restart=on-failure -RestartSec=5 -TimeoutStopSec=15 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/decnet-profiler.service.j2 b/deploy/decnet-profiler.service.j2 deleted file mode 100644 index 3f521627..00000000 --- a/deploy/decnet-profiler.service.j2 +++ /dev/null @@ -1,38 +0,0 @@ -[Unit] -Description=DECNET Profiler (attacker profiling and scoring) -Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#profiler -After=network-online.target decnet-bus.service -Wants=network-online.target decnet-bus.service - -[Service] -Type=simple -User={{ user }} -Group={{ group }} -WorkingDirectory={{ install_dir }} -EnvironmentFile=-{{ install_dir }}/.env.local -Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.profiler.log -ExecStart={{ venv_dir }}/bin/decnet profiler -StandardOutput=append:/var/log/decnet/decnet.profiler.log -StandardError=append:/var/log/decnet/decnet.profiler.log - -CapabilityBoundingSet= -AmbientCapabilities= - -# Security Hardening -NoNewPrivileges=yes -ProtectSystem=full -ProtectHome=read-only -PrivateTmp=yes -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectControlGroups=yes -RestrictSUIDSGID=yes -LockPersonality=yes -ReadWritePaths={{ install_dir }} /var/log/decnet - -Restart=on-failure -RestartSec=5 -TimeoutStopSec=15 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/decnet-reconciler.service.j2 b/deploy/decnet-reconciler.service.j2 deleted file mode 100644 index 47663183..00000000 --- a/deploy/decnet-reconciler.service.j2 +++ /dev/null @@ -1,47 +0,0 @@ -[Unit] -Description=DECNET Fleet Reconciler (converges decnet-state.json ↔ fleet_deckies DB ↔ docker) -Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#reconciler -After=network-online.target decnet-bus.service -Wants=network-online.target decnet-bus.service - -[Service] -Type=simple -User={{ user }} -Group={{ group }} -WorkingDirectory={{ install_dir }} -EnvironmentFile=-{{ install_dir }}/.env.local -Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.reconciler.log -ExecStart={{ venv_dir }}/bin/decnet reconcile -StandardOutput=append:/var/log/decnet/decnet.reconciler.log -StandardError=append:/var/log/decnet/decnet.reconciler.log - -# The reconciler queries the docker daemon (via `docker.from_env()`) to -# observe per-container state. Membership in the docker group lets it -# read /var/run/docker.sock without root. It does NOT exec into -# containers, bind to the network, or spawn new containers. -SupplementaryGroups=docker - -CapabilityBoundingSet= -AmbientCapabilities= - -# Security Hardening -NoNewPrivileges=yes -ProtectSystem=full -ProtectHome=read-only -PrivateTmp=yes -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectControlGroups=yes -RestrictSUIDSGID=yes -LockPersonality=yes -# Read-only access to /var/lib/decnet so we can read decnet-state.json. -# Read-write access only to install_dir + log dir. -ReadOnlyPaths=/var/lib/decnet -ReadWritePaths={{ install_dir }} /var/log/decnet - -Restart=on-failure -RestartSec=5 -TimeoutStopSec=15 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/decnet-reuse-correlator.service.j2 b/deploy/decnet-reuse-correlator.service.j2 deleted file mode 100644 index 23f78988..00000000 --- a/deploy/decnet-reuse-correlator.service.j2 +++ /dev/null @@ -1,41 +0,0 @@ -[Unit] -Description=DECNET Credential-Reuse Correlator (cross-target secret-reuse detection) -Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#reuse-correlator -After=network-online.target decnet-bus.service -Wants=network-online.target decnet-bus.service - -[Service] -Type=simple -User={{ user }} -Group={{ group }} -WorkingDirectory={{ install_dir }} -EnvironmentFile=-{{ install_dir }}/.env.local -Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.reuse-correlator.log -# Subscribes to credential.captured and attacker.observed; falls back to -# a 60s slow-tick poll when the bus is idle or unavailable. Publishes -# credential.reuse.detected once per new/grown finding. -ExecStart={{ venv_dir }}/bin/decnet reuse-correlate -StandardOutput=append:/var/log/decnet/decnet.reuse-correlator.log -StandardError=append:/var/log/decnet/decnet.reuse-correlator.log - -CapabilityBoundingSet= -AmbientCapabilities= - -# Security Hardening -NoNewPrivileges=yes -ProtectSystem=full -ProtectHome=read-only -PrivateTmp=yes -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectControlGroups=yes -RestrictSUIDSGID=yes -LockPersonality=yes -ReadWritePaths={{ install_dir }} /var/log/decnet - -Restart=on-failure -RestartSec=5 -TimeoutStopSec=15 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/decnet-ttp.service.j2 b/deploy/decnet-ttp.service.j2 deleted file mode 100644 index e466084c..00000000 --- a/deploy/decnet-ttp.service.j2 +++ /dev/null @@ -1,57 +0,0 @@ -[Unit] -Description=DECNET TTP Tagger (MITRE ATT&CK technique tagging) -Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers#ttp-tagger -After=network-online.target decnet-bus.service decnet-clusterer.service decnet-enrich.service decnet-reuse-correlator.service -Wants=network-online.target decnet-bus.service - -[Service] -Type=simple -User={{ user }} -Group={{ group }} -WorkingDirectory={{ install_dir }} -EnvironmentFile=-{{ install_dir }}/.env.local -Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.ttp.log -# Subscribes to attacker.session.ended (primary), attacker.observed, -# attacker.intel.enriched, identity.formed, identity.merged, -# credential.reuse.detected, email.received, and canary.> ; falls back -# to a 60s slow-tick poll when the bus is idle or unavailable. Each -# event is dispatched through the CompositeTagger (RuleEngine + -# Behavioral / Intel / Email / CanaryFingerprint / Identity / -# Credential lifters), persisted via the idempotent INSERT OR IGNORE -# repo write, and ttp.tagged + ttp.rule.fired. are -# published only when the insert returned a non-zero rowcount -# (loop-prevention invariant — see TTP_TAGGING.md §"Bus topics"). -# -# Master-only: gated via MASTER_ONLY_COMMANDS in decnet/cli/gating.py. -# Sits one layer above the identity / intel / reuse-correlator -# workers — the After= dependencies ensure their bus topics are live -# before the TTP worker subscribes. -ExecStart={{ venv_dir }}/bin/decnet ttp -StandardOutput=append:/var/log/decnet/decnet.ttp.log -StandardError=append:/var/log/decnet/decnet.ttp.log - -CapabilityBoundingSet= -AmbientCapabilities= - -# Security Hardening -NoNewPrivileges=yes -ProtectSystem=full -# Dev installs under /home need ProtectHome=read-only (the worker -# reads ./rules/ttp/ from the project root, which lives under /home -# on dev boxes — read-only suffices because the FilesystemRuleStore -# only reads YAMLs, never writes). -ProtectHome=read-only -PrivateTmp=yes -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectControlGroups=yes -RestrictSUIDSGID=yes -LockPersonality=yes -ReadWritePaths={{ install_dir }} /var/log/decnet - -Restart=on-failure -RestartSec=5 -TimeoutStopSec=15 - -[Install] -WantedBy=multi-user.target diff --git a/deploy/decnet.target b/deploy/decnet.target index 71f0b25f..960fdd59 100644 --- a/deploy/decnet.target +++ b/deploy/decnet.target @@ -5,23 +5,21 @@ Documentation=https://git.resacachile.cl/anti/DECNET/wiki/Workers # heartbeats to it), then the API + data-plane workers. systemd resolves the # actual ordering via each unit's own After=/Wants= on decnet-bus.service — # this target is a convenience grouping, not an ordering primitive. +# Consolidated since 1.1/1.2: the batch (reconcile/enrich/orchestrate/mutate) +# and cpu (clusterer/campaign-clusterer/attribution/reuse-correlate) supervisor +# groups and the heavy (profiler/ttp) prefork fleet replace their per-worker +# units. The standalone workers below kept their own units. Wants=decnet-bus.service \ decnet-api.service \ decnet-web.service \ decnet-collector.service \ - decnet-profiler.service \ decnet-sniffer.service \ decnet-prober.service \ - decnet-mutator.service \ - decnet-reconciler.service \ - decnet-reuse-correlator.service \ - decnet-enrich.service \ - decnet-clusterer.service \ - decnet-campaign-clusterer.service \ - decnet-ttp.service \ decnet-webhook.service \ decnet-canary.service \ - decnet-orchestrator.service + decnet-supervise-batch.service \ + decnet-supervise-cpu.service \ + decnet-fleet-heavy.service After=decnet-bus.service [Install] diff --git a/tests/deploy/test_orchestrator_unit.py b/tests/deploy/test_orchestrator_unit.py deleted file mode 100644 index 63446bd3..00000000 --- a/tests/deploy/test_orchestrator_unit.py +++ /dev/null @@ -1,152 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-or-later -"""Smoke tests for the orchestrator systemd unit + decnet.target wiring. - -These don't exercise systemd (the test host wouldn't have it); they -just assert the static contents of ``deploy/decnet-orchestrator.service.j2`` -and ``deploy/decnet.target`` match what ``decnet init`` will install. - -Anti-regressions for two specific failure modes: - -1. After the realism migration (stage 5), ``decnet-emailgen.service`` - is gone — the orchestrator covers the email branch. A regression - that re-introduces the emailgen unit file or the ``decnet.target`` - entry would only surface on a fresh host install; cheap to catch - here. -2. The orchestrator unit must ship the ``DECNET_REALISM_*`` env block - so the LLM enrichment + persona-pool path are configurable per - host without editing the .j2. -""" -from __future__ import annotations - -from pathlib import Path - -import pytest - - -REPO = Path(__file__).resolve().parent.parent.parent -DEPLOY = REPO / "deploy" - - -@pytest.fixture -def unit_text() -> str: - return (DEPLOY / "decnet-orchestrator.service.j2").read_text() - - -@pytest.fixture -def target_text() -> str: - return (DEPLOY / "decnet.target").read_text() - - -# ── orchestrator unit ──────────────────────────────────────────────────────── - - -def test_orchestrator_unit_exists(): - assert (DEPLOY / "decnet-orchestrator.service.j2").exists() - - -def test_orchestrator_unit_uses_orchestrate_subcommand(unit_text): - assert "decnet orchestrate" in unit_text - - -def test_orchestrator_unit_has_docker_supplementary_group(unit_text): - """SSHDriver shells `docker exec` against decky containers — without - this group the worker can't reach the docker socket.""" - assert "SupplementaryGroups=docker" in unit_text - - -def test_orchestrator_unit_orders_after_bus(unit_text): - """Bus must be up first so heartbeats publish from the start.""" - assert "After=network-online.target decnet-bus.service" in unit_text - assert "Wants=network-online.target decnet-bus.service" in unit_text - - -def test_orchestrator_unit_has_security_hardening(unit_text): - for directive in ( - "NoNewPrivileges=yes", - "ProtectSystem=full", - "ProtectHome=read-only", - "PrivateTmp=yes", - "ProtectKernelTunables=yes", - "ProtectKernelModules=yes", - "ProtectControlGroups=yes", - "RestrictSUIDSGID=yes", - "LockPersonality=yes", - ): - assert directive in unit_text, f"missing {directive}" - - -def test_orchestrator_unit_writes_to_log_dir(unit_text): - assert "/var/log/decnet/decnet.orchestrator.log" in unit_text - assert "ReadWritePaths={{ install_dir }} /var/log/decnet" in unit_text - - -def test_orchestrator_unit_restart_on_failure(unit_text): - assert "Restart=on-failure" in unit_text - - -def test_orchestrator_unit_carries_realism_env_block(unit_text): - """Stage 5 + 6 contract: the orchestrator's LLM enrichment and - persona-pool path are configured per host via DECNET_REALISM_* - env vars. Shipping them in the .j2 means an operator who never - drops a .env.local still gets sane defaults.""" - for var in ( - "DECNET_REALISM_LLM", - "DECNET_REALISM_MODEL", - "DECNET_REALISM_TIMEOUT", - "DECNET_REALISM_PERSONAS", - ): - assert var in unit_text, f"missing {var} in unit" - - -def test_orchestrator_unit_does_not_carry_legacy_emailgen_envs(unit_text): - """Pre-v1 clean break per the realism migration: the - DECNET_EMAILGEN_* env vars are no longer read. Carrying them in - the unit would mislead operators into thinking they still apply.""" - for legacy in ( - "DECNET_EMAILGEN_LLM", - "DECNET_EMAILGEN_MODEL", - "DECNET_EMAILGEN_TIMEOUT", - "DECNET_EMAILGEN_PERSONAS", - ): - assert legacy not in unit_text, ( - f"legacy env {legacy} still referenced; clean-break broken" - ) - - -# ── decnet.target ──────────────────────────────────────────────────────────── - - -def test_target_wants_orchestrator(target_text): - assert "decnet-orchestrator.service" in target_text - - -def test_target_does_not_want_emailgen(target_text): - """Stage 5 of the realism migration deleted decnet-emailgen.service. - A fresh `decnet init` against a target file that still mentions it - fails systemctl start with `Unit decnet-emailgen.service could not - be found`, blocking the whole target. Anti-regression.""" - assert "decnet-emailgen.service" not in target_text - - -def test_target_wants_canary(target_text): - """Canary worker is a peer of orchestrator; both are part of the - realism + callback story. Bundle check.""" - assert "decnet-canary.service" in target_text - - -def test_target_orders_after_bus(target_text): - """Whole target depends on the bus being up.""" - assert "After=decnet-bus.service" in target_text - - -# ── unit file no longer exists ─────────────────────────────────────────────── - - -def test_emailgen_unit_template_is_gone(): - """The pre-collapse ``deploy/decnet-emailgen.service.j2`` must stay - deleted. A future commit that re-creates it (e.g. by accident - during a partial revert) would break the realism migration's - service-collapse contract.""" - assert not (DEPLOY / "decnet-emailgen.service.j2").exists(), ( - "decnet-emailgen.service.j2 reappeared — service collapse undone?" - ) From c918538f35fea86c2747207cb081bce8cde615ed Mon Sep 17 00:00:00 2001 From: anti Date: Thu, 18 Jun 2026 19:43:53 -0400 Subject: [PATCH 8/8] release: bump to v1.2.0; finalize CHANGELOG Prefork worker consolidation (decnet.prefork + decnet fleet heavy), ATT&CK 19.1 relocation to decnet/data/, and removal of the 10 per-worker unit templates superseded by the supervisor groups + heavy fleet. --- CHANGELOG.md | 13 ++++++++++++- pyproject.toml | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54c35ebe..a7412ae4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to DECNET are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] (1.2.0) +## [1.2.0] - 2026-06-18 Prefork worker consolidation — share the import floor across *separate* processes (own GIL, full isolation) via copy-on-write, for the heavy/isolation-critical @@ -33,6 +33,15 @@ workers the in-process supervisor can't co-host. fetched on demand via `python -m decnet.ttp.attack_stix fetch`, gitignored — not committed). +### Removed +- Per-worker systemd unit templates superseded by consolidation: + `decnet-{reconciler,enrich,orchestrator,mutator}` (→ `supervise batch`), + `decnet-{clusterer,campaign-clusterer,attribution,reuse-correlator}` + (→ `supervise cpu`), and `decnet-{profiler,ttp}` (→ `fleet heavy`). + `decnet.target` now pulls in the 3 consolidated units. The underlying CLI + commands remain for manual/standalone runs; a worker can be re-extracted to its + own unit by editing the group/fleet spec. + ## [1.1.1] - 2026-06-18 ### Fixed @@ -94,5 +103,7 @@ own unit. Initial 1.0 release. See tag `v1.0.0`. +[1.2.0]: https://git.resacachile.cl/anti/DECNET/compare/v1.1.1...v1.2.0 +[1.1.1]: https://git.resacachile.cl/anti/DECNET/compare/v1.1.0...v1.1.1 [1.1.0]: https://git.resacachile.cl/anti/DECNET/compare/v1.0.0...v1.1.0 [1.0.0]: https://git.resacachile.cl/anti/DECNET/releases/tag/v1.0.0 diff --git a/pyproject.toml b/pyproject.toml index 0a4526a8..9cd4e4d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "decnet" -version = "1.2.0.dev0" +version = "1.2.0" description = "Deception network: deploy honeypot deckies that appear as real LAN hosts" readme = "README.md" authors = [{ name = "Samuel Paschuan", email = "samuel.paschuan@xmartlab.com" }]