feat(swarm): per-host microservices as systemd units, mutator off agents

Previously `decnet status` on an agent showed every microservice as DOWN
because deploy's auto-spawn was unihost-scoped and the agent CLI gate
hid the per-host commands. Now:

  - collect, probe, profiler, sniffer drop out of MASTER_ONLY_COMMANDS
    (they run per-host; master-side work stays master-gated).
  - mutate stays master-only (it orchestrates swarm-wide respawns).
  - decnet/mutator/ excluded from agent tarballs — never invoked there.
  - decnet/web exclusion tightened: ship db/ + auth.py + dependencies.py
    (profiler needs the repo singleton), drop api.py, swarm_api.py,
    ingester.py, router/, templates/.
  - Four new systemd unit templates (decnet-collector/prober/profiler/
    sniffer) shipped in every enrollment tarball.
  - enroll_bootstrap.sh enables + starts all four alongside agent and
    forwarder at install time.
  - updater restarts the aux units on code push so they pick up the new
    release (best-effort — legacy enrollments without the units won't
    fail the update).
  - status table hides Mutator + API rows in agent mode.

decnet/cli.py

@@ -1323,6 +1323,11 @@ def status() -> None:
     _status()
 
     registry = _service_registry(str(DECNET_INGEST_LOG_FILE))
+    # On agents, the Mutator runs master-side only (it schedules decky
+    # respawns across the swarm) and the API is never shipped. Hide those
+    # rows so operators aren't chasing permanent DOWN entries.
+    if _agent_mode_active():
+        registry = [r for r in registry if r[0] not in {"Mutator", "API"}]
     svc_table = Table(title="DECNET Services", show_lines=True)
     svc_table.add_column("Service", style="bold cyan")
     svc_table.add_column("Status")
@@ -1762,13 +1767,17 @@ def db_reset(
 # MASTER_ONLY when touching command registration.
 #
 # Worker-legitimate commands (NOT in these sets): agent, updater, forwarder,
-# status (agents run deckies locally and should be able to inspect them).
+# status, collect, probe, profiler, sniffer. Agents run deckies locally and
+# should be able to inspect them + run the per-host microservices (collector
+# streams container logs, prober/profiler characterize attackers hitting
+# this host, sniffer captures traffic). Mutator stays master-only because
+# it orchestrates respawns across the swarm.
 # ───────────────────────────────────────────────────────────────────────────
 MASTER_ONLY_COMMANDS: frozenset[str] = frozenset({
     "api", "swarmctl", "deploy", "redeploy", "teardown",
-    "probe", "collect", "mutate", "listener",
+    "mutate", "listener",
     "services", "distros", "correlate", "archetypes", "web",
-    "profiler", "sniffer", "db-reset",
+    "db-reset",
 })
 MASTER_ONLY_GROUPS: frozenset[str] = frozenset({"swarm"})
 

decnet/updater/executor.py

@@ -237,6 +237,14 @@ def _run_pip(
 AGENT_SYSTEMD_UNIT = "decnet-agent.service"
 FORWARDER_SYSTEMD_UNIT = "decnet-forwarder.service"
 UPDATER_SYSTEMD_UNIT = "decnet-updater.service"
+# Per-host microservices that run out of the same /opt/decnet tree. An
+# update replaces their code, so we must cycle them alongside the agent or
+# they keep serving the pre-update image. Best-effort: legacy enrollments
+# without these units installed shouldn't abort the update.
+AUXILIARY_SYSTEMD_UNITS = (
+    "decnet-collector.service", "decnet-prober.service",
+    "decnet-profiler.service", "decnet-sniffer.service",
+)
 
 
 def _systemd_available() -> bool:
@@ -286,6 +294,13 @@ def _spawn_agent_via_systemd(install_dir: pathlib.Path) -> int:
     )
     if fwd.returncode != 0:
         log.warning("forwarder restart failed (ignored): %s", fwd.stderr.strip())
+    for unit in AUXILIARY_SYSTEMD_UNITS:
+        aux = subprocess.run(  # nosec B603 B607
+            ["systemctl", "restart", unit],
+            check=False, capture_output=True, text=True,
+        )
+        if aux.returncode != 0:
+            log.warning("%s restart failed (ignored): %s", unit, aux.stderr.strip())
     pid_out = subprocess.run(  # nosec B603 B607
         ["systemctl", "show", "--property=MainPID", "--value", AGENT_SYSTEMD_UNIT],
         check=True, capture_output=True, text=True,

decnet/web/router/swarm_mgmt/api_enroll_bundle.py

@@ -63,10 +63,19 @@ _EXCLUDES: tuple[str, ...] = (
     "wiki-checkout", "wiki-checkout/*",
     # Frontend is master-only; agents never serve UI.
     "decnet_web", "decnet_web/*", "decnet_web/**",
-    # Master FastAPI app (API, routers, master-side DB) is not run on agents.
-    # The `agent` / `updater` / `forwarder` commands have their own apps under
-    # decnet/agent, decnet/updater — they don't import decnet.web.
-    "decnet/web", "decnet/web/*", "decnet/web/**",
+    # Master API surface. Agents ship with decnet.web.db + auth + dependencies
+    # (the profiler microservice needs the repo singleton), but the FastAPI
+    # app itself (api.py, swarm_api.py, the full router tree, the ingester,
+    # and the .j2 templates that the master renders into the tarball) has no
+    # business running on a worker.
+    "decnet/web/api.py",
+    "decnet/web/swarm_api.py",
+    "decnet/web/ingester.py",
+    "decnet/web/router", "decnet/web/router/*", "decnet/web/router/**",
+    "decnet/web/templates", "decnet/web/templates/*", "decnet/web/templates/**",
+    # Mutator is master-only (it schedules decky respawns across the swarm);
+    # agents never invoke it. Keep it off the worker.
+    "decnet/mutator", "decnet/mutator/*", "decnet/mutator/**",
     "decnet-state.json",
     "master.log", "master.json",
     "decnet.tar",
@@ -254,7 +263,11 @@ def _build_tarball(
     return buf.getvalue()
 
 
-_SYSTEMD_UNITS = ("decnet-agent", "decnet-forwarder", "decnet-engine", "decnet-updater")
+_SYSTEMD_UNITS = (
+    "decnet-agent", "decnet-forwarder", "decnet-engine", "decnet-updater",
+    # Per-host microservices — activated by enroll_bootstrap.sh.
+    "decnet-collector", "decnet-prober", "decnet-profiler", "decnet-sniffer",
+)
 
 
 def _render_systemd_unit(name: str, agent_name: str, master_host: str) -> bytes:

decnet/web/templates/decnet-collector.service.j2

@@ -0,0 +1,20 @@
+[Unit]
+Description=DECNET container log collector — {{ agent_name }}
+Documentation=https://github.com/anti/DECNET
+After=network-online.target decnet-agent.service
+Wants=network-online.target
+PartOf=decnet-agent.service
+
+[Service]
+Type=simple
+WorkingDirectory=/opt/decnet
+Environment=DECNET_MODE=agent
+Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.collector.log
+ExecStart=/usr/local/bin/decnet collect --log-file /var/log/decnet/decnet.log
+Restart=on-failure
+RestartSec=5
+StandardOutput=append:/var/log/decnet/decnet.collector.log
+StandardError=append:/var/log/decnet/decnet.collector.log
+
+[Install]
+WantedBy=multi-user.target

decnet/web/templates/decnet-prober.service.j2

@@ -0,0 +1,20 @@
+[Unit]
+Description=DECNET attacker prober (JARM/HASSH/TCP fingerprint) — {{ agent_name }}
+Documentation=https://github.com/anti/DECNET
+After=network-online.target decnet-agent.service
+Wants=network-online.target
+PartOf=decnet-agent.service
+
+[Service]
+Type=simple
+WorkingDirectory=/opt/decnet
+Environment=DECNET_MODE=agent
+Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.prober.log
+ExecStart=/usr/local/bin/decnet probe --log-file /var/log/decnet/decnet.log --interval 300
+Restart=on-failure
+RestartSec=5
+StandardOutput=append:/var/log/decnet/decnet.prober.log
+StandardError=append:/var/log/decnet/decnet.prober.log
+
+[Install]
+WantedBy=multi-user.target

decnet/web/templates/decnet-profiler.service.j2

@@ -0,0 +1,20 @@
+[Unit]
+Description=DECNET attacker profiler — {{ agent_name }}
+Documentation=https://github.com/anti/DECNET
+After=network-online.target decnet-agent.service
+Wants=network-online.target
+PartOf=decnet-agent.service
+
+[Service]
+Type=simple
+WorkingDirectory=/opt/decnet
+Environment=DECNET_MODE=agent
+Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.profiler.log
+ExecStart=/usr/local/bin/decnet profiler --interval 30
+Restart=on-failure
+RestartSec=5
+StandardOutput=append:/var/log/decnet/decnet.profiler.log
+StandardError=append:/var/log/decnet/decnet.profiler.log
+
+[Install]
+WantedBy=multi-user.target

decnet/web/templates/decnet-sniffer.service.j2

@@ -0,0 +1,24 @@
+[Unit]
+Description=DECNET network sniffer — {{ agent_name }}
+Documentation=https://github.com/anti/DECNET
+After=network-online.target decnet-agent.service
+Wants=network-online.target
+PartOf=decnet-agent.service
+
+[Service]
+Type=simple
+WorkingDirectory=/opt/decnet
+Environment=DECNET_MODE=agent
+Environment=DECNET_SYSTEM_LOGS=/var/log/decnet/decnet.sniffer.log
+# scapy needs raw sockets; forwarder already runs with these caps, so we
+# mirror the same ambient set here.
+AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW
+CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW
+ExecStart=/usr/local/bin/decnet sniffer --log-file /var/log/decnet/decnet.log
+Restart=on-failure
+RestartSec=5
+StandardOutput=append:/var/log/decnet/decnet.sniffer.log
+StandardError=append:/var/log/decnet/decnet.sniffer.log
+
+[Install]
+WantedBy=multi-user.target

decnet/web/templates/enroll_bootstrap.sh.j2

@@ -60,15 +60,24 @@ chmod 0755 "$VENV_DIR/bin/decnet"
 ln -sf "$VENV_DIR/bin/decnet" /usr/local/bin/decnet
 
 echo "[DECNET] installing systemd units..."
-install -Dm0644 etc/systemd/system/decnet-agent.service /etc/systemd/system/decnet-agent.service
-install -Dm0644 etc/systemd/system/decnet-forwarder.service /etc/systemd/system/decnet-forwarder.service
-install -Dm0644 etc/systemd/system/decnet-engine.service /etc/systemd/system/decnet-engine.service
+for unit in \
+  decnet-agent decnet-forwarder decnet-engine \
+  decnet-collector decnet-prober decnet-profiler decnet-sniffer; do
+  install -Dm0644 "etc/systemd/system/${unit}.service" "/etc/systemd/system/${unit}.service"
+done
 if [[ "$WITH_UPDATER" == "true" ]]; then
   install -Dm0644 etc/systemd/system/decnet-updater.service /etc/systemd/system/decnet-updater.service
 fi
 systemctl daemon-reload
 
-ACTIVE_UNITS=(decnet-agent.service decnet-forwarder.service)
+# Agent + forwarder are the control plane; collector/prober/profiler/sniffer
+# are the per-host microservices that used to require `decnet deploy` to
+# auto-spawn. With systemd units they come up at boot and auto-restart.
+ACTIVE_UNITS=(
+  decnet-agent.service decnet-forwarder.service
+  decnet-collector.service decnet-prober.service
+  decnet-profiler.service decnet-sniffer.service
+)
 if [[ "$WITH_UPDATER" == "true" ]]; then
   ACTIVE_UNITS+=(decnet-updater.service)
 fi