merge: testing → main (reconcile 2-week divergence)

tests/fleet/test_archetypes.py

@@ -0,0 +1,312 @@
+"""
+Tests for machine archetypes and the amount= expansion feature.
+"""
+
+from __future__ import annotations
+
+import textwrap
+import tempfile
+import os
+import pytest
+
+from decnet.archetypes import (
+    ARCHETYPES,
+    all_archetypes,
+    get_archetype,
+    random_archetype,
+)
+from decnet.ini_loader import load_ini
+from decnet.distros import DISTROS
+
+
+# ---------------------------------------------------------------------------
+# Archetype registry
+# ---------------------------------------------------------------------------
+
+def test_all_archetypes_returns_all():
+    result = all_archetypes()
+    assert isinstance(result, dict)
+    assert len(result) == len(ARCHETYPES)
+
+
+def test_get_archetype_known():
+    arch = get_archetype("linux-server")
+    assert arch.slug == "linux-server"
+    assert "ssh" in arch.services
+
+
+def test_get_archetype_unknown_raises():
+    with pytest.raises(ValueError, match="Unknown archetype"):
+        get_archetype("does-not-exist")
+
+
+def test_random_archetype_returns_valid():
+    arch = random_archetype()
+    assert arch.slug in ARCHETYPES
+
+
+def test_every_archetype_has_services():
+    for slug, arch in ARCHETYPES.items():
+        assert arch.services, f"Archetype '{slug}' has no services"
+
+
+def test_every_archetype_has_preferred_distros():
+    for slug, arch in ARCHETYPES.items():
+        assert arch.preferred_distros, f"Archetype '{slug}' has no preferred_distros"
+
+
+def test_every_archetype_preferred_distro_is_valid():
+    valid_slugs = set(DISTROS.keys())
+    for slug, arch in ARCHETYPES.items():
+        for d in arch.preferred_distros:
+            assert d in valid_slugs, (
+                f"Archetype '{slug}' references unknown distro '{d}'"
+            )
+
+
+# ---------------------------------------------------------------------------
+# INI loader — archetype= parsing
+# ---------------------------------------------------------------------------
+
+def _write_ini(content: str) -> str:
+    """Write INI content to a temp file and return the path."""
+    content = textwrap.dedent(content)
+    fd, path = tempfile.mkstemp(suffix=".ini")
+    os.write(fd, content.encode())
+    os.close(fd)
+    return path
+
+
+def test_ini_archetype_parsed():
+    path = _write_ini("""
+        [general]
+        net=10.0.0.0/24
+        gw=10.0.0.1
+
+        [my-server]
+        archetype=linux-server
+    """)
+    cfg = load_ini(path)
+    os.unlink(path)
+    assert len(cfg.deckies) == 1
+    assert cfg.deckies[0].archetype == "linux-server"
+    assert cfg.deckies[0].services is None  # not overridden
+
+
+def test_ini_archetype_with_explicit_services_override():
+    """explicit services= must survive alongside archetype="""
+    path = _write_ini("""
+        [general]
+        net=10.0.0.0/24
+        gw=10.0.0.1
+
+        [my-server]
+        archetype=linux-server
+        services=ftp,smb
+    """)
+    cfg = load_ini(path)
+    os.unlink(path)
+    assert cfg.deckies[0].archetype == "linux-server"
+    assert cfg.deckies[0].services == ["ftp", "smb"]
+
+
+# ---------------------------------------------------------------------------
+# INI loader — amount= expansion
+# ---------------------------------------------------------------------------
+
+def test_ini_amount_one_keeps_section_name():
+    path = _write_ini("""
+        [general]
+        net=10.0.0.0/24
+        gw=10.0.0.1
+
+        [my-printer]
+        archetype=printer
+        amount=1
+    """)
+    cfg = load_ini(path)
+    os.unlink(path)
+    assert len(cfg.deckies) == 1
+    assert cfg.deckies[0].name == "my-printer"
+
+
+def test_ini_amount_expands_deckies():
+    path = _write_ini("""
+        [general]
+        net=10.0.0.0/24
+        gw=10.0.0.1
+
+        [corp-ws]
+        archetype=windows-workstation
+        amount=5
+    """)
+    cfg = load_ini(path)
+    os.unlink(path)
+    assert len(cfg.deckies) == 5
+    for i, d in enumerate(cfg.deckies, start=1):
+        assert d.name == f"corp-ws-{i:02d}"
+        assert d.archetype == "windows-workstation"
+        assert d.ip is None  # auto-allocated
+
+
+def test_ini_amount_with_ip_raises():
+    path = _write_ini("""
+        [general]
+        net=10.0.0.0/24
+        gw=10.0.0.1
+
+        [bad-group]
+        services=ssh
+        ip=10.0.0.50
+        amount=3
+    """)
+    with pytest.raises(ValueError, match="Cannot combine ip="):
+        load_ini(path)
+    os.unlink(path)
+
+
+def test_ini_amount_invalid_value_raises():
+    path = _write_ini("""
+        [general]
+        net=10.0.0.0/24
+        gw=10.0.0.1
+
+        [bad]
+        services=ssh
+        amount=potato
+    """)
+    with pytest.raises(ValueError, match="must be a positive integer"):
+        load_ini(path)
+    os.unlink(path)
+
+
+def test_ini_amount_zero_raises():
+    path = _write_ini("""
+        [general]
+        net=10.0.0.0/24
+        gw=10.0.0.1
+
+        [bad]
+        services=ssh
+        amount=0
+    """)
+    with pytest.raises(ValueError, match="must be a positive integer"):
+        load_ini(path)
+    os.unlink(path)
+
+
+def test_ini_amount_multiple_groups():
+    """Two groups with different amounts expand independently."""
+    path = _write_ini("""
+        [general]
+        net=10.0.0.0/24
+        gw=10.0.0.1
+
+        [workers]
+        archetype=linux-server
+        amount=3
+
+        [printers]
+        archetype=printer
+        amount=2
+    """)
+    cfg = load_ini(path)
+    os.unlink(path)
+    assert len(cfg.deckies) == 5
+    names = [d.name for d in cfg.deckies]
+    assert names == ["workers-01", "workers-02", "workers-03", "printers-01", "printers-02"]
+
+
+# ---------------------------------------------------------------------------
+# INI loader — per-service subsections propagate to expanded deckies
+# ---------------------------------------------------------------------------
+
+def test_ini_subsection_propagates_to_expanded_deckies():
+    """[group.ssh] must apply to group-01, group-02, ..."""
+    path = _write_ini("""
+        [general]
+        net=10.0.0.0/24
+        gw=10.0.0.1
+
+        [linux-hosts]
+        archetype=linux-server
+        amount=3
+
+        [linux-hosts.ssh]
+        kernel_version=5.15.0-76-generic
+    """)
+    cfg = load_ini(path)
+    os.unlink(path)
+    assert len(cfg.deckies) == 3
+    for d in cfg.deckies:
+        assert "ssh" in d.service_config
+        assert d.service_config["ssh"]["kernel_version"] == "5.15.0-76-generic"
+
+
+def test_ini_subsection_direct_match_unaffected():
+    """A direct [decky.svc] subsection must still work when amount=1."""
+    path = _write_ini("""
+        [general]
+        net=10.0.0.0/24
+        gw=10.0.0.1
+
+        [web-01]
+        services=http
+
+        [web-01.http]
+        server_header=Apache/2.4.51
+    """)
+    cfg = load_ini(path)
+    os.unlink(path)
+    assert cfg.deckies[0].service_config["http"]["server_header"] == "Apache/2.4.51"
+
+
+# ---------------------------------------------------------------------------
+# _build_deckies — archetype applied via CLI path
+# ---------------------------------------------------------------------------
+
+def test_build_deckies_archetype_sets_services():
+    from decnet.fleet import build_deckies as _build_deckies
+    from decnet.archetypes import get_archetype
+    arch = get_archetype("mail-server")
+    result = _build_deckies(
+        n=2,
+        ips=["10.0.0.10", "10.0.0.11"],
+        services_explicit=None,
+        randomize_services=False,
+        archetype=arch,
+    )
+    assert len(result) == 2
+    for d in result:
+        assert set(d.services) == set(arch.services)
+        assert d.archetype == "mail-server"
+
+
+def test_build_deckies_archetype_preferred_distros():
+    from decnet.fleet import build_deckies as _build_deckies
+    from decnet.archetypes import get_archetype
+    arch = get_archetype("iot-device")  # preferred_distros=["alpine"]
+    result = _build_deckies(
+        n=3,
+        ips=["10.0.0.10", "10.0.0.11", "10.0.0.12"],
+        services_explicit=None,
+        randomize_services=False,
+        archetype=arch,
+    )
+    for d in result:
+        assert d.distro == "alpine"
+
+
+def test_build_deckies_explicit_services_override_archetype():
+    from decnet.fleet import build_deckies as _build_deckies
+    from decnet.archetypes import get_archetype
+    arch = get_archetype("linux-server")
+    result = _build_deckies(
+        n=1,
+        ips=["10.0.0.10"],
+        services_explicit=["ftp"],
+        randomize_services=False,
+        archetype=arch,
+    )
+    assert result[0].services == ["ftp"]
+    assert result[0].archetype == "linux-server"

tests/fleet/test_auto_spawn.py

@@ -0,0 +1,209 @@
+"""Auto-spawn of forwarder from `decnet agent` (and listener from
+`decnet swarmctl`, added in a later patch).
+
+These tests monkeypatch subprocess.Popen inside decnet.cli so no real
+process is ever forked. We assert on the Popen call shape — argv,
+start_new_session, stdio redirection — plus PID-file correctness.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+
+class _FakePopen:
+    """Minimal Popen stub. Records the call; reports a fake PID."""
+    last_instance: "None | _FakePopen" = None
+
+    def __init__(self, argv: list[str], **kwargs: Any) -> None:
+        self.argv = argv
+        self.kwargs = kwargs
+        self.pid = 424242
+        _FakePopen.last_instance = self
+
+
+@pytest.fixture
+def fake_popen(monkeypatch):
+    import decnet.cli as cli_mod
+    # Patch the subprocess module _spawn_detached reaches via its local
+    # import. Easier: patch subprocess.Popen globally in the subprocess
+    # module, since _spawn_detached uses `import subprocess` locally.
+    import subprocess
+    monkeypatch.setattr(subprocess, "Popen", _FakePopen)
+    _FakePopen.last_instance = None
+    return cli_mod
+
+
+def test_spawn_detached_sets_new_session_and_writes_pid(fake_popen, tmp_path):
+    pid_file = tmp_path / "forwarder.pid"
+    pid = fake_popen._spawn_detached(
+        ["/usr/bin/true", "--flag"], pid_file,
+    )
+    # The helper returns the pid from the Popen instance.
+    assert pid == 424242
+    # PID file exists and contains a valid positive integer.
+    raw = pid_file.read_text().strip()
+    assert raw.isdigit(), f"PID file not numeric: {raw!r}"
+    assert int(raw) > 0, "PID file must contain a positive integer"
+    assert int(raw) == pid
+    # Detach flags were passed.
+    call = _FakePopen.last_instance
+    assert call is not None
+    assert call.kwargs["start_new_session"] is True
+    assert call.kwargs["close_fds"] is True
+    # stdin/stdout/stderr were redirected (file handles, not None).
+    assert call.kwargs["stdin"] is not None
+    assert call.kwargs["stdout"] is not None
+    assert call.kwargs["stderr"] is not None
+
+
+def test_pid_file_parent_is_created(fake_popen, tmp_path):
+    nested = tmp_path / "run" / "decnet" / "forwarder.pid"
+    assert not nested.parent.exists()
+    fake_popen._spawn_detached(["/usr/bin/true"], nested)
+    assert nested.exists()
+    assert int(nested.read_text().strip()) > 0
+
+
+def test_agent_autospawns_forwarder(fake_popen, monkeypatch, tmp_path):
+    """`decnet agent` calls _spawn_detached once with a forwarder argv."""
+    # Isolate PID dir to tmp_path so the test doesn't touch /opt/decnet.
+    from decnet.cli import utils as _cli_utils
+    monkeypatch.setattr(_cli_utils, "_pid_dir", lambda: tmp_path)
+    # Set master host so the auto-spawn branch fires.
+    monkeypatch.setenv("DECNET_SWARM_MASTER_HOST", "10.0.0.1")
+    monkeypatch.setenv("DECNET_SWARM_SYSLOG_PORT", "6514")
+    # Stub the actual agent server so the command body returns fast.
+    from decnet.agent import server as _agent_server
+    monkeypatch.setattr(_agent_server, "run", lambda *a, **k: 0)
+
+    # We also need to re-read DECNET_SWARM_MASTER_HOST through env.py at
+    # call time. env.py already read it at import, so patch on the module.
+    from decnet import env as _env
+    monkeypatch.setattr(_env, "DECNET_SWARM_MASTER_HOST", "10.0.0.1")
+
+    from typer.testing import CliRunner
+    runner = CliRunner()
+    # Invoke the agent command directly (without --daemon to avoid
+    # double-forking the pytest worker).
+    result = runner.invoke(fake_popen.app, ["agent", "--port", "8765"])
+    # Agent server was stubbed → exit=0; the important thing is the Popen
+    # got called with a forwarder argv.
+    assert result.exit_code == 0, result.stdout
+    call = _FakePopen.last_instance
+    assert call is not None, "expected _spawn_detached → Popen to fire"
+    assert "forwarder" in call.argv
+    assert "--master-host" in call.argv
+    assert "10.0.0.1" in call.argv
+    assert "--daemon" in call.argv
+    # PID file was written in the test tmpdir, not /opt/decnet.
+    assert (tmp_path / "forwarder.pid").exists()
+
+
+def test_agent_no_forwarder_flag_suppresses_spawn(fake_popen, monkeypatch, tmp_path):
+    from decnet.cli import utils as _cli_utils
+    monkeypatch.setattr(_cli_utils, "_pid_dir", lambda: tmp_path)
+    monkeypatch.setenv("DECNET_SWARM_MASTER_HOST", "10.0.0.1")
+    from decnet.agent import server as _agent_server
+    monkeypatch.setattr(_agent_server, "run", lambda *a, **k: 0)
+    from decnet import env as _env
+    monkeypatch.setattr(_env, "DECNET_SWARM_MASTER_HOST", "10.0.0.1")
+
+    from typer.testing import CliRunner
+    runner = CliRunner()
+    result = runner.invoke(fake_popen.app, ["agent", "--no-forwarder"])
+    assert result.exit_code == 0, result.stdout
+    assert _FakePopen.last_instance is None, "forwarder should NOT have been spawned"
+    assert not (tmp_path / "forwarder.pid").exists()
+
+
+def test_agent_skips_forwarder_when_master_unset(fake_popen, monkeypatch, tmp_path):
+    """If DECNET_SWARM_MASTER_HOST is not set, auto-spawn is silently
+    skipped — we don't know where to ship logs to."""
+    from decnet.cli import utils as _cli_utils
+    monkeypatch.setattr(_cli_utils, "_pid_dir", lambda: tmp_path)
+    monkeypatch.delenv("DECNET_SWARM_MASTER_HOST", raising=False)
+    from decnet.agent import server as _agent_server
+    monkeypatch.setattr(_agent_server, "run", lambda *a, **k: 0)
+    from decnet import env as _env
+    monkeypatch.setattr(_env, "DECNET_SWARM_MASTER_HOST", None)
+
+    from typer.testing import CliRunner
+    runner = CliRunner()
+    result = runner.invoke(fake_popen.app, ["agent"])
+    assert result.exit_code == 0
+    assert _FakePopen.last_instance is None
+
+
+# ───────────────────────────────────────────────────────────────────────────
+# swarmctl → listener auto-spawn
+# ───────────────────────────────────────────────────────────────────────────
+
+class _FakeUvicornPopen:
+    """Stub for the uvicorn subprocess inside swarmctl — returns immediately
+    so the Typer command body doesn't block on proc.wait()."""
+    def __init__(self, *a, **kw) -> None:
+        self.pid = 999999
+    def wait(self, *a, **kw) -> int:
+        return 0
+
+
+@pytest.fixture
+def fake_swarmctl_popen(monkeypatch):
+    """For swarmctl: record the detached listener spawn via _FakePopen
+    AND stub uvicorn's Popen so swarmctl's body returns immediately."""
+    import decnet.cli as cli_mod
+    import subprocess as _subp
+
+    calls: list[_FakePopen] = []
+
+    def _router(argv, **kwargs):
+        # Only the listener auto-spawn uses start_new_session + DEVNULL stdio.
+        if kwargs.get("start_new_session") and "stdin" in kwargs:
+            inst = _FakePopen(argv, **kwargs)
+            calls.append(inst)
+            return inst
+        # Anything else (the uvicorn child swarmctl blocks on) → cheap stub.
+        return _FakeUvicornPopen()
+
+    monkeypatch.setattr(_subp, "Popen", _router)
+    _FakePopen.last_instance = None
+    return cli_mod, calls
+
+
+def test_swarmctl_autospawns_listener(fake_swarmctl_popen, monkeypatch, tmp_path):
+    cli_mod, calls = fake_swarmctl_popen
+    from decnet.cli import utils as _cli_utils
+    monkeypatch.setattr(_cli_utils, "_pid_dir", lambda: tmp_path)
+    monkeypatch.setenv("DECNET_LISTENER_HOST", "0.0.0.0")
+    monkeypatch.setenv("DECNET_SWARM_SYSLOG_PORT", "6514")
+
+    from typer.testing import CliRunner
+    runner = CliRunner()
+    result = runner.invoke(cli_mod.app, ["swarmctl", "--port", "8770"])
+    assert result.exit_code == 0, result.stdout
+    assert len(calls) == 1, f"expected one detached spawn, got {len(calls)}"
+    argv = calls[0].argv
+    assert "listener" in argv
+    assert "--daemon" in argv
+    assert "--port" in argv and "6514" in argv
+    # PID file written.
+    pid_path = tmp_path / "listener.pid"
+    assert pid_path.exists()
+    assert int(pid_path.read_text().strip()) > 0
+
+
+def test_swarmctl_no_listener_flag_suppresses_spawn(fake_swarmctl_popen, monkeypatch, tmp_path):
+    cli_mod, calls = fake_swarmctl_popen
+    from decnet.cli import utils as _cli_utils
+    monkeypatch.setattr(_cli_utils, "_pid_dir", lambda: tmp_path)
+
+    from typer.testing import CliRunner
+    runner = CliRunner()
+    result = runner.invoke(cli_mod.app, ["swarmctl", "--no-listener"])
+    assert result.exit_code == 0, result.stdout
+    assert calls == [], "listener should NOT have been spawned"
+    assert not (tmp_path / "listener.pid").exists()

tests/fleet/test_composer.py

@@ -0,0 +1,269 @@
+"""
+Tests for the composer — verifies BASE_IMAGE injection and distro heterogeneity.
+"""
+
+import pytest
+from decnet.config import DeckyConfig, DecnetConfig
+from decnet.composer import generate_compose
+from decnet.distros import all_distros, DISTROS
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+APT_COMPATIBLE = {
+    "debian:bookworm-slim",
+    "ubuntu:22.04",
+    "ubuntu:20.04",
+    "kalilinux/kali-rolling",
+}
+
+BUILD_SERVICES = [
+    "ssh", "telnet", "http", "rdp", "smb", "ftp", "smtp", "elasticsearch",
+    "pop3", "imap", "mysql", "mssql", "redis", "mongodb", "postgres",
+    "ldap", "vnc", "docker_api", "k8s", "sip",
+    "mqtt", "llmnr", "snmp", "tftp", "conpot"
+]
+
+UPSTREAM_SERVICES: list = []
+
+
+def _make_config(services, distro="debian", base_image=None, build_base=None):
+    profile = DISTROS[distro]
+    decky = DeckyConfig(
+        name="decky-01",
+        ip="10.0.0.10",
+        services=services,
+        distro=distro,
+        base_image=base_image or profile.image,
+        build_base=build_base or profile.build_base,
+        hostname="test-host",
+    )
+    return DecnetConfig(
+        mode="unihost",
+        interface="eth0",
+        subnet="10.0.0.0/24",
+        gateway="10.0.0.1",
+        deckies=[decky],
+    )
+
+
+# ---------------------------------------------------------------------------
+# BASE_IMAGE injection — build services
+# ---------------------------------------------------------------------------
+
+@pytest.mark.parametrize("svc", BUILD_SERVICES)
+def test_build_service_gets_base_image_arg(svc):
+    """Every build service must have BASE_IMAGE injected in compose args."""
+    config = _make_config([svc], distro="debian")
+    compose = generate_compose(config)
+    key = f"decky-01-{svc}"
+    fragment = compose["services"][key]
+    assert "build" in fragment, f"{svc}: missing 'build' key"
+    assert "args" in fragment["build"], f"{svc}: build section missing 'args'"
+    assert "BASE_IMAGE" in fragment["build"]["args"], f"{svc}: BASE_IMAGE not in args"
+
+
+@pytest.mark.parametrize("distro,expected_build_base", [
+    ("debian",   "debian:bookworm-slim"),
+    ("ubuntu22", "ubuntu:22.04"),
+    ("ubuntu20", "ubuntu:20.04"),
+    ("kali",     "kalilinux/kali-rolling"),
+    ("rocky9",   "debian:bookworm-slim"),
+    ("alpine",   "debian:bookworm-slim"),
+])
+def test_build_service_base_image_matches_distro(distro, expected_build_base):
+    """BASE_IMAGE arg must match the distro's build_base."""
+    config = _make_config(["http"], distro=distro)
+    compose = generate_compose(config)
+    fragment = compose["services"]["decky-01-http"]
+    assert fragment["build"]["args"]["BASE_IMAGE"] == expected_build_base
+
+
+# ---------------------------------------------------------------------------
+# BASE_IMAGE NOT injected for upstream-image services
+# ---------------------------------------------------------------------------
+
+@pytest.mark.parametrize("svc", UPSTREAM_SERVICES)
+def test_upstream_service_has_no_build_section(svc):
+    """Upstream-image services must not receive a build section or BASE_IMAGE."""
+    config = _make_config([svc])
+    compose = generate_compose(config)
+    fragment = compose["services"][f"decky-01-{svc}"]
+    assert "build" not in fragment
+    assert "image" in fragment
+
+
+# ---------------------------------------------------------------------------
+# service_config propagation tests
+# ---------------------------------------------------------------------------
+
+def test_service_config_http_server_header():
+    """service_config for http must inject SERVER_HEADER into compose env."""
+    from decnet.config import DeckyConfig, DecnetConfig
+    from decnet.distros import DISTROS
+    profile = DISTROS["debian"]
+    decky = DeckyConfig(
+        name="decky-01", ip="10.0.0.10",
+        services=["http"], distro="debian",
+        base_image=profile.image, build_base=profile.build_base,
+        hostname="test-host",
+        service_config={"http": {"server_header": "nginx/1.18.0"}},
+    )
+    config = DecnetConfig(
+        mode="unihost", interface="eth0",
+        subnet="10.0.0.0/24", gateway="10.0.0.1",
+        deckies=[decky],
+    )
+    compose = generate_compose(config)
+    env = compose["services"]["decky-01-http"]["environment"]
+    assert env.get("SERVER_HEADER") == "nginx/1.18.0"
+
+
+def test_service_config_ssh_password():
+    """service_config for ssh must inject SSH_ROOT_PASSWORD."""
+    from decnet.config import DeckyConfig, DecnetConfig
+    from decnet.distros import DISTROS
+    profile = DISTROS["debian"]
+    decky = DeckyConfig(
+        name="decky-01", ip="10.0.0.10",
+        services=["ssh"], distro="debian",
+        base_image=profile.image, build_base=profile.build_base,
+        hostname="test-host",
+        service_config={"ssh": {"password": "s3cr3t!"}},
+    )
+    config = DecnetConfig(
+        mode="unihost", interface="eth0",
+        subnet="10.0.0.0/24", gateway="10.0.0.1",
+        deckies=[decky],
+    )
+    compose = generate_compose(config)
+    env = compose["services"]["decky-01-ssh"]["environment"]
+    assert env.get("SSH_ROOT_PASSWORD") == "s3cr3t!"
+    assert not any(k.startswith("COWRIE_") for k in env)
+
+
+def test_service_config_for_one_service_does_not_affect_another():
+    """service_config for http must not bleed into ftp fragment."""
+    from decnet.config import DeckyConfig, DecnetConfig
+    from decnet.distros import DISTROS
+    profile = DISTROS["debian"]
+    decky = DeckyConfig(
+        name="decky-01", ip="10.0.0.10",
+        services=["http", "ftp"], distro="debian",
+        base_image=profile.image, build_base=profile.build_base,
+        hostname="test-host",
+        service_config={"http": {"server_header": "nginx/1.18.0"}},
+    )
+    config = DecnetConfig(
+        mode="unihost", interface="eth0",
+        subnet="10.0.0.0/24", gateway="10.0.0.1",
+        deckies=[decky],
+    )
+    compose = generate_compose(config)
+    ftp_env = compose["services"]["decky-01-ftp"]["environment"]
+    assert "SERVER_HEADER" not in ftp_env
+
+
+def test_no_service_config_produces_no_extra_env():
+    """A decky with no service_config must not have new persona env vars."""
+    config = _make_config(["http", "mysql"])
+    compose = generate_compose(config)
+    for svc in ("http", "mysql"):
+        env = compose["services"][f"decky-01-{svc}"]["environment"]
+        assert "SERVER_HEADER" not in env
+        assert "MYSQL_VERSION" not in env
+
+
+# ---------------------------------------------------------------------------
+# Base container uses distro image, not build_base
+# ---------------------------------------------------------------------------
+
+@pytest.mark.parametrize("distro", list(DISTROS.keys()))
+def test_base_container_uses_full_distro_image(distro):
+    """The IP-holder base container must use distro.image, not build_base."""
+    config = _make_config(["ssh"], distro=distro)
+    compose = generate_compose(config)
+    base = compose["services"]["decky-01"]
+    expected = DISTROS[distro].image
+    assert base["image"] == expected, (
+        f"distro={distro}: base container image '{base['image']}' != '{expected}'"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Distro profile — build_base is always apt-compatible
+# ---------------------------------------------------------------------------
+
+def test_all_distros_have_build_base():
+    for slug, profile in all_distros().items():
+        assert profile.build_base, f"Distro '{slug}' has empty build_base"
+
+
+def test_all_distro_build_bases_are_apt_compatible():
+    for slug, profile in all_distros().items():
+        assert profile.build_base in APT_COMPATIBLE, (
+            f"Distro '{slug}' build_base '{profile.build_base}' is not apt-compatible. "
+            f"Allowed: {APT_COMPATIBLE}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Heterogeneity — multiple deckies with different distros get different images
+# ---------------------------------------------------------------------------
+
+def test_multiple_deckies_different_build_bases():
+    """A multi-decky deployment with ubuntu22 and debian must differ in BASE_IMAGE."""
+    deckies = [
+        DeckyConfig(
+            name="decky-01", ip="10.0.0.10",
+            services=["http"], distro="debian",
+            base_image="debian:bookworm-slim", build_base="debian:bookworm-slim",
+            hostname="host-01",
+        ),
+        DeckyConfig(
+            name="decky-02", ip="10.0.0.11",
+            services=["http"], distro="ubuntu22",
+            base_image="ubuntu:22.04", build_base="ubuntu:22.04",
+            hostname="host-02",
+        ),
+    ]
+    config = DecnetConfig(
+        mode="unihost", interface="eth0",
+        subnet="10.0.0.0/24", gateway="10.0.0.1",
+        deckies=deckies,
+    )
+    compose = generate_compose(config)
+
+    base_img_01 = compose["services"]["decky-01-http"]["build"]["args"]["BASE_IMAGE"]
+    base_img_02 = compose["services"]["decky-02-http"]["build"]["args"]["BASE_IMAGE"]
+
+    assert base_img_01 == "debian:bookworm-slim"
+    assert base_img_02 == "ubuntu:22.04"
+    assert base_img_01 != base_img_02
+
+
+# ---------------------------------------------------------------------------
+# Fleet ownership labels — collector keys off these to recognize freshly-
+# deployed containers without consulting decnet-state.json (the previous
+# state-file lookup race silently dropped containers whose Docker start
+# event arrived before the state write completed).
+# ---------------------------------------------------------------------------
+
+def test_service_container_carries_fleet_labels():
+    config = _make_config(["http"], distro="debian")
+    compose = generate_compose(config)
+    labels = compose["services"]["decky-01-http"]["labels"]
+    assert labels["decnet.fleet.service"] == "true"
+    assert labels["decnet.fleet.decky"] == "decky-01"
+    assert labels["decnet.fleet.service_name"] == "http"
+
+
+def test_base_container_does_not_carry_service_label():
+    """Base containers run sleep — they don't emit logs and must NOT be
+    streamed by the collector, so the service marker stays off them."""
+    config = _make_config(["http"], distro="debian")
+    compose = generate_compose(config)
+    base = compose["services"]["decky-01"]
+    assert "decnet.fleet.service" not in (base.get("labels") or {})

tests/fleet/test_deployer.py

@@ -0,0 +1,664 @@
+"""
+Tests for decnet/engine/deployer.py
+
+Covers _compose, _compose_with_retry, _sync_logging_helper,
+deploy (dry-run and mocked), teardown, status, and _print_status.
+All Docker and subprocess calls are mocked.
+"""
+
+import subprocess
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from decnet.config import DeckyConfig, DecnetConfig
+
+
+@pytest.fixture(autouse=True)
+def _stub_fleet_db_mirror(request):
+    """The DB-mirror helpers are exercised in :class:`TestMirrorFleetToDb`;
+    every other test in this file mocks filesystem + docker but not the DB,
+    so we no-op the mirrors elsewhere to keep the suite self-contained."""
+    if "MirrorFleetToDb" in request.node.nodeid:
+        yield
+        return
+    with patch("decnet.engine.deployer._mirror_fleet_deploy_to_db"), \
+         patch("decnet.engine.deployer._mirror_fleet_teardown_to_db"):
+        yield
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+def _decky(name: str = "decky-01", ip: str = "192.168.1.10",
+           services: list[str] | None = None) -> DeckyConfig:
+    return DeckyConfig(
+        name=name, ip=ip, services=services or ["ssh"],
+        distro="debian", base_image="debian", hostname="test-host",
+        build_base="debian:bookworm-slim", nmap_os="linux",
+    )
+
+
+def _config(deckies: list[DeckyConfig] | None = None, ipvlan: bool = False) -> DecnetConfig:
+    return DecnetConfig(
+        mode="unihost", interface="eth0", subnet="192.168.1.0/24",
+        gateway="192.168.1.1", deckies=deckies or [_decky()],
+        ipvlan=ipvlan,
+    )
+
+
+# ── _compose ──────────────────────────────────────────────────────────────────
+
+class TestCompose:
+    @patch("decnet.engine.deployer.subprocess.run")
+    def test_compose_constructs_correct_command(self, mock_run):
+        from decnet.engine.deployer import _compose
+        mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
+        _compose("up", "-d", compose_file=Path("test.yml"))
+        mock_run.assert_called_once()
+        cmd = mock_run.call_args[0][0]
+        assert cmd[:6] == ["docker", "compose", "-p", "decnet", "-f", "test.yml"]
+        assert "up" in cmd
+        assert "-d" in cmd
+
+    @patch("decnet.engine.deployer.subprocess.run")
+    def test_compose_passes_env(self, mock_run):
+        from decnet.engine.deployer import _compose
+        mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
+        _compose("build", env={"DOCKER_BUILDKIT": "1"})
+        _, kwargs = mock_run.call_args
+        assert "DOCKER_BUILDKIT" in kwargs["env"]
+
+
+# ── _compose_with_retry ───────────────────────────────────────────────────────
+
+@pytest.fixture(autouse=True)
+def _no_leaked_buildkit_mounts(monkeypatch):
+    """Stub out the wedge-detector so dev-host state doesn't bleed into
+    the mocked-subprocess tests below. Tests that exercise the preflight
+    itself patch this function explicitly."""
+    monkeypatch.setattr(
+        "decnet.engine.deployer._count_leaked_buildkit_mounts",
+        lambda: 0,
+    )
+
+
+class TestComposeWithRetry:
+    @patch("decnet.engine.deployer.subprocess.run")
+    def test_success_first_try(self, mock_run):
+        from decnet.engine.deployer import _compose_with_retry
+        mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
+        _compose_with_retry("up", "-d")  # should not raise
+
+    @patch("decnet.engine.deployer.time.sleep")
+    @patch("decnet.engine.deployer.subprocess.run")
+    def test_transient_failure_retries(self, mock_run, mock_sleep):
+        from decnet.engine.deployer import _compose_with_retry
+        fail_result = MagicMock(returncode=1, stdout="", stderr="temporary error")
+        ok_result = MagicMock(returncode=0, stdout="ok", stderr="")
+        mock_run.side_effect = [fail_result, ok_result]
+        _compose_with_retry("up", retries=3)
+        assert mock_run.call_count == 2
+        mock_sleep.assert_called_once()
+
+    @patch("decnet.engine.deployer.time.sleep")
+    @patch("decnet.engine.deployer.subprocess.run")
+    def test_permanent_error_no_retry(self, mock_run, mock_sleep):
+        from decnet.engine.deployer import _compose_with_retry
+        fail_result = MagicMock(returncode=1, stdout="", stderr="manifest unknown error")
+        mock_run.return_value = fail_result
+        with pytest.raises(subprocess.CalledProcessError):
+            _compose_with_retry("pull", retries=3)
+        assert mock_run.call_count == 1
+        mock_sleep.assert_not_called()
+
+    @patch("decnet.engine.deployer.time.sleep")
+    @patch("decnet.engine.deployer.subprocess.run")
+    def test_max_retries_exhausted(self, mock_run, mock_sleep):
+        from decnet.engine.deployer import _compose_with_retry
+        fail_result = MagicMock(returncode=1, stdout="", stderr="connection refused")
+        mock_run.return_value = fail_result
+        with pytest.raises(subprocess.CalledProcessError):
+            _compose_with_retry("up", retries=2)
+        assert mock_run.call_count == 2
+
+    @patch("decnet.engine.deployer.subprocess.run")
+    def test_stdout_printed_on_success(self, mock_run, capsys):
+        from decnet.engine.deployer import _compose_with_retry
+        mock_run.return_value = MagicMock(returncode=0, stdout="done\n", stderr="")
+        _compose_with_retry("build")
+        captured = capsys.readouterr()
+        assert "done" in captured.out
+
+    @patch("decnet.engine.deployer.subprocess.run")
+    def test_buildx_preflight_blocks_when_wedged(self, mock_run, monkeypatch):
+        """Pre-flight: refuse to run a build command when buildx already
+        shows pathological mount leakage — retrying would only leak more."""
+        from decnet.engine import deployer
+        monkeypatch.setattr(deployer, "_count_leaked_buildkit_mounts", lambda: 42)
+        with pytest.raises(subprocess.CalledProcessError) as ei:
+            deployer._compose_with_retry("up", "--build", "-d")
+        mock_run.assert_not_called()
+        assert "Buildx is wedged" in ei.value.stderr
+        # leaked>0 recipe centres on unmount + daemon stop, since
+        # prune+restart alone doesn't evict already-held mounts.
+        assert "umount -l" in ei.value.stderr
+        assert "Detected 42 leaked" in ei.value.stderr
+
+    @patch("decnet.engine.deployer.subprocess.run")
+    def test_buildx_preflight_skipped_for_non_build_cmds(self, mock_run, monkeypatch):
+        """down/stop/etc. don't go through buildx — the preflight must
+        not block them even if mounts are leaked."""
+        from decnet.engine import deployer
+        monkeypatch.setattr(deployer, "_count_leaked_buildkit_mounts", lambda: 999)
+        mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
+        deployer._compose_with_retry("down")  # must not raise
+        mock_run.assert_called_once()
+
+    @patch("decnet.engine.deployer.time.sleep")
+    @patch("decnet.engine.deployer.subprocess.run")
+    def test_buildx_wedge_mid_build_short_circuits_retries(self, mock_run, mock_sleep):
+        """If a build fails with the wedge signature, skip remaining
+        retries and surface the recovery hint in stderr."""
+        from decnet.engine.deployer import _compose_with_retry
+        fail = MagicMock(
+            returncode=1, stdout="",
+            stderr="failed to update builder last activity time: "
+                   "open /home/x/.docker/buildx/activity/.tmp-default: read-only file system",
+        )
+        mock_run.return_value = fail
+        with pytest.raises(subprocess.CalledProcessError) as ei:
+            _compose_with_retry("up", "--build", retries=5)
+        assert mock_run.call_count == 1  # no retry
+        mock_sleep.assert_not_called()
+        assert "Buildx is wedged" in ei.value.stderr
+        # Original stderr is preserved alongside the hint so the user
+        # can see what compose actually said.
+        assert "Original error" in ei.value.stderr
+
+    @patch("decnet.engine.deployer.subprocess.run")
+    def test_buildx_wedge_protecthome_branch(self, mock_run, monkeypatch):
+        """When stderr names a path under /home and no mounts are
+        leaked, the cause is systemd's ProtectHome — recipe should
+        point at DOCKER_CONFIG redirection, not driver rebuild."""
+        from decnet.engine import deployer
+        monkeypatch.setattr(deployer, "_count_leaked_buildkit_mounts", lambda: 0)
+        mock_run.return_value = MagicMock(
+            returncode=1, stdout="",
+            stderr=("failed to update builder last activity time: open "
+                    "/home/anti/.docker/buildx/activity/.tmp-x: read-only file system"),
+        )
+        with pytest.raises(subprocess.CalledProcessError) as ei:
+            deployer._compose_with_retry("up", "--build")
+        assert "ProtectHome=read-only" in ei.value.stderr
+        assert "DOCKER_CONFIG" in ei.value.stderr
+        assert "BUILDX_CONFIG" in ei.value.stderr
+        # Driver-rebuild recipe must NOT be the suggested fix here.
+        assert "buildx create --name decnet-builder" not in ei.value.stderr
+
+    @patch("decnet.engine.deployer.subprocess.run")
+    def test_buildx_wedge_zero_mounts_uses_driver_rebuild_recipe(self, mock_run, monkeypatch):
+        """Wedge signature with 0 leaked mounts means the buildx driver
+        itself is corrupt — recipe should suggest rebuilding it, not
+        unmounting nothing."""
+        from decnet.engine import deployer
+        monkeypatch.setattr(deployer, "_count_leaked_buildkit_mounts", lambda: 0)
+        mock_run.return_value = MagicMock(
+            returncode=1, stdout="",
+            # No /home/ path — driver-rebuild branch, not ProtectHome.
+            stderr="failed to update builder last activity time: open "
+                   "/var/lib/decnet/.docker/buildx/activity/.tmp-x: read-only file system",
+        )
+        with pytest.raises(subprocess.CalledProcessError) as ei:
+            deployer._compose_with_retry("up", "--build")
+        assert "buildx create --name decnet-builder" in ei.value.stderr
+        assert "umount" not in ei.value.stderr
+        assert "No leaked mounts (count=0)" in ei.value.stderr
+
+    @patch("decnet.engine.deployer.time.sleep")
+    @patch("decnet.engine.deployer.subprocess.run")
+    def test_unrelated_erofs_does_not_match_wedge(self, mock_run, mock_sleep):
+        """Stderr containing 'read-only file system' alone (no buildx
+        activity-time phrase) must NOT be classified as a wedge — that
+        was the false-positive that misled the user."""
+        from decnet.engine.deployer import _compose_with_retry
+        fail = MagicMock(
+            returncode=1, stdout="",
+            stderr="open /etc/foo/bar: read-only file system",  # not buildx
+        )
+        mock_run.return_value = fail
+        with pytest.raises(subprocess.CalledProcessError) as ei:
+            _compose_with_retry("up", "--build", retries=2)
+        assert "Buildx is wedged" not in (ei.value.stderr or "")
+        # Treated as a normal transient error → retried until exhausted.
+        assert mock_run.call_count == 2
+
+
+# ── _sync_logging_helper ─────────────────────────────────────────────────────
+
+class TestSyncLoggingHelper:
+    @patch("decnet.engine.deployer.shutil.copy2")
+    @patch("decnet.engine.deployer._CANONICAL_LOGGING")
+    def test_copies_when_file_differs(self, mock_canonical, mock_copy):
+        from decnet.engine.deployer import _sync_logging_helper
+        mock_svc = MagicMock()
+        mock_svc.dockerfile_context.return_value = Path("/tmp/test_ctx")
+        mock_canonical.__truediv__ = Path.__truediv__
+
+        with patch("decnet.services.registry.get_service", return_value=mock_svc):
+            with patch("pathlib.Path.exists", return_value=False):
+                config = _config()
+                _sync_logging_helper(config)
+
+
+# ── deploy ────────────────────────────────────────────────────────────────────
+
+class TestDeploy:
+    @patch("decnet.engine.deployer._print_status")
+    @patch("decnet.engine.deployer._compose_with_retry")
+    @patch("decnet.engine.deployer.save_state")
+    @patch("decnet.engine.deployer.write_compose", return_value=Path("test.yml"))
+    @patch("decnet.engine.deployer._sync_logging_helper")
+    @patch("decnet.engine.deployer.setup_host_macvlan")
+    @patch("decnet.engine.deployer.create_macvlan_network")
+    @patch("decnet.engine.deployer.get_host_ip", return_value="192.168.1.2")
+    @patch("decnet.engine.deployer.ips_to_range", return_value="192.168.1.10/32")
+    @patch("decnet.engine.deployer.docker.from_env")
+    def test_dry_run_no_containers(self, mock_docker, mock_range, mock_hip,
+                                    mock_create, mock_setup, mock_sync,
+                                    mock_compose, mock_save, mock_retry, mock_print):
+        from decnet.engine.deployer import deploy
+        config = _config()
+        deploy(config, dry_run=True)
+        mock_create.assert_not_called()
+        mock_retry.assert_not_called()
+        mock_save.assert_not_called()
+
+    @patch("decnet.engine.deployer._print_status")
+    @patch("decnet.engine.deployer._compose_with_retry")
+    @patch("decnet.engine.deployer.save_state")
+    @patch("decnet.engine.deployer.write_compose", return_value=Path("test.yml"))
+    @patch("decnet.engine.deployer._sync_logging_helper")
+    @patch("decnet.engine.deployer.setup_host_macvlan")
+    @patch("decnet.engine.deployer.create_macvlan_network")
+    @patch("decnet.engine.deployer.get_host_ip", return_value="192.168.1.2")
+    @patch("decnet.engine.deployer.ips_to_range", return_value="192.168.1.10/32")
+    @patch("decnet.engine.deployer.docker.from_env")
+    def test_macvlan_deploy(self, mock_docker, mock_range, mock_hip,
+                             mock_create, mock_setup, mock_sync,
+                             mock_compose, mock_save, mock_retry, mock_print):
+        from decnet.engine.deployer import deploy
+        config = _config(ipvlan=False)
+        deploy(config)
+        mock_create.assert_called_once()
+        mock_setup.assert_called_once()
+        mock_save.assert_called_once()
+        mock_retry.assert_called()
+
+    @patch("decnet.engine.deployer._emit_lifecycle_event")
+    @patch("decnet.engine.deployer._print_status")
+    @patch("decnet.engine.deployer._compose_with_retry")
+    @patch("decnet.engine.deployer.save_state")
+    @patch("decnet.engine.deployer.write_compose", return_value=Path("test.yml"))
+    @patch("decnet.engine.deployer._sync_logging_helper")
+    @patch("decnet.engine.deployer.setup_host_macvlan")
+    @patch("decnet.engine.deployer.create_macvlan_network")
+    @patch("decnet.engine.deployer.get_host_ip", return_value="192.168.1.2")
+    @patch("decnet.engine.deployer.ips_to_range", return_value="192.168.1.10/32")
+    @patch("decnet.engine.deployer.docker.from_env")
+    def test_emits_creation_event_per_decky(
+        self, mock_docker, mock_range, mock_hip, mock_create, mock_setup,
+        mock_sync, mock_compose, mock_save, mock_retry, mock_print, mock_emit,
+    ):
+        from decnet.engine.deployer import deploy
+        deckies = [
+            _decky(name="decky-01", ip="192.168.1.10", services=["ssh"]),
+            _decky(name="decky-02", ip="192.168.1.11", services=["http", "ftp"]),
+        ]
+        deploy(_config(deckies=deckies))
+        assert mock_emit.call_count == 2
+        triggers = [c.kwargs["trigger"] for c in mock_emit.call_args_list]
+        assert triggers == ["creation", "creation"]
+        names = [c.kwargs["decky_name"] for c in mock_emit.call_args_list]
+        assert names == ["decky-01", "decky-02"]
+        # empty-set symmetry: creation has old=[] ⇒ new=<initial>
+        for call in mock_emit.call_args_list:
+            assert call.kwargs["old_services"] == []
+        assert mock_emit.call_args_list[0].kwargs["new_services"] == ["ssh"]
+        assert mock_emit.call_args_list[1].kwargs["new_services"] == ["http", "ftp"]
+
+    @patch("decnet.engine.deployer._emit_lifecycle_event")
+    @patch("decnet.engine.deployer._print_status")
+    @patch("decnet.engine.deployer._compose_with_retry")
+    @patch("decnet.engine.deployer.save_state")
+    @patch("decnet.engine.deployer.write_compose", return_value=Path("test.yml"))
+    @patch("decnet.engine.deployer._sync_logging_helper")
+    @patch("decnet.engine.deployer.setup_host_macvlan")
+    @patch("decnet.engine.deployer.create_macvlan_network")
+    @patch("decnet.engine.deployer.get_host_ip", return_value="192.168.1.2")
+    @patch("decnet.engine.deployer.ips_to_range", return_value="192.168.1.10/32")
+    @patch("decnet.engine.deployer.docker.from_env")
+    def test_dry_run_skips_creation_events(
+        self, mock_docker, mock_range, mock_hip, mock_create, mock_setup,
+        mock_sync, mock_compose, mock_save, mock_retry, mock_print, mock_emit,
+    ):
+        from decnet.engine.deployer import deploy
+        deploy(_config(), dry_run=True)
+        mock_emit.assert_not_called()
+
+    @patch("decnet.engine.deployer._print_status")
+    @patch("decnet.engine.deployer._compose_with_retry")
+    @patch("decnet.engine.deployer.save_state")
+    @patch("decnet.engine.deployer.write_compose", return_value=Path("test.yml"))
+    @patch("decnet.engine.deployer._sync_logging_helper")
+    @patch("decnet.engine.deployer.setup_host_ipvlan")
+    @patch("decnet.engine.deployer.create_ipvlan_network")
+    @patch("decnet.engine.deployer.get_host_ip", return_value="192.168.1.2")
+    @patch("decnet.engine.deployer.ips_to_range", return_value="192.168.1.10/32")
+    @patch("decnet.engine.deployer.docker.from_env")
+    def test_ipvlan_deploy(self, mock_docker, mock_range, mock_hip,
+                            mock_create, mock_setup, mock_sync,
+                            mock_compose, mock_save, mock_retry, mock_print):
+        from decnet.engine.deployer import deploy
+        config = _config(ipvlan=True)
+        deploy(config)
+        mock_create.assert_called_once()
+        mock_setup.assert_called_once()
+
+    @patch("decnet.engine.deployer._print_status")
+    @patch("decnet.engine.deployer._compose_with_retry")
+    @patch("decnet.engine.deployer.save_state")
+    @patch("decnet.engine.deployer.write_compose", return_value=Path("test.yml"))
+    @patch("decnet.engine.deployer._sync_logging_helper")
+    @patch("decnet.engine.deployer.setup_host_macvlan")
+    @patch("decnet.engine.deployer.create_macvlan_network")
+    @patch("decnet.engine.deployer.get_host_ip", return_value="192.168.1.2")
+    @patch("decnet.engine.deployer.ips_to_range", return_value="192.168.1.10/32")
+    @patch("decnet.engine.deployer.docker.from_env")
+    def test_parallel_build(self, mock_docker, mock_range, mock_hip,
+                             mock_create, mock_setup, mock_sync,
+                             mock_compose, mock_save, mock_retry, mock_print):
+        from decnet.engine.deployer import deploy
+        config = _config()
+        deploy(config, parallel=True)
+        # Parallel mode calls _compose_with_retry for "build" and "up" separately
+        calls = mock_retry.call_args_list
+        assert any("build" in str(c) for c in calls)
+
+    @patch("decnet.engine.deployer._print_status")
+    @patch("decnet.engine.deployer._compose_with_retry")
+    @patch("decnet.engine.deployer.save_state")
+    @patch("decnet.engine.deployer.write_compose", return_value=Path("test.yml"))
+    @patch("decnet.engine.deployer._sync_logging_helper")
+    @patch("decnet.engine.deployer.setup_host_macvlan")
+    @patch("decnet.engine.deployer.create_macvlan_network")
+    @patch("decnet.engine.deployer.get_host_ip", return_value="192.168.1.2")
+    @patch("decnet.engine.deployer.ips_to_range", return_value="192.168.1.10/32")
+    @patch("decnet.engine.deployer.docker.from_env")
+    def test_no_cache_build(self, mock_docker, mock_range, mock_hip,
+                             mock_create, mock_setup, mock_sync,
+                             mock_compose, mock_save, mock_retry, mock_print):
+        from decnet.engine.deployer import deploy
+        config = _config()
+        deploy(config, no_cache=True)
+        calls = mock_retry.call_args_list
+        assert any("--no-cache" in str(c) for c in calls)
+
+
+# ── teardown ──────────────────────────────────────────────────────────────────
+
+class TestTeardown:
+    @patch("decnet.engine.deployer.load_state", return_value=None)
+    def test_no_state(self, mock_load):
+        from decnet.engine.deployer import teardown
+        teardown()  # should not raise
+
+    @patch("decnet.engine.deployer.clear_state")
+    @patch("decnet.engine.deployer.remove_macvlan_network")
+    @patch("decnet.engine.deployer.teardown_host_macvlan")
+    @patch("decnet.engine.deployer._compose")
+    @patch("decnet.engine.deployer.ips_to_range", return_value="192.168.1.10/32")
+    @patch("decnet.engine.deployer.docker.from_env")
+    @patch("decnet.engine.deployer.load_state")
+    def test_full_teardown_macvlan(self, mock_load, mock_docker, mock_range,
+                                    mock_compose, mock_td_macvlan, mock_rm_net,
+                                    mock_clear):
+        config = _config()
+        mock_load.return_value = (config, Path("test.yml"))
+        from decnet.engine.deployer import teardown
+        teardown()
+        mock_compose.assert_called_once()
+        mock_td_macvlan.assert_called_once()
+        mock_rm_net.assert_called_once()
+        mock_clear.assert_called_once()
+
+    @patch("decnet.engine.deployer.clear_state")
+    @patch("decnet.engine.deployer.remove_macvlan_network")
+    @patch("decnet.engine.deployer.teardown_host_ipvlan")
+    @patch("decnet.engine.deployer._compose")
+    @patch("decnet.engine.deployer.ips_to_range", return_value="192.168.1.10/32")
+    @patch("decnet.engine.deployer.docker.from_env")
+    @patch("decnet.engine.deployer.load_state")
+    def test_full_teardown_ipvlan(self, mock_load, mock_docker, mock_range,
+                                   mock_compose, mock_td_ipvlan, mock_rm_net,
+                                   mock_clear):
+        config = _config(ipvlan=True)
+        mock_load.return_value = (config, Path("test.yml"))
+        from decnet.engine.deployer import teardown
+        teardown()
+        mock_td_ipvlan.assert_called_once()
+
+    @patch("decnet.engine.deployer._compose")
+    @patch("decnet.engine.deployer.docker.from_env")
+    @patch("decnet.engine.deployer.load_state")
+    def test_single_decky_emits_flat_service_names(
+        self, mock_load, mock_docker, mock_compose,
+    ):
+        """Regression: teardown(decky_id=...) must iterate the matched decky's
+        services, not stringify the services list itself. The old nested
+        comprehension produced `decky3-['sip']` and docker compose choked."""
+        config = _config(deckies=[
+            _decky(name="decky3", ip="192.168.1.13", services=["sip", "ssh"]),
+            _decky(name="decky4", ip="192.168.1.14", services=["http"]),
+        ])
+        mock_load.return_value = (config, Path("test.yml"))
+        from decnet.engine.deployer import teardown
+        teardown(decky_id="decky3")
+
+        # stop + rm, each called with the flat per-service names
+        assert mock_compose.call_count == 2
+        for call in mock_compose.call_args_list:
+            args = call.args
+            svc_names = [a for a in args if a.startswith("decky3-")]
+            assert svc_names == ["decky3-sip", "decky3-ssh"], svc_names
+            for name in svc_names:
+                assert "[" not in name and "'" not in name
+
+    @patch("decnet.engine.deployer._emit_lifecycle_event")
+    @patch("decnet.engine.deployer.clear_state")
+    @patch("decnet.engine.deployer.remove_macvlan_network")
+    @patch("decnet.engine.deployer.teardown_host_macvlan")
+    @patch("decnet.engine.deployer._compose")
+    @patch("decnet.engine.deployer.ips_to_range", return_value="192.168.1.10/32")
+    @patch("decnet.engine.deployer.docker.from_env")
+    @patch("decnet.engine.deployer.load_state")
+    def test_full_teardown_emits_retirement_per_decky(
+        self, mock_load, mock_docker, mock_range, mock_compose,
+        mock_td_macvlan, mock_rm_net, mock_clear, mock_emit,
+    ):
+        deckies = [
+            _decky(name="decky-01", ip="192.168.1.10", services=["ssh"]),
+            _decky(name="decky-02", ip="192.168.1.11", services=["http"]),
+        ]
+        mock_load.return_value = (_config(deckies=deckies), Path("test.yml"))
+        from decnet.engine.deployer import teardown
+        teardown()
+        assert mock_emit.call_count == 2
+        for call in mock_emit.call_args_list:
+            assert call.kwargs["trigger"] == "retirement"
+            assert call.kwargs["new_services"] == []
+        assert mock_emit.call_args_list[0].kwargs["old_services"] == ["ssh"]
+        assert mock_emit.call_args_list[1].kwargs["old_services"] == ["http"]
+
+    @patch("decnet.engine.deployer._emit_lifecycle_event")
+    @patch("decnet.engine.deployer._compose")
+    @patch("decnet.engine.deployer.docker.from_env")
+    @patch("decnet.engine.deployer.load_state")
+    def test_single_decky_teardown_emits_one_retirement(
+        self, mock_load, mock_docker, mock_compose, mock_emit,
+    ):
+        deckies = [
+            _decky(name="decky-01", ip="192.168.1.10", services=["ssh", "ftp"]),
+            _decky(name="decky-02", ip="192.168.1.11", services=["http"]),
+        ]
+        mock_load.return_value = (_config(deckies=deckies), Path("test.yml"))
+        from decnet.engine.deployer import teardown
+        teardown(decky_id="decky-01")
+        assert mock_emit.call_count == 1
+        call = mock_emit.call_args_list[0]
+        assert call.kwargs["decky_name"] == "decky-01"
+        assert call.kwargs["trigger"] == "retirement"
+        assert call.kwargs["old_services"] == ["ssh", "ftp"]
+        assert call.kwargs["new_services"] == []
+
+    @patch("decnet.engine.deployer._compose")
+    @patch("decnet.engine.deployer.docker.from_env")
+    @patch("decnet.engine.deployer.load_state")
+    def test_unknown_decky_id_is_noop(
+        self, mock_load, mock_docker, mock_compose,
+    ):
+        mock_load.return_value = (_config(), Path("test.yml"))
+        from decnet.engine.deployer import teardown
+        teardown(decky_id="does-not-exist")
+        mock_compose.assert_not_called()
+
+
+# ── status ────────────────────────────────────────────────────────────────────
+
+class TestStatus:
+    @patch("decnet.engine.deployer.load_state", return_value=None)
+    def test_no_state(self, mock_load):
+        from decnet.engine.deployer import status
+        status()  # should not raise
+
+    @patch("decnet.engine.deployer.docker.from_env")
+    @patch("decnet.engine.deployer.load_state")
+    def test_with_running_containers(self, mock_load, mock_docker):
+        config = _config()
+        mock_load.return_value = (config, Path("test.yml"))
+        mock_container = MagicMock()
+        mock_container.name = "decky-01-ssh"
+        mock_container.status = "running"
+        mock_docker.return_value.containers.list.return_value = [mock_container]
+        from decnet.engine.deployer import status
+        status()  # should not raise
+
+    @patch("decnet.engine.deployer.docker.from_env")
+    @patch("decnet.engine.deployer.load_state")
+    def test_with_absent_containers(self, mock_load, mock_docker):
+        config = _config()
+        mock_load.return_value = (config, Path("test.yml"))
+        mock_docker.return_value.containers.list.return_value = []
+        from decnet.engine.deployer import status
+        status()  # should not raise
+
+
+# ── _print_status ─────────────────────────────────────────────────────────────
+
+class TestPrintStatus:
+    def test_renders_table(self):
+        from decnet.engine.deployer import _print_status
+        config = _config(deckies=[_decky(), _decky("decky-02", "192.168.1.11")])
+        _print_status(config)  # should not raise
+
+
+# ── DB mirror (engine ↔ fleet_deckies) ────────────────────────────────────────
+
+class TestMirrorFleetToDb:
+    """The mirror helpers are best-effort: they replicate fleet state into
+    the ``fleet_deckies`` table so DB-only consumers (orchestrator, web,
+    REST API) see the same view as JSON consumers, but a DB failure must
+    never abort a CLI deploy."""
+
+    def _make_repo(self):
+        repo = MagicMock()
+
+        async def _upsert(data):
+            self.upserts.append(data)
+        async def _delete(*, host_uuid, name):
+            self.deletes.append((host_uuid, name))
+
+        repo.upsert_fleet_decky = MagicMock(side_effect=_upsert)
+        repo.delete_fleet_decky = MagicMock(side_effect=_delete)
+        return repo
+
+    def setup_method(self) -> None:
+        self.upserts: list[dict] = []
+        self.deletes: list[tuple[str, str]] = []
+
+    @patch("decnet.web.db.factory.get_repository")
+    def test_deploy_mirror_upserts_each_decky(self, mock_get_repo):
+        from decnet.engine.deployer import _mirror_fleet_deploy_to_db
+        mock_get_repo.return_value = self._make_repo()
+        cfg = _config(deckies=[
+            _decky(name="d1", ip="10.0.0.1", services=["ssh"]),
+            _decky(name="d2", ip="10.0.0.2", services=["http", "ftp"]),
+        ])
+        _mirror_fleet_deploy_to_db(cfg)
+        assert len(self.upserts) == 2
+        names = sorted(u["name"] for u in self.upserts)
+        assert names == ["d1", "d2"]
+        u1 = next(u for u in self.upserts if u["name"] == "d1")
+        assert u1["host_uuid"] == "local"
+        assert u1["services"] == ["ssh"]
+        assert u1["state"] == "running"
+        assert u1["decky_ip"] == "10.0.0.1"
+        assert u1["decky_config"]["name"] == "d1"
+
+    @patch("decnet.web.db.factory.get_repository")
+    def test_deploy_mirror_honors_explicit_host_uuid(self, mock_get_repo):
+        from decnet.engine.deployer import _mirror_fleet_deploy_to_db
+        mock_get_repo.return_value = self._make_repo()
+        d = _decky()
+        d.host_uuid = "remote-host-abc"
+        _mirror_fleet_deploy_to_db(_config(deckies=[d]))
+        assert self.upserts[0]["host_uuid"] == "remote-host-abc"
+
+    @patch("decnet.web.db.factory.get_repository")
+    def test_deploy_mirror_swallows_db_failure(self, mock_get_repo):
+        from decnet.engine.deployer import _mirror_fleet_deploy_to_db
+        mock_get_repo.side_effect = RuntimeError("db down")
+        _mirror_fleet_deploy_to_db(_config())  # must not raise
+
+    @patch("decnet.web.db.factory.get_repository")
+    def test_teardown_mirror_deletes_each_decky(self, mock_get_repo):
+        from decnet.engine.deployer import _mirror_fleet_teardown_to_db
+        mock_get_repo.return_value = self._make_repo()
+        deckies = [
+            _decky(name="d1", ip="10.0.0.1"),
+            _decky(name="d2", ip="10.0.0.2"),
+        ]
+        _mirror_fleet_teardown_to_db(deckies)
+        assert sorted(self.deletes) == [("local", "d1"), ("local", "d2")]
+
+    @patch("decnet.web.db.factory.get_repository")
+    def test_teardown_mirror_swallows_db_failure(self, mock_get_repo):
+        from decnet.engine.deployer import _mirror_fleet_teardown_to_db
+        mock_get_repo.side_effect = RuntimeError("db down")
+        _mirror_fleet_teardown_to_db([_decky()])  # must not raise
+
+    def test_run_async_works_with_running_loop(self):
+        """``_run_async`` must work even when the caller is already inside
+        an asyncio loop (the API path calls deploy() from a FastAPI handler)."""
+        import asyncio
+        from decnet.engine.deployer import _run_async
+
+        result: list[int] = []
+
+        async def caller() -> None:
+            async def work() -> None:
+                result.append(42)
+            _run_async(work)
+
+        asyncio.run(caller())
+        assert result == [42]

tests/fleet/test_fleet.py

@@ -0,0 +1,191 @@
+"""
+Tests for decnet/fleet.py — fleet builder logic.
+
+Covers build_deckies, build_deckies_from_ini, resolve_distros,
+and edge cases like IP exhaustion and missing services.
+"""
+
+import pytest
+
+from decnet.archetypes import get_archetype
+from decnet.fleet import (
+    build_deckies,
+    build_deckies_from_ini,
+    resolve_distros,
+)
+from decnet.ini_loader import IniConfig, DeckySpec
+
+
+# ── resolve_distros ───────────────────────────────────────────────────────────
+
+class TestResolveDistros:
+    def test_explicit_distros_cycled(self):
+        result = resolve_distros(["debian", "ubuntu22"], False, 5)
+        assert result == ["debian", "ubuntu22", "debian", "ubuntu22", "debian"]
+
+    def test_explicit_single_distro(self):
+        result = resolve_distros(["rocky9"], False, 3)
+        assert result == ["rocky9", "rocky9", "rocky9"]
+
+    def test_randomize_returns_correct_count(self):
+        result = resolve_distros(None, True, 4)
+        assert len(result) == 4
+        # All returned slugs should be valid distro slugs
+        from decnet.distros import all_distros
+        valid = set(all_distros().keys())
+        for slug in result:
+            assert slug in valid
+
+    def test_archetype_preferred_distros(self):
+        arch = get_archetype("deaddeck")
+        result = resolve_distros(None, False, 3, archetype=arch)
+        for slug in result:
+            assert slug in arch.preferred_distros
+
+    def test_fallback_cycles_all_distros(self):
+        result = resolve_distros(None, False, 2)
+        from decnet.distros import all_distros
+        slugs = list(all_distros().keys())
+        assert result[0] == slugs[0]
+        assert result[1] == slugs[1]
+
+
+# ── build_deckies ─────────────────────────────────────────────────────────────
+
+class TestBuildDeckies:
+    _IPS: list[str] = ["192.168.1.10", "192.168.1.11", "192.168.1.12"]
+
+    def test_explicit_services(self):
+        deckies = build_deckies(3, self._IPS, ["ssh", "http"], False)
+        assert len(deckies) == 3
+        for decky in deckies:
+            assert decky.services == ["ssh", "http"]
+
+    def test_archetype_services(self):
+        arch = get_archetype("deaddeck")
+        deckies = build_deckies(2, self._IPS[:2], None, False, archetype=arch)
+        assert len(deckies) == 2
+        for decky in deckies:
+            assert set(decky.services) == set(arch.services)
+            assert decky.archetype == "deaddeck"
+            assert decky.nmap_os == arch.nmap_os
+
+    def test_randomize_services(self):
+        deckies = build_deckies(3, self._IPS, None, True)
+        assert len(deckies) == 3
+        for decky in deckies:
+            assert len(decky.services) >= 1
+
+    def test_no_services_raises(self):
+        with pytest.raises(ValueError, match="Provide services_explicit"):
+            build_deckies(1, self._IPS[:1], None, False)
+
+    def test_names_sequential(self):
+        deckies = build_deckies(3, self._IPS, ["ssh"], False)
+        assert [d.name for d in deckies] == ["decky-01", "decky-02", "decky-03"]
+
+    def test_ips_assigned_correctly(self):
+        deckies = build_deckies(3, self._IPS, ["ssh"], False)
+        assert [d.ip for d in deckies] == self._IPS
+
+    def test_mutate_interval_propagated(self):
+        deckies = build_deckies(1, self._IPS[:1], ["ssh"], False, mutate_interval=15)
+        assert deckies[0].mutate_interval == 15
+
+    def test_distros_explicit(self):
+        deckies = build_deckies(2, self._IPS[:2], ["ssh"], False, distros_explicit=["rocky9"])
+        for decky in deckies:
+            assert decky.distro == "rocky9"
+
+    def test_randomize_distros(self):
+        deckies = build_deckies(2, self._IPS[:2], ["ssh"], False, randomize_distros=True)
+        from decnet.distros import all_distros
+        valid = set(all_distros().keys())
+        for decky in deckies:
+            assert decky.distro in valid
+
+
+# ── build_deckies_from_ini ────────────────────────────────────────────────────
+
+class TestBuildDeckiesFromIni:
+    _SUBNET: str = "192.168.1.0/24"
+    _GATEWAY: str = "192.168.1.1"
+    _HOST_IP: str = "192.168.1.2"
+
+    def _make_ini(self, deckies: list[DeckySpec], **kwargs) -> IniConfig:
+        defaults: dict = {
+            "interface": "eth0",
+            "subnet": None,
+            "gateway": None,
+            "mutate_interval": None,
+            "custom_services": [],
+        }
+        defaults.update(kwargs)
+        return IniConfig(deckies=deckies, **defaults)
+
+    def test_explicit_ip(self):
+        spec = DeckySpec(name="test-1", ip="192.168.1.50", services=["ssh"])
+        ini = self._make_ini([spec])
+        deckies = build_deckies_from_ini(ini, self._SUBNET, self._GATEWAY, self._HOST_IP, False)
+        assert len(deckies) == 1
+        assert deckies[0].ip == "192.168.1.50"
+
+    def test_auto_ip_allocation(self):
+        spec = DeckySpec(name="test-1", services=["ssh"])
+        ini = self._make_ini([spec])
+        deckies = build_deckies_from_ini(ini, self._SUBNET, self._GATEWAY, self._HOST_IP, False)
+        assert len(deckies) == 1
+        assert deckies[0].ip not in (self._GATEWAY, self._HOST_IP, "192.168.1.0", "192.168.1.255")
+
+    def test_archetype_services(self):
+        spec = DeckySpec(name="test-1", archetype="deaddeck")
+        ini = self._make_ini([spec])
+        deckies = build_deckies_from_ini(ini, self._SUBNET, self._GATEWAY, self._HOST_IP, False)
+        arch = get_archetype("deaddeck")
+        assert set(deckies[0].services) == set(arch.services)
+
+    def test_randomize_services(self):
+        spec = DeckySpec(name="test-1")
+        ini = self._make_ini([spec])
+        deckies = build_deckies_from_ini(ini, self._SUBNET, self._GATEWAY, self._HOST_IP, True)
+        assert len(deckies[0].services) >= 1
+
+    def test_no_services_no_arch_auto_randomizes(self):
+        spec = DeckySpec(name="test-1")
+        ini = self._make_ini([spec])
+        deckies = build_deckies_from_ini(ini, self._SUBNET, self._GATEWAY, self._HOST_IP, False)
+        assert len(deckies[0].services) >= 1
+
+    def test_unknown_service_raises(self):
+        spec = DeckySpec(name="test-1", services=["nonexistent_svc_xyz"])
+        ini = self._make_ini([spec])
+        with pytest.raises(ValueError, match="Unknown service"):
+            build_deckies_from_ini(ini, self._SUBNET, self._GATEWAY, self._HOST_IP, False)
+
+    def test_mutate_interval_from_cli(self):
+        spec = DeckySpec(name="test-1", services=["ssh"])
+        ini = self._make_ini([spec])
+        deckies = build_deckies_from_ini(
+            ini, self._SUBNET, self._GATEWAY, self._HOST_IP, False, cli_mutate_interval=42
+        )
+        assert deckies[0].mutate_interval == 42
+
+    def test_mutate_interval_from_ini(self):
+        spec = DeckySpec(name="test-1", services=["ssh"])
+        ini = self._make_ini([spec], mutate_interval=99)
+        deckies = build_deckies_from_ini(
+            ini, self._SUBNET, self._GATEWAY, self._HOST_IP, False, cli_mutate_interval=None
+        )
+        assert deckies[0].mutate_interval == 99
+
+    def test_nmap_os_from_spec(self):
+        spec = DeckySpec(name="test-1", services=["ssh"], nmap_os="windows")
+        ini = self._make_ini([spec])
+        deckies = build_deckies_from_ini(ini, self._SUBNET, self._GATEWAY, self._HOST_IP, False)
+        assert deckies[0].nmap_os == "windows"
+
+    def test_nmap_os_from_archetype(self):
+        spec = DeckySpec(name="test-1", archetype="deaddeck")
+        ini = self._make_ini([spec])
+        deckies = build_deckies_from_ini(ini, self._SUBNET, self._GATEWAY, self._HOST_IP, False)
+        assert deckies[0].nmap_os == "linux"

tests/fleet/test_fleet_singleton.py

@@ -0,0 +1,78 @@
+"""
+Tests for fleet_singleton service behavior.
+
+Verifies that:
+  - The sniffer is registered but marked as fleet_singleton
+  - fleet_singleton services are excluded from compose generation
+  - fleet_singleton services are excluded from random service assignment
+"""
+
+from decnet.composer import generate_compose
+from decnet.fleet import all_service_names, build_deckies
+from decnet.models import DeckyConfig, DecnetConfig
+from decnet.services.registry import all_services, get_service
+
+
+def test_sniffer_is_fleet_singleton():
+    svc = get_service("sniffer")
+    assert svc.fleet_singleton is True
+
+
+def test_non_sniffer_services_are_not_fleet_singleton():
+    for name, svc in all_services().items():
+        if name == "sniffer":
+            continue
+        assert svc.fleet_singleton is False, f"{name} should not be fleet_singleton"
+
+
+def test_sniffer_excluded_from_all_service_names():
+    names = all_service_names()
+    assert "sniffer" not in names
+
+
+def test_sniffer_still_in_registry():
+    """Sniffer must remain discoverable in the registry even though it's a singleton."""
+    registry = all_services()
+    assert "sniffer" in registry
+
+
+def test_compose_skips_fleet_singleton():
+    """When a decky lists 'sniffer' in its services, compose must not generate a container."""
+    config = DecnetConfig(
+        mode="unihost",
+        interface="eth0",
+        subnet="192.168.1.0/24",
+        gateway="192.168.1.1",
+        host_ip="192.168.1.5",
+        deckies=[
+            DeckyConfig(
+                name="decky-01",
+                ip="192.168.1.10",
+                services=["ssh", "sniffer"],
+                distro="debian",
+                base_image="debian:bookworm-slim",
+                hostname="test-host",
+            ),
+        ],
+    )
+    compose = generate_compose(config)
+    services = compose["services"]
+
+    assert "decky-01" in services  # base container exists
+    assert "decky-01-ssh" in services  # ssh service exists
+    assert "decky-01-sniffer" not in services  # sniffer skipped
+
+
+def test_randomize_never_picks_sniffer():
+    """Random service assignment must never include fleet_singleton services."""
+    all_drawn: set[str] = set()
+    for _ in range(100):
+        deckies = build_deckies(
+            n=1,
+            ips=["10.0.0.10"],
+            services_explicit=None,
+            randomize_services=True,
+        )
+        all_drawn.update(deckies[0].services)
+
+    assert "sniffer" not in all_drawn

tests/fleet/test_reconciler.py

@@ -0,0 +1,382 @@
+"""Tests for decnet.fleet.reconciler — pure-function reconcile pass.
+
+Uses a fake repository (in-memory dict) and a stub docker client so the
+suite never touches MySQL/SQLite or a real docker socket.
+"""
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+
+from decnet.config import DeckyConfig, DecnetConfig
+from decnet.fleet.reconciler import (
+    _aggregate_decky_state,
+    reconcile_once,
+)
+
+
+# ── Fakes ─────────────────────────────────────────────────────────────────────
+
+class FakeRepo:
+    """Minimal in-memory stand-in for the fleet portion of BaseRepository."""
+
+    def __init__(self, rows: list[dict[str, Any]] | None = None):
+        self.rows = list(rows or [])
+        self.upserts: list[dict] = []
+        self.deletes: list[tuple[str, str]] = []
+        self.state_updates: list[dict] = []
+
+    async def list_fleet_deckies(self, *, host_uuid: str | None = None):
+        return [
+            r for r in self.rows
+            if host_uuid is None or r.get("host_uuid") == host_uuid
+        ]
+
+    async def upsert_fleet_decky(self, data: dict[str, Any]) -> None:
+        self.upserts.append(data)
+        # Reflect into rows so subsequent calls see it
+        self.rows = [
+            r for r in self.rows
+            if not (r["host_uuid"] == data["host_uuid"] and r["name"] == data["name"])
+        ]
+        self.rows.append(data)
+
+    async def delete_fleet_decky(self, *, host_uuid: str, name: str) -> None:
+        self.deletes.append((host_uuid, name))
+        self.rows = [
+            r for r in self.rows
+            if not (r["host_uuid"] == host_uuid and r["name"] == name)
+        ]
+
+    async def update_fleet_decky_state(
+        self, *, host_uuid: str, name: str, state: str,
+        last_error: str | None = None,
+    ) -> None:
+        self.state_updates.append({
+            "host_uuid": host_uuid, "name": name, "state": state,
+        })
+        for r in self.rows:
+            if r["host_uuid"] == host_uuid and r["name"] == name:
+                r["state"] = state
+
+
+def _decky(name: str = "decky-01", ip: str = "10.0.0.10",
+           services: list[str] | None = None) -> DeckyConfig:
+    return DeckyConfig(
+        name=name, ip=ip, services=services or ["ssh"],
+        distro="debian", base_image="debian", hostname="h",
+        build_base="debian:bookworm-slim", nmap_os="linux",
+    )
+
+
+def _config(deckies: list[DeckyConfig]) -> DecnetConfig:
+    return DecnetConfig(
+        mode="unihost", interface="eth0", subnet="10.0.0.0/24",
+        gateway="10.0.0.1", deckies=deckies, ipvlan=False,
+    )
+
+
+def _state_loader(deckies: list[DeckyConfig] | None):
+    """Return a fake load_state callable."""
+    if deckies is None:
+        return lambda: None
+    return lambda: (_config(deckies), None)
+
+
+def _docker_factory(container_states: dict[str, str]):
+    """Return a docker client factory that yields the given container states.
+
+    The factory's product mimics ``docker.from_env()`` enough that
+    ``_collect_container_states`` can iterate ``client.containers.list(...)``.
+    """
+    containers = [
+        type("C", (), {"name": name, "status": status})()
+        for name, status in container_states.items()
+    ]
+    client = MagicMock()
+    client.containers.list.return_value = containers
+    return lambda: client
+
+
+# ── _aggregate_decky_state ────────────────────────────────────────────────────
+
+class TestAggregate:
+    def test_all_running(self):
+        s = _aggregate_decky_state("d", ["ssh", "http"], {
+            "d-ssh": "running", "d-http": "running",
+        })
+        assert s == "running"
+
+    def test_partial_running_is_degraded(self):
+        s = _aggregate_decky_state("d", ["ssh", "http"], {
+            "d-ssh": "running", "d-http": "exited",
+        })
+        assert s == "degraded"
+
+    def test_one_service_missing_is_degraded(self):
+        s = _aggregate_decky_state("d", ["ssh", "http"], {
+            "d-ssh": "running",  # d-http never started
+        })
+        assert s == "degraded"
+
+    def test_all_dead_is_failed(self):
+        s = _aggregate_decky_state("d", ["ssh"], {"d-ssh": "exited"})
+        assert s == "failed"
+
+    def test_no_containers_is_torn_down(self):
+        assert _aggregate_decky_state("d", ["ssh"], {}) == "torn_down"
+
+    def test_underscore_in_service_name_normalized_to_dash(self):
+        # The deployer creates container "<decky>-<svc>" with underscores
+        # rewritten to dashes (see deployer.status()).  Aggregate must
+        # follow the same convention or it'll never match.
+        s = _aggregate_decky_state("d", ["smtp_relay"], {
+            "d-smtp-relay": "running",
+        })
+        assert s == "running"
+
+
+# ── reconcile_once ────────────────────────────────────────────────────────────
+
+@pytest.mark.anyio
+@pytest.fixture
+def anyio_backend():
+    return "asyncio"
+
+
+class TestReconcileOnce:
+    @pytest.mark.anyio
+    async def test_inserts_when_json_has_decky_db_does_not(self):
+        repo = FakeRepo()  # DB empty
+        d = _decky(name="solo", ip="10.0.0.5", services=["ssh"])
+        counts = await reconcile_once(
+            repo,
+            load_state_fn=_state_loader([d]),
+            docker_client_factory=_docker_factory({"solo-ssh": "running"}),
+        )
+        assert counts == {"inserted": 1, "deleted": 0, "state_updated": 0}
+        assert len(repo.upserts) == 1
+        u = repo.upserts[0]
+        assert u["host_uuid"] == "local"
+        assert u["name"] == "solo"
+        assert u["services"] == ["ssh"]
+        assert u["decky_ip"] == "10.0.0.5"
+        assert u["state"] == "running"
+
+    @pytest.mark.anyio
+    async def test_deletes_when_db_has_decky_json_does_not(self):
+        repo = FakeRepo([
+            {"host_uuid": "local", "name": "ghost", "services": ["ssh"],
+             "state": "running", "decky_ip": "10.0.0.99"},
+        ])
+        counts = await reconcile_once(
+            repo,
+            load_state_fn=lambda: None,  # no JSON state
+            docker_client_factory=_docker_factory({}),
+        )
+        assert counts == {"inserted": 0, "deleted": 1, "state_updated": 0}
+        assert repo.deletes == [("local", "ghost")]
+
+    @pytest.mark.anyio
+    async def test_updates_state_when_docker_disagrees(self):
+        repo = FakeRepo([
+            {"host_uuid": "local", "name": "d1", "services": ["ssh"],
+             "state": "running", "decky_ip": "10.0.0.10"},
+        ])
+        d = _decky(name="d1", services=["ssh"])
+        counts = await reconcile_once(
+            repo,
+            load_state_fn=_state_loader([d]),
+            docker_client_factory=_docker_factory({"d1-ssh": "exited"}),
+        )
+        assert counts == {"inserted": 0, "deleted": 0, "state_updated": 1}
+        assert repo.state_updates[0]["state"] == "failed"
+
+    @pytest.mark.anyio
+    async def test_no_writes_when_already_converged(self):
+        repo = FakeRepo([
+            {"host_uuid": "local", "name": "d1", "services": ["ssh"],
+             "state": "running", "decky_ip": "10.0.0.10"},
+        ])
+        d = _decky(name="d1", services=["ssh"])
+        counts = await reconcile_once(
+            repo,
+            load_state_fn=_state_loader([d]),
+            docker_client_factory=_docker_factory({"d1-ssh": "running"}),
+        )
+        assert counts == {"inserted": 0, "deleted": 0, "state_updated": 0}
+        assert repo.upserts == [] and repo.deletes == []
+        assert repo.state_updates == []
+
+    @pytest.mark.anyio
+    async def test_skips_state_updates_when_docker_unreachable(self):
+        """Docker socket failure must not torch every row to torn_down —
+        the reconciler returns ``None`` from _collect_container_states and
+        leaves existing DB state alone."""
+        repo = FakeRepo([
+            {"host_uuid": "local", "name": "d1", "services": ["ssh"],
+             "state": "running", "decky_ip": "10.0.0.10"},
+        ])
+        d = _decky(name="d1", services=["ssh"])
+
+        def broken_factory():
+            raise RuntimeError("docker socket unreachable")
+
+        counts = await reconcile_once(
+            repo,
+            load_state_fn=_state_loader([d]),
+            docker_client_factory=broken_factory,
+        )
+        assert counts == {"inserted": 0, "deleted": 0, "state_updated": 0}
+        assert repo.state_updates == []
+
+    @pytest.mark.anyio
+    async def test_host_uuid_scoping_protects_peer_rows(self):
+        """A reconcile on host A must NOT delete rows belonging to host B."""
+        repo = FakeRepo([
+            {"host_uuid": "host-a", "name": "d1", "services": ["ssh"],
+             "state": "running", "decky_ip": "10.0.0.10"},
+            {"host_uuid": "host-b", "name": "d2", "services": ["ssh"],
+             "state": "running", "decky_ip": "10.0.1.10"},
+        ])
+        # Reconciling on host-a with no JSON state
+        counts = await reconcile_once(
+            repo,
+            host_uuid="host-a",
+            load_state_fn=lambda: None,
+            docker_client_factory=_docker_factory({}),
+        )
+        assert counts["deleted"] == 1
+        # Only host-a's row was touched
+        assert repo.deletes == [("host-a", "d1")]
+        # host-b's row survives
+        assert any(r["host_uuid"] == "host-b" for r in repo.rows)
+
+    @pytest.mark.anyio
+    async def test_publishes_decky_state_on_transitions(self):
+        """When *bus* is provided, every insert/delete/state-change must
+        publish on ``decky.<host_uuid:name>.state``."""
+        from decnet.bus.fake import FakeBus
+        bus = FakeBus()
+        await bus.connect()
+
+        published: list = []
+
+        async def collect():
+            async with bus.subscribe("decky.>") as sub:
+                async for ev in sub:
+                    published.append(ev)
+                    if len(published) >= 3:
+                        return
+
+        try:
+            collector = asyncio.create_task(collect())
+            await asyncio.sleep(0)  # let subscription register
+
+            repo = FakeRepo([
+                # An existing row that will be deleted (not in JSON).
+                {"host_uuid": "local", "name": "ghost", "services": ["ssh"],
+                 "state": "running", "decky_ip": "10.0.0.99"},
+                # An existing row whose state will flip running → failed.
+                {"host_uuid": "local", "name": "d-flip", "services": ["ssh"],
+                 "state": "running", "decky_ip": "10.0.0.20"},
+            ])
+            json_deckies = [
+                _decky(name="d-new", ip="10.0.0.30", services=["http"]),
+                _decky(name="d-flip", ip="10.0.0.20", services=["ssh"]),
+            ]
+            await reconcile_once(
+                repo,
+                load_state_fn=_state_loader(json_deckies),
+                docker_client_factory=_docker_factory({
+                    "d-new-http": "running",
+                    "d-flip-ssh": "exited",
+                }),
+                bus=bus,
+            )
+            await asyncio.wait_for(collector, timeout=2.0)
+        finally:
+            await bus.close()
+
+        topics = sorted(ev.topic for ev in published)
+        assert topics == [
+            "decky.local:d-flip.state",
+            "decky.local:d-new.state",
+            "decky.local:ghost.state",
+        ]
+        by_name = {ev.payload["name"]: ev.payload for ev in published}
+        assert by_name["d-new"]["transition"] == "inserted"
+        assert by_name["d-new"]["state"] == "running"
+        assert by_name["ghost"]["transition"] == "deleted"
+        assert by_name["ghost"]["state"] == "torn_down"
+        assert by_name["d-flip"]["transition"] == "state_changed"
+        assert by_name["d-flip"]["state"] == "failed"
+        assert by_name["d-flip"]["previous_state"] == "running"
+
+    @pytest.mark.anyio
+    async def test_no_bus_publish_when_already_converged(self):
+        """Quiet ticks must not publish — otherwise every 30s the bus
+        floods with no-op events."""
+        from decnet.bus.fake import FakeBus
+        bus = FakeBus()
+        await bus.connect()
+        try:
+            published: list = []
+
+            async def collect():
+                async with bus.subscribe("decky.>") as sub:
+                    async for ev in sub:
+                        published.append(ev)
+
+            collector = asyncio.create_task(collect())
+            await asyncio.sleep(0)
+
+            repo = FakeRepo([
+                {"host_uuid": "local", "name": "d1", "services": ["ssh"],
+                 "state": "running", "decky_ip": "10.0.0.10"},
+            ])
+            d = _decky(name="d1", services=["ssh"])
+            await reconcile_once(
+                repo,
+                load_state_fn=_state_loader([d]),
+                docker_client_factory=_docker_factory({"d1-ssh": "running"}),
+                bus=bus,
+            )
+            await asyncio.sleep(0.1)  # give the bus a window to deliver
+            collector.cancel()
+        finally:
+            await bus.close()
+
+        assert published == []
+
+    @pytest.mark.anyio
+    async def test_combined_drift_in_one_pass(self):
+        """JSON has new decky AND DB has stale decky AND third decky's
+        container died — all three converge in a single tick."""
+        repo = FakeRepo([
+            {"host_uuid": "local", "name": "stale", "services": ["ssh"],
+             "state": "running", "decky_ip": "10.0.0.99"},
+            {"host_uuid": "local", "name": "d-existing", "services": ["ssh"],
+             "state": "running", "decky_ip": "10.0.0.20"},
+        ])
+        json_deckies = [
+            _decky(name="d-new", ip="10.0.0.30", services=["http"]),
+            _decky(name="d-existing", ip="10.0.0.20", services=["ssh"]),
+        ]
+        counts = await reconcile_once(
+            repo,
+            load_state_fn=_state_loader(json_deckies),
+            docker_client_factory=_docker_factory({
+                "d-new-http": "running",
+                "d-existing-ssh": "exited",  # crashed
+            }),
+        )
+        assert counts == {"inserted": 1, "deleted": 1, "state_updated": 1}
+        names_inserted = [u["name"] for u in repo.upserts]
+        assert "d-new" in names_inserted
+        assert ("local", "stale") in repo.deletes
+        assert any(s["name"] == "d-existing" and s["state"] == "failed"
+                   for s in repo.state_updates)

tests/fleet/test_reconciler_worker.py

@@ -0,0 +1,72 @@
+"""Worker shutdown smoke test for fleet_reconciler_worker.
+
+The reconcile logic itself is exercised in test_reconciler.py.  This file
+just verifies the worker's lifecycle wrapper (control listener + heartbeat
++ tick loop) exits cleanly when the bus shutdown signal fires.
+"""
+from __future__ import annotations
+
+import asyncio
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from decnet.fleet.reconciler_worker import fleet_reconciler_worker
+
+
+class _FakeRepo:
+    async def list_fleet_deckies(self, *, host_uuid=None):
+        return []
+    async def upsert_fleet_decky(self, data): pass
+    async def delete_fleet_decky(self, **kw): pass
+    async def update_fleet_decky_state(self, **kw): pass
+
+
+@pytest.mark.anyio
+async def test_worker_exits_on_shutdown_event(monkeypatch):
+    # Patch the bus + control listener so the worker doesn't try to bind
+    # to a real socket. The control_task will set `shutdown` once we fire it.
+    fake_bus = AsyncMock()
+    monkeypatch.setattr(
+        "decnet.fleet.reconciler_worker.get_bus",
+        lambda **kw: fake_bus,
+    )
+
+    captured: dict = {}
+
+    async def _capturing_control_listener(bus, name, shutdown_event):
+        captured["shutdown_event"] = shutdown_event
+        # Hold the event loop briefly so the worker enters its tick wait,
+        # then trigger shutdown.
+        await asyncio.sleep(0.05)
+        shutdown_event.set()
+
+    async def _noop_heartbeat(bus, name):
+        await asyncio.sleep(3600)  # never returns naturally
+
+    monkeypatch.setattr(
+        "decnet.fleet.reconciler_worker.run_control_listener",
+        _capturing_control_listener,
+    )
+    monkeypatch.setattr(
+        "decnet.fleet.reconciler_worker.run_health_heartbeat",
+        _noop_heartbeat,
+    )
+    # Skip docker observation entirely — we just need the loop to exit.
+    monkeypatch.setattr(
+        "decnet.fleet.reconciler._real_load_state",
+        lambda: None,
+    )
+    with patch("decnet.fleet.reconciler._collect_container_states",
+               return_value=None):
+        # interval=10 (long) so we exit via shutdown, not via tick completion
+        await asyncio.wait_for(
+            fleet_reconciler_worker(_FakeRepo(), interval=10),
+            timeout=2.0,
+        )
+    assert captured["shutdown_event"].is_set()
+
+
+@pytest.fixture
+def anyio_backend():
+    return "asyncio"