diff --git a/decnet/cli.py b/decnet/cli.py index 3bcf73e..d3dc760 100644 --- a/decnet/cli.py +++ b/decnet/cli.py @@ -8,7 +8,7 @@ Usage: decnet services """ -import random +import signal from typing import Optional import typer @@ -28,7 +28,8 @@ from decnet.config import ( DecnetConfig, random_hostname, ) -from decnet.distros import all_distros, get_distro, random_distro +from decnet.distros import all_distros, get_distro +from decnet.fleet import all_service_names, build_deckies, build_deckies_from_ini from decnet.ini_loader import IniConfig, load_ini from decnet.network import detect_interface, detect_subnet, allocate_ips, get_host_ip from decnet.services.registry import all_services @@ -40,171 +41,31 @@ app = typer.Typer( ) console = Console() -def _all_service_names() -> list[str]: - """Return all registered service names from the live plugin registry.""" - return sorted(all_services().keys()) +def _kill_api() -> None: + """Find and kill any running DECNET API (uvicorn) or mutator processes.""" + import psutil + import os -def _resolve_distros( - distros_explicit: list[str] | None, - randomize_distros: bool, - n: int, - archetype: Archetype | None = None, -) -> list[str]: - """Return a list of n distro slugs based on CLI flags or archetype preference.""" - if distros_explicit: - return [distros_explicit[i % len(distros_explicit)] for i in range(n)] - if randomize_distros: - return [random_distro().slug for _ in range(n)] - if archetype: - pool = archetype.preferred_distros - return [pool[i % len(pool)] for i in range(n)] - # Default: cycle through all distros to maximize heterogeneity - slugs = list(all_distros().keys()) - return [slugs[i % len(slugs)] for i in range(n)] - - -def _build_deckies( - n: int, - ips: list[str], - services_explicit: list[str] | None, - randomize_services: bool, - distros_explicit: list[str] | None = None, - randomize_distros: bool = False, - archetype: Archetype | None = None, -) -> list[DeckyConfig]: - deckies = [] - used_combos: set[frozenset] = set() - distro_slugs = _resolve_distros(distros_explicit, randomize_distros, n, archetype) - - for i, ip in enumerate(ips): - name = f"decky-{i + 1:02d}" - distro = get_distro(distro_slugs[i]) - hostname = random_hostname(distro.slug) - - if services_explicit: - svc_list = services_explicit - elif archetype: - svc_list = list(archetype.services) - elif randomize_services: - svc_pool = _all_service_names() - attempts = 0 - while True: - count = random.randint(1, min(3, len(svc_pool))) # nosec B311 - chosen = frozenset(random.sample(svc_pool, count)) # nosec B311 - attempts += 1 - if chosen not in used_combos or attempts > 20: - break - svc_list = list(chosen) - used_combos.add(chosen) - else: - typer.echo("Error: provide --services, --archetype, or --randomize-services.", err=True) - raise typer.Exit(1) - - deckies.append( - DeckyConfig( - name=name, - ip=ip, - services=svc_list, - distro=distro.slug, - base_image=distro.image, - build_base=distro.build_base, - hostname=hostname, - archetype=archetype.slug if archetype else None, - nmap_os=archetype.nmap_os if archetype else "linux", - ) - ) - return deckies - - -def _build_deckies_from_ini( - ini: IniConfig, - subnet_cidr: str, - gateway: str, - host_ip: str, - randomize: bool, - cli_mutate_interval: int | None = None, -) -> list[DeckyConfig]: - """Build DeckyConfig list from an IniConfig, auto-allocating missing IPs.""" - from ipaddress import IPv4Address, IPv4Network - import time - now = time.time() - - explicit_ips: set[IPv4Address] = { - IPv4Address(s.ip) for s in ini.deckies if s.ip - } - - net = IPv4Network(subnet_cidr, strict=False) - reserved = { - net.network_address, - net.broadcast_address, - IPv4Address(gateway), - IPv4Address(host_ip), - } | explicit_ips - - auto_pool = (str(addr) for addr in net.hosts() if addr not in reserved) - - deckies: list[DeckyConfig] = [] - for spec in ini.deckies: - # Resolve archetype (if any) — explicit services/distro override it - arch: Archetype | None = None - if spec.archetype: - arch = get_archetype(spec.archetype) - - # Distro: archetype preferred list → random → global cycle - distro_pool = arch.preferred_distros if arch else list(all_distros().keys()) - distro = get_distro(distro_pool[len(deckies) % len(distro_pool)]) - hostname = random_hostname(distro.slug) - - ip = spec.ip or next(auto_pool, None) - if ip is None: - raise ValueError(f"Not enough free IPs in {subnet_cidr} while assigning IP for '{spec.name}'.") - - if spec.services: - known = set(_all_service_names()) - unknown = [s for s in spec.services if s not in known] - if unknown: - raise ValueError( - f"Unknown service(s) in [{spec.name}]: {unknown}. " - f"Available: {_all_service_names()}" - ) - svc_list = spec.services - elif arch: - svc_list = list(arch.services) - elif randomize: - svc_pool = _all_service_names() - count = random.randint(1, min(3, len(svc_pool))) # nosec B311 - svc_list = random.sample(svc_pool, count) # nosec B311 - else: - raise ValueError( - f"Decky '[{spec.name}]' has no services= in config. " - "Add services=, archetype=, or use --randomize-services." - ) - - # nmap_os priority: explicit INI key > archetype default > "linux" - resolved_nmap_os = spec.nmap_os or (arch.nmap_os if arch else "linux") - - # mutation interval priority: CLI > per-decky INI > global INI - decky_mutate_interval = cli_mutate_interval - if decky_mutate_interval is None: - decky_mutate_interval = spec.mutate_interval if spec.mutate_interval is not None else ini.mutate_interval - - deckies.append(DeckyConfig( - name=spec.name, - ip=ip, - services=svc_list, - distro=distro.slug, - base_image=distro.image, - build_base=distro.build_base, - hostname=hostname, - archetype=arch.slug if arch else None, - service_config=spec.service_config, - nmap_os=resolved_nmap_os, - mutate_interval=decky_mutate_interval, - last_mutated=now, - )) - return deckies + _killed: bool = False + for _proc in psutil.process_iter(['pid', 'name', 'cmdline']): + try: + _cmd = _proc.info['cmdline'] + if not _cmd: + continue + if "uvicorn" in _cmd and "decnet.web.api:app" in _cmd: + console.print(f"[yellow]Stopping DECNET API (PID {_proc.info['pid']})...[/]") + os.kill(_proc.info['pid'], signal.SIGTERM) + _killed = True + elif "decnet.cli" in _cmd and "mutate" in _cmd and "--watch" in _cmd: + console.print(f"[yellow]Stopping DECNET Mutator Watcher (PID {_proc.info['pid']})...[/]") + os.kill(_proc.info['pid'], signal.SIGTERM) + _killed = True + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + if _killed: + console.print("[green]Background processes stopped.[/]") @app.command() @@ -270,7 +131,6 @@ def deploy( console.print(f"[red]{e}[/]") raise typer.Exit(1) - # CLI flags override INI values when explicitly provided iface = interface or ini.interface or detect_interface() subnet_cidr = subnet or ini.subnet effective_gateway = ini.gateway @@ -284,7 +144,6 @@ def deploy( f"[dim]Subnet:[/] {subnet_cidr} [dim]Gateway:[/] {effective_gateway} " f"[dim]Host IP:[/] {host_ip}") - # Register bring-your-own services from INI before validation if ini.custom_services: from decnet.custom_service import CustomService from decnet.services.registry import register_custom_service @@ -300,7 +159,7 @@ def deploy( effective_log_file = log_file try: - decky_configs = _build_deckies_from_ini( + decky_configs = build_deckies_from_ini( ini, subnet_cidr, effective_gateway, host_ip, randomize_services, cli_mutate_interval=mutate_interval ) except ValueError as e: @@ -316,13 +175,12 @@ def deploy( services_list = [s.strip() for s in services.split(",")] if services else None if services_list: - known = set(_all_service_names()) + known = set(all_service_names()) unknown = [s for s in services_list if s not in known] if unknown: - console.print(f"[red]Unknown service(s): {unknown}. Available: {_all_service_names()}[/]") + console.print(f"[red]Unknown service(s): {unknown}. Available: {all_service_names()}[/]") raise typer.Exit(1) - # Resolve archetype if provided arch: Archetype | None = None if archetype_name: try: @@ -356,14 +214,13 @@ def deploy( raise typer.Exit(1) ips = allocate_ips(subnet_cidr, effective_gateway, host_ip, deckies, ip_start) - decky_configs = _build_deckies( + decky_configs = build_deckies( deckies, ips, services_list, randomize_services, distros_explicit=distros_list, randomize_distros=randomize_distros, archetype=arch, mutate_interval=mutate_interval, ) effective_log_file = log_file - # Handle automatic log file for API if api and not effective_log_file: effective_log_file = os.path.join(os.getcwd(), "decnet.log") console.print(f"[cyan]API mode enabled: defaulting log-file to {effective_log_file}[/]") @@ -379,9 +236,9 @@ def deploy( mutate_interval=mutate_interval, ) - from decnet.deployer import deploy as _deploy + from decnet.engine import deploy as _deploy _deploy(config, dry_run=dry_run, no_cache=no_cache, parallel=parallel) - + if mutate_interval is not None and not dry_run: import subprocess # nosec B404 import sys @@ -396,8 +253,6 @@ def deploy( except (FileNotFoundError, subprocess.SubprocessError): console.print("[red]Failed to start mutator watcher.[/]") - # Start the log collector as a background process unless --api is handling it. - # The collector streams Docker logs → log_file (RFC 5424) + log_file.json. if effective_log_file and not dry_run and not api: import subprocess # noqa: F811 # nosec B404 import sys @@ -436,7 +291,7 @@ def collect( ) -> None: """Stream Docker logs from all running decky service containers to a log file.""" import asyncio - from decnet.web.collector import log_collector_worker + from decnet.collector import log_collector_worker console.print(f"[bold cyan]Collector starting[/] → {log_file}") asyncio.run(log_collector_worker(log_file)) @@ -465,7 +320,7 @@ def mutate( @app.command() def status() -> None: """Show running deckies and their status.""" - from decnet.deployer import status as _status + from decnet.engine import status as _status _status() @@ -479,9 +334,12 @@ def teardown( console.print("[red]Specify --all or --id .[/]") raise typer.Exit(1) - from decnet.deployer import teardown as _teardown + from decnet.engine import teardown as _teardown _teardown(decky_id=id_) + if all_: + _kill_api() + @app.command(name="services") def list_services() -> None: @@ -591,7 +449,6 @@ def serve_web( import socketserver from pathlib import Path - # Assuming decnet_web/dist is relative to the project root dist_dir = Path(__file__).parent.parent / "decnet_web" / "dist" if not dist_dir.exists(): @@ -600,10 +457,8 @@ def serve_web( class SPAHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): def do_GET(self): - # Try to serve the requested file path = self.translate_path(self.path) if not Path(path).exists() or Path(path).is_dir(): - # If not found or is a directory, serve index.html (for React Router) self.path = "/index.html" return super().do_GET() diff --git a/decnet/collector/__init__.py b/decnet/collector/__init__.py new file mode 100644 index 0000000..5dcaf34 --- /dev/null +++ b/decnet/collector/__init__.py @@ -0,0 +1,13 @@ +from decnet.collector.worker import ( + is_service_container, + is_service_event, + log_collector_worker, + parse_rfc5424, +) + +__all__ = [ + "is_service_container", + "is_service_event", + "log_collector_worker", + "parse_rfc5424", +] diff --git a/decnet/web/collector.py b/decnet/collector/worker.py similarity index 98% rename from decnet/web/collector.py rename to decnet/collector/worker.py index 1eb5119..69e2c6b 100644 --- a/decnet/web/collector.py +++ b/decnet/collector/worker.py @@ -14,7 +14,7 @@ from datetime import datetime from pathlib import Path from typing import Any, Optional -logger = logging.getLogger("decnet.web.collector") +logger = logging.getLogger("decnet.collector") # ─── RFC 5424 parser ────────────────────────────────────────────────────────── @@ -175,12 +175,10 @@ async def log_collector_worker(log_file: str) -> None: try: client = docker.from_env() - # Collect from already-running containers for container in client.containers.list(): if is_service_container(container): _spawn(container.id, container.name.lstrip("/")) - # Watch for new containers starting def _watch_events() -> None: for event in client.events( decode=True, diff --git a/decnet/engine/__init__.py b/decnet/engine/__init__.py new file mode 100644 index 0000000..f2edfb1 --- /dev/null +++ b/decnet/engine/__init__.py @@ -0,0 +1,15 @@ +from decnet.engine.deployer import ( + COMPOSE_FILE, + _compose_with_retry, + deploy, + status, + teardown, +) + +__all__ = [ + "COMPOSE_FILE", + "_compose_with_retry", + "deploy", + "status", + "teardown", +] diff --git a/decnet/deployer.py b/decnet/engine/deployer.py similarity index 82% rename from decnet/deployer.py rename to decnet/engine/deployer.py index 92e2fb6..3f03c63 100644 --- a/decnet/deployer.py +++ b/decnet/engine/deployer.py @@ -28,7 +28,7 @@ from decnet.network import ( console = Console() COMPOSE_FILE = Path("decnet-compose.yml") -_CANONICAL_LOGGING = Path(__file__).parent.parent / "templates" / "decnet_logging.py" +_CANONICAL_LOGGING = Path(__file__).parent.parent.parent / "templates" / "decnet_logging.py" def _sync_logging_helper(config: DecnetConfig) -> None: @@ -108,7 +108,6 @@ def _compose_with_retry( def deploy(config: DecnetConfig, dry_run: bool = False, no_cache: bool = False, parallel: bool = False) -> None: client = docker.from_env() - # --- Network setup --- ip_list = [d.ip for d in config.deckies] decky_range = ips_to_range(ip_list) host_ip = get_host_ip(config.interface) @@ -135,10 +134,8 @@ def deploy(config: DecnetConfig, dry_run: bool = False, no_cache: bool = False, ) setup_host_macvlan(config.interface, host_ip, decky_range) - # --- Sync shared logging helper into each template build context --- _sync_logging_helper(config) - # --- Compose generation --- compose_path = write_compose(config, COMPOSE_FILE) console.print(f"[bold cyan]Compose file written[/] → {compose_path}") @@ -146,13 +143,8 @@ def deploy(config: DecnetConfig, dry_run: bool = False, no_cache: bool = False, console.print("[yellow]Dry run — no containers started.[/]") return - # --- Save state before bring-up --- save_state(config, compose_path) - # --- Bring up --- - # With --parallel: force BuildKit, run build explicitly (so all images are - # built concurrently before any container starts), then up without --build. - # Without --parallel: keep the original up --build path. build_env = {"DOCKER_BUILDKIT": "1"} if parallel else {} console.print("[bold cyan]Building images and starting deckies...[/]") @@ -169,37 +161,9 @@ def deploy(config: DecnetConfig, dry_run: bool = False, no_cache: bool = False, _compose_with_retry("build", "--no-cache", compose_file=compose_path) _compose_with_retry("up", "--build", "-d", compose_file=compose_path) - # --- Status summary --- _print_status(config) -def _kill_api() -> None: - """Find and kill any running DECNET API (uvicorn) or mutator processes.""" - import psutil - import signal - import os - - _killed: bool = False - for _proc in psutil.process_iter(['pid', 'name', 'cmdline']): - try: - _cmd = _proc.info['cmdline'] - if not _cmd: - continue - if "uvicorn" in _cmd and "decnet.web.api:app" in _cmd: - console.print(f"[yellow]Stopping DECNET API (PID {_proc.info['pid']})...[/]") - os.kill(_proc.info['pid'], signal.SIGTERM) - _killed = True - elif "decnet.cli" in _cmd and "mutate" in _cmd and "--watch" in _cmd: - console.print(f"[yellow]Stopping DECNET Mutator Watcher (PID {_proc.info['pid']})...[/]") - os.kill(_proc.info['pid'], signal.SIGTERM) - _killed = True - except (psutil.NoSuchProcess, psutil.AccessDenied): - continue - - if _killed: - console.print("[green]Background processes stopped.[/]") - - def teardown(decky_id: str | None = None) -> None: state = load_state() if state is None: @@ -210,7 +174,6 @@ def teardown(decky_id: str | None = None) -> None: client = docker.from_env() if decky_id: - # Bring down only the services matching this decky svc_names = [f"{decky_id}-{svc}" for svc in [d.services for d in config.deckies if d.name == decky_id]] if not svc_names: console.print(f"[red]Decky '{decky_id}' not found in current deployment.[/]") @@ -228,10 +191,7 @@ def teardown(decky_id: str | None = None) -> None: teardown_host_macvlan(decky_range) remove_macvlan_network(client) clear_state() - - # Kill API when doing full teardown - _kill_api() - + net_driver = "IPvlan" if config.ipvlan else "MACVLAN" console.print(f"[green]All deckies torn down. {net_driver} network removed.[/]") diff --git a/decnet/fleet.py b/decnet/fleet.py new file mode 100644 index 0000000..cd9984e --- /dev/null +++ b/decnet/fleet.py @@ -0,0 +1,179 @@ +""" +Fleet builder — shared logic for constructing DeckyConfig lists. + +Used by both the CLI and the web API router to build deckies from +flags or INI config. Lives here (not in cli.py) so that the web layer +and the mutation engine can import it without depending on the CLI. +""" + +import random +from typing import Optional + +from decnet.archetypes import Archetype, get_archetype +from decnet.config import DeckyConfig, random_hostname +from decnet.distros import all_distros, get_distro, random_distro +from decnet.ini_loader import IniConfig +from decnet.services.registry import all_services + + +def all_service_names() -> list[str]: + """Return all registered service names from the live plugin registry.""" + return sorted(all_services().keys()) + + +def resolve_distros( + distros_explicit: list[str] | None, + randomize_distros: bool, + n: int, + archetype: Archetype | None = None, +) -> list[str]: + """Return a list of n distro slugs based on flags or archetype preference.""" + if distros_explicit: + return [distros_explicit[i % len(distros_explicit)] for i in range(n)] + if randomize_distros: + return [random_distro().slug for _ in range(n)] + if archetype: + pool = archetype.preferred_distros + return [pool[i % len(pool)] for i in range(n)] + slugs = list(all_distros().keys()) + return [slugs[i % len(slugs)] for i in range(n)] + + +def build_deckies( + n: int, + ips: list[str], + services_explicit: list[str] | None, + randomize_services: bool, + distros_explicit: list[str] | None = None, + randomize_distros: bool = False, + archetype: Archetype | None = None, + mutate_interval: Optional[int] = None, +) -> list[DeckyConfig]: + """Build a list of DeckyConfigs from CLI-style flags.""" + deckies = [] + used_combos: set[frozenset] = set() + distro_slugs = resolve_distros(distros_explicit, randomize_distros, n, archetype) + + for i, ip in enumerate(ips): + name = f"decky-{i + 1:02d}" + distro = get_distro(distro_slugs[i]) + hostname = random_hostname(distro.slug) + + if services_explicit: + svc_list = services_explicit + elif archetype: + svc_list = list(archetype.services) + elif randomize_services: + svc_pool = all_service_names() + attempts = 0 + while True: + count = random.randint(1, min(3, len(svc_pool))) # nosec B311 + chosen = frozenset(random.sample(svc_pool, count)) # nosec B311 + attempts += 1 + if chosen not in used_combos or attempts > 20: + break + svc_list = list(chosen) + used_combos.add(chosen) + else: + raise ValueError("Provide services_explicit, archetype, or randomize_services=True.") + + deckies.append( + DeckyConfig( + name=name, + ip=ip, + services=svc_list, + distro=distro.slug, + base_image=distro.image, + build_base=distro.build_base, + hostname=hostname, + archetype=archetype.slug if archetype else None, + nmap_os=archetype.nmap_os if archetype else "linux", + mutate_interval=mutate_interval, + ) + ) + return deckies + + +def build_deckies_from_ini( + ini: IniConfig, + subnet_cidr: str, + gateway: str, + host_ip: str, + randomize: bool, + cli_mutate_interval: int | None = None, +) -> list[DeckyConfig]: + """Build DeckyConfig list from an IniConfig, auto-allocating missing IPs.""" + from ipaddress import IPv4Address, IPv4Network + import time + now = time.time() + + explicit_ips: set[IPv4Address] = { + IPv4Address(s.ip) for s in ini.deckies if s.ip + } + + net = IPv4Network(subnet_cidr, strict=False) + reserved = { + net.network_address, + net.broadcast_address, + IPv4Address(gateway), + IPv4Address(host_ip), + } | explicit_ips + + auto_pool = (str(addr) for addr in net.hosts() if addr not in reserved) + + deckies: list[DeckyConfig] = [] + for spec in ini.deckies: + arch: Archetype | None = None + if spec.archetype: + arch = get_archetype(spec.archetype) + + distro_pool = arch.preferred_distros if arch else list(all_distros().keys()) + distro = get_distro(distro_pool[len(deckies) % len(distro_pool)]) + hostname = random_hostname(distro.slug) + + ip = spec.ip or next(auto_pool, None) + if ip is None: + raise ValueError(f"Not enough free IPs in {subnet_cidr} while assigning IP for '{spec.name}'.") + + if spec.services: + known = set(all_service_names()) + unknown = [s for s in spec.services if s not in known] + if unknown: + raise ValueError( + f"Unknown service(s) in [{spec.name}]: {unknown}. " + f"Available: {all_service_names()}" + ) + svc_list = spec.services + elif arch: + svc_list = list(arch.services) + elif randomize: + svc_pool = all_service_names() + count = random.randint(1, min(3, len(svc_pool))) # nosec B311 + svc_list = random.sample(svc_pool, count) # nosec B311 + else: + raise ValueError( + f"Decky '[{spec.name}]' has no services= in config. " + "Add services=, archetype=, or use --randomize-services." + ) + + resolved_nmap_os = spec.nmap_os or (arch.nmap_os if arch else "linux") + + decky_mutate_interval = cli_mutate_interval + if decky_mutate_interval is None: + decky_mutate_interval = spec.mutate_interval if spec.mutate_interval is not None else ini.mutate_interval + + deckies.append(DeckyConfig( + name=spec.name, + ip=ip, + services=svc_list, + distro=distro.slug, + base_image=distro.image, + build_base=distro.build_base, + hostname=hostname, + archetype=arch.slug if arch else None, + service_config=spec.service_config, + nmap_os=resolved_nmap_os, + mutate_interval=decky_mutate_interval, + last_mutated=now, + )) + return deckies diff --git a/decnet/mutator/__init__.py b/decnet/mutator/__init__.py new file mode 100644 index 0000000..41d5792 --- /dev/null +++ b/decnet/mutator/__init__.py @@ -0,0 +1,3 @@ +from decnet.mutator.engine import mutate_all, mutate_decky, run_watch_loop + +__all__ = ["mutate_all", "mutate_decky", "run_watch_loop"] diff --git a/decnet/mutator.py b/decnet/mutator/engine.py similarity index 68% rename from decnet/mutator.py rename to decnet/mutator/engine.py index 9cf857f..fbd3096 100644 --- a/decnet/mutator.py +++ b/decnet/mutator/engine.py @@ -4,43 +4,21 @@ Handles dynamic rotation of exposed honeypot services over time. """ import random -import subprocess # nosec B404 import time -from pathlib import Path from typing import Optional from rich.console import Console from decnet.archetypes import get_archetype -from decnet.cli import _all_service_names +from decnet.fleet import all_service_names from decnet.composer import write_compose from decnet.config import DeckyConfig, load_state, save_state -from decnet.deployer import COMPOSE_FILE +from decnet.engine import COMPOSE_FILE, _compose_with_retry + +import subprocess # nosec B404 console = Console() -def _compose_with_retry( - *args: str, - compose_file: Path = COMPOSE_FILE, - retries: int = 3, - delay: float = 5.0, -) -> None: - """Run a docker compose command, retrying on transient failures.""" - last_exc: subprocess.CalledProcessError | None = None - cmd = ["docker", "compose", "-f", str(compose_file), *args] - for attempt in range(1, retries + 1): - result = subprocess.run(cmd, capture_output=True, text=True) # nosec B603 - if result.returncode == 0: - if result.stdout: - print(result.stdout, end="") - return - last_exc = subprocess.CalledProcessError( - result.returncode, cmd, result.stdout, result.stderr - ) - if attempt < retries: - time.sleep(delay) - delay *= 2 - raise last_exc def mutate_decky(decky_name: str) -> bool: """ @@ -59,23 +37,21 @@ def mutate_decky(decky_name: str) -> bool: console.print(f"[red]Decky '{decky_name}' not found in state.[/]") return False - # Determine allowed services pool if decky.archetype: try: arch = get_archetype(decky.archetype) svc_pool = list(arch.services) except ValueError: - svc_pool = _all_service_names() + svc_pool = all_service_names() else: - svc_pool = _all_service_names() + svc_pool = all_service_names() if not svc_pool: console.print(f"[yellow]No services available for mutating '{decky_name}'.[/]") return False - # Prevent mutating to the exact same set if possible current_services = set(decky.services) - + attempts = 0 while True: count = random.randint(1, min(3, len(svc_pool))) # nosec B311 @@ -87,15 +63,11 @@ def mutate_decky(decky_name: str) -> bool: decky.services = list(chosen) decky.last_mutated = time.time() - # Save new state save_state(config, compose_path) - - # Regenerate compose file write_compose(config, compose_path) console.print(f"[cyan]Mutating '{decky_name}' to services: {', '.join(decky.services)}[/]") - # Bring up the new services and remove old orphans try: _compose_with_retry("up", "-d", "--remove-orphans", compose_file=compose_path) except subprocess.CalledProcessError as e: @@ -104,6 +76,7 @@ def mutate_decky(decky_name: str) -> bool: return True + def mutate_all(force: bool = False) -> None: """ Check all deckies and mutate those that are due. @@ -116,7 +89,7 @@ def mutate_all(force: bool = False) -> None: config, _ = state now = time.time() - + mutated_count = 0 for decky in config.deckies: interval_mins = decky.mutate_interval or config.mutate_interval @@ -133,14 +106,11 @@ def mutate_all(force: bool = False) -> None: success = mutate_decky(decky.name) if success: mutated_count += 1 - # Re-load state for next decky just in case, but mutate_decky saves it. - # However, mutate_decky operates on its own loaded state. - # Since mutate_decky loads and saves the state, our loop over `config.deckies` - # has an outdated `last_mutated` if we don't reload. It's fine because we process one by one. - + if mutated_count == 0 and not force: console.print("[dim]No deckies are due for mutation.[/]") + def run_watch_loop(poll_interval_secs: int = 10) -> None: """Run an infinite loop checking for deckies that need mutation.""" console.print(f"[green]DECNET Mutator Watcher started (polling every {poll_interval_secs}s).[/]") diff --git a/decnet/web/api.py b/decnet/web/api.py index abefa65..29529df 100644 --- a/decnet/web/api.py +++ b/decnet/web/api.py @@ -9,7 +9,7 @@ from fastapi.middleware.cors import CORSMiddleware from decnet.env import DECNET_CORS_ORIGINS, DECNET_DEVELOPER, DECNET_INGEST_LOG_FILE from decnet.web.dependencies import repo -from decnet.web.collector import log_collector_worker +from decnet.collector import log_collector_worker from decnet.web.ingester import log_ingestion_worker from decnet.web.router import api_router diff --git a/decnet/web/router/fleet/api_deploy_deckies.py b/decnet/web/router/fleet/api_deploy_deckies.py index bbe7d1d..c6d011d 100644 --- a/decnet/web/router/fleet/api_deploy_deckies.py +++ b/decnet/web/router/fleet/api_deploy_deckies.py @@ -4,7 +4,7 @@ import os from fastapi import APIRouter, Depends, HTTPException from decnet.config import DEFAULT_MUTATE_INTERVAL, DecnetConfig, load_state -from decnet.deployer import deploy as _deploy +from decnet.engine import deploy as _deploy from decnet.ini_loader import load_ini_from_string from decnet.network import detect_interface, detect_subnet, get_host_ip from decnet.web.dependencies import get_current_user @@ -15,7 +15,7 @@ router = APIRouter() @router.post("/deckies/deploy", tags=["Fleet Management"]) async def api_deploy_deckies(req: DeployIniRequest, current_user: str = Depends(get_current_user)) -> dict[str, str]: - from decnet.cli import _build_deckies_from_ini + from decnet.fleet import build_deckies_from_ini try: ini = load_ini_from_string(req.ini_content) @@ -56,7 +56,7 @@ async def api_deploy_deckies(req: DeployIniRequest, current_user: str = Depends( ) try: - new_decky_configs = _build_deckies_from_ini( + new_decky_configs = build_deckies_from_ini( ini, subnet_cidr, gateway, host_ip, randomize_services, cli_mutate_interval=None ) except ValueError as e: diff --git a/tests/test_archetypes.py b/tests/test_archetypes.py index 1304420..f5a18c9 100644 --- a/tests/test_archetypes.py +++ b/tests/test_archetypes.py @@ -266,7 +266,7 @@ def test_ini_subsection_direct_match_unaffected(): # --------------------------------------------------------------------------- def test_build_deckies_archetype_sets_services(): - from decnet.cli import _build_deckies + from decnet.fleet import build_deckies as _build_deckies from decnet.archetypes import get_archetype arch = get_archetype("mail-server") result = _build_deckies( @@ -283,7 +283,7 @@ def test_build_deckies_archetype_sets_services(): def test_build_deckies_archetype_preferred_distros(): - from decnet.cli import _build_deckies + from decnet.fleet import build_deckies as _build_deckies from decnet.archetypes import get_archetype arch = get_archetype("iot-device") # preferred_distros=["alpine"] result = _build_deckies( @@ -298,7 +298,7 @@ def test_build_deckies_archetype_preferred_distros(): def test_build_deckies_explicit_services_override_archetype(): - from decnet.cli import _build_deckies + from decnet.fleet import build_deckies as _build_deckies from decnet.archetypes import get_archetype arch = get_archetype("linux-server") result = _build_deckies( diff --git a/tests/test_build.py b/tests/test_build.py index fbdc185..7062bb5 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -8,7 +8,13 @@ MODULES = [ "decnet.cli", "decnet.config", "decnet.composer", - "decnet.deployer", + "decnet.engine", + "decnet.engine.deployer", + "decnet.collector", + "decnet.collector.worker", + "decnet.mutator", + "decnet.mutator.engine", + "decnet.fleet", "decnet.network", "decnet.archetypes", "decnet.distros", diff --git a/tests/test_cli_service_pool.py b/tests/test_cli_service_pool.py index 84c9387..6c673a3 100644 --- a/tests/test_cli_service_pool.py +++ b/tests/test_cli_service_pool.py @@ -3,7 +3,7 @@ Tests for the CLI service pool — verifies that --randomize-services draws from all registered services, not just the original hardcoded 5. """ -from decnet.cli import _all_service_names, _build_deckies +from decnet.fleet import all_service_names as _all_service_names, build_deckies as _build_deckies from decnet.services.registry import all_services diff --git a/tests/test_collector.py b/tests/test_collector.py index 5157f2e..edef7f2 100644 --- a/tests/test_collector.py +++ b/tests/test_collector.py @@ -3,7 +3,7 @@ import json from types import SimpleNamespace from unittest.mock import patch -from decnet.web.collector import parse_rfc5424, is_service_container, is_service_event +from decnet.collector import parse_rfc5424, is_service_container, is_service_event _KNOWN_NAMES = {"omega-decky-http", "omega-decky-smtp", "relay-decky-ftp"} @@ -91,42 +91,42 @@ class TestParseRfc5424: class TestIsServiceContainer: def test_known_container_returns_true(self): - with patch("decnet.web.collector._load_service_container_names", return_value=_KNOWN_NAMES): + with patch("decnet.collector.worker._load_service_container_names", return_value=_KNOWN_NAMES): assert is_service_container(_make_container("omega-decky-http")) is True assert is_service_container(_make_container("omega-decky-smtp")) is True assert is_service_container(_make_container("relay-decky-ftp")) is True def test_base_container_returns_false(self): - with patch("decnet.web.collector._load_service_container_names", return_value=_KNOWN_NAMES): + with patch("decnet.collector.worker._load_service_container_names", return_value=_KNOWN_NAMES): assert is_service_container(_make_container("omega-decky")) is False def test_unrelated_container_returns_false(self): - with patch("decnet.web.collector._load_service_container_names", return_value=_KNOWN_NAMES): + with patch("decnet.collector.worker._load_service_container_names", return_value=_KNOWN_NAMES): assert is_service_container(_make_container("nginx")) is False def test_strips_leading_slash(self): - with patch("decnet.web.collector._load_service_container_names", return_value=_KNOWN_NAMES): + with patch("decnet.collector.worker._load_service_container_names", return_value=_KNOWN_NAMES): assert is_service_container(_make_container("/omega-decky-http")) is True assert is_service_container(_make_container("/omega-decky")) is False def test_no_state_returns_false(self): - with patch("decnet.web.collector._load_service_container_names", return_value=set()): + with patch("decnet.collector.worker._load_service_container_names", return_value=set()): assert is_service_container(_make_container("omega-decky-http")) is False class TestIsServiceEvent: def test_known_service_event_returns_true(self): - with patch("decnet.web.collector._load_service_container_names", return_value=_KNOWN_NAMES): + with patch("decnet.collector.worker._load_service_container_names", return_value=_KNOWN_NAMES): assert is_service_event({"name": "omega-decky-smtp"}) is True def test_base_event_returns_false(self): - with patch("decnet.web.collector._load_service_container_names", return_value=_KNOWN_NAMES): + with patch("decnet.collector.worker._load_service_container_names", return_value=_KNOWN_NAMES): assert is_service_event({"name": "omega-decky"}) is False def test_unrelated_event_returns_false(self): - with patch("decnet.web.collector._load_service_container_names", return_value=_KNOWN_NAMES): + with patch("decnet.collector.worker._load_service_container_names", return_value=_KNOWN_NAMES): assert is_service_event({"name": "nginx"}) is False def test_no_state_returns_false(self): - with patch("decnet.web.collector._load_service_container_names", return_value=set()): + with patch("decnet.collector.worker._load_service_container_names", return_value=set()): assert is_service_event({"name": "omega-decky-smtp"}) is False diff --git a/tests/test_mutator.py b/tests/test_mutator.py index f81d7ac..9872883 100644 --- a/tests/test_mutator.py +++ b/tests/test_mutator.py @@ -10,7 +10,8 @@ from unittest.mock import MagicMock, patch import pytest from decnet.config import DeckyConfig, DecnetConfig -from decnet.mutator import _compose_with_retry, mutate_all, mutate_decky +from decnet.engine import _compose_with_retry +from decnet.mutator import mutate_all, mutate_decky # --------------------------------------------------------------------------- @@ -48,37 +49,37 @@ def _make_config(deckies=None, mutate_interval=30): class TestComposeWithRetry: def test_succeeds_on_first_attempt(self): result = MagicMock(returncode=0, stdout="done\n") - with patch("decnet.mutator.subprocess.run", return_value=result) as mock_run: + with patch("decnet.engine.deployer.subprocess.run", return_value=result) as mock_run: _compose_with_retry("up", "-d", compose_file=Path("compose.yml")) mock_run.assert_called_once() def test_retries_on_failure_then_succeeds(self): fail = MagicMock(returncode=1, stdout="", stderr="transient error") ok = MagicMock(returncode=0, stdout="", stderr="") - with patch("decnet.mutator.subprocess.run", side_effect=[fail, ok]) as mock_run, \ - patch("decnet.mutator.time.sleep"): + with patch("decnet.engine.deployer.subprocess.run", side_effect=[fail, ok]) as mock_run, \ + patch("decnet.engine.deployer.time.sleep"): _compose_with_retry("up", "-d", compose_file=Path("compose.yml"), retries=3) assert mock_run.call_count == 2 def test_raises_after_all_retries_exhausted(self): fail = MagicMock(returncode=1, stdout="", stderr="hard error") - with patch("decnet.mutator.subprocess.run", return_value=fail), \ - patch("decnet.mutator.time.sleep"): + with patch("decnet.engine.deployer.subprocess.run", return_value=fail), \ + patch("decnet.engine.deployer.time.sleep"): with pytest.raises(subprocess.CalledProcessError): _compose_with_retry("up", "-d", compose_file=Path("compose.yml"), retries=3) def test_exponential_backoff(self): fail = MagicMock(returncode=1, stdout="", stderr="") sleep_calls = [] - with patch("decnet.mutator.subprocess.run", return_value=fail), \ - patch("decnet.mutator.time.sleep", side_effect=lambda d: sleep_calls.append(d)): + with patch("decnet.engine.deployer.subprocess.run", return_value=fail), \ + patch("decnet.engine.deployer.time.sleep", side_effect=lambda d: sleep_calls.append(d)): with pytest.raises(subprocess.CalledProcessError): _compose_with_retry("up", compose_file=Path("c.yml"), retries=3, delay=1.0) assert sleep_calls == [1.0, 2.0] def test_correct_command_structure(self): ok = MagicMock(returncode=0, stdout="") - with patch("decnet.mutator.subprocess.run", return_value=ok) as mock_run: + with patch("decnet.engine.deployer.subprocess.run", return_value=ok) as mock_run: _compose_with_retry("up", "-d", "--remove-orphans", compose_file=Path("/tmp/compose.yml")) cmd = mock_run.call_args[0][0] @@ -96,14 +97,14 @@ class TestMutateDecky: """Return a context manager that mocks all I/O in mutate_decky.""" cfg = config or _make_config() return ( - patch("decnet.mutator.load_state", return_value=(cfg, compose_path)), - patch("decnet.mutator.save_state"), - patch("decnet.mutator.write_compose"), - patch("decnet.mutator._compose_with_retry"), + patch("decnet.mutator.engine.load_state", return_value=(cfg, compose_path)), + patch("decnet.mutator.engine.save_state"), + patch("decnet.mutator.engine.write_compose"), + patch("decnet.mutator.engine._compose_with_retry"), ) def test_returns_false_when_no_state(self): - with patch("decnet.mutator.load_state", return_value=None): + with patch("decnet.mutator.engine.load_state", return_value=None): assert mutate_decky("decky-01") is False def test_returns_false_when_decky_not_found(self): @@ -118,20 +119,20 @@ class TestMutateDecky: def test_saves_state_after_mutation(self): p = self._patch() - with p[0], patch("decnet.mutator.save_state") as mock_save, p[2], p[3]: + with p[0], patch("decnet.mutator.engine.save_state") as mock_save, p[2], p[3]: mutate_decky("decky-01") mock_save.assert_called_once() def test_regenerates_compose_after_mutation(self): p = self._patch() - with p[0], p[1], patch("decnet.mutator.write_compose") as mock_compose, p[3]: + with p[0], p[1], patch("decnet.mutator.engine.write_compose") as mock_compose, p[3]: mutate_decky("decky-01") mock_compose.assert_called_once() def test_returns_false_on_compose_failure(self): p = self._patch() err = subprocess.CalledProcessError(1, "docker", "", "compose failed") - with p[0], p[1], p[2], patch("decnet.mutator._compose_with_retry", side_effect=err): + with p[0], p[1], p[2], patch("decnet.mutator.engine._compose_with_retry", side_effect=err): assert mutate_decky("decky-01") is False def test_mutation_changes_services(self): @@ -166,15 +167,15 @@ class TestMutateDecky: class TestMutateAll: def test_no_state_returns_early(self): - with patch("decnet.mutator.load_state", return_value=None), \ - patch("decnet.mutator.mutate_decky") as mock_mutate: + with patch("decnet.mutator.engine.load_state", return_value=None), \ + patch("decnet.mutator.engine.mutate_decky") as mock_mutate: mutate_all() mock_mutate.assert_not_called() def test_force_mutates_all_deckies(self): cfg = _make_config(deckies=[_make_decky("d1"), _make_decky("d2")]) - with patch("decnet.mutator.load_state", return_value=(cfg, Path("c.yml"))), \ - patch("decnet.mutator.mutate_decky", return_value=True) as mock_mutate: + with patch("decnet.mutator.engine.load_state", return_value=(cfg, Path("c.yml"))), \ + patch("decnet.mutator.engine.mutate_decky", return_value=True) as mock_mutate: mutate_all(force=True) assert mock_mutate.call_count == 2 @@ -182,8 +183,8 @@ class TestMutateAll: # last_mutated = now, interval = 30 min → not due now = time.time() cfg = _make_config(deckies=[_make_decky(mutate_interval=30, last_mutated=now)]) - with patch("decnet.mutator.load_state", return_value=(cfg, Path("c.yml"))), \ - patch("decnet.mutator.mutate_decky") as mock_mutate: + with patch("decnet.mutator.engine.load_state", return_value=(cfg, Path("c.yml"))), \ + patch("decnet.mutator.engine.mutate_decky") as mock_mutate: mutate_all(force=False) mock_mutate.assert_not_called() @@ -191,8 +192,8 @@ class TestMutateAll: # last_mutated = 2 hours ago, interval = 30 min → due old_ts = time.time() - 7200 cfg = _make_config(deckies=[_make_decky(mutate_interval=30, last_mutated=old_ts)]) - with patch("decnet.mutator.load_state", return_value=(cfg, Path("c.yml"))), \ - patch("decnet.mutator.mutate_decky", return_value=True) as mock_mutate: + with patch("decnet.mutator.engine.load_state", return_value=(cfg, Path("c.yml"))), \ + patch("decnet.mutator.engine.mutate_decky", return_value=True) as mock_mutate: mutate_all(force=False) mock_mutate.assert_called_once_with("decky-01") @@ -201,7 +202,7 @@ class TestMutateAll: deckies=[_make_decky(mutate_interval=None)], mutate_interval=None, ) - with patch("decnet.mutator.load_state", return_value=(cfg, Path("c.yml"))), \ - patch("decnet.mutator.mutate_decky") as mock_mutate: + with patch("decnet.mutator.engine.load_state", return_value=(cfg, Path("c.yml"))), \ + patch("decnet.mutator.engine.mutate_decky") as mock_mutate: mutate_all(force=False) mock_mutate.assert_not_called() diff --git a/tests/test_os_fingerprint.py b/tests/test_os_fingerprint.py index 79e3438..5193716 100644 --- a/tests/test_os_fingerprint.py +++ b/tests/test_os_fingerprint.py @@ -434,7 +434,7 @@ def test_compose_embedded_sysctls_full_set(): def test_build_deckies_windows_archetype_sets_nmap_os(): from decnet.archetypes import get_archetype - from decnet.cli import _build_deckies + from decnet.fleet import build_deckies as _build_deckies arch = get_archetype("windows-workstation") deckies = _build_deckies( @@ -448,7 +448,7 @@ def test_build_deckies_windows_archetype_sets_nmap_os(): def test_build_deckies_no_archetype_defaults_linux(): - from decnet.cli import _build_deckies + from decnet.fleet import build_deckies as _build_deckies deckies = _build_deckies( n=1, @@ -462,7 +462,7 @@ def test_build_deckies_no_archetype_defaults_linux(): def test_build_deckies_embedded_archetype_sets_nmap_os(): from decnet.archetypes import get_archetype - from decnet.cli import _build_deckies + from decnet.fleet import build_deckies as _build_deckies arch = get_archetype("iot-device") deckies = _build_deckies(