feat: replace bind-mount log pipeline with Docker log streaming
Services now print RFC 5424 to stdout; Docker captures via json-file driver. A new host-side collector (decnet.web.collector) streams docker logs from all running decky service containers and writes RFC 5424 + parsed JSON to the host log file. The existing ingester continues to tail the .json file unchanged. rsyslog can consume the .log file independently — no DECNET involvement needed. Removes: bind-mount volume injection, _LOG_NETWORK bridge, log_target config field and --log-target CLI flag, TCP syslog forwarding from service templates.
This commit is contained in:
@@ -6,6 +6,12 @@ Network model:
|
||||
All service containers for that decky share the base's network namespace
|
||||
via `network_mode: "service:<base>"`. From the outside, every service on
|
||||
a given decky appears to come from the same IP — exactly like a real host.
|
||||
|
||||
Logging model:
|
||||
Service containers write RFC 5424 lines to stdout. Docker captures them
|
||||
via the json-file driver. The host-side collector (decnet.web.collector)
|
||||
streams those logs and writes them to the host log file for the ingester
|
||||
and rsyslog to consume. No bind mounts or shared volumes are needed.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
@@ -17,38 +23,19 @@ from decnet.network import MACVLAN_NETWORK_NAME
|
||||
from decnet.os_fingerprint import get_os_sysctls
|
||||
from decnet.services.registry import get_service
|
||||
|
||||
_CONTAINER_LOG_DIR = "/var/log/decnet"
|
||||
|
||||
_LOG_NETWORK = "decnet_logs"
|
||||
|
||||
|
||||
def _resolve_log_file(log_file: str) -> tuple[str, str]:
|
||||
"""
|
||||
Return (host_dir, container_log_path) for a user-supplied log file path.
|
||||
|
||||
The host path is resolved to absolute so Docker can bind-mount it.
|
||||
All containers share the same host directory, mounted at _CONTAINER_LOG_DIR.
|
||||
"""
|
||||
host_path = Path(log_file).resolve()
|
||||
host_dir = str(host_path.parent)
|
||||
container_path = f"{_CONTAINER_LOG_DIR}/{host_path.name}"
|
||||
return host_dir, container_path
|
||||
_DOCKER_LOGGING = {
|
||||
"driver": "json-file",
|
||||
"options": {
|
||||
"max-size": "10m",
|
||||
"max-file": "5",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def generate_compose(config: DecnetConfig) -> dict:
|
||||
"""Build and return the full docker-compose data structure."""
|
||||
services: dict = {}
|
||||
|
||||
log_host_dir: str | None = None
|
||||
log_container_path: str | None = None
|
||||
if config.log_file:
|
||||
log_host_dir, log_container_path = _resolve_log_file(config.log_file)
|
||||
# Ensure the host log directory exists and is writable by the container's
|
||||
# non-root 'decnet' user (DEBT-019). mkdir respects umask, so chmod explicitly.
|
||||
_log_dir = Path(log_host_dir)
|
||||
_log_dir.mkdir(parents=True, exist_ok=True)
|
||||
_log_dir.chmod(0o777)
|
||||
|
||||
for decky in config.deckies:
|
||||
base_key = decky.name # e.g. "decky-01"
|
||||
|
||||
@@ -65,8 +52,6 @@ def generate_compose(config: DecnetConfig) -> dict:
|
||||
}
|
||||
},
|
||||
}
|
||||
if config.log_target:
|
||||
base["networks"][_LOG_NETWORK] = {}
|
||||
|
||||
# Inject TCP/IP stack sysctls to spoof the claimed OS fingerprint.
|
||||
# Only the base container needs this — service containers inherit the
|
||||
@@ -80,9 +65,7 @@ def generate_compose(config: DecnetConfig) -> dict:
|
||||
for svc_name in decky.services:
|
||||
svc = get_service(svc_name)
|
||||
svc_cfg = decky.service_config.get(svc_name, {})
|
||||
fragment = svc.compose_fragment(
|
||||
decky.name, log_target=config.log_target, service_cfg=svc_cfg
|
||||
)
|
||||
fragment = svc.compose_fragment(decky.name, service_cfg=svc_cfg)
|
||||
|
||||
# Inject the per-decky base image into build services so containers
|
||||
# vary by distro and don't all fingerprint as debian:bookworm-slim.
|
||||
@@ -91,12 +74,6 @@ def generate_compose(config: DecnetConfig) -> dict:
|
||||
|
||||
fragment.setdefault("environment", {})
|
||||
fragment["environment"]["HOSTNAME"] = decky.hostname
|
||||
if log_host_dir and log_container_path:
|
||||
fragment["environment"]["DECNET_LOG_FILE"] = log_container_path
|
||||
fragment.setdefault("volumes", [])
|
||||
mount = f"{log_host_dir}:{_CONTAINER_LOG_DIR}"
|
||||
if mount not in fragment["volumes"]:
|
||||
fragment["volumes"].append(mount)
|
||||
|
||||
# Share the base container's network — no own IP needed
|
||||
fragment["network_mode"] = f"service:{base_key}"
|
||||
@@ -106,6 +83,9 @@ def generate_compose(config: DecnetConfig) -> dict:
|
||||
fragment.pop("hostname", None)
|
||||
fragment.pop("networks", None)
|
||||
|
||||
# Rotate Docker logs so disk usage is bounded
|
||||
fragment["logging"] = _DOCKER_LOGGING
|
||||
|
||||
services[f"{decky.name}-{svc_name}"] = fragment
|
||||
|
||||
# Network definitions
|
||||
@@ -114,8 +94,6 @@ def generate_compose(config: DecnetConfig) -> dict:
|
||||
"external": True, # created by network.py before compose up
|
||||
}
|
||||
}
|
||||
if config.log_target:
|
||||
networks[_LOG_NETWORK] = {"driver": "bridge", "internal": True}
|
||||
|
||||
return {
|
||||
"version": "3.8",
|
||||
|
||||
Reference in New Issue
Block a user