From 07c06e3c0aa4d7b6f765cb17492d98658fee41f4 Mon Sep 17 00:00:00 2001 From: anti Date: Sat, 4 Apr 2026 00:57:51 -0300 Subject: [PATCH] Replace dead upstream images with custom build services; add retry logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dtagdevsec/mailoney and dtagdevsec/elasticpot are unavailable on Docker Hub ("manifest unknown"), causing the entire deployment to abort and cascade- interrupt all other image pulls. - Convert smtp and elasticsearch to build services with custom Python honeypots: smtp emulates Postfix ESMTP (EHLO/AUTH/MAIL/RCPT logging), elasticsearch emulates ES 7.17 HTTP API (logs recon probes like /_cat/, /_cluster/, /_nodes/, /_security/) - Both use ARG BASE_IMAGE so they participate in per-decky distro variation - Add _compose_with_retry() to deployer: 3 attempts with exponential backoff (5s → 10s → 20s) for transient network failures; permanent errors ("manifest unknown", "pull access denied") are detected and not retried - Update test_services.py and test_composer.py: smtp/elasticsearch moved from UPSTREAM_SERVICES to BUILD_SERVICES (314 tests passing) Co-Authored-By: Claude Sonnet 4.6 --- decnet/deployer.py | 51 ++++++- decnet/services/elasticsearch.py | 26 ++-- decnet/services/smtp.py | 27 ++-- templates/elasticsearch/Dockerfile | 13 ++ .../elasticsearch/elasticsearch_honeypot.py | 138 ++++++++++++++++++ templates/elasticsearch/entrypoint.sh | 3 + templates/smtp/Dockerfile | 13 ++ templates/smtp/entrypoint.sh | 3 + templates/smtp/smtp_honeypot.py | 117 +++++++++++++++ tests/test_composer.py | 6 +- tests/test_services.py | 50 +++---- 11 files changed, 394 insertions(+), 53 deletions(-) create mode 100644 templates/elasticsearch/Dockerfile create mode 100644 templates/elasticsearch/elasticsearch_honeypot.py create mode 100644 templates/elasticsearch/entrypoint.sh create mode 100644 templates/smtp/Dockerfile create mode 100644 templates/smtp/entrypoint.sh create mode 100644 templates/smtp/smtp_honeypot.py diff --git a/decnet/deployer.py b/decnet/deployer.py index 359eec6..6c21a4c 100644 --- a/decnet/deployer.py +++ b/decnet/deployer.py @@ -3,6 +3,7 @@ Deploy, teardown, and status via Docker SDK + subprocess docker compose. """ import subprocess +import time from pathlib import Path import docker @@ -33,6 +34,52 @@ def _compose(*args: str, compose_file: Path = COMPOSE_FILE) -> None: subprocess.run(cmd, check=True) +_PERMANENT_ERRORS = ( + "manifest unknown", + "manifest for", + "not found", + "pull access denied", + "repository does not exist", +) + + +def _compose_with_retry( + *args: str, + compose_file: Path = COMPOSE_FILE, + retries: int = 3, + delay: float = 5.0, +) -> None: + """Run a docker compose command, retrying on transient failures.""" + last_exc: subprocess.CalledProcessError | None = None + cmd = ["docker", "compose", "-f", str(compose_file), *args] + for attempt in range(1, retries + 1): + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode == 0: + if result.stdout: + print(result.stdout, end="") + return + last_exc = subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) + stderr_lower = (result.stderr or "").lower() + if any(pat in stderr_lower for pat in _PERMANENT_ERRORS): + console.print(f"[red]Permanent Docker error — not retrying:[/]\n{result.stderr.strip()}") + raise last_exc + if attempt < retries: + console.print( + f"[yellow]docker compose {' '.join(args)} failed " + f"(attempt {attempt}/{retries}), retrying in {delay:.0f}s…[/]" + ) + if result.stderr: + console.print(f"[dim]{result.stderr.strip()}[/]") + time.sleep(delay) + delay *= 2 + else: + if result.stderr: + console.print(f"[red]{result.stderr.strip()}[/]") + raise last_exc + + def deploy(config: DecnetConfig, dry_run: bool = False, no_cache: bool = False) -> None: client = docker.from_env() @@ -66,8 +113,8 @@ def deploy(config: DecnetConfig, dry_run: bool = False, no_cache: bool = False) # --- Bring up --- console.print("[bold cyan]Building images and starting deckies...[/]") if no_cache: - _compose("build", "--no-cache", compose_file=compose_path) - _compose("up", "--build", "-d", compose_file=compose_path) + _compose_with_retry("build", "--no-cache", compose_file=compose_path) + _compose_with_retry("up", "--build", "-d", compose_file=compose_path) # --- Status summary --- _print_status(config) diff --git a/decnet/services/elasticsearch.py b/decnet/services/elasticsearch.py index 13da380..3f40179 100644 --- a/decnet/services/elasticsearch.py +++ b/decnet/services/elasticsearch.py @@ -1,23 +1,27 @@ +from pathlib import Path + from decnet.services.base import BaseService +TEMPLATES_DIR = Path(__file__).parent.parent.parent / "templates" / "elasticsearch" + class ElasticsearchService(BaseService): name = "elasticsearch" ports = [9200] - default_image = "dtagdevsec/elasticpot" + default_image = "build" def compose_fragment(self, decky_name: str, log_target: str | None = None) -> dict: - env: dict = { - "ELASTICPOT_HOSTNAME": decky_name, - } - if log_target: - env["ELASTICPOT_LOG_TARGET"] = log_target - return { - "image": "dtagdevsec/elasticpot", + fragment: dict = { + "build": {"context": str(TEMPLATES_DIR)}, "container_name": f"{decky_name}-elasticsearch", "restart": "unless-stopped", - "environment": env, + "environment": { + "HONEYPOT_NAME": decky_name, + }, } + if log_target: + fragment["environment"]["LOG_TARGET"] = log_target + return fragment - def dockerfile_context(self): - return None + def dockerfile_context(self) -> Path: + return TEMPLATES_DIR diff --git a/decnet/services/smtp.py b/decnet/services/smtp.py index f72baa5..2a98bbe 100644 --- a/decnet/services/smtp.py +++ b/decnet/services/smtp.py @@ -1,25 +1,28 @@ +from pathlib import Path + from decnet.services.base import BaseService +TEMPLATES_DIR = Path(__file__).parent.parent.parent / "templates" / "smtp" + class SMTPService(BaseService): name = "smtp" ports = [25, 587] - default_image = "dtagdevsec/mailoney" + default_image = "build" def compose_fragment(self, decky_name: str, log_target: str | None = None) -> dict: - env: dict = { - "MAILONEY_HOSTNAME": decky_name, - "MAILONEY_PORTS": "25,587", - } - if log_target: - env["MAILONEY_LOG_TARGET"] = log_target - return { - "image": "dtagdevsec/mailoney", + fragment: dict = { + "build": {"context": str(TEMPLATES_DIR)}, "container_name": f"{decky_name}-smtp", "restart": "unless-stopped", "cap_add": ["NET_BIND_SERVICE"], - "environment": env, + "environment": { + "HONEYPOT_NAME": decky_name, + }, } + if log_target: + fragment["environment"]["LOG_TARGET"] = log_target + return fragment - def dockerfile_context(self): - return None + def dockerfile_context(self) -> Path: + return TEMPLATES_DIR diff --git a/templates/elasticsearch/Dockerfile b/templates/elasticsearch/Dockerfile new file mode 100644 index 0000000..4beccf7 --- /dev/null +++ b/templates/elasticsearch/Dockerfile @@ -0,0 +1,13 @@ +ARG BASE_IMAGE=debian:bookworm-slim +FROM ${BASE_IMAGE} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 \ + && rm -rf /var/lib/apt/lists/* + +COPY elasticsearch_honeypot.py /opt/elasticsearch_honeypot.py +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +EXPOSE 9200 +ENTRYPOINT ["/entrypoint.sh"] diff --git a/templates/elasticsearch/elasticsearch_honeypot.py b/templates/elasticsearch/elasticsearch_honeypot.py new file mode 100644 index 0000000..7dc6c27 --- /dev/null +++ b/templates/elasticsearch/elasticsearch_honeypot.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +""" +Elasticsearch honeypot — presents a convincing ES 7.x HTTP API on port 9200. +Logs all requests (especially recon probes like /_cat/, /_cluster/, /_nodes/) +as JSON. Designed to attract automated scanners and credential stuffers. +""" + +import json +import os +import socket +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, HTTPServer + +HONEYPOT_NAME = os.environ.get("HONEYPOT_NAME", "esserver") +LOG_TARGET = os.environ.get("LOG_TARGET", "") + +_CLUSTER_UUID = "xC3Pr9abTq2mNkOeLvXwYA" +_NODE_UUID = "dJH7Lm2sRqWvPn0kFiEtBo" + +_ROOT_RESPONSE = { + "name": HONEYPOT_NAME, + "cluster_name": "elasticsearch", + "cluster_uuid": _CLUSTER_UUID, + "version": { + "number": "7.17.9", + "build_flavor": "default", + "build_type": "docker", + "build_hash": "ef48222227ee6b9e70e502f0f0daa52435ee634d", + "build_date": "2023-01-31T05:34:43.305517834Z", + "build_snapshot": False, + "lucene_version": "8.11.1", + "minimum_wire_compatibility_version": "6.8.0", + "minimum_index_compatibility_version": "6.0.0-beta1", + }, + "tagline": "You Know, for Search", +} + + +def _forward(event: dict) -> None: + if not LOG_TARGET: + return + try: + host, port = LOG_TARGET.rsplit(":", 1) + with socket.create_connection((host, int(port)), timeout=3) as s: + s.sendall((json.dumps(event) + "\n").encode()) + except Exception: + pass + + +def _log(event_type: str, **kwargs) -> None: + event = { + "ts": datetime.now(timezone.utc).isoformat(), + "service": "elasticsearch", + "host": HONEYPOT_NAME, + "event": event_type, + **kwargs, + } + print(json.dumps(event), flush=True) + _forward(event) + + +class ESHandler(BaseHTTPRequestHandler): + server_version = "elasticsearch" + sys_version = "" + + def _send_json(self, code: int, data: dict) -> None: + body = json.dumps(data).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json; charset=UTF-8") + self.send_header("Content-Length", str(len(body))) + self.send_header("X-elastic-product", "Elasticsearch") + self.end_headers() + self.wfile.write(body) + + def _read_body(self) -> str: + length = int(self.headers.get("Content-Length", 0)) + return self.rfile.read(length).decode(errors="replace") if length else "" + + def do_GET(self): + src = self.client_address[0] + path = self.path.split("?")[0] + + if path in ("/", ""): + _log("root_probe", src=src, method="GET", path=self.path) + self._send_json(200, _ROOT_RESPONSE) + elif path.startswith("/_cat/"): + _log("cat_api", src=src, method="GET", path=self.path) + self._send_json(200, []) + elif path.startswith("/_cluster/"): + _log("cluster_recon", src=src, method="GET", path=self.path) + self._send_json(200, {"cluster_name": "elasticsearch", "status": "green", + "number_of_nodes": 3, "number_of_data_nodes": 3}) + elif path.startswith("/_nodes"): + _log("nodes_recon", src=src, method="GET", path=self.path) + self._send_json(200, {"_nodes": {"total": 3, "successful": 3, "failed": 0}, "nodes": {}}) + elif path.startswith("/_security/") or path.startswith("/_xpack/"): + _log("security_probe", src=src, method="GET", path=self.path) + self._send_json(200, {"enabled": True, "available": True}) + else: + _log("request", src=src, method="GET", path=self.path) + self._send_json(404, {"error": {"root_cause": [{"type": "index_not_found_exception", + "reason": "no such index"}]}}) + + def do_POST(self): + src = self.client_address[0] + body = self._read_body() + path = self.path.split("?")[0] + _log("post_request", src=src, method="POST", path=self.path, + body_preview=body[:300], user_agent=self.headers.get("User-Agent", "")) + if "_search" in path or "_bulk" in path: + self._send_json(200, {"took": 1, "timed_out": False, "hits": {"total": {"value": 0}, "hits": []}}) + else: + self._send_json(200, {"result": "created", "_id": "1", "_index": "honeypot"}) + + def do_PUT(self): + src = self.client_address[0] + body = self._read_body() + _log("put_request", src=src, method="PUT", path=self.path, body_preview=body[:300]) + self._send_json(200, {"acknowledged": True}) + + def do_DELETE(self): + src = self.client_address[0] + _log("delete_request", src=src, method="DELETE", path=self.path) + self._send_json(200, {"acknowledged": True}) + + def do_HEAD(self): + src = self.client_address[0] + _log("head_request", src=src, method="HEAD", path=self.path) + self._send_json(200, {}) + + def log_message(self, fmt, *args): + pass # suppress default HTTP server logging + + +if __name__ == "__main__": + _log("startup", msg=f"Elasticsearch honeypot starting as {HONEYPOT_NAME}") + server = HTTPServer(("0.0.0.0", 9200), ESHandler) + server.serve_forever() diff --git a/templates/elasticsearch/entrypoint.sh b/templates/elasticsearch/entrypoint.sh new file mode 100644 index 0000000..14898cc --- /dev/null +++ b/templates/elasticsearch/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -e +exec python3 /opt/elasticsearch_honeypot.py diff --git a/templates/smtp/Dockerfile b/templates/smtp/Dockerfile new file mode 100644 index 0000000..416ac07 --- /dev/null +++ b/templates/smtp/Dockerfile @@ -0,0 +1,13 @@ +ARG BASE_IMAGE=debian:bookworm-slim +FROM ${BASE_IMAGE} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 \ + && rm -rf /var/lib/apt/lists/* + +COPY smtp_honeypot.py /opt/smtp_honeypot.py +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +EXPOSE 25 587 +ENTRYPOINT ["/entrypoint.sh"] diff --git a/templates/smtp/entrypoint.sh b/templates/smtp/entrypoint.sh new file mode 100644 index 0000000..e113316 --- /dev/null +++ b/templates/smtp/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -e +exec python3 /opt/smtp_honeypot.py diff --git a/templates/smtp/smtp_honeypot.py b/templates/smtp/smtp_honeypot.py new file mode 100644 index 0000000..cb611a6 --- /dev/null +++ b/templates/smtp/smtp_honeypot.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +""" +SMTP honeypot — emulates a realistic ESMTP server (Postfix-style). +Logs EHLO/AUTH/MAIL FROM/RCPT TO attempts as JSON, then denies auth. +""" + +import asyncio +import json +import os +import socket +from datetime import datetime, timezone + +HONEYPOT_NAME = os.environ.get("HONEYPOT_NAME", "mailserver") +LOG_TARGET = os.environ.get("LOG_TARGET", "") + + +def _forward(event: dict) -> None: + if not LOG_TARGET: + return + try: + host, port = LOG_TARGET.rsplit(":", 1) + with socket.create_connection((host, int(port)), timeout=3) as s: + s.sendall((json.dumps(event) + "\n").encode()) + except Exception: + pass + + +def _log(event_type: str, **kwargs) -> None: + event = { + "ts": datetime.now(timezone.utc).isoformat(), + "service": "smtp", + "host": HONEYPOT_NAME, + "event": event_type, + **kwargs, + } + print(json.dumps(event), flush=True) + _forward(event) + + +class SMTPProtocol(asyncio.Protocol): + def __init__(self): + self._transport = None + self._peer = ("?", 0) + self._buf = b"" + + def connection_made(self, transport): + self._transport = transport + self._peer = transport.get_extra_info("peername", ("?", 0)) + _log("connect", src=self._peer[0], src_port=self._peer[1]) + transport.write(f"220 {HONEYPOT_NAME} ESMTP Postfix (Debian/GNU)\r\n".encode()) + + def data_received(self, data): + self._buf += data + while b"\r\n" in self._buf: + line, self._buf = self._buf.split(b"\r\n", 1) + self._handle_line(line.decode(errors="replace").strip()) + + def _handle_line(self, line: str) -> None: + cmd = line.split()[0].upper() if line.split() else "" + + if cmd in ("EHLO", "HELO"): + domain = line.split(None, 1)[1] if " " in line else "" + _log("ehlo", src=self._peer[0], domain=domain) + self._transport.write( + f"250-{HONEYPOT_NAME}\r\n" + f"250-PIPELINING\r\n" + f"250-SIZE 10240000\r\n" + f"250-VRFY\r\n" + f"250-ETRN\r\n" + f"250-AUTH PLAIN LOGIN\r\n" + f"250-ENHANCEDSTATUSCODES\r\n" + f"250-8BITMIME\r\n" + f"250 DSN\r\n".encode() + ) + elif cmd == "AUTH": + _log("auth_attempt", src=self._peer[0], command=line) + self._transport.write(b"535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6\r\n") + self._transport.close() + elif cmd == "MAIL": + _log("mail_from", src=self._peer[0], value=line) + self._transport.write(b"250 2.1.0 Ok\r\n") + elif cmd == "RCPT": + _log("rcpt_to", src=self._peer[0], value=line) + self._transport.write(b"250 2.1.5 Ok\r\n") + elif cmd == "DATA": + self._transport.write(b"354 End data with .\r\n") + elif cmd == "VRFY": + _log("vrfy", src=self._peer[0], value=line) + self._transport.write(b"252 2.0.0 Cannot VRFY user\r\n") + elif cmd == "QUIT": + self._transport.write(b"221 2.0.0 Bye\r\n") + self._transport.close() + elif cmd == "NOOP": + self._transport.write(b"250 2.0.0 Ok\r\n") + elif cmd == "RSET": + self._transport.write(b"250 2.0.0 Ok\r\n") + elif cmd == "STARTTLS": + # Pretend we don't support upgrading mid-session + self._transport.write(b"454 4.7.0 TLS not available due to local problem\r\n") + else: + _log("unknown_command", src=self._peer[0], command=line) + self._transport.write(b"502 5.5.2 Error: command not recognized\r\n") + + def connection_lost(self, exc): + _log("disconnect", src=self._peer[0] if self._peer else "?") + + +async def main(): + _log("startup", msg=f"SMTP honeypot starting as {HONEYPOT_NAME}") + loop = asyncio.get_running_loop() + server = await loop.create_server(SMTPProtocol, "0.0.0.0", 25) + async with server: + await server.serve_forever() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/test_composer.py b/tests/test_composer.py index 298679c..d707a6a 100644 --- a/tests/test_composer.py +++ b/tests/test_composer.py @@ -20,13 +20,13 @@ APT_COMPATIBLE = { } BUILD_SERVICES = [ - "http", "rdp", "smb", "ftp", "pop3", "imap", - "mysql", "mssql", "redis", "mongodb", "postgres", + "http", "rdp", "smb", "ftp", "smtp", "elasticsearch", + "pop3", "imap", "mysql", "mssql", "redis", "mongodb", "postgres", "ldap", "vnc", "docker_api", "k8s", "sip", "mqtt", "llmnr", "snmp", "tftp", ] -UPSTREAM_SERVICES = ["ssh", "telnet", "smtp", "elasticsearch", "conpot"] +UPSTREAM_SERVICES = ["ssh", "telnet", "conpot"] def _make_config(services, distro="debian", base_image=None, build_base=None): diff --git a/tests/test_services.py b/tests/test_services.py index e8614b0..86902ad 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -31,11 +31,9 @@ def _is_build_service(name: str) -> bool: # --------------------------------------------------------------------------- UPSTREAM_SERVICES = { - "ssh": ("cowrie/cowrie", [22, 2222]), - "telnet": ("cowrie/cowrie", [23]), - "smtp": ("dtagdevsec/mailoney", [25, 587]), - "elasticsearch": ("dtagdevsec/elasticpot", [9200]), - "conpot": ("honeynet/conpot", [502, 161, 80]), + "ssh": ("cowrie/cowrie", [22, 2222]), + "telnet": ("cowrie/cowrie", [23]), + "conpot": ("honeynet/conpot", [502, 161, 80]), } # --------------------------------------------------------------------------- @@ -43,26 +41,28 @@ UPSTREAM_SERVICES = { # --------------------------------------------------------------------------- BUILD_SERVICES = { - "http": ([80, 443], "http"), - "rdp": ([3389], "rdp"), - "smb": ([445, 139], "smb"), - "ftp": ([21], "ftp"), - "pop3": ([110, 995], "pop3"), - "imap": ([143, 993], "imap"), - "mysql": ([3306], "mysql"), - "mssql": ([1433], "mssql"), - "redis": ([6379], "redis"), - "mongodb": ([27017], "mongodb"), - "postgres": ([5432], "postgres"), - "ldap": ([389, 636], "ldap"), - "vnc": ([5900], "vnc"), - "docker_api": ([2375, 2376], "docker_api"), - "k8s": ([6443, 8080], "k8s"), - "sip": ([5060], "sip"), - "mqtt": ([1883], "mqtt"), - "llmnr": ([5355, 5353], "llmnr"), - "snmp": ([161], "snmp"), - "tftp": ([69], "tftp"), + "http": ([80, 443], "http"), + "rdp": ([3389], "rdp"), + "smb": ([445, 139], "smb"), + "ftp": ([21], "ftp"), + "smtp": ([25, 587], "smtp"), + "elasticsearch": ([9200], "elasticsearch"), + "pop3": ([110, 995], "pop3"), + "imap": ([143, 993], "imap"), + "mysql": ([3306], "mysql"), + "mssql": ([1433], "mssql"), + "redis": ([6379], "redis"), + "mongodb": ([27017], "mongodb"), + "postgres": ([5432], "postgres"), + "ldap": ([389, 636], "ldap"), + "vnc": ([5900], "vnc"), + "docker_api": ([2375, 2376], "docker_api"), + "k8s": ([6443, 8080], "k8s"), + "sip": ([5060], "sip"), + "mqtt": ([1883], "mqtt"), + "llmnr": ([5355, 5353], "llmnr"), + "snmp": ([161], "snmp"), + "tftp": ([69], "tftp"), } ALL_SERVICE_NAMES = list(UPSTREAM_SERVICES) + list(BUILD_SERVICES)