Replace dead upstream images with custom build services; add retry logic

dtagdevsec/mailoney and dtagdevsec/elasticpot are unavailable on Docker Hub
("manifest unknown"), causing the entire deployment to abort and cascade-
interrupt all other image pulls.

- Convert smtp and elasticsearch to build services with custom Python
  honeypots: smtp emulates Postfix ESMTP (EHLO/AUTH/MAIL/RCPT logging),
  elasticsearch emulates ES 7.17 HTTP API (logs recon probes like /_cat/,
  /_cluster/, /_nodes/, /_security/)
- Both use ARG BASE_IMAGE so they participate in per-decky distro variation
- Add _compose_with_retry() to deployer: 3 attempts with exponential backoff
  (5s → 10s → 20s) for transient network failures; permanent errors
  ("manifest unknown", "pull access denied") are detected and not retried
- Update test_services.py and test_composer.py: smtp/elasticsearch moved
  from UPSTREAM_SERVICES to BUILD_SERVICES (314 tests passing)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

decnet/deployer.py

@@ -3,6 +3,7 @@ Deploy, teardown, and status via Docker SDK + subprocess docker compose.
 """
 
 import subprocess
+import time
 from pathlib import Path
 
 import docker
@@ -33,6 +34,52 @@ def _compose(*args: str, compose_file: Path = COMPOSE_FILE) -> None:
     subprocess.run(cmd, check=True)
 
 
+_PERMANENT_ERRORS = (
+    "manifest unknown",
+    "manifest for",
+    "not found",
+    "pull access denied",
+    "repository does not exist",
+)
+
+
+def _compose_with_retry(
+    *args: str,
+    compose_file: Path = COMPOSE_FILE,
+    retries: int = 3,
+    delay: float = 5.0,
+) -> None:
+    """Run a docker compose command, retrying on transient failures."""
+    last_exc: subprocess.CalledProcessError | None = None
+    cmd = ["docker", "compose", "-f", str(compose_file), *args]
+    for attempt in range(1, retries + 1):
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        if result.returncode == 0:
+            if result.stdout:
+                print(result.stdout, end="")
+            return
+        last_exc = subprocess.CalledProcessError(
+            result.returncode, cmd, result.stdout, result.stderr
+        )
+        stderr_lower = (result.stderr or "").lower()
+        if any(pat in stderr_lower for pat in _PERMANENT_ERRORS):
+            console.print(f"[red]Permanent Docker error — not retrying:[/]\n{result.stderr.strip()}")
+            raise last_exc
+        if attempt < retries:
+            console.print(
+                f"[yellow]docker compose {' '.join(args)} failed "
+                f"(attempt {attempt}/{retries}), retrying in {delay:.0f}s…[/]"
+            )
+            if result.stderr:
+                console.print(f"[dim]{result.stderr.strip()}[/]")
+            time.sleep(delay)
+            delay *= 2
+        else:
+            if result.stderr:
+                console.print(f"[red]{result.stderr.strip()}[/]")
+    raise last_exc
+
+
 def deploy(config: DecnetConfig, dry_run: bool = False, no_cache: bool = False) -> None:
     client = docker.from_env()
 
@@ -66,8 +113,8 @@ def deploy(config: DecnetConfig, dry_run: bool = False, no_cache: bool = False)
     # --- Bring up ---
     console.print("[bold cyan]Building images and starting deckies...[/]")
     if no_cache:
-        _compose("build", "--no-cache", compose_file=compose_path)
+        _compose_with_retry("build", "--no-cache", compose_file=compose_path)
-    _compose("up", "--build", "-d", compose_file=compose_path)
+    _compose_with_retry("up", "--build", "-d", compose_file=compose_path)
 
     # --- Status summary ---
     _print_status(config)

decnet/services/elasticsearch.py

@@ -1,23 +1,27 @@
+from pathlib import Path
+
 from decnet.services.base import BaseService
 
+TEMPLATES_DIR = Path(__file__).parent.parent.parent / "templates" / "elasticsearch"
+
 
 class ElasticsearchService(BaseService):
     name = "elasticsearch"
     ports = [9200]
-    default_image = "dtagdevsec/elasticpot"
+    default_image = "build"
 
     def compose_fragment(self, decky_name: str, log_target: str | None = None) -> dict:
-        env: dict = {
+        fragment: dict = {
-            "ELASTICPOT_HOSTNAME": decky_name,
+            "build": {"context": str(TEMPLATES_DIR)},
-        }
-        if log_target:
-            env["ELASTICPOT_LOG_TARGET"] = log_target
-        return {
-            "image": "dtagdevsec/elasticpot",
             "container_name": f"{decky_name}-elasticsearch",
             "restart": "unless-stopped",
-            "environment": env,
+            "environment": {
+                "HONEYPOT_NAME": decky_name,
+            },
         }
+        if log_target:
+            fragment["environment"]["LOG_TARGET"] = log_target
+        return fragment
 
-    def dockerfile_context(self):
+    def dockerfile_context(self) -> Path:
-        return None
+        return TEMPLATES_DIR

decnet/services/smtp.py

@@ -1,25 +1,28 @@
+from pathlib import Path
+
 from decnet.services.base import BaseService
 
+TEMPLATES_DIR = Path(__file__).parent.parent.parent / "templates" / "smtp"
+
 
 class SMTPService(BaseService):
     name = "smtp"
     ports = [25, 587]
-    default_image = "dtagdevsec/mailoney"
+    default_image = "build"
 
     def compose_fragment(self, decky_name: str, log_target: str | None = None) -> dict:
-        env: dict = {
+        fragment: dict = {
-            "MAILONEY_HOSTNAME": decky_name,
+            "build": {"context": str(TEMPLATES_DIR)},
-            "MAILONEY_PORTS": "25,587",
-        }
-        if log_target:
-            env["MAILONEY_LOG_TARGET"] = log_target
-        return {
-            "image": "dtagdevsec/mailoney",
             "container_name": f"{decky_name}-smtp",
             "restart": "unless-stopped",
             "cap_add": ["NET_BIND_SERVICE"],
-            "environment": env,
+            "environment": {
+                "HONEYPOT_NAME": decky_name,
+            },
         }
+        if log_target:
+            fragment["environment"]["LOG_TARGET"] = log_target
+        return fragment
 
-    def dockerfile_context(self):
+    def dockerfile_context(self) -> Path:
-        return None
+        return TEMPLATES_DIR

templates/elasticsearch/Dockerfile

@@ -0,0 +1,13 @@
+ARG BASE_IMAGE=debian:bookworm-slim
+FROM ${BASE_IMAGE}
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    python3 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY elasticsearch_honeypot.py /opt/elasticsearch_honeypot.py
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+EXPOSE 9200
+ENTRYPOINT ["/entrypoint.sh"]

templates/elasticsearch/elasticsearch_honeypot.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""
+Elasticsearch honeypot — presents a convincing ES 7.x HTTP API on port 9200.
+Logs all requests (especially recon probes like /_cat/, /_cluster/, /_nodes/)
+as JSON. Designed to attract automated scanners and credential stuffers.
+"""
+
+import json
+import os
+import socket
+from datetime import datetime, timezone
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+HONEYPOT_NAME = os.environ.get("HONEYPOT_NAME", "esserver")
+LOG_TARGET = os.environ.get("LOG_TARGET", "")
+
+_CLUSTER_UUID = "xC3Pr9abTq2mNkOeLvXwYA"
+_NODE_UUID = "dJH7Lm2sRqWvPn0kFiEtBo"
+
+_ROOT_RESPONSE = {
+    "name": HONEYPOT_NAME,
+    "cluster_name": "elasticsearch",
+    "cluster_uuid": _CLUSTER_UUID,
+    "version": {
+        "number": "7.17.9",
+        "build_flavor": "default",
+        "build_type": "docker",
+        "build_hash": "ef48222227ee6b9e70e502f0f0daa52435ee634d",
+        "build_date": "2023-01-31T05:34:43.305517834Z",
+        "build_snapshot": False,
+        "lucene_version": "8.11.1",
+        "minimum_wire_compatibility_version": "6.8.0",
+        "minimum_index_compatibility_version": "6.0.0-beta1",
+    },
+    "tagline": "You Know, for Search",
+}
+
+
+def _forward(event: dict) -> None:
+    if not LOG_TARGET:
+        return
+    try:
+        host, port = LOG_TARGET.rsplit(":", 1)
+        with socket.create_connection((host, int(port)), timeout=3) as s:
+            s.sendall((json.dumps(event) + "\n").encode())
+    except Exception:
+        pass
+
+
+def _log(event_type: str, **kwargs) -> None:
+    event = {
+        "ts": datetime.now(timezone.utc).isoformat(),
+        "service": "elasticsearch",
+        "host": HONEYPOT_NAME,
+        "event": event_type,
+        **kwargs,
+    }
+    print(json.dumps(event), flush=True)
+    _forward(event)
+
+
+class ESHandler(BaseHTTPRequestHandler):
+    server_version = "elasticsearch"
+    sys_version = ""
+
+    def _send_json(self, code: int, data: dict) -> None:
+        body = json.dumps(data).encode()
+        self.send_response(code)
+        self.send_header("Content-Type", "application/json; charset=UTF-8")
+        self.send_header("Content-Length", str(len(body)))
+        self.send_header("X-elastic-product", "Elasticsearch")
+        self.end_headers()
+        self.wfile.write(body)
+
+    def _read_body(self) -> str:
+        length = int(self.headers.get("Content-Length", 0))
+        return self.rfile.read(length).decode(errors="replace") if length else ""
+
+    def do_GET(self):
+        src = self.client_address[0]
+        path = self.path.split("?")[0]
+
+        if path in ("/", ""):
+            _log("root_probe", src=src, method="GET", path=self.path)
+            self._send_json(200, _ROOT_RESPONSE)
+        elif path.startswith("/_cat/"):
+            _log("cat_api", src=src, method="GET", path=self.path)
+            self._send_json(200, [])
+        elif path.startswith("/_cluster/"):
+            _log("cluster_recon", src=src, method="GET", path=self.path)
+            self._send_json(200, {"cluster_name": "elasticsearch", "status": "green",
+                                   "number_of_nodes": 3, "number_of_data_nodes": 3})
+        elif path.startswith("/_nodes"):
+            _log("nodes_recon", src=src, method="GET", path=self.path)
+            self._send_json(200, {"_nodes": {"total": 3, "successful": 3, "failed": 0}, "nodes": {}})
+        elif path.startswith("/_security/") or path.startswith("/_xpack/"):
+            _log("security_probe", src=src, method="GET", path=self.path)
+            self._send_json(200, {"enabled": True, "available": True})
+        else:
+            _log("request", src=src, method="GET", path=self.path)
+            self._send_json(404, {"error": {"root_cause": [{"type": "index_not_found_exception",
+                                                             "reason": "no such index"}]}})
+
+    def do_POST(self):
+        src = self.client_address[0]
+        body = self._read_body()
+        path = self.path.split("?")[0]
+        _log("post_request", src=src, method="POST", path=self.path,
+             body_preview=body[:300], user_agent=self.headers.get("User-Agent", ""))
+        if "_search" in path or "_bulk" in path:
+            self._send_json(200, {"took": 1, "timed_out": False, "hits": {"total": {"value": 0}, "hits": []}})
+        else:
+            self._send_json(200, {"result": "created", "_id": "1", "_index": "honeypot"})
+
+    def do_PUT(self):
+        src = self.client_address[0]
+        body = self._read_body()
+        _log("put_request", src=src, method="PUT", path=self.path, body_preview=body[:300])
+        self._send_json(200, {"acknowledged": True})
+
+    def do_DELETE(self):
+        src = self.client_address[0]
+        _log("delete_request", src=src, method="DELETE", path=self.path)
+        self._send_json(200, {"acknowledged": True})
+
+    def do_HEAD(self):
+        src = self.client_address[0]
+        _log("head_request", src=src, method="HEAD", path=self.path)
+        self._send_json(200, {})
+
+    def log_message(self, fmt, *args):
+        pass  # suppress default HTTP server logging
+
+
+if __name__ == "__main__":
+    _log("startup", msg=f"Elasticsearch honeypot starting as {HONEYPOT_NAME}")
+    server = HTTPServer(("0.0.0.0", 9200), ESHandler)
+    server.serve_forever()

templates/elasticsearch/entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -e
+exec python3 /opt/elasticsearch_honeypot.py

templates/smtp/Dockerfile

@@ -0,0 +1,13 @@
+ARG BASE_IMAGE=debian:bookworm-slim
+FROM ${BASE_IMAGE}
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    python3 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY smtp_honeypot.py /opt/smtp_honeypot.py
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+EXPOSE 25 587
+ENTRYPOINT ["/entrypoint.sh"]

templates/smtp/entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -e
+exec python3 /opt/smtp_honeypot.py

templates/smtp/smtp_honeypot.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""
+SMTP honeypot — emulates a realistic ESMTP server (Postfix-style).
+Logs EHLO/AUTH/MAIL FROM/RCPT TO attempts as JSON, then denies auth.
+"""
+
+import asyncio
+import json
+import os
+import socket
+from datetime import datetime, timezone
+
+HONEYPOT_NAME = os.environ.get("HONEYPOT_NAME", "mailserver")
+LOG_TARGET = os.environ.get("LOG_TARGET", "")
+
+
+def _forward(event: dict) -> None:
+    if not LOG_TARGET:
+        return
+    try:
+        host, port = LOG_TARGET.rsplit(":", 1)
+        with socket.create_connection((host, int(port)), timeout=3) as s:
+            s.sendall((json.dumps(event) + "\n").encode())
+    except Exception:
+        pass
+
+
+def _log(event_type: str, **kwargs) -> None:
+    event = {
+        "ts": datetime.now(timezone.utc).isoformat(),
+        "service": "smtp",
+        "host": HONEYPOT_NAME,
+        "event": event_type,
+        **kwargs,
+    }
+    print(json.dumps(event), flush=True)
+    _forward(event)
+
+
+class SMTPProtocol(asyncio.Protocol):
+    def __init__(self):
+        self._transport = None
+        self._peer = ("?", 0)
+        self._buf = b""
+
+    def connection_made(self, transport):
+        self._transport = transport
+        self._peer = transport.get_extra_info("peername", ("?", 0))
+        _log("connect", src=self._peer[0], src_port=self._peer[1])
+        transport.write(f"220 {HONEYPOT_NAME} ESMTP Postfix (Debian/GNU)\r\n".encode())
+
+    def data_received(self, data):
+        self._buf += data
+        while b"\r\n" in self._buf:
+            line, self._buf = self._buf.split(b"\r\n", 1)
+            self._handle_line(line.decode(errors="replace").strip())
+
+    def _handle_line(self, line: str) -> None:
+        cmd = line.split()[0].upper() if line.split() else ""
+
+        if cmd in ("EHLO", "HELO"):
+            domain = line.split(None, 1)[1] if " " in line else ""
+            _log("ehlo", src=self._peer[0], domain=domain)
+            self._transport.write(
+                f"250-{HONEYPOT_NAME}\r\n"
+                f"250-PIPELINING\r\n"
+                f"250-SIZE 10240000\r\n"
+                f"250-VRFY\r\n"
+                f"250-ETRN\r\n"
+                f"250-AUTH PLAIN LOGIN\r\n"
+                f"250-ENHANCEDSTATUSCODES\r\n"
+                f"250-8BITMIME\r\n"
+                f"250 DSN\r\n".encode()
+            )
+        elif cmd == "AUTH":
+            _log("auth_attempt", src=self._peer[0], command=line)
+            self._transport.write(b"535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6\r\n")
+            self._transport.close()
+        elif cmd == "MAIL":
+            _log("mail_from", src=self._peer[0], value=line)
+            self._transport.write(b"250 2.1.0 Ok\r\n")
+        elif cmd == "RCPT":
+            _log("rcpt_to", src=self._peer[0], value=line)
+            self._transport.write(b"250 2.1.5 Ok\r\n")
+        elif cmd == "DATA":
+            self._transport.write(b"354 End data with <CR><LF>.<CR><LF>\r\n")
+        elif cmd == "VRFY":
+            _log("vrfy", src=self._peer[0], value=line)
+            self._transport.write(b"252 2.0.0 Cannot VRFY user\r\n")
+        elif cmd == "QUIT":
+            self._transport.write(b"221 2.0.0 Bye\r\n")
+            self._transport.close()
+        elif cmd == "NOOP":
+            self._transport.write(b"250 2.0.0 Ok\r\n")
+        elif cmd == "RSET":
+            self._transport.write(b"250 2.0.0 Ok\r\n")
+        elif cmd == "STARTTLS":
+            # Pretend we don't support upgrading mid-session
+            self._transport.write(b"454 4.7.0 TLS not available due to local problem\r\n")
+        else:
+            _log("unknown_command", src=self._peer[0], command=line)
+            self._transport.write(b"502 5.5.2 Error: command not recognized\r\n")
+
+    def connection_lost(self, exc):
+        _log("disconnect", src=self._peer[0] if self._peer else "?")
+
+
+async def main():
+    _log("startup", msg=f"SMTP honeypot starting as {HONEYPOT_NAME}")
+    loop = asyncio.get_running_loop()
+    server = await loop.create_server(SMTPProtocol, "0.0.0.0", 25)
+    async with server:
+        await server.serve_forever()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/test_composer.py

@@ -20,13 +20,13 @@ APT_COMPATIBLE = {
 }
 
 BUILD_SERVICES = [
-    "http", "rdp", "smb", "ftp", "pop3", "imap",
+    "http", "rdp", "smb", "ftp", "smtp", "elasticsearch",
-    "mysql", "mssql", "redis", "mongodb", "postgres",
+    "pop3", "imap", "mysql", "mssql", "redis", "mongodb", "postgres",
     "ldap", "vnc", "docker_api", "k8s", "sip",
     "mqtt", "llmnr", "snmp", "tftp",
 ]
 
-UPSTREAM_SERVICES = ["ssh", "telnet", "smtp", "elasticsearch", "conpot"]
+UPSTREAM_SERVICES = ["ssh", "telnet", "conpot"]
 
 
 def _make_config(services, distro="debian", base_image=None, build_base=None):

tests/test_services.py

@@ -31,11 +31,9 @@ def _is_build_service(name: str) -> bool:
 # ---------------------------------------------------------------------------
 
 UPSTREAM_SERVICES = {
-    "ssh":           ("cowrie/cowrie",       [22, 2222]),
+    "ssh":    ("cowrie/cowrie",  [22, 2222]),
-    "telnet":        ("cowrie/cowrie",       [23]),
+    "telnet": ("cowrie/cowrie",  [23]),
-    "smtp":          ("dtagdevsec/mailoney", [25, 587]),
+    "conpot": ("honeynet/conpot", [502, 161, 80]),
-    "elasticsearch": ("dtagdevsec/elasticpot", [9200]),
-    "conpot":        ("honeynet/conpot",     [502, 161, 80]),
 }
 
 # ---------------------------------------------------------------------------
@@ -43,26 +41,28 @@ UPSTREAM_SERVICES = {
 # ---------------------------------------------------------------------------
 
 BUILD_SERVICES = {
-    "http":       ([80, 443],    "http"),
+    "http":          ([80, 443],    "http"),
-    "rdp":        ([3389],       "rdp"),
+    "rdp":           ([3389],       "rdp"),
-    "smb":        ([445, 139],   "smb"),
+    "smb":           ([445, 139],   "smb"),
-    "ftp":        ([21],         "ftp"),
+    "ftp":           ([21],         "ftp"),
-    "pop3":       ([110, 995],   "pop3"),
+    "smtp":          ([25, 587],    "smtp"),
-    "imap":       ([143, 993],   "imap"),
+    "elasticsearch": ([9200],       "elasticsearch"),
-    "mysql":      ([3306],       "mysql"),
+    "pop3":          ([110, 995],   "pop3"),
-    "mssql":      ([1433],       "mssql"),
+    "imap":          ([143, 993],   "imap"),
-    "redis":      ([6379],       "redis"),
+    "mysql":         ([3306],       "mysql"),
-    "mongodb":    ([27017],      "mongodb"),
+    "mssql":         ([1433],       "mssql"),
-    "postgres":   ([5432],       "postgres"),
+    "redis":         ([6379],       "redis"),
-    "ldap":       ([389, 636],   "ldap"),
+    "mongodb":       ([27017],      "mongodb"),
-    "vnc":        ([5900],       "vnc"),
+    "postgres":      ([5432],       "postgres"),
-    "docker_api": ([2375, 2376], "docker_api"),
+    "ldap":          ([389, 636],   "ldap"),
-    "k8s":        ([6443, 8080], "k8s"),
+    "vnc":           ([5900],       "vnc"),
-    "sip":        ([5060],       "sip"),
+    "docker_api":    ([2375, 2376], "docker_api"),
-    "mqtt":       ([1883],       "mqtt"),
+    "k8s":           ([6443, 8080], "k8s"),
-    "llmnr":      ([5355, 5353], "llmnr"),
+    "sip":           ([5060],       "sip"),
-    "snmp":       ([161],        "snmp"),
+    "mqtt":          ([1883],       "mqtt"),
-    "tftp":       ([69],         "tftp"),
+    "llmnr":         ([5355, 5353], "llmnr"),
+    "snmp":          ([161],        "snmp"),
+    "tftp":          ([69],         "tftp"),
 }
 
 ALL_SERVICE_NAMES = list(UPSTREAM_SERVICES) + list(BUILD_SERVICES)