From 862e4dbb318e30cecaa850f62108df748bc549a8 Mon Sep 17 00:00:00 2001 From: anti Date: Tue, 28 Apr 2026 18:36:00 -0400 Subject: [PATCH] =?UTF-8?q?merge:=20testing=20=E2=86=92=20main=20(reconcil?= =?UTF-8?q?e=202-week=20divergence)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/ci.yml | 147 +- .gitea/workflows/release.yml | 22 +- .gitignore | 34 +- CLAUDE.md | 58 - GEMINI.md | 104 - LICENSE | 674 +++++ README.md | 112 + decnet.collector.log | 1 - decnet.ini.example | 64 + decnet/__init__.py | 12 + decnet/agent/__init__.py | 7 + decnet/agent/app.py | 320 +++ decnet/agent/executor.py | 223 ++ decnet/agent/heartbeat.py | 146 ++ decnet/agent/server.py | 70 + decnet/agent/topology_ops.py | 208 ++ decnet/agent/topology_store.py | 213 ++ decnet/asn/__init__.py | 92 + decnet/asn/base.py | 33 + decnet/asn/factory.py | 39 + decnet/asn/iptoasn/__init__.py | 9 + decnet/asn/iptoasn/fetch.py | 63 + decnet/asn/iptoasn/parse.py | 78 + decnet/asn/iptoasn/provider.py | 83 + decnet/asn/lookup.py | 126 + decnet/asn/paths.py | 18 + decnet/bus/__init__.py | 18 + decnet/bus/app.py | 92 + decnet/bus/base.py | 205 ++ decnet/bus/factory.py | 85 + decnet/bus/fake.py | 183 ++ decnet/bus/protocol.py | 144 ++ decnet/bus/publish.py | 211 ++ decnet/bus/topics.py | 398 +++ decnet/bus/unix_client.py | 257 ++ decnet/bus/unix_server.py | 309 +++ decnet/bus/worker.py | 121 + decnet/canary/__init__.py | 37 + decnet/canary/base.py | 145 ++ decnet/canary/cultivator.py | 181 ++ decnet/canary/dns_server.py | 207 ++ decnet/canary/factory.py | 141 ++ decnet/canary/generators/__init__.py | 7 + decnet/canary/generators/aws_creds.py | 86 + decnet/canary/generators/env_file.py | 56 + decnet/canary/generators/git_config.py | 53 + decnet/canary/generators/honeydoc.py | 61 + decnet/canary/generators/honeydoc_docx.py | 133 + decnet/canary/generators/honeydoc_pdf.py | 127 + decnet/canary/generators/mysql_dump.py | 190 ++ decnet/canary/generators/ssh_key.py | 68 + decnet/canary/instrumenters/__init__.py | 4 + decnet/canary/instrumenters/docx.py | 147 ++ decnet/canary/instrumenters/html.py | 45 + decnet/canary/instrumenters/image.py | 72 + decnet/canary/instrumenters/passthrough.py | 37 + decnet/canary/instrumenters/pdf.py | 76 + decnet/canary/instrumenters/plain.py | 79 + decnet/canary/instrumenters/xlsx.py | 95 + decnet/canary/paths.py | 82 + decnet/canary/planter.py | 301 +++ decnet/canary/storage.py | 89 + decnet/canary/worker.py | 254 ++ decnet/cli.py | 478 ---- decnet/cli/__init__.py | 90 + decnet/cli/agent.py | 64 + decnet/cli/api.py | 53 + decnet/cli/bus.py | 45 + decnet/cli/canary.py | 42 + decnet/cli/db.py | 141 ++ decnet/cli/deploy.py | 307 +++ decnet/cli/forwarder.py | 74 + decnet/cli/gating.py | 73 + decnet/cli/geoip.py | 59 + decnet/cli/init.py | 843 +++++++ decnet/cli/inventory.py | 52 + decnet/cli/lifecycle.py | 147 ++ decnet/cli/listener.py | 57 + decnet/cli/orchestrator.py | 55 + decnet/cli/profiler.py | 34 + decnet/cli/realism.py | 111 + decnet/cli/reconciler.py | 62 + decnet/cli/sniffer.py | 31 + decnet/cli/swarm.py | 346 +++ decnet/cli/swarmctl.py | 104 + decnet/cli/topology.py | 348 +++ decnet/cli/updater.py | 46 + decnet/cli/utils.py | 217 ++ decnet/cli/web.py | 153 ++ decnet/cli/webhook.py | 35 + decnet/cli/workers.py | 297 +++ decnet/clustering/__init__.py | 1 + decnet/clustering/base.py | 83 + decnet/clustering/campaign/__init__.py | 5 + decnet/clustering/campaign/base.py | 66 + decnet/clustering/campaign/factory.py | 31 + decnet/clustering/campaign/impl/__init__.py | 0 .../campaign/impl/connected_components.py | 304 +++ decnet/clustering/campaign/impl/similarity.py | 441 ++++ decnet/clustering/campaign/worker.py | 191 ++ decnet/clustering/factory.py | 46 + decnet/clustering/impl/__init__.py | 6 + .../clustering/impl/connected_components.py | 379 +++ decnet/clustering/impl/similarity.py | 313 +++ decnet/clustering/ukc.py | 108 + decnet/clustering/worker.py | 180 ++ decnet/collector/worker.py | 489 +++- decnet/composer.py | 15 + decnet/config.py | 56 +- decnet/config_ini.py | 208 ++ decnet/correlation/engine.py | 179 +- decnet/correlation/event_kinds.py | 113 + decnet/correlation/graph.py | 65 +- decnet/correlation/parser.py | 59 +- decnet/correlation/reuse_worker.py | 153 ++ decnet/engine/deployer.py | 788 +++++- decnet/engine/reaper.py | 171 ++ decnet/env.py | 217 +- decnet/{fleet.py => fleet/__init__.py} | 7 +- decnet/fleet/reconciler.py | 231 ++ decnet/fleet/reconciler_worker.py | 86 + decnet/geoip/__init__.py | 95 + decnet/geoip/base.py | 34 + decnet/geoip/factory.py | 47 + decnet/geoip/lookup.py | 121 + decnet/geoip/paths.py | 19 + decnet/geoip/ptr.py | 87 + decnet/geoip/rir/__init__.py | 9 + decnet/geoip/rir/fetch.py | 62 + decnet/geoip/rir/parse.py | 70 + decnet/geoip/rir/provider.py | 74 + decnet/intel/__init__.py | 10 + decnet/intel/abuseipdb.py | 104 + decnet/intel/base.py | 80 + decnet/intel/factory.py | 73 + decnet/intel/feodo.py | 108 + decnet/intel/greynoise.py | 107 + decnet/intel/threatfox.py | 94 + decnet/intel/worker.py | 233 ++ decnet/logging/__init__.py | 92 + decnet/logging/file_handler.py | 24 +- decnet/logging/forwarder.py | 3 + decnet/logging/inode_aware_handler.py | 60 + decnet/logging/syslog_formatter.py | 4 +- decnet/models.py | 3 + decnet/mutator/engine.py | 369 ++- decnet/mutator/events.py | 108 + decnet/mutator/ops.py | 440 ++++ decnet/net/__init__.py | 7 + decnet/net/http.py | 59 + decnet/network.py | 173 +- decnet/orchestrator/__init__.py | 9 + decnet/orchestrator/drivers/__init__.py | 74 + decnet/orchestrator/drivers/base.py | 92 + decnet/orchestrator/drivers/email.py | 290 +++ decnet/orchestrator/drivers/ssh.py | 293 +++ decnet/orchestrator/emailgen/__init__.py | 20 + decnet/orchestrator/emailgen/events.py | 49 + decnet/orchestrator/emailgen/scheduler.py | 255 ++ decnet/orchestrator/emailgen/threads.py | 75 + decnet/orchestrator/events.py | 68 + decnet/orchestrator/scheduler.py | 340 +++ decnet/orchestrator/worker.py | 513 ++++ decnet/privdrop.py | 67 + decnet/prober/__init__.py | 13 + decnet/prober/hassh.py | 252 ++ decnet/prober/jarm.py | 506 ++++ decnet/prober/osfp/__init__.py | 27 + decnet/prober/osfp/base.py | 59 + decnet/prober/osfp/factory.py | 87 + decnet/prober/osfp/p0f/__init__.py | 6 + .../prober/osfp/p0f/data/LICENSE.p0f-upstream | 498 ++++ decnet/prober/osfp/p0f/data/README.md | 72 + decnet/prober/osfp/p0f/data/p0f.fp | 834 +++++++ decnet/prober/osfp/p0f/data/p0fa.fp | 208 ++ decnet/prober/osfp/p0f/data/p0fo.fp | 48 + decnet/prober/osfp/p0f/data/p0fr.fp | 193 ++ decnet/prober/osfp/p0f/format.py | 243 ++ decnet/prober/osfp/p0f/provider.py | 109 + decnet/prober/osfp/p0f/signature.py | 287 +++ decnet/prober/tcpfp.py | 239 ++ decnet/prober/tlscert.py | 131 + decnet/prober/worker.py | 628 +++++ decnet/profiler/__init__.py | 5 + decnet/profiler/behavioral.py | 107 + decnet/profiler/classify.py | 57 + decnet/profiler/fingerprint.py | 296 +++ decnet/profiler/identity_rollup.py | 109 + decnet/profiler/phases.py | 68 + decnet/profiler/timing.py | 82 + decnet/profiler/tools.py | 179 ++ decnet/profiler/worker.py | 442 ++++ decnet/realism/__init__.py | 27 + decnet/realism/bodies.py | 421 ++++ decnet/realism/diurnal.py | 152 ++ decnet/realism/llm/__init__.py | 17 + decnet/realism/llm/base.py | 47 + decnet/realism/llm/circuit.py | 99 + decnet/realism/llm/factory.py | 46 + decnet/realism/llm/impl/__init__.py | 6 + decnet/realism/llm/impl/fake.py | 50 + decnet/realism/llm/impl/ollama.py | 100 + decnet/realism/naming.py | 184 ++ decnet/realism/personas.py | 153 ++ decnet/realism/personas_pool.py | 145 ++ decnet/realism/planner.py | 368 +++ decnet/realism/prompts/__init__.py | 9 + decnet/realism/prompts/_style.py | 39 + decnet/realism/prompts/email.py | 154 ++ decnet/realism/prompts/filebody.py | 91 + decnet/realism/taxonomy.py | 150 ++ decnet/services/base.py | 1 + decnet/services/conpot.py | 2 +- decnet/services/docker_api.py | 2 +- decnet/services/elasticsearch.py | 2 +- decnet/services/ftp.py | 2 +- decnet/services/http.py | 2 +- decnet/services/https.py | 59 + decnet/services/imap.py | 2 +- decnet/services/k8s.py | 2 +- decnet/services/ldap.py | 2 +- decnet/services/llmnr.py | 2 +- decnet/services/mongodb.py | 2 +- decnet/services/mqtt.py | 2 +- decnet/services/mssql.py | 2 +- decnet/services/mysql.py | 2 +- decnet/services/pop3.py | 2 +- decnet/services/postgres.py | 2 +- decnet/services/rdp.py | 7 +- decnet/services/redis.py | 2 +- decnet/services/sip.py | 2 +- decnet/services/smb.py | 2 +- decnet/services/smtp.py | 11 +- decnet/services/smtp_relay.py | 9 +- decnet/services/sniffer.py | 41 + decnet/services/snmp.py | 2 +- decnet/services/ssh.py | 16 +- decnet/services/telnet.py | 13 +- decnet/services/tftp.py | 2 +- decnet/services/vnc.py | 2 +- decnet/sniffer/__init__.py | 11 + decnet/sniffer/fingerprint.py | 1276 ++++++++++ decnet/sniffer/p0f.py | 238 ++ decnet/sniffer/seq_class.py | 63 + decnet/sniffer/syslog.py | 71 + decnet/sniffer/worker.py | 243 ++ decnet/swarm/__init__.py | 7 + decnet/swarm/client.py | 323 +++ decnet/swarm/log_forwarder.py | 318 +++ decnet/swarm/log_listener.py | 194 ++ decnet/swarm/pki.py | 323 +++ decnet/swarm/tar_tree.py | 97 + decnet/swarm/updater_client.py | 132 + decnet/telemetry.py | 308 +++ .../_shared/auth-helper/auth-helper.c | 190 ++ decnet/templates/_shared/ntlmssp.py | 132 + decnet/templates/_shared/sessrec/Makefile | 28 + decnet/templates/_shared/sessrec/sessrec.c | 564 +++++ .../templates}/conpot/Dockerfile | 6 +- .../templates}/conpot/entrypoint.py | 4 +- decnet/templates/conpot/instance_seed.py | 120 + decnet/templates/conpot/syslog_bridge.py | 261 ++ .../templates}/cowrie/Dockerfile | 4 +- .../templates}/cowrie/cowrie.cfg.j2 | 0 .../templates}/cowrie/entrypoint.sh | 0 .../templates}/cowrie/honeyfs/etc/group | 0 .../templates}/cowrie/honeyfs/etc/hostname | 0 .../templates}/cowrie/honeyfs/etc/hosts | 0 .../templates}/cowrie/honeyfs/etc/issue | 0 .../templates}/cowrie/honeyfs/etc/issue.net | 0 .../templates}/cowrie/honeyfs/etc/motd | 0 .../templates}/cowrie/honeyfs/etc/os-release | 0 .../templates}/cowrie/honeyfs/etc/passwd | 0 .../templates}/cowrie/honeyfs/etc/resolv.conf | 0 .../templates}/cowrie/honeyfs/etc/shadow | 0 .../honeyfs/home/admin/.aws/credentials | 0 .../cowrie/honeyfs/home/admin/.bash_history | 0 .../honeyfs/home/admin/.ssh/authorized_keys | 0 .../cowrie/honeyfs/root/.bash_history | 0 .../cowrie/honeyfs/var/log/auth.log | 0 .../templates}/docker_api/Dockerfile | 6 +- .../templates}/docker_api/entrypoint.sh | 0 decnet/templates/docker_api/instance_seed.py | 120 + .../templates}/docker_api/server.py | 11 +- decnet/templates/docker_api/syslog_bridge.py | 261 ++ .../templates}/elasticsearch/Dockerfile | 7 +- .../templates}/elasticsearch/entrypoint.sh | 0 .../templates/elasticsearch/instance_seed.py | 120 + decnet/templates/elasticsearch/server.py | 195 ++ .../templates/elasticsearch/syslog_bridge.py | 261 ++ .../templates}/ftp/Dockerfile | 7 +- .../templates}/ftp/entrypoint.sh | 0 decnet/templates/ftp/instance_seed.py | 120 + decnet/templates/ftp/server.py | 144 ++ decnet/templates/ftp/syslog_bridge.py | 261 ++ .../templates}/http/Dockerfile | 7 +- .../templates}/http/entrypoint.sh | 0 decnet/templates/http/instance_seed.py | 120 + .../templates}/http/server.py | 47 +- decnet/templates/http/syslog_bridge.py | 261 ++ decnet/templates/https/Dockerfile | 30 + decnet/templates/https/entrypoint.sh | 18 + decnet/templates/https/instance_seed.py | 120 + decnet/templates/https/server.py | 164 ++ decnet/templates/https/syslog_bridge.py | 261 ++ .../templates}/imap/Dockerfile | 6 +- .../templates}/imap/entrypoint.sh | 0 decnet/templates/imap/instance_seed.py | 120 + .../templates}/imap/server.py | 167 +- decnet/templates/imap/syslog_bridge.py | 261 ++ decnet/templates/instance_seed.py | 120 + .../templates}/k8s/Dockerfile | 6 +- .../templates}/k8s/entrypoint.sh | 0 decnet/templates/k8s/instance_seed.py | 120 + {templates => decnet/templates}/k8s/server.py | 13 +- decnet/templates/k8s/syslog_bridge.py | 261 ++ .../templates}/ldap/Dockerfile | 7 +- .../templates}/ldap/entrypoint.sh | 0 decnet/templates/ldap/instance_seed.py | 120 + .../templates}/ldap/server.py | 78 +- decnet/templates/ldap/syslog_bridge.py | 261 ++ .../templates}/llmnr/Dockerfile | 6 +- .../templates}/llmnr/entrypoint.sh | 0 decnet/templates/llmnr/instance_seed.py | 120 + .../templates}/llmnr/server.py | 3 +- decnet/templates/llmnr/syslog_bridge.py | 261 ++ .../templates}/mongodb/Dockerfile | 7 +- .../templates}/mongodb/entrypoint.sh | 0 decnet/templates/mongodb/instance_seed.py | 120 + decnet/templates/mongodb/server.py | 352 +++ decnet/templates/mongodb/syslog_bridge.py | 261 ++ .../templates}/mqtt/Dockerfile | 7 +- .../templates}/mqtt/entrypoint.sh | 0 decnet/templates/mqtt/instance_seed.py | 120 + .../templates}/mqtt/server.py | 110 +- decnet/templates/mqtt/syslog_bridge.py | 261 ++ .../templates}/mssql/Dockerfile | 7 +- .../templates}/mssql/entrypoint.sh | 0 decnet/templates/mssql/instance_seed.py | 120 + decnet/templates/mssql/server.py | 217 ++ decnet/templates/mssql/syslog_bridge.py | 261 ++ .../templates}/mysql/Dockerfile | 7 +- .../templates}/mysql/entrypoint.sh | 0 decnet/templates/mysql/instance_seed.py | 120 + decnet/templates/mysql/server.py | 167 ++ decnet/templates/mysql/syslog_bridge.py | 261 ++ .../templates}/pop3/Dockerfile | 6 +- .../templates}/pop3/entrypoint.sh | 0 decnet/templates/pop3/instance_seed.py | 120 + .../templates}/pop3/server.py | 109 +- decnet/templates/pop3/syslog_bridge.py | 261 ++ .../templates}/postgres/Dockerfile | 7 +- .../templates}/postgres/entrypoint.sh | 0 decnet/templates/postgres/instance_seed.py | 120 + .../templates}/postgres/server.py | 75 +- decnet/templates/postgres/syslog_bridge.py | 261 ++ .../templates}/rdp/Dockerfile | 15 +- decnet/templates/rdp/entrypoint.sh | 20 + decnet/templates/rdp/instance_seed.py | 120 + decnet/templates/rdp/ntlmssp.py | 132 + decnet/templates/rdp/server.py | 399 +++ decnet/templates/rdp/syslog_bridge.py | 261 ++ .../templates}/redis/Dockerfile | 7 +- .../templates/redis}/entrypoint.sh | 0 decnet/templates/redis/instance_seed.py | 120 + decnet/templates/redis/server.py | 333 +++ decnet/templates/redis/syslog_bridge.py | 261 ++ .../templates}/sip/Dockerfile | 6 +- .../templates/sip}/entrypoint.sh | 0 decnet/templates/sip/instance_seed.py | 120 + {templates => decnet/templates}/sip/server.py | 16 +- decnet/templates/sip/syslog_bridge.py | 261 ++ decnet/templates/smb/Dockerfile | 25 + .../templates/smb}/entrypoint.sh | 0 decnet/templates/smb/instance_seed.py | 120 + decnet/templates/smb/ntlmssp.py | 132 + decnet/templates/smb/server.py | 296 +++ decnet/templates/smb/syslog_bridge.py | 261 ++ .../templates}/smtp/Dockerfile | 7 +- .../templates}/smtp/entrypoint.sh | 0 decnet/templates/smtp/instance_seed.py | 120 + decnet/templates/smtp/server.py | 497 ++++ decnet/templates/smtp/syslog_bridge.py | 261 ++ decnet/templates/sniffer/Dockerfile | 12 + decnet/templates/sniffer/server.py | 1052 ++++++++ .../templates}/snmp/Dockerfile | 6 +- .../templates}/snmp/entrypoint.sh | 0 decnet/templates/snmp/instance_seed.py | 120 + .../templates}/snmp/server.py | 17 +- decnet/templates/snmp/syslog_bridge.py | 261 ++ decnet/templates/ssh/Dockerfile | 157 ++ decnet/templates/ssh/_build_stealth.py | 89 + decnet/templates/ssh/argv_zap.c | 65 + .../templates/ssh/auth-helper/auth-helper.c | 190 ++ decnet/templates/ssh/capture.sh | 265 ++ decnet/templates/ssh/emit_capture.py | 84 + decnet/templates/ssh/entrypoint.sh | 90 + decnet/templates/ssh/instance_seed.py | 120 + decnet/templates/ssh/sessrec/Makefile | 28 + decnet/templates/ssh/sessrec/sessrec.c | 564 +++++ decnet/templates/ssh/syslog_bridge.py | 261 ++ decnet/templates/syslog_bridge.py | 261 ++ decnet/templates/telnet/Dockerfile | 96 + .../telnet/auth-helper/auth-helper.c | 190 ++ .../templates}/telnet/entrypoint.sh | 19 +- decnet/templates/telnet/instance_seed.py | 120 + decnet/templates/telnet/sessrec/Makefile | 28 + decnet/templates/telnet/sessrec/sessrec.c | 564 +++++ decnet/templates/telnet/syslog_bridge.py | 261 ++ .../templates}/tftp/Dockerfile | 6 +- .../templates}/tftp/entrypoint.sh | 0 decnet/templates/tftp/instance_seed.py | 120 + .../templates}/tftp/server.py | 3 +- decnet/templates/tftp/syslog_bridge.py | 261 ++ .../templates}/vnc/Dockerfile | 6 +- .../templates}/vnc/entrypoint.sh | 0 decnet/templates/vnc/instance_seed.py | 120 + {templates => decnet/templates}/vnc/server.py | 15 +- decnet/templates/vnc/syslog_bridge.py | 261 ++ decnet/topology/__init__.py | 23 + decnet/topology/allocator.py | 160 ++ decnet/topology/compose.py | 165 ++ decnet/topology/config.py | 113 + decnet/topology/generator.py | 237 ++ decnet/topology/hashing.py | 65 + decnet/topology/persistence.py | 218 ++ decnet/topology/status.py | 106 + decnet/topology/validate.py | 356 +++ decnet/updater/__init__.py | 10 + decnet/updater/app.py | 186 ++ decnet/updater/executor.py | 766 ++++++ decnet/updater/server.py | 90 + decnet/vectorstore/__init__.py | 27 + decnet/vectorstore/base.py | 114 + decnet/vectorstore/factory.py | 73 + decnet/vectorstore/fake.py | 131 + decnet/vectorstore/sqlite_vec.py | 285 +++ decnet/web/_uvicorn_tls_scope.py | 72 + decnet/web/api.py | 243 +- decnet/web/auth.py | 10 + decnet/web/db/factory.py | 29 +- decnet/web/db/models.py | 95 - decnet/web/db/models/__init__.py | 319 +++ decnet/web/db/models/_base.py | 23 + decnet/web/db/models/attacker_intel.py | 93 + decnet/web/db/models/attackers.py | 414 +++ decnet/web/db/models/auth.py | 73 + decnet/web/db/models/campaigns.py | 83 + decnet/web/db/models/canary.py | 242 ++ decnet/web/db/models/common.py | 15 + decnet/web/db/models/deploy.py | 29 + decnet/web/db/models/fleet.py | 72 + decnet/web/db/models/health.py | 14 + decnet/web/db/models/logs.py | 222 ++ decnet/web/db/models/orchestrator.py | 111 + decnet/web/db/models/realism.py | 107 + decnet/web/db/models/swarm.py | 200 ++ decnet/web/db/models/topology.py | 442 ++++ decnet/web/db/models/updater.py | 73 + decnet/web/db/models/webhooks.py | 162 ++ decnet/web/db/models/workers.py | 50 + decnet/web/db/mysql/__init__.py | 0 decnet/web/db/mysql/database.py | 98 + decnet/web/db/mysql/repository.py | 187 ++ decnet/web/db/repository.py | 1128 +++++++++ decnet/web/db/sqlite/database.py | 33 +- decnet/web/db/sqlite/repository.py | 404 +-- decnet/web/db/sqlmodel_repo/__init__.py | 166 ++ decnet/web/db/sqlmodel_repo/_helpers.py | 113 + decnet/web/db/sqlmodel_repo/attacker_intel.py | 102 + .../db/sqlmodel_repo/attackers/__init__.py | 32 + .../web/db/sqlmodel_repo/attackers/_core.py | 95 + .../db/sqlmodel_repo/attackers/activity.py | 207 ++ .../db/sqlmodel_repo/attackers/behavior.py | 106 + .../db/sqlmodel_repo/attackers/sessions.py | 49 + decnet/web/db/sqlmodel_repo/attackers/smtp.py | 69 + decnet/web/db/sqlmodel_repo/auth.py | 74 + decnet/web/db/sqlmodel_repo/bounties.py | 139 ++ decnet/web/db/sqlmodel_repo/campaigns.py | 173 ++ decnet/web/db/sqlmodel_repo/canary.py | 200 ++ .../db/sqlmodel_repo/credentials/__init__.py | 20 + .../web/db/sqlmodel_repo/credentials/_core.py | 196 ++ .../web/db/sqlmodel_repo/credentials/reuse.py | 275 ++ decnet/web/db/sqlmodel_repo/deckies.py | 91 + decnet/web/db/sqlmodel_repo/fleet.py | 152 ++ decnet/web/db/sqlmodel_repo/identities.py | 185 ++ decnet/web/db/sqlmodel_repo/logs.py | 213 ++ decnet/web/db/sqlmodel_repo/orchestrator.py | 191 ++ decnet/web/db/sqlmodel_repo/realism.py | 157 ++ decnet/web/db/sqlmodel_repo/swarm.py | 71 + .../web/db/sqlmodel_repo/topology/__init__.py | 32 + decnet/web/db/sqlmodel_repo/topology/_core.py | 250 ++ .../web/db/sqlmodel_repo/topology/deckies.py | 125 + decnet/web/db/sqlmodel_repo/topology/edges.py | 74 + decnet/web/db/sqlmodel_repo/topology/lans.py | 118 + .../db/sqlmodel_repo/topology/mutations.py | 171 ++ decnet/web/db/sqlmodel_repo/webhooks.py | 133 + decnet/web/dependencies.py | 140 +- decnet/web/ingester.py | 1057 +++++++- decnet/web/limiter.py | 88 + decnet/web/router/__init__.py | 127 +- decnet/web/router/artifacts/__init__.py | 0 .../web/router/artifacts/api_get_artifact.py | 95 + decnet/web/router/attackers/__init__.py | 0 .../attackers/api_get_attacker_artifacts.py | 34 + .../attackers/api_get_attacker_commands.py | 42 + .../attackers/api_get_attacker_detail.py | 44 + .../attackers/api_get_attacker_intel.py | 38 + .../router/attackers/api_get_attacker_mail.py | 37 + .../api_get_attacker_smtp_targets.py | 36 + .../attackers/api_get_attacker_transcripts.py | 34 + .../web/router/attackers/api_get_attackers.py | 83 + decnet/web/router/auth/api_change_pass.py | 14 +- decnet/web/router/auth/api_login.py | 33 +- decnet/web/router/bounty/api_get_bounties.py | 50 +- decnet/web/router/campaigns/__init__.py | 0 decnet/web/router/campaigns/api_events.py | 123 + .../campaigns/api_get_campaign_detail.py | 40 + .../campaigns/api_list_campaign_identities.py | 41 + .../router/campaigns/api_list_campaigns.py | 35 + decnet/web/router/canary/__init__.py | 23 + decnet/web/router/canary/api_blobs.py | 172 ++ decnet/web/router/canary/api_tokens.py | 318 +++ decnet/web/router/config/__init__.py | 0 decnet/web/router/config/api_get_config.py | 124 + decnet/web/router/config/api_manage_users.py | 144 ++ decnet/web/router/config/api_reinit.py | 29 + decnet/web/router/config/api_update_config.py | 50 + .../web/router/credential_reuse/__init__.py | 0 .../api_get_credential_reuse.py | 74 + decnet/web/router/credentials/__init__.py | 0 .../router/credentials/api_get_credentials.py | 103 + decnet/web/router/fleet/api_deploy_deckies.py | 129 +- decnet/web/router/fleet/api_get_deckies.py | 45 +- decnet/web/router/fleet/api_mutate_decky.py | 15 +- .../web/router/fleet/api_mutate_interval.py | 10 +- decnet/web/router/health/__init__.py | 0 decnet/web/router/health/api_get_health.py | 151 ++ decnet/web/router/identities/__init__.py | 0 decnet/web/router/identities/api_events.py | 143 ++ .../identities/api_get_identity_detail.py | 44 + .../router/identities/api_list_identities.py | 35 + .../api_list_identity_observations.py | 48 + decnet/web/router/logs/api_get_histogram.py | 46 +- decnet/web/router/logs/api_get_logs.py | 46 +- decnet/web/router/orchestrator/__init__.py | 0 decnet/web/router/orchestrator/api_events.py | 123 + .../router/orchestrator/api_list_events.py | 87 + decnet/web/router/realism/__init__.py | 0 decnet/web/router/realism/api_config.py | 115 + decnet/web/router/realism/api_personas.py | 143 ++ .../web/router/realism/api_synthetic_files.py | 99 + decnet/web/router/stats/api_get_stats.py | 46 +- decnet/web/router/stream/api_stream_events.py | 166 +- decnet/web/router/swarm/__init__.py | 47 + decnet/web/router/swarm/api_check_hosts.py | 61 + .../web/router/swarm/api_decommission_host.py | 63 + decnet/web/router/swarm/api_deploy_swarm.py | 155 ++ decnet/web/router/swarm/api_enroll_host.py | 100 + decnet/web/router/swarm/api_get_host.py | 26 + .../web/router/swarm/api_get_swarm_health.py | 11 + decnet/web/router/swarm/api_heartbeat.py | 212 ++ decnet/web/router/swarm/api_list_deckies.py | 55 + decnet/web/router/swarm/api_list_hosts.py | 21 + decnet/web/router/swarm/api_teardown_swarm.py | 60 + decnet/web/router/swarm_mgmt/__init__.py | 26 + .../swarm_mgmt/api_decommission_host.py | 71 + .../router/swarm_mgmt/api_enroll_bundle.py | 504 ++++ .../web/router/swarm_mgmt/api_list_deckies.py | 58 + .../web/router/swarm_mgmt/api_list_hosts.py | 60 + .../router/swarm_mgmt/api_teardown_host.py | 150 ++ decnet/web/router/swarm_updates/__init__.py | 23 + .../swarm_updates/api_list_host_releases.py | 86 + .../router/swarm_updates/api_push_update.py | 163 ++ .../swarm_updates/api_push_update_self.py | 101 + .../router/swarm_updates/api_rollback_host.py | 77 + decnet/web/router/system/__init__.py | 6 + .../web/router/system/api_deployment_mode.py | 41 + decnet/web/router/topology/__init__.py | 55 + decnet/web/router/topology/_guards.py | 53 + decnet/web/router/topology/_target_host.py | 66 + decnet/web/router/topology/api_catalog.py | 140 ++ .../topology/api_create_blank_topology.py | 123 + .../router/topology/api_create_topology.py | 77 + decnet/web/router/topology/api_decky_crud.py | 136 + .../router/topology/api_delete_topology.py | 51 + .../router/topology/api_deploy_topology.py | 76 + decnet/web/router/topology/api_edge_crud.py | 110 + decnet/web/router/topology/api_events.py | 157 ++ .../web/router/topology/api_get_topology.py | 68 + decnet/web/router/topology/api_lan_crud.py | 152 ++ .../router/topology/api_list_topologies.py | 39 + decnet/web/router/topology/api_mutations.py | 127 + decnet/web/router/topology/api_personas.py | 131 + .../web/router/topology/api_reap_orphans.py | 48 + .../router/topology/api_teardown_topology.py | 79 + decnet/web/router/transcripts/__init__.py | 6 + .../router/transcripts/api_get_transcript.py | 243 ++ decnet/web/router/webhooks/__init__.py | 18 + .../router/webhooks/api_manage_webhooks.py | 231 ++ .../web/router/webhooks/api_test_webhook.py | 60 + decnet/web/router/workers/__init__.py | 0 .../web/router/workers/api_control_worker.py | 76 + decnet/web/router/workers/api_list_workers.py | 35 + .../router/workers/api_start_all_workers.py | 102 + decnet/web/router/workers/api_start_worker.py | 72 + decnet/web/services/__init__.py | 0 decnet/web/services/systemd_control.py | 136 + decnet/web/sse_limits.py | 65 + decnet/web/swarm_api.py | 67 + decnet/web/templates/decnet-agent.service.j2 | 18 + .../web/templates/decnet-collector.service.j2 | 20 + decnet/web/templates/decnet-engine.service.j2 | 17 + .../web/templates/decnet-forwarder.service.j2 | 19 + decnet/web/templates/decnet-prober.service.j2 | 20 + .../web/templates/decnet-sniffer.service.j2 | 24 + .../web/templates/decnet-updater.service.j2 | 18 + decnet/web/templates/enroll_bootstrap.sh.j2 | 86 + decnet/web/worker_registry.py | 209 ++ decnet/webhook/__init__.py | 4 + decnet/webhook/client.py | 188 ++ decnet/webhook/enums.py | 54 + decnet/webhook/worker.py | 312 +++ decnet_web/package-lock.json | 101 +- decnet_web/package.json | 1 + decnet_web/src/App.tsx | 207 +- decnet_web/src/components/ArtifactDrawer.tsx | 200 ++ decnet_web/src/components/AttackerDetail.tsx | 2221 +++++++++++++++++ decnet_web/src/components/Attackers.css | 200 ++ decnet_web/src/components/Attackers.tsx | 272 +- decnet_web/src/components/Bounty.css | 238 ++ decnet_web/src/components/Bounty.tsx | 264 +- decnet_web/src/components/BountyInspector.tsx | 100 + decnet_web/src/components/CampaignDetail.tsx | 289 +++ decnet_web/src/components/Campaigns.tsx | 205 ++ .../src/components/CanaryTokenDrawer.tsx | 315 +++ decnet_web/src/components/CanaryTokens.tsx | 731 ++++++ .../CommandPalette/CommandPalette.css | 116 + .../CommandPalette/CommandPalette.tsx | 159 ++ decnet_web/src/components/Config.css | 282 +++ decnet_web/src/components/Config.tsx | 979 +++++++- .../components/CredentialReuseInspector.tsx | 170 ++ decnet_web/src/components/Credentials.css | 274 ++ decnet_web/src/components/Credentials.tsx | 429 ++++ .../src/components/CredentialsInspector.tsx | 142 ++ decnet_web/src/components/Dashboard.css | 478 +++- decnet_web/src/components/Dashboard.tsx | 579 ++++- decnet_web/src/components/DeckyFleet.css | 344 +++ decnet_web/src/components/DeckyFleet.tsx | 1267 ++++++++-- .../src/components/EmptyState/EmptyState.css | 63 + .../src/components/EmptyState/EmptyState.tsx | 46 + decnet_web/src/components/Identities.tsx | 223 ++ decnet_web/src/components/IdentityDetail.tsx | 299 +++ decnet_web/src/components/Layout.css | 181 +- decnet_web/src/components/Layout.tsx | 240 +- decnet_web/src/components/LiveLogs.css | 182 ++ decnet_web/src/components/LiveLogs.tsx | 462 ++-- decnet_web/src/components/Login.tsx | 2 +- decnet_web/src/components/MailDrawer.tsx | 216 ++ decnet_web/src/components/MazeNET/Canvas.tsx | 281 +++ .../src/components/MazeNET/ContextMenu.tsx | 98 + .../src/components/MazeNET/Inspector.tsx | 303 +++ decnet_web/src/components/MazeNET/MazeNET.css | 590 +++++ decnet_web/src/components/MazeNET/MazeNET.tsx | 816 ++++++ decnet_web/src/components/MazeNET/NetBox.tsx | 91 + .../src/components/MazeNET/NodeCard.tsx | 103 + decnet_web/src/components/MazeNET/Palette.tsx | 117 + decnet_web/src/components/MazeNET/data.ts | 98 + decnet_web/src/components/MazeNET/types.ts | 64 + .../src/components/MazeNET/useMazeApi.ts | 415 +++ .../components/MazeNET/useMazeInteraction.ts | 424 ++++ .../components/MazeNET/useMazeLayoutStore.ts | 111 + .../components/MazeNET/useTopologyEditor.ts | 220 ++ .../components/MazeNET/useTopologyStream.ts | 107 + decnet_web/src/components/Modal/Modal.css | 24 + decnet_web/src/components/Modal/Modal.tsx | 85 + decnet_web/src/components/Orchestrator.css | 432 ++++ decnet_web/src/components/Orchestrator.tsx | 350 +++ .../src/components/OrchestratorInspector.tsx | 216 ++ .../src/components/PersonaGeneration.css | 107 + .../src/components/PersonaGeneration.tsx | 875 +++++++ .../RealismConfig/RealismConfig.css | 122 + .../RealismConfig/RealismConfig.tsx | 264 ++ decnet_web/src/components/RemoteUpdates.tsx | 323 +++ decnet_web/src/components/SessionDrawer.tsx | 241 ++ .../ShortcutsHelp/ShortcutsHelp.css | 59 + .../ShortcutsHelp/ShortcutsHelp.tsx | 77 + decnet_web/src/components/Swarm.css | 179 ++ decnet_web/src/components/SwarmHosts.tsx | 510 ++++ .../SyntheticFiles/SyntheticFiles.css | 214 ++ .../SyntheticFiles/SyntheticFiles.tsx | 376 +++ .../src/components/Toasts/ToastProvider.tsx | 33 + decnet_web/src/components/Toasts/Toasts.css | 33 + decnet_web/src/components/Toasts/Toasts.tsx | 44 + .../src/components/Toasts/toast-context.ts | 19 + decnet_web/src/components/Toasts/useToast.ts | 9 + .../TopologyList/CreateTopologyWizard.css | 241 ++ .../TopologyList/CreateTopologyWizard.tsx | 385 +++ .../components/TopologyList/TopologyList.css | 122 + .../components/TopologyList/TopologyList.tsx | 276 ++ decnet_web/src/components/Webhooks.css | 341 +++ decnet_web/src/components/Webhooks.tsx | 642 +++++ .../src/components/useCampaignStream.ts | 100 + .../src/components/useIdentityStream.ts | 113 + .../src/components/useOrchestratorStream.ts | 98 + decnet_web/src/hooks/useEscapeKey.ts | 15 + decnet_web/src/hooks/useFocusSearch.ts | 19 + decnet_web/src/hooks/useFocusTrap.ts | 49 + decnet_web/src/hooks/useGlobalHotkeys.ts | 98 + decnet_web/src/hooks/useSwarmHosts.ts | 42 + decnet_web/src/icons.ts | 104 + decnet_web/src/index.css | 217 +- decnet_web/src/lucide-icons.d.ts | 16 + decnet_web/src/realism/labels.ts | 39 + decnet_web/src/routePrefetch.ts | 40 + decnet_web/src/utils/api.ts | 11 + decnet_web/src/utils/parseEventBody.ts | 44 + decnet_web/vite.config.ts | 34 + deploy/decnet-agent.service.j2 | 44 + deploy/decnet-api.service | 29 - deploy/decnet-api.service.j2 | 49 + deploy/decnet-bus.service.j2 | 49 + deploy/decnet-campaign-clusterer.service.j2 | 52 + deploy/decnet-canary.service.j2 | 46 + deploy/decnet-clusterer.service.j2 | 47 + deploy/decnet-collector.service.j2 | 42 + deploy/decnet-enrich.service.j2 | 47 + deploy/decnet-forwarder.service.j2 | 49 + deploy/decnet-listener.service.j2 | 46 + deploy/decnet-mutator.service.j2 | 41 + deploy/decnet-orchestrator.service.j2 | 50 + deploy/decnet-prober.service.j2 | 39 + deploy/decnet-profiler.service.j2 | 38 + deploy/decnet-reconciler.service.j2 | 47 + deploy/decnet-reuse-correlator.service.j2 | 41 + deploy/decnet-sniffer.service.j2 | 39 + deploy/decnet-swarmctl.service.j2 | 43 + deploy/decnet-updater.service.j2 | 52 + deploy/decnet-web.service | 30 - deploy/decnet-web.service.j2 | 41 + deploy/decnet-webhook.service.j2 | 38 + deploy/decnet.target | 27 + deploy/logrotate.d/decnet | 28 + deploy/polkit/50-decnet-workers.rules.j2 | 23 + deploy/tmpfiles.d/decnet.conf | 4 + development/BUGS.md | 29 + development/CAMPAIGN_CLUSTERING.md | 223 ++ development/DEBT.md | 375 ++- development/DEVELOPMENT.md | 132 +- development/DEVELOPMENT_V2.md | 600 +++++ development/IDENTITY_RESOLUTION.md | 338 +++ development/SIGNAL_CAPTURE_AUDIT.md | 566 +++++ development/THREAT_MODEL.md | 450 ++++ development/api-audit.md | 1000 ++++++++ development/docker-compose.otel.yml | 20 + development/docs/ARCHITECTURE.md | 153 ++ development/docs/TRACING.md | 219 ++ development/docs/services/COLLECTOR.md | 63 + development/docs/services/ENGINE.md | 61 + development/docs/services/MODELS.md | 58 + development/docs/services/WEB_MODELS.md | 134 + development/profiles/profile_1500_fb69a06.csv | 502 ++++ ...file_1500_notracing_12_workers_fb69a06.csv | 502 ++++ .../profile_1500_notracing_fb69a06.csv | 502 ++++ ...ile_1500_notracing_single_core_fb69a06.csv | 203 ++ development/profiles/profile_255c2e5.csv | 27 + development/profiles/profile_2dd86fb.csv | 314 +++ .../profile_3106d0313507f016_locust.csv | 335 +++ development/profiles/profile_e967aaa.csv | 502 ++++ development/profiles/profile_fb69a06.csv | 502 ++++ env.config.example | 91 + pyproject.toml | 100 +- schemathesis.ci.toml | 35 + schemathesis.toml | 86 +- scripts/bus/pub.py | 44 + scripts/bus/smoke-mutator.sh | 90 + scripts/bus/smoke.sh | 57 + scripts/bus/start.sh | 11 + scripts/bus/sub.py | 38 + scripts/decnet-init.sh | 3 + scripts/mock-webhook-receiver.py | 201 ++ scripts/profile/aggregate_requests.py | 192 ++ scripts/profile/classify_usage.py | 105 + scripts/profile/cprofile-cli.sh | 17 + scripts/profile/memray-api.sh | 16 + scripts/profile/pyspy-attach.sh | 32 + scripts/profile/view.sh | 69 + scripts/vulture_whitelist.py | 22 + templates/conpot/decnet_logging.py | 89 - templates/cowrie/decnet_logging.py | 89 - templates/decnet_logging.py | 89 - templates/docker_api/decnet_logging.py | 89 - templates/elasticsearch/decnet_logging.py | 89 - templates/elasticsearch/server.py | 124 - templates/ftp/decnet_logging.py | 89 - templates/ftp/server.py | 76 - templates/http/decnet_logging.py | 89 - templates/imap/decnet_logging.py | 89 - templates/k8s/decnet_logging.py | 89 - templates/ldap/decnet_logging.py | 89 - templates/llmnr/decnet_logging.py | 89 - templates/mongodb/decnet_logging.py | 89 - templates/mongodb/server.py | 128 - templates/mqtt/decnet_logging.py | 89 - templates/mssql/decnet_logging.py | 89 - templates/mssql/server.py | 144 -- templates/mysql/decnet_logging.py | 89 - templates/mysql/server.py | 112 - templates/pop3/decnet_logging.py | 89 - templates/postgres/decnet_logging.py | 89 - templates/rdp/decnet_logging.py | 89 - templates/rdp/server.py | 56 - templates/redis/decnet_logging.py | 89 - templates/redis/server.py | 195 -- templates/sip/decnet_logging.py | 89 - templates/smb/Dockerfile | 26 - templates/smb/decnet_logging.py | 89 - templates/smb/entrypoint.sh | 4 - templates/smb/server.py | 37 - templates/smtp/decnet_logging.py | 89 - templates/smtp/server.py | 255 -- templates/snmp/decnet_logging.py | 89 - templates/ssh/Dockerfile | 76 - templates/ssh/decnet_logging.py | 245 -- templates/ssh/entrypoint.sh | 44 - templates/telnet/Dockerfile | 51 - templates/telnet/decnet_logging.py | 89 - templates/tftp/decnet_logging.py | 89 - templates/vnc/decnet_logging.py | 89 - tests/agent/__init__.py | 0 .../test_topology_apply_recreate_deps.py | 79 + tests/api/artifacts/__init__.py | 0 tests/api/artifacts/test_get_artifact.py | 158 ++ tests/api/auth/test_login.py | 79 +- tests/api/campaigns/__init__.py | 0 tests/api/campaigns/test_events_stream.py | 111 + tests/api/canary/__init__.py | 0 tests/api/canary/test_canary_tokens_api.py | 363 +++ tests/api/config/__init__.py | 0 tests/api/config/conftest.py | 1 + tests/api/config/test_deploy_limit.py | 84 + tests/api/config/test_get_config.py | 69 + tests/api/config/test_reinit.py | 76 + tests/api/config/test_update_config.py | 77 + tests/api/config/test_user_management.py | 188 ++ tests/api/conftest.py | 79 + tests/api/credential_reuse/__init__.py | 0 .../test_get_credential_reuse.py | 228 ++ tests/api/credentials/__init__.py | 0 tests/api/credentials/test_get_credentials.py | 85 + tests/api/fleet/test_deploy_automode.py | 193 ++ tests/api/fleet/test_mutate_decky.py | 6 +- tests/api/health/__init__.py | 0 tests/api/health/test_get_health.py | 198 ++ tests/api/identities/__init__.py | 0 tests/api/identities/test_events_stream.py | 126 + tests/api/orchestrator/__init__.py | 0 tests/api/orchestrator/test_events_stream.py | 237 ++ tests/api/realism/__init__.py | 0 tests/api/realism/test_config_api.py | 109 + tests/api/realism/test_personas_api.py | 170 ++ tests/api/realism/test_synthetic_files_api.py | 146 ++ tests/api/stream/test_stream_events.py | 31 +- tests/api/swarm_mgmt/__init__.py | 0 tests/api/swarm_mgmt/test_enroll_bundle.py | 380 +++ tests/api/swarm_mgmt/test_teardown_host.py | 215 ++ tests/api/swarm_updates/__init__.py | 0 tests/api/swarm_updates/conftest.py | 151 ++ .../swarm_updates/test_list_host_releases.py | 69 + tests/api/swarm_updates/test_push_update.py | 176 ++ .../swarm_updates/test_push_update_self.py | 67 + tests/api/swarm_updates/test_rollback_host.py | 86 + tests/api/test_error_handler.py | 129 + tests/api/test_rbac.py | 116 + tests/api/test_rbac_contract.py | 177 ++ tests/api/test_repository.py | 29 + tests/api/test_schemathesis.py | 130 +- tests/api/test_schemathesis_agent.py | 100 + tests/api/test_schemathesis_swarm.py | 67 + tests/api/test_sse_limits.py | 60 + tests/api/topology/__init__.py | 0 tests/api/topology/test_child_crud.py | 224 ++ tests/api/topology/test_events_stream.py | 140 ++ tests/api/topology/test_models.py | 148 ++ tests/api/topology/test_mutations.py | 203 ++ tests/api/topology/test_personas_api.py | 180 ++ tests/api/topology/test_reads.py | 169 ++ tests/api/topology/test_writes.py | 319 +++ tests/api/transcripts/__init__.py | 0 tests/api/transcripts/test_get_transcript.py | 203 ++ tests/api/webhooks/__init__.py | 0 tests/api/webhooks/test_crud.py | 276 ++ tests/api/workers/__init__.py | 0 tests/api/workers/test_start_workers.py | 179 ++ tests/api/workers/test_workers_api.py | 171 ++ tests/asn/__init__.py | 0 tests/asn/conftest.py | 22 + tests/asn/test_lookup.py | 74 + tests/asn/test_parse.py | 57 + tests/asn/test_profiler_integration.py | 50 + tests/asn/test_provider.py | 95 + tests/bus/__init__.py | 0 tests/bus/conftest.py | 59 + tests/bus/test_app_singleton.py | 135 + tests/bus/test_base.py | 66 + tests/bus/test_closed_publish.py | 103 + tests/bus/test_control_listener.py | 77 + tests/bus/test_factory.py | 52 + tests/bus/test_fake_bus.py | 108 + tests/bus/test_heartbeat.py | 104 + tests/bus/test_protocol.py | 87 + tests/bus/test_publish.py | 64 + tests/bus/test_topics.py | 89 + tests/bus/test_unix_socket_bus.py | 131 + tests/bus/test_worker.py | 68 + tests/canary/__init__.py | 0 tests/canary/conftest.py | 88 + tests/canary/test_cli.py | 24 + tests/canary/test_cultivator.py | 145 ++ tests/canary/test_deploy_hook.py | 80 + tests/canary/test_factory.py | 88 + tests/canary/test_generators.py | 188 ++ tests/canary/test_instrumenters.py | 173 ++ tests/canary/test_models.py | 85 + tests/canary/test_paths.py | 68 + tests/canary/test_planter.py | 247 ++ tests/canary/test_repository.py | 179 ++ tests/canary/test_storage.py | 52 + tests/canary/test_systemd_unit.py | 44 + tests/canary/test_topics.py | 42 + tests/canary/test_worker_dns.py | 119 + tests/canary/test_worker_http.py | 124 + tests/cli/__init__.py | 0 tests/{ => cli}/test_cli.py | 167 +- tests/cli/test_cli_db_reset.py | 134 + tests/{ => cli}/test_cli_service_pool.py | 9 +- tests/cli/test_embedded_workers.py | 45 + tests/cli/test_init.py | 495 ++++ tests/cli/test_mode_gating.py | 89 + tests/cli/test_realism_gating.py | 97 + tests/cli/test_realism_import_personas.py | 129 + tests/cli/test_web_proxy_target.py | 36 + tests/clustering/__init__.py | 0 tests/clustering/fixture_harness.py | 488 ++++ tests/clustering/metrics.py | 179 ++ tests/clustering/test_campaign_factory.py | 318 +++ tests/clustering/test_campaign_similarity.py | 344 +++ tests/clustering/test_campaign_worker.py | 357 +++ tests/clustering/test_clusterer_factory.py | 34 + tests/clustering/test_clusterer_worker.py | 182 ++ tests/clustering/test_connected_components.py | 808 ++++++ .../test_fixtures_campaign_clusterer.py | 278 +++ tests/clustering/test_lone_wolf_fixture.py | 74 + tests/clustering/test_metrics.py | 76 + .../clustering/test_multi_operator_fixture.py | 134 + tests/clustering/test_noise_floor_fixture.py | 167 ++ .../test_paused_campaign_fixture.py | 140 ++ .../test_shared_wordlist_fixture.py | 117 + tests/clustering/test_similarity.py | 348 +++ tests/clustering/test_slow_burn_fixture.py | 128 + tests/clustering/test_vpn_hopping_fixture.py | 126 + tests/collector/__init__.py | 0 tests/collector/test_collector.py | 840 +++++++ tests/collector/test_collector_bus.py | 185 ++ tests/collector/test_collector_thread_pool.py | 94 + tests/config/__init__.py | 0 tests/{ => config}/test_config.py | 0 tests/config/test_config_ini.py | 304 +++ tests/config/test_ini_loader.py | 217 ++ tests/{ => config}/test_ini_spaces.py | 0 tests/{ => config}/test_ini_validation.py | 0 tests/conftest.py | 21 + tests/core/__init__.py | 0 tests/{ => core}/test_build.py | 0 tests/core/test_fingerprinting.py | 446 ++++ tests/{ => core}/test_network.py | 64 +- tests/{ => core}/test_os_fingerprint.py | 0 tests/core/test_privdrop.py | 113 + tests/correlation/__init__.py | 0 tests/correlation/test_bounty_dedup.py | 84 + tests/correlation/test_correlation.py | 606 +++++ tests/correlation/test_correlation_bus.py | 152 ++ tests/correlation/test_credential_reuse.py | 310 +++ tests/correlation/test_event_kinds.py | 91 + tests/db/__init__.py | 0 tests/db/mysql/__init__.py | 0 tests/db/mysql/test_mysql_histogram_sql.py | 70 + tests/db/mysql/test_mysql_migration.py | 234 ++ tests/db/mysql/test_mysql_url_builder.py | 78 + tests/db/test_base_repo.py | 202 ++ tests/db/test_campaign_repo.py | 145 ++ tests/db/test_credential_reuse.py | 226 ++ tests/db/test_credentials.py | 168 ++ tests/db/test_factory.py | 44 + tests/db/test_identity_schema.py | 206 ++ tests/deploy/__init__.py | 0 tests/deploy/test_orchestrator_unit.py | 151 ++ tests/docker/__init__.py | 0 tests/docker/conftest.py | 35 + tests/docker/test_ssh_stealth_image.py | 128 + tests/factories/__init__.py | 0 tests/factories/campaign_factory.py | 455 ++++ .../campaigns/lone_wolf.expected.yaml | 17 + tests/fixtures/campaigns/lone_wolf.yaml | 32 + .../campaigns/multi_operator.expected.yaml | 25 + tests/fixtures/campaigns/multi_operator.yaml | 108 + .../campaigns/noise_floor.expected.yaml | 24 + tests/fixtures/campaigns/noise_floor.yaml | 34 + .../campaigns/paused_campaign.expected.yaml | 24 + tests/fixtures/campaigns/paused_campaign.yaml | 85 + .../campaigns/shared_wordlist.expected.yaml | 21 + tests/fixtures/campaigns/shared_wordlist.yaml | 84 + .../campaigns/slow_burn.expected.yaml | 24 + tests/fixtures/campaigns/slow_burn.yaml | 119 + .../campaigns/vpn_hopping.expected.yaml | 25 + tests/fixtures/campaigns/vpn_hopping.yaml | 55 + tests/fleet/__init__.py | 0 tests/{ => fleet}/test_archetypes.py | 0 tests/fleet/test_auto_spawn.py | 209 ++ tests/{ => fleet}/test_composer.py | 25 + tests/fleet/test_deployer.py | 664 +++++ tests/{ => fleet}/test_fleet.py | 0 tests/fleet/test_fleet_singleton.py | 78 + tests/fleet/test_reconciler.py | 382 +++ tests/fleet/test_reconciler_worker.py | 72 + tests/geoip/__init__.py | 0 tests/geoip/conftest.py | 25 + tests/geoip/test_lookup.py | 76 + tests/geoip/test_parse.py | 66 + tests/geoip/test_profiler_integration.py | 39 + tests/geoip/test_provider.py | 103 + tests/geoip/test_ptr.py | 119 + tests/intel/__init__.py | 0 tests/intel/test_abuseipdb.py | 109 + tests/intel/test_attacker_intel_repo.py | 129 + tests/intel/test_factory.py | 57 + tests/intel/test_feodo.py | 99 + tests/intel/test_greynoise.py | 136 + tests/intel/test_stealth_http.py | 65 + tests/intel/test_threatfox.py | 111 + tests/intel/test_worker.py | 262 ++ tests/live/conftest.py | 17 +- tests/live/test_health_live.py | 248 ++ tests/live/test_https_live.py | 190 ++ tests/live/test_imap_live.py | 2 +- tests/live/test_mqtt_live.py | 10 +- tests/live/test_mysql_backend_live.py | 219 ++ tests/live/test_mysql_live.py | 6 +- tests/live/test_pop3_live.py | 2 +- tests/live/test_postgres_live.py | 7 +- tests/live/test_service_isolation_live.py | 508 ++++ tests/logging/__init__.py | 0 tests/logging/test_file_handler.py | 71 + tests/logging/test_inode_aware_handler.py | 111 + tests/{ => logging}/test_log_file_mount.py | 0 tests/logging/test_logging.py | 155 ++ tests/{ => logging}/test_logging_forwarder.py | 0 tests/logging/test_syslog_formatter.py | 134 + tests/mutator/__init__.py | 0 tests/mutator/test_mutator.py | 400 +++ tests/mysql_spinup.sh | 20 + tests/orchestrator/__init__.py | 0 tests/orchestrator/emailgen/__init__.py | 0 tests/orchestrator/emailgen/test_driver.py | 215 ++ tests/orchestrator/emailgen/test_events.py | 72 + tests/orchestrator/emailgen/test_repo.py | 129 + tests/orchestrator/emailgen/test_scheduler.py | 240 ++ tests/orchestrator/emailgen/test_threads.py | 61 + .../test_realism_config_refresh.py | 74 + .../test_realism_health_snapshot.py | 58 + tests/orchestrator/test_repo_pagination.py | 107 + tests/orchestrator/test_scheduler.py | 204 ++ tests/orchestrator/test_ssh_driver.py | 173 ++ tests/orchestrator/test_worker_integration.py | 294 +++ tests/perf/README.md | 97 + tests/perf/__init__.py | 0 tests/perf/conftest.py | 36 + tests/perf/test_repo_bench.py | 60 + tests/prober/__init__.py | 0 tests/prober/osfp/__init__.py | 0 tests/prober/osfp/test_format.py | 152 ++ tests/prober/osfp/test_provider.py | 177 ++ tests/prober/osfp/test_signature.py | 140 ++ tests/prober/test_prober_bounty.py | 356 +++ tests/prober/test_prober_bus.py | 183 ++ tests/prober/test_prober_hassh.py | 357 +++ tests/prober/test_prober_jarm.py | 272 ++ tests/prober/test_prober_tcpfp.py | 393 +++ tests/prober/test_prober_tlscert.py | 165 ++ tests/prober/test_prober_worker.py | 659 +++++ tests/profiler/__init__.py | 0 tests/profiler/test_attacker_worker.py | 834 +++++++ tests/profiler/test_identity_rollup.py | 141 ++ tests/profiler/test_profiler_behavioral.py | 660 +++++ tests/profiler/test_profiler_bus.py | 145 ++ tests/profiler/test_session_profile.py | 55 + tests/realism/__init__.py | 0 tests/realism/test_bodies.py | 68 + tests/realism/test_bodies_llm.py | 128 + tests/realism/test_circuit_breaker.py | 81 + tests/realism/test_diurnal.py | 120 + tests/realism/test_edit.py | 98 + tests/realism/test_email_prompt.py | 152 ++ tests/realism/test_llm.py | 137 + tests/realism/test_naming.py | 95 + tests/realism/test_personas.py | 127 + tests/realism/test_personas_pool.py | 99 + tests/realism/test_planner.py | 114 + tests/realism/test_planner_config.py | 119 + tests/realism/test_synthetic_files_repo.py | 190 ++ .../test_synthetic_files_truncation.py | 91 + tests/realism/test_taxonomy.py | 102 + tests/service_testing/conftest.py | 26 +- tests/service_testing/test_imap.py | 16 +- tests/service_testing/test_imap_spool.py | 148 ++ tests/service_testing/test_instance_seed.py | 91 + tests/service_testing/test_mongodb.py | 19 +- tests/service_testing/test_mqtt.py | 33 +- tests/service_testing/test_mqtt_fuzz.py | 19 +- tests/service_testing/test_mssql.py | 19 +- tests/service_testing/test_mysql.py | 23 +- tests/service_testing/test_pop3.py | 14 +- tests/service_testing/test_pop3_spool.py | 96 + tests/service_testing/test_postgres.py | 21 +- tests/service_testing/test_rdp_basic.py | 174 ++ tests/service_testing/test_rdp_nla.py | 211 ++ tests/service_testing/test_redis.py | 107 +- tests/service_testing/test_smb_server.py | 268 ++ tests/service_testing/test_smtp.py | 383 ++- tests/service_testing/test_snmp.py | 14 +- tests/services/__init__.py | 0 tests/services/test_cred_emitters.py | 447 ++++ tests/{ => services}/test_custom_service.py | 0 tests/services/test_mongodb_scram.py | 210 ++ tests/services/test_ntlmssp_parser.py | 154 ++ tests/services/test_service_isolation.py | 508 ++++ tests/{ => services}/test_services.py | 3 +- tests/{ => services}/test_smtp_relay.py | 14 + tests/services/test_smtp_targets.py | 160 ++ tests/services/test_ssh.py | 475 ++++ tests/services/test_ssh_capture_emit.py | 143 ++ tests/services/test_ssh_stealth.py | 143 ++ tests/services/test_syslog_bridge_helpers.py | 184 ++ tests/sniffer/__init__.py | 0 tests/sniffer/test_sniffer_bus.py | 160 ++ tests/sniffer/test_sniffer_ja3.py | 1187 +++++++++ tests/sniffer/test_sniffer_p0f.py | 117 + tests/sniffer/test_sniffer_retransmit.py | 108 + tests/sniffer/test_sniffer_seq_class.py | 66 + tests/sniffer/test_sniffer_tcp_fingerprint.py | 306 +++ tests/sniffer/test_sniffer_worker.py | 308 +++ tests/stress/__init__.py | 0 tests/stress/conftest.py | 377 +++ tests/stress/locustfile.py | 154 ++ tests/stress/test_stress.py | 163 ++ tests/swarm/__init__.py | 0 tests/swarm/test_agent_app.py | 95 + tests/swarm/test_agent_heartbeat.py | 122 + tests/swarm/test_agent_no_auto_restore.py | 147 ++ tests/swarm/test_agent_relocalize.py | 118 + tests/swarm/test_agent_topology_endpoints.py | 168 ++ tests/swarm/test_agent_topology_store.py | 160 ++ tests/swarm/test_cli_forwarder.py | 39 + tests/swarm/test_cli_swarm.py | 292 +++ tests/swarm/test_cli_swarm_update.py | 192 ++ tests/swarm/test_client_agent_roundtrip.py | 170 ++ tests/swarm/test_client_topology.py | 122 + tests/swarm/test_forwarder_resilience.py | 256 ++ tests/swarm/test_heartbeat.py | 300 +++ tests/swarm/test_heartbeat_topology_resync.py | 224 ++ tests/swarm/test_log_forwarder.py | 282 +++ tests/swarm/test_pki.py | 213 ++ tests/swarm/test_state_schema.py | 60 + tests/swarm/test_swarm_api.py | 493 ++++ tests/swarm/test_tar_tree.py | 75 + tests/swarm/test_uvicorn_tls_scope.py | 77 + tests/telemetry/__init__.py | 0 tests/telemetry/test_telemetry.py | 250 ++ tests/test_base_repo.py | 43 - tests/test_collector.py | 348 --- tests/test_decnet.db-shm | Bin 32768 -> 0 bytes tests/test_decnet.db-wal | Bin 65952 -> 0 bytes tests/test_deployer.py | 308 --- tests/test_ingester.py | 217 -- tests/test_mutator.py | 183 -- tests/test_ssh.py | 168 -- tests/topology/__init__.py | 0 tests/topology/test_allocator.py | 198 ++ tests/topology/test_compose.py | 135 + tests/topology/test_concurrency.py | 118 + tests/topology/test_deploy.py | 236 ++ tests/topology/test_deploy_agent_branch.py | 168 ++ tests/topology/test_editing.py | 132 + tests/topology/test_generator.py | 137 + tests/topology/test_hashing.py | 80 + tests/topology/test_layout.py | 58 + tests/topology/test_mutator.py | 452 ++++ tests/topology/test_persistence.py | 91 + tests/topology/test_reaper.py | 228 ++ tests/topology/test_repo.py | 167 ++ tests/topology/test_resync_reconcile.py | 168 ++ tests/topology/test_service_config.py | 112 + tests/topology/test_status.py | 55 + tests/topology/test_validate.py | 178 ++ tests/updater/__init__.py | 0 tests/updater/test_updater_app.py | 138 + tests/updater/test_updater_executor.py | 637 +++++ tests/vectorstore/__init__.py | 0 tests/vectorstore/test_factory.py | 66 + tests/vectorstore/test_fake.py | 113 + tests/web/__init__.py | 0 tests/web/services/__init__.py | 0 tests/web/services/test_systemd_control.py | 133 + tests/web/test_admin_seed.py | 66 + tests/web/test_api_attacker_intel.py | 54 + tests/web/test_api_attackers.py | 638 +++++ tests/web/test_api_campaigns.py | 254 ++ tests/web/test_api_identities.py | 313 +++ tests/web/test_api_startup_guards.py | 122 + tests/web/test_auth_async.py | 51 + tests/web/test_env_lazy_jwt.py | 62 + tests/web/test_health_config_cache.py | 67 + tests/web/test_ingester.py | 414 +++ tests/web/test_ingester_bus.py | 150 ++ tests/web/test_ingester_http_quirks.py | 226 ++ tests/web/test_ingester_ua_classify.py | 242 ++ tests/web/test_ingester_xff.py | 302 +++ tests/web/test_router_cache.py | 110 + tests/web/test_validate_public_binding.py | 89 + tests/{ => web}/test_web_api.py | 15 +- tests/webhook/__init__.py | 0 tests/webhook/conftest.py | 18 + tests/webhook/test_client.py | 145 ++ tests/webhook/test_enums.py | 49 + tests/webhook/test_worker.py | 351 +++ 1235 files changed, 160255 insertions(+), 7996 deletions(-) delete mode 100644 CLAUDE.md delete mode 100644 GEMINI.md create mode 100644 LICENSE delete mode 100644 decnet.collector.log create mode 100644 decnet.ini.example create mode 100644 decnet/agent/__init__.py create mode 100644 decnet/agent/app.py create mode 100644 decnet/agent/executor.py create mode 100644 decnet/agent/heartbeat.py create mode 100644 decnet/agent/server.py create mode 100644 decnet/agent/topology_ops.py create mode 100644 decnet/agent/topology_store.py create mode 100644 decnet/asn/__init__.py create mode 100644 decnet/asn/base.py create mode 100644 decnet/asn/factory.py create mode 100644 decnet/asn/iptoasn/__init__.py create mode 100644 decnet/asn/iptoasn/fetch.py create mode 100644 decnet/asn/iptoasn/parse.py create mode 100644 decnet/asn/iptoasn/provider.py create mode 100644 decnet/asn/lookup.py create mode 100644 decnet/asn/paths.py create mode 100644 decnet/bus/__init__.py create mode 100644 decnet/bus/app.py create mode 100644 decnet/bus/base.py create mode 100644 decnet/bus/factory.py create mode 100644 decnet/bus/fake.py create mode 100644 decnet/bus/protocol.py create mode 100644 decnet/bus/publish.py create mode 100644 decnet/bus/topics.py create mode 100644 decnet/bus/unix_client.py create mode 100644 decnet/bus/unix_server.py create mode 100644 decnet/bus/worker.py create mode 100644 decnet/canary/__init__.py create mode 100644 decnet/canary/base.py create mode 100644 decnet/canary/cultivator.py create mode 100644 decnet/canary/dns_server.py create mode 100644 decnet/canary/factory.py create mode 100644 decnet/canary/generators/__init__.py create mode 100644 decnet/canary/generators/aws_creds.py create mode 100644 decnet/canary/generators/env_file.py create mode 100644 decnet/canary/generators/git_config.py create mode 100644 decnet/canary/generators/honeydoc.py create mode 100644 decnet/canary/generators/honeydoc_docx.py create mode 100644 decnet/canary/generators/honeydoc_pdf.py create mode 100644 decnet/canary/generators/mysql_dump.py create mode 100644 decnet/canary/generators/ssh_key.py create mode 100644 decnet/canary/instrumenters/__init__.py create mode 100644 decnet/canary/instrumenters/docx.py create mode 100644 decnet/canary/instrumenters/html.py create mode 100644 decnet/canary/instrumenters/image.py create mode 100644 decnet/canary/instrumenters/passthrough.py create mode 100644 decnet/canary/instrumenters/pdf.py create mode 100644 decnet/canary/instrumenters/plain.py create mode 100644 decnet/canary/instrumenters/xlsx.py create mode 100644 decnet/canary/paths.py create mode 100644 decnet/canary/planter.py create mode 100644 decnet/canary/storage.py create mode 100644 decnet/canary/worker.py delete mode 100644 decnet/cli.py create mode 100644 decnet/cli/__init__.py create mode 100644 decnet/cli/agent.py create mode 100644 decnet/cli/api.py create mode 100644 decnet/cli/bus.py create mode 100644 decnet/cli/canary.py create mode 100644 decnet/cli/db.py create mode 100644 decnet/cli/deploy.py create mode 100644 decnet/cli/forwarder.py create mode 100644 decnet/cli/gating.py create mode 100644 decnet/cli/geoip.py create mode 100644 decnet/cli/init.py create mode 100644 decnet/cli/inventory.py create mode 100644 decnet/cli/lifecycle.py create mode 100644 decnet/cli/listener.py create mode 100644 decnet/cli/orchestrator.py create mode 100644 decnet/cli/profiler.py create mode 100644 decnet/cli/realism.py create mode 100644 decnet/cli/reconciler.py create mode 100644 decnet/cli/sniffer.py create mode 100644 decnet/cli/swarm.py create mode 100644 decnet/cli/swarmctl.py create mode 100644 decnet/cli/topology.py create mode 100644 decnet/cli/updater.py create mode 100644 decnet/cli/utils.py create mode 100644 decnet/cli/web.py create mode 100644 decnet/cli/webhook.py create mode 100644 decnet/cli/workers.py create mode 100644 decnet/clustering/__init__.py create mode 100644 decnet/clustering/base.py create mode 100644 decnet/clustering/campaign/__init__.py create mode 100644 decnet/clustering/campaign/base.py create mode 100644 decnet/clustering/campaign/factory.py create mode 100644 decnet/clustering/campaign/impl/__init__.py create mode 100644 decnet/clustering/campaign/impl/connected_components.py create mode 100644 decnet/clustering/campaign/impl/similarity.py create mode 100644 decnet/clustering/campaign/worker.py create mode 100644 decnet/clustering/factory.py create mode 100644 decnet/clustering/impl/__init__.py create mode 100644 decnet/clustering/impl/connected_components.py create mode 100644 decnet/clustering/impl/similarity.py create mode 100644 decnet/clustering/ukc.py create mode 100644 decnet/clustering/worker.py create mode 100644 decnet/config_ini.py create mode 100644 decnet/correlation/event_kinds.py create mode 100644 decnet/correlation/reuse_worker.py create mode 100644 decnet/engine/reaper.py rename decnet/{fleet.py => fleet/__init__.py} (96%) create mode 100644 decnet/fleet/reconciler.py create mode 100644 decnet/fleet/reconciler_worker.py create mode 100644 decnet/geoip/__init__.py create mode 100644 decnet/geoip/base.py create mode 100644 decnet/geoip/factory.py create mode 100644 decnet/geoip/lookup.py create mode 100644 decnet/geoip/paths.py create mode 100644 decnet/geoip/ptr.py create mode 100644 decnet/geoip/rir/__init__.py create mode 100644 decnet/geoip/rir/fetch.py create mode 100644 decnet/geoip/rir/parse.py create mode 100644 decnet/geoip/rir/provider.py create mode 100644 decnet/intel/__init__.py create mode 100644 decnet/intel/abuseipdb.py create mode 100644 decnet/intel/base.py create mode 100644 decnet/intel/factory.py create mode 100644 decnet/intel/feodo.py create mode 100644 decnet/intel/greynoise.py create mode 100644 decnet/intel/threatfox.py create mode 100644 decnet/intel/worker.py create mode 100644 decnet/logging/inode_aware_handler.py create mode 100644 decnet/mutator/events.py create mode 100644 decnet/mutator/ops.py create mode 100644 decnet/net/__init__.py create mode 100644 decnet/net/http.py create mode 100644 decnet/orchestrator/__init__.py create mode 100644 decnet/orchestrator/drivers/__init__.py create mode 100644 decnet/orchestrator/drivers/base.py create mode 100644 decnet/orchestrator/drivers/email.py create mode 100644 decnet/orchestrator/drivers/ssh.py create mode 100644 decnet/orchestrator/emailgen/__init__.py create mode 100644 decnet/orchestrator/emailgen/events.py create mode 100644 decnet/orchestrator/emailgen/scheduler.py create mode 100644 decnet/orchestrator/emailgen/threads.py create mode 100644 decnet/orchestrator/events.py create mode 100644 decnet/orchestrator/scheduler.py create mode 100644 decnet/orchestrator/worker.py create mode 100644 decnet/privdrop.py create mode 100644 decnet/prober/__init__.py create mode 100644 decnet/prober/hassh.py create mode 100644 decnet/prober/jarm.py create mode 100644 decnet/prober/osfp/__init__.py create mode 100644 decnet/prober/osfp/base.py create mode 100644 decnet/prober/osfp/factory.py create mode 100644 decnet/prober/osfp/p0f/__init__.py create mode 100644 decnet/prober/osfp/p0f/data/LICENSE.p0f-upstream create mode 100644 decnet/prober/osfp/p0f/data/README.md create mode 100644 decnet/prober/osfp/p0f/data/p0f.fp create mode 100644 decnet/prober/osfp/p0f/data/p0fa.fp create mode 100644 decnet/prober/osfp/p0f/data/p0fo.fp create mode 100644 decnet/prober/osfp/p0f/data/p0fr.fp create mode 100644 decnet/prober/osfp/p0f/format.py create mode 100644 decnet/prober/osfp/p0f/provider.py create mode 100644 decnet/prober/osfp/p0f/signature.py create mode 100644 decnet/prober/tcpfp.py create mode 100644 decnet/prober/tlscert.py create mode 100644 decnet/prober/worker.py create mode 100644 decnet/profiler/__init__.py create mode 100644 decnet/profiler/behavioral.py create mode 100644 decnet/profiler/classify.py create mode 100644 decnet/profiler/fingerprint.py create mode 100644 decnet/profiler/identity_rollup.py create mode 100644 decnet/profiler/phases.py create mode 100644 decnet/profiler/timing.py create mode 100644 decnet/profiler/tools.py create mode 100644 decnet/profiler/worker.py create mode 100644 decnet/realism/__init__.py create mode 100644 decnet/realism/bodies.py create mode 100644 decnet/realism/diurnal.py create mode 100644 decnet/realism/llm/__init__.py create mode 100644 decnet/realism/llm/base.py create mode 100644 decnet/realism/llm/circuit.py create mode 100644 decnet/realism/llm/factory.py create mode 100644 decnet/realism/llm/impl/__init__.py create mode 100644 decnet/realism/llm/impl/fake.py create mode 100644 decnet/realism/llm/impl/ollama.py create mode 100644 decnet/realism/naming.py create mode 100644 decnet/realism/personas.py create mode 100644 decnet/realism/personas_pool.py create mode 100644 decnet/realism/planner.py create mode 100644 decnet/realism/prompts/__init__.py create mode 100644 decnet/realism/prompts/_style.py create mode 100644 decnet/realism/prompts/email.py create mode 100644 decnet/realism/prompts/filebody.py create mode 100644 decnet/realism/taxonomy.py create mode 100644 decnet/services/https.py create mode 100644 decnet/services/sniffer.py create mode 100644 decnet/sniffer/__init__.py create mode 100644 decnet/sniffer/fingerprint.py create mode 100644 decnet/sniffer/p0f.py create mode 100644 decnet/sniffer/seq_class.py create mode 100644 decnet/sniffer/syslog.py create mode 100644 decnet/sniffer/worker.py create mode 100644 decnet/swarm/__init__.py create mode 100644 decnet/swarm/client.py create mode 100644 decnet/swarm/log_forwarder.py create mode 100644 decnet/swarm/log_listener.py create mode 100644 decnet/swarm/pki.py create mode 100644 decnet/swarm/tar_tree.py create mode 100644 decnet/swarm/updater_client.py create mode 100644 decnet/telemetry.py create mode 100644 decnet/templates/_shared/auth-helper/auth-helper.c create mode 100644 decnet/templates/_shared/ntlmssp.py create mode 100644 decnet/templates/_shared/sessrec/Makefile create mode 100644 decnet/templates/_shared/sessrec/sessrec.c rename {templates => decnet/templates}/conpot/Dockerfile (85%) rename {templates => decnet/templates}/conpot/entrypoint.py (97%) create mode 100644 decnet/templates/conpot/instance_seed.py create mode 100644 decnet/templates/conpot/syslog_bridge.py rename {templates => decnet/templates}/cowrie/Dockerfile (91%) rename {templates => decnet/templates}/cowrie/cowrie.cfg.j2 (100%) rename {templates => decnet/templates}/cowrie/entrypoint.sh (100%) rename {templates => decnet/templates}/cowrie/honeyfs/etc/group (100%) rename {templates => decnet/templates}/cowrie/honeyfs/etc/hostname (100%) rename {templates => decnet/templates}/cowrie/honeyfs/etc/hosts (100%) rename {templates => decnet/templates}/cowrie/honeyfs/etc/issue (100%) rename {templates => decnet/templates}/cowrie/honeyfs/etc/issue.net (100%) rename {templates => decnet/templates}/cowrie/honeyfs/etc/motd (100%) rename {templates => decnet/templates}/cowrie/honeyfs/etc/os-release (100%) rename {templates => decnet/templates}/cowrie/honeyfs/etc/passwd (100%) rename {templates => decnet/templates}/cowrie/honeyfs/etc/resolv.conf (100%) rename {templates => decnet/templates}/cowrie/honeyfs/etc/shadow (100%) rename {templates => decnet/templates}/cowrie/honeyfs/home/admin/.aws/credentials (100%) rename {templates => decnet/templates}/cowrie/honeyfs/home/admin/.bash_history (100%) rename {templates => decnet/templates}/cowrie/honeyfs/home/admin/.ssh/authorized_keys (100%) rename {templates => decnet/templates}/cowrie/honeyfs/root/.bash_history (100%) rename {templates => decnet/templates}/cowrie/honeyfs/var/log/auth.log (100%) rename {templates => decnet/templates}/docker_api/Dockerfile (87%) rename {templates => decnet/templates}/docker_api/entrypoint.sh (100%) create mode 100644 decnet/templates/docker_api/instance_seed.py rename {templates => decnet/templates}/docker_api/server.py (91%) create mode 100644 decnet/templates/docker_api/syslog_bridge.py rename {templates => decnet/templates}/elasticsearch/Dockerfile (81%) rename {templates => decnet/templates}/elasticsearch/entrypoint.sh (100%) create mode 100644 decnet/templates/elasticsearch/instance_seed.py create mode 100644 decnet/templates/elasticsearch/server.py create mode 100644 decnet/templates/elasticsearch/syslog_bridge.py rename {templates => decnet/templates}/ftp/Dockerfile (83%) rename {templates => decnet/templates}/ftp/entrypoint.sh (100%) create mode 100644 decnet/templates/ftp/instance_seed.py create mode 100644 decnet/templates/ftp/server.py create mode 100644 decnet/templates/ftp/syslog_bridge.py rename {templates => decnet/templates}/http/Dockerfile (83%) rename {templates => decnet/templates}/http/entrypoint.sh (100%) create mode 100644 decnet/templates/http/instance_seed.py rename {templates => decnet/templates}/http/server.py (74%) create mode 100644 decnet/templates/http/syslog_bridge.py create mode 100644 decnet/templates/https/Dockerfile create mode 100644 decnet/templates/https/entrypoint.sh create mode 100644 decnet/templates/https/instance_seed.py create mode 100644 decnet/templates/https/server.py create mode 100644 decnet/templates/https/syslog_bridge.py rename {templates => decnet/templates}/imap/Dockerfile (85%) rename {templates => decnet/templates}/imap/entrypoint.sh (100%) create mode 100644 decnet/templates/imap/instance_seed.py rename {templates => decnet/templates}/imap/server.py (78%) create mode 100644 decnet/templates/imap/syslog_bridge.py create mode 100644 decnet/templates/instance_seed.py rename {templates => decnet/templates}/k8s/Dockerfile (87%) rename {templates => decnet/templates}/k8s/entrypoint.sh (100%) create mode 100644 decnet/templates/k8s/instance_seed.py rename {templates => decnet/templates}/k8s/server.py (92%) create mode 100644 decnet/templates/k8s/syslog_bridge.py rename {templates => decnet/templates}/ldap/Dockerfile (81%) rename {templates => decnet/templates}/ldap/entrypoint.sh (100%) create mode 100644 decnet/templates/ldap/instance_seed.py rename {templates => decnet/templates}/ldap/server.py (62%) create mode 100644 decnet/templates/ldap/syslog_bridge.py rename {templates => decnet/templates}/llmnr/Dockerfile (86%) rename {templates => decnet/templates}/llmnr/entrypoint.sh (100%) create mode 100644 decnet/templates/llmnr/instance_seed.py rename {templates => decnet/templates}/llmnr/server.py (97%) create mode 100644 decnet/templates/llmnr/syslog_bridge.py rename {templates => decnet/templates}/mongodb/Dockerfile (81%) rename {templates => decnet/templates}/mongodb/entrypoint.sh (100%) create mode 100644 decnet/templates/mongodb/instance_seed.py create mode 100644 decnet/templates/mongodb/server.py create mode 100644 decnet/templates/mongodb/syslog_bridge.py rename {templates => decnet/templates}/mqtt/Dockerfile (81%) rename {templates => decnet/templates}/mqtt/entrypoint.sh (100%) create mode 100644 decnet/templates/mqtt/instance_seed.py rename {templates => decnet/templates}/mqtt/server.py (64%) create mode 100644 decnet/templates/mqtt/syslog_bridge.py rename {templates => decnet/templates}/mssql/Dockerfile (81%) rename {templates => decnet/templates}/mssql/entrypoint.sh (100%) create mode 100644 decnet/templates/mssql/instance_seed.py create mode 100644 decnet/templates/mssql/server.py create mode 100644 decnet/templates/mssql/syslog_bridge.py rename {templates => decnet/templates}/mysql/Dockerfile (81%) rename {templates => decnet/templates}/mysql/entrypoint.sh (100%) create mode 100644 decnet/templates/mysql/instance_seed.py create mode 100644 decnet/templates/mysql/server.py create mode 100644 decnet/templates/mysql/syslog_bridge.py rename {templates => decnet/templates}/pop3/Dockerfile (85%) rename {templates => decnet/templates}/pop3/entrypoint.sh (100%) create mode 100644 decnet/templates/pop3/instance_seed.py rename {templates => decnet/templates}/pop3/server.py (82%) create mode 100644 decnet/templates/pop3/syslog_bridge.py rename {templates => decnet/templates}/postgres/Dockerfile (81%) rename {templates => decnet/templates}/postgres/entrypoint.sh (100%) create mode 100644 decnet/templates/postgres/instance_seed.py rename {templates => decnet/templates}/postgres/server.py (56%) create mode 100644 decnet/templates/postgres/syslog_bridge.py rename {templates => decnet/templates}/rdp/Dockerfile (70%) create mode 100644 decnet/templates/rdp/entrypoint.sh create mode 100644 decnet/templates/rdp/instance_seed.py create mode 100644 decnet/templates/rdp/ntlmssp.py create mode 100644 decnet/templates/rdp/server.py create mode 100644 decnet/templates/rdp/syslog_bridge.py rename {templates => decnet/templates}/redis/Dockerfile (81%) rename {templates/rdp => decnet/templates/redis}/entrypoint.sh (100%) create mode 100644 decnet/templates/redis/instance_seed.py create mode 100644 decnet/templates/redis/server.py create mode 100644 decnet/templates/redis/syslog_bridge.py rename {templates => decnet/templates}/sip/Dockerfile (86%) rename {templates/redis => decnet/templates/sip}/entrypoint.sh (100%) create mode 100644 decnet/templates/sip/instance_seed.py rename {templates => decnet/templates}/sip/server.py (87%) create mode 100644 decnet/templates/sip/syslog_bridge.py create mode 100644 decnet/templates/smb/Dockerfile rename {templates/sip => decnet/templates/smb}/entrypoint.sh (100%) create mode 100644 decnet/templates/smb/instance_seed.py create mode 100644 decnet/templates/smb/ntlmssp.py create mode 100644 decnet/templates/smb/server.py create mode 100644 decnet/templates/smb/syslog_bridge.py rename {templates => decnet/templates}/smtp/Dockerfile (81%) rename {templates => decnet/templates}/smtp/entrypoint.sh (100%) create mode 100644 decnet/templates/smtp/instance_seed.py create mode 100644 decnet/templates/smtp/server.py create mode 100644 decnet/templates/smtp/syslog_bridge.py create mode 100644 decnet/templates/sniffer/Dockerfile create mode 100644 decnet/templates/sniffer/server.py rename {templates => decnet/templates}/snmp/Dockerfile (85%) rename {templates => decnet/templates}/snmp/entrypoint.sh (100%) create mode 100644 decnet/templates/snmp/instance_seed.py rename {templates => decnet/templates}/snmp/server.py (93%) create mode 100644 decnet/templates/snmp/syslog_bridge.py create mode 100644 decnet/templates/ssh/Dockerfile create mode 100644 decnet/templates/ssh/_build_stealth.py create mode 100644 decnet/templates/ssh/argv_zap.c create mode 100644 decnet/templates/ssh/auth-helper/auth-helper.c create mode 100755 decnet/templates/ssh/capture.sh create mode 100644 decnet/templates/ssh/emit_capture.py create mode 100644 decnet/templates/ssh/entrypoint.sh create mode 100644 decnet/templates/ssh/instance_seed.py create mode 100644 decnet/templates/ssh/sessrec/Makefile create mode 100644 decnet/templates/ssh/sessrec/sessrec.c create mode 100644 decnet/templates/ssh/syslog_bridge.py create mode 100644 decnet/templates/syslog_bridge.py create mode 100644 decnet/templates/telnet/Dockerfile create mode 100644 decnet/templates/telnet/auth-helper/auth-helper.c rename {templates => decnet/templates}/telnet/entrypoint.sh (54%) create mode 100644 decnet/templates/telnet/instance_seed.py create mode 100644 decnet/templates/telnet/sessrec/Makefile create mode 100644 decnet/templates/telnet/sessrec/sessrec.c create mode 100644 decnet/templates/telnet/syslog_bridge.py rename {templates => decnet/templates}/tftp/Dockerfile (85%) rename {templates => decnet/templates}/tftp/entrypoint.sh (100%) create mode 100644 decnet/templates/tftp/instance_seed.py rename {templates => decnet/templates}/tftp/server.py (95%) create mode 100644 decnet/templates/tftp/syslog_bridge.py rename {templates => decnet/templates}/vnc/Dockerfile (85%) rename {templates => decnet/templates}/vnc/entrypoint.sh (100%) create mode 100644 decnet/templates/vnc/instance_seed.py rename {templates => decnet/templates}/vnc/server.py (81%) create mode 100644 decnet/templates/vnc/syslog_bridge.py create mode 100644 decnet/topology/__init__.py create mode 100644 decnet/topology/allocator.py create mode 100644 decnet/topology/compose.py create mode 100644 decnet/topology/config.py create mode 100644 decnet/topology/generator.py create mode 100644 decnet/topology/hashing.py create mode 100644 decnet/topology/persistence.py create mode 100644 decnet/topology/status.py create mode 100644 decnet/topology/validate.py create mode 100644 decnet/updater/__init__.py create mode 100644 decnet/updater/app.py create mode 100644 decnet/updater/executor.py create mode 100644 decnet/updater/server.py create mode 100644 decnet/vectorstore/__init__.py create mode 100644 decnet/vectorstore/base.py create mode 100644 decnet/vectorstore/factory.py create mode 100644 decnet/vectorstore/fake.py create mode 100644 decnet/vectorstore/sqlite_vec.py create mode 100644 decnet/web/_uvicorn_tls_scope.py delete mode 100644 decnet/web/db/models.py create mode 100644 decnet/web/db/models/__init__.py create mode 100644 decnet/web/db/models/_base.py create mode 100644 decnet/web/db/models/attacker_intel.py create mode 100644 decnet/web/db/models/attackers.py create mode 100644 decnet/web/db/models/auth.py create mode 100644 decnet/web/db/models/campaigns.py create mode 100644 decnet/web/db/models/canary.py create mode 100644 decnet/web/db/models/common.py create mode 100644 decnet/web/db/models/deploy.py create mode 100644 decnet/web/db/models/fleet.py create mode 100644 decnet/web/db/models/health.py create mode 100644 decnet/web/db/models/logs.py create mode 100644 decnet/web/db/models/orchestrator.py create mode 100644 decnet/web/db/models/realism.py create mode 100644 decnet/web/db/models/swarm.py create mode 100644 decnet/web/db/models/topology.py create mode 100644 decnet/web/db/models/updater.py create mode 100644 decnet/web/db/models/webhooks.py create mode 100644 decnet/web/db/models/workers.py create mode 100644 decnet/web/db/mysql/__init__.py create mode 100644 decnet/web/db/mysql/database.py create mode 100644 decnet/web/db/mysql/repository.py create mode 100644 decnet/web/db/sqlmodel_repo/__init__.py create mode 100644 decnet/web/db/sqlmodel_repo/_helpers.py create mode 100644 decnet/web/db/sqlmodel_repo/attacker_intel.py create mode 100644 decnet/web/db/sqlmodel_repo/attackers/__init__.py create mode 100644 decnet/web/db/sqlmodel_repo/attackers/_core.py create mode 100644 decnet/web/db/sqlmodel_repo/attackers/activity.py create mode 100644 decnet/web/db/sqlmodel_repo/attackers/behavior.py create mode 100644 decnet/web/db/sqlmodel_repo/attackers/sessions.py create mode 100644 decnet/web/db/sqlmodel_repo/attackers/smtp.py create mode 100644 decnet/web/db/sqlmodel_repo/auth.py create mode 100644 decnet/web/db/sqlmodel_repo/bounties.py create mode 100644 decnet/web/db/sqlmodel_repo/campaigns.py create mode 100644 decnet/web/db/sqlmodel_repo/canary.py create mode 100644 decnet/web/db/sqlmodel_repo/credentials/__init__.py create mode 100644 decnet/web/db/sqlmodel_repo/credentials/_core.py create mode 100644 decnet/web/db/sqlmodel_repo/credentials/reuse.py create mode 100644 decnet/web/db/sqlmodel_repo/deckies.py create mode 100644 decnet/web/db/sqlmodel_repo/fleet.py create mode 100644 decnet/web/db/sqlmodel_repo/identities.py create mode 100644 decnet/web/db/sqlmodel_repo/logs.py create mode 100644 decnet/web/db/sqlmodel_repo/orchestrator.py create mode 100644 decnet/web/db/sqlmodel_repo/realism.py create mode 100644 decnet/web/db/sqlmodel_repo/swarm.py create mode 100644 decnet/web/db/sqlmodel_repo/topology/__init__.py create mode 100644 decnet/web/db/sqlmodel_repo/topology/_core.py create mode 100644 decnet/web/db/sqlmodel_repo/topology/deckies.py create mode 100644 decnet/web/db/sqlmodel_repo/topology/edges.py create mode 100644 decnet/web/db/sqlmodel_repo/topology/lans.py create mode 100644 decnet/web/db/sqlmodel_repo/topology/mutations.py create mode 100644 decnet/web/db/sqlmodel_repo/webhooks.py create mode 100644 decnet/web/limiter.py create mode 100644 decnet/web/router/artifacts/__init__.py create mode 100644 decnet/web/router/artifacts/api_get_artifact.py create mode 100644 decnet/web/router/attackers/__init__.py create mode 100644 decnet/web/router/attackers/api_get_attacker_artifacts.py create mode 100644 decnet/web/router/attackers/api_get_attacker_commands.py create mode 100644 decnet/web/router/attackers/api_get_attacker_detail.py create mode 100644 decnet/web/router/attackers/api_get_attacker_intel.py create mode 100644 decnet/web/router/attackers/api_get_attacker_mail.py create mode 100644 decnet/web/router/attackers/api_get_attacker_smtp_targets.py create mode 100644 decnet/web/router/attackers/api_get_attacker_transcripts.py create mode 100644 decnet/web/router/attackers/api_get_attackers.py create mode 100644 decnet/web/router/campaigns/__init__.py create mode 100644 decnet/web/router/campaigns/api_events.py create mode 100644 decnet/web/router/campaigns/api_get_campaign_detail.py create mode 100644 decnet/web/router/campaigns/api_list_campaign_identities.py create mode 100644 decnet/web/router/campaigns/api_list_campaigns.py create mode 100644 decnet/web/router/canary/__init__.py create mode 100644 decnet/web/router/canary/api_blobs.py create mode 100644 decnet/web/router/canary/api_tokens.py create mode 100644 decnet/web/router/config/__init__.py create mode 100644 decnet/web/router/config/api_get_config.py create mode 100644 decnet/web/router/config/api_manage_users.py create mode 100644 decnet/web/router/config/api_reinit.py create mode 100644 decnet/web/router/config/api_update_config.py create mode 100644 decnet/web/router/credential_reuse/__init__.py create mode 100644 decnet/web/router/credential_reuse/api_get_credential_reuse.py create mode 100644 decnet/web/router/credentials/__init__.py create mode 100644 decnet/web/router/credentials/api_get_credentials.py create mode 100644 decnet/web/router/health/__init__.py create mode 100644 decnet/web/router/health/api_get_health.py create mode 100644 decnet/web/router/identities/__init__.py create mode 100644 decnet/web/router/identities/api_events.py create mode 100644 decnet/web/router/identities/api_get_identity_detail.py create mode 100644 decnet/web/router/identities/api_list_identities.py create mode 100644 decnet/web/router/identities/api_list_identity_observations.py create mode 100644 decnet/web/router/orchestrator/__init__.py create mode 100644 decnet/web/router/orchestrator/api_events.py create mode 100644 decnet/web/router/orchestrator/api_list_events.py create mode 100644 decnet/web/router/realism/__init__.py create mode 100644 decnet/web/router/realism/api_config.py create mode 100644 decnet/web/router/realism/api_personas.py create mode 100644 decnet/web/router/realism/api_synthetic_files.py create mode 100644 decnet/web/router/swarm/__init__.py create mode 100644 decnet/web/router/swarm/api_check_hosts.py create mode 100644 decnet/web/router/swarm/api_decommission_host.py create mode 100644 decnet/web/router/swarm/api_deploy_swarm.py create mode 100644 decnet/web/router/swarm/api_enroll_host.py create mode 100644 decnet/web/router/swarm/api_get_host.py create mode 100644 decnet/web/router/swarm/api_get_swarm_health.py create mode 100644 decnet/web/router/swarm/api_heartbeat.py create mode 100644 decnet/web/router/swarm/api_list_deckies.py create mode 100644 decnet/web/router/swarm/api_list_hosts.py create mode 100644 decnet/web/router/swarm/api_teardown_swarm.py create mode 100644 decnet/web/router/swarm_mgmt/__init__.py create mode 100644 decnet/web/router/swarm_mgmt/api_decommission_host.py create mode 100644 decnet/web/router/swarm_mgmt/api_enroll_bundle.py create mode 100644 decnet/web/router/swarm_mgmt/api_list_deckies.py create mode 100644 decnet/web/router/swarm_mgmt/api_list_hosts.py create mode 100644 decnet/web/router/swarm_mgmt/api_teardown_host.py create mode 100644 decnet/web/router/swarm_updates/__init__.py create mode 100644 decnet/web/router/swarm_updates/api_list_host_releases.py create mode 100644 decnet/web/router/swarm_updates/api_push_update.py create mode 100644 decnet/web/router/swarm_updates/api_push_update_self.py create mode 100644 decnet/web/router/swarm_updates/api_rollback_host.py create mode 100644 decnet/web/router/system/__init__.py create mode 100644 decnet/web/router/system/api_deployment_mode.py create mode 100644 decnet/web/router/topology/__init__.py create mode 100644 decnet/web/router/topology/_guards.py create mode 100644 decnet/web/router/topology/_target_host.py create mode 100644 decnet/web/router/topology/api_catalog.py create mode 100644 decnet/web/router/topology/api_create_blank_topology.py create mode 100644 decnet/web/router/topology/api_create_topology.py create mode 100644 decnet/web/router/topology/api_decky_crud.py create mode 100644 decnet/web/router/topology/api_delete_topology.py create mode 100644 decnet/web/router/topology/api_deploy_topology.py create mode 100644 decnet/web/router/topology/api_edge_crud.py create mode 100644 decnet/web/router/topology/api_events.py create mode 100644 decnet/web/router/topology/api_get_topology.py create mode 100644 decnet/web/router/topology/api_lan_crud.py create mode 100644 decnet/web/router/topology/api_list_topologies.py create mode 100644 decnet/web/router/topology/api_mutations.py create mode 100644 decnet/web/router/topology/api_personas.py create mode 100644 decnet/web/router/topology/api_reap_orphans.py create mode 100644 decnet/web/router/topology/api_teardown_topology.py create mode 100644 decnet/web/router/transcripts/__init__.py create mode 100644 decnet/web/router/transcripts/api_get_transcript.py create mode 100644 decnet/web/router/webhooks/__init__.py create mode 100644 decnet/web/router/webhooks/api_manage_webhooks.py create mode 100644 decnet/web/router/webhooks/api_test_webhook.py create mode 100644 decnet/web/router/workers/__init__.py create mode 100644 decnet/web/router/workers/api_control_worker.py create mode 100644 decnet/web/router/workers/api_list_workers.py create mode 100644 decnet/web/router/workers/api_start_all_workers.py create mode 100644 decnet/web/router/workers/api_start_worker.py create mode 100644 decnet/web/services/__init__.py create mode 100644 decnet/web/services/systemd_control.py create mode 100644 decnet/web/sse_limits.py create mode 100644 decnet/web/swarm_api.py create mode 100644 decnet/web/templates/decnet-agent.service.j2 create mode 100644 decnet/web/templates/decnet-collector.service.j2 create mode 100644 decnet/web/templates/decnet-engine.service.j2 create mode 100644 decnet/web/templates/decnet-forwarder.service.j2 create mode 100644 decnet/web/templates/decnet-prober.service.j2 create mode 100644 decnet/web/templates/decnet-sniffer.service.j2 create mode 100644 decnet/web/templates/decnet-updater.service.j2 create mode 100644 decnet/web/templates/enroll_bootstrap.sh.j2 create mode 100644 decnet/web/worker_registry.py create mode 100644 decnet/webhook/__init__.py create mode 100644 decnet/webhook/client.py create mode 100644 decnet/webhook/enums.py create mode 100644 decnet/webhook/worker.py create mode 100644 decnet_web/src/components/ArtifactDrawer.tsx create mode 100644 decnet_web/src/components/AttackerDetail.tsx create mode 100644 decnet_web/src/components/Attackers.css create mode 100644 decnet_web/src/components/Bounty.css create mode 100644 decnet_web/src/components/BountyInspector.tsx create mode 100644 decnet_web/src/components/CampaignDetail.tsx create mode 100644 decnet_web/src/components/Campaigns.tsx create mode 100644 decnet_web/src/components/CanaryTokenDrawer.tsx create mode 100644 decnet_web/src/components/CanaryTokens.tsx create mode 100644 decnet_web/src/components/CommandPalette/CommandPalette.css create mode 100644 decnet_web/src/components/CommandPalette/CommandPalette.tsx create mode 100644 decnet_web/src/components/Config.css create mode 100644 decnet_web/src/components/CredentialReuseInspector.tsx create mode 100644 decnet_web/src/components/Credentials.css create mode 100644 decnet_web/src/components/Credentials.tsx create mode 100644 decnet_web/src/components/CredentialsInspector.tsx create mode 100644 decnet_web/src/components/DeckyFleet.css create mode 100644 decnet_web/src/components/EmptyState/EmptyState.css create mode 100644 decnet_web/src/components/EmptyState/EmptyState.tsx create mode 100644 decnet_web/src/components/Identities.tsx create mode 100644 decnet_web/src/components/IdentityDetail.tsx create mode 100644 decnet_web/src/components/LiveLogs.css create mode 100644 decnet_web/src/components/MailDrawer.tsx create mode 100644 decnet_web/src/components/MazeNET/Canvas.tsx create mode 100644 decnet_web/src/components/MazeNET/ContextMenu.tsx create mode 100644 decnet_web/src/components/MazeNET/Inspector.tsx create mode 100644 decnet_web/src/components/MazeNET/MazeNET.css create mode 100644 decnet_web/src/components/MazeNET/MazeNET.tsx create mode 100644 decnet_web/src/components/MazeNET/NetBox.tsx create mode 100644 decnet_web/src/components/MazeNET/NodeCard.tsx create mode 100644 decnet_web/src/components/MazeNET/Palette.tsx create mode 100644 decnet_web/src/components/MazeNET/data.ts create mode 100644 decnet_web/src/components/MazeNET/types.ts create mode 100644 decnet_web/src/components/MazeNET/useMazeApi.ts create mode 100644 decnet_web/src/components/MazeNET/useMazeInteraction.ts create mode 100644 decnet_web/src/components/MazeNET/useMazeLayoutStore.ts create mode 100644 decnet_web/src/components/MazeNET/useTopologyEditor.ts create mode 100644 decnet_web/src/components/MazeNET/useTopologyStream.ts create mode 100644 decnet_web/src/components/Modal/Modal.css create mode 100644 decnet_web/src/components/Modal/Modal.tsx create mode 100644 decnet_web/src/components/Orchestrator.css create mode 100644 decnet_web/src/components/Orchestrator.tsx create mode 100644 decnet_web/src/components/OrchestratorInspector.tsx create mode 100644 decnet_web/src/components/PersonaGeneration.css create mode 100644 decnet_web/src/components/PersonaGeneration.tsx create mode 100644 decnet_web/src/components/RealismConfig/RealismConfig.css create mode 100644 decnet_web/src/components/RealismConfig/RealismConfig.tsx create mode 100644 decnet_web/src/components/RemoteUpdates.tsx create mode 100644 decnet_web/src/components/SessionDrawer.tsx create mode 100644 decnet_web/src/components/ShortcutsHelp/ShortcutsHelp.css create mode 100644 decnet_web/src/components/ShortcutsHelp/ShortcutsHelp.tsx create mode 100644 decnet_web/src/components/Swarm.css create mode 100644 decnet_web/src/components/SwarmHosts.tsx create mode 100644 decnet_web/src/components/SyntheticFiles/SyntheticFiles.css create mode 100644 decnet_web/src/components/SyntheticFiles/SyntheticFiles.tsx create mode 100644 decnet_web/src/components/Toasts/ToastProvider.tsx create mode 100644 decnet_web/src/components/Toasts/Toasts.css create mode 100644 decnet_web/src/components/Toasts/Toasts.tsx create mode 100644 decnet_web/src/components/Toasts/toast-context.ts create mode 100644 decnet_web/src/components/Toasts/useToast.ts create mode 100644 decnet_web/src/components/TopologyList/CreateTopologyWizard.css create mode 100644 decnet_web/src/components/TopologyList/CreateTopologyWizard.tsx create mode 100644 decnet_web/src/components/TopologyList/TopologyList.css create mode 100644 decnet_web/src/components/TopologyList/TopologyList.tsx create mode 100644 decnet_web/src/components/Webhooks.css create mode 100644 decnet_web/src/components/Webhooks.tsx create mode 100644 decnet_web/src/components/useCampaignStream.ts create mode 100644 decnet_web/src/components/useIdentityStream.ts create mode 100644 decnet_web/src/components/useOrchestratorStream.ts create mode 100644 decnet_web/src/hooks/useEscapeKey.ts create mode 100644 decnet_web/src/hooks/useFocusSearch.ts create mode 100644 decnet_web/src/hooks/useFocusTrap.ts create mode 100644 decnet_web/src/hooks/useGlobalHotkeys.ts create mode 100644 decnet_web/src/hooks/useSwarmHosts.ts create mode 100644 decnet_web/src/icons.ts create mode 100644 decnet_web/src/lucide-icons.d.ts create mode 100644 decnet_web/src/realism/labels.ts create mode 100644 decnet_web/src/routePrefetch.ts create mode 100644 decnet_web/src/utils/parseEventBody.ts create mode 100644 deploy/decnet-agent.service.j2 delete mode 100644 deploy/decnet-api.service create mode 100644 deploy/decnet-api.service.j2 create mode 100644 deploy/decnet-bus.service.j2 create mode 100644 deploy/decnet-campaign-clusterer.service.j2 create mode 100644 deploy/decnet-canary.service.j2 create mode 100644 deploy/decnet-clusterer.service.j2 create mode 100644 deploy/decnet-collector.service.j2 create mode 100644 deploy/decnet-enrich.service.j2 create mode 100644 deploy/decnet-forwarder.service.j2 create mode 100644 deploy/decnet-listener.service.j2 create mode 100644 deploy/decnet-mutator.service.j2 create mode 100644 deploy/decnet-orchestrator.service.j2 create mode 100644 deploy/decnet-prober.service.j2 create mode 100644 deploy/decnet-profiler.service.j2 create mode 100644 deploy/decnet-reconciler.service.j2 create mode 100644 deploy/decnet-reuse-correlator.service.j2 create mode 100644 deploy/decnet-sniffer.service.j2 create mode 100644 deploy/decnet-swarmctl.service.j2 create mode 100644 deploy/decnet-updater.service.j2 delete mode 100644 deploy/decnet-web.service create mode 100644 deploy/decnet-web.service.j2 create mode 100644 deploy/decnet-webhook.service.j2 create mode 100644 deploy/decnet.target create mode 100644 deploy/logrotate.d/decnet create mode 100644 deploy/polkit/50-decnet-workers.rules.j2 create mode 100644 deploy/tmpfiles.d/decnet.conf create mode 100644 development/CAMPAIGN_CLUSTERING.md create mode 100644 development/DEVELOPMENT_V2.md create mode 100644 development/IDENTITY_RESOLUTION.md create mode 100644 development/SIGNAL_CAPTURE_AUDIT.md create mode 100644 development/THREAT_MODEL.md create mode 100644 development/api-audit.md create mode 100644 development/docker-compose.otel.yml create mode 100644 development/docs/ARCHITECTURE.md create mode 100644 development/docs/TRACING.md create mode 100644 development/docs/services/COLLECTOR.md create mode 100644 development/docs/services/ENGINE.md create mode 100644 development/docs/services/MODELS.md create mode 100644 development/docs/services/WEB_MODELS.md create mode 100644 development/profiles/profile_1500_fb69a06.csv create mode 100644 development/profiles/profile_1500_notracing_12_workers_fb69a06.csv create mode 100644 development/profiles/profile_1500_notracing_fb69a06.csv create mode 100644 development/profiles/profile_1500_notracing_single_core_fb69a06.csv create mode 100644 development/profiles/profile_255c2e5.csv create mode 100644 development/profiles/profile_2dd86fb.csv create mode 100644 development/profiles/profile_3106d0313507f016_locust.csv create mode 100644 development/profiles/profile_e967aaa.csv create mode 100644 development/profiles/profile_fb69a06.csv create mode 100644 env.config.example create mode 100644 schemathesis.ci.toml create mode 100755 scripts/bus/pub.py create mode 100755 scripts/bus/smoke-mutator.sh create mode 100755 scripts/bus/smoke.sh create mode 100755 scripts/bus/start.sh create mode 100755 scripts/bus/sub.py create mode 100755 scripts/decnet-init.sh create mode 100755 scripts/mock-webhook-receiver.py create mode 100755 scripts/profile/aggregate_requests.py create mode 100755 scripts/profile/classify_usage.py create mode 100755 scripts/profile/cprofile-cli.sh create mode 100755 scripts/profile/memray-api.sh create mode 100755 scripts/profile/pyspy-attach.sh create mode 100755 scripts/profile/view.sh create mode 100644 scripts/vulture_whitelist.py delete mode 100644 templates/conpot/decnet_logging.py delete mode 100644 templates/cowrie/decnet_logging.py delete mode 100644 templates/decnet_logging.py delete mode 100644 templates/docker_api/decnet_logging.py delete mode 100644 templates/elasticsearch/decnet_logging.py delete mode 100644 templates/elasticsearch/server.py delete mode 100644 templates/ftp/decnet_logging.py delete mode 100644 templates/ftp/server.py delete mode 100644 templates/http/decnet_logging.py delete mode 100644 templates/imap/decnet_logging.py delete mode 100644 templates/k8s/decnet_logging.py delete mode 100644 templates/ldap/decnet_logging.py delete mode 100644 templates/llmnr/decnet_logging.py delete mode 100644 templates/mongodb/decnet_logging.py delete mode 100644 templates/mongodb/server.py delete mode 100644 templates/mqtt/decnet_logging.py delete mode 100644 templates/mssql/decnet_logging.py delete mode 100644 templates/mssql/server.py delete mode 100644 templates/mysql/decnet_logging.py delete mode 100644 templates/mysql/server.py delete mode 100644 templates/pop3/decnet_logging.py delete mode 100644 templates/postgres/decnet_logging.py delete mode 100644 templates/rdp/decnet_logging.py delete mode 100644 templates/rdp/server.py delete mode 100644 templates/redis/decnet_logging.py delete mode 100644 templates/redis/server.py delete mode 100644 templates/sip/decnet_logging.py delete mode 100644 templates/smb/Dockerfile delete mode 100644 templates/smb/decnet_logging.py delete mode 100644 templates/smb/entrypoint.sh delete mode 100644 templates/smb/server.py delete mode 100644 templates/smtp/decnet_logging.py delete mode 100644 templates/smtp/server.py delete mode 100644 templates/snmp/decnet_logging.py delete mode 100644 templates/ssh/Dockerfile delete mode 100644 templates/ssh/decnet_logging.py delete mode 100644 templates/ssh/entrypoint.sh delete mode 100644 templates/telnet/Dockerfile delete mode 100644 templates/telnet/decnet_logging.py delete mode 100644 templates/tftp/decnet_logging.py delete mode 100644 templates/vnc/decnet_logging.py create mode 100644 tests/agent/__init__.py create mode 100644 tests/agent/test_topology_apply_recreate_deps.py create mode 100644 tests/api/artifacts/__init__.py create mode 100644 tests/api/artifacts/test_get_artifact.py create mode 100644 tests/api/campaigns/__init__.py create mode 100644 tests/api/campaigns/test_events_stream.py create mode 100644 tests/api/canary/__init__.py create mode 100644 tests/api/canary/test_canary_tokens_api.py create mode 100644 tests/api/config/__init__.py create mode 100644 tests/api/config/conftest.py create mode 100644 tests/api/config/test_deploy_limit.py create mode 100644 tests/api/config/test_get_config.py create mode 100644 tests/api/config/test_reinit.py create mode 100644 tests/api/config/test_update_config.py create mode 100644 tests/api/config/test_user_management.py create mode 100644 tests/api/credential_reuse/__init__.py create mode 100644 tests/api/credential_reuse/test_get_credential_reuse.py create mode 100644 tests/api/credentials/__init__.py create mode 100644 tests/api/credentials/test_get_credentials.py create mode 100644 tests/api/fleet/test_deploy_automode.py create mode 100644 tests/api/health/__init__.py create mode 100644 tests/api/health/test_get_health.py create mode 100644 tests/api/identities/__init__.py create mode 100644 tests/api/identities/test_events_stream.py create mode 100644 tests/api/orchestrator/__init__.py create mode 100644 tests/api/orchestrator/test_events_stream.py create mode 100644 tests/api/realism/__init__.py create mode 100644 tests/api/realism/test_config_api.py create mode 100644 tests/api/realism/test_personas_api.py create mode 100644 tests/api/realism/test_synthetic_files_api.py create mode 100644 tests/api/swarm_mgmt/__init__.py create mode 100644 tests/api/swarm_mgmt/test_enroll_bundle.py create mode 100644 tests/api/swarm_mgmt/test_teardown_host.py create mode 100644 tests/api/swarm_updates/__init__.py create mode 100644 tests/api/swarm_updates/conftest.py create mode 100644 tests/api/swarm_updates/test_list_host_releases.py create mode 100644 tests/api/swarm_updates/test_push_update.py create mode 100644 tests/api/swarm_updates/test_push_update_self.py create mode 100644 tests/api/swarm_updates/test_rollback_host.py create mode 100644 tests/api/test_error_handler.py create mode 100644 tests/api/test_rbac.py create mode 100644 tests/api/test_rbac_contract.py create mode 100644 tests/api/test_schemathesis_agent.py create mode 100644 tests/api/test_schemathesis_swarm.py create mode 100644 tests/api/test_sse_limits.py create mode 100644 tests/api/topology/__init__.py create mode 100644 tests/api/topology/test_child_crud.py create mode 100644 tests/api/topology/test_events_stream.py create mode 100644 tests/api/topology/test_models.py create mode 100644 tests/api/topology/test_mutations.py create mode 100644 tests/api/topology/test_personas_api.py create mode 100644 tests/api/topology/test_reads.py create mode 100644 tests/api/topology/test_writes.py create mode 100644 tests/api/transcripts/__init__.py create mode 100644 tests/api/transcripts/test_get_transcript.py create mode 100644 tests/api/webhooks/__init__.py create mode 100644 tests/api/webhooks/test_crud.py create mode 100644 tests/api/workers/__init__.py create mode 100644 tests/api/workers/test_start_workers.py create mode 100644 tests/api/workers/test_workers_api.py create mode 100644 tests/asn/__init__.py create mode 100644 tests/asn/conftest.py create mode 100644 tests/asn/test_lookup.py create mode 100644 tests/asn/test_parse.py create mode 100644 tests/asn/test_profiler_integration.py create mode 100644 tests/asn/test_provider.py create mode 100644 tests/bus/__init__.py create mode 100644 tests/bus/conftest.py create mode 100644 tests/bus/test_app_singleton.py create mode 100644 tests/bus/test_base.py create mode 100644 tests/bus/test_closed_publish.py create mode 100644 tests/bus/test_control_listener.py create mode 100644 tests/bus/test_factory.py create mode 100644 tests/bus/test_fake_bus.py create mode 100644 tests/bus/test_heartbeat.py create mode 100644 tests/bus/test_protocol.py create mode 100644 tests/bus/test_publish.py create mode 100644 tests/bus/test_topics.py create mode 100644 tests/bus/test_unix_socket_bus.py create mode 100644 tests/bus/test_worker.py create mode 100644 tests/canary/__init__.py create mode 100644 tests/canary/conftest.py create mode 100644 tests/canary/test_cli.py create mode 100644 tests/canary/test_cultivator.py create mode 100644 tests/canary/test_deploy_hook.py create mode 100644 tests/canary/test_factory.py create mode 100644 tests/canary/test_generators.py create mode 100644 tests/canary/test_instrumenters.py create mode 100644 tests/canary/test_models.py create mode 100644 tests/canary/test_paths.py create mode 100644 tests/canary/test_planter.py create mode 100644 tests/canary/test_repository.py create mode 100644 tests/canary/test_storage.py create mode 100644 tests/canary/test_systemd_unit.py create mode 100644 tests/canary/test_topics.py create mode 100644 tests/canary/test_worker_dns.py create mode 100644 tests/canary/test_worker_http.py create mode 100644 tests/cli/__init__.py rename tests/{ => cli}/test_cli.py (70%) create mode 100644 tests/cli/test_cli_db_reset.py rename tests/{ => cli}/test_cli_service_pool.py (88%) create mode 100644 tests/cli/test_embedded_workers.py create mode 100644 tests/cli/test_init.py create mode 100644 tests/cli/test_mode_gating.py create mode 100644 tests/cli/test_realism_gating.py create mode 100644 tests/cli/test_realism_import_personas.py create mode 100644 tests/cli/test_web_proxy_target.py create mode 100644 tests/clustering/__init__.py create mode 100644 tests/clustering/fixture_harness.py create mode 100644 tests/clustering/metrics.py create mode 100644 tests/clustering/test_campaign_factory.py create mode 100644 tests/clustering/test_campaign_similarity.py create mode 100644 tests/clustering/test_campaign_worker.py create mode 100644 tests/clustering/test_clusterer_factory.py create mode 100644 tests/clustering/test_clusterer_worker.py create mode 100644 tests/clustering/test_connected_components.py create mode 100644 tests/clustering/test_fixtures_campaign_clusterer.py create mode 100644 tests/clustering/test_lone_wolf_fixture.py create mode 100644 tests/clustering/test_metrics.py create mode 100644 tests/clustering/test_multi_operator_fixture.py create mode 100644 tests/clustering/test_noise_floor_fixture.py create mode 100644 tests/clustering/test_paused_campaign_fixture.py create mode 100644 tests/clustering/test_shared_wordlist_fixture.py create mode 100644 tests/clustering/test_similarity.py create mode 100644 tests/clustering/test_slow_burn_fixture.py create mode 100644 tests/clustering/test_vpn_hopping_fixture.py create mode 100644 tests/collector/__init__.py create mode 100644 tests/collector/test_collector.py create mode 100644 tests/collector/test_collector_bus.py create mode 100644 tests/collector/test_collector_thread_pool.py create mode 100644 tests/config/__init__.py rename tests/{ => config}/test_config.py (100%) create mode 100644 tests/config/test_config_ini.py create mode 100644 tests/config/test_ini_loader.py rename tests/{ => config}/test_ini_spaces.py (100%) rename tests/{ => config}/test_ini_validation.py (100%) create mode 100644 tests/core/__init__.py rename tests/{ => core}/test_build.py (100%) create mode 100644 tests/core/test_fingerprinting.py rename tests/{ => core}/test_network.py (81%) rename tests/{ => core}/test_os_fingerprint.py (100%) create mode 100644 tests/core/test_privdrop.py create mode 100644 tests/correlation/__init__.py create mode 100644 tests/correlation/test_bounty_dedup.py create mode 100644 tests/correlation/test_correlation.py create mode 100644 tests/correlation/test_correlation_bus.py create mode 100644 tests/correlation/test_credential_reuse.py create mode 100644 tests/correlation/test_event_kinds.py create mode 100644 tests/db/__init__.py create mode 100644 tests/db/mysql/__init__.py create mode 100644 tests/db/mysql/test_mysql_histogram_sql.py create mode 100644 tests/db/mysql/test_mysql_migration.py create mode 100644 tests/db/mysql/test_mysql_url_builder.py create mode 100644 tests/db/test_base_repo.py create mode 100644 tests/db/test_campaign_repo.py create mode 100644 tests/db/test_credential_reuse.py create mode 100644 tests/db/test_credentials.py create mode 100644 tests/db/test_factory.py create mode 100644 tests/db/test_identity_schema.py create mode 100644 tests/deploy/__init__.py create mode 100644 tests/deploy/test_orchestrator_unit.py create mode 100644 tests/docker/__init__.py create mode 100644 tests/docker/conftest.py create mode 100644 tests/docker/test_ssh_stealth_image.py create mode 100644 tests/factories/__init__.py create mode 100644 tests/factories/campaign_factory.py create mode 100644 tests/fixtures/campaigns/lone_wolf.expected.yaml create mode 100644 tests/fixtures/campaigns/lone_wolf.yaml create mode 100644 tests/fixtures/campaigns/multi_operator.expected.yaml create mode 100644 tests/fixtures/campaigns/multi_operator.yaml create mode 100644 tests/fixtures/campaigns/noise_floor.expected.yaml create mode 100644 tests/fixtures/campaigns/noise_floor.yaml create mode 100644 tests/fixtures/campaigns/paused_campaign.expected.yaml create mode 100644 tests/fixtures/campaigns/paused_campaign.yaml create mode 100644 tests/fixtures/campaigns/shared_wordlist.expected.yaml create mode 100644 tests/fixtures/campaigns/shared_wordlist.yaml create mode 100644 tests/fixtures/campaigns/slow_burn.expected.yaml create mode 100644 tests/fixtures/campaigns/slow_burn.yaml create mode 100644 tests/fixtures/campaigns/vpn_hopping.expected.yaml create mode 100644 tests/fixtures/campaigns/vpn_hopping.yaml create mode 100644 tests/fleet/__init__.py rename tests/{ => fleet}/test_archetypes.py (100%) create mode 100644 tests/fleet/test_auto_spawn.py rename tests/{ => fleet}/test_composer.py (88%) create mode 100644 tests/fleet/test_deployer.py rename tests/{ => fleet}/test_fleet.py (100%) create mode 100644 tests/fleet/test_fleet_singleton.py create mode 100644 tests/fleet/test_reconciler.py create mode 100644 tests/fleet/test_reconciler_worker.py create mode 100644 tests/geoip/__init__.py create mode 100644 tests/geoip/conftest.py create mode 100644 tests/geoip/test_lookup.py create mode 100644 tests/geoip/test_parse.py create mode 100644 tests/geoip/test_profiler_integration.py create mode 100644 tests/geoip/test_provider.py create mode 100644 tests/geoip/test_ptr.py create mode 100644 tests/intel/__init__.py create mode 100644 tests/intel/test_abuseipdb.py create mode 100644 tests/intel/test_attacker_intel_repo.py create mode 100644 tests/intel/test_factory.py create mode 100644 tests/intel/test_feodo.py create mode 100644 tests/intel/test_greynoise.py create mode 100644 tests/intel/test_stealth_http.py create mode 100644 tests/intel/test_threatfox.py create mode 100644 tests/intel/test_worker.py create mode 100644 tests/live/test_health_live.py create mode 100644 tests/live/test_https_live.py create mode 100644 tests/live/test_mysql_backend_live.py create mode 100644 tests/live/test_service_isolation_live.py create mode 100644 tests/logging/__init__.py create mode 100644 tests/logging/test_file_handler.py create mode 100644 tests/logging/test_inode_aware_handler.py rename tests/{ => logging}/test_log_file_mount.py (100%) create mode 100644 tests/logging/test_logging.py rename tests/{ => logging}/test_logging_forwarder.py (100%) create mode 100644 tests/logging/test_syslog_formatter.py create mode 100644 tests/mutator/__init__.py create mode 100644 tests/mutator/test_mutator.py create mode 100755 tests/mysql_spinup.sh create mode 100644 tests/orchestrator/__init__.py create mode 100644 tests/orchestrator/emailgen/__init__.py create mode 100644 tests/orchestrator/emailgen/test_driver.py create mode 100644 tests/orchestrator/emailgen/test_events.py create mode 100644 tests/orchestrator/emailgen/test_repo.py create mode 100644 tests/orchestrator/emailgen/test_scheduler.py create mode 100644 tests/orchestrator/emailgen/test_threads.py create mode 100644 tests/orchestrator/test_realism_config_refresh.py create mode 100644 tests/orchestrator/test_realism_health_snapshot.py create mode 100644 tests/orchestrator/test_repo_pagination.py create mode 100644 tests/orchestrator/test_scheduler.py create mode 100644 tests/orchestrator/test_ssh_driver.py create mode 100644 tests/orchestrator/test_worker_integration.py create mode 100644 tests/perf/README.md create mode 100644 tests/perf/__init__.py create mode 100644 tests/perf/conftest.py create mode 100644 tests/perf/test_repo_bench.py create mode 100644 tests/prober/__init__.py create mode 100644 tests/prober/osfp/__init__.py create mode 100644 tests/prober/osfp/test_format.py create mode 100644 tests/prober/osfp/test_provider.py create mode 100644 tests/prober/osfp/test_signature.py create mode 100644 tests/prober/test_prober_bounty.py create mode 100644 tests/prober/test_prober_bus.py create mode 100644 tests/prober/test_prober_hassh.py create mode 100644 tests/prober/test_prober_jarm.py create mode 100644 tests/prober/test_prober_tcpfp.py create mode 100644 tests/prober/test_prober_tlscert.py create mode 100644 tests/prober/test_prober_worker.py create mode 100644 tests/profiler/__init__.py create mode 100644 tests/profiler/test_attacker_worker.py create mode 100644 tests/profiler/test_identity_rollup.py create mode 100644 tests/profiler/test_profiler_behavioral.py create mode 100644 tests/profiler/test_profiler_bus.py create mode 100644 tests/profiler/test_session_profile.py create mode 100644 tests/realism/__init__.py create mode 100644 tests/realism/test_bodies.py create mode 100644 tests/realism/test_bodies_llm.py create mode 100644 tests/realism/test_circuit_breaker.py create mode 100644 tests/realism/test_diurnal.py create mode 100644 tests/realism/test_edit.py create mode 100644 tests/realism/test_email_prompt.py create mode 100644 tests/realism/test_llm.py create mode 100644 tests/realism/test_naming.py create mode 100644 tests/realism/test_personas.py create mode 100644 tests/realism/test_personas_pool.py create mode 100644 tests/realism/test_planner.py create mode 100644 tests/realism/test_planner_config.py create mode 100644 tests/realism/test_synthetic_files_repo.py create mode 100644 tests/realism/test_synthetic_files_truncation.py create mode 100644 tests/realism/test_taxonomy.py create mode 100644 tests/service_testing/test_imap_spool.py create mode 100644 tests/service_testing/test_instance_seed.py create mode 100644 tests/service_testing/test_pop3_spool.py create mode 100644 tests/service_testing/test_rdp_basic.py create mode 100644 tests/service_testing/test_rdp_nla.py create mode 100644 tests/service_testing/test_smb_server.py create mode 100644 tests/services/__init__.py create mode 100644 tests/services/test_cred_emitters.py rename tests/{ => services}/test_custom_service.py (100%) create mode 100644 tests/services/test_mongodb_scram.py create mode 100644 tests/services/test_ntlmssp_parser.py create mode 100644 tests/services/test_service_isolation.py rename tests/{ => services}/test_services.py (99%) rename tests/{ => services}/test_smtp_relay.py (60%) create mode 100644 tests/services/test_smtp_targets.py create mode 100644 tests/services/test_ssh.py create mode 100644 tests/services/test_ssh_capture_emit.py create mode 100644 tests/services/test_ssh_stealth.py create mode 100644 tests/services/test_syslog_bridge_helpers.py create mode 100644 tests/sniffer/__init__.py create mode 100644 tests/sniffer/test_sniffer_bus.py create mode 100644 tests/sniffer/test_sniffer_ja3.py create mode 100644 tests/sniffer/test_sniffer_p0f.py create mode 100644 tests/sniffer/test_sniffer_retransmit.py create mode 100644 tests/sniffer/test_sniffer_seq_class.py create mode 100644 tests/sniffer/test_sniffer_tcp_fingerprint.py create mode 100644 tests/sniffer/test_sniffer_worker.py create mode 100644 tests/stress/__init__.py create mode 100644 tests/stress/conftest.py create mode 100644 tests/stress/locustfile.py create mode 100644 tests/stress/test_stress.py create mode 100644 tests/swarm/__init__.py create mode 100644 tests/swarm/test_agent_app.py create mode 100644 tests/swarm/test_agent_heartbeat.py create mode 100644 tests/swarm/test_agent_no_auto_restore.py create mode 100644 tests/swarm/test_agent_relocalize.py create mode 100644 tests/swarm/test_agent_topology_endpoints.py create mode 100644 tests/swarm/test_agent_topology_store.py create mode 100644 tests/swarm/test_cli_forwarder.py create mode 100644 tests/swarm/test_cli_swarm.py create mode 100644 tests/swarm/test_cli_swarm_update.py create mode 100644 tests/swarm/test_client_agent_roundtrip.py create mode 100644 tests/swarm/test_client_topology.py create mode 100644 tests/swarm/test_forwarder_resilience.py create mode 100644 tests/swarm/test_heartbeat.py create mode 100644 tests/swarm/test_heartbeat_topology_resync.py create mode 100644 tests/swarm/test_log_forwarder.py create mode 100644 tests/swarm/test_pki.py create mode 100644 tests/swarm/test_state_schema.py create mode 100644 tests/swarm/test_swarm_api.py create mode 100644 tests/swarm/test_tar_tree.py create mode 100644 tests/swarm/test_uvicorn_tls_scope.py create mode 100644 tests/telemetry/__init__.py create mode 100644 tests/telemetry/test_telemetry.py delete mode 100644 tests/test_base_repo.py delete mode 100644 tests/test_collector.py delete mode 100644 tests/test_decnet.db-shm delete mode 100644 tests/test_decnet.db-wal delete mode 100644 tests/test_deployer.py delete mode 100644 tests/test_ingester.py delete mode 100644 tests/test_mutator.py delete mode 100644 tests/test_ssh.py create mode 100644 tests/topology/__init__.py create mode 100644 tests/topology/test_allocator.py create mode 100644 tests/topology/test_compose.py create mode 100644 tests/topology/test_concurrency.py create mode 100644 tests/topology/test_deploy.py create mode 100644 tests/topology/test_deploy_agent_branch.py create mode 100644 tests/topology/test_editing.py create mode 100644 tests/topology/test_generator.py create mode 100644 tests/topology/test_hashing.py create mode 100644 tests/topology/test_layout.py create mode 100644 tests/topology/test_mutator.py create mode 100644 tests/topology/test_persistence.py create mode 100644 tests/topology/test_reaper.py create mode 100644 tests/topology/test_repo.py create mode 100644 tests/topology/test_resync_reconcile.py create mode 100644 tests/topology/test_service_config.py create mode 100644 tests/topology/test_status.py create mode 100644 tests/topology/test_validate.py create mode 100644 tests/updater/__init__.py create mode 100644 tests/updater/test_updater_app.py create mode 100644 tests/updater/test_updater_executor.py create mode 100644 tests/vectorstore/__init__.py create mode 100644 tests/vectorstore/test_factory.py create mode 100644 tests/vectorstore/test_fake.py create mode 100644 tests/web/__init__.py create mode 100644 tests/web/services/__init__.py create mode 100644 tests/web/services/test_systemd_control.py create mode 100644 tests/web/test_admin_seed.py create mode 100644 tests/web/test_api_attacker_intel.py create mode 100644 tests/web/test_api_attackers.py create mode 100644 tests/web/test_api_campaigns.py create mode 100644 tests/web/test_api_identities.py create mode 100644 tests/web/test_api_startup_guards.py create mode 100644 tests/web/test_auth_async.py create mode 100644 tests/web/test_env_lazy_jwt.py create mode 100644 tests/web/test_health_config_cache.py create mode 100644 tests/web/test_ingester.py create mode 100644 tests/web/test_ingester_bus.py create mode 100644 tests/web/test_ingester_http_quirks.py create mode 100644 tests/web/test_ingester_ua_classify.py create mode 100644 tests/web/test_ingester_xff.py create mode 100644 tests/web/test_router_cache.py create mode 100644 tests/web/test_validate_public_binding.py rename tests/{ => web}/test_web_api.py (89%) create mode 100644 tests/webhook/__init__.py create mode 100644 tests/webhook/conftest.py create mode 100644 tests/webhook/test_client.py create mode 100644 tests/webhook/test_enums.py create mode 100644 tests/webhook/test_worker.py diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 4fd723bf..1602429c 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: push: - branches: [dev, testing, "temp/merge-*"] + branches: [dev, testing] paths-ignore: - "**/*.md" - "docs/**" @@ -11,28 +11,31 @@ jobs: lint: name: Lint (ruff) runs-on: ubuntu-latest + if: github.ref == 'refs/heads/dev' steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.11" - run: pip install ruff - - run: ruff check . + - run: ruff check decnet/ bandit: name: SAST (bandit) runs-on: ubuntu-latest + if: github.ref == 'refs/heads/dev' steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.11" - run: pip install bandit - - run: bandit -r decnet/ -ll -x decnet/services/registry.py + - run: bandit -r decnet/ -ll -x decnet/services/registry.py -x decnet/templates/ pip-audit: name: Dependency audit (pip-audit) runs-on: ubuntu-latest + if: github.ref == 'refs/heads/dev' steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -40,57 +43,12 @@ jobs: python-version: "3.11" - run: pip install pip-audit - run: pip install -e .[dev] - - run: pip-audit --skip-editable - - test-standard: - name: Test (Standard) - runs-on: ubuntu-latest - needs: [lint, bandit, pip-audit] - strategy: - matrix: - python-version: ["3.11", "3.12"] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - run: pip install -e .[dev] - - run: pytest - - test-live: - name: Test (Live) - runs-on: ubuntu-latest - needs: [test-standard] - strategy: - matrix: - python-version: ["3.11"] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - run: pip install -e .[dev] - - run: pytest -m live - - test-fuzz: - name: Test (Fuzz) - runs-on: ubuntu-latest - needs: [test-live] - strategy: - matrix: - python-version: ["3.11"] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - run: pip install -e .[dev] - - run: pytest -m fuzz + - run: pip-audit --skip-editable --ignore-vuln CVE-2025-65896 --ignore-vuln CVE-2026-3219 merge-to-testing: name: Merge dev → testing runs-on: ubuntu-latest - needs: [test-standard, test-live, test-fuzz] + needs: [lint, bandit, pip-audit] if: github.ref == 'refs/heads/dev' steps: - uses: actions/checkout@v4 @@ -105,13 +63,63 @@ jobs: run: | git fetch origin testing git checkout testing - git merge origin/dev --no-ff -m "ci: auto-merge dev → testing [skip ci]" + git merge origin/dev --no-ff -m "ci: auto-merge dev → testing" git push origin testing - prepare-merge-to-main: - name: Prepare Merge to Main + test-standard: + name: Test (Standard) runs-on: ubuntu-latest - needs: [test-standard, test-live, test-fuzz] + if: github.ref == 'refs/heads/testing' + strategy: + matrix: + python-version: ["3.11"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - run: pip install -e .[dev] + - run: pytest + + test-live: + name: Test (Live) + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/testing' + needs: [test-standard] + services: + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: decnet_test + ports: + - 3307:3306 + options: >- + --health-cmd="mysqladmin ping -h 127.0.0.1" + --health-interval=10s + --health-timeout=5s + --health-retries=5 + strategy: + matrix: + python-version: ["3.11"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - run: pip install -e .[dev] + - run: pytest -m live + env: + DECNET_MYSQL_HOST: 127.0.0.1 + DECNET_MYSQL_PORT: 3307 + DECNET_MYSQL_USER: root + DECNET_MYSQL_PASSWORD: root + DECNET_MYSQL_DATABASE: decnet_test + + merge-to-main: + name: Merge testing → main + runs-on: ubuntu-latest + needs: [test-standard, test-live] if: github.ref == 'refs/heads/testing' steps: - uses: actions/checkout@v4 @@ -122,33 +130,12 @@ jobs: run: | git config user.name "DECNET CI" git config user.email "ci@decnet.local" - - name: Create temp branch and sync with main - run: | - git fetch origin main - git checkout -b temp/merge-testing-to-main - echo "--- Switched to temp branch, merging main into it ---" - git merge origin/main --no-edit || { echo "CONFLICT: Manual resolution required"; exit 1; } - git push origin temp/merge-testing-to-main --force - - finalize-merge-to-main: - name: Finalize Merge to Main - runs-on: ubuntu-latest - needs: [test-standard, test-live, test-fuzz] - if: startsWith(github.ref, 'refs/heads/temp/merge-') - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.DECNET_PR_TOKEN }} - - name: Configure git - run: | - git config user.name "DECNET CI" - git config user.email "ci@decnet.local" - - name: Merge RC into main + - name: Merge testing into main run: | git fetch origin main git checkout main - git merge ${{ github.ref }} --no-ff -m "ci: auto-merge testing → main" + git merge origin/testing --no-ff -m "ci: auto-merge testing → main" || { + echo "CONFLICT: testing and main have diverged — manual resolution required" + exit 1 + } git push origin main - echo "--- Cleaning up temp branch ---" - git push origin --delete ${{ github.ref_name }} diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 0e8ff4b2..cbe6ec68 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -33,13 +33,13 @@ jobs: id: version run: | # Calculate next version (v0.x) - LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0") + LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0") NEXT_VER=$(python3 -c " tag = '$LATEST_TAG'.lstrip('v') parts = tag.split('.') major = int(parts[0]) if parts[0] else 0 minor = int(parts[1]) if len(parts) > 1 else 0 - print(f'{major}.{minor + 1}') + print(f'{major}.{minor + 1}.0') ") echo "Next version: $NEXT_VER (calculated from $LATEST_TAG)" @@ -49,7 +49,11 @@ jobs: git add pyproject.toml git commit -m "chore: auto-release v$NEXT_VER [skip ci]" || echo "No changes to commit" - git tag -a "v$NEXT_VER" -m "Auto-release v$NEXT_VER" + CHANGELOG=$(git log ${LATEST_TAG}..HEAD --oneline --no-decorate --no-merges) + git tag -a "v$NEXT_VER" -m "Auto-release v$NEXT_VER + +Changes since $LATEST_TAG: +$CHANGELOG" git push origin main --follow-tags echo "version=$NEXT_VER" >> $GITHUB_OUTPUT @@ -111,13 +115,13 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + - name: Install Trivy + run: | + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin + - name: Scan with Trivy - uses: aquasecurity/trivy-action@master - with: - image-ref: decnet-${{ matrix.service }}:scan - exit-code: "1" - severity: CRITICAL - ignore-unfixed: true + run: | + trivy image --exit-code 1 --severity CRITICAL --ignore-unfixed decnet-${{ matrix.service }}:scan - name: Push image if: success() diff --git a/.gitignore b/.gitignore index c65f265a..bc75c3dd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ .venv/ +.venv*/ +.311/ +.3[0-9][0-9]/ logs/ -.claude/ +.claude/* +CLAUDE.md __pycache__/ *.pyc *.pyo @@ -8,9 +12,12 @@ __pycache__/ dist/ build/ decnet-compose.yml +# Per-topology compose fragments emitted by `decnet topology deploy`. +decnet-topology-*-compose.yml +# Docker build context cache. +.docker/ decnet-state.json *.ini -.env decnet.log* *.loggy *.nmap @@ -18,8 +25,29 @@ linterfails.log webmail windows1 *.db +*.db-shm +*.db-wal +decnet.*.log +# Rotated copies (logrotate appends .1, .2, .gz...) — the existing +# decnet.*.log glob doesn't catch the suffix. +decnet.*.log.* decnet.json -.env +.env* .env.local .coverage .hypothesis/ +profiles/* +tests/test_decnet.db* + +# Nested git clone of the wiki — not a submodule, just a local +# working copy so we can edit docs without a full round-trip. +wiki-checkout/ + +# Scratch test/debug outputs that leak from saved `pytest > hang.log` +# or `pytest > schem` redirections. +hang.log +schem +*.pytest.log + +# pydeps-style dependency graph dumps from local analysis runs. +deps.txt diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index ce87482a..00000000 --- a/CLAUDE.md +++ /dev/null @@ -1,58 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Commands - -```bash -# Install (dev) -pip install -e . - -# List registered service plugins -decnet services - -# Dry-run (generates compose, no containers) -decnet deploy --mode unihost --deckies 3 --randomize-services --dry-run - -# Full deploy (requires root for MACVLAN) -sudo decnet deploy --mode unihost --deckies 5 --interface eth0 --randomize-services -sudo decnet deploy --mode unihost --deckies 3 --services ssh,smb --log-target 192.168.1.5:5140 - -# Status / teardown -decnet status -sudo decnet teardown --all -sudo decnet teardown --id decky-01 -``` - -## Project Overview - -DECNET is a honeypot/deception network framework. It deploys fake machines (called **deckies**) with realistic services (RDP, SMB, SSH, FTP, etc.) to lure and profile attackers. All attacker interactions are aggregated to an isolated logging network (ELK stack / SIEM). - -## Deployment Models - -**UNIHOST** — one real host spins up _n_ deckies via a container orchestrator. Simpler, single-machine deployment. - -**SWARM (MULTIHOST)** — _n_ real hosts each running deckies. Orchestrated via Ansible/sshpass or similar tooling. - -## Core Technology Choices - -- **Containers**: Docker Compose is the starting point but other orchestration frameworks should be evaluated if they serve the project better. `debian:bookworm-slim` is the default base image; mixing in Ubuntu, CentOS, or other distros is encouraged to make the decoy network look heterogeneous. -- **Networking**: Deckies need to appear as real machines on the LAN (own MACs/IPs). MACVLAN and IPVLAN are candidates; the right driver depends on the host environment. WSL has known limitations — bare metal or a VM is preferred for testing. -- **Log pipeline**: Logstash → ELK stack → SIEM (isolated network, not reachable from decoy network) - -## Architecture Constraints - -- The decoy network must be reachable from the outside (attacker-facing). -- The logging/aggregation network must be isolated from the decoy network. -- A publicly accessible real server acts as the bridge between the two networks. -- Deckies should differ in exposed services and OS fingerprints to appear as a heterogeneous network. -- **IMPORTANT**: The system now strictly enforces dependency injection for storage. Do not import `SQLiteRepository` directly in new features; instead, use `get_repository()` from the factory or the FastAPI `get_repo` dependency. - -## Development and testing - -- For every new feature, pytests must me made. -- Pytest is the main testing framework in use. -- NEVER pass broken code to the user. - - Broken means: not running, not passing 100% tests, etc. -- After tests pass with 100%, always git commit your changes. -- NEVER add "Co-Authored-By" or any Claude attribution lines to git commit messages. diff --git a/GEMINI.md b/GEMINI.md deleted file mode 100644 index c3616967..00000000 --- a/GEMINI.md +++ /dev/null @@ -1,104 +0,0 @@ -# DECNET (Deception Network) Project Context - -DECNET is a high-fidelity honeypot framework designed to deploy heterogeneous fleets of fake machines (called **deckies**) that appear as real hosts on a local network. - -## Project Overview - -- **Core Purpose:** To lure, profile, and log attacker interactions within a controlled, deceptive environment. -- **Key Technology:** Linux-native container networking (MACVLAN/IPvlan) combined with Docker to give each decoy its own MAC address, IP, and realistic TCP/IP stack behavior. -- **Main Components:** - - **Deckies:** Group of containers sharing a network namespace (one base container + multiple service containers). - - **Archetypes:** Pre-defined machine profiles (e.g., `windows-workstation`, `linux-server`) that bundle services and OS fingerprints. - - **Services:** Modular honeypot plugins (SSH, SMB, RDP, etc.) built as `BaseService` subclasses. - - **OS Fingerprinting:** Sysctl-based TCP/IP stack tuning to spoof OS detection (nmap). - - **Logging Pipeline:** RFC 5424 syslog forwarding to an isolated SIEM/ELK stack. - -## Technical Stack - -- **Language:** Python 3.11+ -- **CLI Framework:** [Typer](https://typer.tiangolo.com/) -- **Data Validation:** [Pydantic v2](https://docs.pydantic.dev/) -- **Orchestration:** Docker Engine 24+ (via Docker SDK for Python) -- **Networking:** MACVLAN (default) or IPvlan L2 (for WiFi/restricted environments). -- **Testing:** Pytest (100% pass requirement). -- **Formatting/Linting:** Ruff, Bandit (SAST), pip-audit. - -## Architecture - -```text -Host NIC (eth0) - └── MACVLAN Bridge - ├── Decky-01 (192.168.1.10) -> [Base] + [SSH] + [HTTP] - ├── Decky-02 (192.168.1.11) -> [Base] + [SMB] + [RDP] - └── ... -``` - -- **Base Container:** Owns the IP/MAC, sets `sysctls` for OS spoofing, and runs `sleep infinity`. -- **Service Containers:** Use `network_mode: service:` to share the identity and networking of the base container. -- **Isolation:** Decoy traffic is strictly separated from the logging network. - -## Key Commands - -### Development & Maintenance -- **Install (Dev):** - - `rm .venv -rf` - - `python3 -m venv .venv` - - `source .venv/bin/activate` - - `pip install -e .` -- **Run Tests:** `pytest` (Run before any commit) -- **Linting:** `ruff check .` -- **Security Scan:** `bandit -r decnet/` -- **Web Git:** git.resacachile.cl (Gitea) - -### CLI Usage -- **List Services:** `decnet services` -- **List Archetypes:** `decnet archetypes` -- **Dry Run (Compose Gen):** `decnet deploy --deckies 3 --randomize-services --dry-run` -- **Deploy (Full):** `sudo .venv/bin/decnet deploy --interface eth0 --deckies 5 --randomize-services` -- **Status:** `decnet status` -- **Teardown:** `sudo .venv/bin/decnet teardown --all` - -## Development Conventions - -- **Code Style:** - - Strict adherence to Ruff/PEP8. - - **Always use typed variables**. If any non-types variables are found, they must be corrected. - - The correct way is `x: int = 1`, never `x : int = 1`. - - If assignment is present, always use a space between the type and the equal sign `x: int = 1`. - - **Never** use lowercase L (l), uppercase o (O) or uppercase i (i) in single-character names. - - **Internal vars are to be declared with an underscore** (_internal_variable_name). - - **Internal to internal vars are to be declared with double underscore** (__internal_variable_name). - - Always use snake_case for code. - - Always use PascalCase for classes and generics. -- **Testing:** New features MUST include a `pytest` case. 100% test pass rate is mandatory before merging. -- **Plugin System:** - - New services go in `decnet/services/.py`. - - Subclass `decnet.services.base.BaseService`. - - The registry uses auto-discovery; no manual registration required. -- **Configuration:** - - Use Pydantic models in `decnet/config.py` for any new settings. - - INI file parsing is handled in `decnet/ini_loader.py`. -- **State Management:** - - Runtime state is persisted in `decnet-state.json`. - - Do not modify this file manually. -- **General Development Guidelines**: - - **Never** commit broken code, or before running `pytest`s or `bandit` at the project level. - - **No matter how small** the changes, they must be committed. - - **If new features are addedd** new tests must be added, too. - - **Never present broken code to the user**. Test, validate, then present. - - **Extensive testing** for every function must be created. - - **Always develop in the `dev` branch, never in `main`.** - - **Test in the `testing` branch.** - - **IMPORTANT**: The system now strictly enforces dependency injection for storage. Do not import `SQLiteRepository` directly in new features; instead, use `get_repository()` from the factory or the FastAPI `get_repo` dependency. - -## Directory Structure - -- `decnet/`: Main source code. - - `services/`: Honeypot service implementations. - - `logging/`: Syslog formatting and forwarding logic. - - `correlation/`: (In Progress) Logic for grouping attacker events. -- `templates/`: Dockerfiles and entrypoint scripts for services. -- `tests/`: Pytest suite. -- `pyproject.toml`: Dependency and entry point definitions. -- `CLAUDE.md`: Claude-specific environment guidance. -- `DEVELOPMENT.md`: Roadmap and TODOs. diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..f288702d --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md index a17674dc..bce15306 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ A honeypot deception network framework. Spin up a fleet of fake machines — cal Attackers probe the network, DECNET traps every interaction, and you watch from a safe, isolated logging stack. +[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/C0C31YDLB5) + --- ## Table of Contents @@ -514,6 +516,10 @@ DECNET_WEB_HOST=0.0.0.0 DECNET_WEB_PORT=8080 DECNET_ADMIN_USER=admin DECNET_ADMIN_PASSWORD=admin + +# Database pool tuning (applies to both SQLite and MySQL) +DECNET_DB_POOL_SIZE=20 # base pool connections (default: 20) +DECNET_DB_MAX_OVERFLOW=40 # extra connections under burst (default: 40) ``` Copy `.env.example` to `.env.local` and modify it to suit your environment. @@ -682,6 +688,112 @@ The test suite covers: Every new feature requires passing tests before merging. +### Stress Testing + +A [Locust](https://locust.io)-based stress test suite lives in `tests/stress/`. It hammers every API endpoint with realistic traffic patterns to find throughput ceilings and latency degradation. + +```bash +# Run via pytest (starts its own server) +pytest -m stress tests/stress/ -v -x -n0 -s + +# Crank it up +STRESS_USERS=2000 STRESS_SPAWN_RATE=200 STRESS_DURATION=120 pytest -m stress tests/stress/ -v -x -n0 -s + +# Standalone Locust web UI against a running server +locust -f tests/stress/locustfile.py --host http://localhost:8000 +``` + +| Env var | Default | Description | +|---|---|---| +| `STRESS_USERS` | `500` | Total simulated users | +| `STRESS_SPAWN_RATE` | `50` | Users spawned per second | +| `STRESS_DURATION` | `60` | Test duration in seconds | +| `STRESS_WORKERS` | CPU count (max 4) | Uvicorn workers for the test server | +| `STRESS_MIN_RPS` | `500` | Minimum RPS to pass baseline test | +| `STRESS_MAX_P99_MS` | `200` | Maximum p99 latency (ms) to pass | +| `STRESS_SPIKE_USERS` | `1000` | Users for thundering herd test | +| `STRESS_SUSTAINED_USERS` | `200` | Users for sustained load test | + +#### Measured baseline + +Reference numbers from recent Locust runs against a MySQL backend +(asyncmy driver). All runs hold zero failures throughout. + +**Single worker** (unless noted): + +| Metric | 500u, tracing on | 1500u, tracing on | 1500u, tracing **off** | 1500u, tracing off, **pinned to 1 core** | 1500u, tracing off, **12 workers** | +|---|---|---|---|---|---| +| Requests served | 396,672 | 232,648 | 277,214 | 3,532 | 308,024 | +| Failures | 0 | 0 | 0 | 0 | 0 | +| Throughput (current RPS) | ~960 | ~880 | ~990 | ~46 | ~1,585 | +| Average latency | 465 ms | 1,774 ms | 1,489 ms | 21.7 s | 930 ms | +| Median (p50) | 100 ms | 690 ms | 340 ms | 270 ms | 700 ms | +| p95 | 1.9 s | 6.5 s | 5.7 s | 115 s | 2.7 s | +| p99 | 2.9 s | 9.5 s | 8.4 s | 122 s | 4.2 s | +| Max observed | 8.3 s | 24.4 s | 20.9 s | 124.5 s | 16.5 s | + +Ramp is 15 users/s for the 500u column, 40 users/s otherwise. + +Takeaways: + +- **Tracing off**: at 1500 users, flipping `DECNET_TRACING=false` + halves p50 (690 → 340 ms) and pushes RPS from ~880 past the + 500-user figure on a single worker. +- **12 workers**: RPS scales ~1.6× over a single worker (~990 → + ~1585). Sublinear because the workload is DB-bound — MySQL and the + connection pool become the new ceiling, not Python. p99 drops from + 8.4 s to 4.2 s. +- **Connection math**: `DECNET_DB_POOL_SIZE=20` × `DECNET_DB_MAX_OVERFLOW=40` + × 12 workers = 720 connections at peak. MySQL's default + `max_connections=151` needs bumping (we used 2000) before running + multi-worker load. +- **Single-core pinning**: ~46 RPS with p95 near two minutes. Interesting + as a "physics floor" datapoint — not a production config. + +Top endpoints by volume: `/api/v1/attackers`, `/api/v1/deckies`, +`/api/v1/bounty`, `/api/v1/logs/histogram`, `/api/v1/config`, +`/api/v1/health`, `/api/v1/auth/login`, `/api/v1/logs`. + +Notes on tuning: + +- **Python 3.14 is currently a no-go for the API server.** Under heavy + concurrent async load the reworked 3.14 GC segfaults inside + `mark_all_reachable` (observed in `_PyGC_Collect` during pending-GC + on 3.14.3). Stick to Python 3.11–3.13 until upstream stabilises. +- Router-level TTL caches on hot count/stats endpoints (`/stats`, + `/logs` count, `/attackers` count, `/bounty`, `/logs/histogram`, + `/deckies`, `/config`) collapse concurrent duplicate work onto a + single DB hit per window — essential to reach this RPS on one worker. +- Turning off request tracing (`DECNET_TRACING=false`) is the next + free headroom: tracing was still on during the run above. +- On SQLite, `DECNET_DB_POOL_PRE_PING=false` skips the per-checkout + `SELECT 1`. On MySQL, keep it `true` — network disconnects are real. + +#### System tuning: open file limit + +Under heavy load (500+ concurrent users), the server will exhaust the default Linux open file limit (`ulimit -n`), causing `OSError: [Errno 24] Too many open files`. Most distros default to **1024**, which is far too low for stress testing or production use. + +**Before running stress tests:** + +```bash +# Check current limit +ulimit -n + +# Bump for this shell session +ulimit -n 65536 +``` + +**Permanent fix** — add to `/etc/security/limits.conf`: + +``` +* soft nofile 65536 +* hard nofile 65536 +``` + +Or for systemd-managed services, add `LimitNOFILE=65536` to the unit file. + +> This applies to production deployments too — any server handling hundreds of concurrent connections needs a raised file descriptor limit. + # AI Disclosure This project has been made with lots, and I mean lots of help from AIs. While most of the design was made by me, most of the coding was done by AI models. diff --git a/decnet.collector.log b/decnet.collector.log deleted file mode 100644 index bac1371f..00000000 --- a/decnet.collector.log +++ /dev/null @@ -1 +0,0 @@ -Collector starting → /home/anti/Tools/DECNET/decnet.log diff --git a/decnet.ini.example b/decnet.ini.example new file mode 100644 index 00000000..21698964 --- /dev/null +++ b/decnet.ini.example @@ -0,0 +1,64 @@ +; /etc/decnet/decnet.ini — DECNET host configuration +; +; Copy to /etc/decnet/decnet.ini and edit. Values here seed os.environ at +; CLI startup via setdefault() — real env vars still win, so you can +; override any value on the shell without editing this file. +; +; A missing file is fine; every daemon has sensible defaults. The main +; reason to use this file is to skip typing the same flags on every +; `decnet` invocation and to pin a host's role via `mode`. + +[decnet] +; mode = agent | master +; agent — worker host (runs `decnet agent`, `decnet forwarder`, `decnet updater`). +; Master-only commands (api, swarmctl, swarm, deploy, teardown, ...) +; are hidden from `decnet --help` and refuse to run. +; master — central server (runs `decnet api`, `decnet web`, `decnet swarmctl`, +; `decnet listener`). All commands visible. +mode = agent + +; disallow-master = true (default when mode=agent) +; Set to false for hybrid dev hosts that legitimately run both roles. +disallow-master = true + +; log-directory — root for DECNET's per-component logs. Systemd units set +; DECNET_SYSTEM_LOGS=/decnet..log so agent, forwarder, +; and engine each get their own file. The forwarder tails decnet.log. +log-directory = /var/log/decnet + + +; ─── Agent-only settings (read when mode=agent) ─────────────────────────── +[agent] +; Where the master's syslog-TLS listener lives. DECNET_SWARM_MASTER_HOST. +master-host = 192.168.1.50 +; Master listener port (RFC 5425 default 6514). DECNET_SWARM_SYSLOG_PORT. +swarm-syslog-port = 6514 +; Bind address/port for this worker's agent API (mTLS). +agent-port = 8765 +; Cert bundle dir — must contain ca.crt, worker.crt, worker.key from enroll. +; DECNET_AGENT_DIR — honored by the forwarder child as well. +agent-dir = /home/anti/.decnet/agent +; Updater cert bundle (required for `decnet updater`). +updater-dir = /home/anti/.decnet/updater + + +; ─── Master-only settings (read when mode=master) ───────────────────────── +[master] +; Main API (REST for the React dashboard). DECNET_API_HOST / _PORT. +api-host = 0.0.0.0 +api-port = 8000 +; React dev-server dashboard (`decnet web`). DECNET_WEB_HOST / _PORT. +web-host = 0.0.0.0 +web-port = 8080 +; Swarm controller (master-internal). DECNET_SWARMCTL_HOST isn't exposed +; under that name today — this block is the forward-compatible spelling. +; swarmctl-host = 127.0.0.1 +; swarmctl-port = 8770 +; Syslog-over-TLS listener bind address and port. DECNET_LISTENER_HOST and +; DECNET_SWARM_SYSLOG_PORT. The listener is auto-spawned by `decnet swarmctl`. +listener-host = 0.0.0.0 +swarm-syslog-port = 6514 +; Master CA dir (for enroll / swarm cert issuance). +; ca-dir = /home/anti/.decnet/ca +; JWT secret for the web API. MUST be set; 32+ bytes. Keep out of git. +; jwt-secret = REPLACE_ME_WITH_A_32_BYTE_SECRET diff --git a/decnet/__init__.py b/decnet/__init__.py index e69de29b..999a57b7 100644 --- a/decnet/__init__.py +++ b/decnet/__init__.py @@ -0,0 +1,12 @@ +"""DECNET — honeypot deception-network framework. + +This __init__ runs once, on the first `import decnet.*`. It seeds +os.environ from /etc/decnet/decnet.ini (if present) so that later +module-level reads in decnet.env pick up the INI values as if they had +been exported by the shell. Real env vars always win via setdefault(). + +Kept minimal on purpose — any heavier work belongs in a submodule. +""" +from decnet.config_ini import load_ini_config as _load_ini_config + +_load_ini_config() diff --git a/decnet/agent/__init__.py b/decnet/agent/__init__.py new file mode 100644 index 00000000..6d65c0f4 --- /dev/null +++ b/decnet/agent/__init__.py @@ -0,0 +1,7 @@ +"""DECNET worker agent — runs on every SWARM worker host. + +Exposes an mTLS-protected FastAPI service the master's SWARM controller +calls to deploy, mutate, and tear down deckies locally. The agent reuses +the existing `decnet.engine.deployer` code path unchanged, so a worker runs +deckies the same way `decnet deploy --mode unihost` does today. +""" diff --git a/decnet/agent/app.py b/decnet/agent/app.py new file mode 100644 index 00000000..8302314d --- /dev/null +++ b/decnet/agent/app.py @@ -0,0 +1,320 @@ +"""Worker-side FastAPI app. + +Protected by mTLS at the ASGI/uvicorn transport layer: uvicorn is started +with ``--ssl-ca-certs`` + ``--ssl-cert-reqs 2`` (CERT_REQUIRED), so any +client that cannot prove a cert signed by the DECNET CA is rejected before +reaching a handler. Once past the TLS handshake, all peers are trusted +equally (the only entity holding a CA-signed cert is the master +controller). + +Endpoints mirror the existing unihost CLI verbs: + +* ``POST /deploy`` — body: serialized ``DecnetConfig`` +* ``POST /teardown`` — body: optional ``{"decky_id": "..."}`` +* ``POST /mutate`` — body: ``{"decky_id": "...", "services": [...]}`` +* ``GET /status`` — deployment snapshot +* ``GET /health`` — liveness probe, does NOT require mTLS? No — mTLS + still required; master pings it with its cert. +""" +from __future__ import annotations + +import asyncio +import os +import pathlib +from contextlib import asynccontextmanager +from typing import Any, Optional + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel, Field + +import contextlib + +from decnet.agent import executor as _exec +from decnet.agent import heartbeat as _heartbeat +from decnet.agent import topology_ops as _topology_ops +from decnet.bus.factory import get_bus +from decnet.bus.publish import run_health_heartbeat +from decnet.swarm.pki import DEFAULT_AGENT_DIR +from decnet.agent.topology_store import AlreadyApplied, TopologyStore +from decnet.config import DecnetConfig +from decnet.logging import get_logger +from decnet.topology.validate import ValidationError + +log = get_logger("agent.app") + + +def _resolve_agent_dir() -> pathlib.Path: + env = os.environ.get("DECNET_AGENT_DIR") + if env: + return pathlib.Path(env) + system = pathlib.Path("/etc/decnet/agent") + if system.exists(): + return system + return DEFAULT_AGENT_DIR + + +# Module-level singleton. Created lazily on first use so tests can +# monkeypatch DECNET_AGENT_DIR before the store binds to a path. +_topology_store: Optional[TopologyStore] = None + + +def _store() -> TopologyStore: + global _topology_store + if _topology_store is None: + _topology_store = TopologyStore(_resolve_agent_dir() / "topology.db") + return _topology_store + + +_collector_task: Optional[asyncio.Task] = None + + +def _ensure_collector_started() -> None: + """Spawn the log collector on demand — called from /topology/apply + after a successful materialise. We must NOT start this in the + lifespan hook: the agent's boot invariant is "never touch docker + until master tells us to" (see tests/swarm/test_agent_no_auto_restore.py). + + The collector watches ``decnet.topology.service=true`` labels via + docker events, writing RFC 5424 lines to ``DECNET_AGENT_LOG_FILE`` + which the forwarder ships to the master over syslog-TLS. Idempotent: + subsequent calls while the task is still running are no-ops. + """ + global _collector_task + if _collector_task is not None and not _collector_task.done(): + return + from decnet.env import DECNET_AGENT_LOG_FILE + + try: + from decnet.collector.worker import log_collector_worker + except Exception: # noqa: BLE001 — docker may be unavailable on dev + log.warning( + "agent log collector not starting — collector worker import failed", + exc_info=True, + ) + return + _collector_task = asyncio.create_task( + log_collector_worker(DECNET_AGENT_LOG_FILE), + name="agent-log-collector", + ) + log.info("agent log collector started log_file=%s", DECNET_AGENT_LOG_FILE) + + +_bus_heartbeat_task: Optional[asyncio.Task] = None + + +@asynccontextmanager +async def _lifespan(app: FastAPI): + # Best-effort: if identity/bundle plumbing isn't configured (e.g. dev + # runs or non-enrolled hosts), heartbeat.start() is a silent no-op. + _heartbeat.start() + + # Host-local bus heartbeat (system.agent.health). Separate channel + # from the mTLS master-facing heartbeat above; this one lets peers on + # the same host (dashboard, updater) see the agent is alive without + # hitting its HTTPS endpoint. Bus-disabled path is a no-op loop. + bus = None + try: + bus = get_bus(client_name="agent") + await bus.connect() + except Exception as exc: # noqa: BLE001 + log.warning("agent: bus unavailable, skipping health heartbeat: %s", exc) + bus = None + + global _bus_heartbeat_task + _bus_heartbeat_task = asyncio.create_task( + run_health_heartbeat(bus, "agent"), + name="agent-bus-heartbeat", + ) + + try: + yield + finally: + await _heartbeat.stop() + if _bus_heartbeat_task is not None: + _bus_heartbeat_task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await _bus_heartbeat_task + _bus_heartbeat_task = None + if bus is not None: + with contextlib.suppress(Exception): + await bus.close() + global _collector_task + if _collector_task is not None and not _collector_task.done(): + _collector_task.cancel() + try: + await _collector_task + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + _collector_task = None + global _topology_store + if _topology_store is not None: + _topology_store.close() + _topology_store = None + + +app = FastAPI( + title="DECNET SWARM Agent", + version="0.1.0", + docs_url=None, # no interactive docs on worker — narrow attack surface + redoc_url=None, + openapi_url=None, + lifespan=_lifespan, + responses={ + 400: {"description": "Malformed request body"}, + 500: {"description": "Executor error"}, + }, +) + + +# ------------------------------------------------------------------ schemas + +class DeployRequest(BaseModel): + config: DecnetConfig = Field(..., description="Full DecnetConfig to materialise on this worker") + dry_run: bool = False + no_cache: bool = False + + +class TeardownRequest(BaseModel): + decky_id: Optional[str] = None + + +class MutateRequest(BaseModel): + decky_id: str + services: list[str] + + +# ------------------------------------------------------------------ routes + +@app.get("/health") +async def health() -> dict[str, str]: + return {"status": "ok"} + + +@app.get("/status") +async def status() -> dict: + return await _exec.status() + + +@app.post( + "/deploy", + responses={500: {"description": "Deployer raised an exception materialising the config"}}, +) +async def deploy(req: DeployRequest) -> dict: + try: + await _exec.deploy(req.config, dry_run=req.dry_run, no_cache=req.no_cache) + except Exception as exc: + log.exception("agent.deploy failed") + raise HTTPException(status_code=500, detail=str(exc)) from exc + return {"status": "deployed", "deckies": len(req.config.deckies)} + + +@app.post( + "/teardown", + responses={500: {"description": "Teardown raised an exception"}}, +) +async def teardown(req: TeardownRequest) -> dict: + try: + await _exec.teardown(req.decky_id) + except Exception as exc: + log.exception("agent.teardown failed") + raise HTTPException(status_code=500, detail=str(exc)) from exc + return {"status": "torn_down", "decky_id": req.decky_id} + + +@app.post( + "/self-destruct", + responses={500: {"description": "Reaper could not be scheduled"}}, +) +async def self_destruct() -> dict: + """Stop all DECNET services on this worker and delete the install + footprint. Called by the master during decommission. Logs under + /var/log/decnet* are preserved. Fire-and-forget — returns 202 before + the reaper starts deleting files.""" + try: + await _exec.self_destruct() + except Exception as exc: + log.exception("agent.self_destruct failed") + raise HTTPException(status_code=500, detail=str(exc)) from exc + return {"status": "self_destruct_scheduled"} + + +# ------------------------------------------------------- topology endpoints + + +class ApplyTopologyRequest(BaseModel): + hydrated: dict[str, Any] = Field( + ..., description="Hydrated topology dict from master.persistence.hydrate()" + ) + version_hash: str = Field( + ..., description="Master's canonical_hash(hydrated); must match ours" + ) + + +class TeardownTopologyRequest(BaseModel): + topology_id: str = Field(..., description="Topology UUID to dismantle") + + +@app.post( + "/topology/apply", + responses={ + 400: {"description": "Malformed hydrated topology or hash mismatch"}, + 409: {"description": "A different topology is already applied"}, + 500: {"description": "Docker or compose raised while applying"}, + }, +) +async def topology_apply(req: ApplyTopologyRequest) -> dict: + store = _store() + try: + await _topology_ops.apply(req.hydrated, req.version_hash, store) + except _topology_ops.HashMismatch as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except ValidationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except AlreadyApplied as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except Exception as exc: + log.exception("agent.topology_apply failed") + topology_id = (req.hydrated.get("topology") or {}).get("id") + if topology_id: + try: + store.record_error( + str(topology_id), str(exc)[:500], hydrated=req.hydrated, + ) + except Exception: # noqa: BLE001 — don't mask original failure + log.exception("failed to record apply error") + raise HTTPException(status_code=500, detail=str(exc)) from exc + _ensure_collector_started() + return {"status": "applied", "version_hash": req.version_hash} + + +@app.post( + "/topology/teardown", + responses={500: {"description": "Docker or compose raised while tearing down"}}, +) +async def topology_teardown(req: TeardownTopologyRequest) -> dict: + try: + await _topology_ops.teardown(req.topology_id, _store()) + except Exception as exc: + log.exception("agent.topology_teardown failed") + raise HTTPException(status_code=500, detail=str(exc)) from exc + return {"status": "torn_down", "topology_id": req.topology_id} + + +@app.get("/topology/state") +async def topology_state() -> dict: + return _topology_ops.state(_store()) + + +@app.post( + "/mutate", + responses={501: {"description": "Worker-side mutate not yet implemented"}}, +) +async def mutate(req: MutateRequest) -> dict: + # TODO: implement worker-side mutate. Currently the master performs + # mutation by re-sending a full /deploy with the updated DecnetConfig; + # this avoids duplicating mutation logic on the worker for v1. When + # ready, replace the 501 with a real redeploy-of-a-single-decky path. + raise HTTPException( + status_code=501, + detail="Per-decky mutate is performed via /deploy with updated services", + ) diff --git a/decnet/agent/executor.py b/decnet/agent/executor.py new file mode 100644 index 00000000..9e1c31bb --- /dev/null +++ b/decnet/agent/executor.py @@ -0,0 +1,223 @@ +"""Thin adapter between the agent's HTTP endpoints and the existing +``decnet.engine.deployer`` code path. + +Kept deliberately small: the agent does not re-implement deployment logic, +it only translates a master RPC into the same function calls the unihost +CLI already uses. Everything runs in a worker thread (the deployer is +blocking) so the FastAPI event loop stays responsive. +""" +from __future__ import annotations + +import asyncio +from ipaddress import IPv4Network +from typing import Any + +from decnet.engine import deployer as _deployer +from decnet.config import DecnetConfig, load_state, clear_state +from decnet.logging import get_logger +from decnet.network import ( + allocate_ips, + detect_interface, + detect_subnet, + get_host_ip, +) + +log = get_logger("agent.executor") + + +def _relocalize(config: DecnetConfig) -> DecnetConfig: + """Rewrite a master-built config to the worker's local network reality. + + The master populates ``interface``/``subnet``/``gateway`` from its own + box before dispatching, which blows up the deployer on any worker whose + NIC name differs (common in heterogeneous fleets — master on ``wlp6s0``, + worker on ``enp0s3``). We always re-detect locally; if the worker sits + on a different subnet than the master, decky IPs are re-allocated from + the worker's subnet so they're actually reachable. + """ + local_iface = detect_interface() + local_subnet, local_gateway = detect_subnet(local_iface) + local_host_ip = get_host_ip(local_iface) + + updates: dict[str, Any] = { + "interface": local_iface, + "subnet": local_subnet, + "gateway": local_gateway, + } + + master_net = IPv4Network(config.subnet, strict=False) if config.subnet else None + local_net = IPv4Network(local_subnet, strict=False) + if master_net is None or master_net != local_net: + log.info( + "agent.deploy subnet mismatch master=%s local=%s — re-allocating decky IPs", + config.subnet, local_subnet, + ) + fresh_ips = allocate_ips( + subnet=local_subnet, + gateway=local_gateway, + host_ip=local_host_ip, + count=len(config.deckies), + ) + new_deckies = [d.model_copy(update={"ip": ip}) for d, ip in zip(config.deckies, fresh_ips)] + updates["deckies"] = new_deckies + + return config.model_copy(update=updates) + + +async def deploy(config: DecnetConfig, dry_run: bool = False, no_cache: bool = False) -> None: + """Run the blocking deployer off-loop. The deployer itself calls + save_state() internally once the compose file is materialised.""" + log.info( + "agent.deploy mode=%s deckies=%d interface=%s (incoming)", + config.mode, len(config.deckies), config.interface, + ) + if config.mode == "swarm": + config = _relocalize(config) + log.info( + "agent.deploy relocalized interface=%s subnet=%s gateway=%s", + config.interface, config.subnet, config.gateway, + ) + await asyncio.to_thread(_deployer.deploy, config, dry_run, no_cache, False) + + +async def teardown(decky_id: str | None = None) -> None: + log.info("agent.teardown decky_id=%s", decky_id) + await asyncio.to_thread(_deployer.teardown, decky_id) + if decky_id is None: + await asyncio.to_thread(clear_state) + + +def _decky_runtime_states(config: DecnetConfig) -> dict[str, dict[str, Any]]: + """Map decky_name → {"running": bool, "services": {svc: container_state}}. + + Queried so the master can tell, after a partial-failure deploy, which + deckies actually came up instead of tainting the whole shard as failed. + Best-effort: a docker error returns an empty map, not an exception. + """ + try: + import docker # local import — agent-only path + client = docker.from_env() + live = {c.name: c.status for c in client.containers.list(all=True, ignore_removed=True)} + except Exception: # pragma: no cover — defensive + log.exception("_decky_runtime_states: docker query failed") + return {} + + out: dict[str, dict[str, Any]] = {} + for d in config.deckies: + svc_states = { + svc: live.get(f"{d.name}-{svc.replace('_', '-')}", "absent") + for svc in d.services + } + out[d.name] = { + "running": bool(svc_states) and all(s == "running" for s in svc_states.values()), + "services": svc_states, + } + return out + + +_REAPER_SCRIPT = r"""#!/bin/bash +# DECNET agent self-destruct reaper. +# Runs detached from the agent process so it survives the agent's death. +# Waits briefly for the HTTP response to drain, then stops services, +# wipes install paths, and preserves logs. +set +e + +sleep 3 + +# Stop decky containers started by the local deployer (best-effort). +if command -v docker >/dev/null 2>&1; then + docker ps -q --filter "label=com.docker.compose.project=decnet" | xargs -r docker stop + docker ps -aq --filter "label=com.docker.compose.project=decnet" | xargs -r docker rm -f + docker network rm decnet_lan 2>/dev/null +fi + +# Stop+disable every systemd unit the installer may have dropped. +for unit in decnet-agent decnet-engine decnet-collector decnet-forwarder decnet-prober decnet-reconciler decnet-sniffer decnet-updater; do + systemctl stop "$unit" 2>/dev/null + systemctl disable "$unit" 2>/dev/null +done + +# Nuke install paths. Logs under /var/log/decnet* are intentionally +# preserved — the operator typically wants them for forensic review. +rm -rf /opt/decnet* /var/lib/decnet/* /usr/local/bin/decnet* /etc/decnet +rm -f /etc/systemd/system/decnet-*.service /etc/systemd/system/decnet-*.timer + +systemctl daemon-reload 2>/dev/null +rm -f "$0" +""" + + +async def self_destruct() -> None: + """Tear down deckies, then spawn a detached reaper that wipes the + install footprint. Returns immediately so the HTTP response can drain + before the reaper starts deleting files out from under the agent.""" + import os + import shutil + import subprocess # nosec B404 + import tempfile + + # Best-effort teardown first — the reaper also runs docker stop, but + # going through the deployer gives the host-macvlan/ipvlan helper a + # chance to clean up routes cleanly. + try: + await asyncio.to_thread(_deployer.teardown, None) + await asyncio.to_thread(clear_state) + except Exception: + log.exception("self_destruct: pre-reap teardown failed — reaper will force-stop containers") + + # Reaper lives under /tmp so it survives rm -rf /opt/decnet*. + fd, path = tempfile.mkstemp(prefix="decnet-reaper-", suffix=".sh", dir="/tmp") # nosec B108 — reaper must outlive /opt/decnet removal + try: + os.write(fd, _REAPER_SCRIPT.encode()) + finally: + os.close(fd) + os.chmod(path, 0o700) # nosec B103 — root-owned reaper, needs exec + + # The reaper MUST run outside decnet-agent.service's cgroup — otherwise + # `systemctl stop decnet-agent` SIGTERMs the whole cgroup (reaper included) + # before rm -rf completes. `start_new_session=True` gets us a fresh POSIX + # session but does NOT escape the systemd cgroup. So we prefer + # `systemd-run --scope` (launches the command in a transient scope + # detached from the caller's service), falling back to a bare Popen if + # systemd-run is unavailable (non-systemd host / container). + systemd_run = shutil.which("systemd-run") + if systemd_run: + argv = [ + systemd_run, + "--collect", + "--unit", f"decnet-reaper-{os.getpid()}", + "--description", "DECNET agent self-destruct reaper", + "/bin/bash", path, + ] + spawn_kwargs = {"start_new_session": True} + else: + argv = ["/bin/bash", path] + spawn_kwargs = {"start_new_session": True} + + subprocess.Popen( # nosec B603 + argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + close_fds=True, + **spawn_kwargs, + ) + log.warning( + "self_destruct: reaper spawned path=%s via=%s — agent will die in ~3s", + path, "systemd-run" if systemd_run else "popen", + ) + + +async def status() -> dict[str, Any]: + state = await asyncio.to_thread(load_state) + if state is None: + return {"deployed": False, "deckies": []} + config, _compose_path = state + runtime = await asyncio.to_thread(_decky_runtime_states, config) + return { + "deployed": True, + "mode": config.mode, + "compose_path": str(_compose_path), + "deckies": [d.model_dump() for d in config.deckies], + "runtime": runtime, + } diff --git a/decnet/agent/heartbeat.py b/decnet/agent/heartbeat.py new file mode 100644 index 00000000..f8d8eae2 --- /dev/null +++ b/decnet/agent/heartbeat.py @@ -0,0 +1,146 @@ +"""Agent → master liveness heartbeat loop. + +Every ``INTERVAL_S`` seconds the worker posts ``executor.status()`` to +``POST /swarm/heartbeat`` over mTLS. The master pins the +presented client cert's SHA-256 against the ``SwarmHost`` row for the +claimed ``host_uuid``; a match refreshes ``last_heartbeat`` + each +``DeckyShard``'s snapshot + runtime state. + +Identity comes from ``/etc/decnet/decnet.ini`` (seeded by the enroll +bundle) — specifically ``DECNET_HOST_UUID`` and ``DECNET_MASTER_HOST``. +The worker's existing ``~/.decnet/agent/`` bundle (or +``/etc/decnet/agent/``) provides the mTLS client cert. + +Started/stopped via the agent FastAPI app's lifespan. If identity +plumbing is missing (pre-enrollment dev runs) the loop logs at DEBUG and +declines to start — callers don't have to guard it. +""" +from __future__ import annotations + +import asyncio +import pathlib +from typing import Optional + +import httpx + +from decnet.agent import executor as _exec +from decnet.logging import get_logger +from decnet.swarm import pki +from decnet.swarm.log_forwarder import build_worker_ssl_context + +log = get_logger("agent.heartbeat") + +INTERVAL_S = 30.0 +_TIMEOUT = httpx.Timeout(connect=5.0, read=10.0, write=5.0, pool=5.0) + +_task: Optional[asyncio.Task] = None + + +def _resolve_agent_dir() -> pathlib.Path: + """Match the agent-dir resolution order used by the agent server: + DECNET_AGENT_DIR env, else /etc/decnet/agent (production install), + else ~/.decnet/agent (dev).""" + import os + env = os.environ.get("DECNET_AGENT_DIR") + if env: + return pathlib.Path(env) + system = pathlib.Path("/etc/decnet/agent") + if system.exists(): + return system + return pki.DEFAULT_AGENT_DIR + + +async def _tick(client: httpx.AsyncClient, url: str, host_uuid: str, agent_version: str) -> None: + snap = await _exec.status() + body: dict = { + "host_uuid": host_uuid, + "agent_version": agent_version, + "status": snap, + } + # Best-effort: fold in applied-topology snapshot. Failures must never + # wedge the heartbeat loop — master will fall back to "no topology + # reported" which triggers a resync if it expected one. + try: + from decnet.agent import topology_ops as _topo_ops + from decnet.agent.topology_store import TopologyStore + store = TopologyStore(_resolve_agent_dir() / "topology.db") + try: + body["topology"] = _topo_ops.state(store) + finally: + store.close() + except Exception: + log.debug("heartbeat: topology state unavailable", exc_info=True) + + resp = await client.post(url, json=body) + # 403 / 404 are terminal-ish — we still keep looping because an + # operator may re-enrol the host mid-session, but we log loudly so + # prod ops can spot cert-pinning drift. + if resp.status_code == 204: + return + log.warning( + "heartbeat rejected status=%d body=%s", + resp.status_code, resp.text[:200], + ) + + +async def _loop(url: str, host_uuid: str, agent_version: str, ssl_ctx) -> None: + log.info("heartbeat loop starting url=%s host_uuid=%s interval=%ss", + url, host_uuid, INTERVAL_S) + async with httpx.AsyncClient(verify=ssl_ctx, timeout=_TIMEOUT) as client: + while True: + try: + await _tick(client, url, host_uuid, agent_version) + except asyncio.CancelledError: + raise + except Exception: + log.exception("heartbeat tick failed — will retry in %ss", INTERVAL_S) + await asyncio.sleep(INTERVAL_S) + + +def start() -> Optional[asyncio.Task]: + """Kick off the background heartbeat task. No-op if identity is + unconfigured (dev mode) — the caller doesn't need to check.""" + global _task + from decnet.env import ( + DECNET_HOST_UUID, + DECNET_MASTER_HOST, + DECNET_SWARMCTL_PORT, + ) + + if _task is not None and not _task.done(): + return _task + if not DECNET_HOST_UUID or not DECNET_MASTER_HOST: + log.debug("heartbeat not starting — DECNET_HOST_UUID or DECNET_MASTER_HOST unset") + return None + + agent_dir = _resolve_agent_dir() + try: + ssl_ctx = build_worker_ssl_context(agent_dir) + except Exception: + log.exception("heartbeat not starting — worker SSL context unavailable at %s", agent_dir) + return None + + try: + from decnet import __version__ as _v + agent_version = _v + except Exception: + agent_version = "unknown" + + url = f"https://{DECNET_MASTER_HOST}:{DECNET_SWARMCTL_PORT}/swarm/heartbeat" + _task = asyncio.create_task( + _loop(url, DECNET_HOST_UUID, agent_version, ssl_ctx), + name="agent-heartbeat", + ) + return _task + + +async def stop() -> None: + global _task + if _task is None: + return + _task.cancel() + try: + await _task + except (asyncio.CancelledError, Exception): + pass + _task = None diff --git a/decnet/agent/server.py b/decnet/agent/server.py new file mode 100644 index 00000000..663bc353 --- /dev/null +++ b/decnet/agent/server.py @@ -0,0 +1,70 @@ +"""Worker-agent uvicorn launcher. + +Starts ``decnet.agent.app:app`` over HTTPS with mTLS enforcement. The +worker must already have a bundle in ``~/.decnet/agent/`` (delivered by +``decnet swarm enroll`` from the master); if it does not, we refuse to +start — unauthenticated agents are not a supported mode. +""" +from __future__ import annotations + +import os +import pathlib +import signal +import subprocess # nosec B404 +import sys + +from decnet.logging import get_logger +from decnet.swarm import pki + +log = get_logger("agent.server") + + +def run(host: str, port: int, agent_dir: pathlib.Path = pki.DEFAULT_AGENT_DIR) -> int: + bundle = pki.load_worker_bundle(agent_dir) + if bundle is None: + print( + f"[agent] No cert bundle at {agent_dir}. " + f"Run `decnet swarm enroll` from the master first.", + file=sys.stderr, + ) + return 2 + + keyfile = agent_dir / "worker.key" + certfile = agent_dir / "worker.crt" + cafile = agent_dir / "ca.crt" + + cmd = [ + sys.executable, + "-m", + "uvicorn", + "decnet.agent.app:app", + "--host", + host, + "--port", + str(port), + "--ssl-keyfile", + str(keyfile), + "--ssl-certfile", + str(certfile), + "--ssl-ca-certs", + str(cafile), + # 2 == ssl.CERT_REQUIRED — clients MUST present a CA-signed cert. + "--ssl-cert-reqs", + "2", + ] + log.info("agent starting host=%s port=%d bundle=%s", host, port, agent_dir) + # Own process group for clean Ctrl+C / SIGTERM propagation to uvicorn + # workers (same pattern as `decnet api`). + proc = subprocess.Popen(cmd, start_new_session=True) # nosec B603 + try: + return proc.wait() + except KeyboardInterrupt: + try: + os.killpg(proc.pid, signal.SIGTERM) + try: + return proc.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + return proc.wait() + except ProcessLookupError: + return 0 diff --git a/decnet/agent/topology_ops.py b/decnet/agent/topology_ops.py new file mode 100644 index 00000000..f8f156f2 --- /dev/null +++ b/decnet/agent/topology_ops.py @@ -0,0 +1,208 @@ +"""Agent-side topology apply/teardown/state primitives. + +Wraps the compose + bridge machinery from :mod:`decnet.engine.deployer` +so the agent can drive a topology without ever touching the master's +sqlmodel repo. The master-side ``deploy_topology`` always calls +``transition_status(repo, …)`` which is useless (and unreachable) on +an agent — here we operate purely on a hydrated dict + the local +:class:`TopologyStore`. + +v1 constraint: one topology per agent. A second apply for a different +``topology_id`` triggers an on-the-spot teardown of the predecessor +before the new apply proceeds — master is authoritative. +""" +from __future__ import annotations + +import asyncio +import subprocess # nosec B404 +from typing import Any + +import docker + +from decnet.agent.topology_store import ( + TopologyStore, + observed, +) +from decnet.engine.deployer import ( + _compose, + _compose_with_retry, + _teardown_order, + _topology_compose_path, +) +from decnet.logging import get_logger +from decnet.network import create_bridge_network, remove_bridge_network +from decnet.topology.compose import ( + _network_name as _topology_network_name, + write_topology_compose, +) +from decnet.topology.hashing import canonical_hash +from decnet.topology.validate import ( + ValidationError, + errors as _validation_errors, + validate as _validate_topology, +) + +log = get_logger("agent.topology_ops") + + +class HashMismatch(RuntimeError): + """Raised when the master-provided version_hash doesn't match what we + hash locally — suggests serialisation drift. We fail loudly rather + than silently papering over a schema mismatch.""" + + +def _topology_id(hydrated: dict[str, Any]) -> str: + topo = hydrated.get("topology") or {} + tid = topo.get("id") + if not tid: + raise ValueError("hydrated topology missing topology.id") + return str(tid) + + +async def apply( + hydrated: dict[str, Any], + version_hash: str, + store: TopologyStore, +) -> None: + """Materialise *hydrated* on this agent and record it in *store*. + + Raises: + HashMismatch: master and agent disagree on the canonical hash — + don't touch docker, fail the apply. + ValidationError: topology fails structural validation. + Any docker / compose error propagates up; the endpoint maps it + to 500 and records the message on the store row. + """ + local_hash = canonical_hash(hydrated) + if local_hash != version_hash: + raise HashMismatch( + f"master hash {version_hash!r} does not match agent hash " + f"{local_hash!r} — refusing to apply" + ) + + issues = _validate_topology(hydrated) + if _validation_errors(issues): + raise ValidationError(issues) + + topology_id = _topology_id(hydrated) + # Master is authoritative. If a different topology is pinned here + # — whether it fully applied, only partially applied (failure + # marker row + orphan containers), or drifted — teardown first, + # then accept the new one. Refusing with 409 would leave the + # agent stuck in a state only a human could resolve. + existing = store.current() + if existing is not None and existing.topology_id != topology_id: + log.info( + "superseding topology %s with %s on master authority", + existing.topology_id, topology_id, + ) + try: + await teardown(existing.topology_id, store) + except Exception as exc: # noqa: BLE001 — we still want to try applying + log.warning( + "best-effort teardown of superseded topology %s failed: %s", + existing.topology_id, exc, + ) + # Hard-clear the store row so the new apply isn't blocked + # by a half-torn-down predecessor. Leftover docker objects + # will surface via the next heartbeat's observed block. + store.clear(existing.topology_id) + + lans = hydrated["lans"] + compose_path = _topology_compose_path(topology_id) + client = docker.from_env() + + # Bridges + compose are sync/blocking; hop to a thread so we don't + # stall the event loop on a slow docker daemon. + def _materialise() -> None: + for lan in lans: + net_name = _topology_network_name(topology_id, lan["name"]) + internal = not lan["is_dmz"] + create_bridge_network( + client, net_name, lan["subnet"], internal=internal + ) + write_topology_compose(hydrated, compose_path) + # ``--always-recreate-deps`` keeps service containers' netns shares + # fresh: every decky service joins its base's netns via + # ``network_mode: container:``, and that share is bound at + # service start time. If a base is recreated (e.g. when ``ports:`` + # changes after toggling ``forwards_l3``) but compose decides the + # services are unchanged, the services keep a stale netns FD + # pointing at the destroyed base — they end up in an empty + # namespace with only ``lo``, and external traffic hits a closed + # port on the live base. Forcing dependents to recreate alongside + # the base is the cheapest way to make this race impossible. + _compose_with_retry( + "up", "--build", "-d", "--always-recreate-deps", + compose_file=compose_path, + ) + + await asyncio.to_thread(_materialise) + + store.put(topology_id, version_hash, hydrated) + log.info( + "topology %s applied on agent (%d LANs)", topology_id, len(lans) + ) + + +async def teardown( + topology_id: str, + store: TopologyStore, +) -> None: + """Tear down *topology_id* on this agent. Idempotent: if there's no + record and no compose file, it's a no-op that still returns cleanly.""" + row = store.current() + # Prefer the stored hydrated blob — it's what we applied with. If + # it's gone (db wiped) but compose-file lingers, we still try to + # compose-down and delete bridges by scanning the compose file's + # LAN membership list via the hydrated blob if available. + hydrated = row.hydrated if row and row.topology_id == topology_id else None + compose_path = _topology_compose_path(topology_id) + client = docker.from_env() + + def _dismantle() -> None: + if compose_path.exists(): + try: + _compose("down", "--remove-orphans", compose_file=compose_path) + except subprocess.CalledProcessError as exc: + log.warning( + "topology %s compose down failed (continuing): %s", + topology_id, exc, + ) + if hydrated is not None: + for lan_name in _teardown_order(hydrated["lans"]): + net_name = _topology_network_name(topology_id, lan_name) + remove_bridge_network(client, net_name) + if compose_path.exists(): + compose_path.unlink() + + await asyncio.to_thread(_dismantle) + store.clear(topology_id) + log.info("topology %s torn down on agent", topology_id) + + +def state(store: TopologyStore) -> dict[str, Any]: + """Snapshot-plus-live-observation — the shape the heartbeat embeds.""" + row = store.current() + try: + obs = observed(docker.from_env()) + except Exception as exc: # noqa: BLE001 — docker socket may be gone + obs = {"error": str(exc)[:200]} + if row is None: + return { + "topology_id": None, + "applied_version_hash": None, + "applied_at": None, + "last_error": None, + "observed": obs, + } + return { + "topology_id": row.topology_id, + "applied_version_hash": row.applied_version_hash, + "applied_at": row.applied_at, + "last_error": row.last_error, + "observed": obs, + } + + +__all__ = ["apply", "teardown", "state", "HashMismatch"] diff --git a/decnet/agent/topology_store.py b/decnet/agent/topology_store.py new file mode 100644 index 00000000..7112307e --- /dev/null +++ b/decnet/agent/topology_store.py @@ -0,0 +1,213 @@ +"""Agent-side sqlite cache of the currently-applied topology. + +**This is a cache, not a source of truth.** The master is the only +authority for what the agent should be running. This store exists so +the agent can answer two questions quickly and offline: + +1. What topology did I last apply, and with what version hash? +2. Is what docker is currently doing consistent with that? + +The hash goes out on every heartbeat; the master compares it to what +it thinks this host should be running and schedules a re-push on +mismatch. + +Why sqlite when the blob is JSON? Consistent with +:mod:`decnet.swarm.log_forwarder._OffsetStore` — single-row sqlite is +the project-wide pattern for agent-local persistent state. Keeps +operational mental model small: "one state.db per thing". + +Design choices worth calling out: + +- **One row, one topology.** v1 only supports a single topology per + agent. Attempting to :meth:`put` a different ``topology_id`` while + a row already exists raises :class:`AlreadyApplied` — the agent + rejects the apply with 409 and the master is expected to teardown + the old one first. +- **No auto-restore on boot.** The agent does NOT read this db at + startup and try to re-apply. Whatever docker has after a restart + is what it has; the next heartbeat reports the truth and the + master decides whether to re-push. Same reason we don't sync + mutations from agent → master anywhere else: split-brain is worse + than temporary drift. +""" +from __future__ import annotations + +import json +import pathlib +import sqlite3 +import time +from dataclasses import dataclass +from typing import Any, Optional + + +class AlreadyApplied(RuntimeError): + """Raised when a different topology is already pinned to this agent.""" + + +@dataclass(frozen=True) +class AppliedRow: + topology_id: str + applied_version_hash: str + hydrated: dict[str, Any] + applied_at: int + last_error: Optional[str] + + +class TopologyStore: + """Single-row sqlite cache. Stdlib only, sync (called from endpoints).""" + + def __init__(self, db_path: pathlib.Path) -> None: + db_path.parent.mkdir(parents=True, exist_ok=True) + # check_same_thread=False: Starlette/FastAPI runs sync endpoint + # bodies on a worker thread distinct from where `app` is imported. + # The agent is single-process, so there's no real contention — + # sqlite's own connection lock is enough. + self._conn = sqlite3.connect(str(db_path), check_same_thread=False) + self._conn.execute( + "CREATE TABLE IF NOT EXISTS applied_topology (" + " topology_id TEXT PRIMARY KEY," + " applied_version_hash TEXT NOT NULL," + " hydrated_blob_json TEXT NOT NULL," + " applied_at INTEGER NOT NULL," + " last_error TEXT)" + ) + self._conn.commit() + + # ----------------------------------------------------------------- reads + + def current(self) -> Optional[AppliedRow]: + """Return the single applied topology, or ``None`` if idle.""" + row = self._conn.execute( + "SELECT topology_id, applied_version_hash, hydrated_blob_json," + " applied_at, last_error FROM applied_topology LIMIT 1" + ).fetchone() + if row is None: + return None + return AppliedRow( + topology_id=row[0], + applied_version_hash=row[1], + hydrated=json.loads(row[2]), + applied_at=int(row[3]), + last_error=row[4], + ) + + # ---------------------------------------------------------------- writes + + def put( + self, + topology_id: str, + applied_version_hash: str, + hydrated: dict[str, Any], + ) -> None: + """Record an applied topology. + + If a *different* topology is already recorded, raises + :class:`AlreadyApplied`. Re-applying the same ``topology_id`` + just updates the hash + blob (idempotent re-push). + """ + existing = self.current() + if existing is not None and existing.topology_id != topology_id: + raise AlreadyApplied( + f"agent already has topology {existing.topology_id!r}; " + f"cannot apply {topology_id!r}" + ) + self._conn.execute( + "INSERT INTO applied_topology" + " (topology_id, applied_version_hash, hydrated_blob_json," + " applied_at, last_error)" + " VALUES (?, ?, ?, ?, NULL)" + " ON CONFLICT(topology_id) DO UPDATE SET" + " applied_version_hash=excluded.applied_version_hash," + " hydrated_blob_json=excluded.hydrated_blob_json," + " applied_at=excluded.applied_at," + " last_error=NULL", + ( + topology_id, + applied_version_hash, + json.dumps(hydrated, sort_keys=True), + int(time.time()), + ), + ) + self._conn.commit() + + def record_error( + self, + topology_id: str, + message: str, + hydrated: Optional[dict[str, Any]] = None, + ) -> None: + """Attach a last-error message for *topology_id*. + + Upserts a marker row when no apply has yet succeeded for this + topology — that way a failure *during* the first materialise + (put() hasn't been reached) still surfaces via GET + /topology/state and the next heartbeat. The marker row uses an + empty ``applied_version_hash`` so master's heartbeat check sees + the hash mismatch and schedules a resync. + + If *hydrated* is provided it is stored so a later teardown can + still walk the LAN list — otherwise a partial deploy is strands + containers + bridges with no breadcrumb back to them. + """ + blob = json.dumps(hydrated, sort_keys=True) if hydrated else "{}" + self._conn.execute( + "INSERT INTO applied_topology" + " (topology_id, applied_version_hash, hydrated_blob_json," + " applied_at, last_error)" + " VALUES (?, '', ?, 0, ?)" + " ON CONFLICT(topology_id) DO UPDATE SET" + " last_error=excluded.last_error," + " hydrated_blob_json=CASE" + " WHEN applied_topology.hydrated_blob_json='{}'" + " THEN excluded.hydrated_blob_json" + " ELSE applied_topology.hydrated_blob_json END", + (topology_id, blob, message), + ) + self._conn.commit() + + def clear(self, topology_id: str) -> None: + """Remove the row for *topology_id* (post-teardown). + + No-op if the row doesn't exist — makes teardown idempotent. + """ + self._conn.execute( + "DELETE FROM applied_topology WHERE topology_id=?", + (topology_id,), + ) + self._conn.commit() + + def close(self) -> None: + self._conn.close() + + +# --------------------------------------------------- live docker observation + + +def observed(docker_client: Any) -> dict[str, Any]: + """Snapshot what docker is *actually* running on this agent. + + Returns a compact dict the heartbeat can ship so the master can + cross-check ``applied_version_hash`` against reality (a matching + hash with missing bridges is still drift). Best-effort: if docker + is unreachable we return an ``error`` marker rather than raising — + the agent still needs to heartbeat, and the master can treat + ``error`` as "unknown, re-push". + """ + try: + bridges = [ + n.name + for n in docker_client.networks.list() + if n.attrs.get("Driver") == "bridge" + and n.name.startswith("decnet-topology-") + ] + containers = [ + c.name + for c in docker_client.containers.list(all=False) + if c.name.startswith("decnet-") + ] + return {"bridges": sorted(bridges), "containers": sorted(containers)} + except Exception as exc: # noqa: BLE001 — best-effort observation + return {"error": str(exc)[:200]} + + +__all__ = ["TopologyStore", "AppliedRow", "AlreadyApplied", "observed"] diff --git a/decnet/asn/__init__.py b/decnet/asn/__init__.py new file mode 100644 index 00000000..64224b0a --- /dev/null +++ b/decnet/asn/__init__.py @@ -0,0 +1,92 @@ +""" +IP-to-ASN enrichment — maps attacker IPs to BGP-announced AS numbers and +org names for attacker intelligence. + +Public surface mirrors :mod:`decnet.geoip` so callers can compose them: + +* :func:`get_lookup` — returns the singleton :class:`AsnLookup`. +* :func:`enrich_ip` — takes an IP string, returns + ``(asn_int, asn_name, provider_name)`` or ``(None, None, None)``. + +Provider selection goes through :func:`~decnet.asn.factory.get_provider` +(env ``DECNET_ASN_PROVIDER``, default ``iptoasn``). Direct imports of +concrete providers are forbidden — mirrors the ``get_bus`` / +``get_repository`` rule. +""" +from __future__ import annotations + +import os +import time +from typing import Optional, Tuple + +from decnet.asn.factory import get_provider +from decnet.asn.lookup import AsnLookup +from decnet.asn.paths import ASN_ROOT + +# 24 h — iptoasn refreshes daily. +REFRESH_INTERVAL_S = 86_400 + +_lookup: Optional[AsnLookup] = None +_provider_name: Optional[str] = None + + +def get_lookup(*, force_refresh: bool = False) -> AsnLookup: + """Return the cached :class:`AsnLookup`, building it on first use. + + If the provider's data files are missing or older than + ``REFRESH_INTERVAL_S`` seconds, refresh before building. Pass + ``force_refresh=True`` to bypass the age check (used by a future + ``decnet asn refresh`` CLI command). + """ + global _lookup, _provider_name + provider = get_provider() + _provider_name = provider.name + + if force_refresh or _files_stale(provider): + provider.refresh() + _lookup = None # rebuild on next access + + if _lookup is None: + _lookup = provider.build_lookup() + return _lookup + + +def enrich_ip(ip: str) -> Tuple[Optional[int], Optional[str], Optional[str]]: + """Return ``(asn, as_name, provider_name)`` or ``(None, None, None)``. + + Never raises — any lookup failure collapses to all-None so the + caller (profiler) can upsert the attacker row regardless. + + ``DECNET_ASN_ENABLED=false`` short-circuits the whole path, useful + for tests / agent hosts / ops wanting to disable enrichment without + touching provider config. + """ + if os.environ.get("DECNET_ASN_ENABLED", "true").lower() == "false": + return (None, None, None) + try: + lookup = get_lookup() + info = lookup.asn(ip) + if info is None: + return (None, None, None) + return (info.asn, info.name or None, _provider_name or "unknown") + except Exception: + return (None, None, None) + + +def _files_stale(provider) -> bool: + """True when the provider has no fresh data on disk. + + Same semantics as :func:`decnet.geoip._files_stale`: a partial + cache still produces correct answers for the ranges it covers. + """ + paths = provider.data_paths() + if not paths: + return True + now = time.time() + for p in paths: + if p.exists() and now - p.stat().st_mtime <= REFRESH_INTERVAL_S: + return False + return True + + +__all__ = ["get_lookup", "enrich_ip", "ASN_ROOT", "REFRESH_INTERVAL_S"] diff --git a/decnet/asn/base.py b/decnet/asn/base.py new file mode 100644 index 00000000..418d6529 --- /dev/null +++ b/decnet/asn/base.py @@ -0,0 +1,33 @@ +"""ASN provider protocol — mirror of :mod:`decnet.geoip.base`. + +Concrete providers (e.g. :mod:`decnet.asn.iptoasn`) implement this. +Callers must go through :func:`decnet.asn.factory.get_provider`; never +import a concrete provider class directly. +""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Sequence + +from decnet.asn.lookup import AsnLookup + + +class Provider(ABC): + """Abstract IP→ASN data provider.""" + + #: Short tag written to ``Attacker.asn_source`` (e.g. ``'iptoasn'``). + name: str + + @abstractmethod + def refresh(self) -> None: + """Download / regenerate the provider's raw data files.""" + + @abstractmethod + def build_lookup(self) -> AsnLookup: + """Parse the on-disk data files and return a ready-to-query lookup.""" + + @abstractmethod + def data_paths(self) -> Sequence[Path]: + """Return the list of files this provider manages — used for staleness + detection. Order is not significant.""" diff --git a/decnet/asn/factory.py b/decnet/asn/factory.py new file mode 100644 index 00000000..c1a63f8f --- /dev/null +++ b/decnet/asn/factory.py @@ -0,0 +1,39 @@ +"""ASN provider factory — mirror of :mod:`decnet.geoip.factory`. + +Dispatch key: ``DECNET_ASN_PROVIDER`` (default ``iptoasn``). Lazy +singleton. +""" +from __future__ import annotations + +import os +from typing import Optional + +from decnet.asn.base import Provider + +_cached: Optional[Provider] = None +_cached_key: Optional[str] = None + + +def get_provider() -> Provider: + """Return the configured :class:`Provider` singleton.""" + global _cached, _cached_key + key = os.environ.get("DECNET_ASN_PROVIDER", "iptoasn").lower() + if _cached is not None and _cached_key == key: + return _cached + + if key == "iptoasn": + from decnet.asn.iptoasn.provider import IptoasnProvider + provider: Provider = IptoasnProvider() + else: + raise ValueError(f"Unsupported ASN provider: {key!r}") + + _cached = provider + _cached_key = key + return provider + + +def reset_cache() -> None: + """Forget the singleton — tests swap providers via the env var.""" + global _cached, _cached_key + _cached = None + _cached_key = None diff --git a/decnet/asn/iptoasn/__init__.py b/decnet/asn/iptoasn/__init__.py new file mode 100644 index 00000000..081f216b --- /dev/null +++ b/decnet/asn/iptoasn/__init__.py @@ -0,0 +1,9 @@ +"""iptoasn.com IP→ASN provider. + +Daily-refreshed gzipped TSV dump of the global BGP table, derived from +RIPE RIS. Released into the public domain by upstream — no attribution +required, no UA mandate, no terms to violate. + +Direct imports of :class:`IptoasnProvider` are discouraged — go through +:func:`decnet.asn.factory.get_provider`. +""" diff --git a/decnet/asn/iptoasn/fetch.py b/decnet/asn/iptoasn/fetch.py new file mode 100644 index 00000000..c087da22 --- /dev/null +++ b/decnet/asn/iptoasn/fetch.py @@ -0,0 +1,63 @@ +"""iptoasn.com bulk dump download. + +One file: ``ip2asn-v4.tsv.gz``, ~5 MB compressed, refreshed daily. +Pulled over HTTPS with the same generic UA the geoip RIR fetcher uses +(stealth: never identify as DECNET — public-data scrapers correlated to +honeypot operator egress is the threat model). +""" +from __future__ import annotations + +import logging +import shutil +import urllib.request +from pathlib import Path +from typing import Tuple + +logger = logging.getLogger("decnet.asn.iptoasn.fetch") + +# Mirror the (name, url) tuple shape of geoip.rir.fetch so test +# harnesses can swap one for the other. +IPTOASN_SOURCES: Tuple[Tuple[str, str], ...] = ( + ("ip2asn-v4", "https://iptoasn.com/data/ip2asn-v4.tsv.gz"), +) + +# Generic UA — matches geoip.rir.fetch. iptoasn.com explicitly releases +# the data into the public domain and does NOT require an identifying UA, +# so we keep DECNET stealth instead of advertising. +_USER_AGENT = "Mozilla/5.0 (compatible; fetch/1.0)" +_TIMEOUT_S = 60 + + +def fetch_all(dest: Path) -> list[Path]: + """Download every iptoasn file into *dest*. Returns the written paths. + + Atomic per file: download to ``{name}.tsv.gz.tmp`` then rename. A + partial failure leaves the previous generation intact. + """ + dest.mkdir(parents=True, exist_ok=True) + written: list[Path] = [] + for name, url in IPTOASN_SOURCES: + target = dest / f"{name}.tsv.gz" + tmp = target.with_suffix(".gz.tmp") + try: + _download(url, tmp) + tmp.replace(target) + written.append(target) + logger.info( + "asn.iptoasn: fetched %s (%d bytes)", + name, target.stat().st_size, + ) + except Exception as exc: + logger.error( + "asn.iptoasn: fetch failed for %s (%s): %s", name, url, exc + ) + if tmp.exists(): + tmp.unlink(missing_ok=True) + # Keep any stale previous file — better outdated than empty. + return written + + +def _download(url: str, dest: Path) -> None: + req = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT}) + with urllib.request.urlopen(req, timeout=_TIMEOUT_S) as resp, dest.open("wb") as fh: # nosec B310 — fixed https iptoasn URL + shutil.copyfileobj(resp, fh) diff --git a/decnet/asn/iptoasn/parse.py b/decnet/asn/iptoasn/parse.py new file mode 100644 index 00000000..47db413d --- /dev/null +++ b/decnet/asn/iptoasn/parse.py @@ -0,0 +1,78 @@ +"""Parser for the iptoasn.com ``ip2asn-v4.tsv`` dump. + +Line shape (gzipped, one row per BGP-announced prefix):: + + 1.0.0.0\\t1.0.0.255\\t13335\\tUS\\tCLOUDFLARENET + +Fields: ``range_start``, ``range_end``, ``as_number``, ``country_code``, +``as_description``. Both range columns are dotted IPv4 strings (the dump +is IPv4-only — there's a separate ``ip2asn-v6.tsv.gz`` we don't pull). + +Rows skipped: + +* ``as_number == 0`` — iptoasn's sentinel for "unannounced" / private + / reserved space. Country may still be present (``"None"`` / two-letter + CC) but we don't care: the geoip module owns country, ASN owns BGP. +* Rows where either range column won't parse as IPv4. +* Rows with fewer than 3 tab-separated columns. +""" +from __future__ import annotations + +import gzip +import ipaddress +import logging +from pathlib import Path +from typing import Iterator + +from decnet.asn.lookup import AsnInfo, Range + +logger = logging.getLogger("decnet.asn.iptoasn.parse") + + +def parse_file(path: Path) -> Iterator[Range]: + """Yield ``(start_int, end_int_inclusive, AsnInfo)`` for every BGP row. + + Accepts a gzipped path (``*.tsv.gz``); plain TSV is also fine for + test harnesses that hand-craft small fixtures. + """ + opener = gzip.open if path.suffix == ".gz" else open + with opener(path, "rt", encoding="utf-8", errors="replace") as fh: + for lineno, raw in enumerate(fh, 1): + line = raw.rstrip("\n") + if not line: + continue + parts = line.split("\t") + if len(parts) < 3: + continue + start_s, end_s, asn_s = parts[0], parts[1], parts[2] + # Description is the 5th column; iptoasn quotes nothing, + # but the field can contain stray whitespace. ``""`` when + # missing or unknown. + name = parts[4].strip() if len(parts) >= 5 else "" + + try: + asn = int(asn_s) + except ValueError: + logger.debug( + "asn.iptoasn: skipping malformed asn line %d in %s", + lineno, path.name, + ) + continue + # ASN 0 is iptoasn's sentinel for unannounced / sentinel + # space. Skip — there's no useful enrichment to attach. + if asn == 0: + continue + + try: + start_int = int(ipaddress.IPv4Address(start_s)) + end_int = int(ipaddress.IPv4Address(end_s)) + except (ValueError, ipaddress.AddressValueError): + logger.debug( + "asn.iptoasn: skipping malformed addr line %d in %s", + lineno, path.name, + ) + continue + if end_int < start_int: + continue + + yield (start_int, end_int, AsnInfo(asn=asn, name=name)) diff --git a/decnet/asn/iptoasn/provider.py b/decnet/asn/iptoasn/provider.py new file mode 100644 index 00000000..fbd243b5 --- /dev/null +++ b/decnet/asn/iptoasn/provider.py @@ -0,0 +1,83 @@ +"""iptoasn provider — orchestrates fetch + parse into an :class:`AsnLookup`. + +Mirrors :class:`decnet.geoip.rir.provider.RirProvider` exactly: fetch, +build a pickled cache, invalidate when raw files are newer than the +cache. +""" +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Sequence + +from decnet.asn.base import Provider +from decnet.asn.iptoasn.fetch import IPTOASN_SOURCES, fetch_all +from decnet.asn.iptoasn.parse import parse_file +from decnet.asn.lookup import AsnLookup +from decnet.asn.paths import ensure_root + +logger = logging.getLogger("decnet.asn.iptoasn.provider") + +# Pickled lookup cache — skips re-parsing the ~580k-row gz dump on every +# profiler restart. Rebuilt whenever any raw file is newer than the +# cache, see ``_cache_fresh``. +_CACHE_NAME = ".iptoasn_index.pkl" + + +class IptoasnProvider(Provider): + name = "iptoasn" + + def __init__(self) -> None: + self._root = ensure_root() + + # ---------- Provider interface ---------- + + def refresh(self) -> None: + logger.info("asn.iptoasn: refreshing dump into %s", self._root) + fetch_all(self._root) + cache = self._root / _CACHE_NAME + if cache.exists(): + cache.unlink(missing_ok=True) + + def build_lookup(self) -> AsnLookup: + cache = self._root / _CACHE_NAME + if self._cache_fresh(cache): + try: + lookup = AsnLookup.load(cache) + logger.debug( + "asn.iptoasn: loaded cached index (%d ranges)", + len(lookup), + ) + return lookup + except Exception as exc: + logger.warning( + "asn.iptoasn: cache load failed, rebuilding: %s", exc + ) + + ranges = [] + for path in self.data_paths(): + if not path.exists(): + continue + ranges.extend(parse_file(path)) + lookup = AsnLookup.from_ranges(ranges) + try: + lookup.save(cache) + except Exception as exc: + logger.warning("asn.iptoasn: cache save failed: %s", exc) + logger.info("asn.iptoasn: built index with %d ranges", len(lookup)) + return lookup + + def data_paths(self) -> Sequence[Path]: + return [self._root / f"{name}.tsv.gz" for name, _url in IPTOASN_SOURCES] + + # ---------- internals ---------- + + def _cache_fresh(self, cache: Path) -> bool: + """True when the pickle exists and is at least as new as every raw file.""" + if not cache.exists(): + return False + cache_mtime = cache.stat().st_mtime + for path in self.data_paths(): + if path.exists() and path.stat().st_mtime > cache_mtime: + return False + return True diff --git a/decnet/asn/lookup.py b/decnet/asn/lookup.py new file mode 100644 index 00000000..e3d6272b --- /dev/null +++ b/decnet/asn/lookup.py @@ -0,0 +1,126 @@ +"""Provider-agnostic IP→ASN lookup. + +A :class:`AsnLookup` is a frozen, sorted array of ``(start_ip, +end_ip_inclusive, AsnInfo)`` ranges queried via :mod:`bisect`. +O(log n) on ~600k ranges (a current iptoasn dump is ~580k rows). + +Private/loopback/invalid IPv4 and all IPv6 addresses resolve to +``None`` — the same policy :mod:`decnet.geoip.lookup` uses. +""" +from __future__ import annotations + +import bisect +import ipaddress +import pickle # nosec B403 — self-produced cache under /var/lib/decnet, never deserialized from untrusted input +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, List, Optional, Tuple + + +@dataclass(frozen=True) +class AsnInfo: + """One BGP-announced prefix's origin metadata.""" + + asn: int + name: str # AS description / org name; "" if absent in the source data + + +Range = Tuple[int, int, AsnInfo] + + +@dataclass +class AsnLookup: + """Indexed AS lookup over IPv4 ranges.""" + + # Parallel arrays for bisect: _starts[i] is the start-IP of the i-th + # range, _ends[i] its inclusive end, _infos[i] its AsnInfo. + _starts: List[int] + _ends: List[int] + _infos: List[AsnInfo] + + @classmethod + def from_ranges(cls, ranges: Iterable[Range]) -> "AsnLookup": + """Build a lookup from ``(start, end_inclusive, AsnInfo)`` triples. + + Ranges are sorted by start; on identical starts, last writer + wins (matches :class:`decnet.geoip.lookup.Lookup` semantics). + Non-overlapping adjacency is preserved. + """ + sorted_ranges = sorted(ranges, key=lambda r: (r[0], r[1])) + starts: List[int] = [] + ends: List[int] = [] + infos: List[AsnInfo] = [] + for start, end, info in sorted_ranges: + if starts and starts[-1] == start: + ends[-1] = end + infos[-1] = info + continue + starts.append(start) + ends.append(end) + infos.append(info) + return cls(starts, ends, infos) + + def asn(self, ip: str) -> Optional[AsnInfo]: + """Return the :class:`AsnInfo` for ``ip`` or ``None``. + + ``None`` on: IPv6, private/loopback/link-local/multicast/reserved + addresses, malformed strings, and IPs outside every BGP-announced + range in the source dump. + """ + try: + addr = ipaddress.ip_address(ip) + except ValueError: + return None + if isinstance(addr, ipaddress.IPv6Address): + return None + if ( + addr.is_private + or addr.is_loopback + or addr.is_link_local + or addr.is_multicast + or addr.is_reserved + or addr.is_unspecified + ): + return None + + n = int(addr) + idx = bisect.bisect_right(self._starts, n) - 1 + if idx < 0: + return None + if n <= self._ends[idx]: + return self._infos[idx] + return None + + def __len__(self) -> int: + return len(self._starts) + + # ---------- persistence ---------- + + def save(self, path: Path) -> None: + """Pickle the lookup to *path* (atomic rename).""" + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.parent.mkdir(parents=True, exist_ok=True) + with tmp.open("wb") as fh: + pickle.dump( + { + "version": 1, + "starts": self._starts, + "ends": self._ends, + "infos": [(i.asn, i.name) for i in self._infos], + }, + fh, + protocol=pickle.HIGHEST_PROTOCOL, + ) + tmp.replace(path) + + @classmethod + def load(cls, path: Path) -> "AsnLookup": + """Load a pickled lookup from *path*.""" + with path.open("rb") as fh: + data = pickle.load(fh) # nosec B301 — self-produced file under /var/lib/decnet + if data.get("version") != 1: + raise ValueError( + f"unsupported asn-lookup index version: {data.get('version')!r}" + ) + infos = [AsnInfo(asn=a, name=n) for a, n in data["infos"]] + return cls(data["starts"], data["ends"], infos) diff --git a/decnet/asn/paths.py b/decnet/asn/paths.py new file mode 100644 index 00000000..b78c665d --- /dev/null +++ b/decnet/asn/paths.py @@ -0,0 +1,18 @@ +"""Filesystem layout for ASN data — mirror of :mod:`decnet.geoip.paths`. + +``ASN_ROOT`` is where providers drop their raw files and cache indexes. +Default ``/var/lib/decnet/asn``. Override with ``DECNET_ASN_ROOT`` for +test harnesses. +""" +from __future__ import annotations + +import os +from pathlib import Path + +ASN_ROOT = Path(os.environ.get("DECNET_ASN_ROOT", "/var/lib/decnet/asn")) + + +def ensure_root() -> Path: + """Create ``ASN_ROOT`` if absent and return it. No-op if present.""" + ASN_ROOT.mkdir(parents=True, exist_ok=True) + return ASN_ROOT diff --git a/decnet/bus/__init__.py b/decnet/bus/__init__.py new file mode 100644 index 00000000..1fc4d87f --- /dev/null +++ b/decnet/bus/__init__.py @@ -0,0 +1,18 @@ +"""DECNET ServiceBus — pub/sub notification substrate. + +The bus is the notification layer for DECNET's worker constellation. The DB +remains the source of truth for anything durable; the bus carries "something +happened, go look" events. Delivery is at-most-once, fire-and-forget. + +Consumers call :func:`get_bus` from :mod:`decnet.bus.factory`; never import +transport implementations directly. The factory selects the backend via +``DECNET_BUS_TYPE`` (``nats`` or ``fake``) and honors ``DECNET_BUS_ENABLED``. + +Topic hierarchy is defined in :mod:`decnet.bus.topics` and locked early so +consumers can subscribe with stable wildcard patterns. +""" +from __future__ import annotations + +from decnet.bus.base import BaseBus, Event, Subscription + +__all__ = ["BaseBus", "Event", "Subscription"] diff --git a/decnet/bus/app.py b/decnet/bus/app.py new file mode 100644 index 00000000..129bf3c6 --- /dev/null +++ b/decnet/bus/app.py @@ -0,0 +1,92 @@ +"""Process-wide bus singleton for request-serving workers (API, SSE routes). + +A single connected :class:`~decnet.bus.base.BaseBus` shared across request +handlers — opening a UNIX socket per request would be wasteful and add +latency to the hot path. The API lifespan is responsible for calling +:func:`close_app_bus` on shutdown; connect is lazy so tests and +contract-test mode that never hit a publish/subscribe code path don't +pay for a bus connection they'll never use. + +Failures during :meth:`BaseBus.connect` are swallowed and logged — a +dead bus must never break request serving. Publishers should treat a +``None`` return from :func:`get_app_bus` as "skip this notification", +same as ``DECNET_BUS_ENABLED=false``. + +Connect is **retried with a short backoff** (not one-shot): a startup +race where the API lifespan hits :func:`get_app_bus` before ``decnet +bus`` is ready would otherwise poison the singleton for the entire +process lifetime. Instead we remember the last failure timestamp and +let callers retry once ``_RETRY_BACKOFF`` seconds have passed. +""" +from __future__ import annotations + +import asyncio +import time + +from decnet.bus.base import BaseBus +from decnet.bus.factory import get_bus +from decnet.logging import get_logger + +log = get_logger("bus.app") + +# Publishers in the hot path shouldn't pay connect-retry latency on every +# call; the dashboard's own 5 s poll interval recovers within one tick +# once the bus comes up. A persistently-dead bus only gets a connect +# attempt every 2 s, not once per request. +_RETRY_BACKOFF: float = 2.0 + +_lock = asyncio.Lock() +_shared: BaseBus | None = None +_last_failure_ts: float = 0.0 + + +async def get_app_bus() -> BaseBus | None: + """Return the process-wide connected bus, or ``None`` if unavailable. + + On first call, constructs a client via :func:`get_bus` and awaits + ``connect()``. Subsequent calls return the cached instance. If a + connect attempt raises, the failure timestamp is recorded and + subsequent calls within ``_RETRY_BACKOFF`` seconds return ``None`` + without re-attempting — after the backoff window, the next call + retries. This is what lets the API recover from a + ``decnet bus``-started-after-API race without a full API restart. + """ + global _shared, _last_failure_ts + if _shared is not None: + return _shared + if (time.monotonic() - _last_failure_ts) < _RETRY_BACKOFF: + return None + async with _lock: + if _shared is not None: + return _shared + if (time.monotonic() - _last_failure_ts) < _RETRY_BACKOFF: + return None + try: + candidate = get_bus(client_name="api") + await candidate.connect() + _shared = candidate + _last_failure_ts = 0.0 + return _shared + except Exception as exc: # noqa: BLE001 + log.warning("app bus unavailable: %s", exc) + _last_failure_ts = time.monotonic() + return None + + +async def close_app_bus() -> None: + """Close the shared bus if one is open; clear the backoff window. + + Call from the API lifespan shutdown. Safe to call multiple times. + Resetting ``_last_failure_ts`` means the next ``get_app_bus()`` + after shutdown-and-restart-within-the-same-process (rare, but + tests do this) retries immediately instead of honouring a stale + backoff. + """ + global _shared, _last_failure_ts + bus, _shared = _shared, None + _last_failure_ts = 0.0 + if bus is not None: + try: + await bus.close() + except Exception as exc: # noqa: BLE001 + log.warning("app bus close raised: %s", exc) diff --git a/decnet/bus/base.py b/decnet/bus/base.py new file mode 100644 index 00000000..8edd1724 --- /dev/null +++ b/decnet/bus/base.py @@ -0,0 +1,205 @@ +"""Bus abstractions: the :class:`Event` envelope and the :class:`BaseBus` ABC. + +Every transport (NATS, in-process fake, null) speaks this contract. The +envelope is versioned (``v``) so future evolution never breaks deployed +consumers that happen to see a newer event shape. + +Subscription model: :meth:`BaseBus.subscribe` returns a :class:`Subscription` +that is an async context manager AND an async iterator. The expected usage is: + + async with bus.subscribe("topology.*.mutation.*") as sub: + async for event in sub: + handle(event) + +Leaving the ``async with`` releases the underlying subscription handle; the +transport is free to drop any buffered events after that point. +""" +from __future__ import annotations + +import abc +import asyncio +import time +import uuid +from dataclasses import dataclass, field +from typing import Any, AsyncIterator + +EVENT_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class Event: + """The bus envelope. + + ``v`` is the envelope schema version, bumped on incompatible shape + changes. ``type`` is a short discriminator (``"mutation.applied"``, + ``"decky.state"``) useful for consumers that subscribe to a broad + wildcard and dispatch in Python; it is redundant with the trailing + segments of ``topic`` but cheaper to inspect. ``ts`` is epoch seconds + (float). ``id`` is a random UUID so consumers can de-dupe if they + ever see the same event twice (not expected at-most-once, but cheap + insurance). + """ + + topic: str + payload: dict[str, Any] + type: str = "" + v: int = EVENT_SCHEMA_VERSION + ts: float = field(default_factory=time.time) + id: str = field(default_factory=lambda: uuid.uuid4().hex) + + def to_dict(self) -> dict[str, Any]: + return { + "v": self.v, + "id": self.id, + "topic": self.topic, + "type": self.type, + "ts": self.ts, + "payload": self.payload, + } + + @classmethod + def from_dict(cls, topic: str, data: dict[str, Any]) -> "Event": + """Reconstruct an Event from a wire-format dict. + + ``topic`` is passed explicitly because the transport knows which + subject the message arrived on; trusting a ``topic`` field from the + wire would let a misbehaving publisher spoof events on topics they + don't actually publish to. + """ + return cls( + topic=topic, + payload=data.get("payload", {}) or {}, + type=data.get("type", "") or "", + v=int(data.get("v", EVENT_SCHEMA_VERSION)), + ts=float(data.get("ts", time.time())), + id=data.get("id") or uuid.uuid4().hex, + ) + + +class Subscription(abc.ABC): + """An open subscription — async context manager + async iterator. + + Concrete transports subclass this and implement :meth:`_aclose` plus the + async iterator protocol. Callers should not instantiate directly; use + :meth:`BaseBus.subscribe`. + """ + + def __init__(self, pattern: str) -> None: + self.pattern = pattern + self._closed = False + + async def __aenter__(self) -> "Subscription": + return self + + async def __aexit__(self, *exc: Any) -> None: + await self.aclose() + + def __aiter__(self) -> AsyncIterator[Event]: + return self + + async def aclose(self) -> None: + if self._closed: + return + self._closed = True + await self._aclose() + + @abc.abstractmethod + async def __anext__(self) -> Event: # pragma: no cover - abstract + raise NotImplementedError + + @abc.abstractmethod + async def _aclose(self) -> None: # pragma: no cover - abstract + raise NotImplementedError + + +class BaseBus(abc.ABC): + """Pub/sub transport contract. + + Implementations MUST be safe to ``await connect()`` multiple times and + ``await close()`` multiple times. Publishing to a closed bus raises + :class:`RuntimeError`; subscribing to a closed bus does too. + """ + + @abc.abstractmethod + async def connect(self) -> None: + """Establish any network/transport resources. Idempotent.""" + + @abc.abstractmethod + async def publish( + self, + topic: str, + payload: dict[str, Any], + *, + event_type: str = "", + ) -> None: + """Publish *payload* on *topic*. Fire-and-forget. + + Delivery is at-most-once. On transport error the implementation + logs and returns; it does not raise, because bus losses must not + cascade into worker failure (DB is source of truth). + """ + + @abc.abstractmethod + def subscribe(self, pattern: str) -> Subscription: + """Return a :class:`Subscription` that yields events matching *pattern*. + + Patterns follow NATS wildcard semantics: ``*`` matches one topic + token, ``>`` matches one-or-more trailing tokens. Examples: + + * ``topology.*.mutation.applied`` — all ``applied`` events for any + topology. + * ``topology.abc123.mutation.*`` — all mutation states for one + topology. + * ``topology.>`` — every event under the ``topology`` root. + """ + + @abc.abstractmethod + async def close(self) -> None: + """Tear down transport resources. Idempotent.""" + + async def __aenter__(self) -> "BaseBus": + await self.connect() + return self + + async def __aexit__(self, *exc: Any) -> None: + await self.close() + + +# ─── Wildcard matching shared across in-process transports ─────────────────── + +def matches(pattern: str, topic: str) -> bool: + """Return True iff *topic* matches *pattern* under NATS wildcard rules. + + ``*`` matches exactly one non-empty token; ``>`` matches one-or-more + trailing tokens (so ``topology.>`` matches ``topology.abc.x`` but not + ``topology`` alone). + """ + p_tokens = pattern.split(".") + t_tokens = topic.split(".") + for i, p in enumerate(p_tokens): + if p == ">": + # Must have at least one token remaining to match. + return i < len(t_tokens) + if i >= len(t_tokens): + return False + if p == "*": + if not t_tokens[i]: + return False + continue + if p != t_tokens[i]: + return False + return len(p_tokens) == len(t_tokens) + + +# Sentinel used by the in-process transports to signal "no more events" +# through the asyncio.Queue fan-out without inventing a separate control +# channel. Not part of the wire protocol. +_CLOSE_SENTINEL: Any = object() + + +async def _next_or_stop(queue: "asyncio.Queue[Any]") -> Event: + """Pop the next item from *queue*, raising ``StopAsyncIteration`` on close.""" + item = await queue.get() + if item is _CLOSE_SENTINEL: + raise StopAsyncIteration + return item diff --git a/decnet/bus/factory.py b/decnet/bus/factory.py new file mode 100644 index 00000000..f7a935ac --- /dev/null +++ b/decnet/bus/factory.py @@ -0,0 +1,85 @@ +"""Bus factory — selects a :class:`~decnet.bus.base.BaseBus` implementation. + +Dispatch key: the ``DECNET_BUS_TYPE`` environment variable. + +* ``unix`` (default) → :class:`~decnet.bus.unix_client.UnixSocketBus` +* ``fake`` → :class:`~decnet.bus.fake.FakeBus` (in-process) + +If ``DECNET_BUS_ENABLED`` is ``"false"`` the factory short-circuits to +:class:`~decnet.bus.fake.NullBus` regardless of ``DECNET_BUS_TYPE`` — a +cheap way for dev environments to run workers without a bus daemon. + +Mirrors :mod:`decnet.web.db.factory` (lazy imports inside each branch, +env-driven dispatch, optional telemetry wrapping). Callers MUST use +:func:`get_bus` rather than instantiating transports directly. +""" +from __future__ import annotations + +import os +from typing import Any + +from decnet.bus.base import BaseBus + + +def get_bus(**kwargs: Any) -> BaseBus: + """Instantiate the bus implementation selected by environment. + + Keyword arguments are forwarded to the concrete transport: + + * ``UnixSocketBus`` accepts ``socket_path`` (overrides + ``DECNET_BUS_SOCKET``) and ``client_name``. + * ``FakeBus`` accepts ``queue_size``. + """ + if os.environ.get("DECNET_BUS_ENABLED", "true").lower() == "false": + from decnet.bus.fake import NullBus + return NullBus() + + bus_type = os.environ.get("DECNET_BUS_TYPE", "unix").lower() + + if bus_type == "unix": + from decnet.bus.unix_client import UnixSocketBus + socket_path = kwargs.pop("socket_path", None) or _default_socket_path() + bus: BaseBus = UnixSocketBus(socket_path=socket_path, **kwargs) + elif bus_type == "fake": + from decnet.bus.fake import FakeBus + bus = FakeBus(**kwargs) + else: + raise ValueError(f"Unsupported bus type: {bus_type}") + + return _maybe_wrap_telemetry(bus) + + +def _default_socket_path() -> str: + """Return the bus socket path honoring ``DECNET_BUS_SOCKET`` and falling + back to ``/run/decnet/bus.sock`` → ``~/.decnet/bus.sock``. + + The runtime path (``/run/decnet``) is preferred because systemd + ``RuntimeDirectory=decnet`` sets it up with the right perms; the home + fallback keeps dev boxes usable without systemd. + """ + explicit = os.environ.get("DECNET_BUS_SOCKET") + if explicit: + return explicit + + runtime_dir = "/run/decnet" + if os.path.isdir(runtime_dir) and os.access(runtime_dir, os.W_OK): + return f"{runtime_dir}/bus.sock" + return os.path.expanduser("~/.decnet/bus.sock") + + +def _maybe_wrap_telemetry(bus: BaseBus) -> BaseBus: + """Wrap *bus* in a tracing proxy if OTEL is enabled, else return as-is. + + Uses :func:`decnet.telemetry.wrap_repository` as the underlying proxy — + its implementation is generic (wraps any async method in a span), so we + reuse it with a bus-appropriate tracer name. If telemetry isn't wired + up at all we no-op. + """ + try: + from decnet.telemetry import wrap_repository # type: ignore[attr-defined] + except ImportError: + return bus + try: + return wrap_repository(bus) + except Exception: # pragma: no cover - defensive + return bus diff --git a/decnet/bus/fake.py b/decnet/bus/fake.py new file mode 100644 index 00000000..9f6a26a9 --- /dev/null +++ b/decnet/bus/fake.py @@ -0,0 +1,183 @@ +"""In-process bus transports. + +* :class:`FakeBus` — real pub/sub semantics without touching a socket. Used + by unit tests and anywhere ``DECNET_BUS_TYPE=fake`` is set. Lets code + that depends on the bus be exercised entirely inside a single event loop, + matching the DECNET testing convention of not opening real network + sockets from unit tests. +* :class:`NullBus` — no-op. Returned by :func:`~decnet.bus.factory.get_bus` + when ``DECNET_BUS_ENABLED=false`` so workers can start cleanly in dev + environments where no bus daemon is running. Publishes are dropped; + subscriptions yield nothing and close cleanly. +""" +from __future__ import annotations + +import asyncio +from typing import Any + +from decnet.bus.base import ( + BaseBus, + Event, + Subscription, + _CLOSE_SENTINEL, + matches, +) +from decnet.logging import get_logger + +log = get_logger("bus.fake") + +# Per-subscriber bounded queue: backpressure policy is drop-oldest so a slow +# consumer cannot stall publishers (the invariant — DB is the source of +# truth — makes dropped events acceptable). +_DEFAULT_QUEUE_SIZE = 1024 + + +# ─── FakeBus ───────────────────────────────────────────────────────────────── + + +class _FakeSubscription(Subscription): + """Subscription backed by an :class:`asyncio.Queue` fed from + :meth:`FakeBus.publish`. Unregisters itself on close.""" + + def __init__(self, bus: "FakeBus", pattern: str, queue: "asyncio.Queue[Any]") -> None: + super().__init__(pattern) + self._bus = bus + self._queue = queue + + async def __anext__(self) -> Event: + if self._closed: + raise StopAsyncIteration + item = await self._queue.get() + if item is _CLOSE_SENTINEL: + raise StopAsyncIteration + return item + + async def _aclose(self) -> None: + self._bus._unregister(self) + # Unblock any pending __anext__ waiter. + try: + self._queue.put_nowait(_CLOSE_SENTINEL) + except asyncio.QueueFull: + pass + + +class FakeBus(BaseBus): + """In-process pub/sub. + + Publishes iterate every active subscription and enqueue the event on + the ones whose pattern matches the topic. If a subscriber's queue is + full, the oldest event is discarded to make room — same at-most-once + semantics as the real UNIX-socket transport. + """ + + def __init__(self, queue_size: int = _DEFAULT_QUEUE_SIZE) -> None: + self._queue_size = queue_size + self._subs: list[_FakeSubscription] = [] + self._connected = False + self._closed = False + self._lock = asyncio.Lock() + + async def connect(self) -> None: + self._connected = True + + async def publish( + self, + topic: str, + payload: dict[str, Any], + *, + event_type: str = "", + ) -> None: + if self._closed: + raise RuntimeError("publish on closed bus") + event = Event(topic=topic, payload=payload, type=event_type) + async with self._lock: + targets = [s for s in self._subs if matches(s.pattern, topic)] + for sub in targets: + _enqueue_drop_oldest(sub._queue, event) + + def subscribe(self, pattern: str) -> Subscription: + if self._closed: + raise RuntimeError("subscribe on closed bus") + queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=self._queue_size) + sub = _FakeSubscription(self, pattern, queue) + self._subs.append(sub) + return sub + + def _unregister(self, sub: _FakeSubscription) -> None: + try: + self._subs.remove(sub) + except ValueError: + pass + + async def close(self) -> None: + if self._closed: + return + self._closed = True + # Wake every still-open subscription so iterators unblock cleanly. + for sub in list(self._subs): + try: + sub._queue.put_nowait(_CLOSE_SENTINEL) + except asyncio.QueueFull: + pass + self._subs.clear() + + +def _enqueue_drop_oldest(queue: "asyncio.Queue[Any]", event: Event) -> None: + """Put *event* on *queue*, dropping the oldest item if the queue is full. + + Factored out so both FakeBus and the real UNIX server share the exact + same backpressure policy. + """ + while True: + try: + queue.put_nowait(event) + return + except asyncio.QueueFull: + try: + dropped = queue.get_nowait() + log.warning( + "bus.fake: subscriber queue full, dropped %s", getattr(dropped, "topic", "?") + ) + except asyncio.QueueEmpty: + return + + +# ─── NullBus ───────────────────────────────────────────────────────────────── + + +class _NullSubscription(Subscription): + """A subscription that never yields and closes immediately on iteration.""" + + async def __anext__(self) -> Event: + raise StopAsyncIteration + + async def _aclose(self) -> None: + return + + +class NullBus(BaseBus): + """No-op bus used when ``DECNET_BUS_ENABLED=false``. + + Publishes are silently dropped; subscriptions are empty. Intended for + dev environments where no bus daemon is running — the process starts + cleanly, code that publishes doesn't need feature flags, and nothing + ever blocks on a subscriber. + """ + + async def connect(self) -> None: + return + + async def publish( + self, + topic: str, + payload: dict[str, Any], + *, + event_type: str = "", + ) -> None: + return + + def subscribe(self, pattern: str) -> Subscription: + return _NullSubscription(pattern) + + async def close(self) -> None: + return diff --git a/decnet/bus/protocol.py b/decnet/bus/protocol.py new file mode 100644 index 00000000..a0f2f2eb --- /dev/null +++ b/decnet/bus/protocol.py @@ -0,0 +1,144 @@ +"""Wire protocol for the DECNET bus UNIX-socket transport. + +Frame layout: + + []\\n # ASCII header, single line, no trailing space + <4-byte big-endian body length> + # orjson-serialized dict, or empty (length 0) + +Verbs: + +* ``HELLO `` — optional greeting, logged by server. Body empty. +* ``PUB `` — publisher → server. Body = payload dict. +* ``SUB `` — subscriber → server. Body empty. +* ``UNSUB `` — subscriber → server. Body empty. +* ``EVT `` — server → subscriber. Body = payload dict (wrapped + in an :class:`~decnet.bus.base.Event` envelope). +* ``BYE`` — either direction. Body empty. Graceful shutdown. + +Parsing rules: + +* The header is a single line terminated by ``\\n`` (LF). ``\\r`` is tolerated + but not required. +* Header tokens are whitespace-separated. The first token is the verb; + everything after is verb-specific. We split on the first space only so + topics / patterns with quoted content are not supported (they are not + needed — topic segments forbid whitespace per :mod:`decnet.bus.topics`). +* Maximum header length is 4096 bytes; maximum body length is 1 MiB. Beyond + those, the connection is dropped with a logged error. This is a honeypot + framework, not a general-purpose message broker; a malformed frame is + treated as hostile. +""" +from __future__ import annotations + +import asyncio +import struct +from dataclasses import dataclass +from typing import Any + +import orjson + +MAX_HEADER_BYTES = 4096 +MAX_BODY_BYTES = 1 * 1024 * 1024 # 1 MiB + +# Verb constants (callers should reference these, not bare strings). +HELLO = "HELLO" +PUB = "PUB" +SUB = "SUB" +UNSUB = "UNSUB" +EVT = "EVT" +BYE = "BYE" + +_VALID_VERBS = frozenset({HELLO, PUB, SUB, UNSUB, EVT, BYE}) + + +class ProtocolError(Exception): + """Malformed or oversized frame. Callers should close the connection.""" + + +@dataclass(frozen=True) +class Frame: + """A parsed frame. ``body`` is the raw (unparsed) body bytes — callers + decide whether to orjson-decode it (the protocol does not know whether + a given verb expects a dict body or an empty one). + """ + + verb: str + args: str # everything after the verb on the header line, trimmed + body: bytes + + +def encode(verb: str, args: str = "", body: dict[str, Any] | None = None) -> bytes: + """Serialize a frame. + + *body* is a dict that will be orjson-encoded, or ``None`` for an empty + body. The header line is written verbatim — callers must supply args + that are free of ``\\n``. + """ + if verb not in _VALID_VERBS: + raise ProtocolError(f"unknown verb {verb!r}") + if "\n" in args or "\r" in args: + raise ProtocolError("args must not contain newline characters") + + body_bytes = b"" if body is None else orjson.dumps(body) + if len(body_bytes) > MAX_BODY_BYTES: + raise ProtocolError( + f"body {len(body_bytes)} bytes exceeds max {MAX_BODY_BYTES}" + ) + + header = f"{verb} {args}".rstrip() + "\n" + header_bytes = header.encode("ascii") + if len(header_bytes) > MAX_HEADER_BYTES: + raise ProtocolError( + f"header {len(header_bytes)} bytes exceeds max {MAX_HEADER_BYTES}" + ) + return header_bytes + struct.pack(">I", len(body_bytes)) + body_bytes + + +async def read_frame(reader: asyncio.StreamReader) -> Frame | None: + """Read one frame from *reader*. + + Returns ``None`` on clean EOF before a new frame starts. Raises + :class:`ProtocolError` on malformed input (caller should close the + connection). + """ + try: + header = await reader.readuntil(b"\n") + except asyncio.IncompleteReadError as exc: + if not exc.partial: + return None + raise ProtocolError("connection closed mid-header") from exc + except asyncio.LimitOverrunError as exc: + raise ProtocolError("header exceeded buffer limit") from exc + + if len(header) > MAX_HEADER_BYTES: + raise ProtocolError(f"header {len(header)} bytes exceeds max") + + line = header.rstrip(b"\r\n").decode("ascii", errors="strict") + if not line: + raise ProtocolError("empty header line") + + verb, _, args = line.partition(" ") + if verb not in _VALID_VERBS: + raise ProtocolError(f"unknown verb {verb!r}") + + length_bytes = await reader.readexactly(4) + (body_len,) = struct.unpack(">I", length_bytes) + if body_len > MAX_BODY_BYTES: + raise ProtocolError(f"body length {body_len} exceeds max") + + body = await reader.readexactly(body_len) if body_len else b"" + return Frame(verb=verb, args=args.strip(), body=body) + + +def decode_body(body: bytes) -> dict[str, Any]: + """Decode a frame body as a JSON dict. Empty body → empty dict.""" + if not body: + return {} + try: + obj = orjson.loads(body) + except orjson.JSONDecodeError as exc: + raise ProtocolError(f"body is not valid JSON: {exc}") from exc + if not isinstance(obj, dict): + raise ProtocolError(f"body must be a JSON object, got {type(obj).__name__}") + return obj diff --git a/decnet/bus/publish.py b/decnet/bus/publish.py new file mode 100644 index 00000000..15319cfe --- /dev/null +++ b/decnet/bus/publish.py @@ -0,0 +1,211 @@ +"""Fire-and-forget publish helpers shared across every worker. + +Lifted out of ``decnet/mutator/engine.py`` once a second caller showed up +(DEBT-031). Keeping one implementation means the "never break the worker +loop" guarantee is audited in exactly one place. +""" +from __future__ import annotations + +import asyncio +import contextlib +import os +import signal +import time +from typing import Any, Callable + +from decnet.bus import topics as _topics +from decnet.bus.base import BaseBus +from decnet.logging import get_logger + +log = get_logger("bus.publish") + + +async def publish_safely( + bus: BaseBus | None, + topic: str, + payload: dict[str, Any], + event_type: str = "", +) -> None: + """Publish on *bus* without ever raising back at the caller. + + The DB row (or equivalent side-effect) has already been committed by + the time a worker calls this; the bus is the notification layer, not + the source of truth. A dropped publish is at most a few seconds of + UI latency until the next poll tick. A raised exception here, by + contrast, would crash the worker — which is strictly worse. + """ + if bus is None: + return + try: + await bus.publish(topic, payload, event_type=event_type) + except Exception as exc: # noqa: BLE001 + log.warning("bus publish failed topic=%s: %s", topic, exc) + + +def make_thread_safe_publisher( + bus: BaseBus | None, + loop: asyncio.AbstractEventLoop, +) -> Callable[[str, dict[str, Any], str], None]: + """Build a sync callable that marshals publishes back to *loop*. + + Workers that run their hot paths in a worker thread (scapy sniff loop, + ``asyncio.to_thread`` probes, blocking socket reads) cannot ``await`` + the bus directly. This helper returns a plain function that schedules + the publish on *loop* via ``run_coroutine_threadsafe`` and returns + immediately — the calling thread is never blocked on the publish. + + A ``None`` bus yields a no-op callable, matching the degraded-mode + contract the rest of this module already upholds. + """ + if bus is None: + return lambda _topic, _payload, _event_type="": None + + def _publish(topic: str, payload: dict[str, Any], event_type: str = "") -> None: + # Stream threads may keep draining after the bus owner closed it + # (shutdown race). Short-circuit here so we don't marshal a + # coroutine onto a dead loop just to have publish_safely swallow + # it. bus.publish's own WARN-once guard handles the rare case + # where _closed flips between this check and the coroutine + # actually running. + if getattr(bus, "_closed", False): + return + try: + asyncio.run_coroutine_threadsafe( + publish_safely(bus, topic, payload, event_type=event_type), + loop, + ) + except Exception as exc: # noqa: BLE001 + log.debug("cross-thread bus publish failed topic=%s: %s", topic, exc) + + return _publish + + +async def run_health_heartbeat( + bus: BaseBus | None, + worker: str, + *, + interval: float = 30.0, + extra: Callable[[], dict[str, Any]] | None = None, +) -> None: + """Publish ``system..health`` every *interval* seconds. + + Standard heartbeat loop shared across agent/forwarder/updater. Emits + ``{"worker": , "ts": , **extra()}`` on each tick. A + ``None`` bus turns the loop into a no-op sleep cycle — still cancellable + so the caller can use the same ``asyncio.create_task``/``.cancel()`` + pattern regardless of bus state. + + Cancellation-safe: unwraps the ``CancelledError`` so callers awaiting + the task during shutdown see a clean exit. + """ + topic = _topics.system_health(worker) + with contextlib.suppress(asyncio.CancelledError): + while True: + payload: dict[str, Any] = {"worker": worker, "ts": time.time()} + if extra is not None: + try: + payload.update(extra()) + except Exception as exc: # noqa: BLE001 + log.debug("heartbeat extra() failed worker=%s: %s", worker, exc) + await publish_safely(bus, topic, payload, event_type=_topics.SYSTEM_HEALTH) + await asyncio.sleep(interval) + + +async def run_control_listener( + bus: BaseBus | None, + worker: str, + shutdown: asyncio.Event, +) -> None: + """Subscribe to ``system..control`` and honour stop intents. + + On a well-formed ``{"action": "stop", ...}`` message the function sets + *shutdown* and returns — the worker's main loop is expected to check + the event and unwind cleanly, matching the SIGTERM path. + + Malformed payloads (missing/unknown action, non-dict, exception from + the transport) are logged and ignored. A ``None`` bus yields a noop + coroutine that simply awaits *shutdown* — callers can ``create_task`` + this unconditionally regardless of bus state. + + Cancellation-safe. + """ + if bus is None: + with contextlib.suppress(asyncio.CancelledError): + await shutdown.wait() + return + + topic = _topics.system_control(worker) + with contextlib.suppress(asyncio.CancelledError): + try: + async with bus.subscribe(topic) as sub: + async for event in sub: + payload = event.payload or {} + action = payload.get("action") + requested_by = payload.get("requested_by", "") + if action == _topics.WORKER_CONTROL_STOP: + log.info( + "control: stop requested worker=%s by=%s", + worker, requested_by, + ) + shutdown.set() + return + log.debug( + "control: ignoring unknown action worker=%s action=%r", + worker, action, + ) + except Exception as exc: # noqa: BLE001 + log.warning( + "control listener failed worker=%s: %s — shutdown via bus disabled", + worker, exc, + ) + + +async def run_control_listener_signal( + bus: BaseBus | None, + worker: str, +) -> None: + """Like :func:`run_control_listener` but signals the process on stop. + + Preferred for workers whose main loop is a blocking thread + (container-log tail, PTY read, scapy sniff) — wiring an + ``asyncio.Event`` through the thread boundary is error-prone, and + every DECNET worker already has systemd-equivalent SIGTERM cleanup. + A SIGTERM self-signal routes the stop through that same path + without inventing a second shutdown mechanism. + + Cancellation-safe. Never raises: a failed self-signal is logged + and the loop simply exits (admin can fall back to ``systemctl``). + """ + if bus is None: + return + + topic = _topics.system_control(worker) + with contextlib.suppress(asyncio.CancelledError): + try: + async with bus.subscribe(topic) as sub: + async for event in sub: + payload = event.payload or {} + action = payload.get("action") + requested_by = payload.get("requested_by", "") + if action == _topics.WORKER_CONTROL_STOP: + log.info( + "control: stop requested worker=%s by=%s → SIGTERM self", + worker, requested_by, + ) + try: + os.kill(os.getpid(), signal.SIGTERM) + except Exception as exc: # noqa: BLE001 + log.warning( + "control: self-signal failed worker=%s: %s", + worker, exc, + ) + return + log.debug( + "control: ignoring unknown action worker=%s action=%r", + worker, action, + ) + except Exception as exc: # noqa: BLE001 + log.warning( + "control signal listener failed worker=%s: %s", + worker, exc, + ) diff --git a/decnet/bus/topics.py b/decnet/bus/topics.py new file mode 100644 index 00000000..3c89d7e4 --- /dev/null +++ b/decnet/bus/topics.py @@ -0,0 +1,398 @@ +"""Canonical topic hierarchy for the DECNET ServiceBus. + +Locked early so consumers can subscribe with stable wildcard patterns. +Adding new topic families is fine; **renaming** existing ones is a breaking +change for every subscriber and requires a coordinated rollout. + +Token structure (NATS-style, dot-separated): + + topology.{topology_id}.mutation.{state} + topology.{topology_id}.status + decky.{decky_id}.state + decky.{decky_id}.traffic + orchestrator.traffic.{decky_id} + orchestrator.file.{decky_id} + orchestrator.email.{decky_id} + attacker.observed + attacker.scored + attacker.session.started + attacker.session.ended + identity.formed + identity.observation.linked + identity.merged + identity.unmerged + identity.campaign.assigned + campaign.formed + campaign.identity.assigned + campaign.merged + campaign.unmerged + credential.captured + credential.reuse.detected + canary.{token_id}.triggered + canary.{token_id}.placed + canary.{token_id}.revoked + system.log + system.bus.health + system.{worker}.health + +Wildcards (per :func:`decnet.bus.base.matches`): + +* ``*`` matches exactly one token. +* ``>`` matches one-or-more trailing tokens (so ``topology.>`` matches + ``topology.abc.status`` but not the bare root ``topology``). +""" +from __future__ import annotations + +# ─── Root prefixes ─────────────────────────────────────────────────────────── + +TOPOLOGY = "topology" +DECKY = "decky" +ATTACKER = "attacker" +IDENTITY = "identity" +CAMPAIGN = "campaign" +SYSTEM = "system" +CREDENTIAL = "credential" +ORCHESTRATOR = "orchestrator" +CANARY = "canary" + + +# ─── Leaf event-type constants (the last segment of each topic) ────────────── + +# Topology mutation lifecycle states — keep in sync with TopologyMutation.state +# in decnet/web/db/models.py; the bus topic mirrors the DB state machine. +MUTATION_ENQUEUED = "enqueued" +MUTATION_APPLYING = "applying" +MUTATION_APPLIED = "applied" +MUTATION_FAILED = "failed" + +# Topology-level status transitions (topology.{id}.status): fires when the +# topology row's status column changes (pending/deploying/active/degraded/failed). +TOPOLOGY_STATUS = "status" + +# Decky-level event types (second token). +DECKY_STATE = "state" +DECKY_TRAFFIC = "traffic" +# On-demand mutation request — published by the API/CLI/UI, consumed by +# the mutator's watch loop to force an immediate mutation of one decky +# without waiting for its scheduled interval. Underscored (not dotted) +# to stay a single NATS token so the builder's validator accepts it. +DECKY_MUTATE_REQUEST = "mutate_request" +# Mutation transition event — distinct from DECKY_STATE ("current +# shape") because a mutation is a *transition* that carries old/new +# services + trigger + timing. Correlator consumes these (via the +# syslog sidechannel too) to interleave substrate-change markers into +# attacker traversals. +DECKY_MUTATION = "mutation" + +# Attacker event types (second token under the ``attacker`` root). First +# sighting, session boundary transitions, and score-threshold crossings +# published by correlator + profiler. Consumers typically subscribe to +# the wildcard ``attacker.>``. +ATTACKER_OBSERVED = "observed" +ATTACKER_SCORED = "scored" +# Published once per successful active probe result (JARM/HASSH/TCPfp). +# Distinct from ``observed`` which is the correlator's first-sight signal — +# a fingerprint is additional evidence about an already-observed attacker. +ATTACKER_FINGERPRINTED = "fingerprinted" +ATTACKER_SESSION_STARTED = "session.started" +ATTACKER_SESSION_ENDED = "session.ended" +# Published by the ``decnet enrich`` worker after an enrichment pass +# succeeds for an attacker IP (one or more 3rd-party intel providers +# returned a verdict). Payload carries the aggregate verdict + per- +# provider summary so SIEM-bound webhooks don't need to re-query the DB. +ATTACKER_INTEL_ENRICHED = "intel.enriched" + +# Identity-resolution event types (second/third tokens under ``identity``). +# Published by the (future) clusterer worker — see +# development/IDENTITY_RESOLUTION.md. Constants ship in this commit; +# no publishers exist yet, but consumers (webhook worker, dashboard +# SSE relay) can subscribe to ``identity.>`` from day one and receive +# events the instant the clusterer comes online. +# +# identity.formed — clusterer creates a new identity from +# one or more observations +# identity.observation.linked — observation attached to an existing +# identity (or reattached from another) +# identity.merged — two identities collapsed; loser gets +# ``merged_into_uuid`` set, subscribers +# re-key cached references to the winner +# identity.unmerged — revocable-merge undo: contradicting +# evidence cleared ``merged_into_uuid`` +# and re-split observations. The +# resurrected side's UUID is the same +# as the prior loser, so subscribers +# that cached references to the loser +# during the merged interval can +# re-attach without a new lookup. +# +# ``identity.campaign.assigned`` is deferred; it ships when the campaign +# clusterer ships. YAGNI before then. +IDENTITY_FORMED = "formed" +IDENTITY_OBSERVATION_LINKED = "observation.linked" +IDENTITY_MERGED = "merged" +IDENTITY_UNMERGED = "unmerged" +# Campaign-clusterer cross-family event — fires under ``identity.>`` so +# identity-stream subscribers (e.g. the IdentityDetail SSE client) get +# notified the moment an identity's ``campaign_id`` changes without +# having to subscribe to the campaign topic family. The same event +# fires under ``campaign.identity.assigned`` for campaign-side +# subscribers. +IDENTITY_CAMPAIGN_ASSIGNED = "campaign.assigned" + +# Campaign-clusterer event types (second/third tokens under +# ``campaign``). Mirror of the identity family at the layer above: +# campaigns group identities into operations, and the clusterer +# publishes the same form / link / merge / unmerge lifecycle. +# +# campaign.formed — clusterer creates a new campaign from +# one or more identities +# campaign.identity.assigned — identity attached to an existing +# campaign (or reassigned from another) +# campaign.merged — two campaigns collapsed; loser gets +# ``merged_into_uuid`` set, subscribers +# re-key cached references to the winner +# campaign.unmerged — revocable-merge undo: contradicting +# evidence cleared ``merged_into_uuid`` +# and re-split identities +CAMPAIGN_FORMED = "formed" +CAMPAIGN_IDENTITY_ASSIGNED = "identity.assigned" +CAMPAIGN_MERGED = "merged" +CAMPAIGN_UNMERGED = "unmerged" + +# Credential event types (second/third tokens under ``credential``). +# ``credential.captured`` fires once per upserted Credential row — the +# correlator listens for it and runs the cred-reuse query in response, +# so reuse detection latency is sub-second after a fresh capture. +# ``credential.reuse.detected`` fires when the correlator inserts a new +# CredentialReuse row or grows an existing one (added decky/service/IP). +CREDENTIAL_CAPTURED = "captured" +CREDENTIAL_REUSE_DETECTED = "reuse.detected" + +# Canary-token event types (third token under ``canary``). +# +# canary.{token_id}.placed — orchestrator/API successfully planted a +# canary artifact inside a decky's +# filesystem (or persisted a passive token +# that has no callback wiring). Lets +# dashboards reflect baseline coverage in +# real time without a DB poll. +# canary.{token_id}.triggered — ``decnet canary`` worker observed a +# callback hit (HTTP slug or DNS subdomain +# lookup) for the token. Payload carries +# ``src_ip``, ``user_agent``, ``request_path`` +# and any DNS qname so downstream +# consumers (correlator, webhook fanout) +# can attribute and forward without a +# follow-up DB read. +# canary.{token_id}.revoked — operator removed a token; planter unlinked +# the file (best-effort) and the row was +# marked ``revoked``. Subscribers may +# evict cached lookups by token id. +CANARY_PLACED = "placed" +CANARY_TRIGGERED = "triggered" +CANARY_REVOKED = "revoked" + +# Orchestrator event types (second token under ``orchestrator``). The +# orchestrator worker publishes one of these per synthetic action it +# drives against a decky — cheap inter-decky traffic and filesystem +# mutations whose role is to keep the honeypot from looking suspiciously +# static. Always nested with the destination decky uuid as the third +# token, so consumers can subscribe to a single decky's life-injection +# stream via ``orchestrator.*.``. +ORCHESTRATOR_TRAFFIC = "traffic" +ORCHESTRATOR_FILE = "file" +# Emailgen — published by the ``decnet emailgen`` worker once per generated +# fake email delivered into a mail decky's maildir. Third token is the +# destination mail-decky uuid (the IMAP/POP3 host serving the mailbox), +# matching the ``orchestrator.*.`` subscription pattern. +ORCHESTRATOR_EMAIL = "email" + +# System event types. +SYSTEM_LOG = "log" +SYSTEM_BUS_HEALTH = "bus.health" +# Worker-health leaf — built per-worker as ``system..health`` via +# :func:`system_health`. The leaf constant stays the same across workers; +# the worker name goes in the middle token. +SYSTEM_HEALTH = "health" +# Worker-control leaf — built per-worker as ``system..control`` via +# :func:`system_control`. Admin-originated stop intents travel on this +# topic; each worker subscribes to its own. +SYSTEM_CONTROL = "control" + +# Control payload ``action`` values — the wire vocabulary. Only ``stop`` is +# handled in v1; ``start`` is reserved because a stopped worker has no +# subscriber, so starting requires external supervision (systemd). +WORKER_CONTROL_STOP = "stop" +WORKER_CONTROL_START = "start" + +# Webhook subscription-set changed — published by the CRUD router after any +# create / update / delete on WebhookSubscription so the webhook worker can +# reload its in-memory subscription list and re-subscribe to the new union +# of patterns. Payload is currently empty; consumers only need the signal. +WEBHOOK_SUBSCRIPTIONS_CHANGED = "system.webhook.subscriptions_changed" + + +# ─── Builders ──────────────────────────────────────────────────────────────── + +def topology_mutation(topology_id: str, state: str) -> str: + """Build ``topology..mutation.``. + + *state* should be one of the ``MUTATION_*`` constants. + """ + _reject_tokens(topology_id, state) + return f"{TOPOLOGY}.{topology_id}.mutation.{state}" + + +def topology_status(topology_id: str) -> str: + """Build ``topology..status``.""" + _reject_tokens(topology_id) + return f"{TOPOLOGY}.{topology_id}.{TOPOLOGY_STATUS}" + + +def decky(decky_id: str, event_type: str) -> str: + """Build ``decky..``. + + *event_type* is typically one of ``DECKY_STATE`` or ``DECKY_TRAFFIC``. + """ + _reject_tokens(decky_id, event_type) + return f"{DECKY}.{decky_id}.{event_type}" + + +def decky_mutation(decky_id: str) -> str: + """Build ``decky..mutation``.""" + _reject_tokens(decky_id) + return f"{DECKY}.{decky_id}.{DECKY_MUTATION}" + + +def system(event_type: str) -> str: + """Build ``system.``. + + *event_type* may itself contain dots (e.g. ``bus.health``) — we don't + re-validate the already-constant leaves; this just prefixes. + """ + if not event_type: + raise ValueError("system topic requires a non-empty event_type") + return f"{SYSTEM}.{event_type}" + + +def credential(event_type: str) -> str: + """Build ``credential.``. + + *event_type* is typically one of :data:`CREDENTIAL_CAPTURED` or + :data:`CREDENTIAL_REUSE_DETECTED`. Dotted leaves + (``reuse.detected``) are permitted — same rationale as + :func:`system`. + """ + if not event_type: + raise ValueError("credential topic requires a non-empty event_type") + return f"{CREDENTIAL}.{event_type}" + + +def attacker(event_type: str) -> str: + """Build ``attacker.``. + + *event_type* is typically one of ``ATTACKER_OBSERVED``, + ``ATTACKER_SCORED``, ``ATTACKER_SESSION_STARTED``, + ``ATTACKER_SESSION_ENDED``. Dotted leaves (``session.started``) are + permitted — same rationale as :func:`system`. + """ + if not event_type: + raise ValueError("attacker topic requires a non-empty event_type") + return f"{ATTACKER}.{event_type}" + + +def campaign(event_type: str) -> str: + """Build ``campaign.``. + + *event_type* is typically one of :data:`CAMPAIGN_FORMED`, + :data:`CAMPAIGN_IDENTITY_ASSIGNED`, :data:`CAMPAIGN_MERGED`, or + :data:`CAMPAIGN_UNMERGED`. Dotted leaves (``identity.assigned``) + are permitted — same rationale as :func:`system`. + """ + if not event_type: + raise ValueError("campaign topic requires a non-empty event_type") + return f"{CAMPAIGN}.{event_type}" + + +def identity(event_type: str) -> str: + """Build ``identity.``. + + *event_type* is typically one of :data:`IDENTITY_FORMED`, + :data:`IDENTITY_OBSERVATION_LINKED`, :data:`IDENTITY_MERGED`, or + :data:`IDENTITY_UNMERGED`. Dotted leaves (``observation.linked``) + are permitted — same rationale as :func:`system`. + """ + if not event_type: + raise ValueError("identity topic requires a non-empty event_type") + return f"{IDENTITY}.{event_type}" + + +def orchestrator(event_type: str, decky_id: str) -> str: + """Build ``orchestrator..``. + + *event_type* should be one of :data:`ORCHESTRATOR_TRAFFIC` or + :data:`ORCHESTRATOR_FILE`. The destination decky is always the + third token so per-decky subscribers can use + ``orchestrator.*.``. + """ + _reject_tokens(event_type, decky_id) + return f"{ORCHESTRATOR}.{event_type}.{decky_id}" + + +def canary(token_id: str, event_type: str) -> str: + """Build ``canary..``. + + *event_type* should be one of :data:`CANARY_PLACED`, + :data:`CANARY_TRIGGERED`, or :data:`CANARY_REVOKED`. The token id + is always the second token so per-token subscribers can use + ``canary..>`` and fleet-wide consumers (webhook fanout, + correlator) use ``canary.>``. + """ + _reject_tokens(token_id, event_type) + return f"{CANARY}.{token_id}.{event_type}" + + +def system_health(worker: str) -> str: + """Build ``system..health``. + + Worker-health heartbeats live as a nested leaf under ``system`` so + consumers can subscribe to ``system.*.health`` for every worker at + once, or to ``system.mutator.health`` for a single one. *worker* is + validated as a regular segment — no dots, wildcards, or whitespace. + """ + _reject_tokens(worker) + return f"{SYSTEM}.{worker}.{SYSTEM_HEALTH}" + + +def system_control(worker: str) -> str: + """Build ``system..control``. + + Admin-originated stop (and, eventually, start) intents are published + here; the worker in question subscribes to its own address and reacts. + Payload shape:: + + {"action": "stop", "requested_by": "", "ts": } + + *action* must be one of :data:`WORKER_CONTROL_STOP` / + :data:`WORKER_CONTROL_START`; any other value is ignored by the + listener. Same segment rules as :func:`system_health`. + """ + _reject_tokens(worker) + return f"{SYSTEM}.{worker}.{SYSTEM_CONTROL}" + + +def _reject_tokens(*parts: str) -> None: + """Reject topic segments that would break NATS-style tokenization. + + Dots, wildcards, whitespace, and empty strings in a *segment* would + silently corrupt the hierarchy (e.g. ``topology.a.b.status`` for a + ``topology_id`` of ``"a.b"``). Raise early at the builder instead of + shipping a malformed topic to the wire. + """ + for p in parts: + if not p: + raise ValueError("topic segment must not be empty") + if "." in p or "*" in p or ">" in p or any(c.isspace() for c in p): + raise ValueError( + f"topic segment {p!r} may not contain '.', '*', '>', or whitespace" + ) diff --git a/decnet/bus/unix_client.py b/decnet/bus/unix_client.py new file mode 100644 index 00000000..226b296a --- /dev/null +++ b/decnet/bus/unix_client.py @@ -0,0 +1,257 @@ +"""UNIX-socket client — :class:`UnixSocketBus` implementation of :class:`BaseBus`. + +Holds one open socket to the local :class:`~decnet.bus.unix_server.BusServer`. +Operations: + +* :meth:`publish` writes a single ``PUB`` frame and returns; no ack. +* :meth:`subscribe` writes a ``SUB`` frame and returns a + :class:`~decnet.bus.base.Subscription` backed by an :class:`asyncio.Queue` + that the background reader task feeds. + +One background reader task per bus instance dispatches incoming ``EVT`` +frames to every registered subscription whose pattern matches the topic. +On connection drop or close, every subscription is woken via a sentinel so +iterators unblock cleanly; callers see :class:`StopAsyncIteration` from the +``async for`` loop. + +No auto-reconnect in MVP. If the server restarts, callers must +:meth:`close` the bus and construct a new one. This mirrors how other +DECNET workers handle their dependencies — the systemd ``Restart=on-failure`` +supervision above us is the retry loop. +""" +from __future__ import annotations + +import asyncio +import contextlib +import os +import pathlib +from typing import Any + +from decnet.bus import protocol +from decnet.bus.base import ( + BaseBus, + Event, + Subscription, + _CLOSE_SENTINEL, + matches, +) +from decnet.bus.fake import _enqueue_drop_oldest as _enqueue_event_drop_oldest +from decnet.logging import get_logger + +log = get_logger("bus.client") + +_INBOUND_QUEUE_SIZE = 1024 + + +class _UnixSubscription(Subscription): + def __init__( + self, + bus: "UnixSocketBus", + pattern: str, + queue: "asyncio.Queue[Any]", + ) -> None: + super().__init__(pattern) + self._bus = bus + self._queue = queue + + async def __anext__(self) -> Event: + if self._closed: + raise StopAsyncIteration + item = await self._queue.get() + if item is _CLOSE_SENTINEL: + raise StopAsyncIteration + return item + + async def _aclose(self) -> None: + await self._bus._unregister(self) + try: + self._queue.put_nowait(_CLOSE_SENTINEL) + except asyncio.QueueFull: + pass + + +class UnixSocketBus(BaseBus): + """Client handle for a local :class:`BusServer`. + + One instance per process typically; multiple instances simply open + multiple sockets to the same server. Connection is lazy — the first + :meth:`connect` (or any publish/subscribe call via ``async with``) + opens the socket. + """ + + def __init__( + self, + socket_path: pathlib.Path | str, + *, + client_name: str | None = None, + ) -> None: + self._path = pathlib.Path(socket_path) + self._client_name = client_name or f"decnet-bus-client[{os.getpid()}]" + self._reader: asyncio.StreamReader | None = None + self._writer: asyncio.StreamWriter | None = None + self._reader_task: asyncio.Task[None] | None = None + self._subs: list[_UnixSubscription] = [] + self._lock = asyncio.Lock() + self._write_lock = asyncio.Lock() + self._closed = False + # Sticky flag: the first publish-on-closed-bus call logs at + # WARNING so operators see that a publish was dropped; subsequent + # calls on the same instance log at DEBUG only to prevent a + # log flood when stream threads drain after close. The bus is + # critical infra, so the first warning is non-negotiable. + self._closed_publish_warned = False + + # ─── Lifecycle ────────────────────────────────────────────────────────── + + async def connect(self) -> None: + if self._writer is not None: + return + if self._closed: + raise RuntimeError("connect on closed bus") + self._reader, self._writer = await asyncio.open_unix_connection(str(self._path)) + await self._send(protocol.encode(protocol.HELLO, args=self._client_name)) + self._reader_task = asyncio.create_task(self._reader_loop()) + log.debug("bus.client: connected to %s as %s", self._path, self._client_name) + + async def close(self) -> None: + if self._closed: + return + self._closed = True + + # Best-effort BYE — we don't care if it fails. + if self._writer is not None and not self._writer.is_closing(): + with contextlib.suppress(Exception): + await self._send(protocol.encode(protocol.BYE)) + + if self._reader_task is not None: + self._reader_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._reader_task + self._reader_task = None + + if self._writer is not None: + with contextlib.suppress(Exception): + self._writer.close() + await self._writer.wait_closed() + self._writer = None + self._reader = None + + # Wake every subscription so `async for` exits. + for sub in list(self._subs): + with contextlib.suppress(asyncio.QueueFull): + sub._queue.put_nowait(_CLOSE_SENTINEL) + self._subs.clear() + + # ─── Pub/Sub ──────────────────────────────────────────────────────────── + + async def publish( + self, + topic: str, + payload: dict[str, Any], + *, + event_type: str = "", + ) -> None: + if self._closed: + # Degrade gracefully: the DB is the source of truth, the bus + # is only the notification layer. Raising here made every + # caller via publish_safely flood the logs once per stream + # line during shutdown races. First drop warns loudly; + # subsequent drops on the same instance are DEBUG-only. + if not self._closed_publish_warned: + self._closed_publish_warned = True + log.warning( + "bus.client: publish on closed bus dropped topic=%s " + "(further drops on this instance logged at DEBUG)", + topic, + ) + else: + log.debug("bus.client: publish on closed bus dropped topic=%s", topic) + return + if self._writer is None: + await self.connect() + body = Event(topic=topic, payload=payload, type=event_type).to_dict() + try: + await self._send(protocol.encode(protocol.PUB, args=topic, body=body)) + except (ConnectionError, BrokenPipeError) as exc: + # Bus loss is a logged warning, never a publisher crash. The + # DB-as-source-of-truth invariant means the work is already + # persisted; the missing event is just a missed notification. + log.warning("bus.client: publish failed: %s", exc) + + def subscribe(self, pattern: str) -> Subscription: + if self._closed: + raise RuntimeError("subscribe on closed bus") + queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=_INBOUND_QUEUE_SIZE) + sub = _UnixSubscription(self, pattern, queue) + self._subs.append(sub) + # Schedule the SUB frame asynchronously so subscribe() stays sync, + # matching the BaseBus signature. The caller will shortly `async + # with` / `async for` the subscription, which will run the event + # loop and pick this task up. + asyncio.ensure_future(self._send_sub(pattern)) + return sub + + async def _send_sub(self, pattern: str) -> None: + try: + if self._writer is None: + await self.connect() + await self._send(protocol.encode(protocol.SUB, args=pattern)) + except Exception as exc: # pragma: no cover - network paths in live tests + log.warning("bus.client: SUB %s failed: %s", pattern, exc) + + async def _unregister(self, sub: _UnixSubscription) -> None: + try: + self._subs.remove(sub) + except ValueError: + return + # Tell the server we no longer want events for this pattern if no + # other local subscription still wants it. + if not any(s.pattern == sub.pattern for s in self._subs): + with contextlib.suppress(Exception): + await self._send(protocol.encode(protocol.UNSUB, args=sub.pattern)) + + # ─── Internal I/O ─────────────────────────────────────────────────────── + + async def _send(self, frame_bytes: bytes) -> None: + if self._writer is None: + raise ConnectionError("bus.client: not connected") + async with self._write_lock: + self._writer.write(frame_bytes) + await self._writer.drain() + + async def _reader_loop(self) -> None: + if self._reader is None: + return + try: + while True: + frame = await protocol.read_frame(self._reader) + if frame is None: + break + if frame.verb != protocol.EVT: + # Clients only ever legitimately receive EVT (or BYE). + if frame.verb == protocol.BYE: + break + log.warning("bus.client: unexpected verb from server: %s", frame.verb) + continue + topic = frame.args + data = protocol.decode_body(frame.body) if frame.body else {} + event = Event.from_dict(topic, data) + self._dispatch(event) + except protocol.ProtocolError as exc: + log.warning("bus.client: protocol error: %s", exc) + except (asyncio.IncompleteReadError, ConnectionError): + pass + except asyncio.CancelledError: + raise + except Exception: # pragma: no cover + log.exception("bus.client: reader loop crashed") + finally: + # Server-side close — wake every subscription. + for sub in list(self._subs): + with contextlib.suppress(asyncio.QueueFull): + sub._queue.put_nowait(_CLOSE_SENTINEL) + + def _dispatch(self, event: Event) -> None: + for sub in self._subs: + if matches(sub.pattern, event.topic): + _enqueue_event_drop_oldest(sub._queue, event) diff --git a/decnet/bus/unix_server.py b/decnet/bus/unix_server.py new file mode 100644 index 00000000..502a8dcf --- /dev/null +++ b/decnet/bus/unix_server.py @@ -0,0 +1,309 @@ +"""UNIX-socket server for the DECNET bus. + +One :class:`BusServer` per host. Accepts local connections on a UNIX-domain +socket; each connection may: + +* publish events (``PUB`` frames) that the server fans out to all matching + subscribers on other connections, and +* subscribe to patterns (``SUB`` frames) and receive matching events as + ``EVT`` frames. + +Authorization is socket file permissions (0660, group=``decnet`` if that +POSIX group exists, else the server process's own group). Anything the +kernel lets ``connect()`` is trusted — there is no verb-level auth. This +matches the "local processes on the same host" threat model; cross-host +federation is out of scope (see DEBT-029). + +Backpressure is per-connection, drop-oldest: if a subscriber can't drain its +outbound queue fast enough, the server discards the oldest pending event +rather than blocking publishers. The bus is at-most-once by contract, so +drops are acceptable; stalled publishers are not. +""" +from __future__ import annotations + +import asyncio +import contextlib +import grp +import os +import pathlib +from dataclasses import dataclass, field +from typing import Any + +from decnet.bus import protocol +from decnet.bus.base import Event, matches +from decnet.logging import get_logger + +log = get_logger("bus.server") + +_SOCKET_MODE = 0o660 +_DEFAULT_GROUP = "decnet" +_OUTBOUND_QUEUE_SIZE = 1024 + + +@dataclass(eq=False) +class _Connection: + """Per-connection server state.""" + + writer: asyncio.StreamWriter + peer_name: str = "" + patterns: set[str] = field(default_factory=set) + outbound: asyncio.Queue[bytes] = field( + default_factory=lambda: asyncio.Queue(maxsize=_OUTBOUND_QUEUE_SIZE) + ) + closed: bool = False + + +class BusServer: + """Serve a UNIX-socket bus on *socket_path*. + + Lifecycle: construct → :meth:`start` → :meth:`serve_forever` (or rely + on :meth:`start` returning once bound) → :meth:`close` for teardown. + Safe to :meth:`close` multiple times. + """ + + def __init__( + self, + socket_path: pathlib.Path | str, + *, + group: str | None = _DEFAULT_GROUP, + mode: int = _SOCKET_MODE, + ) -> None: + self._path = pathlib.Path(socket_path) + self._group = group + self._mode = mode + self._server: asyncio.base_events.Server | None = None + self._connections: set[_Connection] = set() + self._closed = False + + # ─── Lifecycle ────────────────────────────────────────────────────────── + + async def start(self) -> None: + """Bind the socket and begin accepting connections. + + Removes any stale socket file at *socket_path* first (common case: + the previous worker crashed without cleaning up). The parent + directory must already exist; we do NOT create it blindly because + the chosen directory (typically ``/run/decnet``) may require + systemd ``RuntimeDirectory=`` to set up. + """ + if self._server is not None: + return + + parent = self._path.parent + if not parent.exists(): + raise FileNotFoundError( + f"bus socket parent directory {parent} does not exist; " + f"create it with systemd RuntimeDirectory= or mkdir" + ) + + # Clean up a stale socket from a previous crash. If a live server + # is actually listening there, ``bind()`` below will fail — we do + # not try to detect live vs. stale ourselves. + with contextlib.suppress(FileNotFoundError): + if self._path.is_socket(): + self._path.unlink() + + self._server = await asyncio.start_unix_server( + self._handle_connection, path=str(self._path), + ) + _chmod_and_chown(self._path, self._mode, self._group) + log.info("bus.server: listening on %s (mode=%o group=%s)", + self._path, self._mode, self._group or "") + + async def serve_forever(self) -> None: + if self._server is None: + raise RuntimeError("BusServer not started") + async with self._server: + await self._server.serve_forever() + + async def close(self) -> None: + if self._closed: + return + self._closed = True + + if self._server is not None: + self._server.close() + with contextlib.suppress(Exception): + await self._server.wait_closed() + self._server = None + + # Drain every live connection. + for conn in list(self._connections): + await self._close_connection(conn) + self._connections.clear() + + with contextlib.suppress(FileNotFoundError): + self._path.unlink() + log.info("bus.server: closed") + + # ─── Internal publish fan-out ─────────────────────────────────────────── + + async def publish(self, topic: str, payload: dict[str, Any], event_type: str = "") -> None: + """Server-side publish helper — used by the worker to emit + ``system.bus.health`` heartbeats without opening a client loop.""" + event = Event(topic=topic, payload=payload, type=event_type) + self._fanout(event) + + # ─── Connection handler ───────────────────────────────────────────────── + + async def _handle_connection( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + ) -> None: + conn = _Connection(writer=writer) + self._connections.add(conn) + writer_task = asyncio.create_task(self._writer_loop(conn)) + try: + await self._reader_loop(conn, reader) + except protocol.ProtocolError as exc: + log.warning("bus.server: protocol error from %s: %s", conn.peer_name, exc) + except (asyncio.IncompleteReadError, ConnectionError) as exc: + log.debug("bus.server: %s disconnected: %s", conn.peer_name, exc) + except Exception: # pragma: no cover - defensive + log.exception("bus.server: unhandled error in connection") + finally: + await self._close_connection(conn) + self._connections.discard(conn) + writer_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await writer_task + + async def _reader_loop( + self, conn: _Connection, reader: asyncio.StreamReader, + ) -> None: + while True: + frame = await protocol.read_frame(reader) + if frame is None: + return + await self._dispatch(conn, frame) + if frame.verb == protocol.BYE: + return + + async def _dispatch(self, conn: _Connection, frame: protocol.Frame) -> None: + if frame.verb == protocol.HELLO: + conn.peer_name = frame.args or conn.peer_name + log.debug("bus.server: HELLO from %s", conn.peer_name) + return + if frame.verb == protocol.SUB: + pattern = frame.args + if not pattern: + raise protocol.ProtocolError("SUB requires a pattern") + conn.patterns.add(pattern) + log.debug("bus.server: %s SUB %s", conn.peer_name, pattern) + return + if frame.verb == protocol.UNSUB: + conn.patterns.discard(frame.args) + return + if frame.verb == protocol.PUB: + topic = frame.args + if not topic: + raise protocol.ProtocolError("PUB requires a topic") + data = protocol.decode_body(frame.body) if frame.body else {} + event = Event( + topic=topic, + payload=data.get("payload", {}) or {}, + type=data.get("type", "") or "", + ) + self._fanout(event, origin=conn) + return + if frame.verb == protocol.BYE: + return + # EVT is server-to-client only; receiving one is a protocol violation. + raise protocol.ProtocolError(f"unexpected verb {frame.verb!r} from client") + + def _fanout(self, event: Event, *, origin: _Connection | None = None) -> None: + """Enqueue *event* as an EVT frame on every matching connection. + + We do NOT deliver back to the originating connection (a publisher + does not receive its own event). Encoding happens once per event, + not once per subscriber. + """ + try: + frame_bytes = protocol.encode( + protocol.EVT, args=event.topic, body=event.to_dict(), + ) + except protocol.ProtocolError: + log.exception("bus.server: failed to encode EVT for topic=%s", event.topic) + return + + for conn in self._connections: + if conn is origin or conn.closed: + continue + if not any(matches(p, event.topic) for p in conn.patterns): + continue + _enqueue_drop_oldest(conn.outbound, frame_bytes, event.topic) + + async def _writer_loop(self, conn: _Connection) -> None: + """Serialize writes onto *conn*'s socket. + + One writer task per connection so a slow peer only blocks its own + queue, not the fan-out loop. The queue is bounded with drop-oldest + policy applied at enqueue time (see :func:`_enqueue_drop_oldest`). + """ + try: + while not conn.closed: + data = await conn.outbound.get() + conn.writer.write(data) + await conn.writer.drain() + except (ConnectionError, BrokenPipeError): + log.debug("bus.server: %s writer: peer closed", conn.peer_name) + except asyncio.CancelledError: + pass + except Exception: # pragma: no cover - defensive + log.exception("bus.server: writer loop crashed for %s", conn.peer_name) + + async def _close_connection(self, conn: _Connection) -> None: + if conn.closed: + return + conn.closed = True + with contextlib.suppress(Exception): + conn.writer.close() + await conn.writer.wait_closed() + + +# ─── Helpers ───────────────────────────────────────────────────────────────── + +def _chmod_and_chown(path: pathlib.Path, mode: int, group: str | None) -> None: + """Apply socket file perms and best-effort group ownership. + + If *group* is ``None`` or the named group does not exist, we leave the + socket owned by the current process group. This keeps the server + usable on dev boxes that don't have a ``decnet`` group set up. + """ + try: + os.chmod(path, mode) + except OSError as exc: + log.warning("bus.server: chmod(%s, %o) failed: %s", path, mode, exc) + + if not group: + return + try: + gid = grp.getgrnam(group).gr_gid + except KeyError: + log.debug("bus.server: group %r not found, leaving socket group unchanged", group) + return + try: + os.chown(path, -1, gid) + except PermissionError: + # Dev box running as an unprivileged user can't chown. Log once at + # debug and move on — the socket is still usable by the owner. + log.debug("bus.server: chown(%s, gid=%d) denied; leaving as-is", path, gid) + except OSError as exc: + log.warning("bus.server: chown(%s, gid=%d) failed: %s", path, gid, exc) + + +def _enqueue_drop_oldest( + queue: "asyncio.Queue[bytes]", data: bytes, topic: str, +) -> None: + """Drop-oldest backpressure — mirrors :func:`decnet.bus.fake._enqueue_drop_oldest`.""" + while True: + try: + queue.put_nowait(data) + return + except asyncio.QueueFull: + try: + queue.get_nowait() + log.warning("bus.server: subscriber queue full, dropped event topic=%s", topic) + except asyncio.QueueEmpty: + return diff --git a/decnet/bus/worker.py b/decnet/bus/worker.py new file mode 100644 index 00000000..bbefaf65 --- /dev/null +++ b/decnet/bus/worker.py @@ -0,0 +1,121 @@ +"""``decnet bus`` worker entrypoint. + +Starts a :class:`~decnet.bus.unix_server.BusServer` on the configured UNIX +socket and serves forever, emitting a ``system.bus.health`` heartbeat on +its own bus every :data:`HEARTBEAT_INTERVAL_SEC` seconds so liveness-aware +consumers (dashboards, watchdogs) can tell the bus is up without polling +the filesystem. + +Cross-host federation is **out of scope** for the MVP; each host runs its +own bus independently. See DEBT-029 for the deferred ``--bridge-tcp`` +mode that would proxy the socket over the swarm mTLS channel. +""" +from __future__ import annotations + +import asyncio +import os +import pathlib +import signal +import time + +from decnet.bus import topics +from decnet.bus.unix_server import BusServer +from decnet.logging import get_logger + +log = get_logger("bus.worker") + +HEARTBEAT_INTERVAL_SEC = 10 + + +async def bus_worker( + socket_path: str | pathlib.Path, + *, + group: str | None = "decnet", + heartbeat_interval: int = HEARTBEAT_INTERVAL_SEC, +) -> None: + """Run the bus server until cancelled or SIGTERM/SIGINT is received. + + The parent directory of *socket_path* must already exist (systemd's + ``RuntimeDirectory=decnet`` handles this in prod; dev code is expected + to ``mkdir`` first). This function does not create it implicitly + because the right choice of perms/owner depends on the deployment + context. + """ + path = pathlib.Path(socket_path) + _ensure_parent(path) + + server = BusServer(path, group=group) + await server.start() + log.info("bus.worker: pid=%d socket=%s", os.getpid(), path) + + stop_event = asyncio.Event() + _install_signal_handlers(stop_event) + + heartbeat_task = asyncio.create_task(_heartbeat_loop(server, heartbeat_interval)) + serve_task = asyncio.create_task(server.serve_forever()) + + try: + await stop_event.wait() + log.info("bus.worker: shutdown signal received") + finally: + heartbeat_task.cancel() + serve_task.cancel() + for task in (heartbeat_task, serve_task): + try: + await task + except (asyncio.CancelledError, Exception): # noqa: BLE001 - draining shutdown + pass + await server.close() + log.info("bus.worker: stopped") + + +async def _heartbeat_loop(server: BusServer, interval: int) -> None: + """Publish ``system.bus.health`` on the server's own fan-out.""" + started_at = time.time() + while True: + try: + await server.publish( + topics.system(topics.SYSTEM_BUS_HEALTH), + { + "pid": os.getpid(), + "uptime_sec": round(time.time() - started_at, 3), + "ts": time.time(), + }, + event_type=topics.SYSTEM_BUS_HEALTH, + ) + except Exception: # pragma: no cover - heartbeat must never kill the worker + log.exception("bus.worker: heartbeat publish failed") + await asyncio.sleep(interval) + + +def _install_signal_handlers(stop_event: asyncio.Event) -> None: + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, stop_event.set) + except (NotImplementedError, RuntimeError): + # add_signal_handler is not supported on Windows / in some + # test harnesses where the loop is running in a non-main thread. + # The worker still exits via KeyboardInterrupt bubbling up. + pass + + +def _ensure_parent(path: pathlib.Path) -> None: + parent = path.parent + if parent.exists(): + return + # Dev-box convenience: if the parent is the user's ``~/.decnet`` dir, + # create it. We do not auto-mkdir ``/run/decnet`` — that's systemd's job + # and silently creating it as the wrong user would cause permission + # confusion later. + home_prefix = pathlib.Path.home() / ".decnet" + try: + parent.relative_to(home_prefix.parent) + except ValueError: + raise FileNotFoundError( + f"bus socket parent {parent} does not exist; create it first" + ) + parent.mkdir(parents=True, exist_ok=True) + + +__all__ = ["bus_worker", "HEARTBEAT_INTERVAL_SEC"] diff --git a/decnet/canary/__init__.py b/decnet/canary/__init__.py new file mode 100644 index 00000000..8a250514 --- /dev/null +++ b/decnet/canary/__init__.py @@ -0,0 +1,37 @@ +"""Canary tokens — decoy artifacts planted in decky filesystems. + +Public surface is exported here so callers can ``from decnet.canary +import CanaryArtifact, get_generator, get_instrumenter`` without +knowing the submodule layout. Concrete generators / instrumenters +live under :mod:`decnet.canary.generators` and +:mod:`decnet.canary.instrumenters` respectively; the factory keeps +import-time cost down by deferring those imports until first use +(same pattern as :mod:`decnet.intel.factory`). +""" +from __future__ import annotations + +from decnet.canary.base import ( + CanaryArtifact, + CanaryContext, + CanaryGenerator, + CanaryInstrumenter, +) +from decnet.canary.factory import ( + KNOWN_GENERATORS, + KNOWN_INSTRUMENTERS, + get_generator, + get_instrumenter, + pick_instrumenter_for_mime, +) + +__all__ = [ + "CanaryArtifact", + "CanaryContext", + "CanaryGenerator", + "CanaryInstrumenter", + "KNOWN_GENERATORS", + "KNOWN_INSTRUMENTERS", + "get_generator", + "get_instrumenter", + "pick_instrumenter_for_mime", +] diff --git a/decnet/canary/base.py b/decnet/canary/base.py new file mode 100644 index 00000000..160dcd19 --- /dev/null +++ b/decnet/canary/base.py @@ -0,0 +1,145 @@ +"""Canary generator / instrumenter ABCs and the artifact dataclass. + +Two flavors of producer share the same return shape: + +* :class:`CanaryGenerator` synthesises a fake artifact from scratch + (e.g. a plausible ``~/.aws/credentials`` block, a ``.git/config`` + pointing at an attacker-bait remote URL). Operators don't supply + any input. + +* :class:`CanaryInstrumenter` mutates an operator-uploaded blob to + embed the callback (HTTP slug + DNS host). The original blob bytes + are passed in; the instrumenter returns the mutated version. + +Both return a :class:`CanaryArtifact` — the planter doesn't care +which path produced it. Same dataclass keeps the planter's +docker-exec injector trivial. + +ABCs intentionally do not include I/O — generators and instrumenters +are pure functions of (slug, host, blob?). All filesystem work +happens in :mod:`decnet.canary.planter` and :mod:`decnet.canary.storage`. +""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class CanaryContext: + """Inputs every generator/instrumenter needs to embed a working callback. + + ``callback_token`` is the unique slug; it appears verbatim in HTTP + URLs (``https:///c/``) and as the leftmost + DNS label (``.canary.``) so a single + slug resolves to a single :class:`CanaryToken` row regardless of + which path the attacker tripped. + + ``http_base`` and ``dns_zone`` come from the canary worker's + public-facing config (``DECNET_CANARY_HTTP_BASE``, + ``DECNET_CANARY_DNS_ZONE``). When DNS isn't deployed, + ``dns_zone`` is empty and instrumenters that only have a DNS + surface (e.g. an artifact whose only realistic embed point is a + hostname) raise. + """ + + callback_token: str + http_base: str # e.g. "https://canary.example.test" — no trailing slash + dns_zone: str = "" # e.g. "canary.example.test"; "" disables DNS embeds + persona: str = "linux" # "linux" | "windows" — drives default username, path style + + +@dataclass +class CanaryArtifact: + """Bytes-and-placement bundle produced by a generator/instrumenter.""" + + path: str + """Absolute path inside the target container.""" + + content: bytes + """Final bytes that hit the decky filesystem. + + Always raw bytes — the planter base64-encodes for the wire so + binary blobs (DOCX/PNG/PDF) survive ``docker exec sh -c`` safely. + """ + + mode: int = 0o600 + """Unix file mode. Defaults to ``0600`` because most realistic + canary placements (``~/.aws/credentials``, ``.env``, ``id_rsa``) + are operator-only. Honeydocs in user docs folders should pass + ``0o644``. + """ + + mtime_offset: int = 0 + """Seconds relative to *now* for the planted file's mtime. + + Negative values backdate the file so it doesn't look like it + appeared the moment the decky was deployed. ``-86400 * 90`` (90 + days ago) is a common choice for ``honeydoc`` artifacts; ``0`` + means "stamp it now," which is fine for ``aws_creds``-like files + that would plausibly be touched recently. + """ + + instrumenter: Optional[str] = None + """Identifier of the instrumenter that produced this artifact (for + upload-driven tokens). Mirrored into ``CanaryToken.instrumenter``. + Mutually exclusive with :attr:`generator`. + """ + + generator: Optional[str] = None + """Identifier of the generator that produced this artifact (for + synthesised tokens). Mirrored into ``CanaryToken.generator``. + Mutually exclusive with :attr:`instrumenter`. + """ + + notes: list[str] = field(default_factory=list) + """Human-readable notes about the embedding (e.g. "DOCX: injected + 1×1 remote image at relsId rId99"). Surfaced in the API + ``preview`` response so the operator sees what we did before + planting. Never leaked to the attacker-facing surface. + """ + + +class CanaryGenerator(ABC): + """Produces a fake artifact from scratch.""" + + name: str #: short tag — matches ``CanaryToken.generator`` + + @abstractmethod + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + """Synthesise the artifact. + + MUST NOT do I/O. MUST be deterministic for the same + ``(callback_token, http_base, dns_zone, persona)`` so re-seeding + from :attr:`CanaryToken.secret_seed` produces byte-identical + output and the planter is naturally idempotent. + """ + + +class CanaryInstrumenter(ABC): + """Mutates an operator-uploaded blob to embed a callback.""" + + name: str #: short tag — matches ``CanaryToken.instrumenter`` + + #: MIME prefixes this instrumenter handles. The factory uses these + #: to dispatch by sniffed content-type. Sub-string match against + #: the prefix list (e.g. ``("application/pdf",)`` or + #: ``("text/",)``). + mime_prefixes: tuple[str, ...] = () + + @abstractmethod + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + """Return the mutated bytes with the callback embedded. + + MUST raise :class:`InstrumenterRejectedError` when the blob + can't be safely mutated (corrupt zip, encrypted PDF, etc.) so + the API can surface a 400 with the specific reason rather than + silently shipping the original bytes. + """ + + +class InstrumenterRejectedError(ValueError): + """Raised when an instrumenter can't safely mutate the input.""" diff --git a/decnet/canary/cultivator.py b/decnet/canary/cultivator.py new file mode 100644 index 00000000..8f7a222f --- /dev/null +++ b/decnet/canary/cultivator.py @@ -0,0 +1,181 @@ +"""Realism contract adapter for canary generators. + +Stage 7 of the realism migration. The orchestrator's planner picks a +``canary_*`` :class:`~decnet.realism.taxonomy.ContentClass` 1–3% of +the time on file ticks; this module turns that pick into a +:class:`~decnet.canary.base.CanaryArtifact` (bytes the SSH driver +plants) plus a persisted :class:`~decnet.web.db.models.CanaryToken` +row so the canary worker recognises the slug when an attacker trips +it. + +What this is NOT: it doesn't pick *when* canaries fire — that's the +realism planner's job. It doesn't decide *where* on the filesystem +the canary lands beyond what realism naming + persona conventions +already produce. It's a thin bytes-and-row factory bolted onto the +realism contract. + +Stealth (per ``feedback_stealth.md``): we never leak the +``DECNET`` literal into anything that survives to the planted file. +The underlying generators are already stealth-clean; this wrapper +must not undo that. +""" +from __future__ import annotations + +import os +import secrets as _secrets +from datetime import datetime, timezone +from typing import Any, Optional + +from decnet.canary.base import CanaryArtifact, CanaryContext +from decnet.canary.factory import get_generator +from decnet.logging import get_logger +from decnet.realism.personas import login_for +from decnet.realism.taxonomy import ContentClass, Plan + +log = get_logger("canary.cultivator") + + +# realism content_class → canary generator name. Mirrors +# :data:`decnet.canary.factory.KNOWN_GENERATORS`. +_CLASS_TO_GENERATOR: dict[ContentClass, str] = { + ContentClass.CANARY_AWS_CREDS: "aws_creds", + ContentClass.CANARY_ENV_FILE: "env_file", + ContentClass.CANARY_GIT_CONFIG: "git_config", + ContentClass.CANARY_SSH_KEY: "ssh_key", + ContentClass.CANARY_HONEYDOC: "honeydoc", + ContentClass.CANARY_HONEYDOC_DOCX: "honeydoc_docx", + ContentClass.CANARY_HONEYDOC_PDF: "honeydoc_pdf", + ContentClass.CANARY_MYSQL_DUMP: "mysql_dump", +} + + +# Generator → CanaryKind. The trip surface (HTTP slug callback / DNS +# resolution / passive bait) determines how the canary worker matches +# an attacker callback to this token. Aligned with +# :data:`decnet.web.db.models.canary.CanaryKind`. +_GENERATOR_TO_KIND: dict[str, str] = { + "aws_creds": "aws_passive", # no embedded callback; passive bait + "env_file": "http", + "git_config": "http", + "honeydoc": "http", + "honeydoc_docx": "http", + "honeydoc_pdf": "http", + "ssh_key": "dns", # trip is DNS resolution of host comment + "mysql_dump": "dns", # trip is DNS resolution of subdomain +} + + +# Path conventions per generator. The realism planner doesn't know +# about decoy-realistic credential locations (``~/.aws/credentials``, +# ``~/.git/config``); we map them per-class here so the planted +# artifact lands somewhere an attacker would actually look. +_DEFAULT_PATH: dict[ContentClass, str] = { + ContentClass.CANARY_AWS_CREDS: "/home/{persona}/.aws/credentials", + ContentClass.CANARY_ENV_FILE: "/home/{persona}/app/.env", + ContentClass.CANARY_GIT_CONFIG: "/home/{persona}/.git/config", + ContentClass.CANARY_SSH_KEY: "/home/{persona}/.ssh/id_rsa", + ContentClass.CANARY_HONEYDOC: "/home/{persona}/Documents/notes.html", + ContentClass.CANARY_HONEYDOC_DOCX: "/home/{persona}/Documents/Q3-Operations-Review.docx", + ContentClass.CANARY_HONEYDOC_PDF: "/home/{persona}/Documents/Q3-Operations-Review.pdf", + ContentClass.CANARY_MYSQL_DUMP: "/var/backups/db_backup.sql", +} + + +def _path_for(plan: Plan) -> str: + """Produce the canary placement path for *plan*. + + The realism planner already filled in ``plan.target_path`` from + the namer, but canary placements have stronger conventions + (``~/.aws/credentials``, ``~/.ssh/id_rsa``) than the realism + namer's vocabulary. When :data:`_DEFAULT_PATH` has an entry, + that wins. + """ + template = _DEFAULT_PATH.get(plan.content_class) + if template is None: + return plan.target_path + return template.format(persona=login_for(plan.persona)) + + +def _new_callback_token() -> str: + """16 url-safe bytes — same shape canary slug fields use elsewhere.""" + return _secrets.token_urlsafe(16) + + +async def cultivate( + plan: Plan, + repo: Any, + *, + http_base: Optional[str] = None, + dns_zone: Optional[str] = None, + created_by: str = "system", +) -> CanaryArtifact: + """Realism-driven canary plant. + + Build a :class:`CanaryContext`, ask the right generator for bytes, + persist a ``canary_tokens`` row so the canary worker can attribute + callbacks to this token, and return the artifact for the SSH + driver to plant. + + *http_base* and *dns_zone* default to ``DECNET_CANARY_HTTP_BASE`` + and ``DECNET_CANARY_DNS_ZONE`` env vars respectively — same + pattern the canary worker uses. When both are empty, generators + that need a callback host (``ssh_key`` DNS, ``mysql_dump``) + raise; the planner's caller logs and falls back to a non-canary + plan. + """ + if not plan.content_class.is_canary(): + raise ValueError( + f"cultivate() called with non-canary content_class=" + f"{plan.content_class!r}" + ) + gen_name = _CLASS_TO_GENERATOR.get(plan.content_class) + if gen_name is None: + raise KeyError( + f"no canary generator mapped for content_class=" + f"{plan.content_class!r}" + ) + + callback_token = _new_callback_token() + ctx = CanaryContext( + callback_token=callback_token, + http_base=http_base or os.environ.get("DECNET_CANARY_HTTP_BASE", ""), + dns_zone=dns_zone or os.environ.get("DECNET_CANARY_DNS_ZONE", ""), + persona="linux", # all our deckies are POSIX in MVP + ) + generator = get_generator(gen_name) + artifact = generator.generate(ctx) + + # The generator returns ``path=""`` (planter fills it normally). + # We have a realism-derived path on hand; stuff it in for the SSH + # driver's plant_file call AND the canary_tokens row. + placement_path = _path_for(plan) + + # Persist the token row before planting so the canary worker can + # attribute a callback if the artifact trips during the plant + # itself (improbable but possible — DOCX viewers can preview + # autoplay-style). + await repo.create_canary_token({ + "kind": _GENERATOR_TO_KIND.get(gen_name, "http"), + "decky_name": plan.decky_name, + "instrumenter": None, + "generator": gen_name, + "placement_path": placement_path, + "callback_token": callback_token, + "secret_seed": callback_token, # deterministic re-seed compatible + "placed_at": datetime.now(timezone.utc), + "created_by": created_by, + "state": "planted", + }) + + # Carry the placement_path on the artifact so the orchestrator's + # plant_file call uses it. We don't mutate the generator's + # original — copy with the new path. + return CanaryArtifact( + path=placement_path, + content=artifact.content, + mode=artifact.mode, + mtime_offset=artifact.mtime_offset, + instrumenter=artifact.instrumenter, + generator=artifact.generator, + notes=list(artifact.notes), + ) diff --git a/decnet/canary/dns_server.py b/decnet/canary/dns_server.py new file mode 100644 index 00000000..65cc6f60 --- /dev/null +++ b/decnet/canary/dns_server.py @@ -0,0 +1,207 @@ +"""Minimal authoritative DNS server for canary tokens (stdlib only). + +We don't need a full resolver — only enough to: + +1. Decode an inbound query's qname. +2. If the qname matches ``.``, log the callback, + publish ``canary..triggered`` on the bus, and return a + plausible A record (any RFC-5737 reserved address would do; we + use 192.0.2.1) so the attacker's resolver doesn't loop on + NXDOMAIN. +3. For unknown qnames return NXDOMAIN. + +DNS-over-UDP wire format is well-trodden: 12-byte header + name +labels + qtype + qclass. We implement just the bits we need. + +This module deliberately avoids the ``dnslib`` PyPI package so the +canary worker has no extra dependency surface. If we ever need +EDNS0, DNSSEC, or other niceties we'll swap to dnslib then. +""" +from __future__ import annotations + +import asyncio +import struct +from dataclasses import dataclass +from typing import Awaitable, Callable, Optional, Tuple + + +@dataclass(frozen=True) +class DNSQuery: + """Decoded query — only the bits the canary worker cares about.""" + + txid: int + qname: str # lowercase, no trailing dot + qtype: int + qclass: int + flags: int + + +def _decode_name(buf: bytes, offset: int) -> Tuple[str, int]: + """Return ``(qname_lowercase_no_dot, bytes_consumed)``. + + Supports compressed pointers (RFC 1035 §4.1.4). Doesn't recurse — + we walk the pointer chain iteratively with a hop cap to avoid + pointer-loop DoS. + """ + labels: list[str] = [] + pos = offset + consumed = 0 + jumped = False + hops = 0 + while True: + if pos >= len(buf): + raise ValueError("truncated DNS name") + length = buf[pos] + if length == 0: + pos += 1 + if not jumped: + consumed = pos - offset + break + if (length & 0xC0) == 0xC0: + # Compression pointer. + if pos + 1 >= len(buf): + raise ValueError("truncated DNS pointer") + ptr = ((length & 0x3F) << 8) | buf[pos + 1] + if not jumped: + consumed = (pos + 2) - offset + pos = ptr + jumped = True + hops += 1 + if hops > 10: + raise ValueError("DNS pointer loop") + continue + pos += 1 + if pos + length > len(buf): + raise ValueError("truncated DNS label") + labels.append(buf[pos:pos + length].decode("ascii", "replace")) + pos += length + return ".".join(labels).lower(), consumed + + +def parse_query(packet: bytes) -> DNSQuery: + """Parse the (single) question of a DNS query packet.""" + if len(packet) < 12: + raise ValueError("DNS packet too short") + txid, flags, qdcount, _ancount, _nscount, _arcount = struct.unpack( + "!HHHHHH", packet[:12] + ) + if qdcount != 1: + raise ValueError(f"expected 1 question, got {qdcount}") + qname, consumed = _decode_name(packet, 12) + pos = 12 + consumed + if pos + 4 > len(packet): + raise ValueError("truncated DNS qtype/qclass") + qtype, qclass = struct.unpack("!HH", packet[pos:pos + 4]) + return DNSQuery( + txid=txid, qname=qname, qtype=qtype, qclass=qclass, flags=flags, + ) + + +def _encode_name(name: str) -> bytes: + out = bytearray() + for label in name.split("."): + if not label: + continue + b = label.encode("ascii", "replace") + out.append(len(b)) + out.extend(b) + out.append(0) + return bytes(out) + + +def _build_response( + query: DNSQuery, + *, + rcode: int = 0, + answer_ip: Optional[str] = None, +) -> bytes: + """Encode a DNS response packet. + + *rcode* 0 = NOERROR, 3 = NXDOMAIN. When *answer_ip* is supplied + and the query was for an A record we include exactly one answer + (TTL 60, class IN). + """ + qd_count = 1 + an_count = 1 if (answer_ip and query.qtype == 1 and rcode == 0) else 0 + flags = 0x8400 | rcode # response + authoritative + RA bit clear + rcode + header = struct.pack( + "!HHHHHH", query.txid, flags, qd_count, an_count, 0, 0, + ) + qname_bytes = _encode_name(query.qname) + question = qname_bytes + struct.pack("!HH", query.qtype, query.qclass) + + answer = b"" + if an_count: + # Use a name pointer back to the question (offset 12). + ptr = struct.pack("!H", 0xC000 | 12) + rdata = bytes(int(o) for o in answer_ip.split(".")) + answer = ptr + struct.pack("!HHIH", 1, 1, 60, 4) + rdata + + return header + question + answer + + +# Hook signature: receives the matched slug + the query; returns +# nothing. The worker uses it to persist a CanaryTrigger row and +# publish the bus event. +TriggerHook = Callable[[str, DNSQuery, str], Awaitable[None]] + + +class CanaryDNSProtocol(asyncio.DatagramProtocol): + """asyncio UDP server endpoint for canary DNS callbacks. + + Constructor takes the canary zone (``"canary.example.test"``) and + a coroutine called when a query matches ``.``. The + hook runs in the event loop's task; we don't block the receive + path on it. + """ + + def __init__( + self, + zone: str, + hook: TriggerHook, + *, + answer_ip: str = "192.0.2.1", + ) -> None: + # Normalise: lowercase, no leading/trailing dot. + self._zone = zone.lower().strip(".") + self._suffix = "." + self._zone if self._zone else "" + self._hook = hook + self._answer_ip = answer_ip + self._transport: Optional[asyncio.DatagramTransport] = None + + def connection_made(self, transport) -> None: # type: ignore[override] + self._transport = transport # type: ignore[assignment] + + def datagram_received( # type: ignore[override] + self, data: bytes, addr: Tuple[str, int], + ) -> None: + try: + query = parse_query(data) + except ValueError: + # Malformed query — drop silently. Returning a FORMERR + # would tip off the attacker that *something* is listening + # on this port; the stealth posture (feedback_stealth) + # prefers radio silence on parse errors. + return + slug = self._slug_for(query.qname) + if slug is None: + # Unknown name — NXDOMAIN. + self._send(addr, _build_response(query, rcode=3)) + return + # Known name — answer with our sinkhole IP, then fire the hook. + self._send(addr, _build_response(query, answer_ip=self._answer_ip)) + asyncio.create_task(self._hook(slug, query, addr[0])) + + def _slug_for(self, qname: str) -> Optional[str]: + if not self._zone or not qname.endswith(self._suffix): + return None + slug = qname[: -len(self._suffix)] + # Single-label slug only; multi-label means the attacker is + # querying a sub-resource we don't model. + if not slug or "." in slug: + return None + return slug + + def _send(self, addr: Tuple[str, int], packet: bytes) -> None: + if self._transport is not None: + self._transport.sendto(packet, addr) diff --git a/decnet/canary/factory.py b/decnet/canary/factory.py new file mode 100644 index 00000000..345443f1 --- /dev/null +++ b/decnet/canary/factory.py @@ -0,0 +1,141 @@ +"""Generator and instrumenter factories. + +Same lazy-import pattern as :mod:`decnet.intel.factory` — concrete +implementations stay un-imported until first use so importing +:mod:`decnet.canary` from a CLI subcommand doesn't drag in +``pikepdf`` / ``python-docx`` / ``Pillow`` for callers that only +need the model layer. +""" +from __future__ import annotations + +from typing import Tuple + +from decnet.canary.base import CanaryGenerator, CanaryInstrumenter + +KNOWN_GENERATORS: Tuple[str, ...] = ( + "git_config", + "env_file", + "ssh_key", + "aws_creds", + "honeydoc", + "honeydoc_docx", + "honeydoc_pdf", + "mysql_dump", +) + +KNOWN_INSTRUMENTERS: Tuple[str, ...] = ( + "docx", + "xlsx", + "pdf", + "html", + "image", + "plain", + "passthrough", +) + + +def get_generator(name: str) -> CanaryGenerator: + """Return the generator registered under ``name``. + + Raises :class:`ValueError` for unknown names so a typo in the API + request surfaces as a 400 rather than silently producing nothing. + """ + if name == "git_config": + from decnet.canary.generators.git_config import GitConfigGenerator + return GitConfigGenerator() + if name == "env_file": + from decnet.canary.generators.env_file import EnvFileGenerator + return EnvFileGenerator() + if name == "ssh_key": + from decnet.canary.generators.ssh_key import SSHKeyGenerator + return SSHKeyGenerator() + if name == "aws_creds": + from decnet.canary.generators.aws_creds import AWSCredsGenerator + return AWSCredsGenerator() + if name == "honeydoc": + from decnet.canary.generators.honeydoc import HoneydocGenerator + return HoneydocGenerator() + if name == "honeydoc_docx": + from decnet.canary.generators.honeydoc_docx import HoneydocDocxGenerator + return HoneydocDocxGenerator() + if name == "honeydoc_pdf": + from decnet.canary.generators.honeydoc_pdf import HoneydocPdfGenerator + return HoneydocPdfGenerator() + if name == "mysql_dump": + from decnet.canary.generators.mysql_dump import MySQLDumpGenerator + return MySQLDumpGenerator() + raise ValueError( + f"Unknown canary generator: {name!r}. Known: {KNOWN_GENERATORS}" + ) + + +def get_instrumenter(name: str) -> CanaryInstrumenter: + """Return the instrumenter registered under ``name``.""" + if name == "docx": + from decnet.canary.instrumenters.docx import DocxInstrumenter + return DocxInstrumenter() + if name == "xlsx": + from decnet.canary.instrumenters.xlsx import XlsxInstrumenter + return XlsxInstrumenter() + if name == "pdf": + from decnet.canary.instrumenters.pdf import PdfInstrumenter + return PdfInstrumenter() + if name == "html": + from decnet.canary.instrumenters.html import HtmlInstrumenter + return HtmlInstrumenter() + if name == "image": + from decnet.canary.instrumenters.image import ImageInstrumenter + return ImageInstrumenter() + if name == "plain": + from decnet.canary.instrumenters.plain import PlainInstrumenter + return PlainInstrumenter() + if name == "passthrough": + from decnet.canary.instrumenters.passthrough import PassthroughInstrumenter + return PassthroughInstrumenter() + raise ValueError( + f"Unknown canary instrumenter: {name!r}. Known: {KNOWN_INSTRUMENTERS}" + ) + + +# MIME → instrumenter dispatch. Order matters: we walk the table +# top-to-bottom and the first prefix match wins, so put the more +# specific (DOCX/XLSX) before the generic (zip/octet-stream). +_MIME_DISPATCH: tuple[tuple[str, str], ...] = ( + # Office Open XML — DOCX/XLSX share a zip structure but expose + # different inner trees, so dispatch by MIME alias rather than + # zip-poking. + ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", "docx"), + ("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx"), + ("application/pdf", "pdf"), + ("text/html", "html"), + ("application/xhtml+xml", "html"), + ("image/png", "image"), + ("image/jpeg", "image"), + ("image/gif", "image"), + # Plaintext catch-alls — config files, .env, .ini, .yaml, .json, + # source code. All handled by the same regex-substitution pass. + ("text/", "plain"), + ("application/json", "plain"), + ("application/x-yaml", "plain"), + ("application/yaml", "plain"), + ("application/toml", "plain"), +) + + +def pick_instrumenter_for_mime(content_type: str) -> str: + """Return the instrumenter name registered for a sniffed MIME. + + Falls back to ``"passthrough"`` for anything we don't have an + embedder for (binary blobs we can't mutate safely — random + container images, archives, executables). ``passthrough`` only + supports DNS-callback tokens (the slug ends up in the filename or + an accompanying README), so the API surfaces that constraint to + the operator before they pick a kind. + """ + if not content_type: + return "passthrough" + lowered = content_type.lower() + for prefix, name in _MIME_DISPATCH: + if lowered.startswith(prefix): + return name + return "passthrough" diff --git a/decnet/canary/generators/__init__.py b/decnet/canary/generators/__init__.py new file mode 100644 index 00000000..cb06c181 --- /dev/null +++ b/decnet/canary/generators/__init__.py @@ -0,0 +1,7 @@ +"""Built-in canary generators (synthesised fake artifacts). + +Concrete classes live in sibling modules and are imported lazily by +:func:`decnet.canary.factory.get_generator` to keep the import-time +cost of :mod:`decnet.canary` cheap for callers that only need the +ABCs. +""" diff --git a/decnet/canary/generators/aws_creds.py b/decnet/canary/generators/aws_creds.py new file mode 100644 index 00000000..f02c201d --- /dev/null +++ b/decnet/canary/generators/aws_creds.py @@ -0,0 +1,86 @@ +"""Fake ``~/.aws/credentials`` block (passive bait). + +This is the **passive** variant — no callback wiring. An attacker +who exfils these keys can't trip a detection unless we run a real +AWS account with a deny-all CloudTrail listener (post-v1). The +realism is the point: the file looks like a routinely used credentials +file, so the rest of the decky's persona feels lived-in. + +If the operator picks ``kind="aws_passive"`` we accept that no slug +will be embedded. If they pick ``kind="http"`` or ``kind="dns"`` for +this generator, the API will reject the combination with a 400 — AWS +keys have no plausible field where a URL or hostname survives a +``grep -E '[A-Z0-9]{20}'`` smell test. +""" +from __future__ import annotations + +import hashlib +from secrets import token_urlsafe + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryGenerator + + +# Stable AWS-style key body derived from the slug. Keeping the +# generator deterministic (per-slug) means re-seeding produces the +# same bytes — the planter is naturally idempotent and an operator +# who runs ``decnet canary verify`` can re-derive the expected file +# without touching the DB. + +def _fake_access_key(seed: str) -> str: + # AWS access keys are 20 chars, uppercase alphanum, AKIA prefix. + body = hashlib.sha256(seed.encode()).hexdigest().upper() + return "AKIA" + body[:16] + + +def _fake_secret_key(seed: str) -> str: + # AWS secret keys are 40 chars, mixed-case base64-ish. We use + # base64-safe characters from token_urlsafe seeded by a SHA-256 + # of the seed so the output is stable per slug. + h = hashlib.sha256(("secret:" + seed).encode()).digest() + # Reuse token_urlsafe for the alphabet but pad to 40 chars from + # the deterministic bytes so we don't depend on os.urandom. + import base64 + return base64.b64encode(h)[:40].decode() + + +class AWSCredsGenerator(CanaryGenerator): + name = "aws_creds" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + seed = ctx.callback_token + access = _fake_access_key(seed) + secret = _fake_secret_key(seed) + body = ( + "[default]\n" + f"aws_access_key_id = {access}\n" + f"aws_secret_access_key = {secret}\n" + "region = us-east-1\n" + "\n" + "[prod]\n" + f"aws_access_key_id = {_fake_access_key('prod-' + seed)}\n" + f"aws_secret_access_key = {_fake_secret_key('prod-' + seed)}\n" + "region = us-west-2\n" + ) + return CanaryArtifact( + path="", # caller (planter) fills this from CanaryToken.placement_path + content=body.encode("utf-8"), + mode=0o600, + mtime_offset=-86400 * 14, # 2 weeks ago — looks lived-in + generator=self.name, + notes=[ + "fake AWS keys; no callback embedded — passive bait only", + f"derived deterministically from slug={seed}", + ], + ) + + +# Re-exported so the slug helper is reusable from the +# instrumenters/passthrough module without an internal import path. +__all__ = ["AWSCredsGenerator", "_fake_access_key", "_fake_secret_key"] + + +# Imports at the bottom keep the public dataclasses on top — pylint +# doesn't run on this repo, but tests do, and putting ``token_urlsafe`` +# in a public symbol confuses readers. Suppress the unused warning by +# referencing it once. +_ = token_urlsafe diff --git a/decnet/canary/generators/env_file.py b/decnet/canary/generators/env_file.py new file mode 100644 index 00000000..979b1dfd --- /dev/null +++ b/decnet/canary/generators/env_file.py @@ -0,0 +1,56 @@ +"""Fake ``.env`` with embedded callback URLs. + +Modern web stacks read environment variables for everything from +database DSNs to webhook URLs, so dropping a few realistic-looking +``KEY=value`` pairs alongside the canary URL is unremarkable. The +slug appears in two fields: + +* ``API_BASE_URL`` — the obvious one; an attacker scripting against + the credentials hits the worker on first invocation. +* ``WEBHOOK_NOTIFY_URL`` — secondary, in case the attacker greps for + ``WEBHOOK`` and pivots there. + +Other fields (``DB_PASSWORD``, ``REDIS_URL``, ``JWT_SECRET``) are +plausible but inert — they're realism filler, not detection +mechanisms. +""" +from __future__ import annotations + +import hashlib + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryGenerator + + +def _stable_token(seed: str, prefix: str = "") -> str: + h = hashlib.sha256((prefix + seed).encode()).hexdigest() + return h[:32] + + +class EnvFileGenerator(CanaryGenerator): + name = "env_file" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + base = ctx.http_base.rstrip("/") + slug = ctx.callback_token + api_url = f"{base}/c/{slug}" + body = ( + "# Production environment — DO NOT COMMIT\n" + f"API_BASE_URL={api_url}\n" + f"WEBHOOK_NOTIFY_URL={api_url}/webhook\n" + f"DB_PASSWORD={_stable_token(slug, 'db:')}\n" + f"REDIS_URL=redis://:{_stable_token(slug, 'redis:')[:16]}@redis.internal:6379/0\n" + f"JWT_SECRET={_stable_token(slug, 'jwt:')}\n" + "LOG_LEVEL=info\n" + "ENVIRONMENT=production\n" + ) + return CanaryArtifact( + path="", + content=body.encode("utf-8"), + mode=0o600, + mtime_offset=-86400 * 7, # last edited a week ago + generator=self.name, + notes=[ + f"API_BASE_URL embeds {api_url}", + f"WEBHOOK_NOTIFY_URL embeds {api_url}/webhook", + ], + ) diff --git a/decnet/canary/generators/git_config.py b/decnet/canary/generators/git_config.py new file mode 100644 index 00000000..297f18ab --- /dev/null +++ b/decnet/canary/generators/git_config.py @@ -0,0 +1,53 @@ +"""Fake ``.git/config`` with an attacker-bait remote URL. + +The ``[remote "origin"]`` ``url`` field is the natural place to embed +an HTTP-callback URL: it's normal for git remotes to be HTTPS, the +URL is read by every git command an attacker runs (``git pull``, +``git fetch``, ``git remote -v``), and the slug fits naturally as +part of a path. + +The generator emits a plausible private-mirror remote (``git.`` +or the canary host's hostname) so an attacker doesn't immediately +recognise it as a honeypot. The slug ends up in the URL path: + + [remote "origin"] + url = https://canary.example.test/c//repo.git +""" +from __future__ import annotations + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryGenerator + + +class GitConfigGenerator(CanaryGenerator): + name = "git_config" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + # Strip trailing slash defensively — operator may have + # configured DECNET_CANARY_HTTP_BASE either way. + base = ctx.http_base.rstrip("/") + slug = ctx.callback_token + # The /c//repo.git suffix gives us a realistic-looking + # path the worker can route on a single ``startswith("/c/")`` + # check, while still surviving a quick grep for the slug. + url = f"{base}/c/{slug}/repo.git" + body = ( + "[core]\n" + "\trepositoryformatversion = 0\n" + "\tfilemode = true\n" + "\tbare = false\n" + "\tlogallrefupdates = true\n" + "[remote \"origin\"]\n" + f"\turl = {url}\n" + "\tfetch = +refs/heads/*:refs/remotes/origin/*\n" + "[branch \"main\"]\n" + "\tremote = origin\n" + "\tmerge = refs/heads/main\n" + ) + return CanaryArtifact( + path="", + content=body.encode("utf-8"), + mode=0o644, + mtime_offset=-86400 * 30, # checked out a month ago + generator=self.name, + notes=[f"git remote 'origin' embeds {url}"], + ) diff --git a/decnet/canary/generators/honeydoc.py b/decnet/canary/generators/honeydoc.py new file mode 100644 index 00000000..455460b3 --- /dev/null +++ b/decnet/canary/generators/honeydoc.py @@ -0,0 +1,61 @@ +"""Built-in honeydoc — a minimal HTML "report" with a tracking pixel. + +This is the *fallback* honeydoc used when the operator hasn't +uploaded a real document. The HTML instrumenter handles operator +uploads via :mod:`decnet.canary.instrumenters.html`; this generator +exists so the deploy-time baseline can plant *something* convincing +without first prompting the operator to drop a file. + +The realism here is intentionally modest: a Documents-folder HTML +page with internal-looking content and a 1×1 remote image at the +bottom whose ``src`` is the canary callback URL. Most desktop +HTML renderers fetch the image as soon as the file is opened in a +browser preview, so opening the doc trips the callback. + +Operators who want a richer artifact should upload their own DOCX +or PDF; the corresponding instrumenter embeds the same callback in +the appropriate format. +""" +from __future__ import annotations + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryGenerator + + +class HoneydocGenerator(CanaryGenerator): + name = "honeydoc" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + base = ctx.http_base.rstrip("/") + slug = ctx.callback_token + pixel_url = f"{base}/c/{slug}" + body = ( + "\n" + "\n" + "\n" + "\n" + "Q3 Operations Review — DRAFT\n" + "\n" + "\n" + "

Q3 Operations Review (DRAFT — DO NOT DISTRIBUTE)

\n" + "

Forecast and remediation timeline below. Numbers are\n" + "preliminary and subject to revision before the all-hands.

\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "
RegionIncidentsMTTR (h)
us-east143.2
us-west94.7
eu-central222.1
\n" + "

Internal contact: " + "secops@internal

\n" + f"\"\"\n" + "\n" + "\n" + ) + return CanaryArtifact( + path="", + content=body.encode("utf-8"), + mode=0o644, # docs are typically world-readable + mtime_offset=-86400 * 21, # 3 weeks ago + generator=self.name, + notes=[f"tracking pixel src={pixel_url}"], + ) diff --git a/decnet/canary/generators/honeydoc_docx.py b/decnet/canary/generators/honeydoc_docx.py new file mode 100644 index 00000000..35456a23 --- /dev/null +++ b/decnet/canary/generators/honeydoc_docx.py @@ -0,0 +1,133 @@ +"""Real-DOCX honeydoc generator. + +Synthesises a minimal but structurally valid DOCX from scratch via +stdlib :mod:`zipfile`, then uses the same external-image relationship +trick that powers :mod:`decnet.canary.instrumenters.docx` to embed +the callback URL. No python-docx dependency. + +The output opens cleanly in Word / LibreOffice; both fetch the +external image relationship on document load. +""" +from __future__ import annotations + +import io +import zipfile + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryGenerator +from decnet.canary.instrumenters.docx import _drawing, _next_rid + + +_CONTENT_TYPES = ( + '' + '' + '' + '' + '' + '' +).encode() + +_PACKAGE_RELS = ( + '' + '' + '' + '' +).encode() + +_BODY_PARAGRAPHS = ( + "Q3 Operations Review (DRAFT — DO NOT DISTRIBUTE)", + "", + "Forecast and remediation timeline below. Numbers are preliminary " + "and subject to revision before the all-hands.", + "", + "Region Incidents MTTR (h)", + "us-east 14 3.2", + "us-west 9 4.7", + "eu-central 22 2.1", + "", + "Internal contact: secops@internal", +) + + +def _document_xml(rid_with_drawing: str | None = None) -> bytes: + """Build the body XML. + + ``rid_with_drawing`` is the rId of the external image relationship; + when set, we append the same ```` element that the DOCX + instrumenter inserts so the body references the external resource. + """ + paragraphs = [] + for line in _BODY_PARAGRAPHS: + if line: + paragraphs.append( + "" + + _xml_escape(line) + + "" + ) + else: + paragraphs.append("") + body = "".join(paragraphs) + drawing = _drawing(rid_with_drawing).decode() if rid_with_drawing else "" + return ( + '' + '' + f'{body}{drawing}' + '' + ).encode() + + +def _xml_escape(s: str) -> str: + return ( + s.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + ) + + +def _document_rels(rid: str, url: str) -> bytes: + return ( + '' + '' + f'' + '' + ).encode() + + +class HoneydocDocxGenerator(CanaryGenerator): + name = "honeydoc_docx" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + url = f"{ctx.http_base.rstrip('/')}/c/{ctx.callback_token}" + # Pick a stable rId — there's only one relationship in the + # synthesised file, so any unused id works. Reuse the + # instrumenter's allocator against the bare relationships + # skeleton for parity with operator-uploaded DOCX flow. + skeleton = ( + b'' + b'' + b'' + ) + rid = _next_rid(skeleton) + + out = io.BytesIO() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr("[Content_Types].xml", _CONTENT_TYPES) + zf.writestr("_rels/.rels", _PACKAGE_RELS) + zf.writestr("word/document.xml", _document_xml(rid)) + zf.writestr("word/_rels/document.xml.rels", _document_rels(rid, url)) + + return CanaryArtifact( + path="", + content=out.getvalue(), + mode=0o644, + mtime_offset=-86400 * 21, + generator=self.name, + notes=[ + "synthesised DOCX with realistic Q3 review body", + f"external-image relationship {rid} -> {url}", + ], + ) diff --git a/decnet/canary/generators/honeydoc_pdf.py b/decnet/canary/generators/honeydoc_pdf.py new file mode 100644 index 00000000..400271ff --- /dev/null +++ b/decnet/canary/generators/honeydoc_pdf.py @@ -0,0 +1,127 @@ +"""Real-PDF honeydoc generator (uses :mod:`pikepdf`). + +Builds a one-page PDF with the same Q3-review body as the HTML/DOCX +flavors and installs an ``/OpenAction`` ``/URI`` action on the +catalog so most viewers fire the callback the moment the document +opens. + +Pikepdf is now a hard dependency for this generator (the operator +installed it explicitly so we can use it). We still surface a +clear :class:`InstrumenterRejectedError` when imports fail, so a +deployment without pikepdf can fall back to the DOCX or HTML +generators rather than crashing the API. +""" +from __future__ import annotations + +import io + +from decnet.canary.base import ( + CanaryArtifact, + CanaryContext, + CanaryGenerator, + InstrumenterRejectedError, +) + + +_BODY_LINES = ( + ("Q3 Operations Review (DRAFT — DO NOT DISTRIBUTE)", 14), + ("", 12), + ("Forecast and remediation timeline below.", 11), + ("Numbers are preliminary, subject to revision.", 11), + ("", 12), + ("Region Incidents MTTR (h)", 11), + ("us-east 14 3.2", 11), + ("us-west 9 4.7", 11), + ("eu-central 22 2.1", 11), + ("", 12), + ("Internal contact: secops@internal", 11), +) + + +class HoneydocPdfGenerator(CanaryGenerator): + name = "honeydoc_pdf" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + try: + from pikepdf import Pdf, Name, Dictionary, String # type: ignore[import-not-found] + except ImportError as e: + raise InstrumenterRejectedError( + "honeydoc_pdf requires pikepdf; install it (`pip install " + "pikepdf`) or pick honeydoc / honeydoc_docx instead." + ) from e + + url = f"{ctx.http_base.rstrip('/')}/c/{ctx.callback_token}" + + pdf = Pdf.new() + # Helvetica is one of the 14 PDF base fonts — every viewer ships + # it, so no font embedding is required. + font = pdf.make_indirect(Dictionary( + Type=Name("/Font"), + Subtype=Name("/Type1"), + BaseFont=Name("/Helvetica"), + )) + + # Build a single content stream that writes each body line at a + # decreasing y-coordinate. PDF coordinates start at the bottom- + # left (US Letter = 612 x 792 points); we lay out lines roughly + # 18 points apart starting near the top. + ops: list[str] = ["BT /F1 12 Tf 72 750 Td"] + first = True + for line, size in _BODY_LINES: + if not first: + ops.append("0 -18 Td") + first = False + ops.append(f"/F1 {size} Tf") + ops.append(f"({_pdf_escape(line)}) Tj") + ops.append("ET") + content_bytes = "\n".join(ops).encode("latin-1") + + content_stream = pdf.make_stream(content_bytes) + + page = pdf.add_blank_page(page_size=(612, 792)) + page[Name("/Resources")] = Dictionary( + Font=Dictionary(F1=font), + ) + page[Name("/Contents")] = content_stream + + # OpenAction fires the URI when the file is opened in Acrobat, + # Preview, the browser PDF viewer, etc. Most viewers prompt + # before fetching; that prompt itself is a tell, and an + # auto-allow viewer fetches silently. + pdf.Root[Name("/OpenAction")] = Dictionary( + Type=Name("/Action"), + S=Name("/URI"), + URI=String(url), + ) + + out = io.BytesIO() + pdf.save(out) + return CanaryArtifact( + path="", + content=out.getvalue(), + mode=0o644, + mtime_offset=-86400 * 21, + generator=self.name, + notes=[ + "synthesised one-page PDF with realistic Q3 review body", + f"/OpenAction /URI -> {url}", + ], + ) + + +def _pdf_escape(s: str) -> str: + """Escape parens and backslashes for PDF literal-string syntax. + + PDF string literals are wrapped in ``( … )``; inner ``(``, ``)``, + and ``\\`` need backslash escapes. Everything else (including + UTF-8 multibyte sequences) round-trips fine because Helvetica's + encoding is WinAnsi-ish — we'll lose exotic glyphs but the + realistic body sticks to ASCII anyway. Em-dashes are downgraded + to ``--`` to avoid the WinAnsi gap. + """ + return ( + s.replace("\\", r"\\") + .replace("(", r"\(") + .replace(")", r"\)") + .replace("—", "--") + ) diff --git a/decnet/canary/generators/mysql_dump.py b/decnet/canary/generators/mysql_dump.py new file mode 100644 index 00000000..ab324137 --- /dev/null +++ b/decnet/canary/generators/mysql_dump.py @@ -0,0 +1,190 @@ +"""Fake ``mysqldump`` output that phones home on import. + +Mirrors the Canarytokens.org MySQL-dump trick. When a victim runs +``mysql < dump.sql``, the trailer block executes a base64-obfuscated +``CHANGE REPLICATION SOURCE TO`` against ``.canary.`` +followed by ``START REPLICA``. The victim's MySQL daemon then: + +1. Resolves the slug subdomain via DNS — this is the trip our + :mod:`decnet.canary.dns_server` already detects. +2. Opens a TCP replica handshake on port 3306, sending its own + ``@@hostname`` and ``@@lc_time_names`` smuggled into the + ``SOURCE_USER`` field via ``CONCAT``. Capturing those bytes + requires a MySQL handshake responder on the worker — out of scope + for v1; the DNS lookup alone is sufficient for detection. + +The base64 wrapper is the camouflage: a plain ``grep canary dump.sql`` +finds nothing. The slug only materialises when the victim's server +runs ``PREPARE … FROM @s2``. + +Because the trip surface is DNS, this generator REQUIRES a non-empty +``dns_zone``. The slug must appear as the leftmost label of the +hostname so a single DNS query identifies the token; the http_base +host is not slug-bearing and can't substitute. +""" +from __future__ import annotations + +import base64 +import hashlib + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryGenerator + + +def _stable_hex(seed: str, prefix: str = "", length: int = 16) -> str: + h = hashlib.sha256((prefix + seed).encode()).hexdigest() + return h[:length] + + +def _build_replica_payload(slug: str, dns_zone: str) -> str: + """Inner SQL that gets base64-wrapped. + + The CONCAT splices ``@@lc_time_names`` and ``@@hostname`` into the + ``SOURCE_USER`` value at PREPARE time so the victim's locale and + hostname travel as the replica username on the 3306 handshake. + """ + host = f"{slug}.{dns_zone}" + return ( + "SET @bb = CONCAT(" + "\"CHANGE REPLICATION SOURCE TO " + "SOURCE_PASSWORD='replica-pw', " + "SOURCE_RETRY_COUNT=1, " + "SOURCE_PORT=3306, " + f"SOURCE_HOST='{host}', " + "SOURCE_SSL=0, " + f"SOURCE_USER='{slug}\", " + "@@lc_time_names, @@hostname, \"';\");" + ) + + +def _build_trailer(slug: str, dns_zone: str) -> str: + inner = _build_replica_payload(slug, dns_zone) + encoded = base64.b64encode(inner.encode("utf-8")).decode("ascii") + return ( + f"SET @b = '{encoded}';\n" + "SET @s2 = FROM_BASE64(@b);\n" + "PREPARE stmt1 FROM @s2;\n" + "EXECUTE stmt1;\n" + "PREPARE stmt2 FROM @bb;\n" + "EXECUTE stmt2;\n" + "START REPLICA;\n" + ) + + +class MySQLDumpGenerator(CanaryGenerator): + name = "mysql_dump" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + if not ctx.dns_zone: + raise ValueError( + "mysql_dump requires a non-empty dns_zone — the trip " + "surface is a DNS lookup of .." + ) + slug = ctx.callback_token + zone = ctx.dns_zone + host = f"{slug}.{zone}" + + # Realism filler: deterministic per-slug fake user rows so two + # runs with the same context produce byte-identical output + # (planter idempotency contract). + u1_hash = _stable_hex(slug, "u1:", 32) + u2_hash = _stable_hex(slug, "u2:", 32) + api_token = _stable_hex(slug, "api:", 40) + + # Synthesised SQL bait below — never executed by us, only by + # whoever runs ``mysql < dump.sql`` against their own server. + # Built with .format() instead of f-strings so bandit's B608 + # heuristic doesn't false-positive on the "INSERT INTO" + var + # pattern. + users_insert = ( + "INSERT INTO `users` VALUES " # nosec B608 + "(1,'alice@app.internal','$2y$10${u1a}.{u1b}','2024-11-12 09:13:44')," + "(2,'bob@app.internal','$2y$10${u2a}.{u2b}','2025-02-03 17:42:08');\n" + ).replace("{u1a}", u1_hash[:22]).replace("{u1b}", u1_hash[22:]) \ + .replace("{u2a}", u2_hash[:22]).replace("{u2b}", u2_hash[22:]) + api_keys_insert = ( + "INSERT INTO `api_keys` VALUES (1,1,'{tok}');\n" # nosec B608 + ).replace("{tok}", api_token) + header = ( + "-- MySQL dump 10.13 Distrib 8.0.35, for Linux (x86_64)\n" + "--\n" + "-- Host: db-prod-01 Database: app_production\n" + "-- ------------------------------------------------------\n" + "-- Server version\t8.0.35\n" + "\n" + "/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;\n" + "/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;\n" + "/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;\n" + "/*!50503 SET NAMES utf8mb4 */;\n" + "/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;\n" + "/*!40103 SET TIME_ZONE='+00:00' */;\n" + "/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;\n" + "/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;\n" + "/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;\n" + "/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;\n" + "\n" + "--\n" + "-- Table structure for table `users`\n" + "--\n" + "\n" + "DROP TABLE IF EXISTS `users`;\n" + "CREATE TABLE `users` (\n" + " `id` int unsigned NOT NULL AUTO_INCREMENT,\n" + " `email` varchar(255) NOT NULL,\n" + " `password_hash` char(60) NOT NULL,\n" + " `created_at` datetime NOT NULL,\n" + " PRIMARY KEY (`id`),\n" + " UNIQUE KEY `uniq_email` (`email`)\n" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n" + "\n" + "LOCK TABLES `users` WRITE;\n" + + users_insert + + "UNLOCK TABLES;\n" + "\n" + "--\n" + "-- Table structure for table `api_keys`\n" + "--\n" + "\n" + "DROP TABLE IF EXISTS `api_keys`;\n" + "CREATE TABLE `api_keys` (\n" + " `id` int unsigned NOT NULL AUTO_INCREMENT,\n" + " `user_id` int unsigned NOT NULL,\n" + " `token` char(40) NOT NULL,\n" + " PRIMARY KEY (`id`)\n" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n" + "\n" + "LOCK TABLES `api_keys` WRITE;\n" + + api_keys_insert + + "UNLOCK TABLES;\n" + "\n" + ) + + trailer_replica = _build_trailer(slug, zone) + + trailer_close = ( + "\n" + "/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;\n" + "/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;\n" + "/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;\n" + "/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;\n" + "/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;\n" + "/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;\n" + "/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;\n" + "/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;\n" + "\n" + "-- Dump completed\n" + ) + + body = header + trailer_replica + trailer_close + + return CanaryArtifact( + path="", + content=body.encode("utf-8"), + mode=0o600, + mtime_offset=-86400 * 7, # last week's backup + generator=self.name, + notes=[ + f"replica payload phones home to {host}:3306 on import", + "base64-wrapped PREPARE/EXECUTE block hides the slug from grep", + "@@hostname and @@lc_time_names smuggled into SOURCE_USER", + ], + ) diff --git a/decnet/canary/generators/ssh_key.py b/decnet/canary/generators/ssh_key.py new file mode 100644 index 00000000..96835aa4 --- /dev/null +++ b/decnet/canary/generators/ssh_key.py @@ -0,0 +1,68 @@ +"""Fake SSH private key with the callback host in the comment. + +OpenSSH private keys carry a free-form comment field — typically +``user@host`` — that's preserved across rounds of ``ssh-keygen -p``. +We embed the canary host as the ``user@host`` so an attacker who +imports the key into their own keyring or runs ``ssh-keygen -lf`` on +it sees a hostname they may then try to reach. + +The key bytes themselves are syntactically valid (PEM envelope, base64 +body) but cryptographically junk — the body is a deterministic SHA-256 +hash of the slug repeated to the right length. We don't ship a real +RSA/Ed25519 key because (a) we don't want a real private key sitting +on disk pretending to be valuable, and (b) the attacker ``cat``-ing +the file or running ``ssh -i`` will trigger the callback regardless +of cryptographic validity. + +The DNS-callback variant uses ``.canary.`` as the +hostname so a bare ``ssh-keygen -lf`` on the file resolves a unique +subdomain even if the attacker never hits HTTP. +""" +from __future__ import annotations + +import base64 +import hashlib + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryGenerator + + +def _fake_key_body(seed: str) -> str: + # Real OpenSSH keys are several hundred base64 chars; we make a + # plausible-looking 24-line block from a SHA-256-derived stream. + h = hashlib.sha256(seed.encode()).digest() + long_stream = (h * 32)[:768] # 768 bytes → ~1024 base64 chars + encoded = base64.b64encode(long_stream).decode() + # Wrap at 70 chars per line — same shape ``ssh-keygen`` produces. + return "\n".join(encoded[i:i + 70] for i in range(0, len(encoded), 70)) + + +class SSHKeyGenerator(CanaryGenerator): + name = "ssh_key" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + slug = ctx.callback_token + body = _fake_key_body(slug) + # Hostname for the comment: prefer DNS-zone form when the + # operator has DNS deployed (so ssh-keygen -lf names a subdomain + # the attacker may resolve); fall back to the http_base host + # otherwise. + if ctx.dns_zone: + host_comment = f"deploy@{slug}.{ctx.dns_zone}" + else: + from urllib.parse import urlparse + host = urlparse(ctx.http_base).hostname or "deploy.local" + host_comment = f"deploy@{host}" + content = ( + "-----BEGIN OPENSSH PRIVATE KEY-----\n" + f"{body}\n" + "-----END OPENSSH PRIVATE KEY-----\n" + f"# {host_comment}\n" + ) + return CanaryArtifact( + path="", + content=content.encode("utf-8"), + mode=0o600, + mtime_offset=-86400 * 60, # 2 months ago + generator=self.name, + notes=[f"comment line embeds {host_comment}"], + ) diff --git a/decnet/canary/instrumenters/__init__.py b/decnet/canary/instrumenters/__init__.py new file mode 100644 index 00000000..905e02b6 --- /dev/null +++ b/decnet/canary/instrumenters/__init__.py @@ -0,0 +1,4 @@ +"""Built-in canary instrumenters (operator-uploaded artifact mutation). + +Lazy-imported by :func:`decnet.canary.factory.get_instrumenter`. +""" diff --git a/decnet/canary/instrumenters/docx.py b/decnet/canary/instrumenters/docx.py new file mode 100644 index 00000000..f0a87903 --- /dev/null +++ b/decnet/canary/instrumenters/docx.py @@ -0,0 +1,147 @@ +"""DOCX instrumenter — inject a remote image into the body. + +DOCX files are zip archives carrying ``word/document.xml`` (the body) +and ``word/_rels/document.xml.rels`` (the relationship table that +maps ``rId`` references to URLs). We: + +1. Add a new relationship of type ``image`` whose target is the + canary callback URL and ``TargetMode="External"``. +2. Add a tiny ```` element referencing that ``rId`` at + the end of ``word/document.xml`` (just before ````). + +Word and LibreOffice both fetch external image relationships when +the document is opened (subject to the user's "trusted source" +toggle, which most enterprise environments disable in favour of +"warn but allow"). + +We use stdlib ``zipfile`` only — no python-docx dependency — because +the surface we touch is two small XML files and we don't need any of +the higher-level abstractions. +""" +from __future__ import annotations + +import io +import re +import zipfile +from typing import Tuple + +from decnet.canary.base import ( + CanaryArtifact, + CanaryContext, + CanaryInstrumenter, + InstrumenterRejectedError, +) + + +_RELS_END = re.compile(rb"", re.IGNORECASE) +_BODY_END = re.compile(rb"", re.IGNORECASE) + + +def _next_rid(rels_xml: bytes) -> str: + """Return an rId not already taken in the relationships file. + + Word's loader tolerates non-sequential ids, so we just pick one + well above the typical range to avoid collisions. + """ + used = set(m.group(1).decode() for m in re.finditer(rb'Id="(rId\d+)"', rels_xml)) + for n in range(900, 9999): + rid = f"rId{n}" + if rid not in used: + return rid + raise InstrumenterRejectedError("DOCX has too many relationships to allocate a new rId") + + +def _inject_relationship(rels_xml: bytes, rid: str, url: str) -> bytes: + rel = ( + f'' + ).encode() + match = _RELS_END.search(rels_xml) + if not match: + raise InstrumenterRejectedError( + "DOCX rels file has no ; refusing to mutate" + ) + return rels_xml[:match.start()] + rel + rels_xml[match.start():] + + +def _drawing(rid: str) -> bytes: + # Minimal w:drawing tree referencing the external image at rid. + # Dimensions are 1 EMU x 1 EMU so the image is invisible; Word + # still fetches the resource on document load. + return ( + '' + '' + '' + '' + '' + '' + '' + '' + f'' + '' + '' + '' + '' + '' + '' + ).encode() + + +def _inject_drawing(document_xml: bytes, rid: str) -> bytes: + match = _BODY_END.search(document_xml) + if not match: + raise InstrumenterRejectedError("DOCX document.xml has no ") + drawing = _drawing(rid) + return document_xml[:match.start()] + drawing + document_xml[match.start():] + + +def _mutate(blob: bytes, url: str) -> Tuple[bytes, str]: + try: + with zipfile.ZipFile(io.BytesIO(blob), "r") as zf: + try: + rels = zf.read("word/_rels/document.xml.rels") + doc = zf.read("word/document.xml") + except KeyError as e: + raise InstrumenterRejectedError( + f"DOCX missing expected member: {e.args[0]!r}" + ) from e + members = [(zi, zf.read(zi.filename)) for zi in zf.infolist()] + except zipfile.BadZipFile as e: + raise InstrumenterRejectedError("uploaded blob is not a valid DOCX zip") from e + + rid = _next_rid(rels) + new_rels = _inject_relationship(rels, rid, url) + new_doc = _inject_drawing(doc, rid) + + out = io.BytesIO() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zf_out: + for zi, data in members: + if zi.filename == "word/_rels/document.xml.rels": + zf_out.writestr(zi.filename, new_rels) + elif zi.filename == "word/document.xml": + zf_out.writestr(zi.filename, new_doc) + else: + zf_out.writestr(zi, data) + return out.getvalue(), rid + + +class DocxInstrumenter(CanaryInstrumenter): + name = "docx" + mime_prefixes = ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ) + + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + url = f"{ctx.http_base.rstrip('/')}/c/{ctx.callback_token}" + mutated, rid = _mutate(blob, url) + return CanaryArtifact( + path=target_path, + content=mutated, + mode=0o644, + mtime_offset=-86400 * 14, + instrumenter=self.name, + notes=[f"injected external-image relationship {rid} -> {url}"], + ) diff --git a/decnet/canary/instrumenters/html.py b/decnet/canary/instrumenters/html.py new file mode 100644 index 00000000..02b4d4e2 --- /dev/null +++ b/decnet/canary/instrumenters/html.py @@ -0,0 +1,45 @@ +"""HTML instrumenter — append a 1×1 tracking pixel. + +Stdlib-only. We don't parse the HTML; we just inject the ```` +tag immediately before the closing ```` (or, failing that, at +the end of the document). Most renderers that support remote images +(email previewers, IDE doc previews, browsers) will fetch it as +soon as the document is opened. +""" +from __future__ import annotations + +import re + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryInstrumenter + + +_BODY_CLOSE = re.compile(rb"", re.IGNORECASE) + + +class HtmlInstrumenter(CanaryInstrumenter): + name = "html" + mime_prefixes = ("text/html", "application/xhtml+xml") + + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + url = f"{ctx.http_base.rstrip('/')}/c/{ctx.callback_token}".encode() + pixel = ( + b"\n" + ) + match = _BODY_CLOSE.search(blob) + if match: + out = blob[:match.start()] + pixel + blob[match.start():] + note = "injected 1x1 pixel before " + else: + out = (blob if blob.endswith(b"\n") else blob + b"\n") + pixel + note = "appended 1x1 pixel (no found)" + return CanaryArtifact( + path=target_path, + content=out, + mode=0o644, + mtime_offset=-86400 * 7, + instrumenter=self.name, + notes=[note, f"pixel src={url.decode()}"], + ) diff --git a/decnet/canary/instrumenters/image.py b/decnet/canary/instrumenters/image.py new file mode 100644 index 00000000..69e31ff4 --- /dev/null +++ b/decnet/canary/instrumenters/image.py @@ -0,0 +1,72 @@ +"""Image instrumenter — requires :mod:`PIL` (optional dependency). + +For PNG/JPEG/GIF we append a tEXt/EXIF chunk carrying the slug so +``exiftool`` / ``identify -verbose`` surface the slug, then route the +detection via a sibling **plain-text companion file**. The image +itself can't really embed an HTTP fetcher — image decoders don't +run network requests on decode — so the realistic detection surface +is "attacker exfils the image, runs metadata tools on it, hits our +URL when curious about the embedded marker." + +When Pillow isn't installed we reject and direct the operator to +``passthrough`` (which preserves the bytes; the slug then lives in +the filename only). +""" +from __future__ import annotations + +import io + +from decnet.canary.base import ( + CanaryArtifact, + CanaryContext, + CanaryInstrumenter, + InstrumenterRejectedError, +) + + +class ImageInstrumenter(CanaryInstrumenter): + name = "image" + mime_prefixes = ("image/png", "image/jpeg", "image/gif") + + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + try: + from PIL import Image, PngImagePlugin # type: ignore[import-not-found] + except ImportError as e: + raise InstrumenterRejectedError( + "image instrumenter requires Pillow; install it (`pip " + "install Pillow`) or re-upload the artifact with " + "kind=passthrough so it ships unmodified." + ) from e + + slug_url = f"{ctx.http_base.rstrip('/')}/c/{ctx.callback_token}" + try: + buf_in = io.BytesIO(blob) + img = Image.open(buf_in) + fmt = (img.format or "").upper() + buf_out = io.BytesIO() + if fmt == "PNG": + meta = PngImagePlugin.PngInfo() + meta.add_text("Comment", f"reference: {slug_url}") + meta.add_text("X-Canary", ctx.callback_token) + img.save(buf_out, format="PNG", pnginfo=meta) + elif fmt in ("JPEG", "JPG"): + # Pillow encodes JPEG comments via the ``comment`` kwarg. + img.save(buf_out, format="JPEG", comment=slug_url.encode()) + else: + # GIF and friends — Pillow doesn't expose comment metadata + # uniformly. Re-encode as-is and skip the metadata embed. + img.save(buf_out, format=fmt or "PNG") + mutated = buf_out.getvalue() + except Exception as e: + raise InstrumenterRejectedError(f"failed to instrument image: {e!s}") from e + + return CanaryArtifact( + path=target_path, + content=mutated, + mode=0o644, + mtime_offset=-86400 * 30, + instrumenter=self.name, + notes=[f"image metadata carries {slug_url} (slug={ctx.callback_token})"], + ) diff --git a/decnet/canary/instrumenters/passthrough.py b/decnet/canary/instrumenters/passthrough.py new file mode 100644 index 00000000..09816d86 --- /dev/null +++ b/decnet/canary/instrumenters/passthrough.py @@ -0,0 +1,37 @@ +"""Passthrough instrumenter — bytes go to disk unchanged. + +Used as the dispatch fallback for content types we can't safely +mutate (random binary blobs, container images, archives we don't +recognise). In passthrough mode the only callback surface is the +:attr:`CanaryToken.placement_path` itself: the operator must use a +DNS-callback token whose slug appears in the filename, so a +listing/access at the OS level resolves the slug as part of the +path (e.g. ``/etc/.canary.example.test/secrets.bin``) when +the attacker greps for hostnames in their loot. + +The instrumenter does not enforce that — the API does, when it sees +``instrumenter=passthrough`` with ``kind=http`` it returns 400. +""" +from __future__ import annotations + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryInstrumenter + + +class PassthroughInstrumenter(CanaryInstrumenter): + name = "passthrough" + mime_prefixes = () # dispatched by fallback in pick_instrumenter_for_mime + + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + return CanaryArtifact( + path=target_path, + content=blob, + mode=0o644, + mtime_offset=-86400 * 7, + instrumenter=self.name, + notes=[ + "passthrough: bytes unchanged — only DNS-callback tokens " + "trip detection (slug must live in the placement path)", + ], + ) diff --git a/decnet/canary/instrumenters/pdf.py b/decnet/canary/instrumenters/pdf.py new file mode 100644 index 00000000..516b6999 --- /dev/null +++ b/decnet/canary/instrumenters/pdf.py @@ -0,0 +1,76 @@ +"""PDF instrumenter — requires :mod:`pikepdf` (optional dependency). + +PDF embedding is non-trivial: the cleanest place to put a callback +is an ``/AA`` (additional actions) ``/O`` (open) entry on the +catalog or a ``/URI`` action on a link annotation. Either path +needs proper xref-table updates — pikepdf handles that for us. + +If pikepdf isn't available in the environment the instrumenter +raises :class:`InstrumenterRejectedError` so the API can return a +clear 400 directing the operator to either install pikepdf or +re-upload as ``passthrough``. + +We don't ship a stdlib fallback because every "naive" PDF mutation +I'm aware of (appending raw bytes, splicing into the trailer, etc.) +breaks the document's xref table and trips a "file is corrupt" +warning in modern viewers — which the attacker will absolutely +notice. +""" +from __future__ import annotations + +from decnet.canary.base import ( + CanaryArtifact, + CanaryContext, + CanaryInstrumenter, + InstrumenterRejectedError, +) + + +class PdfInstrumenter(CanaryInstrumenter): + name = "pdf" + mime_prefixes = ("application/pdf",) + + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + try: + import pikepdf # type: ignore[import-not-found] + except ImportError as e: + raise InstrumenterRejectedError( + "PDF instrumenter requires pikepdf; install it (`pip " + "install pikepdf`) or re-upload the artifact with " + "kind=passthrough so it ships unmodified." + ) from e + + url = f"{ctx.http_base.rstrip('/')}/c/{ctx.callback_token}" + try: + import io + buf = io.BytesIO(blob) + with pikepdf.open(buf) as pdf: + # Add an OpenAction that fires a URI action on document + # open. Most viewers prompt before fetching; that's + # fine — even the prompt itself can trip a "user + # interacted with the document" tell, and an + # auto-allow viewer fetches the URL silently. + action = pikepdf.Dictionary( + Type=pikepdf.Name("/Action"), + S=pikepdf.Name("/URI"), + URI=pikepdf.String(url), + ) + pdf.Root[pikepdf.Name("/OpenAction")] = action + out = io.BytesIO() + pdf.save(out) + mutated = out.getvalue() + except Exception as e: + raise InstrumenterRejectedError( + f"failed to instrument PDF: {e!s}" + ) from e + + return CanaryArtifact( + path=target_path, + content=mutated, + mode=0o644, + mtime_offset=-86400 * 14, + instrumenter=self.name, + notes=[f"installed /OpenAction /URI -> {url}"], + ) diff --git a/decnet/canary/instrumenters/plain.py b/decnet/canary/instrumenters/plain.py new file mode 100644 index 00000000..bfbea7ac --- /dev/null +++ b/decnet/canary/instrumenters/plain.py @@ -0,0 +1,79 @@ +"""Plain-text / config-file instrumenter. + +Two embedding strategies, picked in order: + +1. **Token substitution.** If the blob contains the literal + placeholder ``{{CANARY_URL}}`` or ``{{CANARY_HOST}}``, replace it. + This gives operators full control over where the slug lands — + they can pre-edit the file with placeholders before uploading. +2. **Append.** Otherwise, append a comment line that mentions the + callback URL. The comment style adapts to the file's apparent + syntax (``#`` for shell/yaml/python/dockerfile, ``//`` for json5/ + javascript-ish, ``;`` for ini). + +Operators who want neither behavior should upload the file as +``passthrough``. +""" +from __future__ import annotations + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryInstrumenter + + +_SLASH_HINTS = (b"//", b"function ", b"const ", b"let ", b"var ") +_SEMI_HINTS = (b"[default]", b"[section]", b"\n[") + + +def _comment_prefix(blob: bytes) -> bytes: + head = blob[:512] + if any(h in head for h in _SEMI_HINTS): + return b"; " + if any(h in head for h in _SLASH_HINTS): + return b"// " + # Default to # — the most common comment glyph across config files + # we'd plausibly canary. + return b"# " + + +class PlainInstrumenter(CanaryInstrumenter): + name = "plain" + mime_prefixes = ("text/", "application/json", "application/yaml", "application/toml") + + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + base = ctx.http_base.rstrip("/") + callback_url = f"{base}/c/{ctx.callback_token}".encode() + callback_host = ( + f"{ctx.callback_token}.{ctx.dns_zone}".encode() + if ctx.dns_zone else b"" + ) + notes: list[str] = [] + out = blob + + if b"{{CANARY_URL}}" in blob: + out = out.replace(b"{{CANARY_URL}}", callback_url) + notes.append(f"substituted {{{{CANARY_URL}}}} -> {callback_url.decode()}") + if b"{{CANARY_HOST}}" in blob and callback_host: + out = out.replace(b"{{CANARY_HOST}}", callback_host) + notes.append(f"substituted {{{{CANARY_HOST}}}} -> {callback_host.decode()}") + + if not notes: + # No placeholders — append a comment line at the end. + prefix = _comment_prefix(blob) + tail = ( + b"\n" + prefix + b"see " + callback_url + + b" for the latest version\n" + ) + out = (out if out.endswith(b"\n") else out + b"\n") + tail + notes.append( + f"appended comment line carrying {callback_url.decode()}" + ) + + return CanaryArtifact( + path=target_path, + content=out, + mode=0o644, + mtime_offset=-86400 * 7, + instrumenter=self.name, + notes=notes, + ) diff --git a/decnet/canary/instrumenters/xlsx.py b/decnet/canary/instrumenters/xlsx.py new file mode 100644 index 00000000..ed5cfbc2 --- /dev/null +++ b/decnet/canary/instrumenters/xlsx.py @@ -0,0 +1,95 @@ +"""XLSX instrumenter — embed an external-image link. + +XLSX is structurally identical to DOCX (Office Open XML zip). The +injection target is the workbook's relationships file +(``xl/_rels/workbook.xml.rels``). We add an external image +relationship there; Excel/LibreOffice fetch external images on +workbook open in the same way Word does. + +We don't inject a ```` element into a sheet because that +requires touching ``xl/worksheets/sheetN.xml`` *and* allocating a new +``xl/drawings/drawingN.xml`` part — much higher chance of mangling +the file. An orphan external image relationship is enough: many +Office viewers fetch all relationships at open time regardless of +whether they're referenced from a sheet. + +If the operator wants a stronger trigger (image visible in the +sheet, fetched even by viewers that lazy-load external resources) +they should embed the slug as a hyperlink cell content via the +``plain``/``passthrough`` instrumenters. +""" +from __future__ import annotations + +import io +import zipfile +from typing import Tuple + +from decnet.canary.base import ( + CanaryArtifact, + CanaryContext, + CanaryInstrumenter, + InstrumenterRejectedError, +) +from decnet.canary.instrumenters.docx import _inject_relationship, _next_rid + + +_RELS_PATHS = ( + "xl/_rels/workbook.xml.rels", + "xl/_rels/sharedStrings.xml.rels", +) + + +def _mutate(blob: bytes, url: str) -> Tuple[bytes, str, str]: + try: + with zipfile.ZipFile(io.BytesIO(blob), "r") as zf: + members = [(zi, zf.read(zi.filename)) for zi in zf.infolist()] + except zipfile.BadZipFile as e: + raise InstrumenterRejectedError("uploaded blob is not a valid XLSX zip") from e + + target_rels: str | None = None + for zi, _ in members: + if zi.filename in _RELS_PATHS: + target_rels = zi.filename + break + if not target_rels: + raise InstrumenterRejectedError( + "XLSX has no workbook relationships file to mutate" + ) + + out_members = [] + rid = "" + for zi, data in members: + if zi.filename == target_rels: + rid = _next_rid(data) + data = _inject_relationship(data, rid, url) + out_members.append((zi, data)) + + out = io.BytesIO() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zf_out: + for zi, data in out_members: + zf_out.writestr(zi, data) + return out.getvalue(), rid, target_rels + + +class XlsxInstrumenter(CanaryInstrumenter): + name = "xlsx" + mime_prefixes = ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + url = f"{ctx.http_base.rstrip('/')}/c/{ctx.callback_token}" + mutated, rid, target_rels = _mutate(blob, url) + return CanaryArtifact( + path=target_path, + content=mutated, + mode=0o644, + mtime_offset=-86400 * 14, + instrumenter=self.name, + notes=[ + f"injected external-image relationship {rid} into " + f"{target_rels} -> {url}", + ], + ) diff --git a/decnet/canary/paths.py b/decnet/canary/paths.py new file mode 100644 index 00000000..5700ad0f --- /dev/null +++ b/decnet/canary/paths.py @@ -0,0 +1,82 @@ +"""Persona-aware path resolution for canary artifacts. + +Linux-persona deckies use POSIX-shaped paths under ``/home/``. +"Windows" personas (still Linux containers under the hood — see +:mod:`decnet.archetypes`) use Windows-shaped paths under +``/home//AppData/...`` so an attacker browsing the filesystem +through a planted RDP/SMB session sees the right shape. + +The persona lookup is best-effort: callers pass the +:attr:`decnet.archetypes.Archetype.nmap_os` value (``"linux"`` or +``"windows"``); unknown personas fall through to ``"linux"``. +Operators can always override by passing an explicit +``placement_path`` when creating a token. +""" +from __future__ import annotations + +DEFAULT_LINUX_USER = "admin" +DEFAULT_WINDOWS_USER = "Administrator" + +# Canonical placements for the synthesizer-driven baseline tokens. +# Operators can override per-token via the API, but these are the +# defaults the deploy-time seed uses. +_LINUX_DEFAULTS: dict[str, str] = { + "git_config": "/home/{user}/.git/config", + "env_file": "/home/{user}/.env", + "ssh_key": "/home/{user}/.ssh/id_rsa", + "aws_creds": "/home/{user}/.aws/credentials", + "honeydoc": "/home/{user}/Documents/quarterly_report.html", + "honeydoc_docx": "/home/{user}/Documents/quarterly_report.docx", + "honeydoc_pdf": "/home/{user}/Documents/quarterly_report.pdf", +} + +_WINDOWS_DEFAULTS: dict[str, str] = { + "git_config": "/home/{user}/AppData/Local/Programs/Git/etc/gitconfig", + "env_file": "/home/{user}/Desktop/prod.env", + "ssh_key": "/home/{user}/.ssh/id_rsa", # OpenSSH on Windows uses the same path + "aws_creds": "/home/{user}/.aws/credentials", + "honeydoc": "/home/{user}/Documents/quarterly_report.html", + "honeydoc_docx": "/home/{user}/Documents/quarterly_report.docx", + "honeydoc_pdf": "/home/{user}/Documents/quarterly_report.pdf", +} + + +def default_user(persona: str) -> str: + """Return the conventional unprivileged username for a persona.""" + return DEFAULT_WINDOWS_USER if persona == "windows" else DEFAULT_LINUX_USER + + +def default_path_for(generator: str, persona: str = "linux") -> str: + """Resolve the default placement path for a synthesized token. + + Returns an absolute container path with ``{user}`` already + expanded. Falls back to a sane Linux default for unknown + personas — better to plant *something* than fail the deploy hook. + """ + table = _WINDOWS_DEFAULTS if persona == "windows" else _LINUX_DEFAULTS + template = table.get(generator) + if not template: + # Unknown generator — fall back to a generic /tmp drop so the + # planter still has somewhere to write. The API rejects + # unknown generators upstream, so this branch is defensive. + return f"/tmp/{generator}.canary" # nosec B108 — placement inside attacker-facing decoy container, not host /tmp + return template.format(user=default_user(persona)) + + +def normalize_placement(path: str) -> str: + """Validate and normalize an operator-supplied placement path. + + Forbids relative paths, NUL bytes, and shell metacharacters that + ``docker exec sh -c`` can't safely round-trip. Returns the + sanitised path unchanged when valid; raises :class:`ValueError` + otherwise so the API can return a 400 with a clear message. + """ + if not path or not path.startswith("/"): + raise ValueError("placement_path must be absolute (start with '/')") + if "\x00" in path: + raise ValueError("placement_path may not contain NUL") + if "\n" in path or "\r" in path: + raise ValueError("placement_path may not contain newlines") + if "../" in path or path.endswith("/.."): + raise ValueError("placement_path may not contain '..' segments") + return path diff --git a/decnet/canary/planter.py b/decnet/canary/planter.py new file mode 100644 index 00000000..6beae78b --- /dev/null +++ b/decnet/canary/planter.py @@ -0,0 +1,301 @@ +"""Plant / revoke canary artifacts inside running decky containers. + +Single entry point per operation: + +* :func:`plant` writes a :class:`CanaryArtifact` into one decky's + filesystem via ``docker exec`` (mirroring the SSH driver's + ``_run_file`` pattern), backdates the mtime, sets the requested + mode, and publishes ``canary.{token_id}.placed`` on the bus. +* :func:`revoke` unlinks the file (best-effort) and publishes + ``canary.{token_id}.revoked``. +* :func:`seed_baseline` is the deploy-hook helper: synthesises the + configured baseline set for one decky, persists rows, plants each. + Failures are logged but do **not** abort the deploy (the deployer + hook calls this best-effort). + +We don't reuse :class:`SSHDriver` directly because the orchestrator +driver is tied to its action types (``FileAction`` carries str +content; canary content is bytes). The planter takes the same +shape but speaks bytes-via-base64 over the wire. +""" +from __future__ import annotations + +import asyncio +import base64 +import os +import shlex +import time +from secrets import token_urlsafe +from typing import Any, Iterable, Optional + +from decnet.bus import topics +from decnet.bus.base import BaseBus +from decnet.bus.factory import get_bus +from decnet.canary.base import CanaryArtifact, CanaryContext +from decnet.canary.factory import get_generator +from decnet.canary.paths import default_path_for +from decnet.logging import get_logger +from decnet.web.db.repository import BaseRepository + +log = get_logger("canary.planter") + +_DOCKER = "docker" +_TIMEOUT = 8.0 +# Container suffix — matches the orchestrator SSH driver's convention +# (``-ssh``). Canary placement always happens through the +# ssh container because every decky has one and it carries the most +# realistic filesystem layout. +_SSH_CONTAINER_SUFFIX = "-ssh" + + +def _container_for(decky_name: str) -> str: + return f"{decky_name}{_SSH_CONTAINER_SUFFIX}" + + +def _dirname(path: str) -> str: + idx = path.rfind("/") + if idx <= 0: + return "/" + return path[:idx] + + +async def _run( + argv: list[str], *, stdin_bytes: Optional[bytes] = None, +) -> tuple[int, str, str]: + try: + proc = await asyncio.create_subprocess_exec( + *argv, + stdin=asyncio.subprocess.PIPE if stdin_bytes is not None else None, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except FileNotFoundError as exc: + return 127, "", f"argv[0] not found: {exc}" + try: + stdout, stderr = await asyncio.wait_for( + proc.communicate(input=stdin_bytes), timeout=_TIMEOUT, + ) + except asyncio.TimeoutError: + try: + proc.kill() + except ProcessLookupError: + pass + return 124, "", "timeout" + return ( + proc.returncode if proc.returncode is not None else -1, + stdout.decode("utf-8", "replace"), + stderr.decode("utf-8", "replace"), + ) + + +def _build_plant_command(artifact: CanaryArtifact) -> tuple[str, bytes]: + """Compose the ``sh -c`` script + stdin payload for one artifact. + + Binary safety: we base64-encode on the host and stream the result + over stdin to ``base64 -d`` inside the container, so the bytes + never touch the argv (kernel ARG_MAX would reject anything larger + than ~128KB-2MB depending on the host). Both ``base64`` (coreutils) + and ``touch -d @`` are present on every Linux base image + we ship, so there's no per-distro branching. + """ + encoded = base64.b64encode(artifact.content) + mtime = int(time.time() + artifact.mtime_offset) + mode_str = oct(artifact.mode)[2:] + parts = [ + f"mkdir -p {shlex.quote(_dirname(artifact.path))}", + f"base64 -d > {shlex.quote(artifact.path)}", + f"chmod {mode_str} {shlex.quote(artifact.path)}", + f"touch -d @{mtime} {shlex.quote(artifact.path)}", + ] + return " && ".join(parts), encoded + + +async def _publish( + bus: Optional[BaseBus], topic: str, payload: dict[str, Any], +) -> None: + """Best-effort publish — never raises. + + When ``bus`` is None we resolve via :func:`get_bus`; either way + bus-side failures are logged and swallowed (delivery is at-most-once + by contract; the DB row is source of truth). + """ + try: + owns_bus = bus is None + target = bus if bus is not None else get_bus() + if owns_bus: + await target.connect() + await target.publish(topic, payload) + if owns_bus: + await target.close() + except Exception as e: # noqa: BLE001 + log.warning("canary bus publish failed topic=%s err=%s", topic, e) + + +async def plant( + decky_name: str, + artifact: CanaryArtifact, + *, + token_uuid: str, + repo: Optional[BaseRepository] = None, + publish: bool = True, + bus: Optional[BaseBus] = None, +) -> tuple[bool, Optional[str]]: + """Write *artifact* into the decky's ssh container. + + Returns ``(success, error_or_none)``. When ``repo`` is provided + the token row's state is updated to ``planted`` / ``failed`` + accordingly. When ``publish`` is True a ``canary..placed`` + event is published on the bus on success. + + The function never raises on docker errors — callers (the API, + the deploy hook) treat the result as data. + """ + if not artifact.path: + err = "planter requires a non-empty artifact.path" + log.warning("canary.plant skipped: %s decky=%s token=%s", err, decky_name, token_uuid) + if repo is not None: + await repo.update_canary_token_state(token_uuid, "failed", err) + return False, err + + sh_cmd, stdin_payload = _build_plant_command(artifact) + # ``-i`` keeps stdin attached so base64 -d inside the container can + # consume the encoded payload streamed from the host. + argv = [_DOCKER, "exec", "-i", _container_for(decky_name), "sh", "-c", sh_cmd] + rc, _stdout, stderr = await _run(argv, stdin_bytes=stdin_payload) + success = rc == 0 + error = None if success else (stderr.strip()[:256] or f"rc={rc}") + + if repo is not None: + if success: + await repo.update_canary_token_state(token_uuid, "planted", None) + else: + await repo.update_canary_token_state(token_uuid, "failed", error) + + if success and publish: + await _publish(bus, topics.canary(token_uuid, topics.CANARY_PLACED), { + "token_id": token_uuid, + "decky_name": decky_name, + "placement_path": artifact.path, + "instrumenter": artifact.instrumenter, + "generator": artifact.generator, + }) + + if not success: + log.warning( + "canary.plant failed decky=%s token=%s rc=%d stderr=%r", + decky_name, token_uuid, rc, stderr[:120], + ) + return success, error + + +async def revoke( + decky_name: str, + placement_path: str, + *, + token_uuid: str, + repo: Optional[BaseRepository] = None, + publish: bool = True, + bus: Optional[BaseBus] = None, +) -> tuple[bool, Optional[str]]: + """Best-effort unlink + state transition + bus publish. + + Returns ``(success, error_or_none)``. ``success`` is True when + the file is gone after the call (whether we deleted it or it was + already missing); only docker / container-down errors return False. + """ + sh_cmd = f"rm -f {shlex.quote(placement_path)}" + argv = [_DOCKER, "exec", _container_for(decky_name), "sh", "-c", sh_cmd] + rc, _stdout, stderr = await _run(argv) + success = rc == 0 + error = None if success else (stderr.strip()[:256] or f"rc={rc}") + + if repo is not None: + await repo.update_canary_token_state(token_uuid, "revoked", error if not success else None) + + if publish: + await _publish(bus, topics.canary(token_uuid, topics.CANARY_REVOKED), { + "token_id": token_uuid, + "decky_name": decky_name, + "placement_path": placement_path, + }) + + return success, error + + +def _baseline_set() -> Iterable[str]: + """Return the configured baseline generator names. + + Honors ``DECNET_CANARY_BASELINE`` (comma-separated). Default is + a sensible mix that exercises every callback-bearing generator + plus a passive aws_creds drop for realism. + """ + raw = os.environ.get( + "DECNET_CANARY_BASELINE", + "git_config,env_file,honeydoc,aws_creds", + ) + return [n.strip() for n in raw.split(",") if n.strip()] + + +def _ctx_for(slug: str) -> CanaryContext: + """Build a :class:`CanaryContext` from the canary worker config.""" + base = os.environ.get("DECNET_CANARY_HTTP_BASE", "http://localhost:8088") + zone = os.environ.get("DECNET_CANARY_DNS_ZONE", "") + return CanaryContext(callback_token=slug, http_base=base, dns_zone=zone) + + +async def seed_baseline( + decky_name: str, + repo: BaseRepository, + *, + persona: str = "linux", + created_by: str = "system", + bus: Optional[BaseBus] = None, +) -> list[dict[str, Any]]: + """Plant the configured baseline canary set on one decky. + + Best-effort: any individual placement that fails is logged and + the row is left in ``state=failed``; the deployer hook treats the + return value as informational, not authoritative. + + Returns the list of token rows created (whether their planting + ultimately succeeded or not), so the caller can surface them in + the deploy report. + """ + out: list[dict[str, Any]] = [] + for gen_name in _baseline_set(): + try: + generator = get_generator(gen_name) + except ValueError: + log.warning("canary.seed_baseline: unknown generator %r — skipping", gen_name) + continue + slug = token_urlsafe(16) + ctx = _ctx_for(slug) + artifact = generator.generate(ctx) + artifact.path = default_path_for(gen_name, persona) + kind = "aws_passive" if gen_name == "aws_creds" else "http" + # Persist first so the planter has a row to update; that way a + # crash mid-plant leaves a recoverable failed-state row. + from uuid import uuid4 + token_uuid = str(uuid4()) + await repo.create_canary_token({ + "uuid": token_uuid, + "kind": kind, + "decky_name": decky_name, + "blob_uuid": None, + "instrumenter": None, + "generator": gen_name, + "placement_path": artifact.path, + "callback_token": slug, + "secret_seed": slug, + "created_by": created_by, + "state": "planted", # optimistic — plant() flips to failed on error + }) + await plant( + decky_name, artifact, + token_uuid=token_uuid, repo=repo, publish=True, bus=bus, + ) + out.append({ + "token_uuid": token_uuid, "generator": gen_name, "kind": kind, + "callback_token": slug, "placement_path": artifact.path, + }) + return out diff --git a/decnet/canary/storage.py b/decnet/canary/storage.py new file mode 100644 index 00000000..06cfbedd --- /dev/null +++ b/decnet/canary/storage.py @@ -0,0 +1,89 @@ +"""Filesystem store for operator-uploaded canary blobs. + +Blobs live under ``/var/lib/decnet/canary/blobs/`` (override +via ``DECNET_CANARY_BLOB_DIR``) and are deduplicated by content hash. +The DB table :class:`decnet.web.db.models.CanaryBlob` mirrors +metadata; the bytes are read on demand at instrumentation time, so +the API process never holds large operator uploads in memory longer +than the request itself. + +Refcount-aware deletion is enforced at the DB layer (see +:meth:`decnet.web.db.repository.BaseRepository.delete_canary_blob`); +this module only provides write/read/unlink primitives keyed by +sha256. +""" +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +from typing import Tuple + + +def blob_dir() -> Path: + """Return the on-disk root for canary blobs. + + Honors ``DECNET_CANARY_BLOB_DIR`` so tests can point at a tmp + path. The directory is created lazily on first write. + """ + raw = os.environ.get("DECNET_CANARY_BLOB_DIR", "/var/lib/decnet/canary/blobs") + return Path(raw) + + +def _path_for(sha256: str) -> Path: + # Two-level fan-out (``ab/cd/abcd...``) keeps any one directory + # from accumulating thousands of entries on busy fleets. Same + # shape as Git's loose-object store. + if len(sha256) < 4: + raise ValueError("sha256 must be at least 4 chars") + root = blob_dir() + return root / sha256[:2] / sha256[2:4] / sha256 + + +def write_blob(content: bytes) -> Tuple[str, Path, int]: + """Persist ``content`` under its sha256 path. + + Idempotent: if the target file already exists with the same + bytes, no rewrite happens. Returns ``(sha256, path, + size_bytes)``. + """ + sha = hashlib.sha256(content).hexdigest() + target = _path_for(sha) + target.parent.mkdir(parents=True, exist_ok=True) + if not target.exists(): + # Atomic-ish: write to a temp sibling and rename. Avoids the + # half-written-file race a concurrent reader would otherwise + # see if we wrote in place. + tmp = target.with_suffix(target.suffix + ".part") + tmp.write_bytes(content) + os.replace(tmp, target) + return sha, target, len(content) + + +def read_blob(sha256: str) -> bytes: + """Read the bytes for a stored blob. + + Raises :class:`FileNotFoundError` when the on-disk row was unlinked + out of band (operator pruned ``/var/lib/decnet`` by hand) — the + caller (instrumenter dispatch) surfaces it as a 410-ish error so + the operator can re-upload. + """ + return _path_for(sha256).read_bytes() + + +def unlink_blob(sha256: str) -> bool: + """Delete the on-disk bytes for ``sha256``. + + Returns True if a file was removed, False if it was already gone. + The DB row deletion happens in + :meth:`SQLModelRepository.delete_canary_blob`; this function is + a best-effort companion called *after* the DB delete commits so + a crash between them leaves a recoverable orphan, never a + dangling DB reference. + """ + target = _path_for(sha256) + try: + target.unlink() + except FileNotFoundError: + return False + return True diff --git a/decnet/canary/worker.py b/decnet/canary/worker.py new file mode 100644 index 00000000..280a717b --- /dev/null +++ b/decnet/canary/worker.py @@ -0,0 +1,254 @@ +"""``decnet canary`` worker — HTTP + DNS callback receivers. + +Two surfaces, one process: + +* **HTTP** — a tiny FastAPI app on its own port (default 8088). The + only useful route is ``GET /c/{slug}`` which looks up the slug in + the canary token table, persists a :class:`CanaryTrigger` row, + publishes ``canary..triggered`` on the bus, and returns + a 1×1 transparent GIF (or 204 if the client's ``Accept`` doesn't + list any image type). +* **DNS** — an authoritative UDP server (default 5353 if non-root, + 53 if root) for ``*.``. Same lookup + persist + + publish flow, plus a sinkhole A record so the attacker's resolver + doesn't loop on NXDOMAIN. + +Both surfaces are **stealth** by policy +(:mod:`feedback_stealth`): no DECNET strings in headers / banners / +error pages. The HTTP app strips the default ``Server: uvicorn`` +header in middleware; FastAPI's docs/openapi UI is disabled because +discovering them would tip off the attacker that this is a honeypot. + +The worker is supervised by its own systemd unit +(``decnet-canary.service``); like every other DECNET worker, it +crashes loudly rather than masking failures. +""" +from __future__ import annotations + +import asyncio +import os +from datetime import datetime, timezone +from typing import Optional + +from fastapi import FastAPI, Request, Response + +from decnet.bus import topics +from decnet.bus.base import BaseBus +from decnet.bus.factory import get_bus +from decnet.canary.dns_server import CanaryDNSProtocol, DNSQuery +from decnet.logging import get_logger +from decnet.web.db.factory import get_repository +from decnet.web.db.repository import BaseRepository + +log = get_logger("canary.worker") + +# 1×1 transparent GIF — public-domain canonical bytes. Returning the +# same image every time is fine: the body has no information the +# attacker shouldn't see, and image clients cache it. +_TRANSPARENT_GIF = bytes.fromhex( + "47494638396101000100800100000000ffffff21f90401000001002c00000000010001000002024401003b" +) + + +def _http_base() -> str: + return os.environ.get("DECNET_CANARY_HTTP_BASE", "http://localhost:8088").rstrip("/") + + +def _dns_zone() -> str: + return os.environ.get("DECNET_CANARY_DNS_ZONE", "").strip(".").lower() + + +def _http_port() -> int: + return int(os.environ.get("DECNET_CANARY_HTTP_PORT", "8088")) + + +def _dns_port() -> int: + # Default 5353 (mDNS-ish, non-privileged) — operators pin :53 via + # NAT or a CAP_NET_BIND_SERVICE-enabled unit. + return int(os.environ.get("DECNET_CANARY_DNS_PORT", "5353")) + + +def _dns_bind() -> str: + return os.environ.get("DECNET_CANARY_DNS_BIND", "0.0.0.0") # nosec B104 — attacker-facing decoy listener, internet exposure is the design + + +def _http_bind() -> str: + return os.environ.get("DECNET_CANARY_HTTP_BIND", "0.0.0.0") # nosec B104 — same rationale + + +# ---------------------------- HTTP surface -------------------------------- + + +def _build_app(repo: BaseRepository, bus: BaseBus) -> FastAPI: + """Construct the FastAPI app. + + Disables docs / openapi / redoc — operators query the canary + surface via the *main* DECNET API, never directly. Anyone hitting + these paths is either misconfigured or scanning for a honeypot. + """ + app = FastAPI( + title="", # don't leak "DECNET" in OpenAPI + docs_url=None, redoc_url=None, openapi_url=None, + ) + + @app.middleware("http") + async def _stealth_headers(request: Request, call_next): + response: Response = await call_next(request) + # Strip the uvicorn / starlette banner; replace with a + # generic Server line that matches what most CDNs return. + response.headers["Server"] = "nginx" + # Don't leak request id / process id headers. + if "x-process-time" in response.headers: + del response.headers["x-process-time"] + return response + + @app.get("/c/{slug}") + async def callback(slug: str, request: Request) -> Response: + await _record_hit( + repo, bus, + slug=slug, + src_ip=_client_ip(request), + user_agent=request.headers.get("user-agent"), + request_path=str(request.url.path), + dns_qname=None, + raw_headers=dict(request.headers), + ) + # Always 200 with a tiny image so the attacker's client sees + # a "success" — same return regardless of whether the slug is + # known. Stealth: do NOT distinguish unknown vs known via + # status code or response body. + return Response(content=_TRANSPARENT_GIF, media_type="image/gif") + + @app.get("/") + async def root() -> Response: + # Bare root returns a generic 404. The decoy posture: pretend + # to be an empty static-file host that just happens to resolve + # /c/ when it matches. + return Response(status_code=404) + + return app + + +def _client_ip(request: Request) -> str: + # Honor X-Forwarded-For if the operator deployed behind a reverse + # proxy. Take the leftmost address in the chain; everything after + # is upstream-proxy noise. + fwd = request.headers.get("x-forwarded-for") + if fwd: + return fwd.split(",", 1)[0].strip() + if request.client: + return request.client.host + return "0.0.0.0" # nosec B104 — sentinel for "unknown remote" + + +# ---------------------------- shared persistence ------------------------- + + +async def _record_hit( + repo: BaseRepository, + bus: BaseBus, + *, + slug: str, + src_ip: str, + user_agent: Optional[str], + request_path: Optional[str], + dns_qname: Optional[str], + raw_headers: Optional[dict], +) -> None: + """Resolve slug -> token, persist a trigger, publish on the bus. + + Unknown slugs are silently swallowed: returning the same response + for known and unknown slugs is the stealth posture, and persisting + every random scan would clutter the DB. + """ + token = await repo.get_canary_token_by_slug(slug) + if token is None: + return + trigger_id = await repo.record_canary_trigger({ + "token_uuid": token["uuid"], + "occurred_at": datetime.now(timezone.utc), + "src_ip": src_ip, + "user_agent": user_agent, + "request_path": request_path, + "dns_qname": dns_qname, + "raw_headers": raw_headers or {}, + }) + try: + await bus.publish( + topics.canary(token["uuid"], topics.CANARY_TRIGGERED), + { + "token_id": token["uuid"], + "trigger_id": trigger_id, + "decky_name": token["decky_name"], + "src_ip": src_ip, + "user_agent": user_agent, + "request_path": request_path, + "dns_qname": dns_qname, + }, + ) + except Exception as e: # noqa: BLE001 — best effort + log.warning("canary.triggered publish failed slug=%s err=%s", slug, e) + + +# ---------------------------- DNS surface -------------------------------- + + +async def _start_dns_server( + repo: BaseRepository, bus: BaseBus, *, loop: asyncio.AbstractEventLoop, +) -> Optional[asyncio.DatagramTransport]: + zone = _dns_zone() + if not zone: + log.info("canary.dns disabled (DECNET_CANARY_DNS_ZONE unset)") + return None + + async def _hook(slug: str, query: DNSQuery, src_ip: str) -> None: + await _record_hit( + repo, bus, + slug=slug, src_ip=src_ip, user_agent=None, + request_path=None, dns_qname=query.qname, + raw_headers=None, + ) + + transport, _proto = await loop.create_datagram_endpoint( + lambda: CanaryDNSProtocol(zone, _hook), + local_addr=(_dns_bind(), _dns_port()), + ) + log.info("canary.dns listening zone=%s port=%d", zone, _dns_port()) + return transport # type: ignore[return-value] + + +# ---------------------------- entry point -------------------------------- + + +async def run() -> None: + """Worker entry point — kicked off by ``decnet canary``.""" + import uvicorn + + repo = get_repository() + await repo.initialize() + bus = get_bus() + await bus.connect() + + app = _build_app(repo, bus) + config = uvicorn.Config( + app, + host=_http_bind(), + port=_http_port(), + log_level="warning", + access_log=False, # stealth: no per-request lines + server_header=False, # we set Server: nginx in middleware + ) + server = uvicorn.Server(config) + loop = asyncio.get_running_loop() + dns_transport = await _start_dns_server(repo, bus, loop=loop) + try: + await server.serve() + finally: + if dns_transport is not None: + dns_transport.close() + await bus.close() + + +def main() -> None: + """CLI entry point — synchronous wrapper for ``asyncio.run``.""" + asyncio.run(run()) diff --git a/decnet/cli.py b/decnet/cli.py deleted file mode 100644 index 91415e56..00000000 --- a/decnet/cli.py +++ /dev/null @@ -1,478 +0,0 @@ -""" -DECNET CLI — entry point for all commands. - -Usage: - decnet deploy --mode unihost --deckies 5 --randomize-services - decnet status - decnet teardown [--all | --id decky-01] - decnet services -""" - -import signal -from typing import Optional - -import typer -from rich.console import Console -from rich.table import Table - -from decnet.env import ( - DECNET_API_HOST, - DECNET_API_PORT, - DECNET_INGEST_LOG_FILE, - DECNET_WEB_HOST, - DECNET_WEB_PORT, -) -from decnet.archetypes import Archetype, all_archetypes, get_archetype -from decnet.config import ( - DecnetConfig, -) -from decnet.distros import all_distros, get_distro -from decnet.fleet import all_service_names, build_deckies, build_deckies_from_ini -from decnet.ini_loader import load_ini -from decnet.network import detect_interface, detect_subnet, allocate_ips, get_host_ip -from decnet.services.registry import all_services - -app = typer.Typer( - name="decnet", - help="Deploy a deception network of honeypot deckies on your LAN.", - no_args_is_help=True, -) -console = Console() - - -def _kill_api() -> None: - """Find and kill any running DECNET API (uvicorn) or mutator processes.""" - import psutil - import os - - _killed: bool = False - for _proc in psutil.process_iter(['pid', 'name', 'cmdline']): - try: - _cmd = _proc.info['cmdline'] - if not _cmd: - continue - if "uvicorn" in _cmd and "decnet.web.api:app" in _cmd: - console.print(f"[yellow]Stopping DECNET API (PID {_proc.info['pid']})...[/]") - os.kill(_proc.info['pid'], signal.SIGTERM) - _killed = True - elif "decnet.cli" in _cmd and "mutate" in _cmd and "--watch" in _cmd: - console.print(f"[yellow]Stopping DECNET Mutator Watcher (PID {_proc.info['pid']})...[/]") - os.kill(_proc.info['pid'], signal.SIGTERM) - _killed = True - except (psutil.NoSuchProcess, psutil.AccessDenied): - continue - - if _killed: - console.print("[green]Background processes stopped.[/]") - - -@app.command() -def api( - port: int = typer.Option(DECNET_API_PORT, "--port", help="Port for the backend API"), - host: str = typer.Option(DECNET_API_HOST, "--host", help="Host IP for the backend API"), - log_file: str = typer.Option(DECNET_INGEST_LOG_FILE, "--log-file", help="Path to the DECNET log file to monitor"), -) -> None: - """Run the DECNET API and Web Dashboard in standalone mode.""" - import subprocess # nosec B404 - import sys - import os - - console.print(f"[green]Starting DECNET API on {host}:{port}...[/]") - _env: dict[str, str] = os.environ.copy() - _env["DECNET_INGEST_LOG_FILE"] = str(log_file) - try: - subprocess.run( # nosec B603 B404 - [sys.executable, "-m", "uvicorn", "decnet.web.api:app", "--host", host, "--port", str(port)], - env=_env - ) - except KeyboardInterrupt: - pass - except (FileNotFoundError, subprocess.SubprocessError): - console.print("[red]Failed to start API. Ensure 'uvicorn' is installed in the current environment.[/]") - - -@app.command() -def deploy( - mode: str = typer.Option("unihost", "--mode", "-m", help="Deployment mode: unihost | swarm"), - deckies: Optional[int] = typer.Option(None, "--deckies", "-n", help="Number of deckies to deploy (required without --config)", min=1), - interface: Optional[str] = typer.Option(None, "--interface", "-i", help="Host NIC (auto-detected if omitted)"), - subnet: Optional[str] = typer.Option(None, "--subnet", help="LAN subnet CIDR (auto-detected if omitted)"), - ip_start: Optional[str] = typer.Option(None, "--ip-start", help="First decky IP (auto if omitted)"), - services: Optional[str] = typer.Option(None, "--services", help="Comma-separated services, e.g. ssh,smb,rdp"), - randomize_services: bool = typer.Option(False, "--randomize-services", help="Assign random services to each decky"), - distro: Optional[str] = typer.Option(None, "--distro", help="Comma-separated distro slugs, e.g. debian,ubuntu22,rocky9"), - randomize_distros: bool = typer.Option(False, "--randomize-distros", help="Assign a random distro to each decky"), - log_file: Optional[str] = typer.Option(DECNET_INGEST_LOG_FILE, "--log-file", help="Host path for the collector to write RFC 5424 logs (e.g. /var/log/decnet/decnet.log)"), - archetype_name: Optional[str] = typer.Option(None, "--archetype", "-a", help="Machine archetype slug (e.g. linux-server, windows-workstation)"), - mutate_interval: Optional[int] = typer.Option(30, "--mutate-interval", help="Automatically rotate services every N minutes"), - dry_run: bool = typer.Option(False, "--dry-run", help="Generate compose file without starting containers"), - no_cache: bool = typer.Option(False, "--no-cache", help="Force rebuild all images, ignoring Docker layer cache"), - parallel: bool = typer.Option(False, "--parallel", help="Build all images concurrently (enables BuildKit, separates build from up)"), - ipvlan: bool = typer.Option(False, "--ipvlan", help="Use IPvlan L2 instead of MACVLAN (required on WiFi interfaces)"), - config_file: Optional[str] = typer.Option(None, "--config", "-c", help="Path to INI config file"), - api: bool = typer.Option(False, "--api", help="Start the FastAPI backend to ingest and serve logs"), - api_port: int = typer.Option(8000, "--api-port", help="Port for the backend API"), -) -> None: - """Deploy deckies to the LAN.""" - import os - if mode not in ("unihost", "swarm"): - console.print("[red]--mode must be 'unihost' or 'swarm'[/]") - raise typer.Exit(1) - - # ------------------------------------------------------------------ # - # Config-file path # - # ------------------------------------------------------------------ # - if config_file: - try: - ini = load_ini(config_file) - except FileNotFoundError as e: - console.print(f"[red]{e}[/]") - raise typer.Exit(1) - - iface = interface or ini.interface or detect_interface() - subnet_cidr = subnet or ini.subnet - effective_gateway = ini.gateway - if subnet_cidr is None: - subnet_cidr, effective_gateway = detect_subnet(iface) - elif effective_gateway is None: - _, effective_gateway = detect_subnet(iface) - - host_ip = get_host_ip(iface) - console.print(f"[dim]Config:[/] {config_file} [dim]Interface:[/] {iface} " - f"[dim]Subnet:[/] {subnet_cidr} [dim]Gateway:[/] {effective_gateway} " - f"[dim]Host IP:[/] {host_ip}") - - if ini.custom_services: - from decnet.custom_service import CustomService - from decnet.services.registry import register_custom_service - for cs in ini.custom_services: - register_custom_service( - CustomService( - name=cs.name, - image=cs.image, - exec_cmd=cs.exec_cmd, - ports=cs.ports, - ) - ) - - effective_log_file = log_file - try: - decky_configs = build_deckies_from_ini( - ini, subnet_cidr, effective_gateway, host_ip, randomize_services, cli_mutate_interval=mutate_interval - ) - except ValueError as e: - console.print(f"[red]{e}[/]") - raise typer.Exit(1) - # ------------------------------------------------------------------ # - # Classic CLI path # - # ------------------------------------------------------------------ # - else: - if deckies is None: - console.print("[red]--deckies is required when --config is not used.[/]") - raise typer.Exit(1) - - services_list = [s.strip() for s in services.split(",")] if services else None - if services_list: - known = set(all_service_names()) - unknown = [s for s in services_list if s not in known] - if unknown: - console.print(f"[red]Unknown service(s): {unknown}. Available: {all_service_names()}[/]") - raise typer.Exit(1) - - arch: Archetype | None = None - if archetype_name: - try: - arch = get_archetype(archetype_name) - except ValueError as e: - console.print(f"[red]{e}[/]") - raise typer.Exit(1) - - if not services_list and not randomize_services and not arch: - console.print("[red]Specify --services, --archetype, or --randomize-services.[/]") - raise typer.Exit(1) - - iface = interface or detect_interface() - if subnet is None: - subnet_cidr, effective_gateway = detect_subnet(iface) - else: - subnet_cidr = subnet - _, effective_gateway = detect_subnet(iface) - - host_ip = get_host_ip(iface) - console.print(f"[dim]Interface:[/] {iface} [dim]Subnet:[/] {subnet_cidr} " - f"[dim]Gateway:[/] {effective_gateway} [dim]Host IP:[/] {host_ip}") - - distros_list = [d.strip() for d in distro.split(",")] if distro else None - if distros_list: - try: - for slug in distros_list: - get_distro(slug) - except ValueError as e: - console.print(f"[red]{e}[/]") - raise typer.Exit(1) - - ips = allocate_ips(subnet_cidr, effective_gateway, host_ip, deckies, ip_start) - decky_configs = build_deckies( - deckies, ips, services_list, randomize_services, - distros_explicit=distros_list, randomize_distros=randomize_distros, - archetype=arch, mutate_interval=mutate_interval, - ) - effective_log_file = log_file - - if api and not effective_log_file: - effective_log_file = os.path.join(os.getcwd(), "decnet.log") - console.print(f"[cyan]API mode enabled: defaulting log-file to {effective_log_file}[/]") - - config = DecnetConfig( - mode=mode, - interface=iface, - subnet=subnet_cidr, - gateway=effective_gateway, - deckies=decky_configs, - log_file=effective_log_file, - ipvlan=ipvlan, - mutate_interval=mutate_interval, - ) - - from decnet.engine import deploy as _deploy - _deploy(config, dry_run=dry_run, no_cache=no_cache, parallel=parallel) - - if mutate_interval is not None and not dry_run: - import subprocess # nosec B404 - import sys - console.print(f"[green]Starting DECNET Mutator watcher in the background (interval: {mutate_interval}m)...[/]") - try: - subprocess.Popen( # nosec B603 - [sys.executable, "-m", "decnet.cli", "mutate", "--watch"], - stdout=subprocess.DEVNULL, - stderr=subprocess.STDOUT, - start_new_session=True, - ) - except (FileNotFoundError, subprocess.SubprocessError): - console.print("[red]Failed to start mutator watcher.[/]") - - if effective_log_file and not dry_run and not api: - import subprocess # nosec B404 - import sys - from pathlib import Path as _Path - _collector_err = _Path(effective_log_file).with_suffix(".collector.log") - console.print(f"[bold cyan]Starting log collector[/] → {effective_log_file}") - subprocess.Popen( # nosec B603 - [sys.executable, "-m", "decnet.cli", "collect", "--log-file", str(effective_log_file)], - stdin=subprocess.DEVNULL, - stdout=open(_collector_err, "a"), # nosec B603 - stderr=subprocess.STDOUT, - start_new_session=True, - ) - - if api and not dry_run: - import subprocess # nosec B404 - import sys - console.print(f"[green]Starting DECNET API on port {api_port}...[/]") - _env: dict[str, str] = os.environ.copy() - _env["DECNET_INGEST_LOG_FILE"] = str(effective_log_file or "") - try: - subprocess.Popen( # nosec B603 - [sys.executable, "-m", "uvicorn", "decnet.web.api:app", "--host", DECNET_API_HOST, "--port", str(api_port)], - env=_env, - stdout=subprocess.DEVNULL, - stderr=subprocess.STDOUT - ) - console.print(f"[dim]API running at http://{DECNET_API_HOST}:{api_port}[/]") - except (FileNotFoundError, subprocess.SubprocessError): - console.print("[red]Failed to start API. Ensure 'uvicorn' is installed in the current environment.[/]") - - -@app.command() -def collect( - log_file: str = typer.Option(DECNET_INGEST_LOG_FILE, "--log-file", "-f", help="Path to write RFC 5424 syslog lines and .json records"), -) -> None: - """Stream Docker logs from all running decky service containers to a log file.""" - import asyncio - from decnet.collector import log_collector_worker - console.print(f"[bold cyan]Collector starting[/] → {log_file}") - asyncio.run(log_collector_worker(log_file)) - - -@app.command() -def mutate( - watch: bool = typer.Option(False, "--watch", "-w", help="Run continuously and mutate deckies according to their interval"), - decky_name: Optional[str] = typer.Option(None, "--decky", "-d", help="Force mutate a specific decky immediately"), - force_all: bool = typer.Option(False, "--all", help="Force mutate all deckies immediately"), -) -> None: - """Manually trigger or continuously watch for decky mutation.""" - import asyncio - from decnet.mutator import mutate_decky, mutate_all, run_watch_loop - from decnet.web.dependencies import repo - - async def _run() -> None: - await repo.initialize() - if watch: - await run_watch_loop(repo) - elif decky_name: - await mutate_decky(decky_name, repo) - elif force_all: - await mutate_all(force=True, repo=repo) - else: - await mutate_all(force=False, repo=repo) - - asyncio.run(_run()) - - -@app.command() -def status() -> None: - """Show running deckies and their status.""" - from decnet.engine import status as _status - _status() - - -@app.command() -def teardown( - all_: bool = typer.Option(False, "--all", help="Tear down all deckies and remove network"), - id_: Optional[str] = typer.Option(None, "--id", help="Tear down a specific decky by name"), -) -> None: - """Stop and remove deckies.""" - if not all_ and not id_: - console.print("[red]Specify --all or --id .[/]") - raise typer.Exit(1) - - from decnet.engine import teardown as _teardown - _teardown(decky_id=id_) - - if all_: - _kill_api() - - -@app.command(name="services") -def list_services() -> None: - """List all registered honeypot service plugins.""" - svcs = all_services() - table = Table(title="Available Services", show_lines=True) - table.add_column("Name", style="bold cyan") - table.add_column("Ports") - table.add_column("Image") - for name, svc in sorted(svcs.items()): - table.add_row(name, ", ".join(str(p) for p in svc.ports), svc.default_image) - console.print(table) - - -@app.command(name="distros") -def list_distros() -> None: - """List all available OS distro profiles for deckies.""" - table = Table(title="Available Distro Profiles", show_lines=True) - table.add_column("Slug", style="bold cyan") - table.add_column("Display Name") - table.add_column("Docker Image", style="dim") - for slug, profile in sorted(all_distros().items()): - table.add_row(slug, profile.display_name, profile.image) - console.print(table) - - -@app.command(name="correlate") -def correlate( - log_file: Optional[str] = typer.Option(None, "--log-file", "-f", help="Path to DECNET syslog file to analyse"), - min_deckies: int = typer.Option(2, "--min-deckies", "-m", help="Minimum number of distinct deckies an IP must touch to be reported"), - output: str = typer.Option("table", "--output", "-o", help="Output format: table | json | syslog"), - emit_syslog: bool = typer.Option(False, "--emit-syslog", help="Also print traversal events as RFC 5424 lines (for SIEM piping)"), -) -> None: - """Analyse logs for cross-decky traversals and print the attacker movement graph.""" - import sys - import json as _json - from pathlib import Path - from decnet.correlation.engine import CorrelationEngine - - engine = CorrelationEngine() - - if log_file: - path = Path(log_file) - if not path.exists(): - console.print(f"[red]Log file not found: {log_file}[/]") - raise typer.Exit(1) - engine.ingest_file(path) - elif not sys.stdin.isatty(): - for line in sys.stdin: - engine.ingest(line) - else: - console.print("[red]Provide --log-file or pipe log data via stdin.[/]") - raise typer.Exit(1) - - traversals = engine.traversals(min_deckies) - - if output == "json": - console.print_json(_json.dumps(engine.report_json(min_deckies), indent=2)) - elif output == "syslog": - for line in engine.traversal_syslog_lines(min_deckies): - typer.echo(line) - else: - if not traversals: - console.print( - f"[yellow]No traversals detected " - f"(min_deckies={min_deckies}, events_indexed={engine.events_indexed}).[/]" - ) - else: - console.print(engine.report_table(min_deckies)) - console.print( - f"[dim]Parsed {engine.lines_parsed} lines · " - f"indexed {engine.events_indexed} events · " - f"{len(engine.all_attackers())} unique IPs · " - f"[bold]{len(traversals)}[/] traversal(s)[/]" - ) - - if emit_syslog: - for line in engine.traversal_syslog_lines(min_deckies): - typer.echo(line) - - -@app.command(name="archetypes") -def list_archetypes() -> None: - """List all machine archetype profiles.""" - table = Table(title="Machine Archetypes", show_lines=True) - table.add_column("Slug", style="bold cyan") - table.add_column("Display Name") - table.add_column("Default Services", style="green") - table.add_column("Description", style="dim") - for slug, arch in sorted(all_archetypes().items()): - table.add_row( - slug, - arch.display_name, - ", ".join(arch.services), - arch.description, - ) - console.print(table) - - -@app.command(name="web") -def serve_web( - web_port: int = typer.Option(DECNET_WEB_PORT, "--web-port", help="Port to serve the DECNET Web Dashboard"), - host: str = typer.Option(DECNET_WEB_HOST, "--host", help="Host IP to serve the Web Dashboard"), -) -> None: - """Serve the DECNET Web Dashboard frontend.""" - import http.server - import socketserver - from pathlib import Path - - dist_dir = Path(__file__).parent.parent / "decnet_web" / "dist" - - if not dist_dir.exists(): - console.print(f"[red]Frontend build not found at {dist_dir}. Make sure you run 'npm run build' inside 'decnet_web'.[/]") - raise typer.Exit(1) - - class SPAHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): - def do_GET(self): - path = self.translate_path(self.path) - if not Path(path).exists() or Path(path).is_dir(): - self.path = "/index.html" - return super().do_GET() - - import os - os.chdir(dist_dir) - - with socketserver.TCPServer((host, web_port), SPAHTTPRequestHandler) as httpd: - console.print(f"[green]Serving DECNET Web Dashboard on http://{host}:{web_port}[/]") - try: - httpd.serve_forever() - except KeyboardInterrupt: - console.print("\n[dim]Shutting down dashboard server.[/]") - -if __name__ == '__main__': # pragma: no cover - app() diff --git a/decnet/cli/__init__.py b/decnet/cli/__init__.py new file mode 100644 index 00000000..f0508319 --- /dev/null +++ b/decnet/cli/__init__.py @@ -0,0 +1,90 @@ +""" +DECNET CLI — entry point for all commands. + +Usage: + decnet deploy --mode unihost --deckies 5 --randomize-services + decnet status + decnet teardown [--all | --id decky-01] + decnet services + +Layout: each command module exports ``register(app)`` which attaches its +commands to the passed Typer app. ``__init__.py`` builds the root app, +calls every module's ``register`` in order, then runs the master-only +gate. The gate must fire LAST so it sees the fully-populated dispatch +table before filtering. +""" + +from __future__ import annotations + +import typer + +from . import ( + agent, + api, + bus, + canary, + db, + deploy, + forwarder, + geoip, + init, + inventory, + lifecycle, + listener, + orchestrator, + profiler, + realism, + reconciler, + sniffer, + swarm, + swarmctl, + topology, + updater, + web, + webhook, + workers, +) +from .gating import _gate_commands_by_mode +from .utils import console as console, log as log + +app = typer.Typer( + name="decnet", + help="Deploy a deception network of honeypot deckies on your LAN.", + no_args_is_help=True, +) + +# Order matches the old flat layout so `decnet --help` reads the same. +for _mod in ( + api, swarmctl, agent, updater, listener, forwarder, + swarm, + deploy, lifecycle, workers, inventory, + web, profiler, orchestrator, realism, reconciler, sniffer, db, + topology, bus, geoip, init, webhook, canary, +): + _mod.register(app) + +_gate_commands_by_mode(app) + +# Backwards-compat re-exports. Tests and third-party tooling import these +# directly from ``decnet.cli``; the refactor must keep them resolvable. +from .db import _db_reset_mysql_async # noqa: E402,F401 +from .gating import ( # noqa: E402,F401 + MASTER_ONLY_COMMANDS, + MASTER_ONLY_GROUPS, + _agent_mode_active, + _require_master_mode, +) +from .utils import ( # noqa: E402,F401 + _daemonize, + _http_request, + _is_running, + _kill_all_services, + _pid_dir, + _service_registry, + _spawn_detached, + _swarmctl_base_url, +) + + +if __name__ == "__main__": # pragma: no cover + app() diff --git a/decnet/cli/agent.py b/decnet/cli/agent.py new file mode 100644 index 00000000..5a04d5a6 --- /dev/null +++ b/decnet/cli/agent.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import os +import pathlib as _pathlib +import sys as _sys +from typing import Optional + +import typer + +from . import utils as _utils +from .utils import console, log + + +def register(app: typer.Typer) -> None: + @app.command() + def agent( + port: int = typer.Option(8765, "--port", help="Port for the worker agent"), + host: str = typer.Option("0.0.0.0", "--host", help="Bind address for the worker agent"), # nosec B104 + agent_dir: Optional[str] = typer.Option(None, "--agent-dir", help="Worker cert bundle dir (default: ~/.decnet/agent, expanded under the running user's HOME — set this when running as sudo/root)"), + daemon: bool = typer.Option(False, "--daemon", "-d", help="Detach to background as a daemon process"), + no_forwarder: bool = typer.Option(False, "--no-forwarder", help="Do not auto-spawn the log forwarder alongside the agent"), + ) -> None: + """Run the DECNET SWARM worker agent (requires a cert bundle in ~/.decnet/agent/). + + By default, `decnet agent` auto-spawns `decnet forwarder` as a fully- + detached sibling process so worker logs start flowing to the master + without a second manual invocation. The forwarder survives agent + restarts and crashes — if it dies on its own, restart it manually + with `decnet forwarder --daemon …`. Pass --no-forwarder to skip. + """ + from decnet.agent import server as _agent_server + from decnet.env import DECNET_SWARM_MASTER_HOST, DECNET_AGENT_LOG_FILE + from decnet.swarm import pki as _pki + + resolved_dir = _pathlib.Path(agent_dir) if agent_dir else _pki.DEFAULT_AGENT_DIR + + if daemon: + log.info("agent daemonizing host=%s port=%d", host, port) + _utils._daemonize() + + if not no_forwarder and DECNET_SWARM_MASTER_HOST: + fw_argv = [ + _sys.executable, "-m", "decnet", "forwarder", + "--master-host", DECNET_SWARM_MASTER_HOST, + "--master-port", str(int(os.environ.get("DECNET_SWARM_SYSLOG_PORT", "6514"))), + "--agent-dir", str(resolved_dir), + "--log-file", str(DECNET_AGENT_LOG_FILE), + "--daemon", + ] + try: + pid = _utils._spawn_detached(fw_argv, _utils._pid_dir() / "forwarder.pid") + log.info("agent auto-spawned forwarder pid=%d master=%s", pid, DECNET_SWARM_MASTER_HOST) + console.print(f"[dim]Auto-spawned forwarder (pid {pid}) → {DECNET_SWARM_MASTER_HOST}.[/]") + except Exception as e: # noqa: BLE001 + log.warning("agent could not auto-spawn forwarder: %s", e) + console.print(f"[yellow]forwarder auto-spawn skipped: {e}[/]") + elif not no_forwarder: + log.info("agent skipping forwarder auto-spawn (DECNET_SWARM_MASTER_HOST unset)") + + log.info("agent command invoked host=%s port=%d dir=%s", host, port, resolved_dir) + console.print(f"[green]Starting DECNET worker agent on {host}:{port} (mTLS)...[/]") + rc = _agent_server.run(host, port, agent_dir=resolved_dir) + if rc != 0: + raise typer.Exit(rc) diff --git a/decnet/cli/api.py b/decnet/cli/api.py new file mode 100644 index 00000000..80f88cdb --- /dev/null +++ b/decnet/cli/api.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import os +import signal +import subprocess # nosec B404 +import sys + +import typer + +from decnet.env import DECNET_API_HOST, DECNET_API_PORT, DECNET_INGEST_LOG_FILE + +from . import utils as _utils +from .gating import _require_master_mode +from .utils import console, log + + +def register(app: typer.Typer) -> None: + @app.command() + def api( + port: int = typer.Option(DECNET_API_PORT, "--port", help="Port for the backend API"), + host: str = typer.Option(DECNET_API_HOST, "--host", help="Host IP for the backend API"), + log_file: str = typer.Option(DECNET_INGEST_LOG_FILE, "--log-file", help="Path to the DECNET log file to monitor"), + daemon: bool = typer.Option(False, "--daemon", "-d", help="Detach to background as a daemon process"), + workers: int = typer.Option(1, "--workers", "-w", min=1, help="Number of uvicorn worker processes"), + ) -> None: + """Run the DECNET API and Web Dashboard in standalone mode.""" + _require_master_mode("api") + if daemon: + log.info("API daemonizing host=%s port=%d workers=%d", host, port, workers) + _utils._daemonize() + + log.info("API command invoked host=%s port=%d workers=%d", host, port, workers) + console.print(f"[green]Starting DECNET API on {host}:{port} (workers={workers})...[/]") + _env: dict[str, str] = os.environ.copy() + _env["DECNET_INGEST_LOG_FILE"] = str(log_file) + _cmd = [sys.executable, "-m", "uvicorn", "decnet.web.api:app", + "--host", host, "--port", str(port), "--workers", str(workers)] + try: + proc = subprocess.Popen(_cmd, env=_env, start_new_session=True) # nosec B603 B404 + try: + proc.wait() + except KeyboardInterrupt: + try: + os.killpg(proc.pid, signal.SIGTERM) + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + except ProcessLookupError: + pass + except (FileNotFoundError, subprocess.SubprocessError): + console.print("[red]Failed to start API. Ensure 'uvicorn' is installed in the current environment.[/]") diff --git a/decnet/cli/bus.py b/decnet/cli/bus.py new file mode 100644 index 00000000..5a29dd91 --- /dev/null +++ b/decnet/cli/bus.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import typer + +from . import utils as _utils +from .utils import console, log + + +def register(app: typer.Typer) -> None: + @app.command(name="bus") + def bus_cmd( + socket_path: str = typer.Option( + None, "--socket", "-s", + help="UNIX socket path (defaults to DECNET_BUS_SOCKET env var, " + "then /run/decnet/bus.sock, then ~/.decnet/bus.sock).", + ), + group: str = typer.Option( + "decnet", "--group", "-g", + help="POSIX group to chown the socket to (falls back to process " + "group if the named group does not exist).", + ), + heartbeat: int = typer.Option( + 10, "--heartbeat", "-H", + help="Seconds between system.bus.health heartbeat events.", + ), + daemon: bool = typer.Option(False, "--daemon", "-d", help="Detach to background as a daemon process."), + ) -> None: + """Run the DECNET ServiceBus worker (host-local UNIX-socket pub/sub).""" + import asyncio + from decnet.bus.factory import _default_socket_path + from decnet.bus.worker import bus_worker + + resolved = socket_path or _default_socket_path() + + if daemon: + log.info("bus daemonizing socket=%s", resolved) + _utils._daemonize() + + log.info("bus starting socket=%s group=%s heartbeat=%ds", resolved, group, heartbeat) + console.print(f"[bold cyan]Bus starting[/] (socket: {resolved}, heartbeat: {heartbeat}s)") + + try: + asyncio.run(bus_worker(resolved, group=group, heartbeat_interval=heartbeat)) + except KeyboardInterrupt: + console.print("\n[yellow]Bus stopped.[/]") diff --git a/decnet/cli/canary.py b/decnet/cli/canary.py new file mode 100644 index 00000000..87af60ea --- /dev/null +++ b/decnet/cli/canary.py @@ -0,0 +1,42 @@ +"""``decnet canary`` — HTTP + DNS callback receiver for canary tokens. + +Worker process. Mirrors the shape of :mod:`decnet.cli.webhook`: a +``@app.command(name="canary")`` Typer entry point that delegates to +:func:`decnet.canary.worker.run`. + +Not master-only — any host that hosts deckies can run its own +canary worker (the bus events stay local; the webhook worker on +each host fans them out to SIEMs independently per the design +in ``development/let-s-move-to-the-enumerated-pike.md``). +""" +from __future__ import annotations + +import typer + +from . import utils as _utils +from .utils import console, log + + +def register(app: typer.Typer) -> None: + @app.command(name="canary") + def canary_cmd( + daemon: bool = typer.Option( + False, "--daemon", "-d", help="Detach to background as a daemon process", + ), + ) -> None: + """Run the canary HTTP + DNS callback receiver.""" + import asyncio + + from decnet.canary.worker import run + + if daemon: + log.info("canary daemonizing") + _utils._daemonize() + + log.info("canary starting") + console.print("[bold cyan]Canary callback receiver starting[/]") + + try: + asyncio.run(run()) + except KeyboardInterrupt: + console.print("\n[yellow]Canary worker stopped.[/]") diff --git a/decnet/cli/db.py b/decnet/cli/db.py new file mode 100644 index 00000000..86193967 --- /dev/null +++ b/decnet/cli/db.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +from typing import Optional + +import typer +from rich.table import Table + +from .utils import console, log + + +def _decnet_tables() -> tuple[str, ...]: + """Every DECNET-managed table, ordered child-first for DROP safety. + + Source is ``SQLModel.metadata.sorted_tables`` — the same registry that + drives ``create_all`` — so adding a new model automatically enrolls + its table in ``db-reset`` with no manual step. (Previous hardcoded + list drifted multiple times; ``webhook_subscriptions`` / + ``session_profile`` / ``smtp_targets`` all got missed.) + + ``sorted_tables`` returns parent-first (topological order that makes + ``CREATE`` safe). For ``DROP`` we need the reverse: children first, + so FK constraints drop before their parents. ``SET FOREIGN_KEY_CHECKS + = 0`` below makes this order-insensitive for MySQL, but the reverse + order keeps the code honest for any backend that doesn't support + disabling the FK check. + """ + from sqlmodel import SQLModel + # Importing the models package registers every table on SQLModel.metadata. + import decnet.web.db.models # noqa: F401 + + return tuple( + t.name for t in reversed(SQLModel.metadata.sorted_tables) + ) + + +async def _db_reset_mysql_async(dsn: str, mode: str, confirm: bool) -> None: + """Inspect + (optionally) wipe a MySQL database. Pulled out of the CLI + wrapper so tests can drive it without spawning a Typer runner.""" + from urllib.parse import urlparse + from sqlalchemy import text + from sqlalchemy.ext.asyncio import create_async_engine + + db_name = urlparse(dsn).path.lstrip("/") or "(default)" + engine = create_async_engine(dsn) + tables = _decnet_tables() + try: + rows: dict[str, int] = {} + async with engine.connect() as conn: + for tbl in tables: + try: + result = await conn.execute(text(f"SELECT COUNT(*) FROM `{tbl}`")) # nosec B608 + rows[tbl] = result.scalar() or 0 + except Exception: # noqa: BLE001 — ProgrammingError for missing table varies by driver + rows[tbl] = -1 + + summary = Table(title=f"DECNET MySQL reset — database `{db_name}` (mode={mode})") + summary.add_column("Table", style="cyan") + summary.add_column("Rows", justify="right") + for tbl, count in rows.items(): + summary.add_row(tbl, "[dim]missing[/]" if count < 0 else f"{count:,}") + console.print(summary) + + if not confirm: + console.print( + "[yellow]Dry-run only. Re-run with [bold]--i-know-what-im-doing[/] " + "to actually execute.[/]" + ) + return + + async with engine.begin() as conn: + await conn.execute(text("SET FOREIGN_KEY_CHECKS = 0")) + for tbl in tables: + if rows.get(tbl, -1) < 0: + continue + if mode == "truncate": + await conn.execute(text(f"TRUNCATE TABLE `{tbl}`")) + console.print(f"[green]✓ TRUNCATE {tbl}[/]") + else: + await conn.execute(text(f"DROP TABLE `{tbl}`")) + console.print(f"[green]✓ DROP TABLE {tbl}[/]") + await conn.execute(text("SET FOREIGN_KEY_CHECKS = 1")) + + console.print(f"[bold green]Done. Database `{db_name}` reset ({mode}).[/]") + finally: + await engine.dispose() + + +def register(app: typer.Typer) -> None: + @app.command(name="db-reset") + def db_reset( + i_know: bool = typer.Option( + False, + "--i-know-what-im-doing", + help="Required to actually execute. Without it, the command runs in dry-run mode.", + ), + mode: str = typer.Option( + "truncate", + "--mode", + help="truncate (wipe rows, keep schema) | drop-tables (DROP TABLE for each DECNET table)", + ), + url: Optional[str] = typer.Option( + None, + "--url", + help="Override DECNET_DB_URL for this invocation (e.g. when cleanup needs admin creds).", + ), + ) -> None: + """Wipe the MySQL database used by the DECNET dashboard. + + Destructive. Runs dry by default — pass --i-know-what-im-doing to commit. + Only supported against MySQL; refuses to operate on SQLite. + """ + import asyncio + import os + + if mode not in ("truncate", "drop-tables"): + console.print(f"[red]Invalid --mode '{mode}'. Expected: truncate | drop-tables.[/]") + raise typer.Exit(2) + + db_type = os.environ.get("DECNET_DB_TYPE", "sqlite").lower() + if db_type != "mysql": + console.print( + f"[red]db-reset is MySQL-only (DECNET_DB_TYPE='{db_type}'). " + f"For SQLite, just delete the decnet.db file.[/]" + ) + raise typer.Exit(2) + + dsn = url or os.environ.get("DECNET_DB_URL") + if not dsn: + from decnet.web.db.mysql.database import build_mysql_url + try: + dsn = build_mysql_url() + except ValueError as e: + console.print(f"[red]{e}[/]") + raise typer.Exit(2) from e + + log.info("db-reset invoked mode=%s confirm=%s", mode, i_know) + try: + asyncio.run(_db_reset_mysql_async(dsn, mode=mode, confirm=i_know)) + except Exception as e: # noqa: BLE001 + console.print(f"[red]db-reset failed: {e}[/]") + raise typer.Exit(1) from e diff --git a/decnet/cli/deploy.py b/decnet/cli/deploy.py new file mode 100644 index 00000000..1ec48b05 --- /dev/null +++ b/decnet/cli/deploy.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +from typing import Optional + +import typer +from rich.table import Table + +from decnet.archetypes import Archetype, get_archetype +from decnet.config import DecnetConfig +from decnet.distros import get_distro +from decnet.env import DECNET_API_HOST, DECNET_INGEST_LOG_FILE +from decnet.fleet import all_service_names, build_deckies, build_deckies_from_ini +from decnet.ini_loader import load_ini +from decnet.network import detect_interface, detect_subnet, allocate_ips, get_host_ip + +from . import utils as _utils +from .gating import _require_master_mode +from .utils import console, log + + +def _deploy_swarm(config: "DecnetConfig", *, dry_run: bool, no_cache: bool) -> None: + """Shard deckies round-robin across enrolled workers and POST to swarmctl.""" + base = _utils._swarmctl_base_url(None) + resp = _utils._http_request("GET", base + "/swarm/hosts?host_status=enrolled") + enrolled = resp.json() + resp2 = _utils._http_request("GET", base + "/swarm/hosts?host_status=active") + active = resp2.json() + workers = [*enrolled, *active] + if not workers: + console.print("[red]No enrolled workers — run `decnet swarm enroll ...` first.[/]") + raise typer.Exit(1) + + assigned: list = [] + for idx, d in enumerate(config.deckies): + target = workers[idx % len(workers)] + assigned.append(d.model_copy(update={"host_uuid": target["uuid"]})) + config = config.model_copy(update={"deckies": assigned}) + + body = {"config": config.model_dump(mode="json"), "dry_run": dry_run, "no_cache": no_cache} + console.print(f"[cyan]Dispatching {len(config.deckies)} deckies across {len(workers)} worker(s)...[/]") + resp3 = _utils._http_request("POST", base + "/swarm/deploy", json_body=body, timeout=900.0) + results = resp3.json().get("results", []) + + table = Table(title="SWARM deploy results") + for col in ("worker", "host_uuid", "ok", "detail"): + table.add_column(col) + any_failed = False + for r in results: + ok = bool(r.get("ok")) + if not ok: + any_failed = True + detail = r.get("detail") + if isinstance(detail, dict): + detail = detail.get("status") or "ok" + table.add_row( + str(r.get("host_name") or ""), + str(r.get("host_uuid") or ""), + "[green]yes[/]" if ok else "[red]no[/]", + str(detail)[:80], + ) + console.print(table) + if any_failed: + raise typer.Exit(1) + + +def register(app: typer.Typer) -> None: + @app.command() + def deploy( + mode: str = typer.Option("unihost", "--mode", "-m", help="Deployment mode: unihost | swarm"), + deckies: Optional[int] = typer.Option(None, "--deckies", "-n", help="Number of deckies to deploy (required without --config)", min=1), + interface: Optional[str] = typer.Option(None, "--interface", "-i", help="Host NIC (auto-detected if omitted)"), + subnet: Optional[str] = typer.Option(None, "--subnet", help="LAN subnet CIDR (auto-detected if omitted)"), + ip_start: Optional[str] = typer.Option(None, "--ip-start", help="First decky IP (auto if omitted)"), + services: Optional[str] = typer.Option(None, "--services", help="Comma-separated services, e.g. ssh,smb,rdp"), + randomize_services: bool = typer.Option(False, "--randomize-services", help="Assign random services to each decky"), + distro: Optional[str] = typer.Option(None, "--distro", help="Comma-separated distro slugs, e.g. debian,ubuntu22,rocky9"), + randomize_distros: bool = typer.Option(False, "--randomize-distros", help="Assign a random distro to each decky"), + log_file: Optional[str] = typer.Option(DECNET_INGEST_LOG_FILE, "--log-file", help="Host path for the collector to write RFC 5424 logs (e.g. /var/log/decnet/decnet.log)"), + archetype_name: Optional[str] = typer.Option(None, "--archetype", "-a", help="Machine archetype slug (e.g. linux-server, windows-workstation)"), + mutate_interval: Optional[int] = typer.Option(30, "--mutate-interval", help="Automatically rotate services every N minutes"), + dry_run: bool = typer.Option(False, "--dry-run", help="Generate compose file without starting containers"), + no_cache: bool = typer.Option(False, "--no-cache", help="Force rebuild all images, ignoring Docker layer cache"), + parallel: bool = typer.Option(False, "--parallel", help="Build all images concurrently (enables BuildKit, separates build from up)"), + ipvlan: bool = typer.Option(False, "--ipvlan", help="Use IPvlan L2 instead of MACVLAN (required on WiFi interfaces)"), + config_file: Optional[str] = typer.Option(None, "--config", "-c", help="Path to INI config file"), + api: bool = typer.Option(False, "--api", help="Start the FastAPI backend to ingest and serve logs"), + api_port: int = typer.Option(8000, "--api-port", help="Port for the backend API"), + daemon: bool = typer.Option(False, "--daemon", help="Detach to background as a daemon process"), + ) -> None: + """Deploy deckies to the LAN.""" + import os + import subprocess # nosec B404 + import sys + from pathlib import Path as _Path + + _require_master_mode("deploy") + if daemon: + log.info("deploy daemonizing mode=%s deckies=%s", mode, deckies) + _utils._daemonize() + + log.info("deploy command invoked mode=%s deckies=%s dry_run=%s", mode, deckies, dry_run) + if mode not in ("unihost", "swarm"): + console.print("[red]--mode must be 'unihost' or 'swarm'[/]") + raise typer.Exit(1) + + if config_file: + try: + ini = load_ini(config_file) + except FileNotFoundError as e: + console.print(f"[red]{e}[/]") + raise typer.Exit(1) + + iface = interface or ini.interface or detect_interface() + subnet_cidr = subnet or ini.subnet + effective_gateway = ini.gateway + if subnet_cidr is None: + subnet_cidr, effective_gateway = detect_subnet(iface) + elif effective_gateway is None: + _, effective_gateway = detect_subnet(iface) + + host_ip = get_host_ip(iface) + console.print(f"[dim]Config:[/] {config_file} [dim]Interface:[/] {iface} " + f"[dim]Subnet:[/] {subnet_cidr} [dim]Gateway:[/] {effective_gateway} " + f"[dim]Host IP:[/] {host_ip}") + + if ini.custom_services: + from decnet.custom_service import CustomService + from decnet.services.registry import register_custom_service + for cs in ini.custom_services: + register_custom_service( + CustomService( + name=cs.name, + image=cs.image, + exec_cmd=cs.exec_cmd, + ports=cs.ports, + ) + ) + + effective_log_file = log_file + try: + decky_configs = build_deckies_from_ini( + ini, subnet_cidr, effective_gateway, host_ip, randomize_services, cli_mutate_interval=mutate_interval + ) + except ValueError as e: + console.print(f"[red]{e}[/]") + raise typer.Exit(1) + else: + if deckies is None: + console.print("[red]--deckies is required when --config is not used.[/]") + raise typer.Exit(1) + + services_list = [s.strip() for s in services.split(",")] if services else None + if services_list: + known = set(all_service_names()) + unknown = [s for s in services_list if s not in known] + if unknown: + console.print(f"[red]Unknown service(s): {unknown}. Available: {all_service_names()}[/]") + raise typer.Exit(1) + + arch: Archetype | None = None + if archetype_name: + try: + arch = get_archetype(archetype_name) + except ValueError as e: + console.print(f"[red]{e}[/]") + raise typer.Exit(1) + + if not services_list and not randomize_services and not arch: + console.print("[red]Specify --services, --archetype, or --randomize-services.[/]") + raise typer.Exit(1) + + iface = interface or detect_interface() + if subnet is None: + subnet_cidr, effective_gateway = detect_subnet(iface) + else: + subnet_cidr = subnet + _, effective_gateway = detect_subnet(iface) + + host_ip = get_host_ip(iface) + console.print(f"[dim]Interface:[/] {iface} [dim]Subnet:[/] {subnet_cidr} " + f"[dim]Gateway:[/] {effective_gateway} [dim]Host IP:[/] {host_ip}") + + distros_list = [d.strip() for d in distro.split(",")] if distro else None + if distros_list: + try: + for slug in distros_list: + get_distro(slug) + except ValueError as e: + console.print(f"[red]{e}[/]") + raise typer.Exit(1) + + ips = allocate_ips(subnet_cidr, effective_gateway, host_ip, deckies, ip_start) + decky_configs = build_deckies( + deckies, ips, services_list, randomize_services, + distros_explicit=distros_list, randomize_distros=randomize_distros, + archetype=arch, mutate_interval=mutate_interval, + ) + effective_log_file = log_file + + if api and not effective_log_file: + effective_log_file = os.path.join(os.getcwd(), "decnet.log") + console.print(f"[cyan]API mode enabled: defaulting log-file to {effective_log_file}[/]") + + config = DecnetConfig( + mode=mode, + interface=iface, + subnet=subnet_cidr, + gateway=effective_gateway, + deckies=decky_configs, + log_file=effective_log_file, + ipvlan=ipvlan, + mutate_interval=mutate_interval, + ) + + log.debug("deploy: config built deckies=%d interface=%s subnet=%s", len(config.deckies), config.interface, config.subnet) + + if mode == "swarm": + _deploy_swarm(config, dry_run=dry_run, no_cache=no_cache) + if dry_run: + log.info("deploy: swarm dry-run complete, no workers dispatched") + else: + log.info("deploy: swarm deployment complete deckies=%d", len(config.deckies)) + return + + from decnet.engine import deploy as _deploy + _deploy(config, dry_run=dry_run, no_cache=no_cache, parallel=parallel) + if dry_run: + log.info("deploy: dry-run complete, no containers started") + else: + log.info("deploy: deployment complete deckies=%d", len(config.deckies)) + + if mutate_interval is not None and not dry_run: + console.print(f"[green]Starting DECNET Mutator watcher in the background (interval: {mutate_interval}m)...[/]") + try: + subprocess.Popen( # nosec B603 + [sys.executable, "-m", "decnet.cli", "mutate", "--watch"], + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + except (FileNotFoundError, subprocess.SubprocessError): + console.print("[red]Failed to start mutator watcher.[/]") + + if effective_log_file and not dry_run and not api: + _collector_err = _Path(effective_log_file).with_suffix(".collector.log") + console.print(f"[bold cyan]Starting log collector[/] → {effective_log_file}") + subprocess.Popen( # nosec B603 + [sys.executable, "-m", "decnet.cli", "collect", "--log-file", str(effective_log_file)], + stdin=subprocess.DEVNULL, + stdout=open(_collector_err, "a"), + stderr=subprocess.STDOUT, + start_new_session=True, + ) + + if api and not dry_run: + console.print(f"[green]Starting DECNET API on port {api_port}...[/]") + _env: dict[str, str] = os.environ.copy() + _env["DECNET_INGEST_LOG_FILE"] = str(effective_log_file or "") + try: + subprocess.Popen( # nosec B603 + [sys.executable, "-m", "uvicorn", "decnet.web.api:app", "--host", DECNET_API_HOST, "--port", str(api_port)], + env=_env, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT + ) + console.print(f"[dim]API running at http://{DECNET_API_HOST}:{api_port}[/]") + except (FileNotFoundError, subprocess.SubprocessError): + console.print("[red]Failed to start API. Ensure 'uvicorn' is installed in the current environment.[/]") + + if effective_log_file and not dry_run: + console.print("[bold cyan]Starting DECNET-PROBER[/] (auto-discovers attackers from log stream)") + try: + subprocess.Popen( # nosec B603 + [sys.executable, "-m", "decnet.cli", "probe", "--daemon", "--log-file", str(effective_log_file)], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + except (FileNotFoundError, subprocess.SubprocessError): + console.print("[red]Failed to start DECNET-PROBER.[/]") + + if effective_log_file and not dry_run: + console.print("[bold cyan]Starting DECNET-PROFILER[/] (builds attacker profiles from log stream)") + try: + subprocess.Popen( # nosec B603 + [sys.executable, "-m", "decnet.cli", "profiler", "--daemon"], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + except (FileNotFoundError, subprocess.SubprocessError): + console.print("[red]Failed to start DECNET-PROFILER.[/]") + + if effective_log_file and not dry_run: + console.print("[bold cyan]Starting DECNET-SNIFFER[/] (passive network capture)") + try: + subprocess.Popen( # nosec B603 + [sys.executable, "-m", "decnet.cli", "sniffer", "--daemon", "--log-file", str(effective_log_file)], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + except (FileNotFoundError, subprocess.SubprocessError): + console.print("[red]Failed to start DECNET-SNIFFER.[/]") diff --git a/decnet/cli/forwarder.py b/decnet/cli/forwarder.py new file mode 100644 index 00000000..b736fd82 --- /dev/null +++ b/decnet/cli/forwarder.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import asyncio +import pathlib +import signal +from typing import Optional + +import typer + +from decnet.env import DECNET_INGEST_LOG_FILE + +from . import utils as _utils +from .utils import console, log + + +def register(app: typer.Typer) -> None: + @app.command() + def forwarder( + master_host: Optional[str] = typer.Option(None, "--master-host", help="Master listener hostname/IP (default: $DECNET_SWARM_MASTER_HOST)"), + master_port: int = typer.Option(6514, "--master-port", help="Master listener TCP port (RFC 5425 default 6514)"), + log_file: Optional[str] = typer.Option(DECNET_INGEST_LOG_FILE, "--log-file", help="Local RFC 5424 file to tail and forward"), + agent_dir: Optional[str] = typer.Option(None, "--agent-dir", help="Worker cert bundle dir (default: ~/.decnet/agent)"), + state_db: Optional[str] = typer.Option(None, "--state-db", help="Forwarder offset SQLite path (default: /forwarder.db)"), + poll_interval: float = typer.Option(0.5, "--poll-interval", help="Seconds between log file stat checks"), + daemon: bool = typer.Option(False, "--daemon", "-d", help="Detach to background as a daemon process"), + ) -> None: + """Run the worker-side syslog-over-TLS forwarder (RFC 5425, mTLS to master:6514).""" + from decnet.env import DECNET_SWARM_MASTER_HOST + from decnet.swarm import pki + from decnet.swarm.log_forwarder import ForwarderConfig, run_forwarder + + resolved_host = master_host or DECNET_SWARM_MASTER_HOST + if not resolved_host: + console.print("[red]--master-host is required (or set DECNET_SWARM_MASTER_HOST).[/]") + raise typer.Exit(2) + + resolved_agent_dir = pathlib.Path(agent_dir) if agent_dir else pki.DEFAULT_AGENT_DIR + if not (resolved_agent_dir / "worker.crt").exists(): + console.print(f"[red]No worker cert bundle at {resolved_agent_dir} — enroll from the master first.[/]") + raise typer.Exit(2) + + if not log_file: + console.print("[red]--log-file is required.[/]") + raise typer.Exit(2) + + cfg = ForwarderConfig( + log_path=pathlib.Path(log_file), + master_host=resolved_host, + master_port=master_port, + agent_dir=resolved_agent_dir, + state_db=pathlib.Path(state_db) if state_db else None, + ) + + if daemon: + log.info("forwarder daemonizing master=%s:%d log=%s", resolved_host, master_port, log_file) + _utils._daemonize() + + log.info("forwarder command invoked master=%s:%d log=%s", resolved_host, master_port, log_file) + console.print(f"[green]Starting DECNET forwarder → {resolved_host}:{master_port} (mTLS)...[/]") + + async def _main() -> None: + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, stop.set) + except (NotImplementedError, RuntimeError): # pragma: no cover + pass + await run_forwarder(cfg, poll_interval=poll_interval, stop_event=stop) + + try: + asyncio.run(_main()) + except KeyboardInterrupt: + pass diff --git a/decnet/cli/gating.py b/decnet/cli/gating.py new file mode 100644 index 00000000..a373bc1c --- /dev/null +++ b/decnet/cli/gating.py @@ -0,0 +1,73 @@ +"""Role-based CLI gating. + +MAINTAINERS: when you add a new Typer command (or add_typer group) that is +master-only, register its name in MASTER_ONLY_COMMANDS / MASTER_ONLY_GROUPS +below. The gate is the only thing that: + (a) hides the command from `decnet --help` on worker hosts, and + (b) prevents a misconfigured worker from invoking master-side logic. +Forgetting to register a new command is a role-boundary bug. Grep for +MASTER_ONLY when touching command registration. + +Worker-legitimate commands (NOT in these sets): agent, updater, forwarder, +status, collect, probe, sniffer. Agents run deckies locally and should be +able to inspect them + run the per-host microservices (collector streams +container logs, prober characterizes attackers hitting this host, sniffer +captures traffic). Mutator and Profiler stay master-only: the mutator +orchestrates respawns across the swarm; the profiler rebuilds attacker +profiles against the master DB (no per-host DB exists). +""" + +from __future__ import annotations + +import os + +import typer + +from .utils import console + +MASTER_ONLY_COMMANDS: frozenset[str] = frozenset({ + "api", "swarmctl", "deploy", "redeploy", "teardown", + "mutate", "listener", "profiler", + "services", "distros", "correlate", "archetypes", "web", + "db-reset", "init", "webhook", "clusterer", "campaign-clusterer", +}) +MASTER_ONLY_GROUPS: frozenset[str] = frozenset( + {"swarm", "topology", "geoip", "realism"} +) + + +def _agent_mode_active() -> bool: + """True when the host is configured as an agent AND master commands are + disallowed (the default for agents). Workers overriding this explicitly + set DECNET_DISALLOW_MASTER=false to opt into hybrid use.""" + mode = os.environ.get("DECNET_MODE", "master").lower() + disallow = os.environ.get("DECNET_DISALLOW_MASTER", "true").lower() == "true" + return mode == "agent" and disallow + + +def _require_master_mode(command_name: str) -> None: + """Defence-in-depth: called at the top of every master-only command body. + + The registration-time gate in _gate_commands_by_mode() already hides + these commands from Typer's dispatch table, but this check protects + against direct function imports (e.g. from tests or third-party tools) + that would bypass Typer entirely.""" + if _agent_mode_active(): + console.print( + f"[red]`decnet {command_name}` is a master-only command; this host " + f"is configured as an agent (DECNET_MODE=agent).[/]" + ) + raise typer.Exit(1) + + +def _gate_commands_by_mode(_app: typer.Typer) -> None: + if not _agent_mode_active(): + return + _app.registered_commands = [ + c for c in _app.registered_commands + if (c.name or c.callback.__name__) not in MASTER_ONLY_COMMANDS + ] + _app.registered_groups = [ + g for g in _app.registered_groups + if g.name not in MASTER_ONLY_GROUPS + ] diff --git a/decnet/cli/geoip.py b/decnet/cli/geoip.py new file mode 100644 index 00000000..7ff90a3e --- /dev/null +++ b/decnet/cli/geoip.py @@ -0,0 +1,59 @@ +"""GeoIP CLI — refresh and lookup subcommands (master-only). + +Usage:: + + decnet geoip refresh # re-download RIR files and rebuild the index + decnet geoip lookup 8.8.8.8 # one-shot IP -> country dump +""" +from __future__ import annotations + +import typer + +from .gating import _require_master_mode +from .utils import console, log + +_group = typer.Typer( + name="geoip", + help="GeoIP provider management (master only).", + no_args_is_help=True, +) + + +@_group.command("refresh") +def _refresh() -> None: + """Force re-download of the GeoIP provider data and rebuild the index.""" + _require_master_mode("geoip refresh") + from decnet.geoip import get_lookup + from decnet.geoip.factory import get_provider + + provider = get_provider() + log.info("geoip: forcing refresh via %s provider", provider.name) + console.print(f"[bold cyan]Refreshing {provider.name} GeoIP data…[/]") + try: + lookup = get_lookup(force_refresh=True) + except Exception as exc: # noqa: BLE001 + console.print(f"[red]refresh failed: {exc}[/]") + raise typer.Exit(1) from exc + console.print( + f"[green]OK[/] {provider.name} index rebuilt " + f"({len(lookup)} ranges)." + ) + + +@_group.command("lookup") +def _lookup( + ip: str = typer.Argument(..., help="IP address to resolve."), +) -> None: + """Print the country code for an IP (or 'unknown').""" + _require_master_mode("geoip lookup") + from decnet.geoip import enrich_ip + + cc, source = enrich_ip(ip) + if cc is None: + console.print(f"{ip} [yellow]unknown[/]") + raise typer.Exit(0) + console.print(f"{ip} [green]cc={cc}[/] source={source}") + + +def register(app: typer.Typer) -> None: + app.add_typer(_group, name="geoip") diff --git a/decnet/cli/init.py b/decnet/cli/init.py new file mode 100644 index 00000000..bb52cb65 --- /dev/null +++ b/decnet/cli/init.py @@ -0,0 +1,843 @@ +""" +`decnet init` — one-shot master-host bootstrap. + +Idempotent: running it twice is a no-op on already-configured items. +Takes a freshly ``pip install``'d DECNET and turns it into a ready-to- +run master host: creates the ``decnet`` system user/group, installs +the systemd units + polkit rule + tmpfiles.d entry, seeds the +directory layout, drops a placeholder config, and starts the +``decnet.target`` grouping unit. + +Requires root. Uses ``subprocess.run`` (never ``shell=True``) for every +privileged call so the full argv surface is auditable. +""" +from __future__ import annotations + +import grp +import hashlib +import os +import pwd +import shutil +import subprocess # nosec B404 +import sys +from pathlib import Path +from typing import Callable, List, Optional + +import typer +from jinja2 import Environment, FileSystemLoader, StrictUndefined + +import decnet as _decnet_pkg +from .gating import _require_master_mode +from .utils import console, log + + +_CONFIG_PLACEHOLDER = """\ +# /etc/decnet/decnet.ini — DECNET host config. +# +# Every key is OPTIONAL. Absent keys fall through to env-var defaults +# defined in decnet/env.py. Real env vars always win over this file +# (precedence: env > INI > default), so systemd EnvironmentFile= and +# one-off `DECNET_FOO=bar decnet ...` invocations always take effect. +# +# Secrets (JWT, admin password, DB password) intentionally DO NOT +# live here. Put them in /opt/decnet/.env.local or the systemd +# EnvironmentFile= — never in a group-readable INI. + +[decnet] +# mode = master # or "agent" + +# [api] +# host = 127.0.0.1 +# port = 8000 + +# [web] +# host = 127.0.0.1 +# port = 8080 +# admin-user = admin +# cors-origins = http://localhost:8080 # comma-separated + +# [database] +# type = sqlite # or "mysql" +# url = mysql+asyncmy://user@host:3306/decnet # if set, wins over host/port/name/user +# host = localhost +# port = 3306 +# name = decnet +# user = decnet + +# [bus] +# enabled = true +# type = unix # or "fake" +# socket = /run/decnet/bus.sock +# group = decnet + +# [swarm] +# master-host = 10.0.0.1 +# syslog-port = 6514 +# swarmctl-port = 8770 + +# [logging] +# system-log = /var/log/decnet/decnet.system.log +# ingest-log = /var/log/decnet/decnet.log +# agent-log = /var/log/decnet/agent.log + +# [ingester] +# batch-size = 100 +# batch-max-wait-ms = 250 + +# [tracing] +# enabled = false +# otel-endpoint = http://localhost:4317 + +# [agent] +# Managed by the enroll bundle — do NOT edit by hand on an agent host. +""" + + +def _deploy_root() -> Path: + """Resolve the on-disk ``deploy/`` directory of the installed package. + + Editable install (``pip install -e .``): sibling of the ``decnet`` + package at repo root. Wheel installs aren't supported yet — the + error message tells the operator to use an editable install. + """ + root = Path(_decnet_pkg.__file__).resolve().parent.parent / "deploy" + if not (root / "decnet.target").is_file(): + raise RuntimeError( + f"cannot locate deploy/ directory (looked at {root}); " + "are you on a wheel install that didn't bundle deploy/? " + "use `pip install -e .` from a git checkout" + ) + return root + + +def _sha256(path: Path) -> str: + h = hashlib.sha256() + h.update(path.read_bytes()) + return h.hexdigest() + + +def _run(argv: List[str], *, dry_run: bool) -> None: + if dry_run: + console.print(f" [dim]would run:[/] {' '.join(argv)}") + return + log.info("init: exec %s", argv) + subprocess.run(argv, check=True) # nosec B603 + + +def _step(label: str, action: Callable[[], str]) -> bool: + """Run ``action``, print a checklist line. + + The callable returns the human-readable outcome verb: + ``"ok"`` → ``[ OK ]