diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 5dd8d217..942f853c 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: with: python-version: "3.11" - run: pip install ruff - - run: ruff check . + - run: ruff check decnet/ bandit: name: SAST (bandit) @@ -40,7 +40,7 @@ jobs: python-version: "3.11" - run: pip install pip-audit - run: pip install -e .[dev] - - run: pip-audit --skip-editable --ignore-vuln CVE-2025-65896 + - run: pip-audit --skip-editable --ignore-vuln CVE-2025-65896 --ignore-vuln CVE-2026-3219 test-standard: name: Test (Standard) @@ -61,22 +61,22 @@ jobs: name: Test (Live) runs-on: ubuntu-latest needs: [test-standard] + services: + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: decnet_test + ports: + - 3307:3306 + options: >- + --health-cmd="mysqladmin ping -h 127.0.0.1" + --health-interval=10s + --health-timeout=5s + --health-retries=5 strategy: matrix: python-version: ["3.11"] - services: - mysql: - image: mysql:8.0 - env: - MYSQL_ROOT_PASSWORD: root - MYSQL_DATABASE: decnet_test - ports: - - 3307:3306 - options: >- - --health-cmd="mysqladmin ping -h 127.0.0.1" - --health-interval=10s - --health-timeout=5s - --health-retries=5 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -91,27 +91,10 @@ jobs: DECNET_MYSQL_PASSWORD: root DECNET_MYSQL_DATABASE: decnet_test - test-fuzz: - name: Test (Fuzz) - runs-on: ubuntu-latest - needs: [test-live] - strategy: - matrix: - python-version: ["3.11"] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - run: pip install -e .[dev] - - run: pytest -m fuzz - env: - SCHEMATHESIS_CONFIG: schemathesis.ci.toml - merge-to-testing: name: Merge dev → testing runs-on: ubuntu-latest - needs: [test-standard, test-live, test-fuzz] + needs: [test-standard, test-live] if: github.ref == 'refs/heads/dev' steps: - uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore index 810322d7..bc75c3dd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ .venv/ +.venv*/ +.311/ +.3[0-9][0-9]/ logs/ .claude/* CLAUDE.md @@ -9,6 +12,10 @@ __pycache__/ dist/ build/ decnet-compose.yml +# Per-topology compose fragments emitted by `decnet topology deploy`. +decnet-topology-*-compose.yml +# Docker build context cache. +.docker/ decnet-state.json *.ini decnet.log* @@ -21,6 +28,9 @@ windows1 *.db-shm *.db-wal decnet.*.log +# Rotated copies (logrotate appends .1, .2, .gz...) — the existing +# decnet.*.log glob doesn't catch the suffix. +decnet.*.log.* decnet.json .env* .env.local @@ -28,3 +38,16 @@ decnet.json .hypothesis/ profiles/* tests/test_decnet.db* + +# Nested git clone of the wiki — not a submodule, just a local +# working copy so we can edit docs without a full round-trip. +wiki-checkout/ + +# Scratch test/debug outputs that leak from saved `pytest > hang.log` +# or `pytest > schem` redirections. +hang.log +schem +*.pytest.log + +# pydeps-style dependency graph dumps from local analysis runs. +deps.txt diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..f288702d --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md index 5395f350..21c17ac0 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ A honeypot deception network framework. Spin up a fleet of fake machines — cal Attackers probe the network, DECNET traps every interaction, and you watch from a safe, isolated logging stack. +[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/C0C31YDLB5) + --- ## Table of Contents diff --git a/SIGNAL_CAPTURE_AUDIT.md b/SIGNAL_CAPTURE_AUDIT.md new file mode 100644 index 00000000..d6392064 --- /dev/null +++ b/SIGNAL_CAPTURE_AUDIT.md @@ -0,0 +1,566 @@ +# DECNET Capture Pipeline — Attacker-Profiling Signal Audit + +**Date**: 2026-04-22 +**Scope**: v1 capture readiness for post-v1 profiler extraction +**Methodology**: End-to-end verification (emission → transport → storage) for each signal against active code paths. + +--- + +## Executive Summary + +**Capture Status by Category**: + +| Category | Captured | Partial | Not Captured | n/a | +|----------|----------|---------|--------------|-----| +| Session Environment | 0 | 1 | 3 | 0 | +| Keystroke/Human | 0 | 2 | 6 | 2 | +| SSH Transport | 2 | 2 | 2 | 0 | +| Network/TCP | 3 | 2 | 5 | 0 | +| TLS/L7 | 2 | 2 | 1 | 0 | +| Aggregated/Derived | 0 | 0 | 5 | 0 | +| **TOTAL** | **7** | **9** | **22** | **2** | + +**Critical Pre-v1 Gaps** (blockers if signals are roadmap-committed): + +1. **KEX algorithm ordering** — HASSH hash is stored, but raw `kex_algorithms` string is only emitted to syslog, not persisted to DB. Future extractor must parse syslog archives. +2. **Per-keystroke timing** — Asciinema v2 `"i"` events with `t` timestamps are written to day-shard files on disk, but no database ingestion. Requires filesystem polling + parsing path. +3. **TCP options order** — Captured in PCAP + sniffer logs (`options_sig`), but `options_sig` is a rolled-up signature string, not the raw per-connection sequence. +4. **Terminal size (COLS×ROWS)** — Not captured from pty-req at all; would require SSH protocol-level interception. +5. **SSH client version** — Server-side only sees RFC 4253 banner; full version string would require TLS cert inspection or prober modification. + +**Biggest ROI capture improvements** (cheap, high-value): + +1. Add `ssh_client_banner` column to Attacker table — capture SSH-2.0-* string from pty-req. +2. Ingest asciinema keystroke timing into new `SessionProfile` table (v2 roadmap already designs this). +3. Store raw KEX algorithm lists in `AttackerBehavior.kex_order_raw` (MEDIUMTEXT) instead of relying on syslog dedup. + +--- + +## Per-Signal Classification + +### Per-Session Environment (SessionProfile candidates) + +#### TERM environment variable +- **Status**: `partial` +- **Where**: SSH server can read TERM from pty-req; emitted in syslog by `emit_capture.py` if implemented. +- **Current path**: Not found in active code path. Check `decnet/templates/ssh/emit_capture.py` or syslog bridge. +- **Missing**: Database column in a `SessionProfile` table; no structured ingestion. +- **Cheap fix**: Modify SSH syslog bridge to emit `session_event` with `term=`. Create `SessionProfile` table with `session_term` TEXT column. +- **Priority**: V2 backlog (nice-to-have for human vs. automation, low discriminative power). + +#### LANG / LC_ALL +- **Status**: `not_captured` +- **Why**: Server-side locale is baked into container image, not attacker-controlled. Attacker's client locale is not visible over SSH. +- **Priority**: defer (non-capturable from server vantage point). + +#### SSH client version string (full SSH-2.0-OpenSSH_9.2p1…) +- **Status**: `partial` +- **Where**: RFC 4253 banner string is transmitted in plaintext before encryption. Sniffer could capture it from TCP stream; prober `hassh.py` captures server banner (lines 58–101), not client. +- **Missing**: Client-side banner capture. Sniffer would need TCP stream reconstruction to pluck the SSH banner from the raw payload. +- **Cheap fix**: Extend sniffer to parse SSH banners from TCP stream (before TLS/encryption); emit `ssh_client_banner` event. Store in Attacker.`ssh_client_banners` (JSON list). +- **Priority**: v1 blocker if client-profiling is committed. Currently partial via TLS fingerprint fallback. + +#### Terminal size (COLS × ROWS) +- **Status**: `not_captured` +- **Why**: SSH pty-req extension carries `terminal mode` (COLS, ROWS, speeds); server-side sshd parses this but does not log it by default. Would require patching sshd or intercepting at the protocol layer. +- **Missing**: No access to pty-req payload without protocol-level instrumentation. +- **Cheap fix**: Patch SSH entrypoint to log pty-req to syslog before accepting the request (requires custom OpenSSH build). +- **Priority**: V2 backlog (interesting for typing-space reconstruction, but not blocky). + +--- + +### Per-Session, Keyboard/Human (SessionProfile candidates) + +#### Per-keystroke timing (t in asciinema "i" events) +- **Status**: `partial` +- **Where**: Sessrec pipeline (`decnet/templates/ssh/sessrec/`) writes asciinema v2 day-shards with per-keystroke `"i"` (input) events carrying `t` (timestamp in seconds since session start). Files on disk: `/var/lib/decnet/session_recordings//.json` (or similar). +- **Missing**: No ingestion into database. Extractors must read asciinema files from filesystem and parse the `"i"` event stream post-hoc. +- **Cheap fix**: Ingest keystroke timing stream into new `SessionProfile` table (design already in DEVELOPMENT_V2.md). Add job to parse day-shard files on rotation and compute IKI moments, burst ratio, etc. +- **Priority**: v1 blocker if keystroke dynamics is roadmap-committed. Data exists but not queryable. + +#### Control-character stream (backspace, ^W, ^U, ^C, ^D, arrows, tab) +- **Status**: `partial` +- **Where**: Asciinema captures every keystroke as UTF-8/control byte in `"i"` events. Raw byte sequence is preserved. +- **Missing**: Same as above — files on disk, no DB ingestion. Future extractor can parse control bytes from the `"data"` field of each `"i"` event. +- **Cheap fix**: Same as keystroke timing — ingest asciinema events and compute `kd_ctrl_*` rates in SessionProfile. +- **Priority**: v2 (depends on SessionProfile schema). + +#### Inter-command think time (prompt-return to next-command-start gap) +- **Status**: `not_captured` +- **Why**: Requires prompt boundary detection in the asciinema stream (heuristic: line ending in `$` or `#` + pause > 100ms). No active code marks prompts. +- **Missing**: Prompt-boundary markers in asciinema. Would require ML or regex-based post-processing. +- **Cheap fix**: Add prompt-regex configuration + marker injection during sessrec playback, or post-hoc analysis over asciinema. +- **Priority**: V2 (interesting but requires heuristic or attacker-side annotation). + +#### Pause before sensitive commands +- **Status**: `not_captured` +- **Why**: Requires command-boundary detection (typing a full command, then detecting gap before Enter). Asciinema captures this timing, but no code marks command boundaries. +- **Missing**: Command-line parsing + gap detection logic. +- **Cheap fix**: Off-line analysis: parse `"i"` events, detect Enter (`\r`), measure gap before Enter. Correlate with command content from `"o"` (output) events. +- **Priority**: V2 backlog (post-extraction analysis; interesting for psychological profiling). + +#### Command n-grams +- **Status**: `partial` +- **Where**: SSH service logs individual commands to syslog when pty input is detected. Attacker.`commands` JSON array stores seen commands (but coarse-grained per service/decky, not per-session). +- **Missing**: Per-session, per-command sequencing. No n-gram bigrams/trigrams computed. +- **Cheap fix**: Parse asciinema `"i"` + `"o"` stream to extract full command lines, store as JSON list in SessionProfile.`cmd_sequence` or new `SessionCommand` table. +- **Priority**: V2 (foundation for command chaining fingerprint). + +#### Flag preferences (ls -la vs ls -al, ps -ef vs ps aux) +- **Status**: `not_captured` +- **Why**: Asciinema records the **typed** command line exactly, but no code parses flag ordering or normalizes commands for pattern comparison. +- **Missing**: Canonical command parsing + flag-order extraction. +- **Cheap fix**: Off-line: regex-parse commands from asciinema, extract flag sequences, compute n-grams over flag positions. +- **Priority**: V2 (cheap post-processing, good human-vs-tool separator). + +#### Typo patterns (suod, sl) +- **Status**: `not_captured` +- **Why**: Asciinema records corrected command line after backspacing, not the raw keystrokes with typos visible. +- **Example**: typing `suod` then `ddo` then `o` shows as `sudo` in `"o"` output; the intermediate typos are **visible** in the `"i"` event stream but require careful keystroke-by-keystroke parsing. +- **Missing**: Raw keystroke stream parsing to detect backspace/correction patterns. +- **Cheap fix**: Parse `"i"` events, reconstruct line state keystroke-by-keystroke, log (typed_text, final_text) pairs to detect corrections. +- **Priority**: V2 (unique human fingerprint, but requires manual asciinema parsing). + +#### Editor choice (vi/vim/nano/ed) +- **Status**: `partial` +- **Where**: Command launch (`vi`, `nano`, `ed`) is visible in asciinema `"i"` + `"o"` stream and captured in Attacker.`commands`. +- **Missing**: No aggregation of editor invocations or time-in-editor statistics. +- **Cheap fix**: Post-process commands, count editor launches, extract editor type. Could add to AttackerBehavior.`preferred_editor` or new SessionProfile.`editor_used`. +- **Priority**: V2 (behavioral signal, low priority). + +#### Shell history usage (!!,!$, ^old^new, fc) +- **Status**: `partial` +- **Where**: Command input stream captures the actual invocation (if attacker types `!!`, it's visible in `"i"`). Output `"o"` shows the expanded command. +- **Missing**: No parsing of history expansion syntax; requires post-processing to identify `!` / `^` patterns. +- **Cheap fix**: Regex-scan asciinema input for shell history operators; count occurrences. +- **Priority**: V2 (interesting tool-chain signal, but low volume). + +--- + +### Per-Attacker, SSH Transport (AttackerBehavior candidates) + +#### HASSH / HASSHServer +- **Status**: `captured` +- **Where**: Prober (`decnet/prober/hassh.py`) computes HASSHServer fingerprint; stored as `Attacker.fingerprints` JSON list (generic bounty store). Also emitted to syslog by prober worker. +- **Note**: Roadmap says `[x]` (captured); verified in code at lines 244–252 of `hassh.py`. +- **Storage**: `Attacker.fingerprints` (JSON list of `{type, value, ...}` dicts); not per-attacker-behavior, but queryable. +- **Priority**: ✓ captured; v2: consider normalizing to `AttackerBehavior.hassh_server` for faster lookup. + +#### KEX algorithm preference ORDER (beyond HASSH hash) +- **Status**: `partial` +- **Where**: Sniffer logs raw `kex_algorithms`, `encryption_s2c`, `mac_s2c`, `compression_s2c` strings to syslog in `tls_session` and `tcp_syn_fingerprint` events (fingerprint.py lines 240–252). +- **Missing**: Stored in **syslog only**, not in DB. Attacker table has `fingerprints` (bounty store) but no dedicated `kex_order_raw` column. +- **Path to recovery**: Read syslog archives and parse `kex_algorithms` field. But this is not queryable at scale. +- **Cheap fix**: Add `Attacker.kex_order_raw` (MEDIUMTEXT, JSON string list) and `kd_kex_order_hash` (similar to digraph simhash). Populate during sniffer event ingestion. +- **Priority**: v1 blocker if KEX ordering is committed to roadmap (currently only hash stored, raw data must be re-parsed from syslog). + +#### Public key comment field +- **Status**: `not_captured` +- **Why**: SSH key comment is part of the OpenSSH wire format (only transmitted if key auth is used). Server-side sshd does not log it by default; would require PAM/auth hook instrumentation. +- **Missing**: No interception of public key authentication payloads. +- **Cheap fix**: Patch SSH server to emit auth_pubkey event with key comment extracted from wire format. Or use `net.ssh` library instrumentation. +- **Priority**: V2 backlog (valuable for key reuse fingerprinting, but rare). + +#### Private key type advertised (Ed25519 / RSA / ECDSA) +- **Status**: `partial` +- **Where**: SSH transport carries key type in the public key authentication message. Sniffer cannot decode this (traffic is encrypted after ServerHello). Server-side sshd doesn't log it. +- **Missing**: Requires either passive PCAP of SSH-TRANSPORT (not available; encrypted) or server-side auth hook. +- **Cheap fix**: Patch sshd to emit `auth_pubkey_type` event during authentication. +- **Priority**: V2 (interesting but lower signal than key comment). + +#### Agent forwarding requested? +- **Status**: `not_captured` +- **Why**: Agent forwarding is negotiated via SSH_MSG_SERVICE_REQUEST → ssh-userauth → "ssh-agent@openssh.com" extension. Encrypted after KEX. +- **Missing**: Would require decrypting SSH transport or instrumenting sshd auth hook. +- **Cheap fix**: Sshd can detect `SSH_AUTH_SOCK` or SSH_AGENT_FWD service request; add to syslog. +- **Priority**: V2 (useful for lateral-movement detection). + +#### Channel multiplexing pattern +- **Status**: `partial` +- **Where**: SSH service logs each command separately. Channel open/close events could be tracked, but no code currently does. +- **Missing**: Per-session channel state machine (open channels, their types, lifetime). +- **Cheap fix**: Instrument sshd or use SSH_MSG_CHANNEL_OPEN events in syslog to track simultaneous channels. +- **Priority**: V2 (rare; most attackers use sequential commands). + +#### SSH_CLIENT / SSH_CONNECTION environment variables +- **Status**: `captured` +- **Where**: SSH server **always** sets `SSH_CLIENT` and `SSH_CONNECTION` in the child shell. Server-side user code (bashrc, commands) can read them. If attacker runs `echo $SSH_CLIENT`, it's visible in asciinema output. +- **Missing**: No **automatic** logging of these vars. Requires parsing asciinema for intentional queries or patching sshd to emit them. +- **Cheap fix**: Patch SSH PAM or auth hook to log `SSH_CLIENT` on successful auth. Or parse asciinema for `echo $SSH_*` commands. +- **Priority**: V2 (low value; mostly redundant with src_ip already in logs). + +--- + +### Per-Attacker, Network/Transport (AttackerBehavior candidates) + +#### TCP timestamp clock skew (Kohno 2005) +- **Status**: `partial` +- **Where**: PCAP contains TCP timestamps (if present). Sniffer code extracts MSS, window size, options (fingerprint.py line 77–94). TCP options include timestamp flag (`has_timestamps`). +- **Missing**: Raw timestamp values (`opt_value` for "Timestamp" in scapy) are NOT extracted. Only boolean `has_timestamps` flag is stored. To compute clock skew, need timestamp values across multiple packets. +- **Path to recovery**: Raw PCAP analysis (if PCAPs are retained on disk). Each TCP packet has `[TCP option: Timestamp x, y]` which can be parsed post-hoc. +- **Cheap fix**: Extend sniffer to extract timestamp sequence numbers and RTT deltas. Store as per-flow timing summary in `tcp_flow_timing` event (which already captures flow metrics). +- **Priority**: V2 (requires PCAP or extended sniffer capture; useful for OS fingerprinting). + +#### TCP ISN generator characteristics +- **Status**: `not_captured` +- **Why**: ISN is visible in PCAP (TCP seq number on SYN). Sniffer code tracks flow seqs for retransmit detection (line 850) but does not extract the initial SYN seq across multiple connections to analyze ISN patterns. +- **Missing**: No per-connection ISN logging. Would need to roll up ISN sequences across multiple SYNs to the same port. +- **Cheap fix**: On every SYN, log `syn_seq` in `tcp_syn_fingerprint` event. Post-hoc analysis can compute randomness metrics. +- **Priority**: V2 backlog (weak signal; ISN randomization is standard on modern OS). + +#### TCP options ordering in SYN +- **Status**: `partial` +- **Where**: Sniffer extracts `options_sig` (line 87) via `_extract_options_order()` from scapy TCP options. This is a **signature string** (e.g., `"MSS,WScale,SAckOK,Timestamp"`). +- **Missing**: The signature is **aggregated**; we don't store the raw per-packet ordering. Also, `options_sig` is deduplicated in logs (only one event per unique signature per dedup window). +- **Path to recovery**: Raw PCAP analysis or re-parsing sniffer logs to extract the signature. But the signature is a good enough feature for OS fingerprinting. +- **Cheap fix**: Store `tcp_fingerprint` JSON in AttackerBehavior with raw options list (not just signature). Current schema (models.py line 174–177) only stores aggregated `{window, wscale, mss, options_sig}`. +- **Priority**: v1 improvement (low effort, already have options_sig; add raw list). + +#### Initial congestion window ramp-up +- **Status**: `not_captured` +- **Why**: Requires detailed TCP state machine tracking (SYN, SYN-ACK, ACK sequence with packet sizes). Sniffer tracks `packets` count and `bytes` total per flow (line 844–868), but not per-packet sequence or ACK-clock dynamics. +- **Missing**: Per-packet payload sizes and ACK timing. +- **Cheap fix**: Extend `tcp_flow_timing` event to include per-packet sizes (as JSON list) or CWND estimation from ACK patterns. +- **Priority**: V2 backlog (very niche; useful for Reno vs. Cubic vs. BBR detection, but rare in honeypot context). + +#### Retransmit timing and backoff +- **Status**: `captured` +- **Where**: Sniffer tracks `retransmits` count per flow (lines 873–877, 922). Emitted in `tcp_flow_timing` event. No **timing** of retransmits, only count. +- **Missing**: Timing deltas between retransmit pairs (RTO, exponential backoff pattern). +- **Path to recovery**: Raw PCAP; sequence numbers in `tcp_flow_timing` are not logged. +- **Cheap fix**: Extend event to include retransmit timing deltas (list of RTOs). +- **Priority**: V2 (useful for network condition inference; low value on honeypots). + +#### MTU / path-MTU discovery behavior +- **Status**: `partial` +- **Where**: Sniffer tracks per-flow byte counts (line 868); can infer effective MSS from packet sizes. TCP fingerprint includes extracted MSS (line 77–94, emitted in `tcp_syn_fingerprint`). +- **Missing**: No multi-flow MTU tracking or ICMP fragmentation-needed response detection. Would require ICMP processing. +- **Cheap fix**: Log ICMP unreachable (frag needed) events separately; correlate with TCP flows to infer PMTUD behavior. +- **Priority**: V2 backlog (VPN detection is interesting but niche). + +#### Packet pacing (microsecond-resolution egress timing) +- **Status**: `not_captured` +- **Why**: Sniffer computes mean/min/max inter-arrival time in milliseconds (lines 904–906), not microseconds. Modern pacing requires sub-millisecond precision. +- **Missing**: Sniffer uses `time.monotonic()` (typically millisecond granularity on Linux); would need OS-level timing hooks or PCAP with hardware timestamps. +- **Cheap fix**: Upgrade sniffer to use PCAP timestamps (pcap.ts_resolution) if available; log microsecond-resolution inter-packet gaps. +- **Priority**: V2 backlog (requires infrastructure upgrade; marginal value on honeypots). + +#### Window scaling multipliers +- **Status**: `captured` +- **Where**: Sniffer extracts `wscale` from TCP options (line 80); stored in `tcp_fingerprint` JSON and emitted in `tcp_syn_fingerprint` event. +- **Storage**: AttackerBehavior.`tcp_fingerprint` (JSON: `{window, wscale, mss, ...}`); queryable. +- **Priority**: ✓ captured (sufficient for OS fingerprinting and congestion algorithm inference). + +#### ECN negotiation +- **Status**: `not_captured` +- **Why**: ECN is signaled via TCP flags (CWR, ECE) and the SYN's TCP options. Scapy's TCP layer does not expose ECN flags in the options extraction. +- **Missing**: No code to parse ECN negotiation from TCP header. +- **Cheap fix**: Extend TCP fingerprint extraction to check for ECN flag bits. +- **Priority**: V2 backlog (rarely used; low value). + +--- + +### Per-Attacker, L7 (TLS/HTTP) + +#### TLS fingerprint (JA3/JA4) +- **Status**: `captured` +- **Where**: Sniffer fingerprint engine computes JA3/JA3S/JA4/JA4S (lines 565–662); emitted in syslog and stored in `Attacker.fingerprints` (bounty store). +- **Storage**: Logs are queryable; fingerprints stored as JSON in bounty table (generic). +- **Roadmap**: `[x]` JA3/JA3S, `[x]` JA4+. Verified in code. +- **Priority**: ✓ captured (good). + +#### TLS session resumption behavior +- **Status**: `captured` +- **Where**: Sniffer extracts resumption mechanisms (session_ticket, PSK, early_data, session_id) in `_session_resumption_info()` (lines 675–689). Emitted in `tls_client_hello` event. +- **Storage**: Logged to syslog; `Attacker.fingerprints` stores resumption=`[mechanism list]`. +- **Priority**: ✓ captured (good). + +#### HTTP/2 SETTINGS frame ordering + values +- **Status**: `not_captured` +- **Why**: HTTP/2 is encrypted (after TLS handshake). Sniffer cannot see plaintext SETTINGS frames. +- **Missing**: Would require decryption (not viable passively) or attacker-side TLS instrumentation. +- **Cheap fix**: Instrument HTTP/2 services (h2c, HTTP/2 over plain TCP on rare deployments) or use TLS key log for offline analysis. +- **Priority**: defer (not capturable from passive vantage point). + +#### HTTP/2 stream prioritization +- **Status**: `not_captured` +- **Why**: Encrypted in TLS. +- **Missing**: Same as above. +- **Priority**: defer (not capturable). + +#### HTTP header ordering +- **Status**: `not_captured` +- **Why**: Inside encrypted TLS. Sniffer cannot see plaintext HTTP headers. +- **Missing**: Would require server-side HTTP request logging (not implemented). +- **Cheap fix**: Instrument HTTP service to log raw header order in syslog. +- **Priority**: V2 (useful for bot/tool detection, but requires service-level capture). + +#### Cookie handling behavior (expiry, domain scope) +- **Status**: `not_captured` +- **Why**: Encrypted TLS + requires HTTP state machine tracking (Set-Cookie responses vs. Cookie requests). +- **Missing**: Would need server-side HTTP middleware or browser instrumentation. +- **Cheap fix**: Add cookie jar logging to HTTP service (track which attacker cookies were accepted, rejected, resent). +- **Priority**: V2 (behavioral signal; interesting but niche). + +--- + +### Per-Attacker, Aggregated/Derived (would live in new `AttackerAggregate` table) + +#### Time-of-day activity distribution (chronotyping) +- **Status**: `partial` +- **Where**: Log entries have `timestamp` (datetime). All events are timestamped. Can compute hour-of-day histogram post-hoc. +- **Missing**: No aggregation table or computed features. Would live in new AttackerAggregate. +- **Cheap fix**: Batch job: group events by attacker + hour-of-day, compute distribution histogram. Store as JSON or new table. +- **Priority**: V2 (simple aggregation; good for clustering). + +#### Session duration distribution +- **Status**: `partial` +- **Where**: SessionProfile schema (DEVELOPMENT_V2.md) includes `session_duration_s`. Asciinema files are per-decky-per-day, so duration can be computed. +- **Missing**: No SessionProfile table yet; no aggregation of durations across sessions. +- **Cheap fix**: Implement SessionProfile table + compute per-attacker duration histogram in AttackerAggregate. +- **Priority**: V2 (depends on SessionProfile; good for behavioral clustering). + +#### Recon-to-action ratio +- **Status**: `partial` +- **Where**: Profiler already computes recon vs. exfil phase sequencing (behavioral.py lines 52–62, 188–191). Stored in `AttackerBehavior.phase_sequence` (JSON: `{recon_end, exfil_start, latency}`). +- **Missing**: No per-attacker ratio column in AttackerAggregate. Would be simple division: `exfil_events / recon_events`. +- **Cheap fix**: Compute ratio in profiler job; store in new AttackerAggregate or as extension to AttackerBehavior. +- **Priority**: V2 (low effort; useful for threat level scoring). + +#### Lateral movement style +- **Status**: `not_captured` +- **Why**: Requires graph traversal (attacker hopping between deckies). Correlation engine (correlation/engine.py) should track this, but no explicit "lateral movement style" feature (sequential vs. parallel, target selection heuristic). +- **Missing**: No code analyzing lateral movement pattern (which deckies were touched, in what order, dwell time per decky). +- **Cheap fix**: Extend CorrelationEngine to build per-attacker decky traversal graph; compute metrics (average dwell time, fan-out ratio, revisit frequency). +- **Priority**: V2 (interesting; requires traversal graph extraction from correlation engine). + +#### Persistence-first vs. exfil-first +- **Status**: `not_captured` +- **Why**: Requires semantic tagging of events (is this persistence activity? exfil activity?). Profiler has `EXFIL_EVENT_TYPES` (line 59–62) but no persistence catalog. +- **Missing**: No code to classify persistence attempts (cron jobs, reverse shells, privilege escalation). +- **Cheap fix**: Add PERSISTENCE_EVENT_TYPES list; compute persistence_start vs. exfil_start timestamps; store in AttackerBehavior or AttackerAggregate. +- **Priority**: V2 (requires event taxonomy; valuable for threat classification). + +#### Tool-chain ordering +- **Status**: `partial` +- **Where**: Profiler logs tool guesses in AttackerBehavior.`tool_guesses` (line 183, behavioral.py lines 76–105). Tools are matched by beacon timing + header patterns. +- **Missing**: No **ordering** — tools are listed but not sequenced by first-appearance time. +- **Cheap fix**: Sort tool_guesses by first event timestamp; store as ordered list. Compute tool transition graph (tool A → tool B over time). +- **Priority**: V2 (interesting; small extension to existing tool attribution). + +#### Error-response psychology +- **Status**: `not_captured` +- **Why**: Requires analyzing how attacker reacts to failures (e.g., retry frequency after auth failure, command error recovery). Would need per-command success/failure tracking. +- **Missing**: No error-categorization in logs; would need service-level event typing (auth_failure vs. auth_success, exec_error vs. exec_success). +- **Cheap fix**: Extend service events to include success/failure indicators; compute attacker error-response metrics (retry rate, time-to-recovery, behavior change after error). +- **Priority**: V2 backlog (niche; good for human vs. bot discrimination). + +--- + +## Table Recommendations + +### `AttackerBehavior` — Current & Recommended Additions + +**Currently captured** (verified in models.py lines 161–194): +- `tcp_fingerprint` (JSON) — window, wscale, mss, options_sig +- `timing_stats` (JSON) — mean/median/stdev/min/max IAT +- `phase_sequence` (JSON) — recon_end, exfil_start latency +- `tool_guesses` (JSON list) +- `beacon_interval_s`, `beacon_jitter_pct` +- `behavior_class` (beaconing | interactive | scanning | …) + +**Recommended additions for v1 (pre-v2, no schema bump)**: +- `kex_order_raw` (MEDIUMTEXT, JSON list) — raw KEX algorithm strings from HASSH +- `tls_fingerprints_full` (MEDIUMTEXT, JSON) — full JA3/JA4 raw strings, not just hashes +- `ssh_client_banners` (MEDIUMTEXT, JSON list) — capture from TCP stream + +**Reserved for v2**: +- See SessionProfile below. + +### `SessionProfile` — New Table (v2 roadmap in DEVELOPMENT_V2.md) + +Design is already specified (lines 71–104). Implement in v1 as empty table + stubbed write path, ready for feature extraction post-v1. + +**Columns** (from DEVELOPMENT_V2.md): +- `sid` (TEXT PK) +- `log_id` (FK to logs) +- `schema_version` (INT, required for federation gossip) +- Timing features: `kd_iki_mean`, `kd_iki_stdev`, `kd_iki_p50`, `kd_iki_p95`, `kd_enter_latency_p50`, `kd_enter_latency_p95` +- Ratio features: `kd_burst_ratio`, `kd_think_ratio` +- Control-char rates: `kd_ctrl_backspace`, `kd_ctrl_wkill`, `kd_ctrl_ukill`, `kd_ctrl_abort`, `kd_ctrl_eof`, `kd_arrow_rate`, `kd_tab_rate` +- `kd_digraph_simhash` (BLOB, 8 bytes) +- Derived: `total_keystrokes`, `session_duration_s`, `created_at` + +**Note**: All keystroke-timing values are derivable from existing asciinema day-shard files on disk. Implement ingestion job in v2 (not v1 blocker). + +### `AttackerAggregate` — New Table (v2+) + +Columns (suggested): +- `attacker_uuid` (PK, FK to attackers) +- `activity_dist_by_hour` (JSON) — histogram of event counts by UTC hour +- `session_duration_dist` (JSON) — percentiles of session durations +- `recon_to_action_ratio` (REAL) +- `lateral_movement_graph` (JSON) — decky traversal (src → dst edges with dwell times) +- `tool_sequence` (JSON list) — tools in chronological order +- `is_persistent` (BOOL) — persistence activity detected? +- `updated_at` (TIMESTAMP) + +--- + +## Full Per-Signal Capture Table + +| Signal | Status | Where Captured | What's Missing | Cheap Fix | Priority | +|--------|--------|-----------------|-----------------|-----------|----------| +| **Session Environment** | +| TERM | partial | SSH pty-req, server-readable | No syslog emission, no DB | Patch SSH syslog bridge to emit term= | V2 | +| LANG/LC_ALL | n/a | Server locale, not attacker-controlled | Not visible from server vantage | Defer (not capturable) | defer | +| SSH client version | partial | TCP stream (plaintext banner before TLS) | Sniffer doesn't parse SSH banners; only TLS fingerprints | Extend sniffer to extract SSH banner from TCP stream | v1 blocker | +| Terminal size (COLS×ROWS) | not_captured | SSH pty-req extension | Requires protocol interception or sshd patch | Patch sshd to log pty-req | V2 | +| **Keyboard/Human** | +| Per-keystroke timing | partial | Asciinema "i" events with t timestamps | Files on disk, not ingested to DB | Implement SessionProfile table + ingest job | v1 blocker | +| Control-character stream | partial | Asciinema keystroke bytes | Same as above (files only) | Same as above | v1 blocker | +| Inter-command think time | not_captured | Requires prompt detection | Heuristic (line ending in $/#) not implemented | Post-hoc: regex + gap detection over asciinema | V2 | +| Pause before sensitive cmd | not_captured | Would be in asciinema timing | Requires command-line parsing + gap detection | Off-line analysis of asciinema | V2 | +| Command n-grams | partial | Attacker.commands (generic list) | Per-session structure missing | Parse asciinema I/O; store in SessionProfile | V2 | +| Flag preferences | not_captured | Asciinema input has typed flags | No parsing or normalization | Regex-parse and canonicalize flags from asciinema | V2 | +| Typo patterns | not_captured | Raw keystroke sequence in asciinema "i" | Requires keystroke-by-keystroke reconstruction | Parse "i" events with backspace markers; reconstruct line state | V2 | +| Editor choice | partial | Attacker.commands shows editor launch | No aggregation or time-in-editor | Count editor invocations; store preference in SessionProfile | V2 | +| Shell history usage | partial | Command input shows !, ^, !! | No parsing for history operators | Regex-scan for shell history syntax; count | V2 | +| **SSH Transport** | +| HASSH/HASSHServer | captured | Prober (hassh.py); Attacker.fingerprints | ✓ (hash + raw algorithm strings in syslog) | Already done | — | +| KEX algorithm order | partial | Syslog event kex_algorithms= field | Not persisted to DB (only in syslog) | Add AttackerBehavior.kex_order_raw (MEDIUMTEXT, JSON) | v1 blocker | +| Public key comment | not_captured | SSH wire format (auth_pubkey) | Requires server-side auth hook | Patch sshd to emit auth_pubkey_comment event | V2 | +| Private key type | partial | SSH wire format (auth algorithm OID) | Encrypted after KEX; needs sshd hook | Patch sshd to emit auth_key_type event | V2 | +| Agent forwarding? | not_captured | SSH extension negotiation (encrypted) | Requires sshd instrumentation | Patch sshd to detect ssh-agent@openssh.com | V2 | +| Channel multiplexing | partial | SSH service logs commands separately | No channel state machine | Instrument sshd SSH_MSG_CHANNEL_OPEN events | V2 | +| SSH_CLIENT env vars | captured | Server sets automatically; queryable via shell | No automatic logging | Patch sshd PAM to emit SSH_CLIENT on auth | V2 | +| **Network/Transport** | +| TCP timestamp skew | partial | PCAP + sniffer has has_timestamps flag | Only boolean; not timestamp values | Extract timestamp seq numbers in sniffer | V2 | +| TCP ISN generator | not_captured | PCAP SYN seq field | No per-connection ISN logging | Log syn_seq in tcp_syn_fingerprint event | V2 | +| TCP options ordering | partial | Sniffer extracts options_sig signature | Aggregated string; no raw order per-packet | Extend tcp_fingerprint JSON with raw options list | v1 improvement | +| Initial congestion window | not_captured | Would require per-packet ACK analysis | Not tracked in sniffer | Extend tcp_flow_timing to include payload sizes list | V2 | +| Retransmit timing+backoff | partial | Sniffer counts retransmits; no timing | RTO/backoff timing not logged | Extend event to include RTO deltas | V2 | +| MTU/path-MTU discovery | partial | MSS in TCP SYN; byte counts per flow | No ICMP fragmentation-needed events | Add ICMP processing; correlate with TCP flows | V2 | +| Packet pacing (μs) | not_captured | Sniffer uses millisecond granularity | Needs PCAP hardware timestamps or OS hooks | Upgrade to sub-millisecond timing | V2+ | +| Window scaling | captured | TCP fingerprint; wscale in AttackerBehavior | ✓ queryable | — | — | +| ECN negotiation | not_captured | TCP SYN flags (CWR/ECE) + options | Not extracted from TCP header | Extend TCP fingerprint to parse ECN bits | V2 | +| **L7 (TLS/HTTP)** | +| TLS fingerprint (JA3/JA4) | captured | Sniffer fingerprint.py; Attacker.fingerprints | ✓ hashes stored + syslog | Already done | — | +| HTTP/2 SETTINGS order | not_captured | Encrypted inside TLS | Passive inspection not viable | Defer (not capturable) | defer | +| HTTP/2 prioritization | not_captured | Encrypted | Not capturable | defer | defer | +| HTTP header ordering | not_captured | Encrypted; requires service logging | Service doesn't log raw headers | Patch HTTP service to log header order | V2 | +| Cookie handling | not_captured | Requires HTTP state machine | Not tracked | Add cookie jar logging to HTTP service | V2 | +| **Aggregated/Derived** | +| Time-of-day distribution | partial | Timestamps on all events | No aggregation table | Batch job: hour-of-day histogram → AttackerAggregate | V2 | +| Session duration dist | partial | SessionProfile would have duration | No SessionProfile table yet | Implement SessionProfile + duration stats | V2 | +| Recon-to-action ratio | partial | AttackerBehavior.phase_sequence | No per-attacker ratio column | Compute ratio in profiler; store in AttackerAggregate | V2 | +| Lateral movement style | not_captured | Correlation engine has traversal path | No traversal pattern analysis | Extend engine to compute dwell time + fan-out metrics | V2 | +| Persistence-first vs. exfil | not_captured | No persistence event taxonomy | Needs event-type classification | Add PERSISTENCE_EVENT_TYPES; compute timings | V2 | +| Tool-chain ordering | partial | tool_guesses list exists; unordered | No temporal ordering | Sort by first-event timestamp; build transition graph | V2 | +| Error-response psych | not_captured | No success/failure event tagging | Requires per-command outcome tracking | Extend service events with status=success/failure | V2 | + +--- + +## Pre-v1 Capture Gaps (Actionable, Blocky) + +**Only tackle these if the signal is committed to the v1 roadmap:** + +1. **KEX algorithm ordering** (ssh-transport) + - **Action**: Add `AttackerBehavior.kex_order_raw` (MEDIUMTEXT, JSON list of algorithm strings). + - **Effort**: 2 hrs (schema + sniffer event parser + profiler aggregator). + - **Blocker?**: Only if roadmap demands full KEX analysis (currently only HASSH hash is promised). + +2. **Per-keystroke timing ingestion** (keyboard/human) + - **Action**: Create `SessionProfile` table (design in DEVELOPMENT_V2.md); stub write path with all NULLs. + - **Effort**: 4 hrs (schema + migration + DAL). + - **Blocker?**: Yes, if keystroke dynamics is v1 roadmap. Data exists on disk but is not queryable. + +3. **SSH client banner capture** (ssh-transport) + - **Action**: Extend sniffer to parse SSH banners from TCP stream before TLS; emit ssh_client_hello event. + - **Effort**: 3 hrs (TCP stream parser + sniffer integration). + - **Blocker?**: Yes, if full SSH client profiling is v1 roadmap (currently only server banner via HASSH). + +4. **TCP options raw extraction** (network/transport) + - **Action**: Extend `tcp_fingerprint` JSON to include raw options list (not just signature string). + - **Effort**: 1 hr (minimal schema change + sniffer parser). + - **Blocker?**: No (options_sig is good enough for current p0f-style fingerprinting; nice-to-have). + +--- + +## Non-Capturable Signals (Explicit Deferral) + +These require vantage-point changes or are architecturally infeasible: + +| Signal | Why | Vantage Point Needed | +|--------|-----|----------------------| +| LANG / LC_ALL | Server locale is fixed; attacker's client locale invisible over SSH | Client-side instrumentation | +| HTTP/2 SETTINGS frame order | Encrypted inside TLS stream | Server-side decryption or key log | +| HTTP/2 stream prioritization | Encrypted | Server-side decryption | +| Initial congestion window (CWND) | Requires detailed TCP ACK-clock tracking | Per-packet sniffer instrumentation | +| Packet pacing (μs resolution) | Requires hardware-timestamped PCAP or kernel hooks | OS-level instrumentation | +| Hold time / pressure / velocity (typing biometrics) | Not on SSH wire | Client-side TLS instrumentation | + +--- + +## Summary for v1 Release + +**Ship with these (already captured, queryable)**: +- HASSH/HASSHServer ✓ +- JA3/JA3S/JA4/JA4S ✓ +- TLS session resumption ✓ +- TCP fingerprint (window, wscale, mss, options_sig) ✓ +- Behavioral timing stats (mean/median/stdev IAT) ✓ +- Phase sequencing (recon_end, exfil_start) ✓ +- Tool attribution (beacon timing + headers) ✓ + +**Data exists on disk, not queryable (v1 deferral acceptable)**: +- Per-keystroke timing (asciinema day-shards) — needs SessionProfile ingestion job +- SSH client banner (TCP stream) — needs sniffer enhancement +- KEX algorithm order (syslog) — needs AttackerBehavior.kex_order_raw column + +**Requires infrastructure changes (v2+)**: +- Lateral movement graph analysis +- HTTP header order + cookie jar behavior +- Persistence-first vs. exfil-first classification +- Error-response psychology +- Chronotyping + session duration distribution + +--- + +## Federation & Cross-Operator Gossip (v2 Implications) + +The `SessionProfile` schema (table, schema_version field, numeric features) is designed to be the federation wire format. **No changes needed for v1**, but ensure schema_version is in the table definition from day one so gossip compatibility is straightforward in v2. + +--- + +## Appendices + +### A. Code Paths Audited + +- `decnet/sniffer/fingerprint.py` — TLS + TCP fingerprinting engine +- `decnet/services/ssh.py` — SSH service config + artifact paths +- `decnet/prober/hassh.py` — HASSHServer computation +- `decnet/web/db/models.py` — SQL schema (Attacker, AttackerBehavior, etc.) +- `decnet/profiler/behavioral.py` — Timing + tool attribution +- `decnet/correlation/parser.py` — RFC 5424 syslog ingestion +- `decnet/templates/ssh/` — Session recording (asciinema), syslog bridge, capture.sh + +### B. Storage Destinations Verified + +- **Database**: SQLite/MySQL tables (Attacker, AttackerBehavior, Bounty, Log) +- **Syslog**: RFC 5424 events (parsed by correlation engine, optionally piped to ELK) +- **Disk**: Asciinema day-shards (`/var/lib/decnet/session_recordings/`), raw PCAP (retention TBD) +- **Memory**: Sniffer state (sessions, flows, dedup cache) — lost on restart unless replayed from PCAP + +### C. Roadmap Cross-Reference + +- DEVELOPMENT.md lines 48–133: Attacker Intelligence Collection (TLS, behavioral, protocol fingerprinting, network topology, geolocation, service-level, aggregated). + - `[x]` JA3/JA3S, JA4+, JARM, session resumption, TCP window/scaling, retransmits, beaconing, data exfil timing, HASSH/HASSHServer, HTTP/2 fingerprint, TLS session resumption, TTL values (partial), TCP stack fingerprinting. + - `[ ]` (not v1): ISN patterns, HTTP header ordering, QUIC, DNS, IPv6/mDNS leakage, geolocation, service-level commands, credential reuse, payload signatures. + +- DEVELOPMENT_V2.md: Keystroke dynamics, session profiling, federation. + - SessionProfile schema (lines 71–104) — not yet implemented; ready-to-implement design. + - Correlation via simhash (lines 50–56) — digraph rhythm fingerprinting. + +--- + diff --git a/api-audit.md b/api-audit.md new file mode 100644 index 00000000..b95b32b4 --- /dev/null +++ b/api-audit.md @@ -0,0 +1,1000 @@ +# FastAPI /api/v1 Route Audit Report + +## Executive Summary + +**Total Routes Analyzed**: 77 +**Deletion Candidates**: 54 + - **Zero Callers (dead code)**: 7 + - **Test-Only (replaced routes?)**: 47 + +The audit scanned: +- 77 registered `/api/v1/*` routes across the FastAPI web application +- All sources: frontend TypeScript/React, CLI, worker processes, and test suites +- Frontend path fragment matching (e.g., searching for `/topologies/` in dynamic URLs) + +**Top Deletion Candidates for Review**: +- Attacker detail endpoints (`/attackers/{uuid}*`) — 5 test-only routes, no web/CLI callers +- Decky mutation endpoints (`/deckies/{decky_name}/mutate*`) — 2 zero-caller routes (likely replaced by mutation queue) +- Various CRUD endpoints with test-only usage — likely superseded by newer flows + +--- + +## Full Route Inventory + +| Method | Path | Handler | File | Caller Types | Notes | +|--------|------|---------|------|--------------|-------| +| GET | `/` | `api_list_topologies()` | api_list_topologies.py | cli, test | | +| POST | `/` | `api_create_topology()` | api_create_topology.py | cli, test | | +| GET | `/archetypes` | `api_list_archetypes()` | api_catalog.py | **NONE** | ⚠️ | +| GET | `/artifacts/{decky}/{stored_as}` | `get_artifact()` | api_get_artifact.py | test | ⚠️ | +| GET | `/attackers` | `get_attackers()` | api_get_attackers.py | test, web | | +| GET | `/attackers/{uuid}` | `get_attacker_detail()` | api_get_attacker_detail.py | test | ⚠️ | +| GET | `/attackers/{uuid}/artifacts` | `get_attacker_artifacts()` | api_get_attacker_artifacts.py | test | ⚠️ | +| GET | `/attackers/{uuid}/commands` | `get_attacker_commands()` | api_get_attacker_commands.py | test | ⚠️ | +| GET | `/attackers/{uuid}/transcripts` | `get_attacker_transcripts()` | api_get_attacker_transcripts.py | test | ⚠️ | +| POST | `/auth/change-password` | `change_password()` | api_change_pass.py | test | ⚠️ | +| POST | `/auth/login` | `login()` | api_login.py | test | ⚠️ | +| POST | `/blank` | `api_create_blank_topology()` | api_create_blank_topology.py | test | ⚠️ | +| GET | `/bounty` | `get_bounties()` | api_get_bounties.py | test | ⚠️ | +| POST | `/check` | `api_check_hosts()` | api_check_hosts.py | cli, test | | +| GET | `/config` | `api_get_config()` | api_get_config.py | cli, test, web | | +| PUT | `/config/deployment-limit` | `api_update_deployment_limit()` | api_update_config.py | test, web | | +| PUT | `/config/global-mutation-interval` | `api_update_global_mutation_interval()` | api_update_config.py | test, web | | +| DELETE | `/config/reinit` | `api_reinit()` | api_reinit.py | test, web | | +| POST | `/config/users` | `api_create_user()` | api_manage_users.py | test, web | | +| DELETE | `/config/users/{user_uuid}` | `api_delete_user()` | api_manage_users.py | test | ⚠️ | +| PUT | `/config/users/{user_uuid}/reset-password` | `api_reset_user_password()` | api_manage_users.py | test | ⚠️ | +| PUT | `/config/users/{user_uuid}/role` | `api_update_user_role()` | api_manage_users.py | test | ⚠️ | +| GET | `/deckies` | `get_deckies()` | api_get_deckies.py | cli, test, web | | +| GET | `/deckies` | `api_list_deckies()` | api_list_deckies.py | cli, test, web | | +| GET | `/deckies` | `list_deckies()` | api_list_deckies.py | cli, test, web | | +| POST | `/deckies/deploy` | `api_deploy_deckies()` | api_deploy_deckies.py | test, web | | +| POST | `/deckies/{decky_name}/mutate` | `api_mutate_decky()` | api_mutate_decky.py | **NONE** | ⚠️ | +| PUT | `/deckies/{decky_name}/mutate-interval` | `api_update_mutate_interval()` | api_mutate_interval.py | **NONE** | ⚠️ | +| POST | `/deploy` | `api_deploy_swarm()` | api_deploy_swarm.py | cli, test | | +| GET | `/deployment-mode` | `get_deployment_mode()` | api_deployment_mode.py | test | ⚠️ | +| POST | `/enroll` | `api_enroll_host()` | api_enroll_host.py | cli, test | | +| POST | `/enroll-bundle` | `create_enroll_bundle()` | api_enroll_bundle.py | test | ⚠️ | +| GET | `/enroll-bundle/{token}.sh` | `get_bootstrap()` | api_enroll_bundle.py | test | ⚠️ | +| GET | `/enroll-bundle/{token}.tgz` | `get_payload()` | api_enroll_bundle.py | test | ⚠️ | +| GET | `/health` | `get_health()` | api_get_health.py | cli, test | | +| GET | `/health` | `api_get_swarm_health()` | api_get_swarm_health.py | cli, test | | +| POST | `/heartbeat` | `heartbeat()` | api_heartbeat.py | test | ⚠️ | +| GET | `/hosts` | `api_list_hosts()` | api_list_hosts.py | cli, test | | +| GET | `/hosts` | `list_hosts()` | api_list_hosts.py | cli, test | | +| GET | `/hosts` | `api_list_host_releases()` | api_list_host_releases.py | cli, test | | +| DELETE | `/hosts/{uuid}` | `api_decommission_host()` | api_decommission_host.py | test | ⚠️ | +| DELETE | `/hosts/{uuid}` | `decommission_host()` | api_decommission_host.py | test | ⚠️ | +| GET | `/hosts/{uuid}` | `api_get_host()` | api_get_host.py | test | ⚠️ | +| POST | `/hosts/{uuid}/teardown` | `teardown_host()` | api_teardown_host.py | test | ⚠️ | +| GET | `/logs` | `get_logs()` | api_get_logs.py | test | ⚠️ | +| GET | `/logs/histogram` | `get_logs_histogram()` | api_get_histogram.py | test | ⚠️ | +| GET | `/next-subnet` | `api_next_subnet()` | api_catalog.py | test | ⚠️ | +| POST | `/push` | `api_push_update()` | api_push_update.py | test | ⚠️ | +| POST | `/push-self` | `api_push_update_self()` | api_push_update_self.py | test | ⚠️ | +| POST | `/reap-orphans` | `api_reap_orphans()` | api_reap_orphans.py | test | ⚠️ | +| POST | `/rollback` | `api_rollback_host()` | api_rollback_host.py | test | ⚠️ | +| GET | `/services` | `api_list_services()` | api_catalog.py | test | ⚠️ | +| GET | `/stats` | `get_stats()` | api_get_stats.py | test | ⚠️ | +| GET | `/stream` | `stream_events()` | api_stream_events.py | test, web | | +| POST | `/teardown` | `api_teardown_swarm()` | api_teardown_swarm.py | test | ⚠️ | +| GET | `/transcripts/{decky}/{sid}` | `get_transcript()` | api_get_transcript.py | test | ⚠️ | +| GET | `/workers` | `list_workers()` | api_list_workers.py | test, web | | +| POST | `/workers/start-all` | `start_all_workers()` | api_start_all_workers.py | test, web | | +| POST | `/workers/{name}/start` | `start_worker()` | api_start_worker.py | test | ⚠️ | +| POST | `/workers/{name}/stop` | `stop_worker()` | api_control_worker.py | test | ⚠️ | +| DELETE | `/{topology_id}` | `api_delete_topology()` | api_delete_topology.py | test | ⚠️ | +| GET | `/{topology_id}` | `api_get_topology()` | api_get_topology.py | test | ⚠️ | +| POST | `/{topology_id}/deckies` | `api_create_decky()` | api_decky_crud.py | test | ⚠️ | +| DELETE | `/{topology_id}/deckies/{decky_uuid}` | `api_delete_decky()` | api_decky_crud.py | test | ⚠️ | +| PATCH | `/{topology_id}/deckies/{decky_uuid}` | `api_update_decky()` | api_decky_crud.py | test | ⚠️ | +| POST | `/{topology_id}/deploy` | `api_deploy_topology()` | api_deploy_topology.py | test | ⚠️ | +| POST | `/{topology_id}/edges` | `api_create_edge()` | api_edge_crud.py | test | ⚠️ | +| DELETE | `/{topology_id}/edges/{edge_id}` | `api_delete_edge()` | api_edge_crud.py | test | ⚠️ | +| GET | `/{topology_id}/events` | `api_topology_events()` | api_events.py | **NONE** | ⚠️ | +| POST | `/{topology_id}/lans` | `api_create_lan()` | api_lan_crud.py | test | ⚠️ | +| DELETE | `/{topology_id}/lans/{lan_id}` | `api_delete_lan()` | api_lan_crud.py | test | ⚠️ | +| PATCH | `/{topology_id}/lans/{lan_id}` | `api_update_lan()` | api_lan_crud.py | test | ⚠️ | +| GET | `/{topology_id}/lans/{lan_id}/next-ip` | `api_next_ip()` | api_catalog.py | **NONE** | ⚠️ | +| GET | `/{topology_id}/mutations` | `api_list_mutations()` | api_mutations.py | test | ⚠️ | +| POST | `/{topology_id}/mutations` | `api_enqueue_mutation()` | api_mutations.py | test | ⚠️ | +| GET | `/{topology_id}/status-events` | `api_get_status_events()` | api_get_topology.py | **NONE** | ⚠️ | +| POST | `/{topology_id}/teardown` | `api_teardown_topology()` | api_teardown_topology.py | **NONE** | ⚠️ | + + +--- + +## Deletion Candidates: Zero Callers + +These routes have **no callers anywhere** in the codebase (except their own definition and possibly tests). They are strong candidates for removal. + +### GET `/archetypes` → `api_list_archetypes()` + +**File**: `decnet/web/router/topology/api_catalog.py` +**Callers**: None +**Status**: Dead code — no references in web frontend, CLI, or worker processes. + +**Action**: Safe to delete. If tests exist, they are testing orphaned endpoints. + +--- + +### POST `/deckies/{decky_name}/mutate` → `api_mutate_decky()` + +**File**: `decnet/web/router/fleet/api_mutate_decky.py` +**Callers**: None +**Status**: Dead code — no references in web frontend, CLI, or worker processes. + +**Action**: Safe to delete. If tests exist, they are testing orphaned endpoints. + +--- + +### PUT `/deckies/{decky_name}/mutate-interval` → `api_update_mutate_interval()` + +**File**: `decnet/web/router/fleet/api_mutate_interval.py` +**Callers**: None +**Status**: Dead code — no references in web frontend, CLI, or worker processes. + +**Action**: Safe to delete. If tests exist, they are testing orphaned endpoints. + +--- + +### GET `/{topology_id}/events` → `api_topology_events()` + +**File**: `decnet/web/router/topology/api_events.py` +**Callers**: None +**Status**: Dead code — no references in web frontend, CLI, or worker processes. + +**Action**: Safe to delete. If tests exist, they are testing orphaned endpoints. + +--- + +### GET `/{topology_id}/lans/{lan_id}/next-ip` → `api_next_ip()` + +**File**: `decnet/web/router/topology/api_catalog.py` +**Callers**: None +**Status**: Dead code — no references in web frontend, CLI, or worker processes. + +**Action**: Safe to delete. If tests exist, they are testing orphaned endpoints. + +--- + +### GET `/{topology_id}/status-events` → `api_get_status_events()` + +**File**: `decnet/web/router/topology/api_get_topology.py` +**Callers**: None +**Status**: Dead code — no references in web frontend, CLI, or worker processes. + +**Action**: Safe to delete. If tests exist, they are testing orphaned endpoints. + +--- + +### POST `/{topology_id}/teardown` → `api_teardown_topology()` + +**File**: `decnet/web/router/topology/api_teardown_topology.py` +**Callers**: None +**Status**: Dead code — no references in web frontend, CLI, or worker processes. + +**Action**: Safe to delete. If tests exist, they are testing orphaned endpoints. + +--- + +## Deletion Candidates: Test-Only Routes + +These routes are referenced **only in test files**, not in the actual application. They may have been replaced by newer endpoints and are kept for backward-compatibility testing, or tests simply weren't updated after migration. + +**Count**: 47 routes + + +### Artifacts (1) + +- `GET /artifacts/{decky}/{stored_as}` (api_get_artifact.py) + +### Attackers (4) + +- `GET /attackers/{uuid}` (api_get_attacker_detail.py) +- `GET /attackers/{uuid}/artifacts` (api_get_attacker_artifacts.py) +- `GET /attackers/{uuid}/commands` (api_get_attacker_commands.py) +- ... and 1 more + +### Auth (2) + +- `POST /auth/change-password` (api_change_pass.py) +- `POST /auth/login` (api_login.py) + +### Blank (1) + +- `POST /blank` (api_create_blank_topology.py) + +### Bounty (1) + +- `GET /bounty` (api_get_bounties.py) + +### Config (3) + +- `DELETE /config/users/{user_uuid}` (api_manage_users.py) +- `PUT /config/users/{user_uuid}/reset-password` (api_manage_users.py) +- `PUT /config/users/{user_uuid}/role` (api_manage_users.py) + +### Deployment-Mode (1) + +- `GET /deployment-mode` (api_deployment_mode.py) + +### Enroll-Bundle (3) + +- `POST /enroll-bundle` (api_enroll_bundle.py) +- `GET /enroll-bundle/{token}.sh` (api_enroll_bundle.py) +- `GET /enroll-bundle/{token}.tgz` (api_enroll_bundle.py) + +### Heartbeat (1) + +- `POST /heartbeat` (api_heartbeat.py) + +### Hosts (4) + +- `DELETE /hosts/{uuid}` (api_decommission_host.py) +- `GET /hosts/{uuid}` (api_get_host.py) +- `DELETE /hosts/{uuid}` (api_decommission_host.py) +- ... and 1 more + +### Logs (2) + +- `GET /logs` (api_get_logs.py) +- `GET /logs/histogram` (api_get_histogram.py) + +### Next-Subnet (1) + +- `GET /next-subnet` (api_catalog.py) + +### Push (1) + +- `POST /push` (api_push_update.py) + +### Push-Self (1) + +- `POST /push-self` (api_push_update_self.py) + +### Reap-Orphans (1) + +- `POST /reap-orphans` (api_reap_orphans.py) + +### Rollback (1) + +- `POST /rollback` (api_rollback_host.py) + +### Services (1) + +- `GET /services` (api_catalog.py) + +### Stats (1) + +- `GET /stats` (api_get_stats.py) + +### Teardown (1) + +- `POST /teardown` (api_teardown_swarm.py) + +### Transcripts (1) + +- `GET /transcripts/{decky}/{sid}` (api_get_transcript.py) + +### Workers (2) + +- `POST /workers/{name}/start` (api_start_worker.py) +- `POST /workers/{name}/stop` (api_control_worker.py) + +### {Topology_Id} (13) + +- `DELETE /{topology_id}` (api_delete_topology.py) +- `GET /{topology_id}` (api_get_topology.py) +- `POST /{topology_id}/deckies` (api_decky_crud.py) +- ... and 10 more + + +--- + +## Analysis Notes + +### Context from Recent Work + +Per repo history: +- **Bus-woken mutator** replaced polling — check `/deckies/*` mutation endpoints +- **SSE mutation events** replaced direct CRUD polling — check legacy list endpoints +- **Worker supervisor endpoints** are new — likely need expansion, not deletion +- **MazeNET topologies** are the new feature — older "topology" endpoints may be superseded +- **Direct mutation CRUD for active topologies** replaced by mutation queue + +### Methodology + +- **Web Frontend**: Searched `decnet_web/src/**/*.{ts,tsx}` for literal path references (e.g., `"/attackers/{uuid}"`) +- **CLI**: Searched `decnet/cli/**/*.py` for `/api/v1` calls +- **Workers**: Searched `decnet//**/*.py` (excluding CLI) +- **Tests**: Searched `tests/**/*.py` for path references + +### Caveats + +- Dynamically-built paths (e.g., `${base}/topologies/${id}`) detected via fragment search (e.g., `/topologies/`) +- Method-less references (e.g., just the path string) may miss some usages if not called via fetch/axios +- mTLS/internal worker endpoints (agent API, forwarder, enroll-bundle) deferred to Phase 2 per scope + +--- + +## Possible Duplicates / Overlapping Endpoints + +_To be populated after human review of the candidate list._ + + +--- + +## Phase 2 — Worker / mTLS Endpoints + +### Executive Summary + +**Scope**: Internal worker processes and mTLS-gated inter-process HTTP surfaces: +- Agent FastAPI app (port 8765, mTLS-required) +- Updater FastAPI app (port 8766, mTLS-required, CN-gated) +- Master→Agent client calls via `AgentClient` class +- Master→Updater client calls via `UpdaterClient` class +- Enroll-bundle endpoints (`/swarm/enroll-bundle`) — worker-facing, fetches bootstrap + deployment payload +- Enrollment endpoints (`/swarm/enroll`) — admin-driven, issues certs + +**Total Worker Process Endpoints**: 12 +**Total Deletion Candidates**: 0 (all have active callers) + +--- + +### Worker Process HTTP Endpoints + +#### Agent FastAPI App (`decnet/agent/app.py`) + +**Listener**: Port 8765, mTLS-enforced at ASGI/uvicorn layer (cert required) +**Callers**: Master via `AgentClient`, deployer module, CLI +**Auth**: mTLS only; all authenticated peers trusted equally + +| Method | Path | Handler | Callers | Notes | +|--------|------|---------|---------|-------| +| GET | `/health` | `health()` | master-to-agent, tests | Liveness probe; does NOT skip mTLS | +| GET | `/status` | `status()` | master-to-agent, engine deployer | Deployment snapshot + active topology state | +| POST | `/deploy` | `deploy()` | master-to-agent, engine deployer | Materialise full DecnetConfig (body: `DeployRequest`) | +| POST | `/teardown` | `teardown()` | master-to-agent | Dismantle entire fleet or single decky (body: `TeardownRequest`) | +| POST | `/self-destruct` | `self_destruct()` | master-to-agent | Fire-and-forget reaper; deletes all DECNET footprint (202 response) | +| POST | `/topology/apply` | `topology_apply()` | master-to-agent | Apply a single topology (body: `ApplyTopologyRequest`) | +| POST | `/topology/teardown` | `topology_teardown()` | master-to-agent | Dismantle single topology (body: `TeardownTopologyRequest`) | +| GET | `/topology/state` | `topology_state()` | master-to-agent | Topology-specific state (separate from `/status`) | +| POST | `/mutate` | `mutate()` | (unimplemented, returns 501) | Per-decky mutate; currently done via `/deploy` with updated config | + +**Timeouts**: Deploy/topology-apply 600s read, teardown 300s read (docker compose on slow VMs) + +--- + +#### Updater FastAPI App (`decnet/updater/app.py`) + +**Listener**: Port 8766, mTLS-enforced (cert CN must match `updater@*`) +**Callers**: Master via `UpdaterClient` +**Auth**: mTLS + CN validation (only `updater@` certs allowed) + +| Method | Path | Handler | Callers | Notes | +|--------|------|---------|---------|-------| +| GET | `/health` | `health()` | master-to-updater, dashboard, bus monitor | Returns active + prev release slots | +| GET | `/releases` | `releases()` | master-to-updater | List all available release slots (JSON array) | +| POST | `/update` | `update()` | master-to-updater | Upload + apply tarball (multipart: tarball + sha form) | +| POST | `/update-self` | `update_self()` | master-to-updater | Self-update updater binary (connection drops mid-response) | +| POST | `/rollback` | `rollback()` | master-to-updater | Revert to previous release slot | + +**Timeouts**: `/update` + `/update-self` 180s read (pip install + probe on slow VMs) + +--- + +### Master-Facing Worker Enrollment Endpoints + +#### Enrollment Bundle (`decnet/web/router/swarm_mgmt/api_enroll_bundle.py`) + +**Listener**: Master port 443 (FastAPI web app) +**Callers**: agent (worker fetches payload), admin UI +**Auth**: Token-based (5-min TTL), no mTLS required (public endpoints for worker bootstrap) + +| Method | Path | Handler | Callers | Auth | Notes | +|--------|------|---------|---------|------|-------| +| POST | `/api/v1/swarm/enroll-bundle` | `create_enroll_bundle()` | admin-ui, cli | require_admin | Create bundle (token + shell script + tarball); returns EnrollBundleResponse (201) | +| GET | `/api/v1/swarm/enroll-bundle/{token}.sh` | `get_bootstrap()` | agent-client, curl | token-param | Bootstrap shell script (idempotent, 5-min TTL) | +| GET | `/api/v1/swarm/enroll-bundle/{token}.tgz` | `get_payload()` | agent-client, curl | token-param | Gzipped tarball (one-shot; deletes .sh + .tgz after serving) | + +**Rationale**: Agent's first contact-home; its source IP backfills the `SwarmHost.address` row. + +--- + +#### Simple Enrollment (`decnet/web/router/swarm/api_enroll_host.py`) + +**Listener**: Master port 443 +**Callers**: admin UI, CLI +**Auth**: None (browser-facing, admin dashboard context) + +| Method | Path | Handler | Callers | Auth | Notes | +|--------|------|---------|---------|------|-------| +| POST | `/api/v1/swarm/enroll` | `api_enroll_host()` | admin-ui | (browser auth) | Issue cert bundle + register host row (201) | + +--- + +### Master→Agent RPC Surface (via `AgentClient`) + +Master calls agent via `AgentClient(host).method()` context manager. All calls are mTLS. Called from: + +1. **`api_deploy_swarm.py`**: Deploy topology to all enrolled hosts +2. **`api_teardown_swarm.py`**: Teardown fleet +3. **`api_check_hosts.py`**: Active mTLS probe of all hosts (for dashboard health) +4. **`api_decommission_host.py`** (swarm): Calls agent `/self-destruct` +5. **`api_decommission_host.py`** (swarm_mgmt): Calls agent `/self-destruct` +6. **`api_teardown_host.py`** (swarm_mgmt): Calls agent `/self-destruct` +7. **`api_list_hosts.py`** (swarm_mgmt): Calls agent `/health` on every list request +8. **Engine `deployer.py`**: Direct `/deploy` + `/topology/apply` calls during mutation/materialization + +**Cert Pinning**: Master's cert is CA-signed; workers validate via CA pinning + master hostname-verification disabled (per-operator SANs). + +--- + +### Master→Updater RPC Surface (via `UpdaterClient`) + +Master calls updater via `UpdaterClient(host).method()` context manager. All calls are mTLS. Called from: + +1. **`api_push_update.py`**: Upload new release to updater +2. **`api_push_update_self.py`**: Update the updater binary itself +3. **`api_rollback_host.py`**: Rollback updater to previous release +4. **`api_list_host_releases.py`**: Poll all updaters for active release SHA (dashboard) + +**Connection Drop**: `/update-self` intentionally drops the connection; caller polls `/health` for new SHA. + +--- + +### Agent→Master Heartbeat + +**Endpoint**: `POST /api/v1/swarm/heartbeat` +**Caller**: `decnet/agent/heartbeat.py` module (agent-side daemon) +**Auth**: mTLS + peer cert SHA-256 pinned to `SwarmHost.client_cert_fingerprint` +**Frequency**: ~30 seconds +**Payload**: Host UUID, agent version, executor status dict, optional topology snapshot + +**Security**: Decommissioned workers' still-valid certs must not resurrect ghost shards → cert fingerprint mismatch → 403. + +--- + +### Bus Pub/Sub (Local Only, Not HTTP) + +Per comments in agent/updater app.py: +- Agent publishes `system.agent.health` heartbeat to local bus (separate from mTLS heartbeat) +- Updater publishes `system.updater.health` to local bus +- Bus is host-local UNIX socket — not an external RPC surface + +No HTTP endpoints; no caller analysis needed. + +--- + +### Forwarder + +**Status**: No HTTP endpoints exposed by forwarder process. +The forwarder: +- Consumes RFC 5424 syslog from local log file (written by agent log collector) +- Ships syslog-over-TLS to master port 6514 (outbound, not inbound) +- No master→forwarder calls; no worker-side HTTP surface + +--- + +### Deletion Candidates + +**None.** All identified endpoints have active callers: + +- Agent `/deploy`, `/teardown`, `/self-destruct`, `/topology/*` are called by engine, deployer, master probes +- Updater `/update*`, `/releases`, `/health` are called by master push flow + dashboard +- Enroll-bundle is called by new agents (worker-facing enrollment) +- Simple enroll is called by admin UI + +--- + +### Duplicate / Obsolete Endpoints + +**Potential overlap to review**: + +1. **`/swarm/enroll` vs `/swarm/enroll-bundle`**: Two enrollment flows, both active. + - `/enroll` (old) — admin issues cert + agent curls back for bundle + - `/enroll-bundle` (new) — admin renders bundle upfront, agent one-liners it + - Consider consolidating if old flow is being phased out (need human review of intent). + +2. **Agent `/deploy` + `/teardown` vs `/topology/apply` + `/topology/teardown`**: Both exist. + - `/deploy` — fleet-wide (old unihost verb) + - `/topology/{apply,teardown}` — single topology (newer MazeNET feature) + - No conflict; different scopes. Agent supports both. + +3. **Agent `/mutate` returns 501**: Placeholder for future worker-side mutation. + - Currently master re-sends `/deploy` with updated config. + - Safe to leave as-is (fails closed); can implement later. + +--- + +### Summary Table + +| Process | Count | mTLS | Auth | Notes | +|---------|-------|------|------|-------| +| Agent | 9 | Yes | No (peer auth only) | Port 8765; calls from master + engine | +| Updater | 5 | Yes | Yes (CN-gated) | Port 8766; calls from master | +| Enroll-Bundle | 3 | No | Token (5 min) | Master port 443; agent + admin fetch | +| Enroll | 1 | No | Browser auth | Master port 443; admin UI | +| **Total** | **18** | — | — | — | + +**Caller Types Identified**: +- `master-to-agent`: Master calls agent (9 endpoints) +- `master-to-updater`: Master calls updater (5 endpoints) +- `agent-client`: Agent calls master heartbeat (1 endpoint in Phase 1) +- `admin-client`: Admin calls enroll-bundle POST (1 endpoint) +- `test`: All endpoints have test coverage + +**Zero-Caller Endpoints**: None. + +--- + + +## Phase 3 — CLI Command Surface + +### Summary + +**Total CLI Commands**: 37 +**Master-only Commands**: 27 (via `MASTER_ONLY_COMMANDS` + `MASTER_ONLY_GROUPS`) +**Agent-capable Commands**: 10 (hidden in agent mode when `DECNET_MODE=agent`) +**Commands Hitting API Routes**: 7 (all in `decnet swarm *` group, plus `decnet deploy`) +**Deletion Candidates**: 0 (no deprecated commands found; all are actively used) + +--- + +### Full Command Inventory + +| Command | Handler | Source | Master-only? | Hits API? | Notes | +|---------|---------|--------|--------------|-----------|-------| +| `decnet api` | `api()` | api.py:19 | Yes | No | Start FastAPI backend (uvicorn) | +| `decnet swarmctl` | `swarmctl()` | swarmctl.py:18 | Yes | No | Run SWARM controller + auto-spawn listener | +| `decnet agent` | `agent()` | agent.py:16 | No | No | Worker: run SWARM agent (requires cert bundle) | +| `decnet updater` | `updater()` | updater.py:14 | No | No | Worker: run self-updater daemon | +| `decnet listener` | `listener()` | listener.py:16 | Yes | No | Run syslog-TLS listener (RFC 5425, mTLS) | +| `decnet forwarder` | `forwarder()` | forwarder.py:18 | No | No | Worker: forward syslog to master:6514 (mTLS) | +| `decnet deploy` | `deploy()` | deploy.py:68 | Yes | Yes | Deploy deckies (unihost/swarm mode) | +| `decnet init` | `init_cmd()` | init.py:305 | Yes | No | Bootstrap master: user/group/systemd/config | +| `decnet services` | `list_services()` | inventory.py:15 | No | No | List available service plugins | +| `decnet distros` | `list_distros()` | inventory.py:27 | No | No | List available OS distro profiles | +| `decnet archetypes` | `list_archetypes()` | inventory.py:38 | Yes | No | List machine archetype profiles | +| `decnet redeploy` | `redeploy()` | lifecycle.py:18 | No | No | Check services + relaunch any down | +| `decnet status` | `status()` | lifecycle.py:57 | No | No | Show running deckies + service status | +| `decnet teardown` | `teardown()` | lifecycle.py:81 | Yes | No | Stop/remove deckies (--all or --id) | +| `decnet probe` | `probe()` | workers.py:15 | No | No | Fingerprint attackers (JARM/HASSH) | +| `decnet collect` | `collect()` | workers.py:40 | No | No | Stream Docker logs to RFC 5424 file | +| `decnet mutate` | `mutate()` | workers.py:57 | Yes | No | Trigger/watch decky mutation | +| `decnet correlate` | `correlate()` | workers.py:86 | Yes | No | Analyse logs for cross-decky traversals | +| `decnet web` | `serve_web()` | web.py:13 | Yes | No | Serve frontend SPA + proxy /api/* | +| `decnet profiler` | `profiler_cmd()` | profiler.py:11 | Yes | No | Build attacker profiles from log stream | +| `decnet sniffer` | `sniffer_cmd()` | sniffer.py:12 | Yes | No | Passive network sniffer | +| `decnet db-reset` | `db_reset()` | db.py:86 | Yes | No | Wipe MySQL database (truncate or drop-tables) | +| `decnet bus` | `bus_cmd()` | bus.py:11 | No | No | Run UNIX-socket pub/sub bus worker | +| `decnet swarm enroll` | `swarm_enroll()` | swarm.py:23 | Yes | Yes | Enroll worker + issue mTLS bundle → POST `/swarm/enroll` | +| `decnet swarm list` | `swarm_list()` | swarm.py:85 | Yes | Yes | List enrolled workers → GET `/swarm/hosts` | +| `decnet swarm check` | `swarm_check()` | swarm.py:111 | Yes | Yes | Probe worker status → POST `/swarm/check` | +| `decnet swarm update` | `swarm_update()` | swarm.py:149 | Yes | Yes | Push tarball to workers → GET `/swarm/hosts` + updater client | +| `decnet swarm deckies` | `swarm_deckies()` | swarm.py:256 | Yes | Yes | List deckies across swarm → GET `/swarm/deckies` | +| `decnet swarm decommission` | `swarm_decommission()` | swarm.py:315 | Yes | Yes | Remove worker from swarm → DELETE `/swarm/hosts/{uuid}` | +| `decnet topology generate` | `_generate()` | topology.py:35 | Yes | No | Generate topology plan (persist as pending) | +| `decnet topology list` | `_list()` | topology.py:94 | Yes | No | List all topologies | +| `decnet topology show` | `_show()` | topology.py:121 | Yes | No | Print topology structure | +| `decnet topology deploy` | `_deploy()` | topology.py:177 | Yes | No | Deploy pending topology | +| `decnet topology teardown` | `_teardown()` | topology.py:194 | Yes | No | Tear down active topology | +| `decnet topology delete` | `_delete()` | topology.py:210 | Yes | No | Delete topology + cascade (LANs/deckies/edges) | +| `decnet topology mutate` | `_mutate()` | topology.py:265 | Yes | No | Enqueue live topology mutation | +| `decnet topology mutations` | `_mutations()` | topology.py:310 | Yes | No | List queued/applied mutations | + +--- + +### Commands Hitting API Routes + +All 7 commands that call HTTP endpoints go through **swarmctl** (not the main `/api/v1` backend). These are: + +1. **`decnet deploy`** (swarm mode) + - Hits: `GET /swarm/hosts?host_status=enrolled`, `GET /swarm/hosts?host_status=active`, `POST /swarm/deploy` + - Route source: `decnet/web/swarm_api.py` (Swarmctl API, not Phase 1 audit scope) + +2. **`decnet swarm enroll`** + - Hits: `POST /swarm/enroll` + +3. **`decnet swarm list`** + - Hits: `GET /swarm/hosts` + +4. **`decnet swarm check`** + - Hits: `POST /swarm/check` + +5. **`decnet swarm update`** + - Hits: `GET /swarm/hosts` + direct mTLS to updater port 8766 + +6. **`decnet swarm deckies`** + - Hits: `GET /swarm/deckies` + +7. **`decnet swarm decommission`** + - Hits: `DELETE /swarm/hosts/{uuid}` + +**Note**: Swarmctl API endpoints (`/swarm/*`) are **not** in the Phase 1 audit (Phase 1 scanned `/api/v1/*` only). These routes are stable and not candidates for deletion. + +--- + +### Deletion Candidates + +**Count: 0** + +**Rationale**: +- No commands are marked `@deprecated` in docstrings. +- No old "v1" flavors replaced by newer flows (e.g., no `decnet deploy-v1` vs `decnet deploy-v2`). +- All commands in `MASTER_ONLY_COMMANDS` + `MASTER_ONLY_GROUPS` are actively referenced and tested. +- Worker-capable commands (`agent`, `updater`, `forwarder`, `bus`, `probe`, `collect`, `redeploy`, `status`, `services`, `distros`) are essential for field operation. +- Recent additions (`decnet init`, `decnet swarm *`, `decnet topology *`) are part of the SWARM/MazeNET bootstrap flow and have no predecessors. + +--- + +### CLI → API Deletion Chains + +No CLI command is the **only caller** of a Phase 1 API route marked `cli` or `zero`. All Phase 1 routes with `cli` callers have multiple paths: + +- Phase 1 example: `/health` — called by both CLI (`decnet status`) and web/test +- Phase 1 example: `/deckies` — called by CLI (`swarm deckies`) + web + test + +**Implication**: Deleting a CLI command does NOT unlock any Phase 1 API route deletions. + +--- + +### Gating Configuration + +Master-only enforcement lives in `decnet/cli/gating.py`: + +**MASTER_ONLY_COMMANDS** (25 command names): +``` +"api", "swarmctl", "deploy", "redeploy", "teardown", +"mutate", "listener", "profiler", +"services", "distros", "correlate", "archetypes", "web", +"db-reset", "init", +``` +Plus subcommand groups: + +**MASTER_ONLY_GROUPS** (2 group names): +``` +"swarm", "topology" +``` + +**Defense-in-depth**: +- Registration-time filter hides commands from `decnet --help` on agents (when `DECNET_MODE=agent`). +- Runtime gate in each command body calls `_require_master_mode()` to block direct function imports. + +--- + +### Recent Additions (Phase Context) + +Per repo memory and recent commits: + +- **`decnet init` + `--deinit`**: Bootstrap + teardown systemd/polkit/tmpfiles. Idempotent. +- **`decnet swarm *`**: Enroll workers, list status, push updates, manage deckies. All talk to swarmctl, not `/api/v1`. +- **`decnet topology *`**: MazeNET nested-topology commands. Direct DB calls (no HTTP). Replaces old flat `/topologies` CRUD. +- **`decnet bus`**: New ServiceBus worker. UNIX-socket pub/sub, not HTTP. +- **Worker supervisors** (`probe`, `collect`, `correlate`, `sniffer`, `profiler`): Field microservices. Spawned by `decnet deploy` as detached processes. + +None are marked for removal; all have active use cases. + +--- + +### Output Modes + +CLI output is **structured text** (Rich tables, JSON, syslog-format lines). All commands respect: +- `--json` flag where applicable (e.g., `decnet swarm check --json`) +- Scriptable structured output (e.g., `decnet correlate --output json`) + +Web dashboard visualization is **not** in CLI scope (per repo design: CLI outputs text, dashboard ingests data via API). + + +--- + +## Phase 4 — Consolidated Cleanup Plan + +### Executive Summary + +**CRITICAL FINDING**: Phase 1's "test-only routes" classification is **fundamentally unreliable**. Of 8 sampled test-only routes, **6 showed active web UI callers** — the Phase 1 grep methodology failed to catch TypeScript/TSX frontend API calls. + +**Phase 1 zero-caller candidates**: **REVISED DOWNWARD** from 7 to **3 actual deletions**: +- 4 routes flagged as zero-callers actually have active web UI callers: `/archetypes`, `/deckies/{decky_name}/mutate`, `/deckies/{decky_name}/mutate-interval`, and `/teardown` +- Remaining true zero-callers: `GET /{topology_id}/events`, `GET /{topology_id}/status-events`, `GET /{topology_id}/lans/{lan_id}/next-ip` + +**Recommendation**: Do NOT use the Phase 1 "47 test-only" list as a deletion target without manual verification of EACH route against the TypeScript frontend code. + +--- + +### Phase 4 Verification Results + +#### Zero-Caller Candidates — Fresh Grep Results + +| Route | Handler | Phase 1 Status | Phase 4 Finding | Verdict | +|-------|---------|----------------|-----------------|---------| +| `GET /archetypes` | `api_list_archetypes()` | Zero callers | **FOUND**: `DeckyFleet.tsx:833` calls `/topologies/archetypes` | **KEEP** | +| `POST /deckies/{decky_name}/mutate` | `api_mutate_decky()` | Zero callers | **FOUND**: `DeckyFleet.tsx:850` calls `/deckies/${name}/mutate` | **KEEP** | +| `PUT /deckies/{decky_name}/mutate-interval` | `api_update_mutate_interval()` | Zero callers | **FOUND**: `DeckyFleet.tsx:898` calls `/deckies/${name}/mutate-interval` | **KEEP** | +| `GET /{topology_id}/events` | `api_topology_events()` | Zero callers | **NO CALLERS FOUND** (only test mock) | **DELETE** | +| `GET /{topology_id}/lans/{lan_id}/next-ip` | `api_next_ip()` | Zero callers | **NO CALLERS FOUND** | **DELETE** | +| `GET /{topology_id}/status-events` | `api_get_status_events()` | Zero callers | **NO CALLERS FOUND** | **DELETE** | +| `POST /{topology_id}/teardown` | `api_teardown_topology()` | Zero callers | **FOUND**: `TopologyList.tsx` calls `/topologies/${id}/teardown` | **KEEP** | + +**Revised zero-caller count**: **3 routes** (not 7) + +--- + +#### Test-Only Routes — Spot-Check Results + +Sampled 8 of 47 "test-only" routes: + +| Route | Phase 1 Sample | Web Frontend Caller | Verdict | +|-------|----------------|-------------------|---------| +| `GET /artifacts/{decky}/{stored_as}` | test-only | **FOUND**: `ArtifactDrawer.tsx` | **FALSE POSITIVE** | +| `POST /auth/change-password` | test-only | **FOUND**: `Login.tsx` | **FALSE POSITIVE** | +| `POST /auth/login` | test-only | **FOUND**: `Login.tsx` | **FALSE POSITIVE** | +| `POST /blank` | test-only | **FOUND**: `MazeNET/useMazeApi.ts` + `TopologyList.tsx` | **FALSE POSITIVE** | +| `GET /bounty` | test-only | **FOUND**: `Bounty.tsx`, `CommandPalette.tsx` | **FALSE POSITIVE** | +| `GET /deployment-mode` | test-only | **FOUND**: `DeckyFleet.tsx` | **FALSE POSITIVE** | +| `DELETE /config/users/{user_uuid}` | test-only | **FOUND**: `Config.tsx` | **FALSE POSITIVE** | +| `GET /logs` | test-only | **FOUND**: `LiveLogs.tsx` | **FALSE POSITIVE** | + +**Verdict**: The "47 test-only routes" number is **unreliable**. At least **6/8 sampled routes have active web callers** that Phase 1's grep missed. The methodology failed because: +1. Phase 1 grepped Python/test files only; it did **not systematically scan TypeScript/TSX**. +2. Dynamic path construction (e.g., `` api.post(`/topologies/${id}/teardown`) ``) requires careful regex; simple string matching misses them. +3. Frontend developers split concerns across files (components/hooks/utils); no single grep layer caught all call sites. + +**Recommendation**: **Do not trust the "47 test-only" list.** Before deleting ANY route marked test-only, manually verify: +```bash +# For each route, run: +grep -r "" decnet_web/src --include="*.ts" --include="*.tsx" +``` + +--- + +### Enroll Flow Consolidation + +#### `POST /swarm/enroll` vs `POST /swarm/enroll-bundle` + +**Current state**: +- **`/swarm/enroll`** (simple): Master-driven, admin issues cert bundle, returns full bundle in response (201 Created). +- **`/swarm/enroll-bundle`** (new): Token-based workflow — admin builds token, renders `.sh` + `.tgz`, agent curls both (Wazuh-style one-liner). + +**Web UI caller analysis**: +- `SwarmHosts.tsx` calls **ONLY** `POST /swarm/enroll-bundle` (new flow). +- No web caller for `POST /swarm/enroll` (old flow) found. + +**CLI caller analysis**: +- `decnet swarm enroll` (Phase 3 audit) calls `POST /swarm/enroll` (line 572 of Phase 3 summary). + +**Recommendation**: **DEPRECATE simple `/swarm/enroll`** +1. Keep both endpoints for now (CLI still uses simple). +2. Mark `POST /swarm/enroll` as `@deprecated` in docstring; note that new deployments should use `POST /swarm/enroll-bundle`. +3. Update CLI (`decnet swarm enroll`) to call `/swarm/enroll-bundle` in a follow-up PR. +4. Only DELETE simple `/swarm/enroll` **after** CLI migration is merged and tested. + +**Why not delete now**: CLI is the only caller; deleting breaks backward compatibility for operators with scripts or runbooks calling the simple flow. Deprecate first, migrate CLI, then delete. + +--- + +### Ordered PR Plan (Kill List) + +**Three independent deletions** — run tests after each. Do NOT combine; each is a commit-shaped change. + +--- + +#### PR #1: Remove `/api/v1/{topology_id}/events` endpoint + +**Scope**: One endpoint, one handler module, test module, no other imports. + +**Files to delete**: +- `decnet/web/router/topology/api_events.py` (handler + schema) +- `tests/api/topology/test_events_stream.py` (test file) + +**Files to modify**: +- `decnet/web/router/topology/__init__.py` — remove two lines: + ```python + # DELETE: from .api_events import router as events_router + # DELETE: include_router(events_router) + ``` + +**Blast radius**: ~120 lines deleted, 2 import lines in router init. + +**Verification before deleting**: +```bash +grep -r "api_topology_events\|/events" --include="*.py" --include="*.ts" --include="*.tsx" \ + decnet/ decnet_web/ tests/ --exclude-dir=.claude | grep -v "def api_topology_events" | grep -v "test_events" +# Should return ZERO results except in files being deleted +``` + +**Test plan**: +```bash +pytest tests/api/topology/ -v # Topology suite still passes +pytest tests/api/ -k "not test_events_stream" --tb=short # Full API suite minus events +``` + +--- + +#### PR #2: Remove `/api/v1/{topology_id}/status-events` endpoint + +**Scope**: One endpoint, one handler (shares module with `GET /{topology_id}`), test code. + +**Files to modify**: +- `decnet/web/router/topology/api_get_topology.py` — remove function and route decorator: + ```python + # DELETE: @router.get("/{topology_id}/status-events", ...) + # DELETE: async def api_get_status_events(...): ... [~30 lines] + ``` + +**Files to modify (tests)**: +- `tests/api/topology/test_reads.py` — remove test cases that call `status-events`. + +**Blast radius**: ~40 lines (one function + docstring + route decorator). + +**Verification before deleting**: +```bash +grep -r "api_get_status_events\|/status-events" --include="*.py" --include="*.ts" --include="*.tsx" \ + decnet/ decnet_web/ tests/ --exclude-dir=.claude | grep -v "def api_get_status_events" +# Should return ZERO results except in deleted test code +``` + +**Test plan**: +```bash +pytest tests/api/topology/test_reads.py -v # Should pass after removing status-events test case +``` + +--- + +#### PR #3: Remove `/api/v1/{topology_id}/lans/{lan_id}/next-ip` endpoint + +**Scope**: One endpoint, one handler (shares module with catalog endpoints), test code. + +**Files to modify**: +- `decnet/web/router/topology/api_catalog.py` — remove function and route decorator: + ```python + # DELETE: @router.get("/{topology_id}/lans/{lan_id}/next-ip", ...) + # DELETE: async def api_next_ip(...): ... [~40 lines] + ``` + +**Files to modify (tests)**: +- `tests/api/topology/test_reads.py` — remove test cases that call `next-ip`. + +**Blast radius**: ~60 lines (one function + route + docstring). + +**Verification before deleting**: +```bash +grep -r "api_next_ip\|/next-ip" --include="*.py" --include="*.ts" --include="*.tsx" \ + decnet/ decnet_web/ tests/ --exclude-dir=.claude | grep -v "def api_next_ip" +# Should return ZERO results except in deleted test code +``` + +**Test plan**: +```bash +pytest tests/api/topology/test_reads.py -v # Should pass after removing next-ip test case +``` + +--- + +### Known Risks / Routes NOT Deleted (Had Callers) + +These routes were **flagged as zero-callers by Phase 1 but DO have active callers** — listed here so the human knows they were considered and verified: + +| Route | Handler | Caller Location | Decision | +|-------|---------|------------------|----------| +| `GET /archetypes` | `api_list_archetypes()` | `DeckyFleet.tsx:833` | KEEP | +| `POST /deckies/{decky_name}/mutate` | `api_mutate_decky()` | `DeckyFleet.tsx:850` | KEEP | +| `PUT /deckies/{decky_name}/mutate-interval` | `api_update_mutate_interval()` | `DeckyFleet.tsx:898` | KEEP | +| `POST /{topology_id}/teardown` | `api_teardown_topology()` | `TopologyList.tsx` | KEEP | + +--- + +### Summary Table + +| PR | Deletion | Files | Lines | Risk | Phase | +|----|----------|-------|-------|------|-------| +| #1 | `GET /{topology_id}/events` | 2 (handler + test) | ~120 | Low | 4a | +| #2 | `GET /{topology_id}/status-events` | 1 (shared module + test edit) | ~40 | Low | 4b | +| #3 | `GET /{topology_id}/lans/{lan_id}/next-ip` | 1 (shared module + test edit) | ~60 | Low | 4c | +| — | `POST /swarm/enroll` (simple) | 1 (handler) | ~100 | **Medium** | **Deferred** | + +**Total committed lines of code deleted**: ~220 lines (handler + tests) +**Total test files touched**: 3 (api_events.py deletion + test_events_stream.py deletion + test_reads.py edits) +**Estimated review time per PR**: 10–15 minutes +**Total estimated project time**: 1 hour (including test runs) + +--- + +### Why This Order + +1. **PR #1** removes the most isolated endpoint (dedicated handler module + test). No shared code, lowest risk. +2. **PR #2** modifies a shared catalog module but removes only one function. Can be reviewed with test edits. +3. **PR #3** similar scope to #2 (catalog module). Groups naturally with #2's test file edit strategy. +4. **Enroll consolidation deferred**: Requires CLI change first (`decnet swarm enroll` → `/swarm/enroll-bundle`). Plan for Phase 5. + +--- + +### Testing Strategy for Each PR + +1. **Before deletion**: Run the verification grep command above. Should return zero results except in files being deleted. +2. **After deletion**: + - Run `pytest tests/api/ -v` to verify no regressions in other routes. + - Spot-check web UI in dev (`decnet web`, then visit `/topologies` page). + - Verify CLI still works: `decnet --help` (not affected by these deletions). + - Final check: `grep -r ""` should be empty in decnet/, decnet_web/, tests/ (except deleted files). + +--- + +### Critical Lessons for Future Audits + +1. **Phase 1 methodology is insufficient**: Future audits must: + - Grep TypeScript/TSX sources **systematically** (not as an afterthought in Phase 4). + - Audit `decnet_web/src` for every route with same rigor as Python backend. + - Use IDE symbol search (e.g., VSCode "Find All References") for very high confidence on dynamic paths. + +2. **Do NOT bulk-delete "test-only" routes**: The "47 test-only" number is a **red flag, not a deletion target**. Each requires individual verification against web UI code. + +3. **Consolidation opportunities**: The simple `/swarm/enroll` is now deprecated but NOT deleted (requires CLI migration first). Document these as "Phase N+1" work, not in the main kill list. + + +--- + +## Phase 4.5 — Redundancy callout + +A follow-up pass (beyond zero-caller deletions) flagged three redundancy classes worth explicit documentation. These are orthogonal to the kill list in Phase 4 — they're about *ambiguity in the surface*, not dead code. + +### 1. Triple-registered `GET /deckies` ⚠️ HIGH PRIORITY + +The Phase 1 route table shows the same path + method bound to **three** handlers: + +| Method | Path | Handler | File | +|---|---|---|---| +| GET | `/deckies` | `get_deckies()` | `api_get_deckies.py` | +| GET | `/deckies` | `api_list_deckies()` | `api_list_deckies.py` | +| GET | `/deckies` | `list_deckies()` | `api_list_deckies.py` | + +**Why it matters**: +- FastAPI resolves same-path duplicates to whichever is registered last. The other two are dead but still appear in the OpenAPI schema. +- Two handlers in the same file (`api_list_deckies.py`) is a strong smell of a leftover-from-rename refactor. +- Schemathesis sees the duplicates and generates overlapping cases, inflating the 30-minute run time. + +**Verification TODO** (before deletion): +1. `grep -n "get_deckies\|api_list_deckies\|list_deckies" decnet/web/router/fleet/` — identify which is actually wired in the router `__init__.py` / include statements. +2. Determine whether the canonical handler is `get_deckies` or `api_list_deckies` (check which the web frontend's response shape matches). +3. Delete the two losers + their tests. Keep one canonical handler. + +**Risk**: Low. Only one handler is live; removing dead registrations can't change runtime behavior. + +--- + +### 2. Two enrollment flows — `/swarm/enroll` vs `/swarm/enroll-bundle` + +Already covered in [§ Enroll Flow Consolidation](#enroll-flow-consolidation) above. Reiterated here so all redundancies live in one place. + +- **`POST /swarm/enroll`** — legacy, simple, still called by `decnet swarm enroll` CLI. +- **`POST /swarm/enroll-bundle`** (+ `.sh` / `.tgz`) — new token-based flow, sole web-UI caller. +- **Recommendation**: mark simple as deprecated, migrate CLI to bundle flow, delete simple in a Phase 5 pass. Not on the current kill list. + +--- + +### 3. Mutation-verb confusion + +After Phase 4's zero-caller deletions land, four "mutate" endpoints currently coexist with overlapping names but different semantics: + +| Endpoint | Status | Scope | +|---|---|---| +| `POST /api/v1/deckies/{decky_name}/mutate` | **dead** (kill list) | single decky, fleet-wide | +| `PUT /api/v1/deckies/{decky_name}/mutate-interval` | **dead** (kill list) | single decky, fleet-wide | +| `POST /api/v1/{topology_id}/mutations` | **live** (mutation queue, bus-woken) | topology-scoped | +| Agent `POST /mutate` (port 8765) | **501 placeholder** | agent-local, unused | + +**Why it matters**: a reader new to the codebase sees four mutate-verbs and has to figure out which is canonical. After the kill list lands, only two remain: +- **Master**: `POST /{topology_id}/mutations` — the canonical live-mutation API. +- **Agent**: `POST /mutate` (501) — reserved for future worker-side mutation (currently master re-sends `/deploy`). + +**Action**: no code change needed *beyond the Phase 4 kill list*. Once dead routes are gone, this section stops being confusing on its own. + +--- + +### Explicitly NOT redundant + +For the record — these look like pairs but are not: + +- **Agent `/deploy` + `/teardown` vs `/topology/apply` + `/topology/teardown`** — fleet-wide vs single-topology scopes. Both serve agent, different purposes. Keep. +- **`POST /deckies/deploy` vs `POST /{topology_id}/deploy`** — same as above: fleet-wide deploy vs topology-scoped deploy. Keep. diff --git a/decnet/agent/app.py b/decnet/agent/app.py index 16639f40..8302314d 100644 --- a/decnet/agent/app.py +++ b/decnet/agent/app.py @@ -18,29 +18,138 @@ Endpoints mirror the existing unihost CLI verbs: """ from __future__ import annotations +import asyncio +import os +import pathlib from contextlib import asynccontextmanager -from typing import Optional +from typing import Any, Optional from fastapi import FastAPI, HTTPException from pydantic import BaseModel, Field +import contextlib + from decnet.agent import executor as _exec from decnet.agent import heartbeat as _heartbeat +from decnet.agent import topology_ops as _topology_ops +from decnet.bus.factory import get_bus +from decnet.bus.publish import run_health_heartbeat +from decnet.swarm.pki import DEFAULT_AGENT_DIR +from decnet.agent.topology_store import AlreadyApplied, TopologyStore from decnet.config import DecnetConfig from decnet.logging import get_logger +from decnet.topology.validate import ValidationError log = get_logger("agent.app") +def _resolve_agent_dir() -> pathlib.Path: + env = os.environ.get("DECNET_AGENT_DIR") + if env: + return pathlib.Path(env) + system = pathlib.Path("/etc/decnet/agent") + if system.exists(): + return system + return DEFAULT_AGENT_DIR + + +# Module-level singleton. Created lazily on first use so tests can +# monkeypatch DECNET_AGENT_DIR before the store binds to a path. +_topology_store: Optional[TopologyStore] = None + + +def _store() -> TopologyStore: + global _topology_store + if _topology_store is None: + _topology_store = TopologyStore(_resolve_agent_dir() / "topology.db") + return _topology_store + + +_collector_task: Optional[asyncio.Task] = None + + +def _ensure_collector_started() -> None: + """Spawn the log collector on demand — called from /topology/apply + after a successful materialise. We must NOT start this in the + lifespan hook: the agent's boot invariant is "never touch docker + until master tells us to" (see tests/swarm/test_agent_no_auto_restore.py). + + The collector watches ``decnet.topology.service=true`` labels via + docker events, writing RFC 5424 lines to ``DECNET_AGENT_LOG_FILE`` + which the forwarder ships to the master over syslog-TLS. Idempotent: + subsequent calls while the task is still running are no-ops. + """ + global _collector_task + if _collector_task is not None and not _collector_task.done(): + return + from decnet.env import DECNET_AGENT_LOG_FILE + + try: + from decnet.collector.worker import log_collector_worker + except Exception: # noqa: BLE001 — docker may be unavailable on dev + log.warning( + "agent log collector not starting — collector worker import failed", + exc_info=True, + ) + return + _collector_task = asyncio.create_task( + log_collector_worker(DECNET_AGENT_LOG_FILE), + name="agent-log-collector", + ) + log.info("agent log collector started log_file=%s", DECNET_AGENT_LOG_FILE) + + +_bus_heartbeat_task: Optional[asyncio.Task] = None + + @asynccontextmanager async def _lifespan(app: FastAPI): # Best-effort: if identity/bundle plumbing isn't configured (e.g. dev # runs or non-enrolled hosts), heartbeat.start() is a silent no-op. _heartbeat.start() + + # Host-local bus heartbeat (system.agent.health). Separate channel + # from the mTLS master-facing heartbeat above; this one lets peers on + # the same host (dashboard, updater) see the agent is alive without + # hitting its HTTPS endpoint. Bus-disabled path is a no-op loop. + bus = None + try: + bus = get_bus(client_name="agent") + await bus.connect() + except Exception as exc: # noqa: BLE001 + log.warning("agent: bus unavailable, skipping health heartbeat: %s", exc) + bus = None + + global _bus_heartbeat_task + _bus_heartbeat_task = asyncio.create_task( + run_health_heartbeat(bus, "agent"), + name="agent-bus-heartbeat", + ) + try: yield finally: await _heartbeat.stop() + if _bus_heartbeat_task is not None: + _bus_heartbeat_task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await _bus_heartbeat_task + _bus_heartbeat_task = None + if bus is not None: + with contextlib.suppress(Exception): + await bus.close() + global _collector_task + if _collector_task is not None and not _collector_task.done(): + _collector_task.cancel() + try: + await _collector_task + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + _collector_task = None + global _topology_store + if _topology_store is not None: + _topology_store.close() + _topology_store = None app = FastAPI( @@ -129,6 +238,73 @@ async def self_destruct() -> dict: return {"status": "self_destruct_scheduled"} +# ------------------------------------------------------- topology endpoints + + +class ApplyTopologyRequest(BaseModel): + hydrated: dict[str, Any] = Field( + ..., description="Hydrated topology dict from master.persistence.hydrate()" + ) + version_hash: str = Field( + ..., description="Master's canonical_hash(hydrated); must match ours" + ) + + +class TeardownTopologyRequest(BaseModel): + topology_id: str = Field(..., description="Topology UUID to dismantle") + + +@app.post( + "/topology/apply", + responses={ + 400: {"description": "Malformed hydrated topology or hash mismatch"}, + 409: {"description": "A different topology is already applied"}, + 500: {"description": "Docker or compose raised while applying"}, + }, +) +async def topology_apply(req: ApplyTopologyRequest) -> dict: + store = _store() + try: + await _topology_ops.apply(req.hydrated, req.version_hash, store) + except _topology_ops.HashMismatch as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except ValidationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except AlreadyApplied as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except Exception as exc: + log.exception("agent.topology_apply failed") + topology_id = (req.hydrated.get("topology") or {}).get("id") + if topology_id: + try: + store.record_error( + str(topology_id), str(exc)[:500], hydrated=req.hydrated, + ) + except Exception: # noqa: BLE001 — don't mask original failure + log.exception("failed to record apply error") + raise HTTPException(status_code=500, detail=str(exc)) from exc + _ensure_collector_started() + return {"status": "applied", "version_hash": req.version_hash} + + +@app.post( + "/topology/teardown", + responses={500: {"description": "Docker or compose raised while tearing down"}}, +) +async def topology_teardown(req: TeardownTopologyRequest) -> dict: + try: + await _topology_ops.teardown(req.topology_id, _store()) + except Exception as exc: + log.exception("agent.topology_teardown failed") + raise HTTPException(status_code=500, detail=str(exc)) from exc + return {"status": "torn_down", "topology_id": req.topology_id} + + +@app.get("/topology/state") +async def topology_state() -> dict: + return _topology_ops.state(_store()) + + @app.post( "/mutate", responses={501: {"description": "Worker-side mutate not yet implemented"}}, diff --git a/decnet/agent/executor.py b/decnet/agent/executor.py index 69851439..9e1c31bb 100644 --- a/decnet/agent/executor.py +++ b/decnet/agent/executor.py @@ -132,7 +132,7 @@ if command -v docker >/dev/null 2>&1; then fi # Stop+disable every systemd unit the installer may have dropped. -for unit in decnet-agent decnet-engine decnet-collector decnet-forwarder decnet-prober decnet-sniffer decnet-updater; do +for unit in decnet-agent decnet-engine decnet-collector decnet-forwarder decnet-prober decnet-reconciler decnet-sniffer decnet-updater; do systemctl stop "$unit" 2>/dev/null systemctl disable "$unit" 2>/dev/null done diff --git a/decnet/agent/heartbeat.py b/decnet/agent/heartbeat.py index bbc00aad..f8d8eae2 100644 --- a/decnet/agent/heartbeat.py +++ b/decnet/agent/heartbeat.py @@ -52,14 +52,26 @@ def _resolve_agent_dir() -> pathlib.Path: async def _tick(client: httpx.AsyncClient, url: str, host_uuid: str, agent_version: str) -> None: snap = await _exec.status() - resp = await client.post( - url, - json={ - "host_uuid": host_uuid, - "agent_version": agent_version, - "status": snap, - }, - ) + body: dict = { + "host_uuid": host_uuid, + "agent_version": agent_version, + "status": snap, + } + # Best-effort: fold in applied-topology snapshot. Failures must never + # wedge the heartbeat loop — master will fall back to "no topology + # reported" which triggers a resync if it expected one. + try: + from decnet.agent import topology_ops as _topo_ops + from decnet.agent.topology_store import TopologyStore + store = TopologyStore(_resolve_agent_dir() / "topology.db") + try: + body["topology"] = _topo_ops.state(store) + finally: + store.close() + except Exception: + log.debug("heartbeat: topology state unavailable", exc_info=True) + + resp = await client.post(url, json=body) # 403 / 404 are terminal-ish — we still keep looping because an # operator may re-enrol the host mid-session, but we log loudly so # prod ops can spot cert-pinning drift. diff --git a/decnet/agent/topology_ops.py b/decnet/agent/topology_ops.py new file mode 100644 index 00000000..f8f156f2 --- /dev/null +++ b/decnet/agent/topology_ops.py @@ -0,0 +1,208 @@ +"""Agent-side topology apply/teardown/state primitives. + +Wraps the compose + bridge machinery from :mod:`decnet.engine.deployer` +so the agent can drive a topology without ever touching the master's +sqlmodel repo. The master-side ``deploy_topology`` always calls +``transition_status(repo, …)`` which is useless (and unreachable) on +an agent — here we operate purely on a hydrated dict + the local +:class:`TopologyStore`. + +v1 constraint: one topology per agent. A second apply for a different +``topology_id`` triggers an on-the-spot teardown of the predecessor +before the new apply proceeds — master is authoritative. +""" +from __future__ import annotations + +import asyncio +import subprocess # nosec B404 +from typing import Any + +import docker + +from decnet.agent.topology_store import ( + TopologyStore, + observed, +) +from decnet.engine.deployer import ( + _compose, + _compose_with_retry, + _teardown_order, + _topology_compose_path, +) +from decnet.logging import get_logger +from decnet.network import create_bridge_network, remove_bridge_network +from decnet.topology.compose import ( + _network_name as _topology_network_name, + write_topology_compose, +) +from decnet.topology.hashing import canonical_hash +from decnet.topology.validate import ( + ValidationError, + errors as _validation_errors, + validate as _validate_topology, +) + +log = get_logger("agent.topology_ops") + + +class HashMismatch(RuntimeError): + """Raised when the master-provided version_hash doesn't match what we + hash locally — suggests serialisation drift. We fail loudly rather + than silently papering over a schema mismatch.""" + + +def _topology_id(hydrated: dict[str, Any]) -> str: + topo = hydrated.get("topology") or {} + tid = topo.get("id") + if not tid: + raise ValueError("hydrated topology missing topology.id") + return str(tid) + + +async def apply( + hydrated: dict[str, Any], + version_hash: str, + store: TopologyStore, +) -> None: + """Materialise *hydrated* on this agent and record it in *store*. + + Raises: + HashMismatch: master and agent disagree on the canonical hash — + don't touch docker, fail the apply. + ValidationError: topology fails structural validation. + Any docker / compose error propagates up; the endpoint maps it + to 500 and records the message on the store row. + """ + local_hash = canonical_hash(hydrated) + if local_hash != version_hash: + raise HashMismatch( + f"master hash {version_hash!r} does not match agent hash " + f"{local_hash!r} — refusing to apply" + ) + + issues = _validate_topology(hydrated) + if _validation_errors(issues): + raise ValidationError(issues) + + topology_id = _topology_id(hydrated) + # Master is authoritative. If a different topology is pinned here + # — whether it fully applied, only partially applied (failure + # marker row + orphan containers), or drifted — teardown first, + # then accept the new one. Refusing with 409 would leave the + # agent stuck in a state only a human could resolve. + existing = store.current() + if existing is not None and existing.topology_id != topology_id: + log.info( + "superseding topology %s with %s on master authority", + existing.topology_id, topology_id, + ) + try: + await teardown(existing.topology_id, store) + except Exception as exc: # noqa: BLE001 — we still want to try applying + log.warning( + "best-effort teardown of superseded topology %s failed: %s", + existing.topology_id, exc, + ) + # Hard-clear the store row so the new apply isn't blocked + # by a half-torn-down predecessor. Leftover docker objects + # will surface via the next heartbeat's observed block. + store.clear(existing.topology_id) + + lans = hydrated["lans"] + compose_path = _topology_compose_path(topology_id) + client = docker.from_env() + + # Bridges + compose are sync/blocking; hop to a thread so we don't + # stall the event loop on a slow docker daemon. + def _materialise() -> None: + for lan in lans: + net_name = _topology_network_name(topology_id, lan["name"]) + internal = not lan["is_dmz"] + create_bridge_network( + client, net_name, lan["subnet"], internal=internal + ) + write_topology_compose(hydrated, compose_path) + # ``--always-recreate-deps`` keeps service containers' netns shares + # fresh: every decky service joins its base's netns via + # ``network_mode: container:``, and that share is bound at + # service start time. If a base is recreated (e.g. when ``ports:`` + # changes after toggling ``forwards_l3``) but compose decides the + # services are unchanged, the services keep a stale netns FD + # pointing at the destroyed base — they end up in an empty + # namespace with only ``lo``, and external traffic hits a closed + # port on the live base. Forcing dependents to recreate alongside + # the base is the cheapest way to make this race impossible. + _compose_with_retry( + "up", "--build", "-d", "--always-recreate-deps", + compose_file=compose_path, + ) + + await asyncio.to_thread(_materialise) + + store.put(topology_id, version_hash, hydrated) + log.info( + "topology %s applied on agent (%d LANs)", topology_id, len(lans) + ) + + +async def teardown( + topology_id: str, + store: TopologyStore, +) -> None: + """Tear down *topology_id* on this agent. Idempotent: if there's no + record and no compose file, it's a no-op that still returns cleanly.""" + row = store.current() + # Prefer the stored hydrated blob — it's what we applied with. If + # it's gone (db wiped) but compose-file lingers, we still try to + # compose-down and delete bridges by scanning the compose file's + # LAN membership list via the hydrated blob if available. + hydrated = row.hydrated if row and row.topology_id == topology_id else None + compose_path = _topology_compose_path(topology_id) + client = docker.from_env() + + def _dismantle() -> None: + if compose_path.exists(): + try: + _compose("down", "--remove-orphans", compose_file=compose_path) + except subprocess.CalledProcessError as exc: + log.warning( + "topology %s compose down failed (continuing): %s", + topology_id, exc, + ) + if hydrated is not None: + for lan_name in _teardown_order(hydrated["lans"]): + net_name = _topology_network_name(topology_id, lan_name) + remove_bridge_network(client, net_name) + if compose_path.exists(): + compose_path.unlink() + + await asyncio.to_thread(_dismantle) + store.clear(topology_id) + log.info("topology %s torn down on agent", topology_id) + + +def state(store: TopologyStore) -> dict[str, Any]: + """Snapshot-plus-live-observation — the shape the heartbeat embeds.""" + row = store.current() + try: + obs = observed(docker.from_env()) + except Exception as exc: # noqa: BLE001 — docker socket may be gone + obs = {"error": str(exc)[:200]} + if row is None: + return { + "topology_id": None, + "applied_version_hash": None, + "applied_at": None, + "last_error": None, + "observed": obs, + } + return { + "topology_id": row.topology_id, + "applied_version_hash": row.applied_version_hash, + "applied_at": row.applied_at, + "last_error": row.last_error, + "observed": obs, + } + + +__all__ = ["apply", "teardown", "state", "HashMismatch"] diff --git a/decnet/agent/topology_store.py b/decnet/agent/topology_store.py new file mode 100644 index 00000000..7112307e --- /dev/null +++ b/decnet/agent/topology_store.py @@ -0,0 +1,213 @@ +"""Agent-side sqlite cache of the currently-applied topology. + +**This is a cache, not a source of truth.** The master is the only +authority for what the agent should be running. This store exists so +the agent can answer two questions quickly and offline: + +1. What topology did I last apply, and with what version hash? +2. Is what docker is currently doing consistent with that? + +The hash goes out on every heartbeat; the master compares it to what +it thinks this host should be running and schedules a re-push on +mismatch. + +Why sqlite when the blob is JSON? Consistent with +:mod:`decnet.swarm.log_forwarder._OffsetStore` — single-row sqlite is +the project-wide pattern for agent-local persistent state. Keeps +operational mental model small: "one state.db per thing". + +Design choices worth calling out: + +- **One row, one topology.** v1 only supports a single topology per + agent. Attempting to :meth:`put` a different ``topology_id`` while + a row already exists raises :class:`AlreadyApplied` — the agent + rejects the apply with 409 and the master is expected to teardown + the old one first. +- **No auto-restore on boot.** The agent does NOT read this db at + startup and try to re-apply. Whatever docker has after a restart + is what it has; the next heartbeat reports the truth and the + master decides whether to re-push. Same reason we don't sync + mutations from agent → master anywhere else: split-brain is worse + than temporary drift. +""" +from __future__ import annotations + +import json +import pathlib +import sqlite3 +import time +from dataclasses import dataclass +from typing import Any, Optional + + +class AlreadyApplied(RuntimeError): + """Raised when a different topology is already pinned to this agent.""" + + +@dataclass(frozen=True) +class AppliedRow: + topology_id: str + applied_version_hash: str + hydrated: dict[str, Any] + applied_at: int + last_error: Optional[str] + + +class TopologyStore: + """Single-row sqlite cache. Stdlib only, sync (called from endpoints).""" + + def __init__(self, db_path: pathlib.Path) -> None: + db_path.parent.mkdir(parents=True, exist_ok=True) + # check_same_thread=False: Starlette/FastAPI runs sync endpoint + # bodies on a worker thread distinct from where `app` is imported. + # The agent is single-process, so there's no real contention — + # sqlite's own connection lock is enough. + self._conn = sqlite3.connect(str(db_path), check_same_thread=False) + self._conn.execute( + "CREATE TABLE IF NOT EXISTS applied_topology (" + " topology_id TEXT PRIMARY KEY," + " applied_version_hash TEXT NOT NULL," + " hydrated_blob_json TEXT NOT NULL," + " applied_at INTEGER NOT NULL," + " last_error TEXT)" + ) + self._conn.commit() + + # ----------------------------------------------------------------- reads + + def current(self) -> Optional[AppliedRow]: + """Return the single applied topology, or ``None`` if idle.""" + row = self._conn.execute( + "SELECT topology_id, applied_version_hash, hydrated_blob_json," + " applied_at, last_error FROM applied_topology LIMIT 1" + ).fetchone() + if row is None: + return None + return AppliedRow( + topology_id=row[0], + applied_version_hash=row[1], + hydrated=json.loads(row[2]), + applied_at=int(row[3]), + last_error=row[4], + ) + + # ---------------------------------------------------------------- writes + + def put( + self, + topology_id: str, + applied_version_hash: str, + hydrated: dict[str, Any], + ) -> None: + """Record an applied topology. + + If a *different* topology is already recorded, raises + :class:`AlreadyApplied`. Re-applying the same ``topology_id`` + just updates the hash + blob (idempotent re-push). + """ + existing = self.current() + if existing is not None and existing.topology_id != topology_id: + raise AlreadyApplied( + f"agent already has topology {existing.topology_id!r}; " + f"cannot apply {topology_id!r}" + ) + self._conn.execute( + "INSERT INTO applied_topology" + " (topology_id, applied_version_hash, hydrated_blob_json," + " applied_at, last_error)" + " VALUES (?, ?, ?, ?, NULL)" + " ON CONFLICT(topology_id) DO UPDATE SET" + " applied_version_hash=excluded.applied_version_hash," + " hydrated_blob_json=excluded.hydrated_blob_json," + " applied_at=excluded.applied_at," + " last_error=NULL", + ( + topology_id, + applied_version_hash, + json.dumps(hydrated, sort_keys=True), + int(time.time()), + ), + ) + self._conn.commit() + + def record_error( + self, + topology_id: str, + message: str, + hydrated: Optional[dict[str, Any]] = None, + ) -> None: + """Attach a last-error message for *topology_id*. + + Upserts a marker row when no apply has yet succeeded for this + topology — that way a failure *during* the first materialise + (put() hasn't been reached) still surfaces via GET + /topology/state and the next heartbeat. The marker row uses an + empty ``applied_version_hash`` so master's heartbeat check sees + the hash mismatch and schedules a resync. + + If *hydrated* is provided it is stored so a later teardown can + still walk the LAN list — otherwise a partial deploy is strands + containers + bridges with no breadcrumb back to them. + """ + blob = json.dumps(hydrated, sort_keys=True) if hydrated else "{}" + self._conn.execute( + "INSERT INTO applied_topology" + " (topology_id, applied_version_hash, hydrated_blob_json," + " applied_at, last_error)" + " VALUES (?, '', ?, 0, ?)" + " ON CONFLICT(topology_id) DO UPDATE SET" + " last_error=excluded.last_error," + " hydrated_blob_json=CASE" + " WHEN applied_topology.hydrated_blob_json='{}'" + " THEN excluded.hydrated_blob_json" + " ELSE applied_topology.hydrated_blob_json END", + (topology_id, blob, message), + ) + self._conn.commit() + + def clear(self, topology_id: str) -> None: + """Remove the row for *topology_id* (post-teardown). + + No-op if the row doesn't exist — makes teardown idempotent. + """ + self._conn.execute( + "DELETE FROM applied_topology WHERE topology_id=?", + (topology_id,), + ) + self._conn.commit() + + def close(self) -> None: + self._conn.close() + + +# --------------------------------------------------- live docker observation + + +def observed(docker_client: Any) -> dict[str, Any]: + """Snapshot what docker is *actually* running on this agent. + + Returns a compact dict the heartbeat can ship so the master can + cross-check ``applied_version_hash`` against reality (a matching + hash with missing bridges is still drift). Best-effort: if docker + is unreachable we return an ``error`` marker rather than raising — + the agent still needs to heartbeat, and the master can treat + ``error`` as "unknown, re-push". + """ + try: + bridges = [ + n.name + for n in docker_client.networks.list() + if n.attrs.get("Driver") == "bridge" + and n.name.startswith("decnet-topology-") + ] + containers = [ + c.name + for c in docker_client.containers.list(all=False) + if c.name.startswith("decnet-") + ] + return {"bridges": sorted(bridges), "containers": sorted(containers)} + except Exception as exc: # noqa: BLE001 — best-effort observation + return {"error": str(exc)[:200]} + + +__all__ = ["TopologyStore", "AppliedRow", "AlreadyApplied", "observed"] diff --git a/decnet/asn/__init__.py b/decnet/asn/__init__.py new file mode 100644 index 00000000..64224b0a --- /dev/null +++ b/decnet/asn/__init__.py @@ -0,0 +1,92 @@ +""" +IP-to-ASN enrichment — maps attacker IPs to BGP-announced AS numbers and +org names for attacker intelligence. + +Public surface mirrors :mod:`decnet.geoip` so callers can compose them: + +* :func:`get_lookup` — returns the singleton :class:`AsnLookup`. +* :func:`enrich_ip` — takes an IP string, returns + ``(asn_int, asn_name, provider_name)`` or ``(None, None, None)``. + +Provider selection goes through :func:`~decnet.asn.factory.get_provider` +(env ``DECNET_ASN_PROVIDER``, default ``iptoasn``). Direct imports of +concrete providers are forbidden — mirrors the ``get_bus`` / +``get_repository`` rule. +""" +from __future__ import annotations + +import os +import time +from typing import Optional, Tuple + +from decnet.asn.factory import get_provider +from decnet.asn.lookup import AsnLookup +from decnet.asn.paths import ASN_ROOT + +# 24 h — iptoasn refreshes daily. +REFRESH_INTERVAL_S = 86_400 + +_lookup: Optional[AsnLookup] = None +_provider_name: Optional[str] = None + + +def get_lookup(*, force_refresh: bool = False) -> AsnLookup: + """Return the cached :class:`AsnLookup`, building it on first use. + + If the provider's data files are missing or older than + ``REFRESH_INTERVAL_S`` seconds, refresh before building. Pass + ``force_refresh=True`` to bypass the age check (used by a future + ``decnet asn refresh`` CLI command). + """ + global _lookup, _provider_name + provider = get_provider() + _provider_name = provider.name + + if force_refresh or _files_stale(provider): + provider.refresh() + _lookup = None # rebuild on next access + + if _lookup is None: + _lookup = provider.build_lookup() + return _lookup + + +def enrich_ip(ip: str) -> Tuple[Optional[int], Optional[str], Optional[str]]: + """Return ``(asn, as_name, provider_name)`` or ``(None, None, None)``. + + Never raises — any lookup failure collapses to all-None so the + caller (profiler) can upsert the attacker row regardless. + + ``DECNET_ASN_ENABLED=false`` short-circuits the whole path, useful + for tests / agent hosts / ops wanting to disable enrichment without + touching provider config. + """ + if os.environ.get("DECNET_ASN_ENABLED", "true").lower() == "false": + return (None, None, None) + try: + lookup = get_lookup() + info = lookup.asn(ip) + if info is None: + return (None, None, None) + return (info.asn, info.name or None, _provider_name or "unknown") + except Exception: + return (None, None, None) + + +def _files_stale(provider) -> bool: + """True when the provider has no fresh data on disk. + + Same semantics as :func:`decnet.geoip._files_stale`: a partial + cache still produces correct answers for the ranges it covers. + """ + paths = provider.data_paths() + if not paths: + return True + now = time.time() + for p in paths: + if p.exists() and now - p.stat().st_mtime <= REFRESH_INTERVAL_S: + return False + return True + + +__all__ = ["get_lookup", "enrich_ip", "ASN_ROOT", "REFRESH_INTERVAL_S"] diff --git a/decnet/asn/base.py b/decnet/asn/base.py new file mode 100644 index 00000000..418d6529 --- /dev/null +++ b/decnet/asn/base.py @@ -0,0 +1,33 @@ +"""ASN provider protocol — mirror of :mod:`decnet.geoip.base`. + +Concrete providers (e.g. :mod:`decnet.asn.iptoasn`) implement this. +Callers must go through :func:`decnet.asn.factory.get_provider`; never +import a concrete provider class directly. +""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Sequence + +from decnet.asn.lookup import AsnLookup + + +class Provider(ABC): + """Abstract IP→ASN data provider.""" + + #: Short tag written to ``Attacker.asn_source`` (e.g. ``'iptoasn'``). + name: str + + @abstractmethod + def refresh(self) -> None: + """Download / regenerate the provider's raw data files.""" + + @abstractmethod + def build_lookup(self) -> AsnLookup: + """Parse the on-disk data files and return a ready-to-query lookup.""" + + @abstractmethod + def data_paths(self) -> Sequence[Path]: + """Return the list of files this provider manages — used for staleness + detection. Order is not significant.""" diff --git a/decnet/asn/factory.py b/decnet/asn/factory.py new file mode 100644 index 00000000..c1a63f8f --- /dev/null +++ b/decnet/asn/factory.py @@ -0,0 +1,39 @@ +"""ASN provider factory — mirror of :mod:`decnet.geoip.factory`. + +Dispatch key: ``DECNET_ASN_PROVIDER`` (default ``iptoasn``). Lazy +singleton. +""" +from __future__ import annotations + +import os +from typing import Optional + +from decnet.asn.base import Provider + +_cached: Optional[Provider] = None +_cached_key: Optional[str] = None + + +def get_provider() -> Provider: + """Return the configured :class:`Provider` singleton.""" + global _cached, _cached_key + key = os.environ.get("DECNET_ASN_PROVIDER", "iptoasn").lower() + if _cached is not None and _cached_key == key: + return _cached + + if key == "iptoasn": + from decnet.asn.iptoasn.provider import IptoasnProvider + provider: Provider = IptoasnProvider() + else: + raise ValueError(f"Unsupported ASN provider: {key!r}") + + _cached = provider + _cached_key = key + return provider + + +def reset_cache() -> None: + """Forget the singleton — tests swap providers via the env var.""" + global _cached, _cached_key + _cached = None + _cached_key = None diff --git a/decnet/asn/iptoasn/__init__.py b/decnet/asn/iptoasn/__init__.py new file mode 100644 index 00000000..081f216b --- /dev/null +++ b/decnet/asn/iptoasn/__init__.py @@ -0,0 +1,9 @@ +"""iptoasn.com IP→ASN provider. + +Daily-refreshed gzipped TSV dump of the global BGP table, derived from +RIPE RIS. Released into the public domain by upstream — no attribution +required, no UA mandate, no terms to violate. + +Direct imports of :class:`IptoasnProvider` are discouraged — go through +:func:`decnet.asn.factory.get_provider`. +""" diff --git a/decnet/asn/iptoasn/fetch.py b/decnet/asn/iptoasn/fetch.py new file mode 100644 index 00000000..c087da22 --- /dev/null +++ b/decnet/asn/iptoasn/fetch.py @@ -0,0 +1,63 @@ +"""iptoasn.com bulk dump download. + +One file: ``ip2asn-v4.tsv.gz``, ~5 MB compressed, refreshed daily. +Pulled over HTTPS with the same generic UA the geoip RIR fetcher uses +(stealth: never identify as DECNET — public-data scrapers correlated to +honeypot operator egress is the threat model). +""" +from __future__ import annotations + +import logging +import shutil +import urllib.request +from pathlib import Path +from typing import Tuple + +logger = logging.getLogger("decnet.asn.iptoasn.fetch") + +# Mirror the (name, url) tuple shape of geoip.rir.fetch so test +# harnesses can swap one for the other. +IPTOASN_SOURCES: Tuple[Tuple[str, str], ...] = ( + ("ip2asn-v4", "https://iptoasn.com/data/ip2asn-v4.tsv.gz"), +) + +# Generic UA — matches geoip.rir.fetch. iptoasn.com explicitly releases +# the data into the public domain and does NOT require an identifying UA, +# so we keep DECNET stealth instead of advertising. +_USER_AGENT = "Mozilla/5.0 (compatible; fetch/1.0)" +_TIMEOUT_S = 60 + + +def fetch_all(dest: Path) -> list[Path]: + """Download every iptoasn file into *dest*. Returns the written paths. + + Atomic per file: download to ``{name}.tsv.gz.tmp`` then rename. A + partial failure leaves the previous generation intact. + """ + dest.mkdir(parents=True, exist_ok=True) + written: list[Path] = [] + for name, url in IPTOASN_SOURCES: + target = dest / f"{name}.tsv.gz" + tmp = target.with_suffix(".gz.tmp") + try: + _download(url, tmp) + tmp.replace(target) + written.append(target) + logger.info( + "asn.iptoasn: fetched %s (%d bytes)", + name, target.stat().st_size, + ) + except Exception as exc: + logger.error( + "asn.iptoasn: fetch failed for %s (%s): %s", name, url, exc + ) + if tmp.exists(): + tmp.unlink(missing_ok=True) + # Keep any stale previous file — better outdated than empty. + return written + + +def _download(url: str, dest: Path) -> None: + req = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT}) + with urllib.request.urlopen(req, timeout=_TIMEOUT_S) as resp, dest.open("wb") as fh: # nosec B310 — fixed https iptoasn URL + shutil.copyfileobj(resp, fh) diff --git a/decnet/asn/iptoasn/parse.py b/decnet/asn/iptoasn/parse.py new file mode 100644 index 00000000..47db413d --- /dev/null +++ b/decnet/asn/iptoasn/parse.py @@ -0,0 +1,78 @@ +"""Parser for the iptoasn.com ``ip2asn-v4.tsv`` dump. + +Line shape (gzipped, one row per BGP-announced prefix):: + + 1.0.0.0\\t1.0.0.255\\t13335\\tUS\\tCLOUDFLARENET + +Fields: ``range_start``, ``range_end``, ``as_number``, ``country_code``, +``as_description``. Both range columns are dotted IPv4 strings (the dump +is IPv4-only — there's a separate ``ip2asn-v6.tsv.gz`` we don't pull). + +Rows skipped: + +* ``as_number == 0`` — iptoasn's sentinel for "unannounced" / private + / reserved space. Country may still be present (``"None"`` / two-letter + CC) but we don't care: the geoip module owns country, ASN owns BGP. +* Rows where either range column won't parse as IPv4. +* Rows with fewer than 3 tab-separated columns. +""" +from __future__ import annotations + +import gzip +import ipaddress +import logging +from pathlib import Path +from typing import Iterator + +from decnet.asn.lookup import AsnInfo, Range + +logger = logging.getLogger("decnet.asn.iptoasn.parse") + + +def parse_file(path: Path) -> Iterator[Range]: + """Yield ``(start_int, end_int_inclusive, AsnInfo)`` for every BGP row. + + Accepts a gzipped path (``*.tsv.gz``); plain TSV is also fine for + test harnesses that hand-craft small fixtures. + """ + opener = gzip.open if path.suffix == ".gz" else open + with opener(path, "rt", encoding="utf-8", errors="replace") as fh: + for lineno, raw in enumerate(fh, 1): + line = raw.rstrip("\n") + if not line: + continue + parts = line.split("\t") + if len(parts) < 3: + continue + start_s, end_s, asn_s = parts[0], parts[1], parts[2] + # Description is the 5th column; iptoasn quotes nothing, + # but the field can contain stray whitespace. ``""`` when + # missing or unknown. + name = parts[4].strip() if len(parts) >= 5 else "" + + try: + asn = int(asn_s) + except ValueError: + logger.debug( + "asn.iptoasn: skipping malformed asn line %d in %s", + lineno, path.name, + ) + continue + # ASN 0 is iptoasn's sentinel for unannounced / sentinel + # space. Skip — there's no useful enrichment to attach. + if asn == 0: + continue + + try: + start_int = int(ipaddress.IPv4Address(start_s)) + end_int = int(ipaddress.IPv4Address(end_s)) + except (ValueError, ipaddress.AddressValueError): + logger.debug( + "asn.iptoasn: skipping malformed addr line %d in %s", + lineno, path.name, + ) + continue + if end_int < start_int: + continue + + yield (start_int, end_int, AsnInfo(asn=asn, name=name)) diff --git a/decnet/asn/iptoasn/provider.py b/decnet/asn/iptoasn/provider.py new file mode 100644 index 00000000..fbd243b5 --- /dev/null +++ b/decnet/asn/iptoasn/provider.py @@ -0,0 +1,83 @@ +"""iptoasn provider — orchestrates fetch + parse into an :class:`AsnLookup`. + +Mirrors :class:`decnet.geoip.rir.provider.RirProvider` exactly: fetch, +build a pickled cache, invalidate when raw files are newer than the +cache. +""" +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Sequence + +from decnet.asn.base import Provider +from decnet.asn.iptoasn.fetch import IPTOASN_SOURCES, fetch_all +from decnet.asn.iptoasn.parse import parse_file +from decnet.asn.lookup import AsnLookup +from decnet.asn.paths import ensure_root + +logger = logging.getLogger("decnet.asn.iptoasn.provider") + +# Pickled lookup cache — skips re-parsing the ~580k-row gz dump on every +# profiler restart. Rebuilt whenever any raw file is newer than the +# cache, see ``_cache_fresh``. +_CACHE_NAME = ".iptoasn_index.pkl" + + +class IptoasnProvider(Provider): + name = "iptoasn" + + def __init__(self) -> None: + self._root = ensure_root() + + # ---------- Provider interface ---------- + + def refresh(self) -> None: + logger.info("asn.iptoasn: refreshing dump into %s", self._root) + fetch_all(self._root) + cache = self._root / _CACHE_NAME + if cache.exists(): + cache.unlink(missing_ok=True) + + def build_lookup(self) -> AsnLookup: + cache = self._root / _CACHE_NAME + if self._cache_fresh(cache): + try: + lookup = AsnLookup.load(cache) + logger.debug( + "asn.iptoasn: loaded cached index (%d ranges)", + len(lookup), + ) + return lookup + except Exception as exc: + logger.warning( + "asn.iptoasn: cache load failed, rebuilding: %s", exc + ) + + ranges = [] + for path in self.data_paths(): + if not path.exists(): + continue + ranges.extend(parse_file(path)) + lookup = AsnLookup.from_ranges(ranges) + try: + lookup.save(cache) + except Exception as exc: + logger.warning("asn.iptoasn: cache save failed: %s", exc) + logger.info("asn.iptoasn: built index with %d ranges", len(lookup)) + return lookup + + def data_paths(self) -> Sequence[Path]: + return [self._root / f"{name}.tsv.gz" for name, _url in IPTOASN_SOURCES] + + # ---------- internals ---------- + + def _cache_fresh(self, cache: Path) -> bool: + """True when the pickle exists and is at least as new as every raw file.""" + if not cache.exists(): + return False + cache_mtime = cache.stat().st_mtime + for path in self.data_paths(): + if path.exists() and path.stat().st_mtime > cache_mtime: + return False + return True diff --git a/decnet/asn/lookup.py b/decnet/asn/lookup.py new file mode 100644 index 00000000..e3d6272b --- /dev/null +++ b/decnet/asn/lookup.py @@ -0,0 +1,126 @@ +"""Provider-agnostic IP→ASN lookup. + +A :class:`AsnLookup` is a frozen, sorted array of ``(start_ip, +end_ip_inclusive, AsnInfo)`` ranges queried via :mod:`bisect`. +O(log n) on ~600k ranges (a current iptoasn dump is ~580k rows). + +Private/loopback/invalid IPv4 and all IPv6 addresses resolve to +``None`` — the same policy :mod:`decnet.geoip.lookup` uses. +""" +from __future__ import annotations + +import bisect +import ipaddress +import pickle # nosec B403 — self-produced cache under /var/lib/decnet, never deserialized from untrusted input +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, List, Optional, Tuple + + +@dataclass(frozen=True) +class AsnInfo: + """One BGP-announced prefix's origin metadata.""" + + asn: int + name: str # AS description / org name; "" if absent in the source data + + +Range = Tuple[int, int, AsnInfo] + + +@dataclass +class AsnLookup: + """Indexed AS lookup over IPv4 ranges.""" + + # Parallel arrays for bisect: _starts[i] is the start-IP of the i-th + # range, _ends[i] its inclusive end, _infos[i] its AsnInfo. + _starts: List[int] + _ends: List[int] + _infos: List[AsnInfo] + + @classmethod + def from_ranges(cls, ranges: Iterable[Range]) -> "AsnLookup": + """Build a lookup from ``(start, end_inclusive, AsnInfo)`` triples. + + Ranges are sorted by start; on identical starts, last writer + wins (matches :class:`decnet.geoip.lookup.Lookup` semantics). + Non-overlapping adjacency is preserved. + """ + sorted_ranges = sorted(ranges, key=lambda r: (r[0], r[1])) + starts: List[int] = [] + ends: List[int] = [] + infos: List[AsnInfo] = [] + for start, end, info in sorted_ranges: + if starts and starts[-1] == start: + ends[-1] = end + infos[-1] = info + continue + starts.append(start) + ends.append(end) + infos.append(info) + return cls(starts, ends, infos) + + def asn(self, ip: str) -> Optional[AsnInfo]: + """Return the :class:`AsnInfo` for ``ip`` or ``None``. + + ``None`` on: IPv6, private/loopback/link-local/multicast/reserved + addresses, malformed strings, and IPs outside every BGP-announced + range in the source dump. + """ + try: + addr = ipaddress.ip_address(ip) + except ValueError: + return None + if isinstance(addr, ipaddress.IPv6Address): + return None + if ( + addr.is_private + or addr.is_loopback + or addr.is_link_local + or addr.is_multicast + or addr.is_reserved + or addr.is_unspecified + ): + return None + + n = int(addr) + idx = bisect.bisect_right(self._starts, n) - 1 + if idx < 0: + return None + if n <= self._ends[idx]: + return self._infos[idx] + return None + + def __len__(self) -> int: + return len(self._starts) + + # ---------- persistence ---------- + + def save(self, path: Path) -> None: + """Pickle the lookup to *path* (atomic rename).""" + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.parent.mkdir(parents=True, exist_ok=True) + with tmp.open("wb") as fh: + pickle.dump( + { + "version": 1, + "starts": self._starts, + "ends": self._ends, + "infos": [(i.asn, i.name) for i in self._infos], + }, + fh, + protocol=pickle.HIGHEST_PROTOCOL, + ) + tmp.replace(path) + + @classmethod + def load(cls, path: Path) -> "AsnLookup": + """Load a pickled lookup from *path*.""" + with path.open("rb") as fh: + data = pickle.load(fh) # nosec B301 — self-produced file under /var/lib/decnet + if data.get("version") != 1: + raise ValueError( + f"unsupported asn-lookup index version: {data.get('version')!r}" + ) + infos = [AsnInfo(asn=a, name=n) for a, n in data["infos"]] + return cls(data["starts"], data["ends"], infos) diff --git a/decnet/asn/paths.py b/decnet/asn/paths.py new file mode 100644 index 00000000..b78c665d --- /dev/null +++ b/decnet/asn/paths.py @@ -0,0 +1,18 @@ +"""Filesystem layout for ASN data — mirror of :mod:`decnet.geoip.paths`. + +``ASN_ROOT`` is where providers drop their raw files and cache indexes. +Default ``/var/lib/decnet/asn``. Override with ``DECNET_ASN_ROOT`` for +test harnesses. +""" +from __future__ import annotations + +import os +from pathlib import Path + +ASN_ROOT = Path(os.environ.get("DECNET_ASN_ROOT", "/var/lib/decnet/asn")) + + +def ensure_root() -> Path: + """Create ``ASN_ROOT`` if absent and return it. No-op if present.""" + ASN_ROOT.mkdir(parents=True, exist_ok=True) + return ASN_ROOT diff --git a/decnet/bus/__init__.py b/decnet/bus/__init__.py new file mode 100644 index 00000000..1fc4d87f --- /dev/null +++ b/decnet/bus/__init__.py @@ -0,0 +1,18 @@ +"""DECNET ServiceBus — pub/sub notification substrate. + +The bus is the notification layer for DECNET's worker constellation. The DB +remains the source of truth for anything durable; the bus carries "something +happened, go look" events. Delivery is at-most-once, fire-and-forget. + +Consumers call :func:`get_bus` from :mod:`decnet.bus.factory`; never import +transport implementations directly. The factory selects the backend via +``DECNET_BUS_TYPE`` (``nats`` or ``fake``) and honors ``DECNET_BUS_ENABLED``. + +Topic hierarchy is defined in :mod:`decnet.bus.topics` and locked early so +consumers can subscribe with stable wildcard patterns. +""" +from __future__ import annotations + +from decnet.bus.base import BaseBus, Event, Subscription + +__all__ = ["BaseBus", "Event", "Subscription"] diff --git a/decnet/bus/app.py b/decnet/bus/app.py new file mode 100644 index 00000000..129bf3c6 --- /dev/null +++ b/decnet/bus/app.py @@ -0,0 +1,92 @@ +"""Process-wide bus singleton for request-serving workers (API, SSE routes). + +A single connected :class:`~decnet.bus.base.BaseBus` shared across request +handlers — opening a UNIX socket per request would be wasteful and add +latency to the hot path. The API lifespan is responsible for calling +:func:`close_app_bus` on shutdown; connect is lazy so tests and +contract-test mode that never hit a publish/subscribe code path don't +pay for a bus connection they'll never use. + +Failures during :meth:`BaseBus.connect` are swallowed and logged — a +dead bus must never break request serving. Publishers should treat a +``None`` return from :func:`get_app_bus` as "skip this notification", +same as ``DECNET_BUS_ENABLED=false``. + +Connect is **retried with a short backoff** (not one-shot): a startup +race where the API lifespan hits :func:`get_app_bus` before ``decnet +bus`` is ready would otherwise poison the singleton for the entire +process lifetime. Instead we remember the last failure timestamp and +let callers retry once ``_RETRY_BACKOFF`` seconds have passed. +""" +from __future__ import annotations + +import asyncio +import time + +from decnet.bus.base import BaseBus +from decnet.bus.factory import get_bus +from decnet.logging import get_logger + +log = get_logger("bus.app") + +# Publishers in the hot path shouldn't pay connect-retry latency on every +# call; the dashboard's own 5 s poll interval recovers within one tick +# once the bus comes up. A persistently-dead bus only gets a connect +# attempt every 2 s, not once per request. +_RETRY_BACKOFF: float = 2.0 + +_lock = asyncio.Lock() +_shared: BaseBus | None = None +_last_failure_ts: float = 0.0 + + +async def get_app_bus() -> BaseBus | None: + """Return the process-wide connected bus, or ``None`` if unavailable. + + On first call, constructs a client via :func:`get_bus` and awaits + ``connect()``. Subsequent calls return the cached instance. If a + connect attempt raises, the failure timestamp is recorded and + subsequent calls within ``_RETRY_BACKOFF`` seconds return ``None`` + without re-attempting — after the backoff window, the next call + retries. This is what lets the API recover from a + ``decnet bus``-started-after-API race without a full API restart. + """ + global _shared, _last_failure_ts + if _shared is not None: + return _shared + if (time.monotonic() - _last_failure_ts) < _RETRY_BACKOFF: + return None + async with _lock: + if _shared is not None: + return _shared + if (time.monotonic() - _last_failure_ts) < _RETRY_BACKOFF: + return None + try: + candidate = get_bus(client_name="api") + await candidate.connect() + _shared = candidate + _last_failure_ts = 0.0 + return _shared + except Exception as exc: # noqa: BLE001 + log.warning("app bus unavailable: %s", exc) + _last_failure_ts = time.monotonic() + return None + + +async def close_app_bus() -> None: + """Close the shared bus if one is open; clear the backoff window. + + Call from the API lifespan shutdown. Safe to call multiple times. + Resetting ``_last_failure_ts`` means the next ``get_app_bus()`` + after shutdown-and-restart-within-the-same-process (rare, but + tests do this) retries immediately instead of honouring a stale + backoff. + """ + global _shared, _last_failure_ts + bus, _shared = _shared, None + _last_failure_ts = 0.0 + if bus is not None: + try: + await bus.close() + except Exception as exc: # noqa: BLE001 + log.warning("app bus close raised: %s", exc) diff --git a/decnet/bus/base.py b/decnet/bus/base.py new file mode 100644 index 00000000..8edd1724 --- /dev/null +++ b/decnet/bus/base.py @@ -0,0 +1,205 @@ +"""Bus abstractions: the :class:`Event` envelope and the :class:`BaseBus` ABC. + +Every transport (NATS, in-process fake, null) speaks this contract. The +envelope is versioned (``v``) so future evolution never breaks deployed +consumers that happen to see a newer event shape. + +Subscription model: :meth:`BaseBus.subscribe` returns a :class:`Subscription` +that is an async context manager AND an async iterator. The expected usage is: + + async with bus.subscribe("topology.*.mutation.*") as sub: + async for event in sub: + handle(event) + +Leaving the ``async with`` releases the underlying subscription handle; the +transport is free to drop any buffered events after that point. +""" +from __future__ import annotations + +import abc +import asyncio +import time +import uuid +from dataclasses import dataclass, field +from typing import Any, AsyncIterator + +EVENT_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class Event: + """The bus envelope. + + ``v`` is the envelope schema version, bumped on incompatible shape + changes. ``type`` is a short discriminator (``"mutation.applied"``, + ``"decky.state"``) useful for consumers that subscribe to a broad + wildcard and dispatch in Python; it is redundant with the trailing + segments of ``topic`` but cheaper to inspect. ``ts`` is epoch seconds + (float). ``id`` is a random UUID so consumers can de-dupe if they + ever see the same event twice (not expected at-most-once, but cheap + insurance). + """ + + topic: str + payload: dict[str, Any] + type: str = "" + v: int = EVENT_SCHEMA_VERSION + ts: float = field(default_factory=time.time) + id: str = field(default_factory=lambda: uuid.uuid4().hex) + + def to_dict(self) -> dict[str, Any]: + return { + "v": self.v, + "id": self.id, + "topic": self.topic, + "type": self.type, + "ts": self.ts, + "payload": self.payload, + } + + @classmethod + def from_dict(cls, topic: str, data: dict[str, Any]) -> "Event": + """Reconstruct an Event from a wire-format dict. + + ``topic`` is passed explicitly because the transport knows which + subject the message arrived on; trusting a ``topic`` field from the + wire would let a misbehaving publisher spoof events on topics they + don't actually publish to. + """ + return cls( + topic=topic, + payload=data.get("payload", {}) or {}, + type=data.get("type", "") or "", + v=int(data.get("v", EVENT_SCHEMA_VERSION)), + ts=float(data.get("ts", time.time())), + id=data.get("id") or uuid.uuid4().hex, + ) + + +class Subscription(abc.ABC): + """An open subscription — async context manager + async iterator. + + Concrete transports subclass this and implement :meth:`_aclose` plus the + async iterator protocol. Callers should not instantiate directly; use + :meth:`BaseBus.subscribe`. + """ + + def __init__(self, pattern: str) -> None: + self.pattern = pattern + self._closed = False + + async def __aenter__(self) -> "Subscription": + return self + + async def __aexit__(self, *exc: Any) -> None: + await self.aclose() + + def __aiter__(self) -> AsyncIterator[Event]: + return self + + async def aclose(self) -> None: + if self._closed: + return + self._closed = True + await self._aclose() + + @abc.abstractmethod + async def __anext__(self) -> Event: # pragma: no cover - abstract + raise NotImplementedError + + @abc.abstractmethod + async def _aclose(self) -> None: # pragma: no cover - abstract + raise NotImplementedError + + +class BaseBus(abc.ABC): + """Pub/sub transport contract. + + Implementations MUST be safe to ``await connect()`` multiple times and + ``await close()`` multiple times. Publishing to a closed bus raises + :class:`RuntimeError`; subscribing to a closed bus does too. + """ + + @abc.abstractmethod + async def connect(self) -> None: + """Establish any network/transport resources. Idempotent.""" + + @abc.abstractmethod + async def publish( + self, + topic: str, + payload: dict[str, Any], + *, + event_type: str = "", + ) -> None: + """Publish *payload* on *topic*. Fire-and-forget. + + Delivery is at-most-once. On transport error the implementation + logs and returns; it does not raise, because bus losses must not + cascade into worker failure (DB is source of truth). + """ + + @abc.abstractmethod + def subscribe(self, pattern: str) -> Subscription: + """Return a :class:`Subscription` that yields events matching *pattern*. + + Patterns follow NATS wildcard semantics: ``*`` matches one topic + token, ``>`` matches one-or-more trailing tokens. Examples: + + * ``topology.*.mutation.applied`` — all ``applied`` events for any + topology. + * ``topology.abc123.mutation.*`` — all mutation states for one + topology. + * ``topology.>`` — every event under the ``topology`` root. + """ + + @abc.abstractmethod + async def close(self) -> None: + """Tear down transport resources. Idempotent.""" + + async def __aenter__(self) -> "BaseBus": + await self.connect() + return self + + async def __aexit__(self, *exc: Any) -> None: + await self.close() + + +# ─── Wildcard matching shared across in-process transports ─────────────────── + +def matches(pattern: str, topic: str) -> bool: + """Return True iff *topic* matches *pattern* under NATS wildcard rules. + + ``*`` matches exactly one non-empty token; ``>`` matches one-or-more + trailing tokens (so ``topology.>`` matches ``topology.abc.x`` but not + ``topology`` alone). + """ + p_tokens = pattern.split(".") + t_tokens = topic.split(".") + for i, p in enumerate(p_tokens): + if p == ">": + # Must have at least one token remaining to match. + return i < len(t_tokens) + if i >= len(t_tokens): + return False + if p == "*": + if not t_tokens[i]: + return False + continue + if p != t_tokens[i]: + return False + return len(p_tokens) == len(t_tokens) + + +# Sentinel used by the in-process transports to signal "no more events" +# through the asyncio.Queue fan-out without inventing a separate control +# channel. Not part of the wire protocol. +_CLOSE_SENTINEL: Any = object() + + +async def _next_or_stop(queue: "asyncio.Queue[Any]") -> Event: + """Pop the next item from *queue*, raising ``StopAsyncIteration`` on close.""" + item = await queue.get() + if item is _CLOSE_SENTINEL: + raise StopAsyncIteration + return item diff --git a/decnet/bus/factory.py b/decnet/bus/factory.py new file mode 100644 index 00000000..f7a935ac --- /dev/null +++ b/decnet/bus/factory.py @@ -0,0 +1,85 @@ +"""Bus factory — selects a :class:`~decnet.bus.base.BaseBus` implementation. + +Dispatch key: the ``DECNET_BUS_TYPE`` environment variable. + +* ``unix`` (default) → :class:`~decnet.bus.unix_client.UnixSocketBus` +* ``fake`` → :class:`~decnet.bus.fake.FakeBus` (in-process) + +If ``DECNET_BUS_ENABLED`` is ``"false"`` the factory short-circuits to +:class:`~decnet.bus.fake.NullBus` regardless of ``DECNET_BUS_TYPE`` — a +cheap way for dev environments to run workers without a bus daemon. + +Mirrors :mod:`decnet.web.db.factory` (lazy imports inside each branch, +env-driven dispatch, optional telemetry wrapping). Callers MUST use +:func:`get_bus` rather than instantiating transports directly. +""" +from __future__ import annotations + +import os +from typing import Any + +from decnet.bus.base import BaseBus + + +def get_bus(**kwargs: Any) -> BaseBus: + """Instantiate the bus implementation selected by environment. + + Keyword arguments are forwarded to the concrete transport: + + * ``UnixSocketBus`` accepts ``socket_path`` (overrides + ``DECNET_BUS_SOCKET``) and ``client_name``. + * ``FakeBus`` accepts ``queue_size``. + """ + if os.environ.get("DECNET_BUS_ENABLED", "true").lower() == "false": + from decnet.bus.fake import NullBus + return NullBus() + + bus_type = os.environ.get("DECNET_BUS_TYPE", "unix").lower() + + if bus_type == "unix": + from decnet.bus.unix_client import UnixSocketBus + socket_path = kwargs.pop("socket_path", None) or _default_socket_path() + bus: BaseBus = UnixSocketBus(socket_path=socket_path, **kwargs) + elif bus_type == "fake": + from decnet.bus.fake import FakeBus + bus = FakeBus(**kwargs) + else: + raise ValueError(f"Unsupported bus type: {bus_type}") + + return _maybe_wrap_telemetry(bus) + + +def _default_socket_path() -> str: + """Return the bus socket path honoring ``DECNET_BUS_SOCKET`` and falling + back to ``/run/decnet/bus.sock`` → ``~/.decnet/bus.sock``. + + The runtime path (``/run/decnet``) is preferred because systemd + ``RuntimeDirectory=decnet`` sets it up with the right perms; the home + fallback keeps dev boxes usable without systemd. + """ + explicit = os.environ.get("DECNET_BUS_SOCKET") + if explicit: + return explicit + + runtime_dir = "/run/decnet" + if os.path.isdir(runtime_dir) and os.access(runtime_dir, os.W_OK): + return f"{runtime_dir}/bus.sock" + return os.path.expanduser("~/.decnet/bus.sock") + + +def _maybe_wrap_telemetry(bus: BaseBus) -> BaseBus: + """Wrap *bus* in a tracing proxy if OTEL is enabled, else return as-is. + + Uses :func:`decnet.telemetry.wrap_repository` as the underlying proxy — + its implementation is generic (wraps any async method in a span), so we + reuse it with a bus-appropriate tracer name. If telemetry isn't wired + up at all we no-op. + """ + try: + from decnet.telemetry import wrap_repository # type: ignore[attr-defined] + except ImportError: + return bus + try: + return wrap_repository(bus) + except Exception: # pragma: no cover - defensive + return bus diff --git a/decnet/bus/fake.py b/decnet/bus/fake.py new file mode 100644 index 00000000..9f6a26a9 --- /dev/null +++ b/decnet/bus/fake.py @@ -0,0 +1,183 @@ +"""In-process bus transports. + +* :class:`FakeBus` — real pub/sub semantics without touching a socket. Used + by unit tests and anywhere ``DECNET_BUS_TYPE=fake`` is set. Lets code + that depends on the bus be exercised entirely inside a single event loop, + matching the DECNET testing convention of not opening real network + sockets from unit tests. +* :class:`NullBus` — no-op. Returned by :func:`~decnet.bus.factory.get_bus` + when ``DECNET_BUS_ENABLED=false`` so workers can start cleanly in dev + environments where no bus daemon is running. Publishes are dropped; + subscriptions yield nothing and close cleanly. +""" +from __future__ import annotations + +import asyncio +from typing import Any + +from decnet.bus.base import ( + BaseBus, + Event, + Subscription, + _CLOSE_SENTINEL, + matches, +) +from decnet.logging import get_logger + +log = get_logger("bus.fake") + +# Per-subscriber bounded queue: backpressure policy is drop-oldest so a slow +# consumer cannot stall publishers (the invariant — DB is the source of +# truth — makes dropped events acceptable). +_DEFAULT_QUEUE_SIZE = 1024 + + +# ─── FakeBus ───────────────────────────────────────────────────────────────── + + +class _FakeSubscription(Subscription): + """Subscription backed by an :class:`asyncio.Queue` fed from + :meth:`FakeBus.publish`. Unregisters itself on close.""" + + def __init__(self, bus: "FakeBus", pattern: str, queue: "asyncio.Queue[Any]") -> None: + super().__init__(pattern) + self._bus = bus + self._queue = queue + + async def __anext__(self) -> Event: + if self._closed: + raise StopAsyncIteration + item = await self._queue.get() + if item is _CLOSE_SENTINEL: + raise StopAsyncIteration + return item + + async def _aclose(self) -> None: + self._bus._unregister(self) + # Unblock any pending __anext__ waiter. + try: + self._queue.put_nowait(_CLOSE_SENTINEL) + except asyncio.QueueFull: + pass + + +class FakeBus(BaseBus): + """In-process pub/sub. + + Publishes iterate every active subscription and enqueue the event on + the ones whose pattern matches the topic. If a subscriber's queue is + full, the oldest event is discarded to make room — same at-most-once + semantics as the real UNIX-socket transport. + """ + + def __init__(self, queue_size: int = _DEFAULT_QUEUE_SIZE) -> None: + self._queue_size = queue_size + self._subs: list[_FakeSubscription] = [] + self._connected = False + self._closed = False + self._lock = asyncio.Lock() + + async def connect(self) -> None: + self._connected = True + + async def publish( + self, + topic: str, + payload: dict[str, Any], + *, + event_type: str = "", + ) -> None: + if self._closed: + raise RuntimeError("publish on closed bus") + event = Event(topic=topic, payload=payload, type=event_type) + async with self._lock: + targets = [s for s in self._subs if matches(s.pattern, topic)] + for sub in targets: + _enqueue_drop_oldest(sub._queue, event) + + def subscribe(self, pattern: str) -> Subscription: + if self._closed: + raise RuntimeError("subscribe on closed bus") + queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=self._queue_size) + sub = _FakeSubscription(self, pattern, queue) + self._subs.append(sub) + return sub + + def _unregister(self, sub: _FakeSubscription) -> None: + try: + self._subs.remove(sub) + except ValueError: + pass + + async def close(self) -> None: + if self._closed: + return + self._closed = True + # Wake every still-open subscription so iterators unblock cleanly. + for sub in list(self._subs): + try: + sub._queue.put_nowait(_CLOSE_SENTINEL) + except asyncio.QueueFull: + pass + self._subs.clear() + + +def _enqueue_drop_oldest(queue: "asyncio.Queue[Any]", event: Event) -> None: + """Put *event* on *queue*, dropping the oldest item if the queue is full. + + Factored out so both FakeBus and the real UNIX server share the exact + same backpressure policy. + """ + while True: + try: + queue.put_nowait(event) + return + except asyncio.QueueFull: + try: + dropped = queue.get_nowait() + log.warning( + "bus.fake: subscriber queue full, dropped %s", getattr(dropped, "topic", "?") + ) + except asyncio.QueueEmpty: + return + + +# ─── NullBus ───────────────────────────────────────────────────────────────── + + +class _NullSubscription(Subscription): + """A subscription that never yields and closes immediately on iteration.""" + + async def __anext__(self) -> Event: + raise StopAsyncIteration + + async def _aclose(self) -> None: + return + + +class NullBus(BaseBus): + """No-op bus used when ``DECNET_BUS_ENABLED=false``. + + Publishes are silently dropped; subscriptions are empty. Intended for + dev environments where no bus daemon is running — the process starts + cleanly, code that publishes doesn't need feature flags, and nothing + ever blocks on a subscriber. + """ + + async def connect(self) -> None: + return + + async def publish( + self, + topic: str, + payload: dict[str, Any], + *, + event_type: str = "", + ) -> None: + return + + def subscribe(self, pattern: str) -> Subscription: + return _NullSubscription(pattern) + + async def close(self) -> None: + return diff --git a/decnet/bus/protocol.py b/decnet/bus/protocol.py new file mode 100644 index 00000000..a0f2f2eb --- /dev/null +++ b/decnet/bus/protocol.py @@ -0,0 +1,144 @@ +"""Wire protocol for the DECNET bus UNIX-socket transport. + +Frame layout: + + []\\n # ASCII header, single line, no trailing space + <4-byte big-endian body length> + # orjson-serialized dict, or empty (length 0) + +Verbs: + +* ``HELLO `` — optional greeting, logged by server. Body empty. +* ``PUB `` — publisher → server. Body = payload dict. +* ``SUB `` — subscriber → server. Body empty. +* ``UNSUB `` — subscriber → server. Body empty. +* ``EVT `` — server → subscriber. Body = payload dict (wrapped + in an :class:`~decnet.bus.base.Event` envelope). +* ``BYE`` — either direction. Body empty. Graceful shutdown. + +Parsing rules: + +* The header is a single line terminated by ``\\n`` (LF). ``\\r`` is tolerated + but not required. +* Header tokens are whitespace-separated. The first token is the verb; + everything after is verb-specific. We split on the first space only so + topics / patterns with quoted content are not supported (they are not + needed — topic segments forbid whitespace per :mod:`decnet.bus.topics`). +* Maximum header length is 4096 bytes; maximum body length is 1 MiB. Beyond + those, the connection is dropped with a logged error. This is a honeypot + framework, not a general-purpose message broker; a malformed frame is + treated as hostile. +""" +from __future__ import annotations + +import asyncio +import struct +from dataclasses import dataclass +from typing import Any + +import orjson + +MAX_HEADER_BYTES = 4096 +MAX_BODY_BYTES = 1 * 1024 * 1024 # 1 MiB + +# Verb constants (callers should reference these, not bare strings). +HELLO = "HELLO" +PUB = "PUB" +SUB = "SUB" +UNSUB = "UNSUB" +EVT = "EVT" +BYE = "BYE" + +_VALID_VERBS = frozenset({HELLO, PUB, SUB, UNSUB, EVT, BYE}) + + +class ProtocolError(Exception): + """Malformed or oversized frame. Callers should close the connection.""" + + +@dataclass(frozen=True) +class Frame: + """A parsed frame. ``body`` is the raw (unparsed) body bytes — callers + decide whether to orjson-decode it (the protocol does not know whether + a given verb expects a dict body or an empty one). + """ + + verb: str + args: str # everything after the verb on the header line, trimmed + body: bytes + + +def encode(verb: str, args: str = "", body: dict[str, Any] | None = None) -> bytes: + """Serialize a frame. + + *body* is a dict that will be orjson-encoded, or ``None`` for an empty + body. The header line is written verbatim — callers must supply args + that are free of ``\\n``. + """ + if verb not in _VALID_VERBS: + raise ProtocolError(f"unknown verb {verb!r}") + if "\n" in args or "\r" in args: + raise ProtocolError("args must not contain newline characters") + + body_bytes = b"" if body is None else orjson.dumps(body) + if len(body_bytes) > MAX_BODY_BYTES: + raise ProtocolError( + f"body {len(body_bytes)} bytes exceeds max {MAX_BODY_BYTES}" + ) + + header = f"{verb} {args}".rstrip() + "\n" + header_bytes = header.encode("ascii") + if len(header_bytes) > MAX_HEADER_BYTES: + raise ProtocolError( + f"header {len(header_bytes)} bytes exceeds max {MAX_HEADER_BYTES}" + ) + return header_bytes + struct.pack(">I", len(body_bytes)) + body_bytes + + +async def read_frame(reader: asyncio.StreamReader) -> Frame | None: + """Read one frame from *reader*. + + Returns ``None`` on clean EOF before a new frame starts. Raises + :class:`ProtocolError` on malformed input (caller should close the + connection). + """ + try: + header = await reader.readuntil(b"\n") + except asyncio.IncompleteReadError as exc: + if not exc.partial: + return None + raise ProtocolError("connection closed mid-header") from exc + except asyncio.LimitOverrunError as exc: + raise ProtocolError("header exceeded buffer limit") from exc + + if len(header) > MAX_HEADER_BYTES: + raise ProtocolError(f"header {len(header)} bytes exceeds max") + + line = header.rstrip(b"\r\n").decode("ascii", errors="strict") + if not line: + raise ProtocolError("empty header line") + + verb, _, args = line.partition(" ") + if verb not in _VALID_VERBS: + raise ProtocolError(f"unknown verb {verb!r}") + + length_bytes = await reader.readexactly(4) + (body_len,) = struct.unpack(">I", length_bytes) + if body_len > MAX_BODY_BYTES: + raise ProtocolError(f"body length {body_len} exceeds max") + + body = await reader.readexactly(body_len) if body_len else b"" + return Frame(verb=verb, args=args.strip(), body=body) + + +def decode_body(body: bytes) -> dict[str, Any]: + """Decode a frame body as a JSON dict. Empty body → empty dict.""" + if not body: + return {} + try: + obj = orjson.loads(body) + except orjson.JSONDecodeError as exc: + raise ProtocolError(f"body is not valid JSON: {exc}") from exc + if not isinstance(obj, dict): + raise ProtocolError(f"body must be a JSON object, got {type(obj).__name__}") + return obj diff --git a/decnet/bus/publish.py b/decnet/bus/publish.py new file mode 100644 index 00000000..15319cfe --- /dev/null +++ b/decnet/bus/publish.py @@ -0,0 +1,211 @@ +"""Fire-and-forget publish helpers shared across every worker. + +Lifted out of ``decnet/mutator/engine.py`` once a second caller showed up +(DEBT-031). Keeping one implementation means the "never break the worker +loop" guarantee is audited in exactly one place. +""" +from __future__ import annotations + +import asyncio +import contextlib +import os +import signal +import time +from typing import Any, Callable + +from decnet.bus import topics as _topics +from decnet.bus.base import BaseBus +from decnet.logging import get_logger + +log = get_logger("bus.publish") + + +async def publish_safely( + bus: BaseBus | None, + topic: str, + payload: dict[str, Any], + event_type: str = "", +) -> None: + """Publish on *bus* without ever raising back at the caller. + + The DB row (or equivalent side-effect) has already been committed by + the time a worker calls this; the bus is the notification layer, not + the source of truth. A dropped publish is at most a few seconds of + UI latency until the next poll tick. A raised exception here, by + contrast, would crash the worker — which is strictly worse. + """ + if bus is None: + return + try: + await bus.publish(topic, payload, event_type=event_type) + except Exception as exc: # noqa: BLE001 + log.warning("bus publish failed topic=%s: %s", topic, exc) + + +def make_thread_safe_publisher( + bus: BaseBus | None, + loop: asyncio.AbstractEventLoop, +) -> Callable[[str, dict[str, Any], str], None]: + """Build a sync callable that marshals publishes back to *loop*. + + Workers that run their hot paths in a worker thread (scapy sniff loop, + ``asyncio.to_thread`` probes, blocking socket reads) cannot ``await`` + the bus directly. This helper returns a plain function that schedules + the publish on *loop* via ``run_coroutine_threadsafe`` and returns + immediately — the calling thread is never blocked on the publish. + + A ``None`` bus yields a no-op callable, matching the degraded-mode + contract the rest of this module already upholds. + """ + if bus is None: + return lambda _topic, _payload, _event_type="": None + + def _publish(topic: str, payload: dict[str, Any], event_type: str = "") -> None: + # Stream threads may keep draining after the bus owner closed it + # (shutdown race). Short-circuit here so we don't marshal a + # coroutine onto a dead loop just to have publish_safely swallow + # it. bus.publish's own WARN-once guard handles the rare case + # where _closed flips between this check and the coroutine + # actually running. + if getattr(bus, "_closed", False): + return + try: + asyncio.run_coroutine_threadsafe( + publish_safely(bus, topic, payload, event_type=event_type), + loop, + ) + except Exception as exc: # noqa: BLE001 + log.debug("cross-thread bus publish failed topic=%s: %s", topic, exc) + + return _publish + + +async def run_health_heartbeat( + bus: BaseBus | None, + worker: str, + *, + interval: float = 30.0, + extra: Callable[[], dict[str, Any]] | None = None, +) -> None: + """Publish ``system..health`` every *interval* seconds. + + Standard heartbeat loop shared across agent/forwarder/updater. Emits + ``{"worker": , "ts": , **extra()}`` on each tick. A + ``None`` bus turns the loop into a no-op sleep cycle — still cancellable + so the caller can use the same ``asyncio.create_task``/``.cancel()`` + pattern regardless of bus state. + + Cancellation-safe: unwraps the ``CancelledError`` so callers awaiting + the task during shutdown see a clean exit. + """ + topic = _topics.system_health(worker) + with contextlib.suppress(asyncio.CancelledError): + while True: + payload: dict[str, Any] = {"worker": worker, "ts": time.time()} + if extra is not None: + try: + payload.update(extra()) + except Exception as exc: # noqa: BLE001 + log.debug("heartbeat extra() failed worker=%s: %s", worker, exc) + await publish_safely(bus, topic, payload, event_type=_topics.SYSTEM_HEALTH) + await asyncio.sleep(interval) + + +async def run_control_listener( + bus: BaseBus | None, + worker: str, + shutdown: asyncio.Event, +) -> None: + """Subscribe to ``system..control`` and honour stop intents. + + On a well-formed ``{"action": "stop", ...}`` message the function sets + *shutdown* and returns — the worker's main loop is expected to check + the event and unwind cleanly, matching the SIGTERM path. + + Malformed payloads (missing/unknown action, non-dict, exception from + the transport) are logged and ignored. A ``None`` bus yields a noop + coroutine that simply awaits *shutdown* — callers can ``create_task`` + this unconditionally regardless of bus state. + + Cancellation-safe. + """ + if bus is None: + with contextlib.suppress(asyncio.CancelledError): + await shutdown.wait() + return + + topic = _topics.system_control(worker) + with contextlib.suppress(asyncio.CancelledError): + try: + async with bus.subscribe(topic) as sub: + async for event in sub: + payload = event.payload or {} + action = payload.get("action") + requested_by = payload.get("requested_by", "") + if action == _topics.WORKER_CONTROL_STOP: + log.info( + "control: stop requested worker=%s by=%s", + worker, requested_by, + ) + shutdown.set() + return + log.debug( + "control: ignoring unknown action worker=%s action=%r", + worker, action, + ) + except Exception as exc: # noqa: BLE001 + log.warning( + "control listener failed worker=%s: %s — shutdown via bus disabled", + worker, exc, + ) + + +async def run_control_listener_signal( + bus: BaseBus | None, + worker: str, +) -> None: + """Like :func:`run_control_listener` but signals the process on stop. + + Preferred for workers whose main loop is a blocking thread + (container-log tail, PTY read, scapy sniff) — wiring an + ``asyncio.Event`` through the thread boundary is error-prone, and + every DECNET worker already has systemd-equivalent SIGTERM cleanup. + A SIGTERM self-signal routes the stop through that same path + without inventing a second shutdown mechanism. + + Cancellation-safe. Never raises: a failed self-signal is logged + and the loop simply exits (admin can fall back to ``systemctl``). + """ + if bus is None: + return + + topic = _topics.system_control(worker) + with contextlib.suppress(asyncio.CancelledError): + try: + async with bus.subscribe(topic) as sub: + async for event in sub: + payload = event.payload or {} + action = payload.get("action") + requested_by = payload.get("requested_by", "") + if action == _topics.WORKER_CONTROL_STOP: + log.info( + "control: stop requested worker=%s by=%s → SIGTERM self", + worker, requested_by, + ) + try: + os.kill(os.getpid(), signal.SIGTERM) + except Exception as exc: # noqa: BLE001 + log.warning( + "control: self-signal failed worker=%s: %s", + worker, exc, + ) + return + log.debug( + "control: ignoring unknown action worker=%s action=%r", + worker, action, + ) + except Exception as exc: # noqa: BLE001 + log.warning( + "control signal listener failed worker=%s: %s", + worker, exc, + ) diff --git a/decnet/bus/topics.py b/decnet/bus/topics.py new file mode 100644 index 00000000..3c89d7e4 --- /dev/null +++ b/decnet/bus/topics.py @@ -0,0 +1,398 @@ +"""Canonical topic hierarchy for the DECNET ServiceBus. + +Locked early so consumers can subscribe with stable wildcard patterns. +Adding new topic families is fine; **renaming** existing ones is a breaking +change for every subscriber and requires a coordinated rollout. + +Token structure (NATS-style, dot-separated): + + topology.{topology_id}.mutation.{state} + topology.{topology_id}.status + decky.{decky_id}.state + decky.{decky_id}.traffic + orchestrator.traffic.{decky_id} + orchestrator.file.{decky_id} + orchestrator.email.{decky_id} + attacker.observed + attacker.scored + attacker.session.started + attacker.session.ended + identity.formed + identity.observation.linked + identity.merged + identity.unmerged + identity.campaign.assigned + campaign.formed + campaign.identity.assigned + campaign.merged + campaign.unmerged + credential.captured + credential.reuse.detected + canary.{token_id}.triggered + canary.{token_id}.placed + canary.{token_id}.revoked + system.log + system.bus.health + system.{worker}.health + +Wildcards (per :func:`decnet.bus.base.matches`): + +* ``*`` matches exactly one token. +* ``>`` matches one-or-more trailing tokens (so ``topology.>`` matches + ``topology.abc.status`` but not the bare root ``topology``). +""" +from __future__ import annotations + +# ─── Root prefixes ─────────────────────────────────────────────────────────── + +TOPOLOGY = "topology" +DECKY = "decky" +ATTACKER = "attacker" +IDENTITY = "identity" +CAMPAIGN = "campaign" +SYSTEM = "system" +CREDENTIAL = "credential" +ORCHESTRATOR = "orchestrator" +CANARY = "canary" + + +# ─── Leaf event-type constants (the last segment of each topic) ────────────── + +# Topology mutation lifecycle states — keep in sync with TopologyMutation.state +# in decnet/web/db/models.py; the bus topic mirrors the DB state machine. +MUTATION_ENQUEUED = "enqueued" +MUTATION_APPLYING = "applying" +MUTATION_APPLIED = "applied" +MUTATION_FAILED = "failed" + +# Topology-level status transitions (topology.{id}.status): fires when the +# topology row's status column changes (pending/deploying/active/degraded/failed). +TOPOLOGY_STATUS = "status" + +# Decky-level event types (second token). +DECKY_STATE = "state" +DECKY_TRAFFIC = "traffic" +# On-demand mutation request — published by the API/CLI/UI, consumed by +# the mutator's watch loop to force an immediate mutation of one decky +# without waiting for its scheduled interval. Underscored (not dotted) +# to stay a single NATS token so the builder's validator accepts it. +DECKY_MUTATE_REQUEST = "mutate_request" +# Mutation transition event — distinct from DECKY_STATE ("current +# shape") because a mutation is a *transition* that carries old/new +# services + trigger + timing. Correlator consumes these (via the +# syslog sidechannel too) to interleave substrate-change markers into +# attacker traversals. +DECKY_MUTATION = "mutation" + +# Attacker event types (second token under the ``attacker`` root). First +# sighting, session boundary transitions, and score-threshold crossings +# published by correlator + profiler. Consumers typically subscribe to +# the wildcard ``attacker.>``. +ATTACKER_OBSERVED = "observed" +ATTACKER_SCORED = "scored" +# Published once per successful active probe result (JARM/HASSH/TCPfp). +# Distinct from ``observed`` which is the correlator's first-sight signal — +# a fingerprint is additional evidence about an already-observed attacker. +ATTACKER_FINGERPRINTED = "fingerprinted" +ATTACKER_SESSION_STARTED = "session.started" +ATTACKER_SESSION_ENDED = "session.ended" +# Published by the ``decnet enrich`` worker after an enrichment pass +# succeeds for an attacker IP (one or more 3rd-party intel providers +# returned a verdict). Payload carries the aggregate verdict + per- +# provider summary so SIEM-bound webhooks don't need to re-query the DB. +ATTACKER_INTEL_ENRICHED = "intel.enriched" + +# Identity-resolution event types (second/third tokens under ``identity``). +# Published by the (future) clusterer worker — see +# development/IDENTITY_RESOLUTION.md. Constants ship in this commit; +# no publishers exist yet, but consumers (webhook worker, dashboard +# SSE relay) can subscribe to ``identity.>`` from day one and receive +# events the instant the clusterer comes online. +# +# identity.formed — clusterer creates a new identity from +# one or more observations +# identity.observation.linked — observation attached to an existing +# identity (or reattached from another) +# identity.merged — two identities collapsed; loser gets +# ``merged_into_uuid`` set, subscribers +# re-key cached references to the winner +# identity.unmerged — revocable-merge undo: contradicting +# evidence cleared ``merged_into_uuid`` +# and re-split observations. The +# resurrected side's UUID is the same +# as the prior loser, so subscribers +# that cached references to the loser +# during the merged interval can +# re-attach without a new lookup. +# +# ``identity.campaign.assigned`` is deferred; it ships when the campaign +# clusterer ships. YAGNI before then. +IDENTITY_FORMED = "formed" +IDENTITY_OBSERVATION_LINKED = "observation.linked" +IDENTITY_MERGED = "merged" +IDENTITY_UNMERGED = "unmerged" +# Campaign-clusterer cross-family event — fires under ``identity.>`` so +# identity-stream subscribers (e.g. the IdentityDetail SSE client) get +# notified the moment an identity's ``campaign_id`` changes without +# having to subscribe to the campaign topic family. The same event +# fires under ``campaign.identity.assigned`` for campaign-side +# subscribers. +IDENTITY_CAMPAIGN_ASSIGNED = "campaign.assigned" + +# Campaign-clusterer event types (second/third tokens under +# ``campaign``). Mirror of the identity family at the layer above: +# campaigns group identities into operations, and the clusterer +# publishes the same form / link / merge / unmerge lifecycle. +# +# campaign.formed — clusterer creates a new campaign from +# one or more identities +# campaign.identity.assigned — identity attached to an existing +# campaign (or reassigned from another) +# campaign.merged — two campaigns collapsed; loser gets +# ``merged_into_uuid`` set, subscribers +# re-key cached references to the winner +# campaign.unmerged — revocable-merge undo: contradicting +# evidence cleared ``merged_into_uuid`` +# and re-split identities +CAMPAIGN_FORMED = "formed" +CAMPAIGN_IDENTITY_ASSIGNED = "identity.assigned" +CAMPAIGN_MERGED = "merged" +CAMPAIGN_UNMERGED = "unmerged" + +# Credential event types (second/third tokens under ``credential``). +# ``credential.captured`` fires once per upserted Credential row — the +# correlator listens for it and runs the cred-reuse query in response, +# so reuse detection latency is sub-second after a fresh capture. +# ``credential.reuse.detected`` fires when the correlator inserts a new +# CredentialReuse row or grows an existing one (added decky/service/IP). +CREDENTIAL_CAPTURED = "captured" +CREDENTIAL_REUSE_DETECTED = "reuse.detected" + +# Canary-token event types (third token under ``canary``). +# +# canary.{token_id}.placed — orchestrator/API successfully planted a +# canary artifact inside a decky's +# filesystem (or persisted a passive token +# that has no callback wiring). Lets +# dashboards reflect baseline coverage in +# real time without a DB poll. +# canary.{token_id}.triggered — ``decnet canary`` worker observed a +# callback hit (HTTP slug or DNS subdomain +# lookup) for the token. Payload carries +# ``src_ip``, ``user_agent``, ``request_path`` +# and any DNS qname so downstream +# consumers (correlator, webhook fanout) +# can attribute and forward without a +# follow-up DB read. +# canary.{token_id}.revoked — operator removed a token; planter unlinked +# the file (best-effort) and the row was +# marked ``revoked``. Subscribers may +# evict cached lookups by token id. +CANARY_PLACED = "placed" +CANARY_TRIGGERED = "triggered" +CANARY_REVOKED = "revoked" + +# Orchestrator event types (second token under ``orchestrator``). The +# orchestrator worker publishes one of these per synthetic action it +# drives against a decky — cheap inter-decky traffic and filesystem +# mutations whose role is to keep the honeypot from looking suspiciously +# static. Always nested with the destination decky uuid as the third +# token, so consumers can subscribe to a single decky's life-injection +# stream via ``orchestrator.*.``. +ORCHESTRATOR_TRAFFIC = "traffic" +ORCHESTRATOR_FILE = "file" +# Emailgen — published by the ``decnet emailgen`` worker once per generated +# fake email delivered into a mail decky's maildir. Third token is the +# destination mail-decky uuid (the IMAP/POP3 host serving the mailbox), +# matching the ``orchestrator.*.`` subscription pattern. +ORCHESTRATOR_EMAIL = "email" + +# System event types. +SYSTEM_LOG = "log" +SYSTEM_BUS_HEALTH = "bus.health" +# Worker-health leaf — built per-worker as ``system..health`` via +# :func:`system_health`. The leaf constant stays the same across workers; +# the worker name goes in the middle token. +SYSTEM_HEALTH = "health" +# Worker-control leaf — built per-worker as ``system..control`` via +# :func:`system_control`. Admin-originated stop intents travel on this +# topic; each worker subscribes to its own. +SYSTEM_CONTROL = "control" + +# Control payload ``action`` values — the wire vocabulary. Only ``stop`` is +# handled in v1; ``start`` is reserved because a stopped worker has no +# subscriber, so starting requires external supervision (systemd). +WORKER_CONTROL_STOP = "stop" +WORKER_CONTROL_START = "start" + +# Webhook subscription-set changed — published by the CRUD router after any +# create / update / delete on WebhookSubscription so the webhook worker can +# reload its in-memory subscription list and re-subscribe to the new union +# of patterns. Payload is currently empty; consumers only need the signal. +WEBHOOK_SUBSCRIPTIONS_CHANGED = "system.webhook.subscriptions_changed" + + +# ─── Builders ──────────────────────────────────────────────────────────────── + +def topology_mutation(topology_id: str, state: str) -> str: + """Build ``topology..mutation.``. + + *state* should be one of the ``MUTATION_*`` constants. + """ + _reject_tokens(topology_id, state) + return f"{TOPOLOGY}.{topology_id}.mutation.{state}" + + +def topology_status(topology_id: str) -> str: + """Build ``topology..status``.""" + _reject_tokens(topology_id) + return f"{TOPOLOGY}.{topology_id}.{TOPOLOGY_STATUS}" + + +def decky(decky_id: str, event_type: str) -> str: + """Build ``decky..``. + + *event_type* is typically one of ``DECKY_STATE`` or ``DECKY_TRAFFIC``. + """ + _reject_tokens(decky_id, event_type) + return f"{DECKY}.{decky_id}.{event_type}" + + +def decky_mutation(decky_id: str) -> str: + """Build ``decky..mutation``.""" + _reject_tokens(decky_id) + return f"{DECKY}.{decky_id}.{DECKY_MUTATION}" + + +def system(event_type: str) -> str: + """Build ``system.``. + + *event_type* may itself contain dots (e.g. ``bus.health``) — we don't + re-validate the already-constant leaves; this just prefixes. + """ + if not event_type: + raise ValueError("system topic requires a non-empty event_type") + return f"{SYSTEM}.{event_type}" + + +def credential(event_type: str) -> str: + """Build ``credential.``. + + *event_type* is typically one of :data:`CREDENTIAL_CAPTURED` or + :data:`CREDENTIAL_REUSE_DETECTED`. Dotted leaves + (``reuse.detected``) are permitted — same rationale as + :func:`system`. + """ + if not event_type: + raise ValueError("credential topic requires a non-empty event_type") + return f"{CREDENTIAL}.{event_type}" + + +def attacker(event_type: str) -> str: + """Build ``attacker.``. + + *event_type* is typically one of ``ATTACKER_OBSERVED``, + ``ATTACKER_SCORED``, ``ATTACKER_SESSION_STARTED``, + ``ATTACKER_SESSION_ENDED``. Dotted leaves (``session.started``) are + permitted — same rationale as :func:`system`. + """ + if not event_type: + raise ValueError("attacker topic requires a non-empty event_type") + return f"{ATTACKER}.{event_type}" + + +def campaign(event_type: str) -> str: + """Build ``campaign.``. + + *event_type* is typically one of :data:`CAMPAIGN_FORMED`, + :data:`CAMPAIGN_IDENTITY_ASSIGNED`, :data:`CAMPAIGN_MERGED`, or + :data:`CAMPAIGN_UNMERGED`. Dotted leaves (``identity.assigned``) + are permitted — same rationale as :func:`system`. + """ + if not event_type: + raise ValueError("campaign topic requires a non-empty event_type") + return f"{CAMPAIGN}.{event_type}" + + +def identity(event_type: str) -> str: + """Build ``identity.``. + + *event_type* is typically one of :data:`IDENTITY_FORMED`, + :data:`IDENTITY_OBSERVATION_LINKED`, :data:`IDENTITY_MERGED`, or + :data:`IDENTITY_UNMERGED`. Dotted leaves (``observation.linked``) + are permitted — same rationale as :func:`system`. + """ + if not event_type: + raise ValueError("identity topic requires a non-empty event_type") + return f"{IDENTITY}.{event_type}" + + +def orchestrator(event_type: str, decky_id: str) -> str: + """Build ``orchestrator..``. + + *event_type* should be one of :data:`ORCHESTRATOR_TRAFFIC` or + :data:`ORCHESTRATOR_FILE`. The destination decky is always the + third token so per-decky subscribers can use + ``orchestrator.*.``. + """ + _reject_tokens(event_type, decky_id) + return f"{ORCHESTRATOR}.{event_type}.{decky_id}" + + +def canary(token_id: str, event_type: str) -> str: + """Build ``canary..``. + + *event_type* should be one of :data:`CANARY_PLACED`, + :data:`CANARY_TRIGGERED`, or :data:`CANARY_REVOKED`. The token id + is always the second token so per-token subscribers can use + ``canary..>`` and fleet-wide consumers (webhook fanout, + correlator) use ``canary.>``. + """ + _reject_tokens(token_id, event_type) + return f"{CANARY}.{token_id}.{event_type}" + + +def system_health(worker: str) -> str: + """Build ``system..health``. + + Worker-health heartbeats live as a nested leaf under ``system`` so + consumers can subscribe to ``system.*.health`` for every worker at + once, or to ``system.mutator.health`` for a single one. *worker* is + validated as a regular segment — no dots, wildcards, or whitespace. + """ + _reject_tokens(worker) + return f"{SYSTEM}.{worker}.{SYSTEM_HEALTH}" + + +def system_control(worker: str) -> str: + """Build ``system..control``. + + Admin-originated stop (and, eventually, start) intents are published + here; the worker in question subscribes to its own address and reacts. + Payload shape:: + + {"action": "stop", "requested_by": "", "ts": } + + *action* must be one of :data:`WORKER_CONTROL_STOP` / + :data:`WORKER_CONTROL_START`; any other value is ignored by the + listener. Same segment rules as :func:`system_health`. + """ + _reject_tokens(worker) + return f"{SYSTEM}.{worker}.{SYSTEM_CONTROL}" + + +def _reject_tokens(*parts: str) -> None: + """Reject topic segments that would break NATS-style tokenization. + + Dots, wildcards, whitespace, and empty strings in a *segment* would + silently corrupt the hierarchy (e.g. ``topology.a.b.status`` for a + ``topology_id`` of ``"a.b"``). Raise early at the builder instead of + shipping a malformed topic to the wire. + """ + for p in parts: + if not p: + raise ValueError("topic segment must not be empty") + if "." in p or "*" in p or ">" in p or any(c.isspace() for c in p): + raise ValueError( + f"topic segment {p!r} may not contain '.', '*', '>', or whitespace" + ) diff --git a/decnet/bus/unix_client.py b/decnet/bus/unix_client.py new file mode 100644 index 00000000..226b296a --- /dev/null +++ b/decnet/bus/unix_client.py @@ -0,0 +1,257 @@ +"""UNIX-socket client — :class:`UnixSocketBus` implementation of :class:`BaseBus`. + +Holds one open socket to the local :class:`~decnet.bus.unix_server.BusServer`. +Operations: + +* :meth:`publish` writes a single ``PUB`` frame and returns; no ack. +* :meth:`subscribe` writes a ``SUB`` frame and returns a + :class:`~decnet.bus.base.Subscription` backed by an :class:`asyncio.Queue` + that the background reader task feeds. + +One background reader task per bus instance dispatches incoming ``EVT`` +frames to every registered subscription whose pattern matches the topic. +On connection drop or close, every subscription is woken via a sentinel so +iterators unblock cleanly; callers see :class:`StopAsyncIteration` from the +``async for`` loop. + +No auto-reconnect in MVP. If the server restarts, callers must +:meth:`close` the bus and construct a new one. This mirrors how other +DECNET workers handle their dependencies — the systemd ``Restart=on-failure`` +supervision above us is the retry loop. +""" +from __future__ import annotations + +import asyncio +import contextlib +import os +import pathlib +from typing import Any + +from decnet.bus import protocol +from decnet.bus.base import ( + BaseBus, + Event, + Subscription, + _CLOSE_SENTINEL, + matches, +) +from decnet.bus.fake import _enqueue_drop_oldest as _enqueue_event_drop_oldest +from decnet.logging import get_logger + +log = get_logger("bus.client") + +_INBOUND_QUEUE_SIZE = 1024 + + +class _UnixSubscription(Subscription): + def __init__( + self, + bus: "UnixSocketBus", + pattern: str, + queue: "asyncio.Queue[Any]", + ) -> None: + super().__init__(pattern) + self._bus = bus + self._queue = queue + + async def __anext__(self) -> Event: + if self._closed: + raise StopAsyncIteration + item = await self._queue.get() + if item is _CLOSE_SENTINEL: + raise StopAsyncIteration + return item + + async def _aclose(self) -> None: + await self._bus._unregister(self) + try: + self._queue.put_nowait(_CLOSE_SENTINEL) + except asyncio.QueueFull: + pass + + +class UnixSocketBus(BaseBus): + """Client handle for a local :class:`BusServer`. + + One instance per process typically; multiple instances simply open + multiple sockets to the same server. Connection is lazy — the first + :meth:`connect` (or any publish/subscribe call via ``async with``) + opens the socket. + """ + + def __init__( + self, + socket_path: pathlib.Path | str, + *, + client_name: str | None = None, + ) -> None: + self._path = pathlib.Path(socket_path) + self._client_name = client_name or f"decnet-bus-client[{os.getpid()}]" + self._reader: asyncio.StreamReader | None = None + self._writer: asyncio.StreamWriter | None = None + self._reader_task: asyncio.Task[None] | None = None + self._subs: list[_UnixSubscription] = [] + self._lock = asyncio.Lock() + self._write_lock = asyncio.Lock() + self._closed = False + # Sticky flag: the first publish-on-closed-bus call logs at + # WARNING so operators see that a publish was dropped; subsequent + # calls on the same instance log at DEBUG only to prevent a + # log flood when stream threads drain after close. The bus is + # critical infra, so the first warning is non-negotiable. + self._closed_publish_warned = False + + # ─── Lifecycle ────────────────────────────────────────────────────────── + + async def connect(self) -> None: + if self._writer is not None: + return + if self._closed: + raise RuntimeError("connect on closed bus") + self._reader, self._writer = await asyncio.open_unix_connection(str(self._path)) + await self._send(protocol.encode(protocol.HELLO, args=self._client_name)) + self._reader_task = asyncio.create_task(self._reader_loop()) + log.debug("bus.client: connected to %s as %s", self._path, self._client_name) + + async def close(self) -> None: + if self._closed: + return + self._closed = True + + # Best-effort BYE — we don't care if it fails. + if self._writer is not None and not self._writer.is_closing(): + with contextlib.suppress(Exception): + await self._send(protocol.encode(protocol.BYE)) + + if self._reader_task is not None: + self._reader_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._reader_task + self._reader_task = None + + if self._writer is not None: + with contextlib.suppress(Exception): + self._writer.close() + await self._writer.wait_closed() + self._writer = None + self._reader = None + + # Wake every subscription so `async for` exits. + for sub in list(self._subs): + with contextlib.suppress(asyncio.QueueFull): + sub._queue.put_nowait(_CLOSE_SENTINEL) + self._subs.clear() + + # ─── Pub/Sub ──────────────────────────────────────────────────────────── + + async def publish( + self, + topic: str, + payload: dict[str, Any], + *, + event_type: str = "", + ) -> None: + if self._closed: + # Degrade gracefully: the DB is the source of truth, the bus + # is only the notification layer. Raising here made every + # caller via publish_safely flood the logs once per stream + # line during shutdown races. First drop warns loudly; + # subsequent drops on the same instance are DEBUG-only. + if not self._closed_publish_warned: + self._closed_publish_warned = True + log.warning( + "bus.client: publish on closed bus dropped topic=%s " + "(further drops on this instance logged at DEBUG)", + topic, + ) + else: + log.debug("bus.client: publish on closed bus dropped topic=%s", topic) + return + if self._writer is None: + await self.connect() + body = Event(topic=topic, payload=payload, type=event_type).to_dict() + try: + await self._send(protocol.encode(protocol.PUB, args=topic, body=body)) + except (ConnectionError, BrokenPipeError) as exc: + # Bus loss is a logged warning, never a publisher crash. The + # DB-as-source-of-truth invariant means the work is already + # persisted; the missing event is just a missed notification. + log.warning("bus.client: publish failed: %s", exc) + + def subscribe(self, pattern: str) -> Subscription: + if self._closed: + raise RuntimeError("subscribe on closed bus") + queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=_INBOUND_QUEUE_SIZE) + sub = _UnixSubscription(self, pattern, queue) + self._subs.append(sub) + # Schedule the SUB frame asynchronously so subscribe() stays sync, + # matching the BaseBus signature. The caller will shortly `async + # with` / `async for` the subscription, which will run the event + # loop and pick this task up. + asyncio.ensure_future(self._send_sub(pattern)) + return sub + + async def _send_sub(self, pattern: str) -> None: + try: + if self._writer is None: + await self.connect() + await self._send(protocol.encode(protocol.SUB, args=pattern)) + except Exception as exc: # pragma: no cover - network paths in live tests + log.warning("bus.client: SUB %s failed: %s", pattern, exc) + + async def _unregister(self, sub: _UnixSubscription) -> None: + try: + self._subs.remove(sub) + except ValueError: + return + # Tell the server we no longer want events for this pattern if no + # other local subscription still wants it. + if not any(s.pattern == sub.pattern for s in self._subs): + with contextlib.suppress(Exception): + await self._send(protocol.encode(protocol.UNSUB, args=sub.pattern)) + + # ─── Internal I/O ─────────────────────────────────────────────────────── + + async def _send(self, frame_bytes: bytes) -> None: + if self._writer is None: + raise ConnectionError("bus.client: not connected") + async with self._write_lock: + self._writer.write(frame_bytes) + await self._writer.drain() + + async def _reader_loop(self) -> None: + if self._reader is None: + return + try: + while True: + frame = await protocol.read_frame(self._reader) + if frame is None: + break + if frame.verb != protocol.EVT: + # Clients only ever legitimately receive EVT (or BYE). + if frame.verb == protocol.BYE: + break + log.warning("bus.client: unexpected verb from server: %s", frame.verb) + continue + topic = frame.args + data = protocol.decode_body(frame.body) if frame.body else {} + event = Event.from_dict(topic, data) + self._dispatch(event) + except protocol.ProtocolError as exc: + log.warning("bus.client: protocol error: %s", exc) + except (asyncio.IncompleteReadError, ConnectionError): + pass + except asyncio.CancelledError: + raise + except Exception: # pragma: no cover + log.exception("bus.client: reader loop crashed") + finally: + # Server-side close — wake every subscription. + for sub in list(self._subs): + with contextlib.suppress(asyncio.QueueFull): + sub._queue.put_nowait(_CLOSE_SENTINEL) + + def _dispatch(self, event: Event) -> None: + for sub in self._subs: + if matches(sub.pattern, event.topic): + _enqueue_event_drop_oldest(sub._queue, event) diff --git a/decnet/bus/unix_server.py b/decnet/bus/unix_server.py new file mode 100644 index 00000000..502a8dcf --- /dev/null +++ b/decnet/bus/unix_server.py @@ -0,0 +1,309 @@ +"""UNIX-socket server for the DECNET bus. + +One :class:`BusServer` per host. Accepts local connections on a UNIX-domain +socket; each connection may: + +* publish events (``PUB`` frames) that the server fans out to all matching + subscribers on other connections, and +* subscribe to patterns (``SUB`` frames) and receive matching events as + ``EVT`` frames. + +Authorization is socket file permissions (0660, group=``decnet`` if that +POSIX group exists, else the server process's own group). Anything the +kernel lets ``connect()`` is trusted — there is no verb-level auth. This +matches the "local processes on the same host" threat model; cross-host +federation is out of scope (see DEBT-029). + +Backpressure is per-connection, drop-oldest: if a subscriber can't drain its +outbound queue fast enough, the server discards the oldest pending event +rather than blocking publishers. The bus is at-most-once by contract, so +drops are acceptable; stalled publishers are not. +""" +from __future__ import annotations + +import asyncio +import contextlib +import grp +import os +import pathlib +from dataclasses import dataclass, field +from typing import Any + +from decnet.bus import protocol +from decnet.bus.base import Event, matches +from decnet.logging import get_logger + +log = get_logger("bus.server") + +_SOCKET_MODE = 0o660 +_DEFAULT_GROUP = "decnet" +_OUTBOUND_QUEUE_SIZE = 1024 + + +@dataclass(eq=False) +class _Connection: + """Per-connection server state.""" + + writer: asyncio.StreamWriter + peer_name: str = "" + patterns: set[str] = field(default_factory=set) + outbound: asyncio.Queue[bytes] = field( + default_factory=lambda: asyncio.Queue(maxsize=_OUTBOUND_QUEUE_SIZE) + ) + closed: bool = False + + +class BusServer: + """Serve a UNIX-socket bus on *socket_path*. + + Lifecycle: construct → :meth:`start` → :meth:`serve_forever` (or rely + on :meth:`start` returning once bound) → :meth:`close` for teardown. + Safe to :meth:`close` multiple times. + """ + + def __init__( + self, + socket_path: pathlib.Path | str, + *, + group: str | None = _DEFAULT_GROUP, + mode: int = _SOCKET_MODE, + ) -> None: + self._path = pathlib.Path(socket_path) + self._group = group + self._mode = mode + self._server: asyncio.base_events.Server | None = None + self._connections: set[_Connection] = set() + self._closed = False + + # ─── Lifecycle ────────────────────────────────────────────────────────── + + async def start(self) -> None: + """Bind the socket and begin accepting connections. + + Removes any stale socket file at *socket_path* first (common case: + the previous worker crashed without cleaning up). The parent + directory must already exist; we do NOT create it blindly because + the chosen directory (typically ``/run/decnet``) may require + systemd ``RuntimeDirectory=`` to set up. + """ + if self._server is not None: + return + + parent = self._path.parent + if not parent.exists(): + raise FileNotFoundError( + f"bus socket parent directory {parent} does not exist; " + f"create it with systemd RuntimeDirectory= or mkdir" + ) + + # Clean up a stale socket from a previous crash. If a live server + # is actually listening there, ``bind()`` below will fail — we do + # not try to detect live vs. stale ourselves. + with contextlib.suppress(FileNotFoundError): + if self._path.is_socket(): + self._path.unlink() + + self._server = await asyncio.start_unix_server( + self._handle_connection, path=str(self._path), + ) + _chmod_and_chown(self._path, self._mode, self._group) + log.info("bus.server: listening on %s (mode=%o group=%s)", + self._path, self._mode, self._group or "") + + async def serve_forever(self) -> None: + if self._server is None: + raise RuntimeError("BusServer not started") + async with self._server: + await self._server.serve_forever() + + async def close(self) -> None: + if self._closed: + return + self._closed = True + + if self._server is not None: + self._server.close() + with contextlib.suppress(Exception): + await self._server.wait_closed() + self._server = None + + # Drain every live connection. + for conn in list(self._connections): + await self._close_connection(conn) + self._connections.clear() + + with contextlib.suppress(FileNotFoundError): + self._path.unlink() + log.info("bus.server: closed") + + # ─── Internal publish fan-out ─────────────────────────────────────────── + + async def publish(self, topic: str, payload: dict[str, Any], event_type: str = "") -> None: + """Server-side publish helper — used by the worker to emit + ``system.bus.health`` heartbeats without opening a client loop.""" + event = Event(topic=topic, payload=payload, type=event_type) + self._fanout(event) + + # ─── Connection handler ───────────────────────────────────────────────── + + async def _handle_connection( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter, + ) -> None: + conn = _Connection(writer=writer) + self._connections.add(conn) + writer_task = asyncio.create_task(self._writer_loop(conn)) + try: + await self._reader_loop(conn, reader) + except protocol.ProtocolError as exc: + log.warning("bus.server: protocol error from %s: %s", conn.peer_name, exc) + except (asyncio.IncompleteReadError, ConnectionError) as exc: + log.debug("bus.server: %s disconnected: %s", conn.peer_name, exc) + except Exception: # pragma: no cover - defensive + log.exception("bus.server: unhandled error in connection") + finally: + await self._close_connection(conn) + self._connections.discard(conn) + writer_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await writer_task + + async def _reader_loop( + self, conn: _Connection, reader: asyncio.StreamReader, + ) -> None: + while True: + frame = await protocol.read_frame(reader) + if frame is None: + return + await self._dispatch(conn, frame) + if frame.verb == protocol.BYE: + return + + async def _dispatch(self, conn: _Connection, frame: protocol.Frame) -> None: + if frame.verb == protocol.HELLO: + conn.peer_name = frame.args or conn.peer_name + log.debug("bus.server: HELLO from %s", conn.peer_name) + return + if frame.verb == protocol.SUB: + pattern = frame.args + if not pattern: + raise protocol.ProtocolError("SUB requires a pattern") + conn.patterns.add(pattern) + log.debug("bus.server: %s SUB %s", conn.peer_name, pattern) + return + if frame.verb == protocol.UNSUB: + conn.patterns.discard(frame.args) + return + if frame.verb == protocol.PUB: + topic = frame.args + if not topic: + raise protocol.ProtocolError("PUB requires a topic") + data = protocol.decode_body(frame.body) if frame.body else {} + event = Event( + topic=topic, + payload=data.get("payload", {}) or {}, + type=data.get("type", "") or "", + ) + self._fanout(event, origin=conn) + return + if frame.verb == protocol.BYE: + return + # EVT is server-to-client only; receiving one is a protocol violation. + raise protocol.ProtocolError(f"unexpected verb {frame.verb!r} from client") + + def _fanout(self, event: Event, *, origin: _Connection | None = None) -> None: + """Enqueue *event* as an EVT frame on every matching connection. + + We do NOT deliver back to the originating connection (a publisher + does not receive its own event). Encoding happens once per event, + not once per subscriber. + """ + try: + frame_bytes = protocol.encode( + protocol.EVT, args=event.topic, body=event.to_dict(), + ) + except protocol.ProtocolError: + log.exception("bus.server: failed to encode EVT for topic=%s", event.topic) + return + + for conn in self._connections: + if conn is origin or conn.closed: + continue + if not any(matches(p, event.topic) for p in conn.patterns): + continue + _enqueue_drop_oldest(conn.outbound, frame_bytes, event.topic) + + async def _writer_loop(self, conn: _Connection) -> None: + """Serialize writes onto *conn*'s socket. + + One writer task per connection so a slow peer only blocks its own + queue, not the fan-out loop. The queue is bounded with drop-oldest + policy applied at enqueue time (see :func:`_enqueue_drop_oldest`). + """ + try: + while not conn.closed: + data = await conn.outbound.get() + conn.writer.write(data) + await conn.writer.drain() + except (ConnectionError, BrokenPipeError): + log.debug("bus.server: %s writer: peer closed", conn.peer_name) + except asyncio.CancelledError: + pass + except Exception: # pragma: no cover - defensive + log.exception("bus.server: writer loop crashed for %s", conn.peer_name) + + async def _close_connection(self, conn: _Connection) -> None: + if conn.closed: + return + conn.closed = True + with contextlib.suppress(Exception): + conn.writer.close() + await conn.writer.wait_closed() + + +# ─── Helpers ───────────────────────────────────────────────────────────────── + +def _chmod_and_chown(path: pathlib.Path, mode: int, group: str | None) -> None: + """Apply socket file perms and best-effort group ownership. + + If *group* is ``None`` or the named group does not exist, we leave the + socket owned by the current process group. This keeps the server + usable on dev boxes that don't have a ``decnet`` group set up. + """ + try: + os.chmod(path, mode) + except OSError as exc: + log.warning("bus.server: chmod(%s, %o) failed: %s", path, mode, exc) + + if not group: + return + try: + gid = grp.getgrnam(group).gr_gid + except KeyError: + log.debug("bus.server: group %r not found, leaving socket group unchanged", group) + return + try: + os.chown(path, -1, gid) + except PermissionError: + # Dev box running as an unprivileged user can't chown. Log once at + # debug and move on — the socket is still usable by the owner. + log.debug("bus.server: chown(%s, gid=%d) denied; leaving as-is", path, gid) + except OSError as exc: + log.warning("bus.server: chown(%s, gid=%d) failed: %s", path, gid, exc) + + +def _enqueue_drop_oldest( + queue: "asyncio.Queue[bytes]", data: bytes, topic: str, +) -> None: + """Drop-oldest backpressure — mirrors :func:`decnet.bus.fake._enqueue_drop_oldest`.""" + while True: + try: + queue.put_nowait(data) + return + except asyncio.QueueFull: + try: + queue.get_nowait() + log.warning("bus.server: subscriber queue full, dropped event topic=%s", topic) + except asyncio.QueueEmpty: + return diff --git a/decnet/bus/worker.py b/decnet/bus/worker.py new file mode 100644 index 00000000..bbefaf65 --- /dev/null +++ b/decnet/bus/worker.py @@ -0,0 +1,121 @@ +"""``decnet bus`` worker entrypoint. + +Starts a :class:`~decnet.bus.unix_server.BusServer` on the configured UNIX +socket and serves forever, emitting a ``system.bus.health`` heartbeat on +its own bus every :data:`HEARTBEAT_INTERVAL_SEC` seconds so liveness-aware +consumers (dashboards, watchdogs) can tell the bus is up without polling +the filesystem. + +Cross-host federation is **out of scope** for the MVP; each host runs its +own bus independently. See DEBT-029 for the deferred ``--bridge-tcp`` +mode that would proxy the socket over the swarm mTLS channel. +""" +from __future__ import annotations + +import asyncio +import os +import pathlib +import signal +import time + +from decnet.bus import topics +from decnet.bus.unix_server import BusServer +from decnet.logging import get_logger + +log = get_logger("bus.worker") + +HEARTBEAT_INTERVAL_SEC = 10 + + +async def bus_worker( + socket_path: str | pathlib.Path, + *, + group: str | None = "decnet", + heartbeat_interval: int = HEARTBEAT_INTERVAL_SEC, +) -> None: + """Run the bus server until cancelled or SIGTERM/SIGINT is received. + + The parent directory of *socket_path* must already exist (systemd's + ``RuntimeDirectory=decnet`` handles this in prod; dev code is expected + to ``mkdir`` first). This function does not create it implicitly + because the right choice of perms/owner depends on the deployment + context. + """ + path = pathlib.Path(socket_path) + _ensure_parent(path) + + server = BusServer(path, group=group) + await server.start() + log.info("bus.worker: pid=%d socket=%s", os.getpid(), path) + + stop_event = asyncio.Event() + _install_signal_handlers(stop_event) + + heartbeat_task = asyncio.create_task(_heartbeat_loop(server, heartbeat_interval)) + serve_task = asyncio.create_task(server.serve_forever()) + + try: + await stop_event.wait() + log.info("bus.worker: shutdown signal received") + finally: + heartbeat_task.cancel() + serve_task.cancel() + for task in (heartbeat_task, serve_task): + try: + await task + except (asyncio.CancelledError, Exception): # noqa: BLE001 - draining shutdown + pass + await server.close() + log.info("bus.worker: stopped") + + +async def _heartbeat_loop(server: BusServer, interval: int) -> None: + """Publish ``system.bus.health`` on the server's own fan-out.""" + started_at = time.time() + while True: + try: + await server.publish( + topics.system(topics.SYSTEM_BUS_HEALTH), + { + "pid": os.getpid(), + "uptime_sec": round(time.time() - started_at, 3), + "ts": time.time(), + }, + event_type=topics.SYSTEM_BUS_HEALTH, + ) + except Exception: # pragma: no cover - heartbeat must never kill the worker + log.exception("bus.worker: heartbeat publish failed") + await asyncio.sleep(interval) + + +def _install_signal_handlers(stop_event: asyncio.Event) -> None: + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, stop_event.set) + except (NotImplementedError, RuntimeError): + # add_signal_handler is not supported on Windows / in some + # test harnesses where the loop is running in a non-main thread. + # The worker still exits via KeyboardInterrupt bubbling up. + pass + + +def _ensure_parent(path: pathlib.Path) -> None: + parent = path.parent + if parent.exists(): + return + # Dev-box convenience: if the parent is the user's ``~/.decnet`` dir, + # create it. We do not auto-mkdir ``/run/decnet`` — that's systemd's job + # and silently creating it as the wrong user would cause permission + # confusion later. + home_prefix = pathlib.Path.home() / ".decnet" + try: + parent.relative_to(home_prefix.parent) + except ValueError: + raise FileNotFoundError( + f"bus socket parent {parent} does not exist; create it first" + ) + parent.mkdir(parents=True, exist_ok=True) + + +__all__ = ["bus_worker", "HEARTBEAT_INTERVAL_SEC"] diff --git a/decnet/canary/__init__.py b/decnet/canary/__init__.py new file mode 100644 index 00000000..8a250514 --- /dev/null +++ b/decnet/canary/__init__.py @@ -0,0 +1,37 @@ +"""Canary tokens — decoy artifacts planted in decky filesystems. + +Public surface is exported here so callers can ``from decnet.canary +import CanaryArtifact, get_generator, get_instrumenter`` without +knowing the submodule layout. Concrete generators / instrumenters +live under :mod:`decnet.canary.generators` and +:mod:`decnet.canary.instrumenters` respectively; the factory keeps +import-time cost down by deferring those imports until first use +(same pattern as :mod:`decnet.intel.factory`). +""" +from __future__ import annotations + +from decnet.canary.base import ( + CanaryArtifact, + CanaryContext, + CanaryGenerator, + CanaryInstrumenter, +) +from decnet.canary.factory import ( + KNOWN_GENERATORS, + KNOWN_INSTRUMENTERS, + get_generator, + get_instrumenter, + pick_instrumenter_for_mime, +) + +__all__ = [ + "CanaryArtifact", + "CanaryContext", + "CanaryGenerator", + "CanaryInstrumenter", + "KNOWN_GENERATORS", + "KNOWN_INSTRUMENTERS", + "get_generator", + "get_instrumenter", + "pick_instrumenter_for_mime", +] diff --git a/decnet/canary/base.py b/decnet/canary/base.py new file mode 100644 index 00000000..160dcd19 --- /dev/null +++ b/decnet/canary/base.py @@ -0,0 +1,145 @@ +"""Canary generator / instrumenter ABCs and the artifact dataclass. + +Two flavors of producer share the same return shape: + +* :class:`CanaryGenerator` synthesises a fake artifact from scratch + (e.g. a plausible ``~/.aws/credentials`` block, a ``.git/config`` + pointing at an attacker-bait remote URL). Operators don't supply + any input. + +* :class:`CanaryInstrumenter` mutates an operator-uploaded blob to + embed the callback (HTTP slug + DNS host). The original blob bytes + are passed in; the instrumenter returns the mutated version. + +Both return a :class:`CanaryArtifact` — the planter doesn't care +which path produced it. Same dataclass keeps the planter's +docker-exec injector trivial. + +ABCs intentionally do not include I/O — generators and instrumenters +are pure functions of (slug, host, blob?). All filesystem work +happens in :mod:`decnet.canary.planter` and :mod:`decnet.canary.storage`. +""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class CanaryContext: + """Inputs every generator/instrumenter needs to embed a working callback. + + ``callback_token`` is the unique slug; it appears verbatim in HTTP + URLs (``https:///c/``) and as the leftmost + DNS label (``.canary.``) so a single + slug resolves to a single :class:`CanaryToken` row regardless of + which path the attacker tripped. + + ``http_base`` and ``dns_zone`` come from the canary worker's + public-facing config (``DECNET_CANARY_HTTP_BASE``, + ``DECNET_CANARY_DNS_ZONE``). When DNS isn't deployed, + ``dns_zone`` is empty and instrumenters that only have a DNS + surface (e.g. an artifact whose only realistic embed point is a + hostname) raise. + """ + + callback_token: str + http_base: str # e.g. "https://canary.example.test" — no trailing slash + dns_zone: str = "" # e.g. "canary.example.test"; "" disables DNS embeds + persona: str = "linux" # "linux" | "windows" — drives default username, path style + + +@dataclass +class CanaryArtifact: + """Bytes-and-placement bundle produced by a generator/instrumenter.""" + + path: str + """Absolute path inside the target container.""" + + content: bytes + """Final bytes that hit the decky filesystem. + + Always raw bytes — the planter base64-encodes for the wire so + binary blobs (DOCX/PNG/PDF) survive ``docker exec sh -c`` safely. + """ + + mode: int = 0o600 + """Unix file mode. Defaults to ``0600`` because most realistic + canary placements (``~/.aws/credentials``, ``.env``, ``id_rsa``) + are operator-only. Honeydocs in user docs folders should pass + ``0o644``. + """ + + mtime_offset: int = 0 + """Seconds relative to *now* for the planted file's mtime. + + Negative values backdate the file so it doesn't look like it + appeared the moment the decky was deployed. ``-86400 * 90`` (90 + days ago) is a common choice for ``honeydoc`` artifacts; ``0`` + means "stamp it now," which is fine for ``aws_creds``-like files + that would plausibly be touched recently. + """ + + instrumenter: Optional[str] = None + """Identifier of the instrumenter that produced this artifact (for + upload-driven tokens). Mirrored into ``CanaryToken.instrumenter``. + Mutually exclusive with :attr:`generator`. + """ + + generator: Optional[str] = None + """Identifier of the generator that produced this artifact (for + synthesised tokens). Mirrored into ``CanaryToken.generator``. + Mutually exclusive with :attr:`instrumenter`. + """ + + notes: list[str] = field(default_factory=list) + """Human-readable notes about the embedding (e.g. "DOCX: injected + 1×1 remote image at relsId rId99"). Surfaced in the API + ``preview`` response so the operator sees what we did before + planting. Never leaked to the attacker-facing surface. + """ + + +class CanaryGenerator(ABC): + """Produces a fake artifact from scratch.""" + + name: str #: short tag — matches ``CanaryToken.generator`` + + @abstractmethod + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + """Synthesise the artifact. + + MUST NOT do I/O. MUST be deterministic for the same + ``(callback_token, http_base, dns_zone, persona)`` so re-seeding + from :attr:`CanaryToken.secret_seed` produces byte-identical + output and the planter is naturally idempotent. + """ + + +class CanaryInstrumenter(ABC): + """Mutates an operator-uploaded blob to embed a callback.""" + + name: str #: short tag — matches ``CanaryToken.instrumenter`` + + #: MIME prefixes this instrumenter handles. The factory uses these + #: to dispatch by sniffed content-type. Sub-string match against + #: the prefix list (e.g. ``("application/pdf",)`` or + #: ``("text/",)``). + mime_prefixes: tuple[str, ...] = () + + @abstractmethod + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + """Return the mutated bytes with the callback embedded. + + MUST raise :class:`InstrumenterRejectedError` when the blob + can't be safely mutated (corrupt zip, encrypted PDF, etc.) so + the API can surface a 400 with the specific reason rather than + silently shipping the original bytes. + """ + + +class InstrumenterRejectedError(ValueError): + """Raised when an instrumenter can't safely mutate the input.""" diff --git a/decnet/canary/cultivator.py b/decnet/canary/cultivator.py new file mode 100644 index 00000000..8f7a222f --- /dev/null +++ b/decnet/canary/cultivator.py @@ -0,0 +1,181 @@ +"""Realism contract adapter for canary generators. + +Stage 7 of the realism migration. The orchestrator's planner picks a +``canary_*`` :class:`~decnet.realism.taxonomy.ContentClass` 1–3% of +the time on file ticks; this module turns that pick into a +:class:`~decnet.canary.base.CanaryArtifact` (bytes the SSH driver +plants) plus a persisted :class:`~decnet.web.db.models.CanaryToken` +row so the canary worker recognises the slug when an attacker trips +it. + +What this is NOT: it doesn't pick *when* canaries fire — that's the +realism planner's job. It doesn't decide *where* on the filesystem +the canary lands beyond what realism naming + persona conventions +already produce. It's a thin bytes-and-row factory bolted onto the +realism contract. + +Stealth (per ``feedback_stealth.md``): we never leak the +``DECNET`` literal into anything that survives to the planted file. +The underlying generators are already stealth-clean; this wrapper +must not undo that. +""" +from __future__ import annotations + +import os +import secrets as _secrets +from datetime import datetime, timezone +from typing import Any, Optional + +from decnet.canary.base import CanaryArtifact, CanaryContext +from decnet.canary.factory import get_generator +from decnet.logging import get_logger +from decnet.realism.personas import login_for +from decnet.realism.taxonomy import ContentClass, Plan + +log = get_logger("canary.cultivator") + + +# realism content_class → canary generator name. Mirrors +# :data:`decnet.canary.factory.KNOWN_GENERATORS`. +_CLASS_TO_GENERATOR: dict[ContentClass, str] = { + ContentClass.CANARY_AWS_CREDS: "aws_creds", + ContentClass.CANARY_ENV_FILE: "env_file", + ContentClass.CANARY_GIT_CONFIG: "git_config", + ContentClass.CANARY_SSH_KEY: "ssh_key", + ContentClass.CANARY_HONEYDOC: "honeydoc", + ContentClass.CANARY_HONEYDOC_DOCX: "honeydoc_docx", + ContentClass.CANARY_HONEYDOC_PDF: "honeydoc_pdf", + ContentClass.CANARY_MYSQL_DUMP: "mysql_dump", +} + + +# Generator → CanaryKind. The trip surface (HTTP slug callback / DNS +# resolution / passive bait) determines how the canary worker matches +# an attacker callback to this token. Aligned with +# :data:`decnet.web.db.models.canary.CanaryKind`. +_GENERATOR_TO_KIND: dict[str, str] = { + "aws_creds": "aws_passive", # no embedded callback; passive bait + "env_file": "http", + "git_config": "http", + "honeydoc": "http", + "honeydoc_docx": "http", + "honeydoc_pdf": "http", + "ssh_key": "dns", # trip is DNS resolution of host comment + "mysql_dump": "dns", # trip is DNS resolution of subdomain +} + + +# Path conventions per generator. The realism planner doesn't know +# about decoy-realistic credential locations (``~/.aws/credentials``, +# ``~/.git/config``); we map them per-class here so the planted +# artifact lands somewhere an attacker would actually look. +_DEFAULT_PATH: dict[ContentClass, str] = { + ContentClass.CANARY_AWS_CREDS: "/home/{persona}/.aws/credentials", + ContentClass.CANARY_ENV_FILE: "/home/{persona}/app/.env", + ContentClass.CANARY_GIT_CONFIG: "/home/{persona}/.git/config", + ContentClass.CANARY_SSH_KEY: "/home/{persona}/.ssh/id_rsa", + ContentClass.CANARY_HONEYDOC: "/home/{persona}/Documents/notes.html", + ContentClass.CANARY_HONEYDOC_DOCX: "/home/{persona}/Documents/Q3-Operations-Review.docx", + ContentClass.CANARY_HONEYDOC_PDF: "/home/{persona}/Documents/Q3-Operations-Review.pdf", + ContentClass.CANARY_MYSQL_DUMP: "/var/backups/db_backup.sql", +} + + +def _path_for(plan: Plan) -> str: + """Produce the canary placement path for *plan*. + + The realism planner already filled in ``plan.target_path`` from + the namer, but canary placements have stronger conventions + (``~/.aws/credentials``, ``~/.ssh/id_rsa``) than the realism + namer's vocabulary. When :data:`_DEFAULT_PATH` has an entry, + that wins. + """ + template = _DEFAULT_PATH.get(plan.content_class) + if template is None: + return plan.target_path + return template.format(persona=login_for(plan.persona)) + + +def _new_callback_token() -> str: + """16 url-safe bytes — same shape canary slug fields use elsewhere.""" + return _secrets.token_urlsafe(16) + + +async def cultivate( + plan: Plan, + repo: Any, + *, + http_base: Optional[str] = None, + dns_zone: Optional[str] = None, + created_by: str = "system", +) -> CanaryArtifact: + """Realism-driven canary plant. + + Build a :class:`CanaryContext`, ask the right generator for bytes, + persist a ``canary_tokens`` row so the canary worker can attribute + callbacks to this token, and return the artifact for the SSH + driver to plant. + + *http_base* and *dns_zone* default to ``DECNET_CANARY_HTTP_BASE`` + and ``DECNET_CANARY_DNS_ZONE`` env vars respectively — same + pattern the canary worker uses. When both are empty, generators + that need a callback host (``ssh_key`` DNS, ``mysql_dump``) + raise; the planner's caller logs and falls back to a non-canary + plan. + """ + if not plan.content_class.is_canary(): + raise ValueError( + f"cultivate() called with non-canary content_class=" + f"{plan.content_class!r}" + ) + gen_name = _CLASS_TO_GENERATOR.get(plan.content_class) + if gen_name is None: + raise KeyError( + f"no canary generator mapped for content_class=" + f"{plan.content_class!r}" + ) + + callback_token = _new_callback_token() + ctx = CanaryContext( + callback_token=callback_token, + http_base=http_base or os.environ.get("DECNET_CANARY_HTTP_BASE", ""), + dns_zone=dns_zone or os.environ.get("DECNET_CANARY_DNS_ZONE", ""), + persona="linux", # all our deckies are POSIX in MVP + ) + generator = get_generator(gen_name) + artifact = generator.generate(ctx) + + # The generator returns ``path=""`` (planter fills it normally). + # We have a realism-derived path on hand; stuff it in for the SSH + # driver's plant_file call AND the canary_tokens row. + placement_path = _path_for(plan) + + # Persist the token row before planting so the canary worker can + # attribute a callback if the artifact trips during the plant + # itself (improbable but possible — DOCX viewers can preview + # autoplay-style). + await repo.create_canary_token({ + "kind": _GENERATOR_TO_KIND.get(gen_name, "http"), + "decky_name": plan.decky_name, + "instrumenter": None, + "generator": gen_name, + "placement_path": placement_path, + "callback_token": callback_token, + "secret_seed": callback_token, # deterministic re-seed compatible + "placed_at": datetime.now(timezone.utc), + "created_by": created_by, + "state": "planted", + }) + + # Carry the placement_path on the artifact so the orchestrator's + # plant_file call uses it. We don't mutate the generator's + # original — copy with the new path. + return CanaryArtifact( + path=placement_path, + content=artifact.content, + mode=artifact.mode, + mtime_offset=artifact.mtime_offset, + instrumenter=artifact.instrumenter, + generator=artifact.generator, + notes=list(artifact.notes), + ) diff --git a/decnet/canary/dns_server.py b/decnet/canary/dns_server.py new file mode 100644 index 00000000..65cc6f60 --- /dev/null +++ b/decnet/canary/dns_server.py @@ -0,0 +1,207 @@ +"""Minimal authoritative DNS server for canary tokens (stdlib only). + +We don't need a full resolver — only enough to: + +1. Decode an inbound query's qname. +2. If the qname matches ``.``, log the callback, + publish ``canary..triggered`` on the bus, and return a + plausible A record (any RFC-5737 reserved address would do; we + use 192.0.2.1) so the attacker's resolver doesn't loop on + NXDOMAIN. +3. For unknown qnames return NXDOMAIN. + +DNS-over-UDP wire format is well-trodden: 12-byte header + name +labels + qtype + qclass. We implement just the bits we need. + +This module deliberately avoids the ``dnslib`` PyPI package so the +canary worker has no extra dependency surface. If we ever need +EDNS0, DNSSEC, or other niceties we'll swap to dnslib then. +""" +from __future__ import annotations + +import asyncio +import struct +from dataclasses import dataclass +from typing import Awaitable, Callable, Optional, Tuple + + +@dataclass(frozen=True) +class DNSQuery: + """Decoded query — only the bits the canary worker cares about.""" + + txid: int + qname: str # lowercase, no trailing dot + qtype: int + qclass: int + flags: int + + +def _decode_name(buf: bytes, offset: int) -> Tuple[str, int]: + """Return ``(qname_lowercase_no_dot, bytes_consumed)``. + + Supports compressed pointers (RFC 1035 §4.1.4). Doesn't recurse — + we walk the pointer chain iteratively with a hop cap to avoid + pointer-loop DoS. + """ + labels: list[str] = [] + pos = offset + consumed = 0 + jumped = False + hops = 0 + while True: + if pos >= len(buf): + raise ValueError("truncated DNS name") + length = buf[pos] + if length == 0: + pos += 1 + if not jumped: + consumed = pos - offset + break + if (length & 0xC0) == 0xC0: + # Compression pointer. + if pos + 1 >= len(buf): + raise ValueError("truncated DNS pointer") + ptr = ((length & 0x3F) << 8) | buf[pos + 1] + if not jumped: + consumed = (pos + 2) - offset + pos = ptr + jumped = True + hops += 1 + if hops > 10: + raise ValueError("DNS pointer loop") + continue + pos += 1 + if pos + length > len(buf): + raise ValueError("truncated DNS label") + labels.append(buf[pos:pos + length].decode("ascii", "replace")) + pos += length + return ".".join(labels).lower(), consumed + + +def parse_query(packet: bytes) -> DNSQuery: + """Parse the (single) question of a DNS query packet.""" + if len(packet) < 12: + raise ValueError("DNS packet too short") + txid, flags, qdcount, _ancount, _nscount, _arcount = struct.unpack( + "!HHHHHH", packet[:12] + ) + if qdcount != 1: + raise ValueError(f"expected 1 question, got {qdcount}") + qname, consumed = _decode_name(packet, 12) + pos = 12 + consumed + if pos + 4 > len(packet): + raise ValueError("truncated DNS qtype/qclass") + qtype, qclass = struct.unpack("!HH", packet[pos:pos + 4]) + return DNSQuery( + txid=txid, qname=qname, qtype=qtype, qclass=qclass, flags=flags, + ) + + +def _encode_name(name: str) -> bytes: + out = bytearray() + for label in name.split("."): + if not label: + continue + b = label.encode("ascii", "replace") + out.append(len(b)) + out.extend(b) + out.append(0) + return bytes(out) + + +def _build_response( + query: DNSQuery, + *, + rcode: int = 0, + answer_ip: Optional[str] = None, +) -> bytes: + """Encode a DNS response packet. + + *rcode* 0 = NOERROR, 3 = NXDOMAIN. When *answer_ip* is supplied + and the query was for an A record we include exactly one answer + (TTL 60, class IN). + """ + qd_count = 1 + an_count = 1 if (answer_ip and query.qtype == 1 and rcode == 0) else 0 + flags = 0x8400 | rcode # response + authoritative + RA bit clear + rcode + header = struct.pack( + "!HHHHHH", query.txid, flags, qd_count, an_count, 0, 0, + ) + qname_bytes = _encode_name(query.qname) + question = qname_bytes + struct.pack("!HH", query.qtype, query.qclass) + + answer = b"" + if an_count: + # Use a name pointer back to the question (offset 12). + ptr = struct.pack("!H", 0xC000 | 12) + rdata = bytes(int(o) for o in answer_ip.split(".")) + answer = ptr + struct.pack("!HHIH", 1, 1, 60, 4) + rdata + + return header + question + answer + + +# Hook signature: receives the matched slug + the query; returns +# nothing. The worker uses it to persist a CanaryTrigger row and +# publish the bus event. +TriggerHook = Callable[[str, DNSQuery, str], Awaitable[None]] + + +class CanaryDNSProtocol(asyncio.DatagramProtocol): + """asyncio UDP server endpoint for canary DNS callbacks. + + Constructor takes the canary zone (``"canary.example.test"``) and + a coroutine called when a query matches ``.``. The + hook runs in the event loop's task; we don't block the receive + path on it. + """ + + def __init__( + self, + zone: str, + hook: TriggerHook, + *, + answer_ip: str = "192.0.2.1", + ) -> None: + # Normalise: lowercase, no leading/trailing dot. + self._zone = zone.lower().strip(".") + self._suffix = "." + self._zone if self._zone else "" + self._hook = hook + self._answer_ip = answer_ip + self._transport: Optional[asyncio.DatagramTransport] = None + + def connection_made(self, transport) -> None: # type: ignore[override] + self._transport = transport # type: ignore[assignment] + + def datagram_received( # type: ignore[override] + self, data: bytes, addr: Tuple[str, int], + ) -> None: + try: + query = parse_query(data) + except ValueError: + # Malformed query — drop silently. Returning a FORMERR + # would tip off the attacker that *something* is listening + # on this port; the stealth posture (feedback_stealth) + # prefers radio silence on parse errors. + return + slug = self._slug_for(query.qname) + if slug is None: + # Unknown name — NXDOMAIN. + self._send(addr, _build_response(query, rcode=3)) + return + # Known name — answer with our sinkhole IP, then fire the hook. + self._send(addr, _build_response(query, answer_ip=self._answer_ip)) + asyncio.create_task(self._hook(slug, query, addr[0])) + + def _slug_for(self, qname: str) -> Optional[str]: + if not self._zone or not qname.endswith(self._suffix): + return None + slug = qname[: -len(self._suffix)] + # Single-label slug only; multi-label means the attacker is + # querying a sub-resource we don't model. + if not slug or "." in slug: + return None + return slug + + def _send(self, addr: Tuple[str, int], packet: bytes) -> None: + if self._transport is not None: + self._transport.sendto(packet, addr) diff --git a/decnet/canary/factory.py b/decnet/canary/factory.py new file mode 100644 index 00000000..345443f1 --- /dev/null +++ b/decnet/canary/factory.py @@ -0,0 +1,141 @@ +"""Generator and instrumenter factories. + +Same lazy-import pattern as :mod:`decnet.intel.factory` — concrete +implementations stay un-imported until first use so importing +:mod:`decnet.canary` from a CLI subcommand doesn't drag in +``pikepdf`` / ``python-docx`` / ``Pillow`` for callers that only +need the model layer. +""" +from __future__ import annotations + +from typing import Tuple + +from decnet.canary.base import CanaryGenerator, CanaryInstrumenter + +KNOWN_GENERATORS: Tuple[str, ...] = ( + "git_config", + "env_file", + "ssh_key", + "aws_creds", + "honeydoc", + "honeydoc_docx", + "honeydoc_pdf", + "mysql_dump", +) + +KNOWN_INSTRUMENTERS: Tuple[str, ...] = ( + "docx", + "xlsx", + "pdf", + "html", + "image", + "plain", + "passthrough", +) + + +def get_generator(name: str) -> CanaryGenerator: + """Return the generator registered under ``name``. + + Raises :class:`ValueError` for unknown names so a typo in the API + request surfaces as a 400 rather than silently producing nothing. + """ + if name == "git_config": + from decnet.canary.generators.git_config import GitConfigGenerator + return GitConfigGenerator() + if name == "env_file": + from decnet.canary.generators.env_file import EnvFileGenerator + return EnvFileGenerator() + if name == "ssh_key": + from decnet.canary.generators.ssh_key import SSHKeyGenerator + return SSHKeyGenerator() + if name == "aws_creds": + from decnet.canary.generators.aws_creds import AWSCredsGenerator + return AWSCredsGenerator() + if name == "honeydoc": + from decnet.canary.generators.honeydoc import HoneydocGenerator + return HoneydocGenerator() + if name == "honeydoc_docx": + from decnet.canary.generators.honeydoc_docx import HoneydocDocxGenerator + return HoneydocDocxGenerator() + if name == "honeydoc_pdf": + from decnet.canary.generators.honeydoc_pdf import HoneydocPdfGenerator + return HoneydocPdfGenerator() + if name == "mysql_dump": + from decnet.canary.generators.mysql_dump import MySQLDumpGenerator + return MySQLDumpGenerator() + raise ValueError( + f"Unknown canary generator: {name!r}. Known: {KNOWN_GENERATORS}" + ) + + +def get_instrumenter(name: str) -> CanaryInstrumenter: + """Return the instrumenter registered under ``name``.""" + if name == "docx": + from decnet.canary.instrumenters.docx import DocxInstrumenter + return DocxInstrumenter() + if name == "xlsx": + from decnet.canary.instrumenters.xlsx import XlsxInstrumenter + return XlsxInstrumenter() + if name == "pdf": + from decnet.canary.instrumenters.pdf import PdfInstrumenter + return PdfInstrumenter() + if name == "html": + from decnet.canary.instrumenters.html import HtmlInstrumenter + return HtmlInstrumenter() + if name == "image": + from decnet.canary.instrumenters.image import ImageInstrumenter + return ImageInstrumenter() + if name == "plain": + from decnet.canary.instrumenters.plain import PlainInstrumenter + return PlainInstrumenter() + if name == "passthrough": + from decnet.canary.instrumenters.passthrough import PassthroughInstrumenter + return PassthroughInstrumenter() + raise ValueError( + f"Unknown canary instrumenter: {name!r}. Known: {KNOWN_INSTRUMENTERS}" + ) + + +# MIME → instrumenter dispatch. Order matters: we walk the table +# top-to-bottom and the first prefix match wins, so put the more +# specific (DOCX/XLSX) before the generic (zip/octet-stream). +_MIME_DISPATCH: tuple[tuple[str, str], ...] = ( + # Office Open XML — DOCX/XLSX share a zip structure but expose + # different inner trees, so dispatch by MIME alias rather than + # zip-poking. + ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", "docx"), + ("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx"), + ("application/pdf", "pdf"), + ("text/html", "html"), + ("application/xhtml+xml", "html"), + ("image/png", "image"), + ("image/jpeg", "image"), + ("image/gif", "image"), + # Plaintext catch-alls — config files, .env, .ini, .yaml, .json, + # source code. All handled by the same regex-substitution pass. + ("text/", "plain"), + ("application/json", "plain"), + ("application/x-yaml", "plain"), + ("application/yaml", "plain"), + ("application/toml", "plain"), +) + + +def pick_instrumenter_for_mime(content_type: str) -> str: + """Return the instrumenter name registered for a sniffed MIME. + + Falls back to ``"passthrough"`` for anything we don't have an + embedder for (binary blobs we can't mutate safely — random + container images, archives, executables). ``passthrough`` only + supports DNS-callback tokens (the slug ends up in the filename or + an accompanying README), so the API surfaces that constraint to + the operator before they pick a kind. + """ + if not content_type: + return "passthrough" + lowered = content_type.lower() + for prefix, name in _MIME_DISPATCH: + if lowered.startswith(prefix): + return name + return "passthrough" diff --git a/decnet/canary/generators/__init__.py b/decnet/canary/generators/__init__.py new file mode 100644 index 00000000..cb06c181 --- /dev/null +++ b/decnet/canary/generators/__init__.py @@ -0,0 +1,7 @@ +"""Built-in canary generators (synthesised fake artifacts). + +Concrete classes live in sibling modules and are imported lazily by +:func:`decnet.canary.factory.get_generator` to keep the import-time +cost of :mod:`decnet.canary` cheap for callers that only need the +ABCs. +""" diff --git a/decnet/canary/generators/aws_creds.py b/decnet/canary/generators/aws_creds.py new file mode 100644 index 00000000..f02c201d --- /dev/null +++ b/decnet/canary/generators/aws_creds.py @@ -0,0 +1,86 @@ +"""Fake ``~/.aws/credentials`` block (passive bait). + +This is the **passive** variant — no callback wiring. An attacker +who exfils these keys can't trip a detection unless we run a real +AWS account with a deny-all CloudTrail listener (post-v1). The +realism is the point: the file looks like a routinely used credentials +file, so the rest of the decky's persona feels lived-in. + +If the operator picks ``kind="aws_passive"`` we accept that no slug +will be embedded. If they pick ``kind="http"`` or ``kind="dns"`` for +this generator, the API will reject the combination with a 400 — AWS +keys have no plausible field where a URL or hostname survives a +``grep -E '[A-Z0-9]{20}'`` smell test. +""" +from __future__ import annotations + +import hashlib +from secrets import token_urlsafe + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryGenerator + + +# Stable AWS-style key body derived from the slug. Keeping the +# generator deterministic (per-slug) means re-seeding produces the +# same bytes — the planter is naturally idempotent and an operator +# who runs ``decnet canary verify`` can re-derive the expected file +# without touching the DB. + +def _fake_access_key(seed: str) -> str: + # AWS access keys are 20 chars, uppercase alphanum, AKIA prefix. + body = hashlib.sha256(seed.encode()).hexdigest().upper() + return "AKIA" + body[:16] + + +def _fake_secret_key(seed: str) -> str: + # AWS secret keys are 40 chars, mixed-case base64-ish. We use + # base64-safe characters from token_urlsafe seeded by a SHA-256 + # of the seed so the output is stable per slug. + h = hashlib.sha256(("secret:" + seed).encode()).digest() + # Reuse token_urlsafe for the alphabet but pad to 40 chars from + # the deterministic bytes so we don't depend on os.urandom. + import base64 + return base64.b64encode(h)[:40].decode() + + +class AWSCredsGenerator(CanaryGenerator): + name = "aws_creds" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + seed = ctx.callback_token + access = _fake_access_key(seed) + secret = _fake_secret_key(seed) + body = ( + "[default]\n" + f"aws_access_key_id = {access}\n" + f"aws_secret_access_key = {secret}\n" + "region = us-east-1\n" + "\n" + "[prod]\n" + f"aws_access_key_id = {_fake_access_key('prod-' + seed)}\n" + f"aws_secret_access_key = {_fake_secret_key('prod-' + seed)}\n" + "region = us-west-2\n" + ) + return CanaryArtifact( + path="", # caller (planter) fills this from CanaryToken.placement_path + content=body.encode("utf-8"), + mode=0o600, + mtime_offset=-86400 * 14, # 2 weeks ago — looks lived-in + generator=self.name, + notes=[ + "fake AWS keys; no callback embedded — passive bait only", + f"derived deterministically from slug={seed}", + ], + ) + + +# Re-exported so the slug helper is reusable from the +# instrumenters/passthrough module without an internal import path. +__all__ = ["AWSCredsGenerator", "_fake_access_key", "_fake_secret_key"] + + +# Imports at the bottom keep the public dataclasses on top — pylint +# doesn't run on this repo, but tests do, and putting ``token_urlsafe`` +# in a public symbol confuses readers. Suppress the unused warning by +# referencing it once. +_ = token_urlsafe diff --git a/decnet/canary/generators/env_file.py b/decnet/canary/generators/env_file.py new file mode 100644 index 00000000..979b1dfd --- /dev/null +++ b/decnet/canary/generators/env_file.py @@ -0,0 +1,56 @@ +"""Fake ``.env`` with embedded callback URLs. + +Modern web stacks read environment variables for everything from +database DSNs to webhook URLs, so dropping a few realistic-looking +``KEY=value`` pairs alongside the canary URL is unremarkable. The +slug appears in two fields: + +* ``API_BASE_URL`` — the obvious one; an attacker scripting against + the credentials hits the worker on first invocation. +* ``WEBHOOK_NOTIFY_URL`` — secondary, in case the attacker greps for + ``WEBHOOK`` and pivots there. + +Other fields (``DB_PASSWORD``, ``REDIS_URL``, ``JWT_SECRET``) are +plausible but inert — they're realism filler, not detection +mechanisms. +""" +from __future__ import annotations + +import hashlib + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryGenerator + + +def _stable_token(seed: str, prefix: str = "") -> str: + h = hashlib.sha256((prefix + seed).encode()).hexdigest() + return h[:32] + + +class EnvFileGenerator(CanaryGenerator): + name = "env_file" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + base = ctx.http_base.rstrip("/") + slug = ctx.callback_token + api_url = f"{base}/c/{slug}" + body = ( + "# Production environment — DO NOT COMMIT\n" + f"API_BASE_URL={api_url}\n" + f"WEBHOOK_NOTIFY_URL={api_url}/webhook\n" + f"DB_PASSWORD={_stable_token(slug, 'db:')}\n" + f"REDIS_URL=redis://:{_stable_token(slug, 'redis:')[:16]}@redis.internal:6379/0\n" + f"JWT_SECRET={_stable_token(slug, 'jwt:')}\n" + "LOG_LEVEL=info\n" + "ENVIRONMENT=production\n" + ) + return CanaryArtifact( + path="", + content=body.encode("utf-8"), + mode=0o600, + mtime_offset=-86400 * 7, # last edited a week ago + generator=self.name, + notes=[ + f"API_BASE_URL embeds {api_url}", + f"WEBHOOK_NOTIFY_URL embeds {api_url}/webhook", + ], + ) diff --git a/decnet/canary/generators/git_config.py b/decnet/canary/generators/git_config.py new file mode 100644 index 00000000..297f18ab --- /dev/null +++ b/decnet/canary/generators/git_config.py @@ -0,0 +1,53 @@ +"""Fake ``.git/config`` with an attacker-bait remote URL. + +The ``[remote "origin"]`` ``url`` field is the natural place to embed +an HTTP-callback URL: it's normal for git remotes to be HTTPS, the +URL is read by every git command an attacker runs (``git pull``, +``git fetch``, ``git remote -v``), and the slug fits naturally as +part of a path. + +The generator emits a plausible private-mirror remote (``git.`` +or the canary host's hostname) so an attacker doesn't immediately +recognise it as a honeypot. The slug ends up in the URL path: + + [remote "origin"] + url = https://canary.example.test/c//repo.git +""" +from __future__ import annotations + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryGenerator + + +class GitConfigGenerator(CanaryGenerator): + name = "git_config" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + # Strip trailing slash defensively — operator may have + # configured DECNET_CANARY_HTTP_BASE either way. + base = ctx.http_base.rstrip("/") + slug = ctx.callback_token + # The /c//repo.git suffix gives us a realistic-looking + # path the worker can route on a single ``startswith("/c/")`` + # check, while still surviving a quick grep for the slug. + url = f"{base}/c/{slug}/repo.git" + body = ( + "[core]\n" + "\trepositoryformatversion = 0\n" + "\tfilemode = true\n" + "\tbare = false\n" + "\tlogallrefupdates = true\n" + "[remote \"origin\"]\n" + f"\turl = {url}\n" + "\tfetch = +refs/heads/*:refs/remotes/origin/*\n" + "[branch \"main\"]\n" + "\tremote = origin\n" + "\tmerge = refs/heads/main\n" + ) + return CanaryArtifact( + path="", + content=body.encode("utf-8"), + mode=0o644, + mtime_offset=-86400 * 30, # checked out a month ago + generator=self.name, + notes=[f"git remote 'origin' embeds {url}"], + ) diff --git a/decnet/canary/generators/honeydoc.py b/decnet/canary/generators/honeydoc.py new file mode 100644 index 00000000..455460b3 --- /dev/null +++ b/decnet/canary/generators/honeydoc.py @@ -0,0 +1,61 @@ +"""Built-in honeydoc — a minimal HTML "report" with a tracking pixel. + +This is the *fallback* honeydoc used when the operator hasn't +uploaded a real document. The HTML instrumenter handles operator +uploads via :mod:`decnet.canary.instrumenters.html`; this generator +exists so the deploy-time baseline can plant *something* convincing +without first prompting the operator to drop a file. + +The realism here is intentionally modest: a Documents-folder HTML +page with internal-looking content and a 1×1 remote image at the +bottom whose ``src`` is the canary callback URL. Most desktop +HTML renderers fetch the image as soon as the file is opened in a +browser preview, so opening the doc trips the callback. + +Operators who want a richer artifact should upload their own DOCX +or PDF; the corresponding instrumenter embeds the same callback in +the appropriate format. +""" +from __future__ import annotations + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryGenerator + + +class HoneydocGenerator(CanaryGenerator): + name = "honeydoc" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + base = ctx.http_base.rstrip("/") + slug = ctx.callback_token + pixel_url = f"{base}/c/{slug}" + body = ( + "\n" + "\n" + "\n" + "\n" + "Q3 Operations Review — DRAFT\n" + "\n" + "\n" + "

Q3 Operations Review (DRAFT — DO NOT DISTRIBUTE)

\n" + "

Forecast and remediation timeline below. Numbers are\n" + "preliminary and subject to revision before the all-hands.

\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "
RegionIncidentsMTTR (h)
us-east143.2
us-west94.7
eu-central222.1
\n" + "

Internal contact: " + "secops@internal

\n" + f"\"\"\n" + "\n" + "\n" + ) + return CanaryArtifact( + path="", + content=body.encode("utf-8"), + mode=0o644, # docs are typically world-readable + mtime_offset=-86400 * 21, # 3 weeks ago + generator=self.name, + notes=[f"tracking pixel src={pixel_url}"], + ) diff --git a/decnet/canary/generators/honeydoc_docx.py b/decnet/canary/generators/honeydoc_docx.py new file mode 100644 index 00000000..35456a23 --- /dev/null +++ b/decnet/canary/generators/honeydoc_docx.py @@ -0,0 +1,133 @@ +"""Real-DOCX honeydoc generator. + +Synthesises a minimal but structurally valid DOCX from scratch via +stdlib :mod:`zipfile`, then uses the same external-image relationship +trick that powers :mod:`decnet.canary.instrumenters.docx` to embed +the callback URL. No python-docx dependency. + +The output opens cleanly in Word / LibreOffice; both fetch the +external image relationship on document load. +""" +from __future__ import annotations + +import io +import zipfile + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryGenerator +from decnet.canary.instrumenters.docx import _drawing, _next_rid + + +_CONTENT_TYPES = ( + '' + '' + '' + '' + '' + '' +).encode() + +_PACKAGE_RELS = ( + '' + '' + '' + '' +).encode() + +_BODY_PARAGRAPHS = ( + "Q3 Operations Review (DRAFT — DO NOT DISTRIBUTE)", + "", + "Forecast and remediation timeline below. Numbers are preliminary " + "and subject to revision before the all-hands.", + "", + "Region Incidents MTTR (h)", + "us-east 14 3.2", + "us-west 9 4.7", + "eu-central 22 2.1", + "", + "Internal contact: secops@internal", +) + + +def _document_xml(rid_with_drawing: str | None = None) -> bytes: + """Build the body XML. + + ``rid_with_drawing`` is the rId of the external image relationship; + when set, we append the same ```` element that the DOCX + instrumenter inserts so the body references the external resource. + """ + paragraphs = [] + for line in _BODY_PARAGRAPHS: + if line: + paragraphs.append( + "" + + _xml_escape(line) + + "" + ) + else: + paragraphs.append("") + body = "".join(paragraphs) + drawing = _drawing(rid_with_drawing).decode() if rid_with_drawing else "" + return ( + '' + '' + f'{body}{drawing}' + '' + ).encode() + + +def _xml_escape(s: str) -> str: + return ( + s.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + ) + + +def _document_rels(rid: str, url: str) -> bytes: + return ( + '' + '' + f'' + '' + ).encode() + + +class HoneydocDocxGenerator(CanaryGenerator): + name = "honeydoc_docx" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + url = f"{ctx.http_base.rstrip('/')}/c/{ctx.callback_token}" + # Pick a stable rId — there's only one relationship in the + # synthesised file, so any unused id works. Reuse the + # instrumenter's allocator against the bare relationships + # skeleton for parity with operator-uploaded DOCX flow. + skeleton = ( + b'' + b'' + b'' + ) + rid = _next_rid(skeleton) + + out = io.BytesIO() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr("[Content_Types].xml", _CONTENT_TYPES) + zf.writestr("_rels/.rels", _PACKAGE_RELS) + zf.writestr("word/document.xml", _document_xml(rid)) + zf.writestr("word/_rels/document.xml.rels", _document_rels(rid, url)) + + return CanaryArtifact( + path="", + content=out.getvalue(), + mode=0o644, + mtime_offset=-86400 * 21, + generator=self.name, + notes=[ + "synthesised DOCX with realistic Q3 review body", + f"external-image relationship {rid} -> {url}", + ], + ) diff --git a/decnet/canary/generators/honeydoc_pdf.py b/decnet/canary/generators/honeydoc_pdf.py new file mode 100644 index 00000000..400271ff --- /dev/null +++ b/decnet/canary/generators/honeydoc_pdf.py @@ -0,0 +1,127 @@ +"""Real-PDF honeydoc generator (uses :mod:`pikepdf`). + +Builds a one-page PDF with the same Q3-review body as the HTML/DOCX +flavors and installs an ``/OpenAction`` ``/URI`` action on the +catalog so most viewers fire the callback the moment the document +opens. + +Pikepdf is now a hard dependency for this generator (the operator +installed it explicitly so we can use it). We still surface a +clear :class:`InstrumenterRejectedError` when imports fail, so a +deployment without pikepdf can fall back to the DOCX or HTML +generators rather than crashing the API. +""" +from __future__ import annotations + +import io + +from decnet.canary.base import ( + CanaryArtifact, + CanaryContext, + CanaryGenerator, + InstrumenterRejectedError, +) + + +_BODY_LINES = ( + ("Q3 Operations Review (DRAFT — DO NOT DISTRIBUTE)", 14), + ("", 12), + ("Forecast and remediation timeline below.", 11), + ("Numbers are preliminary, subject to revision.", 11), + ("", 12), + ("Region Incidents MTTR (h)", 11), + ("us-east 14 3.2", 11), + ("us-west 9 4.7", 11), + ("eu-central 22 2.1", 11), + ("", 12), + ("Internal contact: secops@internal", 11), +) + + +class HoneydocPdfGenerator(CanaryGenerator): + name = "honeydoc_pdf" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + try: + from pikepdf import Pdf, Name, Dictionary, String # type: ignore[import-not-found] + except ImportError as e: + raise InstrumenterRejectedError( + "honeydoc_pdf requires pikepdf; install it (`pip install " + "pikepdf`) or pick honeydoc / honeydoc_docx instead." + ) from e + + url = f"{ctx.http_base.rstrip('/')}/c/{ctx.callback_token}" + + pdf = Pdf.new() + # Helvetica is one of the 14 PDF base fonts — every viewer ships + # it, so no font embedding is required. + font = pdf.make_indirect(Dictionary( + Type=Name("/Font"), + Subtype=Name("/Type1"), + BaseFont=Name("/Helvetica"), + )) + + # Build a single content stream that writes each body line at a + # decreasing y-coordinate. PDF coordinates start at the bottom- + # left (US Letter = 612 x 792 points); we lay out lines roughly + # 18 points apart starting near the top. + ops: list[str] = ["BT /F1 12 Tf 72 750 Td"] + first = True + for line, size in _BODY_LINES: + if not first: + ops.append("0 -18 Td") + first = False + ops.append(f"/F1 {size} Tf") + ops.append(f"({_pdf_escape(line)}) Tj") + ops.append("ET") + content_bytes = "\n".join(ops).encode("latin-1") + + content_stream = pdf.make_stream(content_bytes) + + page = pdf.add_blank_page(page_size=(612, 792)) + page[Name("/Resources")] = Dictionary( + Font=Dictionary(F1=font), + ) + page[Name("/Contents")] = content_stream + + # OpenAction fires the URI when the file is opened in Acrobat, + # Preview, the browser PDF viewer, etc. Most viewers prompt + # before fetching; that prompt itself is a tell, and an + # auto-allow viewer fetches silently. + pdf.Root[Name("/OpenAction")] = Dictionary( + Type=Name("/Action"), + S=Name("/URI"), + URI=String(url), + ) + + out = io.BytesIO() + pdf.save(out) + return CanaryArtifact( + path="", + content=out.getvalue(), + mode=0o644, + mtime_offset=-86400 * 21, + generator=self.name, + notes=[ + "synthesised one-page PDF with realistic Q3 review body", + f"/OpenAction /URI -> {url}", + ], + ) + + +def _pdf_escape(s: str) -> str: + """Escape parens and backslashes for PDF literal-string syntax. + + PDF string literals are wrapped in ``( … )``; inner ``(``, ``)``, + and ``\\`` need backslash escapes. Everything else (including + UTF-8 multibyte sequences) round-trips fine because Helvetica's + encoding is WinAnsi-ish — we'll lose exotic glyphs but the + realistic body sticks to ASCII anyway. Em-dashes are downgraded + to ``--`` to avoid the WinAnsi gap. + """ + return ( + s.replace("\\", r"\\") + .replace("(", r"\(") + .replace(")", r"\)") + .replace("—", "--") + ) diff --git a/decnet/canary/generators/mysql_dump.py b/decnet/canary/generators/mysql_dump.py new file mode 100644 index 00000000..ab324137 --- /dev/null +++ b/decnet/canary/generators/mysql_dump.py @@ -0,0 +1,190 @@ +"""Fake ``mysqldump`` output that phones home on import. + +Mirrors the Canarytokens.org MySQL-dump trick. When a victim runs +``mysql < dump.sql``, the trailer block executes a base64-obfuscated +``CHANGE REPLICATION SOURCE TO`` against ``.canary.`` +followed by ``START REPLICA``. The victim's MySQL daemon then: + +1. Resolves the slug subdomain via DNS — this is the trip our + :mod:`decnet.canary.dns_server` already detects. +2. Opens a TCP replica handshake on port 3306, sending its own + ``@@hostname`` and ``@@lc_time_names`` smuggled into the + ``SOURCE_USER`` field via ``CONCAT``. Capturing those bytes + requires a MySQL handshake responder on the worker — out of scope + for v1; the DNS lookup alone is sufficient for detection. + +The base64 wrapper is the camouflage: a plain ``grep canary dump.sql`` +finds nothing. The slug only materialises when the victim's server +runs ``PREPARE … FROM @s2``. + +Because the trip surface is DNS, this generator REQUIRES a non-empty +``dns_zone``. The slug must appear as the leftmost label of the +hostname so a single DNS query identifies the token; the http_base +host is not slug-bearing and can't substitute. +""" +from __future__ import annotations + +import base64 +import hashlib + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryGenerator + + +def _stable_hex(seed: str, prefix: str = "", length: int = 16) -> str: + h = hashlib.sha256((prefix + seed).encode()).hexdigest() + return h[:length] + + +def _build_replica_payload(slug: str, dns_zone: str) -> str: + """Inner SQL that gets base64-wrapped. + + The CONCAT splices ``@@lc_time_names`` and ``@@hostname`` into the + ``SOURCE_USER`` value at PREPARE time so the victim's locale and + hostname travel as the replica username on the 3306 handshake. + """ + host = f"{slug}.{dns_zone}" + return ( + "SET @bb = CONCAT(" + "\"CHANGE REPLICATION SOURCE TO " + "SOURCE_PASSWORD='replica-pw', " + "SOURCE_RETRY_COUNT=1, " + "SOURCE_PORT=3306, " + f"SOURCE_HOST='{host}', " + "SOURCE_SSL=0, " + f"SOURCE_USER='{slug}\", " + "@@lc_time_names, @@hostname, \"';\");" + ) + + +def _build_trailer(slug: str, dns_zone: str) -> str: + inner = _build_replica_payload(slug, dns_zone) + encoded = base64.b64encode(inner.encode("utf-8")).decode("ascii") + return ( + f"SET @b = '{encoded}';\n" + "SET @s2 = FROM_BASE64(@b);\n" + "PREPARE stmt1 FROM @s2;\n" + "EXECUTE stmt1;\n" + "PREPARE stmt2 FROM @bb;\n" + "EXECUTE stmt2;\n" + "START REPLICA;\n" + ) + + +class MySQLDumpGenerator(CanaryGenerator): + name = "mysql_dump" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + if not ctx.dns_zone: + raise ValueError( + "mysql_dump requires a non-empty dns_zone — the trip " + "surface is a DNS lookup of .." + ) + slug = ctx.callback_token + zone = ctx.dns_zone + host = f"{slug}.{zone}" + + # Realism filler: deterministic per-slug fake user rows so two + # runs with the same context produce byte-identical output + # (planter idempotency contract). + u1_hash = _stable_hex(slug, "u1:", 32) + u2_hash = _stable_hex(slug, "u2:", 32) + api_token = _stable_hex(slug, "api:", 40) + + # Synthesised SQL bait below — never executed by us, only by + # whoever runs ``mysql < dump.sql`` against their own server. + # Built with .format() instead of f-strings so bandit's B608 + # heuristic doesn't false-positive on the "INSERT INTO" + var + # pattern. + users_insert = ( + "INSERT INTO `users` VALUES " # nosec B608 + "(1,'alice@app.internal','$2y$10${u1a}.{u1b}','2024-11-12 09:13:44')," + "(2,'bob@app.internal','$2y$10${u2a}.{u2b}','2025-02-03 17:42:08');\n" + ).replace("{u1a}", u1_hash[:22]).replace("{u1b}", u1_hash[22:]) \ + .replace("{u2a}", u2_hash[:22]).replace("{u2b}", u2_hash[22:]) + api_keys_insert = ( + "INSERT INTO `api_keys` VALUES (1,1,'{tok}');\n" # nosec B608 + ).replace("{tok}", api_token) + header = ( + "-- MySQL dump 10.13 Distrib 8.0.35, for Linux (x86_64)\n" + "--\n" + "-- Host: db-prod-01 Database: app_production\n" + "-- ------------------------------------------------------\n" + "-- Server version\t8.0.35\n" + "\n" + "/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;\n" + "/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;\n" + "/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;\n" + "/*!50503 SET NAMES utf8mb4 */;\n" + "/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;\n" + "/*!40103 SET TIME_ZONE='+00:00' */;\n" + "/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;\n" + "/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;\n" + "/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;\n" + "/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;\n" + "\n" + "--\n" + "-- Table structure for table `users`\n" + "--\n" + "\n" + "DROP TABLE IF EXISTS `users`;\n" + "CREATE TABLE `users` (\n" + " `id` int unsigned NOT NULL AUTO_INCREMENT,\n" + " `email` varchar(255) NOT NULL,\n" + " `password_hash` char(60) NOT NULL,\n" + " `created_at` datetime NOT NULL,\n" + " PRIMARY KEY (`id`),\n" + " UNIQUE KEY `uniq_email` (`email`)\n" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n" + "\n" + "LOCK TABLES `users` WRITE;\n" + + users_insert + + "UNLOCK TABLES;\n" + "\n" + "--\n" + "-- Table structure for table `api_keys`\n" + "--\n" + "\n" + "DROP TABLE IF EXISTS `api_keys`;\n" + "CREATE TABLE `api_keys` (\n" + " `id` int unsigned NOT NULL AUTO_INCREMENT,\n" + " `user_id` int unsigned NOT NULL,\n" + " `token` char(40) NOT NULL,\n" + " PRIMARY KEY (`id`)\n" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n" + "\n" + "LOCK TABLES `api_keys` WRITE;\n" + + api_keys_insert + + "UNLOCK TABLES;\n" + "\n" + ) + + trailer_replica = _build_trailer(slug, zone) + + trailer_close = ( + "\n" + "/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;\n" + "/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;\n" + "/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;\n" + "/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;\n" + "/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;\n" + "/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;\n" + "/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;\n" + "/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;\n" + "\n" + "-- Dump completed\n" + ) + + body = header + trailer_replica + trailer_close + + return CanaryArtifact( + path="", + content=body.encode("utf-8"), + mode=0o600, + mtime_offset=-86400 * 7, # last week's backup + generator=self.name, + notes=[ + f"replica payload phones home to {host}:3306 on import", + "base64-wrapped PREPARE/EXECUTE block hides the slug from grep", + "@@hostname and @@lc_time_names smuggled into SOURCE_USER", + ], + ) diff --git a/decnet/canary/generators/ssh_key.py b/decnet/canary/generators/ssh_key.py new file mode 100644 index 00000000..96835aa4 --- /dev/null +++ b/decnet/canary/generators/ssh_key.py @@ -0,0 +1,68 @@ +"""Fake SSH private key with the callback host in the comment. + +OpenSSH private keys carry a free-form comment field — typically +``user@host`` — that's preserved across rounds of ``ssh-keygen -p``. +We embed the canary host as the ``user@host`` so an attacker who +imports the key into their own keyring or runs ``ssh-keygen -lf`` on +it sees a hostname they may then try to reach. + +The key bytes themselves are syntactically valid (PEM envelope, base64 +body) but cryptographically junk — the body is a deterministic SHA-256 +hash of the slug repeated to the right length. We don't ship a real +RSA/Ed25519 key because (a) we don't want a real private key sitting +on disk pretending to be valuable, and (b) the attacker ``cat``-ing +the file or running ``ssh -i`` will trigger the callback regardless +of cryptographic validity. + +The DNS-callback variant uses ``.canary.`` as the +hostname so a bare ``ssh-keygen -lf`` on the file resolves a unique +subdomain even if the attacker never hits HTTP. +""" +from __future__ import annotations + +import base64 +import hashlib + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryGenerator + + +def _fake_key_body(seed: str) -> str: + # Real OpenSSH keys are several hundred base64 chars; we make a + # plausible-looking 24-line block from a SHA-256-derived stream. + h = hashlib.sha256(seed.encode()).digest() + long_stream = (h * 32)[:768] # 768 bytes → ~1024 base64 chars + encoded = base64.b64encode(long_stream).decode() + # Wrap at 70 chars per line — same shape ``ssh-keygen`` produces. + return "\n".join(encoded[i:i + 70] for i in range(0, len(encoded), 70)) + + +class SSHKeyGenerator(CanaryGenerator): + name = "ssh_key" + + def generate(self, ctx: CanaryContext) -> CanaryArtifact: + slug = ctx.callback_token + body = _fake_key_body(slug) + # Hostname for the comment: prefer DNS-zone form when the + # operator has DNS deployed (so ssh-keygen -lf names a subdomain + # the attacker may resolve); fall back to the http_base host + # otherwise. + if ctx.dns_zone: + host_comment = f"deploy@{slug}.{ctx.dns_zone}" + else: + from urllib.parse import urlparse + host = urlparse(ctx.http_base).hostname or "deploy.local" + host_comment = f"deploy@{host}" + content = ( + "-----BEGIN OPENSSH PRIVATE KEY-----\n" + f"{body}\n" + "-----END OPENSSH PRIVATE KEY-----\n" + f"# {host_comment}\n" + ) + return CanaryArtifact( + path="", + content=content.encode("utf-8"), + mode=0o600, + mtime_offset=-86400 * 60, # 2 months ago + generator=self.name, + notes=[f"comment line embeds {host_comment}"], + ) diff --git a/decnet/canary/instrumenters/__init__.py b/decnet/canary/instrumenters/__init__.py new file mode 100644 index 00000000..905e02b6 --- /dev/null +++ b/decnet/canary/instrumenters/__init__.py @@ -0,0 +1,4 @@ +"""Built-in canary instrumenters (operator-uploaded artifact mutation). + +Lazy-imported by :func:`decnet.canary.factory.get_instrumenter`. +""" diff --git a/decnet/canary/instrumenters/docx.py b/decnet/canary/instrumenters/docx.py new file mode 100644 index 00000000..f0a87903 --- /dev/null +++ b/decnet/canary/instrumenters/docx.py @@ -0,0 +1,147 @@ +"""DOCX instrumenter — inject a remote image into the body. + +DOCX files are zip archives carrying ``word/document.xml`` (the body) +and ``word/_rels/document.xml.rels`` (the relationship table that +maps ``rId`` references to URLs). We: + +1. Add a new relationship of type ``image`` whose target is the + canary callback URL and ``TargetMode="External"``. +2. Add a tiny ```` element referencing that ``rId`` at + the end of ``word/document.xml`` (just before ````). + +Word and LibreOffice both fetch external image relationships when +the document is opened (subject to the user's "trusted source" +toggle, which most enterprise environments disable in favour of +"warn but allow"). + +We use stdlib ``zipfile`` only — no python-docx dependency — because +the surface we touch is two small XML files and we don't need any of +the higher-level abstractions. +""" +from __future__ import annotations + +import io +import re +import zipfile +from typing import Tuple + +from decnet.canary.base import ( + CanaryArtifact, + CanaryContext, + CanaryInstrumenter, + InstrumenterRejectedError, +) + + +_RELS_END = re.compile(rb"", re.IGNORECASE) +_BODY_END = re.compile(rb"", re.IGNORECASE) + + +def _next_rid(rels_xml: bytes) -> str: + """Return an rId not already taken in the relationships file. + + Word's loader tolerates non-sequential ids, so we just pick one + well above the typical range to avoid collisions. + """ + used = set(m.group(1).decode() for m in re.finditer(rb'Id="(rId\d+)"', rels_xml)) + for n in range(900, 9999): + rid = f"rId{n}" + if rid not in used: + return rid + raise InstrumenterRejectedError("DOCX has too many relationships to allocate a new rId") + + +def _inject_relationship(rels_xml: bytes, rid: str, url: str) -> bytes: + rel = ( + f'' + ).encode() + match = _RELS_END.search(rels_xml) + if not match: + raise InstrumenterRejectedError( + "DOCX rels file has no ; refusing to mutate" + ) + return rels_xml[:match.start()] + rel + rels_xml[match.start():] + + +def _drawing(rid: str) -> bytes: + # Minimal w:drawing tree referencing the external image at rid. + # Dimensions are 1 EMU x 1 EMU so the image is invisible; Word + # still fetches the resource on document load. + return ( + '' + '' + '' + '' + '' + '' + '' + '' + f'' + '' + '' + '' + '' + '' + '' + ).encode() + + +def _inject_drawing(document_xml: bytes, rid: str) -> bytes: + match = _BODY_END.search(document_xml) + if not match: + raise InstrumenterRejectedError("DOCX document.xml has no ") + drawing = _drawing(rid) + return document_xml[:match.start()] + drawing + document_xml[match.start():] + + +def _mutate(blob: bytes, url: str) -> Tuple[bytes, str]: + try: + with zipfile.ZipFile(io.BytesIO(blob), "r") as zf: + try: + rels = zf.read("word/_rels/document.xml.rels") + doc = zf.read("word/document.xml") + except KeyError as e: + raise InstrumenterRejectedError( + f"DOCX missing expected member: {e.args[0]!r}" + ) from e + members = [(zi, zf.read(zi.filename)) for zi in zf.infolist()] + except zipfile.BadZipFile as e: + raise InstrumenterRejectedError("uploaded blob is not a valid DOCX zip") from e + + rid = _next_rid(rels) + new_rels = _inject_relationship(rels, rid, url) + new_doc = _inject_drawing(doc, rid) + + out = io.BytesIO() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zf_out: + for zi, data in members: + if zi.filename == "word/_rels/document.xml.rels": + zf_out.writestr(zi.filename, new_rels) + elif zi.filename == "word/document.xml": + zf_out.writestr(zi.filename, new_doc) + else: + zf_out.writestr(zi, data) + return out.getvalue(), rid + + +class DocxInstrumenter(CanaryInstrumenter): + name = "docx" + mime_prefixes = ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ) + + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + url = f"{ctx.http_base.rstrip('/')}/c/{ctx.callback_token}" + mutated, rid = _mutate(blob, url) + return CanaryArtifact( + path=target_path, + content=mutated, + mode=0o644, + mtime_offset=-86400 * 14, + instrumenter=self.name, + notes=[f"injected external-image relationship {rid} -> {url}"], + ) diff --git a/decnet/canary/instrumenters/html.py b/decnet/canary/instrumenters/html.py new file mode 100644 index 00000000..02b4d4e2 --- /dev/null +++ b/decnet/canary/instrumenters/html.py @@ -0,0 +1,45 @@ +"""HTML instrumenter — append a 1×1 tracking pixel. + +Stdlib-only. We don't parse the HTML; we just inject the ```` +tag immediately before the closing ```` (or, failing that, at +the end of the document). Most renderers that support remote images +(email previewers, IDE doc previews, browsers) will fetch it as +soon as the document is opened. +""" +from __future__ import annotations + +import re + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryInstrumenter + + +_BODY_CLOSE = re.compile(rb"", re.IGNORECASE) + + +class HtmlInstrumenter(CanaryInstrumenter): + name = "html" + mime_prefixes = ("text/html", "application/xhtml+xml") + + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + url = f"{ctx.http_base.rstrip('/')}/c/{ctx.callback_token}".encode() + pixel = ( + b"\n" + ) + match = _BODY_CLOSE.search(blob) + if match: + out = blob[:match.start()] + pixel + blob[match.start():] + note = "injected 1x1 pixel before " + else: + out = (blob if blob.endswith(b"\n") else blob + b"\n") + pixel + note = "appended 1x1 pixel (no found)" + return CanaryArtifact( + path=target_path, + content=out, + mode=0o644, + mtime_offset=-86400 * 7, + instrumenter=self.name, + notes=[note, f"pixel src={url.decode()}"], + ) diff --git a/decnet/canary/instrumenters/image.py b/decnet/canary/instrumenters/image.py new file mode 100644 index 00000000..69e31ff4 --- /dev/null +++ b/decnet/canary/instrumenters/image.py @@ -0,0 +1,72 @@ +"""Image instrumenter — requires :mod:`PIL` (optional dependency). + +For PNG/JPEG/GIF we append a tEXt/EXIF chunk carrying the slug so +``exiftool`` / ``identify -verbose`` surface the slug, then route the +detection via a sibling **plain-text companion file**. The image +itself can't really embed an HTTP fetcher — image decoders don't +run network requests on decode — so the realistic detection surface +is "attacker exfils the image, runs metadata tools on it, hits our +URL when curious about the embedded marker." + +When Pillow isn't installed we reject and direct the operator to +``passthrough`` (which preserves the bytes; the slug then lives in +the filename only). +""" +from __future__ import annotations + +import io + +from decnet.canary.base import ( + CanaryArtifact, + CanaryContext, + CanaryInstrumenter, + InstrumenterRejectedError, +) + + +class ImageInstrumenter(CanaryInstrumenter): + name = "image" + mime_prefixes = ("image/png", "image/jpeg", "image/gif") + + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + try: + from PIL import Image, PngImagePlugin # type: ignore[import-not-found] + except ImportError as e: + raise InstrumenterRejectedError( + "image instrumenter requires Pillow; install it (`pip " + "install Pillow`) or re-upload the artifact with " + "kind=passthrough so it ships unmodified." + ) from e + + slug_url = f"{ctx.http_base.rstrip('/')}/c/{ctx.callback_token}" + try: + buf_in = io.BytesIO(blob) + img = Image.open(buf_in) + fmt = (img.format or "").upper() + buf_out = io.BytesIO() + if fmt == "PNG": + meta = PngImagePlugin.PngInfo() + meta.add_text("Comment", f"reference: {slug_url}") + meta.add_text("X-Canary", ctx.callback_token) + img.save(buf_out, format="PNG", pnginfo=meta) + elif fmt in ("JPEG", "JPG"): + # Pillow encodes JPEG comments via the ``comment`` kwarg. + img.save(buf_out, format="JPEG", comment=slug_url.encode()) + else: + # GIF and friends — Pillow doesn't expose comment metadata + # uniformly. Re-encode as-is and skip the metadata embed. + img.save(buf_out, format=fmt or "PNG") + mutated = buf_out.getvalue() + except Exception as e: + raise InstrumenterRejectedError(f"failed to instrument image: {e!s}") from e + + return CanaryArtifact( + path=target_path, + content=mutated, + mode=0o644, + mtime_offset=-86400 * 30, + instrumenter=self.name, + notes=[f"image metadata carries {slug_url} (slug={ctx.callback_token})"], + ) diff --git a/decnet/canary/instrumenters/passthrough.py b/decnet/canary/instrumenters/passthrough.py new file mode 100644 index 00000000..09816d86 --- /dev/null +++ b/decnet/canary/instrumenters/passthrough.py @@ -0,0 +1,37 @@ +"""Passthrough instrumenter — bytes go to disk unchanged. + +Used as the dispatch fallback for content types we can't safely +mutate (random binary blobs, container images, archives we don't +recognise). In passthrough mode the only callback surface is the +:attr:`CanaryToken.placement_path` itself: the operator must use a +DNS-callback token whose slug appears in the filename, so a +listing/access at the OS level resolves the slug as part of the +path (e.g. ``/etc/.canary.example.test/secrets.bin``) when +the attacker greps for hostnames in their loot. + +The instrumenter does not enforce that — the API does, when it sees +``instrumenter=passthrough`` with ``kind=http`` it returns 400. +""" +from __future__ import annotations + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryInstrumenter + + +class PassthroughInstrumenter(CanaryInstrumenter): + name = "passthrough" + mime_prefixes = () # dispatched by fallback in pick_instrumenter_for_mime + + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + return CanaryArtifact( + path=target_path, + content=blob, + mode=0o644, + mtime_offset=-86400 * 7, + instrumenter=self.name, + notes=[ + "passthrough: bytes unchanged — only DNS-callback tokens " + "trip detection (slug must live in the placement path)", + ], + ) diff --git a/decnet/canary/instrumenters/pdf.py b/decnet/canary/instrumenters/pdf.py new file mode 100644 index 00000000..516b6999 --- /dev/null +++ b/decnet/canary/instrumenters/pdf.py @@ -0,0 +1,76 @@ +"""PDF instrumenter — requires :mod:`pikepdf` (optional dependency). + +PDF embedding is non-trivial: the cleanest place to put a callback +is an ``/AA`` (additional actions) ``/O`` (open) entry on the +catalog or a ``/URI`` action on a link annotation. Either path +needs proper xref-table updates — pikepdf handles that for us. + +If pikepdf isn't available in the environment the instrumenter +raises :class:`InstrumenterRejectedError` so the API can return a +clear 400 directing the operator to either install pikepdf or +re-upload as ``passthrough``. + +We don't ship a stdlib fallback because every "naive" PDF mutation +I'm aware of (appending raw bytes, splicing into the trailer, etc.) +breaks the document's xref table and trips a "file is corrupt" +warning in modern viewers — which the attacker will absolutely +notice. +""" +from __future__ import annotations + +from decnet.canary.base import ( + CanaryArtifact, + CanaryContext, + CanaryInstrumenter, + InstrumenterRejectedError, +) + + +class PdfInstrumenter(CanaryInstrumenter): + name = "pdf" + mime_prefixes = ("application/pdf",) + + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + try: + import pikepdf # type: ignore[import-not-found] + except ImportError as e: + raise InstrumenterRejectedError( + "PDF instrumenter requires pikepdf; install it (`pip " + "install pikepdf`) or re-upload the artifact with " + "kind=passthrough so it ships unmodified." + ) from e + + url = f"{ctx.http_base.rstrip('/')}/c/{ctx.callback_token}" + try: + import io + buf = io.BytesIO(blob) + with pikepdf.open(buf) as pdf: + # Add an OpenAction that fires a URI action on document + # open. Most viewers prompt before fetching; that's + # fine — even the prompt itself can trip a "user + # interacted with the document" tell, and an + # auto-allow viewer fetches the URL silently. + action = pikepdf.Dictionary( + Type=pikepdf.Name("/Action"), + S=pikepdf.Name("/URI"), + URI=pikepdf.String(url), + ) + pdf.Root[pikepdf.Name("/OpenAction")] = action + out = io.BytesIO() + pdf.save(out) + mutated = out.getvalue() + except Exception as e: + raise InstrumenterRejectedError( + f"failed to instrument PDF: {e!s}" + ) from e + + return CanaryArtifact( + path=target_path, + content=mutated, + mode=0o644, + mtime_offset=-86400 * 14, + instrumenter=self.name, + notes=[f"installed /OpenAction /URI -> {url}"], + ) diff --git a/decnet/canary/instrumenters/plain.py b/decnet/canary/instrumenters/plain.py new file mode 100644 index 00000000..bfbea7ac --- /dev/null +++ b/decnet/canary/instrumenters/plain.py @@ -0,0 +1,79 @@ +"""Plain-text / config-file instrumenter. + +Two embedding strategies, picked in order: + +1. **Token substitution.** If the blob contains the literal + placeholder ``{{CANARY_URL}}`` or ``{{CANARY_HOST}}``, replace it. + This gives operators full control over where the slug lands — + they can pre-edit the file with placeholders before uploading. +2. **Append.** Otherwise, append a comment line that mentions the + callback URL. The comment style adapts to the file's apparent + syntax (``#`` for shell/yaml/python/dockerfile, ``//`` for json5/ + javascript-ish, ``;`` for ini). + +Operators who want neither behavior should upload the file as +``passthrough``. +""" +from __future__ import annotations + +from decnet.canary.base import CanaryArtifact, CanaryContext, CanaryInstrumenter + + +_SLASH_HINTS = (b"//", b"function ", b"const ", b"let ", b"var ") +_SEMI_HINTS = (b"[default]", b"[section]", b"\n[") + + +def _comment_prefix(blob: bytes) -> bytes: + head = blob[:512] + if any(h in head for h in _SEMI_HINTS): + return b"; " + if any(h in head for h in _SLASH_HINTS): + return b"// " + # Default to # — the most common comment glyph across config files + # we'd plausibly canary. + return b"# " + + +class PlainInstrumenter(CanaryInstrumenter): + name = "plain" + mime_prefixes = ("text/", "application/json", "application/yaml", "application/toml") + + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + base = ctx.http_base.rstrip("/") + callback_url = f"{base}/c/{ctx.callback_token}".encode() + callback_host = ( + f"{ctx.callback_token}.{ctx.dns_zone}".encode() + if ctx.dns_zone else b"" + ) + notes: list[str] = [] + out = blob + + if b"{{CANARY_URL}}" in blob: + out = out.replace(b"{{CANARY_URL}}", callback_url) + notes.append(f"substituted {{{{CANARY_URL}}}} -> {callback_url.decode()}") + if b"{{CANARY_HOST}}" in blob and callback_host: + out = out.replace(b"{{CANARY_HOST}}", callback_host) + notes.append(f"substituted {{{{CANARY_HOST}}}} -> {callback_host.decode()}") + + if not notes: + # No placeholders — append a comment line at the end. + prefix = _comment_prefix(blob) + tail = ( + b"\n" + prefix + b"see " + callback_url + + b" for the latest version\n" + ) + out = (out if out.endswith(b"\n") else out + b"\n") + tail + notes.append( + f"appended comment line carrying {callback_url.decode()}" + ) + + return CanaryArtifact( + path=target_path, + content=out, + mode=0o644, + mtime_offset=-86400 * 7, + instrumenter=self.name, + notes=notes, + ) diff --git a/decnet/canary/instrumenters/xlsx.py b/decnet/canary/instrumenters/xlsx.py new file mode 100644 index 00000000..ed5cfbc2 --- /dev/null +++ b/decnet/canary/instrumenters/xlsx.py @@ -0,0 +1,95 @@ +"""XLSX instrumenter — embed an external-image link. + +XLSX is structurally identical to DOCX (Office Open XML zip). The +injection target is the workbook's relationships file +(``xl/_rels/workbook.xml.rels``). We add an external image +relationship there; Excel/LibreOffice fetch external images on +workbook open in the same way Word does. + +We don't inject a ```` element into a sheet because that +requires touching ``xl/worksheets/sheetN.xml`` *and* allocating a new +``xl/drawings/drawingN.xml`` part — much higher chance of mangling +the file. An orphan external image relationship is enough: many +Office viewers fetch all relationships at open time regardless of +whether they're referenced from a sheet. + +If the operator wants a stronger trigger (image visible in the +sheet, fetched even by viewers that lazy-load external resources) +they should embed the slug as a hyperlink cell content via the +``plain``/``passthrough`` instrumenters. +""" +from __future__ import annotations + +import io +import zipfile +from typing import Tuple + +from decnet.canary.base import ( + CanaryArtifact, + CanaryContext, + CanaryInstrumenter, + InstrumenterRejectedError, +) +from decnet.canary.instrumenters.docx import _inject_relationship, _next_rid + + +_RELS_PATHS = ( + "xl/_rels/workbook.xml.rels", + "xl/_rels/sharedStrings.xml.rels", +) + + +def _mutate(blob: bytes, url: str) -> Tuple[bytes, str, str]: + try: + with zipfile.ZipFile(io.BytesIO(blob), "r") as zf: + members = [(zi, zf.read(zi.filename)) for zi in zf.infolist()] + except zipfile.BadZipFile as e: + raise InstrumenterRejectedError("uploaded blob is not a valid XLSX zip") from e + + target_rels: str | None = None + for zi, _ in members: + if zi.filename in _RELS_PATHS: + target_rels = zi.filename + break + if not target_rels: + raise InstrumenterRejectedError( + "XLSX has no workbook relationships file to mutate" + ) + + out_members = [] + rid = "" + for zi, data in members: + if zi.filename == target_rels: + rid = _next_rid(data) + data = _inject_relationship(data, rid, url) + out_members.append((zi, data)) + + out = io.BytesIO() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zf_out: + for zi, data in out_members: + zf_out.writestr(zi, data) + return out.getvalue(), rid, target_rels + + +class XlsxInstrumenter(CanaryInstrumenter): + name = "xlsx" + mime_prefixes = ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + + def instrument( + self, blob: bytes, ctx: CanaryContext, *, target_path: str, + ) -> CanaryArtifact: + url = f"{ctx.http_base.rstrip('/')}/c/{ctx.callback_token}" + mutated, rid, target_rels = _mutate(blob, url) + return CanaryArtifact( + path=target_path, + content=mutated, + mode=0o644, + mtime_offset=-86400 * 14, + instrumenter=self.name, + notes=[ + f"injected external-image relationship {rid} into " + f"{target_rels} -> {url}", + ], + ) diff --git a/decnet/canary/paths.py b/decnet/canary/paths.py new file mode 100644 index 00000000..5700ad0f --- /dev/null +++ b/decnet/canary/paths.py @@ -0,0 +1,82 @@ +"""Persona-aware path resolution for canary artifacts. + +Linux-persona deckies use POSIX-shaped paths under ``/home/``. +"Windows" personas (still Linux containers under the hood — see +:mod:`decnet.archetypes`) use Windows-shaped paths under +``/home//AppData/...`` so an attacker browsing the filesystem +through a planted RDP/SMB session sees the right shape. + +The persona lookup is best-effort: callers pass the +:attr:`decnet.archetypes.Archetype.nmap_os` value (``"linux"`` or +``"windows"``); unknown personas fall through to ``"linux"``. +Operators can always override by passing an explicit +``placement_path`` when creating a token. +""" +from __future__ import annotations + +DEFAULT_LINUX_USER = "admin" +DEFAULT_WINDOWS_USER = "Administrator" + +# Canonical placements for the synthesizer-driven baseline tokens. +# Operators can override per-token via the API, but these are the +# defaults the deploy-time seed uses. +_LINUX_DEFAULTS: dict[str, str] = { + "git_config": "/home/{user}/.git/config", + "env_file": "/home/{user}/.env", + "ssh_key": "/home/{user}/.ssh/id_rsa", + "aws_creds": "/home/{user}/.aws/credentials", + "honeydoc": "/home/{user}/Documents/quarterly_report.html", + "honeydoc_docx": "/home/{user}/Documents/quarterly_report.docx", + "honeydoc_pdf": "/home/{user}/Documents/quarterly_report.pdf", +} + +_WINDOWS_DEFAULTS: dict[str, str] = { + "git_config": "/home/{user}/AppData/Local/Programs/Git/etc/gitconfig", + "env_file": "/home/{user}/Desktop/prod.env", + "ssh_key": "/home/{user}/.ssh/id_rsa", # OpenSSH on Windows uses the same path + "aws_creds": "/home/{user}/.aws/credentials", + "honeydoc": "/home/{user}/Documents/quarterly_report.html", + "honeydoc_docx": "/home/{user}/Documents/quarterly_report.docx", + "honeydoc_pdf": "/home/{user}/Documents/quarterly_report.pdf", +} + + +def default_user(persona: str) -> str: + """Return the conventional unprivileged username for a persona.""" + return DEFAULT_WINDOWS_USER if persona == "windows" else DEFAULT_LINUX_USER + + +def default_path_for(generator: str, persona: str = "linux") -> str: + """Resolve the default placement path for a synthesized token. + + Returns an absolute container path with ``{user}`` already + expanded. Falls back to a sane Linux default for unknown + personas — better to plant *something* than fail the deploy hook. + """ + table = _WINDOWS_DEFAULTS if persona == "windows" else _LINUX_DEFAULTS + template = table.get(generator) + if not template: + # Unknown generator — fall back to a generic /tmp drop so the + # planter still has somewhere to write. The API rejects + # unknown generators upstream, so this branch is defensive. + return f"/tmp/{generator}.canary" # nosec B108 — placement inside attacker-facing decoy container, not host /tmp + return template.format(user=default_user(persona)) + + +def normalize_placement(path: str) -> str: + """Validate and normalize an operator-supplied placement path. + + Forbids relative paths, NUL bytes, and shell metacharacters that + ``docker exec sh -c`` can't safely round-trip. Returns the + sanitised path unchanged when valid; raises :class:`ValueError` + otherwise so the API can return a 400 with a clear message. + """ + if not path or not path.startswith("/"): + raise ValueError("placement_path must be absolute (start with '/')") + if "\x00" in path: + raise ValueError("placement_path may not contain NUL") + if "\n" in path or "\r" in path: + raise ValueError("placement_path may not contain newlines") + if "../" in path or path.endswith("/.."): + raise ValueError("placement_path may not contain '..' segments") + return path diff --git a/decnet/canary/planter.py b/decnet/canary/planter.py new file mode 100644 index 00000000..6beae78b --- /dev/null +++ b/decnet/canary/planter.py @@ -0,0 +1,301 @@ +"""Plant / revoke canary artifacts inside running decky containers. + +Single entry point per operation: + +* :func:`plant` writes a :class:`CanaryArtifact` into one decky's + filesystem via ``docker exec`` (mirroring the SSH driver's + ``_run_file`` pattern), backdates the mtime, sets the requested + mode, and publishes ``canary.{token_id}.placed`` on the bus. +* :func:`revoke` unlinks the file (best-effort) and publishes + ``canary.{token_id}.revoked``. +* :func:`seed_baseline` is the deploy-hook helper: synthesises the + configured baseline set for one decky, persists rows, plants each. + Failures are logged but do **not** abort the deploy (the deployer + hook calls this best-effort). + +We don't reuse :class:`SSHDriver` directly because the orchestrator +driver is tied to its action types (``FileAction`` carries str +content; canary content is bytes). The planter takes the same +shape but speaks bytes-via-base64 over the wire. +""" +from __future__ import annotations + +import asyncio +import base64 +import os +import shlex +import time +from secrets import token_urlsafe +from typing import Any, Iterable, Optional + +from decnet.bus import topics +from decnet.bus.base import BaseBus +from decnet.bus.factory import get_bus +from decnet.canary.base import CanaryArtifact, CanaryContext +from decnet.canary.factory import get_generator +from decnet.canary.paths import default_path_for +from decnet.logging import get_logger +from decnet.web.db.repository import BaseRepository + +log = get_logger("canary.planter") + +_DOCKER = "docker" +_TIMEOUT = 8.0 +# Container suffix — matches the orchestrator SSH driver's convention +# (``-ssh``). Canary placement always happens through the +# ssh container because every decky has one and it carries the most +# realistic filesystem layout. +_SSH_CONTAINER_SUFFIX = "-ssh" + + +def _container_for(decky_name: str) -> str: + return f"{decky_name}{_SSH_CONTAINER_SUFFIX}" + + +def _dirname(path: str) -> str: + idx = path.rfind("/") + if idx <= 0: + return "/" + return path[:idx] + + +async def _run( + argv: list[str], *, stdin_bytes: Optional[bytes] = None, +) -> tuple[int, str, str]: + try: + proc = await asyncio.create_subprocess_exec( + *argv, + stdin=asyncio.subprocess.PIPE if stdin_bytes is not None else None, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except FileNotFoundError as exc: + return 127, "", f"argv[0] not found: {exc}" + try: + stdout, stderr = await asyncio.wait_for( + proc.communicate(input=stdin_bytes), timeout=_TIMEOUT, + ) + except asyncio.TimeoutError: + try: + proc.kill() + except ProcessLookupError: + pass + return 124, "", "timeout" + return ( + proc.returncode if proc.returncode is not None else -1, + stdout.decode("utf-8", "replace"), + stderr.decode("utf-8", "replace"), + ) + + +def _build_plant_command(artifact: CanaryArtifact) -> tuple[str, bytes]: + """Compose the ``sh -c`` script + stdin payload for one artifact. + + Binary safety: we base64-encode on the host and stream the result + over stdin to ``base64 -d`` inside the container, so the bytes + never touch the argv (kernel ARG_MAX would reject anything larger + than ~128KB-2MB depending on the host). Both ``base64`` (coreutils) + and ``touch -d @`` are present on every Linux base image + we ship, so there's no per-distro branching. + """ + encoded = base64.b64encode(artifact.content) + mtime = int(time.time() + artifact.mtime_offset) + mode_str = oct(artifact.mode)[2:] + parts = [ + f"mkdir -p {shlex.quote(_dirname(artifact.path))}", + f"base64 -d > {shlex.quote(artifact.path)}", + f"chmod {mode_str} {shlex.quote(artifact.path)}", + f"touch -d @{mtime} {shlex.quote(artifact.path)}", + ] + return " && ".join(parts), encoded + + +async def _publish( + bus: Optional[BaseBus], topic: str, payload: dict[str, Any], +) -> None: + """Best-effort publish — never raises. + + When ``bus`` is None we resolve via :func:`get_bus`; either way + bus-side failures are logged and swallowed (delivery is at-most-once + by contract; the DB row is source of truth). + """ + try: + owns_bus = bus is None + target = bus if bus is not None else get_bus() + if owns_bus: + await target.connect() + await target.publish(topic, payload) + if owns_bus: + await target.close() + except Exception as e: # noqa: BLE001 + log.warning("canary bus publish failed topic=%s err=%s", topic, e) + + +async def plant( + decky_name: str, + artifact: CanaryArtifact, + *, + token_uuid: str, + repo: Optional[BaseRepository] = None, + publish: bool = True, + bus: Optional[BaseBus] = None, +) -> tuple[bool, Optional[str]]: + """Write *artifact* into the decky's ssh container. + + Returns ``(success, error_or_none)``. When ``repo`` is provided + the token row's state is updated to ``planted`` / ``failed`` + accordingly. When ``publish`` is True a ``canary..placed`` + event is published on the bus on success. + + The function never raises on docker errors — callers (the API, + the deploy hook) treat the result as data. + """ + if not artifact.path: + err = "planter requires a non-empty artifact.path" + log.warning("canary.plant skipped: %s decky=%s token=%s", err, decky_name, token_uuid) + if repo is not None: + await repo.update_canary_token_state(token_uuid, "failed", err) + return False, err + + sh_cmd, stdin_payload = _build_plant_command(artifact) + # ``-i`` keeps stdin attached so base64 -d inside the container can + # consume the encoded payload streamed from the host. + argv = [_DOCKER, "exec", "-i", _container_for(decky_name), "sh", "-c", sh_cmd] + rc, _stdout, stderr = await _run(argv, stdin_bytes=stdin_payload) + success = rc == 0 + error = None if success else (stderr.strip()[:256] or f"rc={rc}") + + if repo is not None: + if success: + await repo.update_canary_token_state(token_uuid, "planted", None) + else: + await repo.update_canary_token_state(token_uuid, "failed", error) + + if success and publish: + await _publish(bus, topics.canary(token_uuid, topics.CANARY_PLACED), { + "token_id": token_uuid, + "decky_name": decky_name, + "placement_path": artifact.path, + "instrumenter": artifact.instrumenter, + "generator": artifact.generator, + }) + + if not success: + log.warning( + "canary.plant failed decky=%s token=%s rc=%d stderr=%r", + decky_name, token_uuid, rc, stderr[:120], + ) + return success, error + + +async def revoke( + decky_name: str, + placement_path: str, + *, + token_uuid: str, + repo: Optional[BaseRepository] = None, + publish: bool = True, + bus: Optional[BaseBus] = None, +) -> tuple[bool, Optional[str]]: + """Best-effort unlink + state transition + bus publish. + + Returns ``(success, error_or_none)``. ``success`` is True when + the file is gone after the call (whether we deleted it or it was + already missing); only docker / container-down errors return False. + """ + sh_cmd = f"rm -f {shlex.quote(placement_path)}" + argv = [_DOCKER, "exec", _container_for(decky_name), "sh", "-c", sh_cmd] + rc, _stdout, stderr = await _run(argv) + success = rc == 0 + error = None if success else (stderr.strip()[:256] or f"rc={rc}") + + if repo is not None: + await repo.update_canary_token_state(token_uuid, "revoked", error if not success else None) + + if publish: + await _publish(bus, topics.canary(token_uuid, topics.CANARY_REVOKED), { + "token_id": token_uuid, + "decky_name": decky_name, + "placement_path": placement_path, + }) + + return success, error + + +def _baseline_set() -> Iterable[str]: + """Return the configured baseline generator names. + + Honors ``DECNET_CANARY_BASELINE`` (comma-separated). Default is + a sensible mix that exercises every callback-bearing generator + plus a passive aws_creds drop for realism. + """ + raw = os.environ.get( + "DECNET_CANARY_BASELINE", + "git_config,env_file,honeydoc,aws_creds", + ) + return [n.strip() for n in raw.split(",") if n.strip()] + + +def _ctx_for(slug: str) -> CanaryContext: + """Build a :class:`CanaryContext` from the canary worker config.""" + base = os.environ.get("DECNET_CANARY_HTTP_BASE", "http://localhost:8088") + zone = os.environ.get("DECNET_CANARY_DNS_ZONE", "") + return CanaryContext(callback_token=slug, http_base=base, dns_zone=zone) + + +async def seed_baseline( + decky_name: str, + repo: BaseRepository, + *, + persona: str = "linux", + created_by: str = "system", + bus: Optional[BaseBus] = None, +) -> list[dict[str, Any]]: + """Plant the configured baseline canary set on one decky. + + Best-effort: any individual placement that fails is logged and + the row is left in ``state=failed``; the deployer hook treats the + return value as informational, not authoritative. + + Returns the list of token rows created (whether their planting + ultimately succeeded or not), so the caller can surface them in + the deploy report. + """ + out: list[dict[str, Any]] = [] + for gen_name in _baseline_set(): + try: + generator = get_generator(gen_name) + except ValueError: + log.warning("canary.seed_baseline: unknown generator %r — skipping", gen_name) + continue + slug = token_urlsafe(16) + ctx = _ctx_for(slug) + artifact = generator.generate(ctx) + artifact.path = default_path_for(gen_name, persona) + kind = "aws_passive" if gen_name == "aws_creds" else "http" + # Persist first so the planter has a row to update; that way a + # crash mid-plant leaves a recoverable failed-state row. + from uuid import uuid4 + token_uuid = str(uuid4()) + await repo.create_canary_token({ + "uuid": token_uuid, + "kind": kind, + "decky_name": decky_name, + "blob_uuid": None, + "instrumenter": None, + "generator": gen_name, + "placement_path": artifact.path, + "callback_token": slug, + "secret_seed": slug, + "created_by": created_by, + "state": "planted", # optimistic — plant() flips to failed on error + }) + await plant( + decky_name, artifact, + token_uuid=token_uuid, repo=repo, publish=True, bus=bus, + ) + out.append({ + "token_uuid": token_uuid, "generator": gen_name, "kind": kind, + "callback_token": slug, "placement_path": artifact.path, + }) + return out diff --git a/decnet/canary/storage.py b/decnet/canary/storage.py new file mode 100644 index 00000000..06cfbedd --- /dev/null +++ b/decnet/canary/storage.py @@ -0,0 +1,89 @@ +"""Filesystem store for operator-uploaded canary blobs. + +Blobs live under ``/var/lib/decnet/canary/blobs/`` (override +via ``DECNET_CANARY_BLOB_DIR``) and are deduplicated by content hash. +The DB table :class:`decnet.web.db.models.CanaryBlob` mirrors +metadata; the bytes are read on demand at instrumentation time, so +the API process never holds large operator uploads in memory longer +than the request itself. + +Refcount-aware deletion is enforced at the DB layer (see +:meth:`decnet.web.db.repository.BaseRepository.delete_canary_blob`); +this module only provides write/read/unlink primitives keyed by +sha256. +""" +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +from typing import Tuple + + +def blob_dir() -> Path: + """Return the on-disk root for canary blobs. + + Honors ``DECNET_CANARY_BLOB_DIR`` so tests can point at a tmp + path. The directory is created lazily on first write. + """ + raw = os.environ.get("DECNET_CANARY_BLOB_DIR", "/var/lib/decnet/canary/blobs") + return Path(raw) + + +def _path_for(sha256: str) -> Path: + # Two-level fan-out (``ab/cd/abcd...``) keeps any one directory + # from accumulating thousands of entries on busy fleets. Same + # shape as Git's loose-object store. + if len(sha256) < 4: + raise ValueError("sha256 must be at least 4 chars") + root = blob_dir() + return root / sha256[:2] / sha256[2:4] / sha256 + + +def write_blob(content: bytes) -> Tuple[str, Path, int]: + """Persist ``content`` under its sha256 path. + + Idempotent: if the target file already exists with the same + bytes, no rewrite happens. Returns ``(sha256, path, + size_bytes)``. + """ + sha = hashlib.sha256(content).hexdigest() + target = _path_for(sha) + target.parent.mkdir(parents=True, exist_ok=True) + if not target.exists(): + # Atomic-ish: write to a temp sibling and rename. Avoids the + # half-written-file race a concurrent reader would otherwise + # see if we wrote in place. + tmp = target.with_suffix(target.suffix + ".part") + tmp.write_bytes(content) + os.replace(tmp, target) + return sha, target, len(content) + + +def read_blob(sha256: str) -> bytes: + """Read the bytes for a stored blob. + + Raises :class:`FileNotFoundError` when the on-disk row was unlinked + out of band (operator pruned ``/var/lib/decnet`` by hand) — the + caller (instrumenter dispatch) surfaces it as a 410-ish error so + the operator can re-upload. + """ + return _path_for(sha256).read_bytes() + + +def unlink_blob(sha256: str) -> bool: + """Delete the on-disk bytes for ``sha256``. + + Returns True if a file was removed, False if it was already gone. + The DB row deletion happens in + :meth:`SQLModelRepository.delete_canary_blob`; this function is + a best-effort companion called *after* the DB delete commits so + a crash between them leaves a recoverable orphan, never a + dangling DB reference. + """ + target = _path_for(sha256) + try: + target.unlink() + except FileNotFoundError: + return False + return True diff --git a/decnet/canary/worker.py b/decnet/canary/worker.py new file mode 100644 index 00000000..280a717b --- /dev/null +++ b/decnet/canary/worker.py @@ -0,0 +1,254 @@ +"""``decnet canary`` worker — HTTP + DNS callback receivers. + +Two surfaces, one process: + +* **HTTP** — a tiny FastAPI app on its own port (default 8088). The + only useful route is ``GET /c/{slug}`` which looks up the slug in + the canary token table, persists a :class:`CanaryTrigger` row, + publishes ``canary..triggered`` on the bus, and returns + a 1×1 transparent GIF (or 204 if the client's ``Accept`` doesn't + list any image type). +* **DNS** — an authoritative UDP server (default 5353 if non-root, + 53 if root) for ``*.``. Same lookup + persist + + publish flow, plus a sinkhole A record so the attacker's resolver + doesn't loop on NXDOMAIN. + +Both surfaces are **stealth** by policy +(:mod:`feedback_stealth`): no DECNET strings in headers / banners / +error pages. The HTTP app strips the default ``Server: uvicorn`` +header in middleware; FastAPI's docs/openapi UI is disabled because +discovering them would tip off the attacker that this is a honeypot. + +The worker is supervised by its own systemd unit +(``decnet-canary.service``); like every other DECNET worker, it +crashes loudly rather than masking failures. +""" +from __future__ import annotations + +import asyncio +import os +from datetime import datetime, timezone +from typing import Optional + +from fastapi import FastAPI, Request, Response + +from decnet.bus import topics +from decnet.bus.base import BaseBus +from decnet.bus.factory import get_bus +from decnet.canary.dns_server import CanaryDNSProtocol, DNSQuery +from decnet.logging import get_logger +from decnet.web.db.factory import get_repository +from decnet.web.db.repository import BaseRepository + +log = get_logger("canary.worker") + +# 1×1 transparent GIF — public-domain canonical bytes. Returning the +# same image every time is fine: the body has no information the +# attacker shouldn't see, and image clients cache it. +_TRANSPARENT_GIF = bytes.fromhex( + "47494638396101000100800100000000ffffff21f90401000001002c00000000010001000002024401003b" +) + + +def _http_base() -> str: + return os.environ.get("DECNET_CANARY_HTTP_BASE", "http://localhost:8088").rstrip("/") + + +def _dns_zone() -> str: + return os.environ.get("DECNET_CANARY_DNS_ZONE", "").strip(".").lower() + + +def _http_port() -> int: + return int(os.environ.get("DECNET_CANARY_HTTP_PORT", "8088")) + + +def _dns_port() -> int: + # Default 5353 (mDNS-ish, non-privileged) — operators pin :53 via + # NAT or a CAP_NET_BIND_SERVICE-enabled unit. + return int(os.environ.get("DECNET_CANARY_DNS_PORT", "5353")) + + +def _dns_bind() -> str: + return os.environ.get("DECNET_CANARY_DNS_BIND", "0.0.0.0") # nosec B104 — attacker-facing decoy listener, internet exposure is the design + + +def _http_bind() -> str: + return os.environ.get("DECNET_CANARY_HTTP_BIND", "0.0.0.0") # nosec B104 — same rationale + + +# ---------------------------- HTTP surface -------------------------------- + + +def _build_app(repo: BaseRepository, bus: BaseBus) -> FastAPI: + """Construct the FastAPI app. + + Disables docs / openapi / redoc — operators query the canary + surface via the *main* DECNET API, never directly. Anyone hitting + these paths is either misconfigured or scanning for a honeypot. + """ + app = FastAPI( + title="", # don't leak "DECNET" in OpenAPI + docs_url=None, redoc_url=None, openapi_url=None, + ) + + @app.middleware("http") + async def _stealth_headers(request: Request, call_next): + response: Response = await call_next(request) + # Strip the uvicorn / starlette banner; replace with a + # generic Server line that matches what most CDNs return. + response.headers["Server"] = "nginx" + # Don't leak request id / process id headers. + if "x-process-time" in response.headers: + del response.headers["x-process-time"] + return response + + @app.get("/c/{slug}") + async def callback(slug: str, request: Request) -> Response: + await _record_hit( + repo, bus, + slug=slug, + src_ip=_client_ip(request), + user_agent=request.headers.get("user-agent"), + request_path=str(request.url.path), + dns_qname=None, + raw_headers=dict(request.headers), + ) + # Always 200 with a tiny image so the attacker's client sees + # a "success" — same return regardless of whether the slug is + # known. Stealth: do NOT distinguish unknown vs known via + # status code or response body. + return Response(content=_TRANSPARENT_GIF, media_type="image/gif") + + @app.get("/") + async def root() -> Response: + # Bare root returns a generic 404. The decoy posture: pretend + # to be an empty static-file host that just happens to resolve + # /c/ when it matches. + return Response(status_code=404) + + return app + + +def _client_ip(request: Request) -> str: + # Honor X-Forwarded-For if the operator deployed behind a reverse + # proxy. Take the leftmost address in the chain; everything after + # is upstream-proxy noise. + fwd = request.headers.get("x-forwarded-for") + if fwd: + return fwd.split(",", 1)[0].strip() + if request.client: + return request.client.host + return "0.0.0.0" # nosec B104 — sentinel for "unknown remote" + + +# ---------------------------- shared persistence ------------------------- + + +async def _record_hit( + repo: BaseRepository, + bus: BaseBus, + *, + slug: str, + src_ip: str, + user_agent: Optional[str], + request_path: Optional[str], + dns_qname: Optional[str], + raw_headers: Optional[dict], +) -> None: + """Resolve slug -> token, persist a trigger, publish on the bus. + + Unknown slugs are silently swallowed: returning the same response + for known and unknown slugs is the stealth posture, and persisting + every random scan would clutter the DB. + """ + token = await repo.get_canary_token_by_slug(slug) + if token is None: + return + trigger_id = await repo.record_canary_trigger({ + "token_uuid": token["uuid"], + "occurred_at": datetime.now(timezone.utc), + "src_ip": src_ip, + "user_agent": user_agent, + "request_path": request_path, + "dns_qname": dns_qname, + "raw_headers": raw_headers or {}, + }) + try: + await bus.publish( + topics.canary(token["uuid"], topics.CANARY_TRIGGERED), + { + "token_id": token["uuid"], + "trigger_id": trigger_id, + "decky_name": token["decky_name"], + "src_ip": src_ip, + "user_agent": user_agent, + "request_path": request_path, + "dns_qname": dns_qname, + }, + ) + except Exception as e: # noqa: BLE001 — best effort + log.warning("canary.triggered publish failed slug=%s err=%s", slug, e) + + +# ---------------------------- DNS surface -------------------------------- + + +async def _start_dns_server( + repo: BaseRepository, bus: BaseBus, *, loop: asyncio.AbstractEventLoop, +) -> Optional[asyncio.DatagramTransport]: + zone = _dns_zone() + if not zone: + log.info("canary.dns disabled (DECNET_CANARY_DNS_ZONE unset)") + return None + + async def _hook(slug: str, query: DNSQuery, src_ip: str) -> None: + await _record_hit( + repo, bus, + slug=slug, src_ip=src_ip, user_agent=None, + request_path=None, dns_qname=query.qname, + raw_headers=None, + ) + + transport, _proto = await loop.create_datagram_endpoint( + lambda: CanaryDNSProtocol(zone, _hook), + local_addr=(_dns_bind(), _dns_port()), + ) + log.info("canary.dns listening zone=%s port=%d", zone, _dns_port()) + return transport # type: ignore[return-value] + + +# ---------------------------- entry point -------------------------------- + + +async def run() -> None: + """Worker entry point — kicked off by ``decnet canary``.""" + import uvicorn + + repo = get_repository() + await repo.initialize() + bus = get_bus() + await bus.connect() + + app = _build_app(repo, bus) + config = uvicorn.Config( + app, + host=_http_bind(), + port=_http_port(), + log_level="warning", + access_log=False, # stealth: no per-request lines + server_header=False, # we set Server: nginx in middleware + ) + server = uvicorn.Server(config) + loop = asyncio.get_running_loop() + dns_transport = await _start_dns_server(repo, bus, loop=loop) + try: + await server.serve() + finally: + if dns_transport is not None: + dns_transport.close() + await bus.close() + + +def main() -> None: + """CLI entry point — synchronous wrapper for ``asyncio.run``.""" + asyncio.run(run()) diff --git a/decnet/cli/__init__.py b/decnet/cli/__init__.py index e2976c8f..f0508319 100644 --- a/decnet/cli/__init__.py +++ b/decnet/cli/__init__.py @@ -21,18 +21,27 @@ import typer from . import ( agent, api, + bus, + canary, db, deploy, forwarder, + geoip, + init, inventory, lifecycle, listener, + orchestrator, profiler, + realism, + reconciler, sniffer, swarm, swarmctl, + topology, updater, web, + webhook, workers, ) from .gating import _gate_commands_by_mode @@ -49,7 +58,8 @@ for _mod in ( api, swarmctl, agent, updater, listener, forwarder, swarm, deploy, lifecycle, workers, inventory, - web, profiler, sniffer, db, + web, profiler, orchestrator, realism, reconciler, sniffer, db, + topology, bus, geoip, init, webhook, canary, ): _mod.register(app) diff --git a/decnet/cli/agent.py b/decnet/cli/agent.py index ae89a464..5a04d5a6 100644 --- a/decnet/cli/agent.py +++ b/decnet/cli/agent.py @@ -29,7 +29,7 @@ def register(app: typer.Typer) -> None: with `decnet forwarder --daemon …`. Pass --no-forwarder to skip. """ from decnet.agent import server as _agent_server - from decnet.env import DECNET_SWARM_MASTER_HOST, DECNET_INGEST_LOG_FILE + from decnet.env import DECNET_SWARM_MASTER_HOST, DECNET_AGENT_LOG_FILE from decnet.swarm import pki as _pki resolved_dir = _pathlib.Path(agent_dir) if agent_dir else _pki.DEFAULT_AGENT_DIR @@ -44,7 +44,7 @@ def register(app: typer.Typer) -> None: "--master-host", DECNET_SWARM_MASTER_HOST, "--master-port", str(int(os.environ.get("DECNET_SWARM_SYSLOG_PORT", "6514"))), "--agent-dir", str(resolved_dir), - "--log-file", str(DECNET_INGEST_LOG_FILE), + "--log-file", str(DECNET_AGENT_LOG_FILE), "--daemon", ] try: diff --git a/decnet/cli/bus.py b/decnet/cli/bus.py new file mode 100644 index 00000000..5a29dd91 --- /dev/null +++ b/decnet/cli/bus.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import typer + +from . import utils as _utils +from .utils import console, log + + +def register(app: typer.Typer) -> None: + @app.command(name="bus") + def bus_cmd( + socket_path: str = typer.Option( + None, "--socket", "-s", + help="UNIX socket path (defaults to DECNET_BUS_SOCKET env var, " + "then /run/decnet/bus.sock, then ~/.decnet/bus.sock).", + ), + group: str = typer.Option( + "decnet", "--group", "-g", + help="POSIX group to chown the socket to (falls back to process " + "group if the named group does not exist).", + ), + heartbeat: int = typer.Option( + 10, "--heartbeat", "-H", + help="Seconds between system.bus.health heartbeat events.", + ), + daemon: bool = typer.Option(False, "--daemon", "-d", help="Detach to background as a daemon process."), + ) -> None: + """Run the DECNET ServiceBus worker (host-local UNIX-socket pub/sub).""" + import asyncio + from decnet.bus.factory import _default_socket_path + from decnet.bus.worker import bus_worker + + resolved = socket_path or _default_socket_path() + + if daemon: + log.info("bus daemonizing socket=%s", resolved) + _utils._daemonize() + + log.info("bus starting socket=%s group=%s heartbeat=%ds", resolved, group, heartbeat) + console.print(f"[bold cyan]Bus starting[/] (socket: {resolved}, heartbeat: {heartbeat}s)") + + try: + asyncio.run(bus_worker(resolved, group=group, heartbeat_interval=heartbeat)) + except KeyboardInterrupt: + console.print("\n[yellow]Bus stopped.[/]") diff --git a/decnet/cli/canary.py b/decnet/cli/canary.py new file mode 100644 index 00000000..87af60ea --- /dev/null +++ b/decnet/cli/canary.py @@ -0,0 +1,42 @@ +"""``decnet canary`` — HTTP + DNS callback receiver for canary tokens. + +Worker process. Mirrors the shape of :mod:`decnet.cli.webhook`: a +``@app.command(name="canary")`` Typer entry point that delegates to +:func:`decnet.canary.worker.run`. + +Not master-only — any host that hosts deckies can run its own +canary worker (the bus events stay local; the webhook worker on +each host fans them out to SIEMs independently per the design +in ``development/let-s-move-to-the-enumerated-pike.md``). +""" +from __future__ import annotations + +import typer + +from . import utils as _utils +from .utils import console, log + + +def register(app: typer.Typer) -> None: + @app.command(name="canary") + def canary_cmd( + daemon: bool = typer.Option( + False, "--daemon", "-d", help="Detach to background as a daemon process", + ), + ) -> None: + """Run the canary HTTP + DNS callback receiver.""" + import asyncio + + from decnet.canary.worker import run + + if daemon: + log.info("canary daemonizing") + _utils._daemonize() + + log.info("canary starting") + console.print("[bold cyan]Canary callback receiver starting[/]") + + try: + asyncio.run(run()) + except KeyboardInterrupt: + console.print("\n[yellow]Canary worker stopped.[/]") diff --git a/decnet/cli/db.py b/decnet/cli/db.py index 73c9430c..86193967 100644 --- a/decnet/cli/db.py +++ b/decnet/cli/db.py @@ -8,19 +8,29 @@ from rich.table import Table from .utils import console, log -_DB_RESET_TABLES: tuple[str, ...] = ( - # Order matters for DROP TABLE: child FKs first. - # - attacker_behavior FK-references attackers. - # - decky_shards FK-references swarm_hosts. - "attacker_behavior", - "attackers", - "logs", - "bounty", - "state", - "users", - "decky_shards", - "swarm_hosts", -) +def _decnet_tables() -> tuple[str, ...]: + """Every DECNET-managed table, ordered child-first for DROP safety. + + Source is ``SQLModel.metadata.sorted_tables`` — the same registry that + drives ``create_all`` — so adding a new model automatically enrolls + its table in ``db-reset`` with no manual step. (Previous hardcoded + list drifted multiple times; ``webhook_subscriptions`` / + ``session_profile`` / ``smtp_targets`` all got missed.) + + ``sorted_tables`` returns parent-first (topological order that makes + ``CREATE`` safe). For ``DROP`` we need the reverse: children first, + so FK constraints drop before their parents. ``SET FOREIGN_KEY_CHECKS + = 0`` below makes this order-insensitive for MySQL, but the reverse + order keeps the code honest for any backend that doesn't support + disabling the FK check. + """ + from sqlmodel import SQLModel + # Importing the models package registers every table on SQLModel.metadata. + import decnet.web.db.models # noqa: F401 + + return tuple( + t.name for t in reversed(SQLModel.metadata.sorted_tables) + ) async def _db_reset_mysql_async(dsn: str, mode: str, confirm: bool) -> None: @@ -32,10 +42,11 @@ async def _db_reset_mysql_async(dsn: str, mode: str, confirm: bool) -> None: db_name = urlparse(dsn).path.lstrip("/") or "(default)" engine = create_async_engine(dsn) + tables = _decnet_tables() try: rows: dict[str, int] = {} async with engine.connect() as conn: - for tbl in _DB_RESET_TABLES: + for tbl in tables: try: result = await conn.execute(text(f"SELECT COUNT(*) FROM `{tbl}`")) # nosec B608 rows[tbl] = result.scalar() or 0 @@ -58,7 +69,7 @@ async def _db_reset_mysql_async(dsn: str, mode: str, confirm: bool) -> None: async with engine.begin() as conn: await conn.execute(text("SET FOREIGN_KEY_CHECKS = 0")) - for tbl in _DB_RESET_TABLES: + for tbl in tables: if rows.get(tbl, -1) < 0: continue if mode == "truncate": diff --git a/decnet/cli/gating.py b/decnet/cli/gating.py index 5c153522..a373bc1c 100644 --- a/decnet/cli/gating.py +++ b/decnet/cli/gating.py @@ -29,9 +29,11 @@ MASTER_ONLY_COMMANDS: frozenset[str] = frozenset({ "api", "swarmctl", "deploy", "redeploy", "teardown", "mutate", "listener", "profiler", "services", "distros", "correlate", "archetypes", "web", - "db-reset", + "db-reset", "init", "webhook", "clusterer", "campaign-clusterer", }) -MASTER_ONLY_GROUPS: frozenset[str] = frozenset({"swarm"}) +MASTER_ONLY_GROUPS: frozenset[str] = frozenset( + {"swarm", "topology", "geoip", "realism"} +) def _agent_mode_active() -> bool: diff --git a/decnet/cli/geoip.py b/decnet/cli/geoip.py new file mode 100644 index 00000000..7ff90a3e --- /dev/null +++ b/decnet/cli/geoip.py @@ -0,0 +1,59 @@ +"""GeoIP CLI — refresh and lookup subcommands (master-only). + +Usage:: + + decnet geoip refresh # re-download RIR files and rebuild the index + decnet geoip lookup 8.8.8.8 # one-shot IP -> country dump +""" +from __future__ import annotations + +import typer + +from .gating import _require_master_mode +from .utils import console, log + +_group = typer.Typer( + name="geoip", + help="GeoIP provider management (master only).", + no_args_is_help=True, +) + + +@_group.command("refresh") +def _refresh() -> None: + """Force re-download of the GeoIP provider data and rebuild the index.""" + _require_master_mode("geoip refresh") + from decnet.geoip import get_lookup + from decnet.geoip.factory import get_provider + + provider = get_provider() + log.info("geoip: forcing refresh via %s provider", provider.name) + console.print(f"[bold cyan]Refreshing {provider.name} GeoIP data…[/]") + try: + lookup = get_lookup(force_refresh=True) + except Exception as exc: # noqa: BLE001 + console.print(f"[red]refresh failed: {exc}[/]") + raise typer.Exit(1) from exc + console.print( + f"[green]OK[/] {provider.name} index rebuilt " + f"({len(lookup)} ranges)." + ) + + +@_group.command("lookup") +def _lookup( + ip: str = typer.Argument(..., help="IP address to resolve."), +) -> None: + """Print the country code for an IP (or 'unknown').""" + _require_master_mode("geoip lookup") + from decnet.geoip import enrich_ip + + cc, source = enrich_ip(ip) + if cc is None: + console.print(f"{ip} [yellow]unknown[/]") + raise typer.Exit(0) + console.print(f"{ip} [green]cc={cc}[/] source={source}") + + +def register(app: typer.Typer) -> None: + app.add_typer(_group, name="geoip") diff --git a/decnet/cli/init.py b/decnet/cli/init.py new file mode 100644 index 00000000..bb52cb65 --- /dev/null +++ b/decnet/cli/init.py @@ -0,0 +1,843 @@ +""" +`decnet init` — one-shot master-host bootstrap. + +Idempotent: running it twice is a no-op on already-configured items. +Takes a freshly ``pip install``'d DECNET and turns it into a ready-to- +run master host: creates the ``decnet`` system user/group, installs +the systemd units + polkit rule + tmpfiles.d entry, seeds the +directory layout, drops a placeholder config, and starts the +``decnet.target`` grouping unit. + +Requires root. Uses ``subprocess.run`` (never ``shell=True``) for every +privileged call so the full argv surface is auditable. +""" +from __future__ import annotations + +import grp +import hashlib +import os +import pwd +import shutil +import subprocess # nosec B404 +import sys +from pathlib import Path +from typing import Callable, List, Optional + +import typer +from jinja2 import Environment, FileSystemLoader, StrictUndefined + +import decnet as _decnet_pkg +from .gating import _require_master_mode +from .utils import console, log + + +_CONFIG_PLACEHOLDER = """\ +# /etc/decnet/decnet.ini — DECNET host config. +# +# Every key is OPTIONAL. Absent keys fall through to env-var defaults +# defined in decnet/env.py. Real env vars always win over this file +# (precedence: env > INI > default), so systemd EnvironmentFile= and +# one-off `DECNET_FOO=bar decnet ...` invocations always take effect. +# +# Secrets (JWT, admin password, DB password) intentionally DO NOT +# live here. Put them in /opt/decnet/.env.local or the systemd +# EnvironmentFile= — never in a group-readable INI. + +[decnet] +# mode = master # or "agent" + +# [api] +# host = 127.0.0.1 +# port = 8000 + +# [web] +# host = 127.0.0.1 +# port = 8080 +# admin-user = admin +# cors-origins = http://localhost:8080 # comma-separated + +# [database] +# type = sqlite # or "mysql" +# url = mysql+asyncmy://user@host:3306/decnet # if set, wins over host/port/name/user +# host = localhost +# port = 3306 +# name = decnet +# user = decnet + +# [bus] +# enabled = true +# type = unix # or "fake" +# socket = /run/decnet/bus.sock +# group = decnet + +# [swarm] +# master-host = 10.0.0.1 +# syslog-port = 6514 +# swarmctl-port = 8770 + +# [logging] +# system-log = /var/log/decnet/decnet.system.log +# ingest-log = /var/log/decnet/decnet.log +# agent-log = /var/log/decnet/agent.log + +# [ingester] +# batch-size = 100 +# batch-max-wait-ms = 250 + +# [tracing] +# enabled = false +# otel-endpoint = http://localhost:4317 + +# [agent] +# Managed by the enroll bundle — do NOT edit by hand on an agent host. +""" + + +def _deploy_root() -> Path: + """Resolve the on-disk ``deploy/`` directory of the installed package. + + Editable install (``pip install -e .``): sibling of the ``decnet`` + package at repo root. Wheel installs aren't supported yet — the + error message tells the operator to use an editable install. + """ + root = Path(_decnet_pkg.__file__).resolve().parent.parent / "deploy" + if not (root / "decnet.target").is_file(): + raise RuntimeError( + f"cannot locate deploy/ directory (looked at {root}); " + "are you on a wheel install that didn't bundle deploy/? " + "use `pip install -e .` from a git checkout" + ) + return root + + +def _sha256(path: Path) -> str: + h = hashlib.sha256() + h.update(path.read_bytes()) + return h.hexdigest() + + +def _run(argv: List[str], *, dry_run: bool) -> None: + if dry_run: + console.print(f" [dim]would run:[/] {' '.join(argv)}") + return + log.info("init: exec %s", argv) + subprocess.run(argv, check=True) # nosec B603 + + +def _step(label: str, action: Callable[[], str]) -> bool: + """Run ``action``, print a checklist line. + + The callable returns the human-readable outcome verb: + ``"ok"`` → ``[ OK ]