diff --git a/.gitignore b/.gitignore index 23784f91..bc49506f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,8 @@ decnet-topology-*-compose.yml .docker/ decnet-state.json *.ini +# tracked: Alembic CLI config (migrations live in decnet/web/db/migrations) +!alembic.ini decnet.log* *.loggy *.nmap diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..4535bbe9 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,147 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/decnet/web/db/migrations + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s +# Or organize into date-based subdirectories (requires recursive_version_locations = true) +# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the tzdata library which can be installed by adding +# `alembic[tz]` to the pip requirements. +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# NOTE: no sqlalchemy.url here on purpose. env.py selects the engine from +# DECNET_DB_TYPE (sqlite|mysql), mirroring decnet/web/db/factory.py. + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module +# hooks = ruff +# ruff.type = module +# ruff.module = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Alternatively, use the exec runner to execute a binary found on your PATH +# hooks = ruff +# ruff.type = exec +# ruff.executable = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/decnet/web/db/migrations/README b/decnet/web/db/migrations/README new file mode 100644 index 00000000..98e4f9c4 --- /dev/null +++ b/decnet/web/db/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/decnet/web/db/migrations/env.py b/decnet/web/db/migrations/env.py new file mode 100644 index 00000000..4c8fab02 --- /dev/null +++ b/decnet/web/db/migrations/env.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Alembic environment — async, dual-backend (sqlite | mysql). + +Two entry shapes: + +* **Programmatic** (app boot): :func:`decnet.web.db.migrate.run_migrations` + passes the app's own sync ``Connection`` via ``config.attributes`` so the + upgrade rides the existing engine — no second connection, no extra driver. +* **Standalone** (``alembic`` CLI: autogenerate, upgrade, history): builds its + own async engine from ``DECNET_DB_TYPE``, mirroring ``db/factory.py``. +""" +from __future__ import annotations + +import asyncio +import os +from logging.config import fileConfig + +from alembic import context +from sqlalchemy.engine import Connection +from sqlmodel import SQLModel + +# Importing the models package registers every table on SQLModel.metadata, +# which is what autogenerate diffs against. +import decnet.web.db.models # noqa: F401 + +config = context.config + +# Standalone CLI runs configure logging from alembic.ini; the programmatic +# path builds a Config with no file, so guard on it. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = SQLModel.metadata + + +def _build_async_engine(): + """Standalone-only: pick an async engine the way db/factory.py does.""" + db_type = os.environ.get("DECNET_DB_TYPE", "sqlite").lower() + if db_type == "sqlite": + from decnet.config import _ROOT + from decnet.web.db.sqlite.database import get_async_engine as sqlite_engine + db_path = os.environ.get("DECNET_DB_PATH", str(_ROOT / "decnet.db")) + return sqlite_engine(db_path) + if db_type == "mysql": + from decnet.web.db.mysql.database import get_async_engine as mysql_engine + return mysql_engine() + raise ValueError(f"Unsupported database type: {db_type}") + + +def _configure_and_run(connection: Connection) -> None: + context.configure( + connection=connection, + target_metadata=target_metadata, + # SQLite can't ALTER in place; batch mode rewrites the table so future + # migrations (drop/alter column) work on both backends. + render_as_batch=connection.dialect.name == "sqlite", + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +async def _run_standalone() -> None: + engine = _build_async_engine() + async with engine.connect() as connection: + await connection.run_sync(_configure_and_run) + await engine.dispose() + + +def run_migrations_online() -> None: + connection = config.attributes.get("connection", None) + if connection is not None: + # Programmatic: app handed us a live sync Connection (via run_sync). + _configure_and_run(connection) + else: + asyncio.run(_run_standalone()) + + +if context.is_offline_mode(): + # Offline (--sql) mode: emit DDL without a DB. Cheap to support and keeps + # `alembic upgrade head --sql` working for operators who want to review SQL. + context.configure( + url=os.environ.get("DECNET_DB_URL"), + target_metadata=target_metadata, + literal_binds=True, + ) + with context.begin_transaction(): + context.run_migrations() +else: + run_migrations_online() diff --git a/decnet/web/db/migrations/script.py.mako b/decnet/web/db/migrations/script.py.mako new file mode 100644 index 00000000..263971dd --- /dev/null +++ b/decnet/web/db/migrations/script.py.mako @@ -0,0 +1,29 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel # SQLModel column types (AutoString, …) referenced by autogenerate +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/decnet/web/db/migrations/versions/4a914b1d62a0_baseline_schema.py b/decnet/web/db/migrations/versions/4a914b1d62a0_baseline_schema.py new file mode 100644 index 00000000..d873f5a7 --- /dev/null +++ b/decnet/web/db/migrations/versions/4a914b1d62a0_baseline_schema.py @@ -0,0 +1,1109 @@ +"""baseline schema + +Revision ID: 4a914b1d62a0 +Revises: +Create Date: 2026-06-16 16:24:28.972499 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel # SQLModel column types (AutoString, …) referenced by autogenerate +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision: str = '4a914b1d62a0' +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('bounty', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('timestamp', sa.DateTime(), nullable=False), + sa.Column('decky', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('service', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('attacker_ip', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('bounty_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('payload', sa.Text(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('bounty', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_bounty_attacker_ip'), ['attacker_ip'], unique=False) + batch_op.create_index(batch_op.f('ix_bounty_bounty_type'), ['bounty_type'], unique=False) + batch_op.create_index(batch_op.f('ix_bounty_decky'), ['decky'], unique=False) + batch_op.create_index(batch_op.f('ix_bounty_service'), ['service'], unique=False) + batch_op.create_index(batch_op.f('ix_bounty_timestamp'), ['timestamp'], unique=False) + + op.create_table('campaigns', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('schema_version', sa.Integer(), nullable=False), + sa.Column('first_seen_at', sa.DateTime(), nullable=True), + sa.Column('last_seen_at', sa.DateTime(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.Column('confidence', sa.Float(), nullable=True), + sa.Column('identity_count', sa.Integer(), nullable=False), + sa.Column('ja3_hashes', sa.Text(), nullable=True), + sa.Column('hassh_hashes', sa.Text(), nullable=True), + sa.Column('tls_cert_sha256', sa.Text(), nullable=True), + sa.Column('payload_simhashes', sa.Text(), nullable=True), + sa.Column('c2_endpoints', sa.Text(), nullable=True), + sa.Column('merged_into_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('notes', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['merged_into_uuid'], ['campaigns.uuid'], ), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('campaigns', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_campaigns_created_at'), ['created_at'], unique=False) + batch_op.create_index(batch_op.f('ix_campaigns_first_seen_at'), ['first_seen_at'], unique=False) + batch_op.create_index(batch_op.f('ix_campaigns_last_seen_at'), ['last_seen_at'], unique=False) + batch_op.create_index(batch_op.f('ix_campaigns_merged_into_uuid'), ['merged_into_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_campaigns_updated_at'), ['updated_at'], unique=False) + + op.create_table('canary_blobs', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('sha256', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('filename', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('content_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('size_bytes', sa.Integer(), nullable=False), + sa.Column('uploaded_by', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('uploaded_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('canary_blobs', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_canary_blobs_sha256'), ['sha256'], unique=True) + batch_op.create_index(batch_op.f('ix_canary_blobs_uploaded_by'), ['uploaded_by'], unique=False) + + op.create_table('credential_reuse', + sa.Column('id', sqlmodel.sql.sqltypes.AutoString(length=36), nullable=False), + sa.Column('secret_sha256', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False), + sa.Column('secret_kind', sqlmodel.sql.sqltypes.AutoString(length=32), nullable=False), + sa.Column('principal', sqlmodel.sql.sqltypes.AutoString(length=256), nullable=True), + sa.Column('principal_key', sqlmodel.sql.sqltypes.AutoString(length=256), nullable=False), + sa.Column('attacker_uuids', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('attacker_ips', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('deckies', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('services', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('target_count', sa.Integer(), nullable=False), + sa.Column('attempt_count', sa.Integer(), nullable=False), + sa.Column('confidence', sa.Float(), nullable=False), + sa.Column('first_seen', sa.DateTime(), nullable=False), + sa.Column('last_seen', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('secret_sha256', 'secret_kind', 'principal_key', name='uq_credential_reuse_secret_principal') + ) + with op.batch_alter_table('credential_reuse', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_credential_reuse_first_seen'), ['first_seen'], unique=False) + batch_op.create_index(batch_op.f('ix_credential_reuse_last_seen'), ['last_seen'], unique=False) + batch_op.create_index(batch_op.f('ix_credential_reuse_secret_kind'), ['secret_kind'], unique=False) + batch_op.create_index(batch_op.f('ix_credential_reuse_secret_sha256'), ['secret_sha256'], unique=False) + batch_op.create_index(batch_op.f('ix_credential_reuse_target_count'), ['target_count'], unique=False) + batch_op.create_index(batch_op.f('ix_credential_reuse_updated_at'), ['updated_at'], unique=False) + + op.create_table('decky_lifecycle', + sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('decky_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('host_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('operation', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('status', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('error', sa.Text(), nullable=True), + sa.Column('started_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.Column('completed_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('decky_lifecycle', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_decky_lifecycle_decky_name'), ['decky_name'], unique=False) + batch_op.create_index(batch_op.f('ix_decky_lifecycle_host_uuid'), ['host_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_decky_lifecycle_operation'), ['operation'], unique=False) + batch_op.create_index(batch_op.f('ix_decky_lifecycle_status'), ['status'], unique=False) + + op.create_table('fleet_deckies', + sa.Column('host_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('services', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('decky_config', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=True), + sa.Column('decky_ip', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('state', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('last_error', sa.Text(), nullable=True), + sa.Column('compose_hash', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('last_seen', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('host_uuid', 'name') + ) + with op.batch_alter_table('fleet_deckies', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_fleet_deckies_host_uuid'), ['host_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_fleet_deckies_state'), ['state'], unique=False) + + op.create_table('logs', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('timestamp', sa.DateTime(), nullable=False), + sa.Column('decky', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('service', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('event_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('attacker_ip', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('raw_line', sa.Text(), nullable=False), + sa.Column('fields', sa.Text(), nullable=False), + sa.Column('msg', sa.Text(), nullable=True), + sa.Column('trace_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('span_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('logs', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_logs_attacker_ip'), ['attacker_ip'], unique=False) + batch_op.create_index(batch_op.f('ix_logs_decky'), ['decky'], unique=False) + batch_op.create_index(batch_op.f('ix_logs_event_type'), ['event_type'], unique=False) + batch_op.create_index(batch_op.f('ix_logs_service'), ['service'], unique=False) + batch_op.create_index(batch_op.f('ix_logs_timestamp'), ['timestamp'], unique=False) + + op.create_table('observed_attachments', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('sha256', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False), + sa.Column('first_seen', sa.DateTime(), nullable=False), + sa.Column('last_seen', sa.DateTime(), nullable=False), + sa.Column('observation_count', sa.Integer(), nullable=False), + sa.Column('first_seen_decky_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('first_seen_attacker_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('last_seen_attacker_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('extensions', sa.JSON(), nullable=False), + sa.Column('first_subject', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('mal_hash_match', sa.Boolean(), nullable=True), + sa.Column('mal_hash_match_provider', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=True), + sa.Column('mal_hash_match_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('observed_attachments', schema=None) as batch_op: + batch_op.create_index('ix_observed_attachments_first_seen', ['first_seen'], unique=False) + batch_op.create_index(batch_op.f('ix_observed_attachments_first_seen_attacker_uuid'), ['first_seen_attacker_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_observed_attachments_first_seen_decky_uuid'), ['first_seen_decky_uuid'], unique=False) + batch_op.create_index('ix_observed_attachments_last_seen', ['last_seen'], unique=False) + batch_op.create_index(batch_op.f('ix_observed_attachments_last_seen_attacker_uuid'), ['last_seen_attacker_uuid'], unique=False) + batch_op.create_index('ix_observed_attachments_mal_hash_match', ['mal_hash_match'], unique=False) + batch_op.create_index(batch_op.f('ix_observed_attachments_sha256'), ['sha256'], unique=True) + + op.create_table('orchestrator_emails', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('ts', sa.DateTime(), nullable=False), + sa.Column('mail_decky_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('thread_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('message_id', sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False), + sa.Column('in_reply_to', sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True), + sa.Column('sender_email', sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False), + sa.Column('recipient_email', sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False), + sa.Column('subject', sqlmodel.sql.sqltypes.AutoString(length=512), nullable=False), + sa.Column('language', sqlmodel.sql.sqltypes.AutoString(length=8), nullable=False), + sa.Column('eml_path', sqlmodel.sql.sqltypes.AutoString(length=1024), nullable=False), + sa.Column('success', sa.Boolean(), nullable=False), + sa.Column('payload', sa.Text(), nullable=False), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('orchestrator_emails', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_orchestrator_emails_mail_decky_uuid'), ['mail_decky_uuid'], unique=False) + batch_op.create_index('ix_orchestrator_emails_mail_ts', ['mail_decky_uuid', 'ts'], unique=False) + batch_op.create_index(batch_op.f('ix_orchestrator_emails_recipient_email'), ['recipient_email'], unique=False) + batch_op.create_index(batch_op.f('ix_orchestrator_emails_sender_email'), ['sender_email'], unique=False) + batch_op.create_index(batch_op.f('ix_orchestrator_emails_success'), ['success'], unique=False) + batch_op.create_index('ix_orchestrator_emails_thread', ['thread_id'], unique=False) + batch_op.create_index(batch_op.f('ix_orchestrator_emails_thread_id'), ['thread_id'], unique=False) + batch_op.create_index(batch_op.f('ix_orchestrator_emails_ts'), ['ts'], unique=False) + + op.create_table('orchestrator_events', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('ts', sa.DateTime(), nullable=False), + sa.Column('kind', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False), + sa.Column('protocol', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False), + sa.Column('action', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False), + sa.Column('src_decky_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('dst_decky_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('success', sa.Boolean(), nullable=False), + sa.Column('payload', sa.Text(), nullable=False), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('orchestrator_events', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_orchestrator_events_dst_decky_uuid'), ['dst_decky_uuid'], unique=False) + batch_op.create_index('ix_orchestrator_events_dst_ts', ['dst_decky_uuid', 'ts'], unique=False) + batch_op.create_index(batch_op.f('ix_orchestrator_events_kind'), ['kind'], unique=False) + batch_op.create_index(batch_op.f('ix_orchestrator_events_protocol'), ['protocol'], unique=False) + batch_op.create_index(batch_op.f('ix_orchestrator_events_src_decky_uuid'), ['src_decky_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_orchestrator_events_success'), ['success'], unique=False) + batch_op.create_index(batch_op.f('ix_orchestrator_events_ts'), ['ts'], unique=False) + + op.create_table('realism_config', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('key', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False), + sa.Column('value', sa.Text(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('realism_config', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_realism_config_key'), ['key'], unique=True) + + op.create_table('revoked_tokens', + sa.Column('jti', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('user_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('expires_at', sa.DateTime(), nullable=False), + sa.Column('revoked_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('jti') + ) + with op.batch_alter_table('revoked_tokens', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_revoked_tokens_expires_at'), ['expires_at'], unique=False) + batch_op.create_index(batch_op.f('ix_revoked_tokens_user_uuid'), ['user_uuid'], unique=False) + + op.create_table('state', + sa.Column('key', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('value', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.PrimaryKeyConstraint('key') + ) + op.create_table('swarm_hosts', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('address', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('agent_port', sa.Integer(), nullable=False), + sa.Column('status', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('last_heartbeat', sa.DateTime(), nullable=True), + sa.Column('client_cert_fingerprint', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('updater_cert_fingerprint', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('cert_bundle_path', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('enrolled_at', sa.DateTime(), nullable=False), + sa.Column('notes', sa.Text(), nullable=True), + sa.Column('use_ipvlan', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('swarm_hosts', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_swarm_hosts_name'), ['name'], unique=True) + batch_op.create_index(batch_op.f('ix_swarm_hosts_status'), ['status'], unique=False) + + op.create_table('synthetic_files', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('decky_uuid', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False), + sa.Column('path', sqlmodel.sql.sqltypes.AutoString(length=512), nullable=False), + sa.Column('persona', sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False), + sa.Column('content_class', sqlmodel.sql.sqltypes.AutoString(length=32), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('last_modified', sa.DateTime(), nullable=False), + sa.Column('edit_count', sa.Integer(), nullable=False), + sa.Column('content_hash', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False), + sa.Column('last_body', sa.Text(), nullable=False), + sa.PrimaryKeyConstraint('uuid'), + sa.UniqueConstraint('decky_uuid', 'path', name='uq_synthetic_files_decky_path') + ) + with op.batch_alter_table('synthetic_files', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_synthetic_files_content_class'), ['content_class'], unique=False) + batch_op.create_index(batch_op.f('ix_synthetic_files_created_at'), ['created_at'], unique=False) + batch_op.create_index('ix_synthetic_files_decky_modified', ['decky_uuid', 'last_modified'], unique=False) + batch_op.create_index(batch_op.f('ix_synthetic_files_decky_uuid'), ['decky_uuid'], unique=False) + + op.create_table('tarpit_rules', + sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('decky_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('ports', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('delay_ms', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('created_by', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('tarpit_rules', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_tarpit_rules_decky_name'), ['decky_name'], unique=True) + + op.create_table('ttp_rule', + sa.Column('rule_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('rule_version', sa.Integer(), nullable=False), + sa.Column('source_path', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('yaml_content', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.Column('updated_by', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('rule_id') + ) + op.create_table('ttp_rule_state', + sa.Column('rule_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('state', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('confidence_max', sa.Float(), nullable=True), + sa.Column('expires_at', sa.DateTime(), nullable=True), + sa.Column('reason', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('set_by', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('set_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('rule_id') + ) + op.create_table('users', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('username', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('password_hash', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('role', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('must_change_password', sa.Boolean(), nullable=False), + sa.Column('tokens_valid_from', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_users_username'), ['username'], unique=True) + + op.create_table('webhook_subscriptions', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('url', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('secret', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('topic_patterns', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('enabled', sa.Boolean(), nullable=False), + sa.Column('consecutive_failures', sa.Integer(), nullable=False), + sa.Column('last_success_at', sa.DateTime(), nullable=True), + sa.Column('last_failure_at', sa.DateTime(), nullable=True), + sa.Column('last_error', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('auto_disabled_at', sa.DateTime(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('webhook_subscriptions', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_webhook_subscriptions_enabled'), ['enabled'], unique=False) + batch_op.create_index(batch_op.f('ix_webhook_subscriptions_name'), ['name'], unique=True) + + op.create_table('attacker_identities', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('schema_version', sa.Integer(), nullable=False), + sa.Column('campaign_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('first_seen_at', sa.DateTime(), nullable=True), + sa.Column('last_seen_at', sa.DateTime(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.Column('confidence', sa.Float(), nullable=True), + sa.Column('observation_count', sa.Integer(), nullable=False), + sa.Column('ja3_hashes', sa.Text(), nullable=True), + sa.Column('hassh_hashes', sa.Text(), nullable=True), + sa.Column('ja4h_hashes', sa.Text(), nullable=True), + sa.Column('ja4_quic_hashes', sa.Text(), nullable=True), + sa.Column('http_versions_seen', sa.Text(), nullable=True), + sa.Column('tls_cert_sha256', sa.Text(), nullable=True), + sa.Column('ipv6_link_local_iids', sa.Text(), nullable=True), + sa.Column('payload_simhashes', sa.Text(), nullable=True), + sa.Column('c2_endpoints', sa.Text(), nullable=True), + sa.Column('kd_digraph_simhash', sa.BINARY(length=8).with_variant(sa.LargeBinary(), 'sqlite'), nullable=True), + sa.Column('merged_into_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('notes', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['campaign_id'], ['campaigns.uuid'], ), + sa.ForeignKeyConstraint(['merged_into_uuid'], ['attacker_identities.uuid'], ), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('attacker_identities', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_attacker_identities_campaign_id'), ['campaign_id'], unique=False) + batch_op.create_index(batch_op.f('ix_attacker_identities_created_at'), ['created_at'], unique=False) + batch_op.create_index(batch_op.f('ix_attacker_identities_first_seen_at'), ['first_seen_at'], unique=False) + batch_op.create_index(batch_op.f('ix_attacker_identities_kd_digraph_simhash'), ['kd_digraph_simhash'], unique=False) + batch_op.create_index(batch_op.f('ix_attacker_identities_last_seen_at'), ['last_seen_at'], unique=False) + batch_op.create_index(batch_op.f('ix_attacker_identities_merged_into_uuid'), ['merged_into_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_attacker_identities_updated_at'), ['updated_at'], unique=False) + + op.create_table('canary_tokens', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('kind', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('decky_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('topology_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('blob_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('instrumenter', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('generator', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('placement_path', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('callback_token', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('secret_seed', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('placed_at', sa.DateTime(), nullable=False), + sa.Column('last_triggered_at', sa.DateTime(), nullable=True), + sa.Column('trigger_count', sa.Integer(), nullable=False), + sa.Column('created_by', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('state', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('last_error', sa.Text(), nullable=True), + sa.Column('fingerprint_nonce', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=True), + sa.ForeignKeyConstraint(['blob_uuid'], ['canary_blobs.uuid'], ), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('canary_tokens', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_canary_tokens_blob_uuid'), ['blob_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_canary_tokens_callback_token'), ['callback_token'], unique=True) + batch_op.create_index(batch_op.f('ix_canary_tokens_created_by'), ['created_by'], unique=False) + batch_op.create_index('ix_canary_tokens_decky', ['decky_name', 'state'], unique=False) + batch_op.create_index(batch_op.f('ix_canary_tokens_decky_name'), ['decky_name'], unique=False) + batch_op.create_index(batch_op.f('ix_canary_tokens_kind'), ['kind'], unique=False) + batch_op.create_index(batch_op.f('ix_canary_tokens_last_triggered_at'), ['last_triggered_at'], unique=False) + batch_op.create_index(batch_op.f('ix_canary_tokens_state'), ['state'], unique=False) + batch_op.create_index(batch_op.f('ix_canary_tokens_topology_id'), ['topology_id'], unique=False) + + op.create_table('decky_shards', + sa.Column('decky_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('host_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('services', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('decky_config', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=True), + sa.Column('decky_ip', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('state', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('last_error', sa.Text(), nullable=True), + sa.Column('compose_hash', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('last_seen', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['host_uuid'], ['swarm_hosts.uuid'], ), + sa.PrimaryKeyConstraint('decky_name') + ) + with op.batch_alter_table('decky_shards', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_decky_shards_host_uuid'), ['host_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_decky_shards_state'), ['state'], unique=False) + + op.create_table('topologies', + sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('mode', sa.String(length=16), nullable=False), + sa.Column('target_host_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('config_snapshot', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('status', sa.String(length=32), nullable=False), + sa.Column('status_changed_at', sa.DateTime(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('version', sa.Integer(), nullable=False), + sa.Column('needs_resync', sa.Boolean(), nullable=False), + sa.Column('email_personas', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('language_default', sqlmodel.sql.sqltypes.AutoString(length=8), nullable=False), + sa.ForeignKeyConstraint(['target_host_uuid'], ['swarm_hosts.uuid'], ), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('topologies', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_topologies_created_at'), ['created_at'], unique=False) + batch_op.create_index(batch_op.f('ix_topologies_name'), ['name'], unique=True) + batch_op.create_index(batch_op.f('ix_topologies_status'), ['status'], unique=False) + batch_op.create_index(batch_op.f('ix_topologies_target_host_uuid'), ['target_host_uuid'], unique=False) + + op.create_table('attackers', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('ip', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('identity_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('first_seen', sa.DateTime(), nullable=False), + sa.Column('last_seen', sa.DateTime(), nullable=False), + sa.Column('event_count', sa.Integer(), nullable=False), + sa.Column('service_count', sa.Integer(), nullable=False), + sa.Column('decky_count', sa.Integer(), nullable=False), + sa.Column('services', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('deckies', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('traversal_path', sa.Text(), nullable=True), + sa.Column('is_traversal', sa.Boolean(), nullable=False), + sa.Column('bounty_count', sa.Integer(), nullable=False), + sa.Column('credential_count', sa.Integer(), nullable=False), + sa.Column('fingerprints', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('commands', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('country_code', sqlmodel.sql.sqltypes.AutoString(length=2), nullable=True), + sa.Column('country_source', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=True), + sa.Column('asn', sa.Integer(), nullable=True), + sa.Column('as_name', sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True), + sa.Column('bgp_prefix', sqlmodel.sql.sqltypes.AutoString(length=43), nullable=True), + sa.Column('asn_source', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=True), + sa.Column('rpki_status', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=True), + sa.Column('rpki_source', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=True), + sa.Column('ptr_record', sqlmodel.sql.sqltypes.AutoString(length=256), nullable=True), + sa.Column('rotation_count', sa.Integer(), nullable=False), + sa.Column('last_rotation_at', sa.DateTime(), nullable=True), + sa.Column('ipv6_leak_count', sa.Integer(), nullable=False), + sa.Column('last_ipv6_leak_at', sa.DateTime(), nullable=True), + sa.Column('last_ipv6_link_local', sqlmodel.sql.sqltypes.AutoString(length=45), nullable=True), + sa.Column('last_ipv6_iid_kind', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=True), + sa.Column('last_ipv6_mac_oui', sqlmodel.sql.sqltypes.AutoString(length=8), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['identity_id'], ['attacker_identities.uuid'], ), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('attackers', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_attackers_asn'), ['asn'], unique=False) + batch_op.create_index(batch_op.f('ix_attackers_bgp_prefix'), ['bgp_prefix'], unique=False) + batch_op.create_index(batch_op.f('ix_attackers_country_code'), ['country_code'], unique=False) + batch_op.create_index(batch_op.f('ix_attackers_first_seen'), ['first_seen'], unique=False) + batch_op.create_index(batch_op.f('ix_attackers_identity_id'), ['identity_id'], unique=False) + batch_op.create_index(batch_op.f('ix_attackers_ip'), ['ip'], unique=False) + batch_op.create_index(batch_op.f('ix_attackers_last_ipv6_leak_at'), ['last_ipv6_leak_at'], unique=False) + batch_op.create_index(batch_op.f('ix_attackers_last_rotation_at'), ['last_rotation_at'], unique=False) + batch_op.create_index(batch_op.f('ix_attackers_last_seen'), ['last_seen'], unique=False) + batch_op.create_index(batch_op.f('ix_attackers_updated_at'), ['updated_at'], unique=False) + + op.create_table('attribution_state', + sa.Column('identity_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('primitive', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('current_value', sa.JSON(), nullable=False), + sa.Column('state', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('confidence', sa.Float(), nullable=False), + sa.Column('observation_count', sa.Integer(), nullable=False), + sa.Column('last_change_ts', sa.Float(), nullable=False), + sa.Column('last_observation_ts', sa.Float(), nullable=False), + sa.Column('schema_version', sa.Integer(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['identity_uuid'], ['attacker_identities.uuid'], ), + sa.PrimaryKeyConstraint('identity_uuid', 'primitive') + ) + with op.batch_alter_table('attribution_state', schema=None) as batch_op: + batch_op.create_index('ix_attribution_state_identity_state', ['identity_uuid', 'state'], unique=False) + batch_op.create_index('ix_attribution_state_last_change', ['last_change_ts'], unique=False) + batch_op.create_index('ix_attribution_state_state', ['state'], unique=False) + + op.create_table('canary_triggers', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('token_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('occurred_at', sa.DateTime(), nullable=False), + sa.Column('src_ip', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('user_agent', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('request_path', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('dns_qname', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('raw_headers', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('attacker_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.ForeignKeyConstraint(['token_uuid'], ['canary_tokens.uuid'], ), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('canary_triggers', schema=None) as batch_op: + batch_op.create_index('ix_canary_triggers_attacker', ['attacker_id'], unique=False) + batch_op.create_index(batch_op.f('ix_canary_triggers_attacker_id'), ['attacker_id'], unique=False) + batch_op.create_index(batch_op.f('ix_canary_triggers_src_ip'), ['src_ip'], unique=False) + batch_op.create_index('ix_canary_triggers_token_ts', ['token_uuid', 'occurred_at'], unique=False) + batch_op.create_index(batch_op.f('ix_canary_triggers_token_uuid'), ['token_uuid'], unique=False) + + op.create_table('lans', + sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('topology_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('docker_network_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('subnet', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('is_dmz', sa.Boolean(), nullable=False), + sa.Column('x', sa.Float(), nullable=True), + sa.Column('y', sa.Float(), nullable=True), + sa.ForeignKeyConstraint(['topology_id'], ['topologies.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('topology_id', 'name', name='uq_lan_topology_name') + ) + with op.batch_alter_table('lans', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_lans_topology_id'), ['topology_id'], unique=False) + + op.create_table('topology_deckies', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('topology_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('services', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('decky_config', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=True), + sa.Column('ip', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('state', sa.String(length=32), nullable=False), + sa.Column('last_error', sa.Text(), nullable=True), + sa.Column('compose_hash', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('last_seen', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.Column('x', sa.Float(), nullable=True), + sa.Column('y', sa.Float(), nullable=True), + sa.ForeignKeyConstraint(['topology_id'], ['topologies.id'], ), + sa.PrimaryKeyConstraint('uuid'), + sa.UniqueConstraint('topology_id', 'name', name='uq_topology_decky_name') + ) + with op.batch_alter_table('topology_deckies', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_topology_deckies_state'), ['state'], unique=False) + batch_op.create_index(batch_op.f('ix_topology_deckies_topology_id'), ['topology_id'], unique=False) + + op.create_table('topology_mutations', + sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('topology_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('op', sa.String(length=32), nullable=False), + sa.Column('payload', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('state', sa.String(length=32), nullable=False), + sa.Column('requested_at', sa.DateTime(), nullable=False), + sa.Column('applied_at', sa.DateTime(), nullable=True), + sa.Column('reason', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['topology_id'], ['topologies.id'], ), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('topology_mutations', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_topology_mutations_op'), ['op'], unique=False) + batch_op.create_index(batch_op.f('ix_topology_mutations_requested_at'), ['requested_at'], unique=False) + batch_op.create_index(batch_op.f('ix_topology_mutations_state'), ['state'], unique=False) + batch_op.create_index('ix_topology_mutations_state_topology', ['state', 'topology_id'], unique=False) + batch_op.create_index(batch_op.f('ix_topology_mutations_topology_id'), ['topology_id'], unique=False) + + op.create_table('topology_status_events', + sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('topology_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('from_status', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('to_status', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('at', sa.DateTime(), nullable=False), + sa.Column('reason', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['topology_id'], ['topologies.id'], ), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('topology_status_events', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_topology_status_events_at'), ['at'], unique=False) + batch_op.create_index(batch_op.f('ix_topology_status_events_topology_id'), ['topology_id'], unique=False) + + op.create_table('attacker_behavior', + sa.Column('attacker_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('os_guess', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('hop_distance', sa.Integer(), nullable=True), + sa.Column('tcp_fingerprint', sa.Text(), nullable=False), + sa.Column('kex_order_raw', sa.Text(), nullable=True), + sa.Column('ssh_client_banners', sa.Text(), nullable=True), + sa.Column('retransmit_count', sa.Integer(), nullable=False), + sa.Column('behavior_class', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('beacon_interval_s', sa.Float(), nullable=True), + sa.Column('beacon_jitter_pct', sa.Float(), nullable=True), + sa.Column('tool_guesses', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('timing_stats', sa.Text(), nullable=False), + sa.Column('phase_sequence', sa.Text(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['attacker_uuid'], ['attackers.uuid'], ), + sa.PrimaryKeyConstraint('attacker_uuid') + ) + with op.batch_alter_table('attacker_behavior', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_attacker_behavior_updated_at'), ['updated_at'], unique=False) + + op.create_table('attacker_fingerprint_state', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('attacker_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('port', sa.Integer(), nullable=False), + sa.Column('probe_type', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False), + sa.Column('last_hash', sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False), + sa.Column('last_seen', sa.DateTime(), nullable=False), + sa.Column('rotation_count', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['attacker_uuid'], ['attackers.uuid'], ), + sa.PrimaryKeyConstraint('uuid'), + sa.UniqueConstraint('attacker_uuid', 'port', 'probe_type', name='uq_attacker_fingerprint_state_natural') + ) + with op.batch_alter_table('attacker_fingerprint_state', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_attacker_fingerprint_state_attacker_uuid'), ['attacker_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_attacker_fingerprint_state_last_seen'), ['last_seen'], unique=False) + + op.create_table('attacker_intel', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('attacker_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('attacker_ip', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('schema_version', sa.Integer(), nullable=False), + sa.Column('greynoise_classification', sqlmodel.sql.sqltypes.AutoString(length=32), nullable=True), + sa.Column('greynoise_name', sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True), + sa.Column('greynoise_tags', sa.JSON(), nullable=False), + sa.Column('greynoise_raw', sa.JSON(), nullable=False), + sa.Column('greynoise_queried_at', sa.DateTime(), nullable=True), + sa.Column('abuseipdb_score', sa.Integer(), nullable=True), + sa.Column('abuseipdb_categories', sa.JSON(), nullable=False), + sa.Column('abuseipdb_raw', sa.JSON(), nullable=False), + sa.Column('abuseipdb_queried_at', sa.DateTime(), nullable=True), + sa.Column('feodo_listed', sa.Boolean(), nullable=True), + sa.Column('feodo_malware_family', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=True), + sa.Column('feodo_raw', sa.JSON(), nullable=False), + sa.Column('feodo_queried_at', sa.DateTime(), nullable=True), + sa.Column('threatfox_listed', sa.Boolean(), nullable=True), + sa.Column('threatfox_threat_types', sa.JSON(), nullable=False), + sa.Column('threatfox_ioc_types', sa.JSON(), nullable=False), + sa.Column('threatfox_malware_families', sa.JSON(), nullable=False), + sa.Column('threatfox_raw', sa.JSON(), nullable=False), + sa.Column('threatfox_queried_at', sa.DateTime(), nullable=True), + sa.Column('aggregate_verdict', sqlmodel.sql.sqltypes.AutoString(length=32), nullable=True), + sa.Column('cached_at', sa.DateTime(), nullable=False), + sa.Column('expires_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['attacker_uuid'], ['attackers.uuid'], ), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('attacker_intel', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_attacker_intel_aggregate_verdict'), ['aggregate_verdict'], unique=False) + batch_op.create_index(batch_op.f('ix_attacker_intel_attacker_ip'), ['attacker_ip'], unique=False) + batch_op.create_index(batch_op.f('ix_attacker_intel_attacker_uuid'), ['attacker_uuid'], unique=True) + batch_op.create_index(batch_op.f('ix_attacker_intel_cached_at'), ['cached_at'], unique=False) + batch_op.create_index(batch_op.f('ix_attacker_intel_expires_at'), ['expires_at'], unique=False) + + op.create_table('credentials', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('attacker_ip', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('attacker_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('decky_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('service', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('principal', sqlmodel.sql.sqltypes.AutoString(length=256), nullable=True), + sa.Column('principal_key', sqlmodel.sql.sqltypes.AutoString(length=256), nullable=False), + sa.Column('secret_kind', sqlmodel.sql.sqltypes.AutoString(length=32), nullable=False), + sa.Column('secret_sha256', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False), + sa.Column('secret_b64', sqlmodel.sql.sqltypes.AutoString(length=2048), nullable=True), + sa.Column('secret_printable', sqlmodel.sql.sqltypes.AutoString(length=512), nullable=True), + sa.Column('outcome', sqlmodel.sql.sqltypes.AutoString(length=16), nullable=True), + sa.Column('fields', sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.Column('first_seen', sa.DateTime(), nullable=False), + sa.Column('last_seen', sa.DateTime(), nullable=False), + sa.Column('attempt_count', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['attacker_uuid'], ['attackers.uuid'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('attacker_ip', 'decky_name', 'service', 'secret_kind', 'secret_sha256', 'principal_key', name='uq_credentials_dedup') + ) + with op.batch_alter_table('credentials', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_credentials_attacker_ip'), ['attacker_ip'], unique=False) + batch_op.create_index(batch_op.f('ix_credentials_attacker_uuid'), ['attacker_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_credentials_decky_name'), ['decky_name'], unique=False) + batch_op.create_index(batch_op.f('ix_credentials_first_seen'), ['first_seen'], unique=False) + batch_op.create_index(batch_op.f('ix_credentials_last_seen'), ['last_seen'], unique=False) + batch_op.create_index(batch_op.f('ix_credentials_principal'), ['principal'], unique=False) + batch_op.create_index('ix_credentials_principal_service', ['principal', 'service'], unique=False) + batch_op.create_index(batch_op.f('ix_credentials_secret_kind'), ['secret_kind'], unique=False) + batch_op.create_index('ix_credentials_secret_service', ['secret_sha256', 'service'], unique=False) + batch_op.create_index(batch_op.f('ix_credentials_secret_sha256'), ['secret_sha256'], unique=False) + batch_op.create_index(batch_op.f('ix_credentials_service'), ['service'], unique=False) + + op.create_table('observations', + sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('identity_ref', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('primitive', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('value', sa.JSON(), nullable=False), + sa.Column('confidence', sa.Float(), nullable=False), + sa.Column('window_start_ts', sa.Float(), nullable=False), + sa.Column('window_end_ts', sa.Float(), nullable=False), + sa.Column('source', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('evidence_ref', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('envelope_v', sa.Integer(), nullable=False), + sa.Column('ts', sa.Float(), nullable=False), + sa.Column('attacker_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.ForeignKeyConstraint(['attacker_uuid'], ['attackers.uuid'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('evidence_ref', 'primitive', name='uq_observations_evidence_primitive') + ) + with op.batch_alter_table('observations', schema=None) as batch_op: + batch_op.create_index('ix_observations_attacker_primitive_ts', ['attacker_uuid', 'primitive', 'ts'], unique=False) + batch_op.create_index(batch_op.f('ix_observations_attacker_uuid'), ['attacker_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_observations_primitive'), ['primitive'], unique=False) + batch_op.create_index('ix_observations_primitive_ts', ['primitive', 'ts'], unique=False) + batch_op.create_index(batch_op.f('ix_observations_ts'), ['ts'], unique=False) + + op.create_table('smtp_targets', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('attacker_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('domain', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('first_seen', sa.DateTime(), nullable=False), + sa.Column('last_seen', sa.DateTime(), nullable=False), + sa.Column('count', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['attacker_uuid'], ['attackers.uuid'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('attacker_uuid', 'domain', name='uq_smtp_targets_attacker_domain') + ) + with op.batch_alter_table('smtp_targets', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_smtp_targets_attacker_uuid'), ['attacker_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_smtp_targets_domain'), ['domain'], unique=False) + batch_op.create_index(batch_op.f('ix_smtp_targets_last_seen'), ['last_seen'], unique=False) + + op.create_table('topology_edges', + sa.Column('id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('topology_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('decky_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('lan_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('is_bridge', sa.Boolean(), nullable=False), + sa.Column('forwards_l3', sa.Boolean(), nullable=False), + sa.ForeignKeyConstraint(['decky_uuid'], ['topology_deckies.uuid'], ), + sa.ForeignKeyConstraint(['lan_id'], ['lans.id'], ), + sa.ForeignKeyConstraint(['topology_id'], ['topologies.id'], ), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('topology_edges', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_topology_edges_decky_uuid'), ['decky_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_topology_edges_lan_id'), ['lan_id'], unique=False) + batch_op.create_index(batch_op.f('ix_topology_edges_topology_id'), ['topology_id'], unique=False) + + op.create_table('ttp_tag', + sa.Column('uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('source_kind', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('source_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('attacker_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('identity_uuid', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('session_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('decky_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('tactic', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('technique_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('sub_technique_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('confidence', sa.Float(), nullable=False), + sa.Column('rule_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('rule_version', sa.Integer(), nullable=False), + sa.Column('evidence', sa.JSON(), nullable=False), + sa.Column('attack_release', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('mitre_url', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.CheckConstraint('attacker_uuid IS NOT NULL OR identity_uuid IS NOT NULL', name='ttp_tag_has_anchor'), + sa.CheckConstraint('confidence >= 0.0 AND confidence <= 1.0', name='ttp_tag_confidence_range'), + sa.ForeignKeyConstraint(['attacker_uuid'], ['attackers.uuid'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['identity_uuid'], ['attacker_identities.uuid'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('uuid') + ) + with op.batch_alter_table('ttp_tag', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_ttp_tag_attack_release'), ['attack_release'], unique=False) + batch_op.create_index('ix_ttp_tag_attacker_technique', ['attacker_uuid', 'technique_id'], unique=False) + batch_op.create_index(batch_op.f('ix_ttp_tag_attacker_uuid'), ['attacker_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_ttp_tag_created_at'), ['created_at'], unique=False) + batch_op.create_index(batch_op.f('ix_ttp_tag_decky_id'), ['decky_id'], unique=False) + batch_op.create_index('ix_ttp_tag_identity_technique', ['identity_uuid', 'technique_id'], unique=False) + batch_op.create_index(batch_op.f('ix_ttp_tag_identity_uuid'), ['identity_uuid'], unique=False) + batch_op.create_index(batch_op.f('ix_ttp_tag_rule_id'), ['rule_id'], unique=False) + batch_op.create_index(batch_op.f('ix_ttp_tag_session_id'), ['session_id'], unique=False) + batch_op.create_index(batch_op.f('ix_ttp_tag_sub_technique_id'), ['sub_technique_id'], unique=False) + batch_op.create_index(batch_op.f('ix_ttp_tag_tactic'), ['tactic'], unique=False) + batch_op.create_index('ix_ttp_tag_technique_created', ['technique_id', 'created_at'], unique=False) + batch_op.create_index(batch_op.f('ix_ttp_tag_technique_id'), ['technique_id'], unique=False) + + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('ttp_tag', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_ttp_tag_technique_id')) + batch_op.drop_index('ix_ttp_tag_technique_created') + batch_op.drop_index(batch_op.f('ix_ttp_tag_tactic')) + batch_op.drop_index(batch_op.f('ix_ttp_tag_sub_technique_id')) + batch_op.drop_index(batch_op.f('ix_ttp_tag_session_id')) + batch_op.drop_index(batch_op.f('ix_ttp_tag_rule_id')) + batch_op.drop_index(batch_op.f('ix_ttp_tag_identity_uuid')) + batch_op.drop_index('ix_ttp_tag_identity_technique') + batch_op.drop_index(batch_op.f('ix_ttp_tag_decky_id')) + batch_op.drop_index(batch_op.f('ix_ttp_tag_created_at')) + batch_op.drop_index(batch_op.f('ix_ttp_tag_attacker_uuid')) + batch_op.drop_index('ix_ttp_tag_attacker_technique') + batch_op.drop_index(batch_op.f('ix_ttp_tag_attack_release')) + + op.drop_table('ttp_tag') + with op.batch_alter_table('topology_edges', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_topology_edges_topology_id')) + batch_op.drop_index(batch_op.f('ix_topology_edges_lan_id')) + batch_op.drop_index(batch_op.f('ix_topology_edges_decky_uuid')) + + op.drop_table('topology_edges') + with op.batch_alter_table('smtp_targets', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_smtp_targets_last_seen')) + batch_op.drop_index(batch_op.f('ix_smtp_targets_domain')) + batch_op.drop_index(batch_op.f('ix_smtp_targets_attacker_uuid')) + + op.drop_table('smtp_targets') + with op.batch_alter_table('observations', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_observations_ts')) + batch_op.drop_index('ix_observations_primitive_ts') + batch_op.drop_index(batch_op.f('ix_observations_primitive')) + batch_op.drop_index(batch_op.f('ix_observations_attacker_uuid')) + batch_op.drop_index('ix_observations_attacker_primitive_ts') + + op.drop_table('observations') + with op.batch_alter_table('credentials', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_credentials_service')) + batch_op.drop_index(batch_op.f('ix_credentials_secret_sha256')) + batch_op.drop_index('ix_credentials_secret_service') + batch_op.drop_index(batch_op.f('ix_credentials_secret_kind')) + batch_op.drop_index('ix_credentials_principal_service') + batch_op.drop_index(batch_op.f('ix_credentials_principal')) + batch_op.drop_index(batch_op.f('ix_credentials_last_seen')) + batch_op.drop_index(batch_op.f('ix_credentials_first_seen')) + batch_op.drop_index(batch_op.f('ix_credentials_decky_name')) + batch_op.drop_index(batch_op.f('ix_credentials_attacker_uuid')) + batch_op.drop_index(batch_op.f('ix_credentials_attacker_ip')) + + op.drop_table('credentials') + with op.batch_alter_table('attacker_intel', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_attacker_intel_expires_at')) + batch_op.drop_index(batch_op.f('ix_attacker_intel_cached_at')) + batch_op.drop_index(batch_op.f('ix_attacker_intel_attacker_uuid')) + batch_op.drop_index(batch_op.f('ix_attacker_intel_attacker_ip')) + batch_op.drop_index(batch_op.f('ix_attacker_intel_aggregate_verdict')) + + op.drop_table('attacker_intel') + with op.batch_alter_table('attacker_fingerprint_state', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_attacker_fingerprint_state_last_seen')) + batch_op.drop_index(batch_op.f('ix_attacker_fingerprint_state_attacker_uuid')) + + op.drop_table('attacker_fingerprint_state') + with op.batch_alter_table('attacker_behavior', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_attacker_behavior_updated_at')) + + op.drop_table('attacker_behavior') + with op.batch_alter_table('topology_status_events', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_topology_status_events_topology_id')) + batch_op.drop_index(batch_op.f('ix_topology_status_events_at')) + + op.drop_table('topology_status_events') + with op.batch_alter_table('topology_mutations', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_topology_mutations_topology_id')) + batch_op.drop_index('ix_topology_mutations_state_topology') + batch_op.drop_index(batch_op.f('ix_topology_mutations_state')) + batch_op.drop_index(batch_op.f('ix_topology_mutations_requested_at')) + batch_op.drop_index(batch_op.f('ix_topology_mutations_op')) + + op.drop_table('topology_mutations') + with op.batch_alter_table('topology_deckies', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_topology_deckies_topology_id')) + batch_op.drop_index(batch_op.f('ix_topology_deckies_state')) + + op.drop_table('topology_deckies') + with op.batch_alter_table('lans', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_lans_topology_id')) + + op.drop_table('lans') + with op.batch_alter_table('canary_triggers', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_canary_triggers_token_uuid')) + batch_op.drop_index('ix_canary_triggers_token_ts') + batch_op.drop_index(batch_op.f('ix_canary_triggers_src_ip')) + batch_op.drop_index(batch_op.f('ix_canary_triggers_attacker_id')) + batch_op.drop_index('ix_canary_triggers_attacker') + + op.drop_table('canary_triggers') + with op.batch_alter_table('attribution_state', schema=None) as batch_op: + batch_op.drop_index('ix_attribution_state_state') + batch_op.drop_index('ix_attribution_state_last_change') + batch_op.drop_index('ix_attribution_state_identity_state') + + op.drop_table('attribution_state') + with op.batch_alter_table('attackers', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_attackers_updated_at')) + batch_op.drop_index(batch_op.f('ix_attackers_last_seen')) + batch_op.drop_index(batch_op.f('ix_attackers_last_rotation_at')) + batch_op.drop_index(batch_op.f('ix_attackers_last_ipv6_leak_at')) + batch_op.drop_index(batch_op.f('ix_attackers_ip')) + batch_op.drop_index(batch_op.f('ix_attackers_identity_id')) + batch_op.drop_index(batch_op.f('ix_attackers_first_seen')) + batch_op.drop_index(batch_op.f('ix_attackers_country_code')) + batch_op.drop_index(batch_op.f('ix_attackers_bgp_prefix')) + batch_op.drop_index(batch_op.f('ix_attackers_asn')) + + op.drop_table('attackers') + with op.batch_alter_table('topologies', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_topologies_target_host_uuid')) + batch_op.drop_index(batch_op.f('ix_topologies_status')) + batch_op.drop_index(batch_op.f('ix_topologies_name')) + batch_op.drop_index(batch_op.f('ix_topologies_created_at')) + + op.drop_table('topologies') + with op.batch_alter_table('decky_shards', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_decky_shards_state')) + batch_op.drop_index(batch_op.f('ix_decky_shards_host_uuid')) + + op.drop_table('decky_shards') + with op.batch_alter_table('canary_tokens', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_canary_tokens_topology_id')) + batch_op.drop_index(batch_op.f('ix_canary_tokens_state')) + batch_op.drop_index(batch_op.f('ix_canary_tokens_last_triggered_at')) + batch_op.drop_index(batch_op.f('ix_canary_tokens_kind')) + batch_op.drop_index(batch_op.f('ix_canary_tokens_decky_name')) + batch_op.drop_index('ix_canary_tokens_decky') + batch_op.drop_index(batch_op.f('ix_canary_tokens_created_by')) + batch_op.drop_index(batch_op.f('ix_canary_tokens_callback_token')) + batch_op.drop_index(batch_op.f('ix_canary_tokens_blob_uuid')) + + op.drop_table('canary_tokens') + with op.batch_alter_table('attacker_identities', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_attacker_identities_updated_at')) + batch_op.drop_index(batch_op.f('ix_attacker_identities_merged_into_uuid')) + batch_op.drop_index(batch_op.f('ix_attacker_identities_last_seen_at')) + batch_op.drop_index(batch_op.f('ix_attacker_identities_kd_digraph_simhash')) + batch_op.drop_index(batch_op.f('ix_attacker_identities_first_seen_at')) + batch_op.drop_index(batch_op.f('ix_attacker_identities_created_at')) + batch_op.drop_index(batch_op.f('ix_attacker_identities_campaign_id')) + + op.drop_table('attacker_identities') + with op.batch_alter_table('webhook_subscriptions', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_webhook_subscriptions_name')) + batch_op.drop_index(batch_op.f('ix_webhook_subscriptions_enabled')) + + op.drop_table('webhook_subscriptions') + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_users_username')) + + op.drop_table('users') + op.drop_table('ttp_rule_state') + op.drop_table('ttp_rule') + with op.batch_alter_table('tarpit_rules', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_tarpit_rules_decky_name')) + + op.drop_table('tarpit_rules') + with op.batch_alter_table('synthetic_files', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_synthetic_files_decky_uuid')) + batch_op.drop_index('ix_synthetic_files_decky_modified') + batch_op.drop_index(batch_op.f('ix_synthetic_files_created_at')) + batch_op.drop_index(batch_op.f('ix_synthetic_files_content_class')) + + op.drop_table('synthetic_files') + with op.batch_alter_table('swarm_hosts', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_swarm_hosts_status')) + batch_op.drop_index(batch_op.f('ix_swarm_hosts_name')) + + op.drop_table('swarm_hosts') + op.drop_table('state') + with op.batch_alter_table('revoked_tokens', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_revoked_tokens_user_uuid')) + batch_op.drop_index(batch_op.f('ix_revoked_tokens_expires_at')) + + op.drop_table('revoked_tokens') + with op.batch_alter_table('realism_config', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_realism_config_key')) + + op.drop_table('realism_config') + with op.batch_alter_table('orchestrator_events', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_orchestrator_events_ts')) + batch_op.drop_index(batch_op.f('ix_orchestrator_events_success')) + batch_op.drop_index(batch_op.f('ix_orchestrator_events_src_decky_uuid')) + batch_op.drop_index(batch_op.f('ix_orchestrator_events_protocol')) + batch_op.drop_index(batch_op.f('ix_orchestrator_events_kind')) + batch_op.drop_index('ix_orchestrator_events_dst_ts') + batch_op.drop_index(batch_op.f('ix_orchestrator_events_dst_decky_uuid')) + + op.drop_table('orchestrator_events') + with op.batch_alter_table('orchestrator_emails', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_orchestrator_emails_ts')) + batch_op.drop_index(batch_op.f('ix_orchestrator_emails_thread_id')) + batch_op.drop_index('ix_orchestrator_emails_thread') + batch_op.drop_index(batch_op.f('ix_orchestrator_emails_success')) + batch_op.drop_index(batch_op.f('ix_orchestrator_emails_sender_email')) + batch_op.drop_index(batch_op.f('ix_orchestrator_emails_recipient_email')) + batch_op.drop_index('ix_orchestrator_emails_mail_ts') + batch_op.drop_index(batch_op.f('ix_orchestrator_emails_mail_decky_uuid')) + + op.drop_table('orchestrator_emails') + with op.batch_alter_table('observed_attachments', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_observed_attachments_sha256')) + batch_op.drop_index('ix_observed_attachments_mal_hash_match') + batch_op.drop_index(batch_op.f('ix_observed_attachments_last_seen_attacker_uuid')) + batch_op.drop_index('ix_observed_attachments_last_seen') + batch_op.drop_index(batch_op.f('ix_observed_attachments_first_seen_decky_uuid')) + batch_op.drop_index(batch_op.f('ix_observed_attachments_first_seen_attacker_uuid')) + batch_op.drop_index('ix_observed_attachments_first_seen') + + op.drop_table('observed_attachments') + with op.batch_alter_table('logs', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_logs_timestamp')) + batch_op.drop_index(batch_op.f('ix_logs_service')) + batch_op.drop_index(batch_op.f('ix_logs_event_type')) + batch_op.drop_index(batch_op.f('ix_logs_decky')) + batch_op.drop_index(batch_op.f('ix_logs_attacker_ip')) + + op.drop_table('logs') + with op.batch_alter_table('fleet_deckies', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_fleet_deckies_state')) + batch_op.drop_index(batch_op.f('ix_fleet_deckies_host_uuid')) + + op.drop_table('fleet_deckies') + with op.batch_alter_table('decky_lifecycle', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_decky_lifecycle_status')) + batch_op.drop_index(batch_op.f('ix_decky_lifecycle_operation')) + batch_op.drop_index(batch_op.f('ix_decky_lifecycle_host_uuid')) + batch_op.drop_index(batch_op.f('ix_decky_lifecycle_decky_name')) + + op.drop_table('decky_lifecycle') + with op.batch_alter_table('credential_reuse', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_credential_reuse_updated_at')) + batch_op.drop_index(batch_op.f('ix_credential_reuse_target_count')) + batch_op.drop_index(batch_op.f('ix_credential_reuse_secret_sha256')) + batch_op.drop_index(batch_op.f('ix_credential_reuse_secret_kind')) + batch_op.drop_index(batch_op.f('ix_credential_reuse_last_seen')) + batch_op.drop_index(batch_op.f('ix_credential_reuse_first_seen')) + + op.drop_table('credential_reuse') + with op.batch_alter_table('canary_blobs', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_canary_blobs_uploaded_by')) + batch_op.drop_index(batch_op.f('ix_canary_blobs_sha256')) + + op.drop_table('canary_blobs') + with op.batch_alter_table('campaigns', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_campaigns_updated_at')) + batch_op.drop_index(batch_op.f('ix_campaigns_merged_into_uuid')) + batch_op.drop_index(batch_op.f('ix_campaigns_last_seen_at')) + batch_op.drop_index(batch_op.f('ix_campaigns_first_seen_at')) + batch_op.drop_index(batch_op.f('ix_campaigns_created_at')) + + op.drop_table('campaigns') + with op.batch_alter_table('bounty', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_bounty_timestamp')) + batch_op.drop_index(batch_op.f('ix_bounty_service')) + batch_op.drop_index(batch_op.f('ix_bounty_decky')) + batch_op.drop_index(batch_op.f('ix_bounty_bounty_type')) + batch_op.drop_index(batch_op.f('ix_bounty_attacker_ip')) + + op.drop_table('bounty') + # ### end Alembic commands ### diff --git a/decnet/web/db/models/attackers.py b/decnet/web/db/models/attackers.py index b7455bbb..ef0b47a1 100644 --- a/decnet/web/db/models/attackers.py +++ b/decnet/web/db/models/attackers.py @@ -13,7 +13,7 @@ from datetime import datetime, timezone from typing import Any, List, Optional from pydantic import BaseModel -from sqlalchemy import BINARY, Column, Text, UniqueConstraint +from sqlalchemy import BINARY, Column, LargeBinary, Text, UniqueConstraint from sqlmodel import Field, SQLModel from ._base import _BIG_TEXT @@ -238,10 +238,18 @@ class AttackerIdentity(SQLModel, table=True): # registry); this column is the rollup the (future) attribution # engine will write into so the federation gossip layer # has one identity-level fingerprint to compare across operators. - # BINARY(8) so MySQL can index without a prefix length. + # BINARY(8) so MySQL can index without a prefix length. SQLite has no + # fixed-width binary type (BINARY → NUMERIC affinity, which reflects back + # as NUMERIC and trips `alembic check`), so use a BLOB variant there — + # bytes round-trip identically and the type matches what SQLite reports. kd_digraph_simhash: Optional[bytes] = Field( default=None, - sa_column=Column("kd_digraph_simhash", BINARY(8), nullable=True, index=True), + sa_column=Column( + "kd_digraph_simhash", + BINARY(8).with_variant(LargeBinary(), "sqlite"), + nullable=True, + index=True, + ), ) # Soft-merge audit trail. When the clusterer collapses two # identities, the loser's row stays in place with this set to the diff --git a/pyproject.toml b/pyproject.toml index 398724af..71217f0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,9 @@ dependencies = [ "psutil>=5.9.0", "python-dotenv>=1.0.0", "sqlmodel>=0.0.16", + # Schema migrations. Runtime dep (not dev-only): the API runs + # `alembic upgrade head` at boot for managed DBs (see db/migrate.py). + "alembic>=1.13", "scapy>=2.6.1", "orjson>=3.10", "cryptography>=48.0.1",