refactor(tests): move flat tests/*.py into per-subsystem subfolders

Groups every flat test_*.py under the module it exercises, matching the
existing tests/{profiler,sniffer,prober,collector,correlation,cli,web,
topology,swarm,bus,updater,api,docker,geoip,...} layout. New folders:
services/, fleet/, config/, logging/, db/ (+ db/mysql/), telemetry/,
mutator/, core/.

Path-dependent __file__ references bumped an extra .parent in three
files that moved one level deeper:
- tests/sniffer/test_sniffer_ja3.py   (template path)
- tests/services/test_ssh_capture_emit.py (template path)
- tests/cli/test_mode_gating.py  (REPO root)
- tests/web/test_env_lazy_jwt.py (repo var)

Also drops two SQLite runtime artifacts (test_decnet.db-{shm,wal}) that
were leaking into the repo from a previous test run.

Fixes two test_service_isolation cases that patched asyncio.sleep (no
longer on the profiler main-loop hot path — same pre-existing bug I
fixed earlier in test_attacker_worker.py) by patching asyncio.wait_for
and passing interval=0.
This commit is contained in:
2026-04-23 21:34:25 -04:00
parent 21e6820714
commit ea95a009df
78 changed files with 18 additions and 10 deletions

0
tests/db/__init__.py Normal file
View File

View File

View File

@@ -0,0 +1,70 @@
"""
Inspection-level tests for the MySQL-dialect SQL emitted by MySQLRepository.
We compile the SQLAlchemy statements against the MySQL dialect and assert on
the string form — no live MySQL server is required.
"""
import pytest
from sqlalchemy import func, select, literal_column
from sqlalchemy.dialects import mysql
from sqlmodel.sql.expression import SelectOfScalar
from decnet.web.db.models import Log
def _compile(stmt) -> str:
"""Compile a statement to MySQL-dialect SQL with literal values inlined."""
return str(stmt.compile(
dialect=mysql.dialect(),
compile_kwargs={"literal_binds": True},
))
def test_mysql_histogram_uses_from_unixtime_bucket():
"""The MySQL dialect must bucket with UNIX_TIMESTAMP DIV N * N wrapped in FROM_UNIXTIME."""
bucket_seconds = 900 # 15 min
bucket_expr = literal_column(
f"FROM_UNIXTIME((UNIX_TIMESTAMP(timestamp) DIV {bucket_seconds}) * {bucket_seconds})"
).label("bucket_time")
stmt: SelectOfScalar = select(bucket_expr, func.count().label("count")).select_from(Log)
sql = _compile(stmt)
assert "FROM_UNIXTIME" in sql
assert "UNIX_TIMESTAMP" in sql
assert "DIV 900" in sql
# Sanity: SQLite-only strftime must NOT appear in the MySQL-dialect output.
assert "strftime" not in sql
assert "unixepoch" not in sql
def test_mysql_json_unquote_predicate_shape():
"""MySQL JSON filter uses JSON_UNQUOTE(JSON_EXTRACT(...))."""
from decnet.web.db.mysql.repository import MySQLRepository
# Build a dummy instance without touching the engine. We only need _json_field_equals,
# which is a pure function of the key.
repo = MySQLRepository.__new__(MySQLRepository) # bypass __init__ / no DB connection
predicate = repo._json_field_equals("username")
# text() objects carry their literal SQL in .text
assert "JSON_UNQUOTE" in predicate.text
assert "JSON_EXTRACT(fields, '$.username')" in predicate.text
assert ":val" in predicate.text
@pytest.mark.parametrize("key", ["user", "port", "sess_id"])
def test_mysql_json_predicate_safe_for_reasonable_keys(key):
"""Keys matching [A-Za-z0-9_]+ are inserted verbatim; verify no SQL breakage."""
from decnet.web.db.mysql.repository import MySQLRepository
repo = MySQLRepository.__new__(MySQLRepository)
pred = repo._json_field_equals(key)
assert f"'$.{key}'" in pred.text
def test_sqlite_histogram_still_uses_strftime():
"""Regression guard — SQLite implementation must keep its strftime-based bucket."""
from decnet.web.db.sqlite.repository import SQLiteRepository
import inspect
src = inspect.getsource(SQLiteRepository.get_log_histogram)
assert "strftime" in src
assert "unixepoch" in src

View File

@@ -0,0 +1,224 @@
"""
Tests for MySQLRepository._migrate_column_types().
No live MySQL server required — uses an in-memory SQLite engine that exposes
the same information_schema-style query surface via a mocked connection, plus
an integration-style test using a real async engine over aiosqlite (which
ignores the TEXT/MEDIUMTEXT distinction but verifies the ALTER path is called
and idempotent).
The ALTER TABLE branch is tested via unittest.mock: we intercept the
information_schema query result and assert the correct MODIFY COLUMN
statements are issued.
"""
from __future__ import annotations
import pytest
from unittest.mock import AsyncMock, MagicMock, patch, call
from decnet.web.db.mysql.repository import MySQLRepository
# ── helpers ──────────────────────────────────────────────────────────────────
def _make_repo() -> MySQLRepository:
"""Construct a MySQLRepository without touching any real DB."""
return MySQLRepository.__new__(MySQLRepository)
# ── _migrate_column_types ─────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_migrate_column_types_issues_alter_for_text_columns():
"""When information_schema reports TEXT columns, ALTER TABLE is called for each."""
repo = _make_repo()
# Rows returned by the information_schema query: two TEXT columns found
fake_rows = [
("attackers", "commands"),
("attackers", "fingerprints"),
("state", "value"),
]
exec_results: list[str] = []
async def fake_execute(stmt):
sql = str(stmt)
if "information_schema" in sql:
result = MagicMock()
result.fetchall.return_value = fake_rows
return result
# Capture ALTER TABLE calls
exec_results.append(sql)
return MagicMock()
fake_conn = AsyncMock()
fake_conn.execute.side_effect = fake_execute
fake_ctx = AsyncMock()
fake_ctx.__aenter__ = AsyncMock(return_value=fake_conn)
fake_ctx.__aexit__ = AsyncMock(return_value=False)
repo.engine = MagicMock()
repo.engine.begin.return_value = fake_ctx
await repo._migrate_column_types()
# Three ALTER TABLE statements expected, one per TEXT column returned
assert len(exec_results) == 3
assert any("`commands` MEDIUMTEXT" in s for s in exec_results)
assert any("`fingerprints` MEDIUMTEXT" in s for s in exec_results)
assert any("`value` MEDIUMTEXT" in s for s in exec_results)
# Verify NOT NULL is preserved
assert all("NOT NULL" in s for s in exec_results)
@pytest.mark.asyncio
async def test_migrate_column_types_no_alter_when_already_mediumtext():
"""When information_schema returns no TEXT rows, no ALTER is issued."""
repo = _make_repo()
exec_results: list[str] = []
async def fake_execute(stmt):
sql = str(stmt)
if "information_schema" in sql:
result = MagicMock()
result.fetchall.return_value = [] # nothing to migrate
return result
exec_results.append(sql)
return MagicMock()
fake_conn = AsyncMock()
fake_conn.execute.side_effect = fake_execute
fake_ctx = AsyncMock()
fake_ctx.__aenter__ = AsyncMock(return_value=fake_conn)
fake_ctx.__aexit__ = AsyncMock(return_value=False)
repo.engine = MagicMock()
repo.engine.begin.return_value = fake_ctx
await repo._migrate_column_types()
assert exec_results == [], "No ALTER TABLE should be issued when columns are already MEDIUMTEXT"
@pytest.mark.asyncio
async def test_migrate_column_types_idempotent_on_repeated_calls():
"""Calling _migrate_column_types twice is safe: second call is a no-op."""
repo = _make_repo()
call_count = 0
async def fake_execute(stmt):
nonlocal call_count
sql = str(stmt)
if "information_schema" in sql:
result = MagicMock()
# First call: two TEXT columns; second call: zero (already migrated)
call_count += 1
result.fetchall.return_value = (
[("attackers", "commands")] if call_count == 1 else []
)
return result
return MagicMock()
def _make_ctx():
fake_conn = AsyncMock()
fake_conn.execute.side_effect = fake_execute
ctx = AsyncMock()
ctx.__aenter__ = AsyncMock(return_value=fake_conn)
ctx.__aexit__ = AsyncMock(return_value=False)
return ctx
repo.engine = MagicMock()
repo.engine.begin.side_effect = _make_ctx
await repo._migrate_column_types()
await repo._migrate_column_types() # second call must not raise
@pytest.mark.asyncio
async def test_migrate_column_types_default_clause_per_column():
"""Each attacker column gets DEFAULT '[]'; state.value gets no DEFAULT."""
repo = _make_repo()
all_text_rows = [
("attackers", "commands"),
("attackers", "fingerprints"),
("attackers", "services"),
("attackers", "deckies"),
("state", "value"),
]
alter_stmts: list[str] = []
async def fake_execute(stmt):
sql = str(stmt)
if "information_schema" in sql:
result = MagicMock()
result.fetchall.return_value = all_text_rows
return result
alter_stmts.append(sql)
return MagicMock()
fake_conn = AsyncMock()
fake_conn.execute.side_effect = fake_execute
fake_ctx = AsyncMock()
fake_ctx.__aenter__ = AsyncMock(return_value=fake_conn)
fake_ctx.__aexit__ = AsyncMock(return_value=False)
repo.engine = MagicMock()
repo.engine.begin.return_value = fake_ctx
await repo._migrate_column_types()
attacker_alters = [s for s in alter_stmts if "`attackers`" in s]
state_alters = [s for s in alter_stmts if "`state`" in s]
assert len(attacker_alters) == 4
assert len(state_alters) == 1
for stmt in attacker_alters:
assert "DEFAULT '[]'" in stmt, f"Missing DEFAULT '[]' in: {stmt}"
# state.value has no DEFAULT in the schema
assert "DEFAULT" not in state_alters[0], \
f"Unexpected DEFAULT in state.value alter: {state_alters[0]}"
# ── initialize override ───────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_mysql_initialize_calls_migrate_column_types():
"""MySQLRepository.initialize() must invoke _migrate_column_types after _migrate_attackers_table."""
repo = _make_repo()
call_order: list[str] = []
async def fake_migrate_attackers():
call_order.append("migrate_attackers")
async def fake_migrate_column_types():
call_order.append("migrate_column_types")
async def fake_ensure_admin():
call_order.append("ensure_admin")
repo._migrate_attackers_table = fake_migrate_attackers
repo._migrate_column_types = fake_migrate_column_types
repo._ensure_admin_user = fake_ensure_admin
# Stub engine.begin() so create_all is a no-op
fake_conn = AsyncMock()
fake_conn.run_sync = AsyncMock()
fake_ctx = AsyncMock()
fake_ctx.__aenter__ = AsyncMock(return_value=fake_conn)
fake_ctx.__aexit__ = AsyncMock(return_value=False)
repo.engine = MagicMock()
repo.engine.begin.return_value = fake_ctx
await repo.initialize()
assert call_order == ["migrate_attackers", "migrate_column_types", "ensure_admin"], \
f"Unexpected call order: {call_order}"

View File

@@ -0,0 +1,78 @@
"""
Unit tests for decnet.web.db.mysql.database.build_mysql_url / resolve_url.
No MySQL server is required — these are pure URL-construction tests.
"""
import pytest
from decnet.web.db.mysql.database import build_mysql_url, resolve_url
def test_build_url_defaults(monkeypatch):
for v in ("DECNET_DB_HOST", "DECNET_DB_PORT", "DECNET_DB_NAME",
"DECNET_DB_USER", "DECNET_DB_PASSWORD", "DECNET_DB_URL"):
monkeypatch.delenv(v, raising=False)
# PYTEST_* is set by pytest itself, so empty password is allowed here.
url = build_mysql_url()
assert url == "mysql+asyncmy://decnet:@localhost:3306/decnet"
def test_build_url_from_env(monkeypatch):
monkeypatch.setenv("DECNET_DB_HOST", "db.internal")
monkeypatch.setenv("DECNET_DB_PORT", "3307")
monkeypatch.setenv("DECNET_DB_NAME", "decnet_prod")
monkeypatch.setenv("DECNET_DB_USER", "svc_decnet")
monkeypatch.setenv("DECNET_DB_PASSWORD", "hunter2")
url = build_mysql_url()
assert url == "mysql+asyncmy://svc_decnet:hunter2@db.internal:3307/decnet_prod"
def test_build_url_percent_encodes_password(monkeypatch):
"""Passwords with @ : / # etc must not break URL parsing."""
monkeypatch.setenv("DECNET_DB_PASSWORD", "p@ss:word/!#")
url = build_mysql_url(user="u", host="h", port=3306, database="d")
# @ → %40, : → %3A, / → %2F, # → %23, ! → %21
assert "p%40ss%3Aword%2F%21%23" in url
assert url.startswith("mysql+asyncmy://u:")
assert url.endswith("@h:3306/d")
def test_build_url_component_args_override_env(monkeypatch):
monkeypatch.setenv("DECNET_DB_HOST", "ignored")
monkeypatch.setenv("DECNET_DB_PASSWORD", "env-pw")
url = build_mysql_url(host="arg.host", user="arg-user", password="arg-pw",
port=9999, database="arg-db")
assert url == "mysql+asyncmy://arg-user:arg-pw@arg.host:9999/arg-db"
def test_resolve_url_prefers_explicit_arg(monkeypatch):
monkeypatch.setenv("DECNET_DB_URL", "mysql+asyncmy://env-url/x")
assert resolve_url("mysql+asyncmy://explicit/y") == "mysql+asyncmy://explicit/y"
def test_resolve_url_uses_env_url_before_components(monkeypatch):
monkeypatch.setenv("DECNET_DB_URL", "mysql+asyncmy://env-user:env-pw@env-host/env-db")
monkeypatch.setenv("DECNET_DB_HOST", "ignored.host")
assert resolve_url() == "mysql+asyncmy://env-user:env-pw@env-host/env-db"
def test_resolve_url_falls_back_to_components(monkeypatch):
monkeypatch.delenv("DECNET_DB_URL", raising=False)
monkeypatch.setenv("DECNET_DB_HOST", "fallback.host")
monkeypatch.setenv("DECNET_DB_PASSWORD", "pw")
url = resolve_url()
assert "fallback.host" in url
assert url.startswith("mysql+asyncmy://")
def test_build_url_requires_password_outside_pytest(monkeypatch):
"""Without a password and not in a pytest run, construction must fail loudly."""
for v in ("DECNET_DB_URL", "DECNET_DB_PASSWORD"):
monkeypatch.delenv(v, raising=False)
# Strip every PYTEST_* env var so the safety check trips.
import os
for k in list(os.environ):
if k.startswith("PYTEST"):
monkeypatch.delenv(k, raising=False)
with pytest.raises(ValueError, match="DECNET_DB_PASSWORD is not set"):
build_mysql_url()

123
tests/db/test_base_repo.py Normal file
View File

@@ -0,0 +1,123 @@
"""
Mock test for BaseRepository to ensure coverage of abstract pass lines.
"""
import pytest
from decnet.web.db.repository import BaseRepository
class DummyRepo(BaseRepository):
async def initialize(self) -> None: await super().initialize()
async def add_log(self, data): await super().add_log(data)
async def get_logs(self, **kw): await super().get_logs(**kw)
async def get_total_logs(self, **kw): await super().get_total_logs(**kw)
async def get_stats_summary(self): await super().get_stats_summary()
async def get_deckies(self): await super().get_deckies()
async def get_user_by_username(self, u): await super().get_user_by_username(u)
async def get_user_by_uuid(self, u): await super().get_user_by_uuid(u)
async def create_user(self, d): await super().create_user(d)
async def update_user_password(self, *a, **kw): await super().update_user_password(*a, **kw)
async def add_bounty(self, d): await super().add_bounty(d)
async def get_bounties(self, **kw): await super().get_bounties(**kw)
async def get_total_bounties(self, **kw): await super().get_total_bounties(**kw)
async def get_state(self, k): await super().get_state(k)
async def set_state(self, k, v): await super().set_state(k, v)
async def get_max_log_id(self): await super().get_max_log_id()
async def get_logs_after_id(self, last_id, limit=500): await super().get_logs_after_id(last_id, limit)
async def get_all_bounties_by_ip(self): await super().get_all_bounties_by_ip()
async def get_bounties_for_ips(self, ips): await super().get_bounties_for_ips(ips)
async def upsert_attacker(self, d): await super().upsert_attacker(d); return ""
async def upsert_attacker_behavior(self, u, d): await super().upsert_attacker_behavior(u, d)
async def get_attacker_behavior(self, u): await super().get_attacker_behavior(u)
async def get_behaviors_for_ips(self, ips): await super().get_behaviors_for_ips(ips)
async def upsert_session_profile(self, sid, data): await super().upsert_session_profile(sid, data)
async def get_session_profile(self, sid): await super().get_session_profile(sid)
async def increment_smtp_target(self, u, d): await super().increment_smtp_target(u, d)
async def list_smtp_targets(self, u): await super().list_smtp_targets(u)
async def get_attacker_stored_mail(self, u): await super().get_attacker_stored_mail(u)
async def smtp_target_seen(self, d): await super().smtp_target_seen(d)
async def get_attacker_by_uuid(self, u): await super().get_attacker_by_uuid(u)
async def get_attackers(self, **kw): await super().get_attackers(**kw)
async def get_total_attackers(self, **kw): await super().get_total_attackers(**kw)
async def get_attacker_commands(self, **kw): await super().get_attacker_commands(**kw)
async def list_users(self): await super().list_users()
async def delete_user(self, u): await super().delete_user(u)
async def update_user_role(self, u, r): await super().update_user_role(u, r)
async def purge_logs_and_bounties(self): await super().purge_logs_and_bounties()
async def get_attacker_artifacts(self, uuid): await super().get_attacker_artifacts(uuid)
async def get_attacker_transcripts(self, uuid): await super().get_attacker_transcripts(uuid)
async def get_session_log(self, sid): await super().get_session_log(sid)
@pytest.mark.asyncio
async def test_base_repo_coverage():
dr = DummyRepo()
# Call all to hit 'pass' statements
await dr.initialize()
await dr.add_log({})
await dr.get_logs()
await dr.get_total_logs()
await dr.get_stats_summary()
await dr.get_deckies()
await dr.get_user_by_username("a")
await dr.get_user_by_uuid("a")
await dr.create_user({})
await dr.update_user_password("a", "b")
await dr.add_bounty({})
await dr.get_bounties()
await dr.get_total_bounties()
await dr.get_state("k")
await dr.set_state("k", "v")
await dr.get_max_log_id()
await dr.get_logs_after_id(0)
await dr.get_all_bounties_by_ip()
await dr.get_bounties_for_ips({"1.1.1.1"})
await dr.upsert_attacker({})
await dr.upsert_attacker_behavior("a", {})
await dr.get_attacker_behavior("a")
await dr.get_behaviors_for_ips({"1.1.1.1"})
await dr.upsert_session_profile("sid", {})
await dr.get_session_profile("sid")
await dr.increment_smtp_target("uuid", "corp.com")
await dr.list_smtp_targets("uuid")
await dr.get_attacker_stored_mail("uuid")
await dr.smtp_target_seen("corp.com")
await dr.get_attacker_by_uuid("a")
await dr.get_attackers()
await dr.get_total_attackers()
await dr.get_attacker_commands(uuid="a")
await dr.list_users()
await dr.delete_user("a")
await dr.update_user_role("a", "admin")
await dr.purge_logs_and_bounties()
await dr.get_attacker_artifacts("a")
await dr.get_attacker_transcripts("a")
await dr.get_session_log("a")
# Swarm methods: default NotImplementedError on BaseRepository. Covering
# them here keeps the coverage contract honest for the swarm CRUD surface.
for coro, args in [
(dr.add_swarm_host, ({},)),
(dr.get_swarm_host_by_name, ("w",)),
(dr.get_swarm_host_by_uuid, ("u",)),
(dr.list_swarm_hosts, ()),
(dr.update_swarm_host, ("u", {})),
(dr.delete_swarm_host, ("u",)),
(dr.upsert_decky_shard, ({},)),
(dr.list_decky_shards, ()),
(dr.delete_decky_shards_for_host, ("u",)),
(dr.create_topology, ({},)),
(dr.get_topology, ("t",)),
(dr.list_topologies, ()),
(dr.update_topology_status, ("t", "active")),
(dr.delete_topology_cascade, ("t",)),
(dr.add_lan, ({},)),
(dr.update_lan, ("l", {})),
(dr.list_lans_for_topology, ("t",)),
(dr.add_topology_decky, ({},)),
(dr.update_topology_decky, ("d", {})),
(dr.list_topology_deckies, ("t",)),
(dr.add_topology_edge, ({},)),
(dr.list_topology_edges, ("t",)),
(dr.list_topology_status_events, ("t",)),
]:
with pytest.raises(NotImplementedError):
await coro(*args)

44
tests/db/test_factory.py Normal file
View File

@@ -0,0 +1,44 @@
"""
Unit tests for the repository factory — dispatch on DECNET_DB_TYPE.
"""
import pytest
from decnet.web.db.factory import get_repository
from decnet.web.db.sqlite.repository import SQLiteRepository
from decnet.web.db.mysql.repository import MySQLRepository
def test_factory_defaults_to_sqlite(monkeypatch, tmp_path):
monkeypatch.delenv("DECNET_DB_TYPE", raising=False)
repo = get_repository(db_path=str(tmp_path / "t.db"))
assert isinstance(repo, SQLiteRepository)
def test_factory_sqlite_explicit(monkeypatch, tmp_path):
monkeypatch.setenv("DECNET_DB_TYPE", "sqlite")
repo = get_repository(db_path=str(tmp_path / "t.db"))
assert isinstance(repo, SQLiteRepository)
def test_factory_mysql_branch(monkeypatch):
"""MySQL branch must import and instantiate without a live server.
Engine creation is lazy in SQLAlchemy — no socket is opened until the
first query — so the repository constructs cleanly here.
"""
monkeypatch.setenv("DECNET_DB_TYPE", "mysql")
monkeypatch.setenv("DECNET_DB_URL", "mysql+asyncmy://u:p@127.0.0.1:3306/x")
repo = get_repository()
assert isinstance(repo, MySQLRepository)
def test_factory_is_case_insensitive(monkeypatch, tmp_path):
monkeypatch.setenv("DECNET_DB_TYPE", "SQLite")
repo = get_repository(db_path=str(tmp_path / "t.db"))
assert isinstance(repo, SQLiteRepository)
def test_factory_rejects_unknown_type(monkeypatch):
monkeypatch.setenv("DECNET_DB_TYPE", "cassandra")
with pytest.raises(ValueError, match="Unsupported database type"):
get_repository()