feat(ttp): E.3.6 DatabaseRuleStore — ttp_rule/ttp_rule_state + master sync
Implements the DB-backed rule store body left empty at contract phase:
load_compiled reads from ttp_rule + ttp_rule_state; get_state /
set_state hit ttp_rule_state with the same expires_at auto-revert and
bus-event semantics as the FS backend; subscribe_changes returns a
per-subscriber queue. State persists across process restarts — the
swarm property the FS backend deliberately doesn't have.
Also lands two swarm-mode helpers:
- sync_from_filesystem(fs_store) — master-side, subscribes to a
FilesystemRuleStore and projects each RuleChange onto a ttp_rule
upsert/delete.
- tail_db(poll_interval) — worker-side, watermark poll over
ttp_rule.updated_at; emits RuleChange("definition", ...) for each
row that moved.
Why: swarm mode needs rule definitions and operator state to
propagate across hosts. The filesystem backend (E.3.5) was the
single-host-dev variant; this one survives restart and serves N
workers from a shared DB.
Notes:
- DatabaseRuleStore() with no args lazy-inits an in-memory SQLite
repo so the conformance fixture works without test plumbing. In
production the worker bootstrap (E.3.14) passes an explicit repo.
- The conftest.py rule_store fixture became async (pytest_asyncio),
per-backend creates/initializes a SQLite repo for the DB run.
- Adds a `seed_rule(store, rule_id, yaml)` helper to bridge backend
semantics: drop a YAML file (FS) vs insert a ttp_rule row (DB).
Used by the parametrized load_compiled conformance test.
- Late-bound _tracer() in both backends (was module-level get_tracer
binding) so test_tracing's monkeypatch of decnet.telemetry.get_tracer
actually affects span output.
xfails flipped: tests/ttp/store/test_database.py set_state-writes-to-
ttp_rule_state + filesystem-to-DB sync; tests/ttp/store/test_conformance.py
DB-side load_compiled / set_state isolation / round-trip / per-rule
fan-out / expired-state revert / set_state failure / get_state default
(was xfail-only-on-DB); tests/ttp/test_tracing.py set_state span
hierarchy.
Tests: 208 passed, 25 xfailed (gated on E.3.7 + lifters).
mypy: clean on all touched files.
This commit is contained in:
@@ -2,8 +2,7 @@
|
||||
|
||||
Per ``development/TTP_TAGGING.md`` §E.2.14b: the database backend's
|
||||
tests run against BOTH SQLite and MySQL via the ``db_backends``
|
||||
fixture in :mod:`tests.web.db.conftest`. Today the database store's
|
||||
methods raise ``NotImplementedError`` so most assertions xfail.
|
||||
fixture in :mod:`tests.web.db.conftest`.
|
||||
|
||||
The cross-backend conformance assertions (load_compiled equality,
|
||||
get_state default, set_state isolation/round-trip,
|
||||
@@ -13,16 +12,26 @@ via the parametrized ``rule_store`` fixture in :mod:`conftest`.
|
||||
|
||||
This module pins behavior that's *only* meaningful for the database
|
||||
backend — specifically the propagation of state via the underlying
|
||||
``ttp_rule_state`` table, which conformance tests exercise but don't
|
||||
introspect at the SQL level.
|
||||
``ttp_rule_state`` table and the master-side filesystem→DB sync
|
||||
helper.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from decnet.ttp.impl.rule_engine import CompiledRule
|
||||
from decnet.ttp.store.base import RuleState
|
||||
from decnet.ttp.store.impl.database import DatabaseRuleStore
|
||||
from decnet.web.db.models import TTPRule, TTPRuleState
|
||||
from sqlalchemy import select as sa_select
|
||||
from sqlmodel import col
|
||||
|
||||
|
||||
def test_database_store_constructs_without_platform_guard() -> None:
|
||||
@@ -52,29 +61,148 @@ def test_async_methods_are_coroutines() -> None:
|
||||
assert inspect.iscoroutinefunction(member)
|
||||
|
||||
|
||||
@pytest.mark.xfail(
|
||||
strict=True,
|
||||
reason="impl phase E.3.6 — DatabaseRuleStore needs to write "
|
||||
"into ttp_rule_state via the repository session; today the "
|
||||
"method body raises NotImplementedError",
|
||||
)
|
||||
async def test_set_state_writes_to_ttp_rule_state_table() -> None:
|
||||
@pytest_asyncio.fixture
|
||||
async def db_store(tmp_path: Path) -> Any:
|
||||
from decnet.web.db.sqlite.repository import SQLiteRepository
|
||||
|
||||
repo = SQLiteRepository(db_path=str(tmp_path / "ttp_db_store.db"))
|
||||
await repo.initialize()
|
||||
store = DatabaseRuleStore(repo=repo)
|
||||
try:
|
||||
yield store
|
||||
finally:
|
||||
try:
|
||||
await repo.engine.dispose()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
|
||||
async def test_set_state_writes_to_ttp_rule_state_table(
|
||||
db_store: DatabaseRuleStore, tmp_path: Path,
|
||||
) -> None:
|
||||
"""``set_state`` writes / upserts a row in the ``ttp_rule_state``
|
||||
table. After the write, a fresh ``DatabaseRuleStore`` instance
|
||||
sees the same value via :meth:`get_state` (state survives
|
||||
process restart — that's the whole point of the database
|
||||
backend over the filesystem one)."""
|
||||
pytest.fail("DatabaseRuleStore.set_state not yet implemented")
|
||||
table. After the write, a fresh :class:`DatabaseRuleStore`
|
||||
instance pointing at the same DB sees the same value via
|
||||
:meth:`get_state` — state survives process restart, which is the
|
||||
whole point of the DB backend over the filesystem one."""
|
||||
await db_store.set_state(
|
||||
"R0001",
|
||||
RuleState(state="disabled", reason="probation"),
|
||||
set_by="anti",
|
||||
)
|
||||
repo = db_store._repo
|
||||
assert repo is not None
|
||||
async with repo._session() as session: # type: ignore[attr-defined]
|
||||
row = (
|
||||
await session.execute(
|
||||
sa_select(TTPRuleState).where(
|
||||
col(TTPRuleState.rule_id) == "R0001",
|
||||
),
|
||||
)
|
||||
).scalars().first()
|
||||
assert row is not None
|
||||
assert row.state == "disabled"
|
||||
assert row.reason == "probation"
|
||||
assert row.set_by == "anti"
|
||||
|
||||
# Fresh store instance against the same engine — state survives.
|
||||
fresh = DatabaseRuleStore(repo=repo)
|
||||
state = await fresh.get_state("R0001")
|
||||
assert state.state == "disabled"
|
||||
assert state.reason == "probation"
|
||||
|
||||
|
||||
@pytest.mark.xfail(
|
||||
strict=True,
|
||||
reason="impl phase E.3.6 — master-side filesystem→DB sync of "
|
||||
"ttp_rule lands with the swarm-mode wiring",
|
||||
)
|
||||
async def test_filesystem_to_db_sync_populates_ttp_rule() -> None:
|
||||
async def test_filesystem_to_db_sync_populates_ttp_rule(
|
||||
db_store: DatabaseRuleStore, tmp_path: Path,
|
||||
) -> None:
|
||||
"""In swarm mode, the master watches ``./rules/ttp/`` and
|
||||
syncs each YAML edit into the ``ttp_rule`` table; workers
|
||||
tail the DB. This test pins the half of the contract that
|
||||
only the database backend implements."""
|
||||
pytest.fail("master-side fs→DB sync not yet implemented")
|
||||
only the database backend implements: a CompiledRule fed to
|
||||
:meth:`upsert_rule` lands as a ``ttp_rule`` row whose
|
||||
``yaml_content`` round-trips through :meth:`load_compiled`."""
|
||||
compiled = CompiledRule(
|
||||
rule_id="R0001",
|
||||
rule_version=1,
|
||||
name="brute force ssh",
|
||||
applies_to=frozenset({"command"}),
|
||||
match_spec={"pattern": "hydra"},
|
||||
emits=(("T1110", None),),
|
||||
evidence_fields=("matched_tokens",),
|
||||
state=RuleState(),
|
||||
)
|
||||
await db_store.upsert_rule(
|
||||
compiled,
|
||||
source_path="./rules/ttp/R0001.yaml",
|
||||
updated_by="filesystem",
|
||||
)
|
||||
repo = db_store._repo
|
||||
assert repo is not None
|
||||
async with repo._session() as session: # type: ignore[attr-defined]
|
||||
row = (
|
||||
await session.execute(
|
||||
sa_select(TTPRule).where(col(TTPRule.rule_id) == "R0001"),
|
||||
)
|
||||
).scalars().first()
|
||||
assert row is not None
|
||||
assert row.rule_version == 1
|
||||
assert row.updated_by == "filesystem"
|
||||
# Round-trip through load_compiled.
|
||||
loaded = await db_store.load_compiled()
|
||||
assert len(loaded) == 1
|
||||
assert loaded[0].rule_id == "R0001"
|
||||
assert loaded[0].emits == (("T1110", None),)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform != "linux",
|
||||
reason="FilesystemRuleStore is Linux-only (inotify dep)",
|
||||
)
|
||||
async def test_sync_from_filesystem_propagates_changes(
|
||||
db_store: DatabaseRuleStore, tmp_path: Path,
|
||||
) -> None:
|
||||
"""The master-side helper :meth:`sync_from_filesystem` projects
|
||||
every :class:`RuleChange` from a :class:`FilesystemRuleStore`
|
||||
onto a ``ttp_rule`` upsert. Validates the swarm-mode
|
||||
bootstrap path: master watches disk, workers tail DB."""
|
||||
import asyncio # noqa: PLC0415
|
||||
from decnet.ttp.store.impl.filesystem import FilesystemRuleStore # noqa: PLC0415
|
||||
|
||||
rules_dir = tmp_path / "rules"
|
||||
rules_dir.mkdir()
|
||||
fs_store = FilesystemRuleStore(rules_dir=rules_dir)
|
||||
|
||||
sync_task = asyncio.create_task(
|
||||
db_store.sync_from_filesystem(fs_store, updated_by="git"),
|
||||
)
|
||||
try:
|
||||
async with fs_store:
|
||||
await asyncio.sleep(0.05)
|
||||
(rules_dir / "R0042.yaml").write_text(
|
||||
"""rule_id: R0042
|
||||
rule_version: 1
|
||||
name: test
|
||||
applies_to: [command]
|
||||
match:
|
||||
pattern: 'whoami'
|
||||
emits:
|
||||
- technique_id: T1033
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
# Give the sync task a moment to project the change.
|
||||
for _ in range(20):
|
||||
await asyncio.sleep(0.05)
|
||||
loaded = await db_store.load_compiled()
|
||||
if any(c.rule_id == "R0042" for c in loaded):
|
||||
break
|
||||
else:
|
||||
pytest.fail("sync_from_filesystem did not project the edit")
|
||||
ids = {c.rule_id for c in loaded}
|
||||
assert "R0042" in ids
|
||||
finally:
|
||||
sync_task.cancel()
|
||||
try:
|
||||
await sync_task
|
||||
except (asyncio.CancelledError, Exception): # noqa: BLE001
|
||||
pass
|
||||
|
||||
Reference in New Issue
Block a user