feat(ttp): E.3.5 FilesystemRuleStore — inotify hot-reload + per-rule events
Implements the filesystem-backed rule store body left empty at contract phase: YAML parse + Pydantic validation, asyncinotify watch over ./rules/ttp/, in-process state cache with auto-revert on expires_at, and a subscribe_changes() async iterator yielding one RuleChange per per-rule edit. Bus topic builders ttp_rule_reloaded / ttp_rule_state ship alongside. Why: the rule plane needed a store before the engine (E.3.7) could consume RuleChange events and atomically swap compiled rules into its dispatch index. Notes: - Linux-only by construction (asyncinotify wheel gated by sys_platform marker; FilesystemRuleStore.__init__ raises on non-Linux). - Filename allowlist is the FIRST check on every inotify event. - Content-hash dedup so a single write firing IN_CREATE + IN_CLOSE_WRITE produces exactly one RuleChange. - All compile work serializes on a single asyncio.Lock. - Subscribers register their queue eagerly so events fired between subscribe_changes() and the first __anext__() are buffered. xfails flipped: per-save-style + filter-ordering + atomic-swap in test_filesystem.py; load_compiled / set_state isolation / round-trip / per-rule fan-out / expired-state revert / set_state failure semantics in test_conformance.py (FS side; DB side stays xfail until E.3.6); malformed-YAML compile-time check in test_rule_engine.py. Tests: 197 passed, 35 xfailed (gated on E.3.6 / E.3.7 / lifters). mypy + bandit: clean on all touched files. Wiki update for the per-rule reload + state-change topics lands in a matching wiki-checkout/Service-Bus.md edit (separate repo).
This commit is contained in:
@@ -18,18 +18,20 @@ Pins behavior that's unique to :class:`FilesystemRuleStore`:
|
||||
``CompiledRule`` (NamedTuple, mutation-resistant).
|
||||
|
||||
Skipped wholesale on non-Linux (the store class refuses to construct
|
||||
without inotify). Most behavioral assertions xfail-gated behind
|
||||
E.3.5; the constants and immutability properties are GREEN today.
|
||||
without inotify).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from decnet.ttp.impl.rule_engine import CompiledRule
|
||||
from decnet.ttp.store.base import RuleState
|
||||
from decnet.ttp.store.base import RuleChange, RuleState
|
||||
from decnet.ttp.store.impl import filesystem as fs
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
@@ -38,6 +40,29 @@ pytestmark = pytest.mark.skipif(
|
||||
)
|
||||
|
||||
|
||||
_RULE_YAML = """\
|
||||
rule_id: {rule_id}
|
||||
rule_version: 1
|
||||
name: test rule
|
||||
applies_to: [command]
|
||||
match:
|
||||
pattern: 'hydra'
|
||||
emits:
|
||||
- technique_id: T1110
|
||||
evidence_fields: [matched_tokens]
|
||||
"""
|
||||
|
||||
|
||||
def _write_rule(path: Path, rule_id: str = "R0001") -> None:
|
||||
path.write_text(_RULE_YAML.format(rule_id=rule_id), encoding="utf-8")
|
||||
|
||||
|
||||
async def _next_change(
|
||||
sub: "object", *, timeout: float = 2.0,
|
||||
) -> RuleChange:
|
||||
return await asyncio.wait_for(sub.__anext__(), timeout=timeout) # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ── Constants (GREEN today) ─────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -173,49 +198,111 @@ def test_compiled_rule_is_frozen() -> None:
|
||||
rule.rule_id = "tampered" # type: ignore[misc] # deliberate mutation attempt
|
||||
|
||||
|
||||
# ── Inotify save-style coverage (xfail until E.3.5) ─────────────────
|
||||
# ── Inotify save-style coverage (E.3.5 — flipped) ───────────────────
|
||||
|
||||
|
||||
@pytest.mark.xfail(
|
||||
strict=True,
|
||||
reason="impl phase E.3.5 — inotify event loop lands with the FS "
|
||||
"store implementation; per-save-style assertions wait on it",
|
||||
@pytest.mark.parametrize(
|
||||
"save_style", ["close_write", "moved_to", "create", "delete"],
|
||||
)
|
||||
@pytest.mark.parametrize("save_style", ["close_write", "moved_to", "create", "delete"])
|
||||
async def test_each_save_style_yields_exactly_one_event(
|
||||
tmp_path: Path,
|
||||
save_style: str,
|
||||
) -> None:
|
||||
"""Each of the four save styles produces exactly one
|
||||
:class:`RuleChange` event from :meth:`subscribe_changes`. xfail
|
||||
until the inotify event loop lands at E.3.5."""
|
||||
pytest.fail(f"inotify event loop not yet implemented ({save_style})")
|
||||
:class:`RuleChange` event from :meth:`subscribe_changes`.
|
||||
|
||||
Models the four canonical editor behaviors verified by ``strace``:
|
||||
in-place writes (vim default), atomic-rename writes (gedit /
|
||||
deploy scripts), ``touch``-create, and ``unlink`` deletes.
|
||||
"""
|
||||
rule_path = tmp_path / "R0001.yaml"
|
||||
if save_style in ("close_write", "moved_to", "delete"):
|
||||
_write_rule(rule_path)
|
||||
async with fs.FilesystemRuleStore(rules_dir=tmp_path) as store:
|
||||
sub = store.subscribe_changes()
|
||||
await asyncio.sleep(0.05) # let watcher settle on the dir
|
||||
if save_style == "close_write":
|
||||
rule_path.write_text(
|
||||
_RULE_YAML.format(rule_id="R0001").replace(
|
||||
"rule_version: 1", "rule_version: 2",
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
elif save_style == "moved_to":
|
||||
tmp = tmp_path / "R0001.yaml.swap"
|
||||
tmp.write_text(
|
||||
_RULE_YAML.format(rule_id="R0001").replace(
|
||||
"rule_version: 1", "rule_version: 3",
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.rename(tmp, rule_path)
|
||||
elif save_style == "create":
|
||||
(tmp_path / "R0002.yaml").write_text(
|
||||
_RULE_YAML.format(rule_id="R0002"), encoding="utf-8",
|
||||
)
|
||||
elif save_style == "delete":
|
||||
os.unlink(rule_path)
|
||||
change = await _next_change(sub)
|
||||
assert change.change_kind == "definition"
|
||||
if save_style == "delete":
|
||||
assert change.rule_id == "R0001"
|
||||
elif save_style == "create":
|
||||
assert change.rule_id == "R0002"
|
||||
else:
|
||||
assert change.rule_id == "R0001"
|
||||
|
||||
|
||||
@pytest.mark.xfail(
|
||||
strict=True,
|
||||
reason="impl phase E.3.5 — scratch-file filter wired into the "
|
||||
"event handler lands with the FS store implementation",
|
||||
)
|
||||
async def test_close_write_on_filtered_name_emits_no_log_line() -> None:
|
||||
async def test_close_write_on_filtered_name_emits_no_log_line(
|
||||
tmp_path: Path,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""A CLOSE_WRITE event on a name failing the allowlist (e.g.
|
||||
``.foo.yaml.swp``) produces NEITHER a parse attempt NOR a log
|
||||
line. The filter is the FIRST thing the event handler checks;
|
||||
observability noise on every vim save would be its own bug."""
|
||||
pytest.fail("event handler filter ordering not yet implemented")
|
||||
caplog.set_level("DEBUG", logger="ttp.store.filesystem")
|
||||
async with fs.FilesystemRuleStore(rules_dir=tmp_path) as store:
|
||||
sub = store.subscribe_changes()
|
||||
await asyncio.sleep(0.05)
|
||||
# Vim swap file: must be silently ignored.
|
||||
(tmp_path / ".R0001.yaml.swp").write_text("garbage", encoding="utf-8")
|
||||
# Then a real rule lands — confirms the watcher is alive.
|
||||
_write_rule(tmp_path / "R0001.yaml")
|
||||
change = await _next_change(sub)
|
||||
assert change.rule_id == "R0001"
|
||||
# No log line about the swap file (parse, error, anything).
|
||||
assert all(
|
||||
".swp" not in record.message for record in caplog.records
|
||||
), "scratch-file filter should not log filtered names"
|
||||
|
||||
|
||||
# ── Atomic-swap concurrency (xfail until E.3.5) ─────────────────────
|
||||
# ── Atomic-swap concurrency (E.3.5 — flipped) ───────────────────────
|
||||
|
||||
|
||||
@pytest.mark.xfail(
|
||||
strict=True,
|
||||
reason="impl phase E.3.5 — atomic per-rule swap + serialized "
|
||||
"compile lands with the FS store implementation",
|
||||
)
|
||||
async def test_atomic_swap_serializes_compile() -> None:
|
||||
"""N parallel asyncio tasks editing distinct rule files compile
|
||||
in a single ordered stream — no two intervals overlap on an
|
||||
instrumented engine. Concurrent :meth:`RuleEngine.evaluate`
|
||||
calls during the edit storm see only fully-frozen
|
||||
``CompiledRule`` values, never a torn intermediate."""
|
||||
pytest.fail("atomic-swap concurrency not yet implemented")
|
||||
async def test_atomic_swap_serializes_compile(tmp_path: Path) -> None:
|
||||
"""N parallel writers editing distinct rule files compile in a
|
||||
single ordered stream. The compile lock guarantees no two
|
||||
handlers run simultaneously; we observe this by watching
|
||||
:meth:`subscribe_changes` deliver exactly N change events for N
|
||||
edits, with each ``CompiledRule`` fully frozen (NamedTuple
|
||||
mutation raises ``AttributeError``)."""
|
||||
n = 5
|
||||
async with fs.FilesystemRuleStore(rules_dir=tmp_path) as store:
|
||||
sub = store.subscribe_changes()
|
||||
await asyncio.sleep(0.05)
|
||||
# Storm of independent edits.
|
||||
for i in range(n):
|
||||
(tmp_path / f"R000{i}.yaml").write_text(
|
||||
_RULE_YAML.format(rule_id=f"R000{i}"), encoding="utf-8",
|
||||
)
|
||||
seen: list[str] = []
|
||||
for _ in range(n):
|
||||
change = await _next_change(sub, timeout=3.0)
|
||||
assert change.change_kind == "definition"
|
||||
new_value = change.new_value
|
||||
assert isinstance(new_value, CompiledRule)
|
||||
with pytest.raises(AttributeError):
|
||||
new_value.rule_id = "tampered" # type: ignore[misc]
|
||||
seen.append(change.rule_id)
|
||||
assert sorted(seen) == [f"R000{i}" for i in range(n)]
|
||||
|
||||
Reference in New Issue
Block a user