merge: testing → main (reconcile 2-week divergence)

decnet/correlation/parser.py

@@ -6,7 +6,7 @@ the fields needed for cross-decky correlation: attacker IP, decky name,
 service, event type, and timestamp.
 
 Log format (produced by decnet.logging.syslog_formatter):
-  <PRI>1 TIMESTAMP HOSTNAME APP-NAME - MSGID [decnet@55555 k1="v1" k2="v2"] [MSG]
+  <PRI>1 TIMESTAMP HOSTNAME APP-NAME - MSGID [relay@55555 k1="v1" k2="v2"] [MSG]
 
 The attacker IP may appear under several field names depending on service:
   src_ip  — ftp, smtp, http, most services
@@ -17,8 +17,9 @@ The attacker IP may appear under several field names depending on service:
 from __future__ import annotations
 
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime
+from typing import Literal
 
 # RFC 5424 line structure
 _RFC5424_RE = re.compile(
@@ -31,14 +32,31 @@ _RFC5424_RE = re.compile(
     r"(.+)$",       # 5: SD element + optional MSG
 )
 
-# Structured data block: [decnet@55555 k="v" ...]
-_SD_BLOCK_RE = re.compile(r'\[decnet@55555\s+(.*?)\]', re.DOTALL)
+# Structured data block: [relay@55555 k="v" ...]
+_SD_BLOCK_RE = re.compile(r'\[relay@55555\s+(.*?)\]', re.DOTALL)
 
 # Individual param: key="value" (with escaped chars inside value)
 _PARAM_RE = re.compile(r'(\w+)="((?:[^"\\]|\\.)*)"')
 
 # Field names to probe for attacker IP, in priority order
-_IP_FIELDS = ("src_ip", "src", "client_ip", "remote_ip", "ip")
+_IP_FIELDS = ("src_ip", "src", "client_ip", "remote_ip", "remote_addr", "target_ip", "ip")
+
+# Native syslog producers (sshd, pam_unix routed through rsyslog) emit
+# free prose with no SD block. Pull the remote address out of idiomatic
+# anchors first ("from <ip>", "rhost=<ip>"), then fall back to the first
+# IPv4 in the line. Anchored matches keep us from picking the local
+# listener in "Connection from X port Y on Z port 22".
+_IPV4 = r"\d{1,3}(?:\.\d{1,3}){3}"
+_IPV6 = r"[0-9a-fA-F:]+:[0-9a-fA-F:]+"
+_IP_RE = rf"(?:{_IPV4}|{_IPV6})"
+_MSG_IP_ANCHORED_RE = re.compile(
+    rf"\b(?:from|rhost[:=]|client[:=]|src[:=])\s*({_IP_RE})",
+    re.IGNORECASE,
+)
+_MSG_IP_BARE_RE = re.compile(rf"\b({_IPV4})\b")
+
+
+EventKind = Literal["attacker", "mutation"]
 
 
 @dataclass
@@ -52,6 +70,12 @@ class LogEvent:
     attacker_ip: str | None  # extracted from SD params; None if not present
     fields: dict[str, str]   # all structured data params
     raw: str            # original log line (stripped)
+    # ``attacker`` = service-emitted event keyed on a source IP (the
+    # existing correlation input).  ``mutation`` = ``mutator`` worker
+    # event — same RFC 5424 wire format but routed into a separate
+    # per-decky index so substrate transitions can be interleaved into
+    # attacker traversals without polluting the per-IP event stream.
+    kind: EventKind = field(default="attacker")
 
 
 def _parse_sd_params(sd_rest: str) -> dict[str, str]:
@@ -66,10 +90,17 @@ def _parse_sd_params(sd_rest: str) -> dict[str, str]:
     return params
 
 
-def _extract_attacker_ip(fields: dict[str, str]) -> str | None:
+def _extract_attacker_ip(fields: dict[str, str], msg: str = "") -> str | None:
     for fname in _IP_FIELDS:
         if fname in fields:
             return fields[fname]
+    if msg:
+        anchored = _MSG_IP_ANCHORED_RE.search(msg)
+        if anchored:
+            return anchored.group(1)
+        bare = _MSG_IP_BARE_RE.search(msg)
+        if bare:
+            return bare.group(1)
     return None
 
 
@@ -99,7 +130,20 @@ def parse_line(line: str) -> LogEvent | None:
         return None
 
     fields = _parse_sd_params(sd_rest)
-    attacker_ip = _extract_attacker_ip(fields)
+    if sd_rest.startswith("-"):
+        msg = sd_rest[1:].lstrip()
+    else:
+        tail = re.search(r'\]\s+(.+)$', sd_rest)
+        msg = tail.group(1).strip() if tail else ""
+    attacker_ip = _extract_attacker_ip(fields, msg)
+
+    # Mutator-emitted transitions arrive on the same ingest stream but
+    # belong in the substrate-state index, not the per-IP attacker one.
+    kind: EventKind = (
+        "mutation"
+        if service == "mutator" and event_type == "decky_mutated"
+        else "attacker"
+    )
 
     return LogEvent(
         timestamp=timestamp,
@@ -109,4 +153,5 @@ def parse_line(line: str) -> LogEvent | None:
         attacker_ip=attacker_ip,
         fields=fields,
         raw=line,
+        kind=kind,
     )