feat: switch to JSON-based log ingestion for higher reliability

decnet/web/ingester.py

@@ -1,36 +1,36 @@
 import asyncio
 import os
 import logging
+import json
 from typing import Any
 from pathlib import Path
 
-from decnet.correlation.parser import parse_line
 from decnet.web.repository import BaseRepository
 
 logger = logging.getLogger("decnet.web.ingester")
 
 async def log_ingestion_worker(repo: BaseRepository) -> None:
     """
-    Background task that tails the DECNET_INGEST_LOG_FILE and
-    inserts parsed LogEvents into the SQLite repository.
+    Background task that tails the DECNET_INGEST_LOG_FILE.json and
+    inserts structured JSON logs into the SQLite repository.
     """
-    log_file_path_str = os.environ.get("DECNET_INGEST_LOG_FILE")
-    if not log_file_path_str:
+    base_log_file = os.environ.get("DECNET_INGEST_LOG_FILE")
+    if not base_log_file:
         logger.warning("DECNET_INGEST_LOG_FILE not set. Log ingestion disabled.")
         return
 
-    log_path = Path(log_file_path_str)
+    json_log_path = Path(base_log_file).with_suffix(".json")
     position = 0
     
-    logger.info(f"Starting log ingestion from {log_path}")
+    logger.info(f"Starting JSON log ingestion from {json_log_path}")
 
     while True:
         try:
-            if not log_path.exists():
+            if not json_log_path.exists():
                 await asyncio.sleep(2)
                 continue
                 
-            stat = log_path.stat()
+            stat = json_log_path.stat()
             if stat.st_size < position:
                 # File rotated or truncated
                 position = 0
@@ -40,26 +40,26 @@ async def log_ingestion_worker(repo: BaseRepository) -> None:
                 await asyncio.sleep(1)
                 continue
                 
-            with open(log_path, "r", encoding="utf-8", errors="replace") as f:
+            with open(json_log_path, "r", encoding="utf-8", errors="replace") as f:
                 f.seek(position)
                 while True:
                     line = f.readline()
                     if not line:
                         break # EOF reached
                     
-                    event = parse_line(line)
-                    if event:
-                        log_data = {
-                            "timestamp": event.timestamp.strftime("%Y-%m-%d %H:%M:%S"),
-                            "decky": event.decky,
-                            "service": event.service,
-                            "event_type": event.event_type,
-                            "attacker_ip": event.attacker_ip or "Unknown",
-                            "raw_line": event.raw
-                        }
+                    if not line.endswith('\n'):
+                        # Partial line read, don't process yet, don't advance position
+                        break
+
+                    try:
+                        log_data = json.loads(line.strip())
                         await repo.add_log(log_data)
-                        
-                position = f.tell()
+                    except json.JSONDecodeError:
+                        logger.error(f"Failed to decode JSON log line: {line}")
+                        continue
+                    
+                    # Update position after successful line read
+                    position = f.tell()
                 
         except Exception as e:
             logger.error(f"Error in log ingestion worker: {e}")