anti
48f486ac97
- Core Telegram monitoring pipeline (scraper, processor, notifier, downloaders) - Textual TUI frontend with thread-safe event bus - SQLite persistence, severity scoring, dedup cache - Fixed ULP parser: handles https:// truncation, port+path URLs, semicolon separator - Test suite: 88 tests across scorer, cache, database, processor
1017 lines
40 KiB
Python
1017 lines
40 KiB
Python
"""
|
||
tui.py — Textual TUI for the ULP credential monitor.
|
||
|
||
Layout (main screen):
|
||
┌──────────────────────────────────┬──────────────────────────────────┐
|
||
│ 📥 Downloads │ 🎯 Hits [N] │
|
||
│ (live tdl output + status log) │ (color-coded hit log) │
|
||
├──────────────────────────────────┴──────────────────────────────────┤
|
||
│ 📊 Stats bar (live DB counters, auto-refresh every 10 s) │
|
||
├─────────────────────────────────────────────────────────────────────┤
|
||
│ 📡 Channels (add / remove entries; applied immediately) │
|
||
└─────────────────────────────────────────────────────────────────────┘
|
||
│ Footer (keybindings) │
|
||
└─────────────────────────────────────────────────────────────────────┘
|
||
|
||
Additional screens (push/pop via keybindings):
|
||
• SearchScreen — full-text search across hits DB [s]
|
||
• HitsDBScreen — paginated recent / severity viewer [h]
|
||
• KeywordsScreen — live-edit TARGET_KEYWORDS regex list [k]
|
||
|
||
Architecture:
|
||
- The entire bot backend runs as a Textual Worker (asyncio task inside the
|
||
TUI event loop — no threading needed).
|
||
- A second Worker runs _bus_consumer(), reading events from tui_events.queue
|
||
and dispatching to the right panel.
|
||
- Channel add/remove from the UI immediately re-registers Telethon handlers
|
||
via asyncio.Event signalling into the bot worker.
|
||
- tdl output is piped (not terminal-inherited) and relayed via EvTdlOutput
|
||
into the download panel's RichLog.
|
||
- StatsPanel polls database.stats() every 10 s via set_interval().
|
||
- Keyword changes are applied in-memory immediately (scorer caches rebuilt);
|
||
NOT auto-persisted to config.py — a notice banner reminds the user.
|
||
- Live patterns are recompiled from config.TARGET_KEYWORDS on every message
|
||
so keyword changes take effect without a handler restart.
|
||
"""
|
||
|
||
import asyncio
|
||
import logging
|
||
import queue
|
||
import shutil
|
||
import threading
|
||
from datetime import datetime, timezone
|
||
|
||
from textual.app import App, ComposeResult, Screen
|
||
from textual.binding import Binding
|
||
from textual.containers import Horizontal, Vertical
|
||
from textual.widgets import (
|
||
Footer, Header, Label, Input, Button,
|
||
ListView, ListItem, RichLog, DataTable, Static,
|
||
)
|
||
from textual.reactive import reactive
|
||
|
||
from . import events as bus
|
||
from config import WATCHED_CHANNELS, SESSION_NAME
|
||
import config
|
||
|
||
log = logging.getLogger(__name__)
|
||
|
||
# ─── Colour maps ──────────────────────────────────────────────────────────────
|
||
|
||
SEV_COLOUR = {
|
||
"CRITICAL": "bold red",
|
||
"HIGH": "bold orange1",
|
||
"MEDIUM": "bold yellow",
|
||
"LOW": "bold green",
|
||
}
|
||
SEV_EMOJI = {
|
||
"CRITICAL": "🔴", "HIGH": "🟠", "MEDIUM": "🟡", "LOW": "🟢",
|
||
}
|
||
DL_COLOUR = {
|
||
"queued": "dim white",
|
||
"downloading": "bold cyan",
|
||
"done_tdl": "bold green",
|
||
"done_tel": "green",
|
||
"failed": "bold red",
|
||
}
|
||
DL_ICON = {
|
||
"queued": "⏳", "downloading": "⬇️ ",
|
||
"done_tdl": "✅", "done_tel": "✅", "failed": "❌",
|
||
}
|
||
|
||
|
||
def _now() -> str:
|
||
return datetime.now(timezone.utc).strftime("%H:%M:%S")
|
||
|
||
|
||
# ─── Download panel ───────────────────────────────────────────────────────────
|
||
|
||
class DownloadPanel(Vertical):
|
||
"""
|
||
Left panel — two sub-logs stacked vertically:
|
||
• top: tdl raw output (stripped ANSI), scrolling
|
||
• bottom: our own structured status entries
|
||
"""
|
||
|
||
DEFAULT_CSS = """
|
||
DownloadPanel {
|
||
border: solid $accent;
|
||
height: 100%;
|
||
width: 1fr;
|
||
}
|
||
DownloadPanel Label.panel-title {
|
||
background: $accent;
|
||
color: $text;
|
||
padding: 0 1;
|
||
width: 100%;
|
||
}
|
||
DownloadPanel Label.sub-title {
|
||
background: $surface;
|
||
color: $text-muted;
|
||
padding: 0 1;
|
||
width: 100%;
|
||
}
|
||
DownloadPanel RichLog {
|
||
padding: 0 1;
|
||
}
|
||
#tdl-out {
|
||
height: 1fr;
|
||
border-bottom: dashed $accent-darken-2;
|
||
}
|
||
#dl-log {
|
||
height: 1fr;
|
||
}
|
||
"""
|
||
|
||
def compose(self) -> ComposeResult:
|
||
yield Label("📥 Downloads", classes="panel-title")
|
||
yield Label(" tdl output", classes="sub-title")
|
||
yield RichLog(highlight=False, markup=False, wrap=True, id="tdl-out")
|
||
yield Label(" status", classes="sub-title")
|
||
yield RichLog(highlight=True, markup=True, wrap=True, id="dl-log")
|
||
|
||
def tdl_line(self, line: str) -> None:
|
||
self.query_one("#tdl-out", RichLog).write(line)
|
||
|
||
def queued(self, filename: str, size_mb: float, source: str,
|
||
password: str | None) -> None:
|
||
pw = f" 🔑 [dim]{password}[/dim]" if password else ""
|
||
self.query_one("#dl-log", RichLog).write(
|
||
f"[{DL_COLOUR['queued']}]{DL_ICON['queued']} {_now()} "
|
||
f"{filename}[/{DL_COLOUR['queued']}]"
|
||
f" [dim]{size_mb:.1f} MB {source}[/dim]{pw}"
|
||
)
|
||
|
||
def status(self, filename: str, state: str, via: str = "") -> None:
|
||
colour = DL_COLOUR.get(state, "white")
|
||
icon = DL_ICON.get(state, "•")
|
||
suffix = f" [dim]via {via}[/dim]" if via else ""
|
||
self.query_one("#dl-log", RichLog).write(
|
||
f" [dim]↳[/dim] [{colour}]{icon} {filename}[/{colour}]{suffix}"
|
||
)
|
||
|
||
def clear_logs(self) -> None:
|
||
self.query_one("#tdl-out", RichLog).clear()
|
||
self.query_one("#dl-log", RichLog).clear()
|
||
|
||
|
||
# ─── Hits panel ───────────────────────────────────────────────────────────────
|
||
|
||
class HitsPanel(Vertical):
|
||
"""Right panel — scrollable color-coded hit log with live counter badge."""
|
||
|
||
hit_count: reactive[int] = reactive(0)
|
||
|
||
DEFAULT_CSS = """
|
||
HitsPanel {
|
||
border: solid $error;
|
||
height: 100%;
|
||
width: 1fr;
|
||
}
|
||
HitsPanel Label.panel-title {
|
||
background: $error;
|
||
color: $text;
|
||
padding: 0 1;
|
||
width: 100%;
|
||
}
|
||
HitsPanel RichLog {
|
||
height: 1fr;
|
||
padding: 0 1;
|
||
}
|
||
"""
|
||
|
||
def compose(self) -> ComposeResult:
|
||
yield Label("🎯 Hits", classes="panel-title")
|
||
yield RichLog(highlight=True, markup=True, wrap=True, id="hits-log")
|
||
|
||
def watch_hit_count(self, count: int) -> None:
|
||
self.query_one(".panel-title", Label).update(f"🎯 Hits [{count}]")
|
||
|
||
def add_hit(self, severity: str, raw: str, source: str,
|
||
filename: str, reasons: list[str]) -> None:
|
||
colour = SEV_COLOUR.get(severity, "white")
|
||
emoji = SEV_EMOJI.get(severity, "⚪")
|
||
self.query_one("#hits-log", RichLog).write(
|
||
f"{emoji} [{colour}]{severity}[/{colour}] [dim]{_now()}[/dim]\n"
|
||
f" [bold]{raw}[/bold]\n"
|
||
f" [dim]↳ {' | '.join(reasons)}[/dim]\n"
|
||
f" [dim]📁 {filename} 📢 {source}[/dim]"
|
||
)
|
||
self.hit_count += 1
|
||
|
||
def clear_log(self) -> None:
|
||
self.query_one("#hits-log", RichLog).clear()
|
||
self.hit_count = 0
|
||
|
||
|
||
# ─── Stats panel ──────────────────────────────────────────────────────────────
|
||
|
||
class StatsPanel(Horizontal):
|
||
"""
|
||
Slim bar — shows live DB stats, refreshed every 10 s.
|
||
Also refreshed immediately whenever a new hit arrives.
|
||
"""
|
||
|
||
DEFAULT_CSS = """
|
||
StatsPanel {
|
||
border: solid $primary-darken-2;
|
||
height: 3;
|
||
width: 100%;
|
||
padding: 0 1;
|
||
background: $surface;
|
||
}
|
||
StatsPanel Static {
|
||
width: 1fr;
|
||
content-align: center middle;
|
||
color: $text-muted;
|
||
}
|
||
StatsPanel Static.stat-critical { color: red; }
|
||
StatsPanel Static.stat-high { color: orange; }
|
||
StatsPanel Static.stat-medium { color: yellow; }
|
||
StatsPanel Static.stat-low { color: green; }
|
||
"""
|
||
|
||
def compose(self) -> ComposeResult:
|
||
yield Static("📊 DB Stats", id="stat-label")
|
||
yield Static("🔴 —", classes="stat-critical", id="stat-critical")
|
||
yield Static("🟠 —", classes="stat-high", id="stat-high")
|
||
yield Static("🟡 —", classes="stat-medium", id="stat-medium")
|
||
yield Static("🟢 —", classes="stat-low", id="stat-low")
|
||
yield Static("total: —", id="stat-total")
|
||
yield Static("unique: —", id="stat-unique")
|
||
yield Static("dupes: —", id="stat-dupes")
|
||
yield Static("sources: —", id="stat-sources")
|
||
|
||
def on_mount(self) -> None:
|
||
self.set_interval(10, self.refresh_stats)
|
||
self.refresh_stats()
|
||
|
||
def refresh_stats(self) -> None:
|
||
try:
|
||
from utils.database import stats
|
||
s = stats()
|
||
self.query_one("#stat-critical", Static).update(f"🔴 {s['critical']}")
|
||
self.query_one("#stat-high", Static).update(f"🟠 {s['high']}")
|
||
self.query_one("#stat-medium", Static).update(f"🟡 {s['medium']}")
|
||
self.query_one("#stat-low", Static).update(f"🟢 {s['low']}")
|
||
self.query_one("#stat-total", Static).update(f"total: {s['total']}")
|
||
self.query_one("#stat-unique", Static).update(f"unique: {s['unique']}")
|
||
self.query_one("#stat-dupes", Static).update(f"dupes: {s['duplicates']}")
|
||
self.query_one("#stat-sources", Static).update(f"sources: {s['sources']}")
|
||
except Exception:
|
||
pass # DB not ready yet on first paint
|
||
|
||
|
||
# ─── Channel panel ────────────────────────────────────────────────────────────
|
||
|
||
class ChannelPanel(Vertical):
|
||
"""
|
||
Bottom panel — live-editable channel list.
|
||
|
||
Changes are applied immediately (Telethon handlers are re-registered).
|
||
To make them permanent, edit config.py's WATCHED_CHANNELS manually.
|
||
"""
|
||
|
||
DEFAULT_CSS = """
|
||
ChannelPanel {
|
||
border: solid $warning;
|
||
height: 14;
|
||
width: 100%;
|
||
}
|
||
ChannelPanel Label.panel-title {
|
||
background: $warning;
|
||
color: $text;
|
||
padding: 0 1;
|
||
width: 100%;
|
||
}
|
||
ChannelPanel Horizontal.controls {
|
||
height: 3;
|
||
padding: 0 1;
|
||
}
|
||
ChannelPanel Horizontal.controls Input {
|
||
width: 1fr;
|
||
}
|
||
ChannelPanel Horizontal.controls Button {
|
||
width: auto;
|
||
margin-left: 1;
|
||
}
|
||
ChannelPanel Horizontal.list-row {
|
||
height: 1fr;
|
||
}
|
||
ChannelPanel Horizontal.list-row ListView {
|
||
width: 1fr;
|
||
height: 100%;
|
||
}
|
||
ChannelPanel Horizontal.list-row Button {
|
||
width: 14;
|
||
margin: 0 1;
|
||
}
|
||
"""
|
||
|
||
def __init__(self, initial_channels: list, **kwargs):
|
||
super().__init__(**kwargs)
|
||
self._channels: list[str | int] = list(initial_channels)
|
||
|
||
def compose(self) -> ComposeResult:
|
||
yield Label(
|
||
"📡 Channels — changes apply immediately | edit config.py to persist",
|
||
classes="panel-title",
|
||
)
|
||
with Horizontal(classes="controls"):
|
||
yield Input(placeholder="channel username or -100xxxxxxxxxx", id="ch-input")
|
||
yield Button("➕ Add", id="ch-add", variant="success")
|
||
with Horizontal(classes="list-row"):
|
||
yield ListView(id="ch-list")
|
||
yield Button("🗑 Remove", id="ch-remove", variant="error")
|
||
|
||
def on_mount(self) -> None:
|
||
self._refresh_list()
|
||
|
||
def _refresh_list(self) -> None:
|
||
lv = self.query_one("#ch-list", ListView)
|
||
lv.clear()
|
||
for ch in self._channels:
|
||
lv.append(ListItem(Label(str(ch))))
|
||
|
||
def on_button_pressed(self, event: Button.Pressed) -> None:
|
||
if event.button.id == "ch-add":
|
||
inp = self.query_one("#ch-input", Input)
|
||
raw = inp.value.strip()
|
||
if not raw:
|
||
return
|
||
channel: str | int = int(raw) if raw.lstrip("-").isdigit() else raw
|
||
if channel not in self._channels:
|
||
self._channels.append(channel)
|
||
self._refresh_list()
|
||
bus.post(bus.EvChannelAdded(channel=channel))
|
||
self.app.notify(f"Added: {channel}", severity="information")
|
||
inp.value = ""
|
||
|
||
elif event.button.id == "ch-remove":
|
||
lv = self.query_one("#ch-list", ListView)
|
||
idx = lv.index
|
||
if idx is None or not (0 <= idx < len(self._channels)):
|
||
self.app.notify("Select a channel first", severity="warning")
|
||
return
|
||
removed = self._channels.pop(idx)
|
||
self._refresh_list()
|
||
bus.post(bus.EvChannelRemoved(channel=removed))
|
||
self.app.notify(f"Removed: {removed}", severity="warning")
|
||
|
||
@property
|
||
def channels(self) -> list[str | int]:
|
||
return list(self._channels)
|
||
|
||
|
||
# ─── Search screen ────────────────────────────────────────────────────────────
|
||
|
||
class SearchScreen(Screen):
|
||
"""Full-text search across the hits database (url, username, raw line)."""
|
||
|
||
BINDINGS = [Binding("escape", "dismiss", "Back")]
|
||
|
||
DEFAULT_CSS = """
|
||
SearchScreen { background: $background; }
|
||
SearchScreen Label.screen-title {
|
||
background: $primary;
|
||
color: $text;
|
||
padding: 0 1;
|
||
width: 100%;
|
||
}
|
||
SearchScreen #search-bar {
|
||
height: 3;
|
||
padding: 0 1;
|
||
}
|
||
SearchScreen #search-bar Input { width: 1fr; }
|
||
SearchScreen #search-bar Button { width: 14; margin-left: 1; }
|
||
SearchScreen #result-count { padding: 0 1; color: $text-muted; }
|
||
SearchScreen #results-table { height: 1fr; margin: 0 1 1 1; }
|
||
"""
|
||
|
||
def compose(self) -> ComposeResult:
|
||
yield Header()
|
||
yield Label("🔍 Search Hits Database", classes="screen-title")
|
||
with Horizontal(id="search-bar"):
|
||
yield Input(placeholder="keyword, domain, username, IP…", id="search-input")
|
||
yield Button("Search", id="search-btn", variant="primary")
|
||
yield Label("Enter a keyword and press Search or ↵", id="result-count")
|
||
yield DataTable(id="results-table", zebra_stripes=True, cursor_type="row")
|
||
yield Footer()
|
||
|
||
def on_mount(self) -> None:
|
||
t = self.query_one("#results-table", DataTable)
|
||
t.add_columns("Sev", "Time", "URL", "Username", "Password", "Source", "File")
|
||
self.query_one("#search-input", Input).focus()
|
||
|
||
def on_button_pressed(self, event: Button.Pressed) -> None:
|
||
if event.button.id == "search-btn":
|
||
self._run_search()
|
||
|
||
def on_input_submitted(self, event: Input.Submitted) -> None:
|
||
if event.input.id == "search-input":
|
||
self._run_search()
|
||
|
||
def _run_search(self) -> None:
|
||
kw = self.query_one("#search-input", Input).value.strip()
|
||
if not kw:
|
||
return
|
||
try:
|
||
from utils.database import search
|
||
rows = search(kw)
|
||
except Exception as e:
|
||
self.app.notify(f"Search error: {e}", severity="error")
|
||
return
|
||
|
||
t = self.query_one("#results-table", DataTable)
|
||
t.clear()
|
||
for row in rows:
|
||
emoji = SEV_EMOJI.get(row["severity"], "⚪")
|
||
t.add_row(
|
||
f"{emoji} {row['severity']}",
|
||
row["timestamp"],
|
||
(row["url"] or "")[:45],
|
||
(row["username"] or "")[:30],
|
||
(row["password"] or "")[:20],
|
||
(row["source"] or "")[:20],
|
||
(row["filename"] or "")[:25],
|
||
)
|
||
self.query_one("#result-count", Label).update(
|
||
f" {len(rows)} result(s) for '{kw}'"
|
||
)
|
||
|
||
def action_dismiss(self) -> None:
|
||
self.app.pop_screen()
|
||
|
||
|
||
# ─── Hits DB viewer screen ────────────────────────────────────────────────────
|
||
|
||
class HitsDBScreen(Screen):
|
||
"""
|
||
Paginated viewer for DB hits.
|
||
Toolbar buttons + number-key bindings filter by severity.
|
||
"""
|
||
|
||
BINDINGS = [
|
||
Binding("escape", "dismiss", "Back"),
|
||
Binding("r", "load_recent", "Recent 50"),
|
||
Binding("1", "filter_critical", "CRITICAL"),
|
||
Binding("2", "filter_high", "HIGH"),
|
||
Binding("3", "filter_medium", "MEDIUM"),
|
||
Binding("4", "filter_low", "LOW"),
|
||
]
|
||
|
||
DEFAULT_CSS = """
|
||
HitsDBScreen { background: $background; }
|
||
HitsDBScreen Label.screen-title {
|
||
background: $error;
|
||
color: $text;
|
||
padding: 0 1;
|
||
width: 100%;
|
||
}
|
||
HitsDBScreen #toolbar {
|
||
height: 3;
|
||
padding: 0 1;
|
||
background: $surface;
|
||
}
|
||
HitsDBScreen #toolbar Button { margin-right: 1; width: auto; }
|
||
HitsDBScreen #db-status { padding: 0 1; color: $text-muted; }
|
||
HitsDBScreen #hits-db-table { height: 1fr; margin: 0 1 1 1; }
|
||
"""
|
||
|
||
def compose(self) -> ComposeResult:
|
||
yield Header()
|
||
yield Label("📋 Hits Database Viewer", classes="screen-title")
|
||
with Horizontal(id="toolbar"):
|
||
yield Button("Recent 50", id="btn-recent", variant="default")
|
||
yield Button("🔴 CRITICAL", id="btn-critical", variant="error")
|
||
yield Button("🟠 HIGH", id="btn-high", variant="warning")
|
||
yield Button("🟡 MEDIUM", id="btn-medium", variant="default")
|
||
yield Button("🟢 LOW", id="btn-low", variant="success")
|
||
yield Label("", id="db-status")
|
||
yield DataTable(id="hits-db-table", zebra_stripes=True, cursor_type="row")
|
||
yield Footer()
|
||
|
||
def on_mount(self) -> None:
|
||
t = self.query_one("#hits-db-table", DataTable)
|
||
t.add_columns("ID", "Sev", "Timestamp", "URL", "Username", "Source", "Status")
|
||
self._load_recent()
|
||
|
||
def on_button_pressed(self, event: Button.Pressed) -> None:
|
||
dispatch = {
|
||
"btn-recent": self._load_recent,
|
||
"btn-critical": lambda: self._load_severity("CRITICAL"),
|
||
"btn-high": lambda: self._load_severity("HIGH"),
|
||
"btn-medium": lambda: self._load_severity("MEDIUM"),
|
||
"btn-low": lambda: self._load_severity("LOW"),
|
||
}
|
||
fn = dispatch.get(event.button.id)
|
||
if fn:
|
||
fn()
|
||
|
||
def _populate(self, rows, label: str) -> None:
|
||
t = self.query_one("#hits-db-table", DataTable)
|
||
t.clear()
|
||
for row in rows:
|
||
emoji = SEV_EMOJI.get(row["severity"], "⚪")
|
||
status = "dup" if row["seen_before"] else "new"
|
||
t.add_row(
|
||
str(row["id"]),
|
||
f"{emoji} {row['severity']}",
|
||
row["timestamp"],
|
||
(row["url"] or "")[:45],
|
||
(row["username"] or "")[:30],
|
||
(row["source"] or "")[:20],
|
||
status,
|
||
)
|
||
self.query_one("#db-status", Label).update(
|
||
f" {len(rows)} row(s) — {label}"
|
||
)
|
||
|
||
def _load_recent(self) -> None:
|
||
try:
|
||
from utils.database import recent
|
||
self._populate(recent(50), "most recent 50")
|
||
except Exception as e:
|
||
self.app.notify(f"DB error: {e}", severity="error")
|
||
|
||
def _load_severity(self, sev: str) -> None:
|
||
try:
|
||
from utils.database import by_severity
|
||
self._populate(by_severity(sev), f"severity = {sev} (unique only)")
|
||
except Exception as e:
|
||
self.app.notify(f"DB error: {e}", severity="error")
|
||
|
||
def action_dismiss(self) : self.app.pop_screen()
|
||
def action_load_recent(self) : self._load_recent()
|
||
def action_filter_critical(self): self._load_severity("CRITICAL")
|
||
def action_filter_high(self) : self._load_severity("HIGH")
|
||
def action_filter_medium(self) : self._load_severity("MEDIUM")
|
||
def action_filter_low(self) : self._load_severity("LOW")
|
||
|
||
|
||
# ─── Keywords screen ──────────────────────────────────────────────────────────
|
||
|
||
class KeywordsScreen(Screen):
|
||
"""
|
||
Live-edit TARGET_KEYWORDS regex patterns.
|
||
|
||
Additions / removals apply immediately:
|
||
• config.TARGET_KEYWORDS is mutated in place
|
||
• scorer's domain caches are rebuilt
|
||
• The bot handler recompiles patterns on the next message automatically
|
||
|
||
Changes are NOT written back to config.py — a notice banner says so.
|
||
"""
|
||
|
||
BINDINGS = [Binding("escape", "dismiss", "Back")]
|
||
|
||
DEFAULT_CSS = """
|
||
KeywordsScreen { background: $background; }
|
||
KeywordsScreen Label.screen-title {
|
||
background: $success;
|
||
color: $text;
|
||
padding: 0 1;
|
||
width: 100%;
|
||
}
|
||
KeywordsScreen Label.notice {
|
||
background: $warning;
|
||
color: $text;
|
||
padding: 0 1;
|
||
width: 100%;
|
||
}
|
||
KeywordsScreen #kw-controls {
|
||
height: 3;
|
||
padding: 0 1;
|
||
}
|
||
KeywordsScreen #kw-controls Input { width: 1fr; }
|
||
KeywordsScreen #kw-controls Button { width: auto; margin-left: 1; }
|
||
KeywordsScreen #kw-list-row {
|
||
height: 1fr;
|
||
padding: 0 1;
|
||
}
|
||
KeywordsScreen #kw-list {
|
||
width: 1fr;
|
||
height: 100%;
|
||
border: solid $primary;
|
||
}
|
||
KeywordsScreen #kw-list-row Button { width: 16; margin-left: 1; }
|
||
"""
|
||
|
||
def compose(self) -> ComposeResult:
|
||
yield Header()
|
||
yield Label("🔑 Keyword / Pattern Editor", classes="screen-title")
|
||
yield Label(
|
||
"⚠ Changes are in-memory only — copy patterns to config.py to persist across restarts.",
|
||
classes="notice",
|
||
)
|
||
with Horizontal(id="kw-controls"):
|
||
yield Input(
|
||
placeholder="regex e.g. @myorg\\.com or 192\\.168\\.10\\.",
|
||
id="kw-input",
|
||
)
|
||
yield Button("➕ Add", id="kw-add", variant="success")
|
||
with Horizontal(id="kw-list-row"):
|
||
yield ListView(id="kw-list")
|
||
yield Button("🗑 Remove", id="kw-remove", variant="error")
|
||
yield Footer()
|
||
|
||
def on_mount(self) -> None:
|
||
self._refresh_list()
|
||
self.query_one("#kw-input", Input).focus()
|
||
|
||
def _refresh_list(self) -> None:
|
||
lv = self.query_one("#kw-list", ListView)
|
||
lv.clear()
|
||
for kw in config.TARGET_KEYWORDS:
|
||
lv.append(ListItem(Label(kw)))
|
||
|
||
def on_button_pressed(self, event: Button.Pressed) -> None:
|
||
if event.button.id == "kw-add":
|
||
inp = self.query_one("#kw-input", Input)
|
||
raw = inp.value.strip()
|
||
if not raw:
|
||
return
|
||
import re
|
||
try:
|
||
re.compile(raw, re.IGNORECASE)
|
||
except re.error as e:
|
||
self.app.notify(f"Invalid regex: {e}", severity="error")
|
||
return
|
||
if raw not in config.TARGET_KEYWORDS:
|
||
config.TARGET_KEYWORDS.append(raw)
|
||
self._rebuild_scorer()
|
||
self._refresh_list()
|
||
self.app.notify(f"Pattern added: {raw}", severity="information")
|
||
inp.value = ""
|
||
|
||
elif event.button.id == "kw-remove":
|
||
lv = self.query_one("#kw-list", ListView)
|
||
idx = lv.index
|
||
if idx is None or not (0 <= idx < len(config.TARGET_KEYWORDS)):
|
||
self.app.notify("Select a pattern first", severity="warning")
|
||
return
|
||
removed = config.TARGET_KEYWORDS.pop(idx)
|
||
self._rebuild_scorer()
|
||
self._refresh_list()
|
||
self.app.notify(f"Pattern removed: {removed}", severity="warning")
|
||
|
||
def on_input_submitted(self, event: Input.Submitted) -> None:
|
||
if event.input.id == "kw-input":
|
||
# Simulate Add button press
|
||
self.on_button_pressed(
|
||
Button.Pressed(self.query_one("#kw-add", Button))
|
||
)
|
||
|
||
def _rebuild_scorer(self) -> None:
|
||
"""Rebuild scorer's cached domain patterns after a keyword change."""
|
||
try:
|
||
import scorer
|
||
scorer.EMPLOYEE_DOMAINS = scorer._build_employee_domains()
|
||
scorer.ORG_DOMAINS = scorer._build_org_domains()
|
||
except Exception as e:
|
||
log.warning(f"Could not rebuild scorer caches: {e}")
|
||
bus.post(bus.EvStatus(
|
||
f"Keywords updated — {len(config.TARGET_KEYWORDS)} pattern(s) active"
|
||
))
|
||
|
||
def action_dismiss(self) -> None:
|
||
self.app.pop_screen()
|
||
|
||
|
||
# ─── Main application ─────────────────────────────────────────────────────────
|
||
|
||
class MonitorApp(App):
|
||
|
||
CSS = """
|
||
Screen { layout: vertical; }
|
||
#top-row { layout: horizontal; height: 1fr; }
|
||
"""
|
||
|
||
BINDINGS = [
|
||
Binding("q", "quit", "Quit", priority=True),
|
||
Binding("ctrl+c", "quit", "Quit", priority=True),
|
||
Binding("s", "push_search", "Search DB"),
|
||
Binding("h", "push_hits_db", "Hits DB"),
|
||
Binding("k", "push_keywords", "Keywords"),
|
||
Binding("c", "clear_logs", "Clear Logs"),
|
||
Binding("r", "refresh_stats", "Refresh Stats"),
|
||
]
|
||
|
||
TITLE = "ULP Credential Monitor"
|
||
SUB_TITLE = f"session: {SESSION_NAME}"
|
||
|
||
def __init__(self):
|
||
super().__init__()
|
||
self._live_channels: list[str | int] = list(WATCHED_CHANNELS)
|
||
# Set by _drain_bus (Textual loop), read by _bot_main (bot loop)
|
||
# via call_soon_threadsafe so the asyncio.Event is set on the right loop.
|
||
self._bot_loop_channel_event: asyncio.Event | None = None
|
||
self._bot_loop: asyncio.AbstractEventLoop | None = None
|
||
|
||
def compose(self) -> ComposeResult:
|
||
yield Header()
|
||
with Horizontal(id="top-row"):
|
||
yield DownloadPanel(id="dl-panel")
|
||
yield HitsPanel(id="hits-panel")
|
||
yield StatsPanel(id="stats-panel")
|
||
yield ChannelPanel(initial_channels=WATCHED_CHANNELS, id="ch-panel")
|
||
yield Footer()
|
||
|
||
def on_mount(self) -> None:
|
||
# The bot backend runs in its own thread with its own asyncio event
|
||
# loop, completely isolated from Textual. Telethon spawns background
|
||
# tasks via asyncio.ensure_future() and calls connect() which returns
|
||
# only after its receiver loop is scheduled — both of these deadlock
|
||
# inside Textual's managed loop. Running in a dedicated thread
|
||
# sidesteps all of that.
|
||
#
|
||
# Communication uses a thread-safe queue.Queue (see tui_events.py).
|
||
# The TUI polls it every 100 ms via set_interval().
|
||
bus.init_bus()
|
||
self._bot_thread = threading.Thread(
|
||
target=self._run_bot_thread,
|
||
name="bot-thread",
|
||
daemon=True,
|
||
)
|
||
self._bot_thread.start()
|
||
# Poll the thread-safe queue and dispatch to widgets
|
||
self.set_interval(0.1, self._drain_bus)
|
||
|
||
# ── Screen navigation ─────────────────────────────────────────────────────
|
||
|
||
def action_push_search(self) : self.push_screen(SearchScreen())
|
||
def action_push_hits_db(self) : self.push_screen(HitsDBScreen())
|
||
def action_push_keywords(self) : self.push_screen(KeywordsScreen())
|
||
|
||
def action_clear_logs(self) -> None:
|
||
self.query_one("#dl-panel", DownloadPanel).clear_logs()
|
||
self.query_one("#hits-panel", HitsPanel).clear_log()
|
||
self.notify("Logs cleared", severity="information")
|
||
|
||
def action_refresh_stats(self) -> None:
|
||
self.query_one("#stats-panel", StatsPanel).refresh_stats()
|
||
self.notify("Stats refreshed", severity="information")
|
||
|
||
# ── Event bus consumer ────────────────────────────────────────────────────
|
||
|
||
def _signal_channel_changed(self) -> None:
|
||
"""Thread-safely set the channel-change event on the bot loop."""
|
||
ev = self._bot_loop_channel_event
|
||
loop = self._bot_loop
|
||
if ev is not None and loop is not None and loop.is_running():
|
||
loop.call_soon_threadsafe(ev.set)
|
||
|
||
# ── Bus drain (runs on Textual's loop via set_interval) ──────────────────
|
||
|
||
def _drain_bus(self) -> None:
|
||
"""
|
||
Called every 100 ms by set_interval(). Drains all pending events
|
||
from the thread-safe queue and dispatches them to the right widget.
|
||
Runs on Textual's event loop — safe to call widget methods directly.
|
||
"""
|
||
q = bus.get_bus()
|
||
if q is None:
|
||
return
|
||
|
||
try:
|
||
dl = self.query_one("#dl-panel", DownloadPanel)
|
||
hit = self.query_one("#hits-panel", HitsPanel)
|
||
stats = self.query_one("#stats-panel", StatsPanel)
|
||
except Exception:
|
||
return # widgets not mounted yet
|
||
|
||
# Drain everything currently in the queue in one pass
|
||
while True:
|
||
try:
|
||
ev = q.get_nowait()
|
||
except queue.Empty:
|
||
break
|
||
|
||
try:
|
||
if isinstance(ev, bus.EvTdlOutput):
|
||
dl.tdl_line(ev.line)
|
||
|
||
elif isinstance(ev, bus.EvDownloadQueued):
|
||
dl.queued(ev.filename, ev.size_mb, ev.source, ev.password)
|
||
|
||
elif isinstance(ev, bus.EvDownloadStarted):
|
||
dl.status(ev.filename, "downloading")
|
||
|
||
elif isinstance(ev, bus.EvDownloadDone):
|
||
dl.status(ev.filename,
|
||
"done_tdl" if ev.via == "tdl" else "done_tel",
|
||
via=ev.via)
|
||
|
||
elif isinstance(ev, bus.EvDownloadFailed):
|
||
dl.status(ev.filename, "failed")
|
||
|
||
elif isinstance(ev, bus.EvHit):
|
||
hit.add_hit(ev.severity, ev.raw, ev.source, ev.filename, ev.reasons)
|
||
stats.refresh_stats()
|
||
|
||
elif isinstance(ev, bus.EvChannelAdded):
|
||
if ev.channel not in self._live_channels:
|
||
self._live_channels.append(ev.channel)
|
||
self._signal_channel_changed()
|
||
|
||
elif isinstance(ev, bus.EvChannelRemoved):
|
||
self._live_channels = [
|
||
c for c in self._live_channels if c != ev.channel
|
||
]
|
||
self._signal_channel_changed()
|
||
|
||
elif isinstance(ev, bus.EvStatus):
|
||
log.info(f"[bus] EvStatus: {ev.text}")
|
||
severity = {"error": "error", "warning": "warning"}.get(
|
||
ev.level, "information"
|
||
)
|
||
self.notify(ev.text, severity=severity)
|
||
|
||
else:
|
||
log.warning(f"[bus] Unknown event type: {type(ev)}")
|
||
|
||
except Exception as e:
|
||
log.error(f"[bus] Dispatch error for {type(ev).__name__}: {e}", exc_info=True)
|
||
|
||
# ── Bot thread ────────────────────────────────────────────────────────────
|
||
|
||
def _run_bot_thread(self) -> None:
|
||
"""
|
||
Entry point for the bot background thread.
|
||
Creates a brand-new asyncio event loop for Telethon to use,
|
||
completely isolated from Textual's loop.
|
||
"""
|
||
loop = asyncio.new_event_loop()
|
||
asyncio.set_event_loop(loop)
|
||
self._bot_loop = loop
|
||
try:
|
||
loop.run_until_complete(self._bot_main())
|
||
except Exception as e:
|
||
log.error(f"[bot-thread] Unhandled exception: {e}", exc_info=True)
|
||
bus.post(bus.EvStatus(f"Bot thread crashed: {e}", level="error"))
|
||
finally:
|
||
loop.close()
|
||
|
||
async def _bot_main(self) -> None:
|
||
"""
|
||
Full bot backend — runs inside the bot thread's own event loop.
|
||
Telethon is free to schedule background tasks without interfering
|
||
with Textual's loop.
|
||
"""
|
||
import shutil as _shutil
|
||
from telethon import TelegramClient
|
||
from telethon import events as tl_events
|
||
from core.processor import compile_patterns
|
||
from core.notifier import send_status
|
||
from core.scraper import backfill_all, warm_entity_cache
|
||
from utils.database import init_db
|
||
|
||
init_db()
|
||
patterns = compile_patterns(config.TARGET_KEYWORDS)
|
||
|
||
bus.post(bus.EvStatus(
|
||
f"Starting — {len(config.WATCHED_CHANNELS)} channel(s), "
|
||
f"{len(patterns)} pattern(s)"
|
||
))
|
||
|
||
user_client = TelegramClient(
|
||
config.SESSION_NAME, config.API_ID, config.API_HASH,
|
||
connection_retries=5, auto_reconnect=True, request_retries=5,
|
||
)
|
||
bot_client = TelegramClient(
|
||
"bot_session", config.API_ID, config.API_HASH,
|
||
)
|
||
|
||
try:
|
||
log.info("[bot] Connecting bot_client...")
|
||
await bot_client.connect()
|
||
log.info("[bot] bot_client connected, authorizing...")
|
||
if not await bot_client.is_user_authorized():
|
||
await bot_client.sign_in(bot_token=config.BOT_TOKEN)
|
||
log.info("[bot] bot_client ready")
|
||
|
||
log.info("[bot] Connecting user_client...")
|
||
await user_client.connect()
|
||
log.info("[bot] user_client connected, checking auth...")
|
||
if not await user_client.is_user_authorized():
|
||
log.error("[bot] user_client not authorized — run: python main.py --no-tui")
|
||
bus.post(bus.EvStatus(
|
||
"Not authorized — run --no-tui once to complete login",
|
||
level="error",
|
||
))
|
||
return
|
||
log.info("[bot] user_client ready")
|
||
|
||
try:
|
||
me = await user_client.get_me()
|
||
bus.post(bus.EvStatus(f"Connected as {me.first_name} (@{me.username})"))
|
||
await send_status(
|
||
bot_client,
|
||
f"✅ *Monitor started* (TUI)\n"
|
||
f"User: `{me.first_name}`\n"
|
||
f"Channels: `{len(config.WATCHED_CHANNELS)}`\n"
|
||
f"Patterns: `{len(patterns)}`",
|
||
)
|
||
|
||
await warm_entity_cache(user_client)
|
||
|
||
_current_handler = [None]
|
||
|
||
def _make_handler(channels):
|
||
if _current_handler[0] is not None:
|
||
user_client.remove_event_handler(_current_handler[0])
|
||
|
||
from core.bot_downloader import (
|
||
handle_bot_download_message,
|
||
has_download_button,
|
||
extract_password,
|
||
)
|
||
from core.scraper import handle_message
|
||
from telethon.tl.types import MessageMediaDocument
|
||
|
||
_channel_passwords: dict[int, str] = {}
|
||
|
||
@user_client.on(tl_events.NewMessage(chats=channels))
|
||
async def _handler(event):
|
||
msg = event.message
|
||
try:
|
||
source = event.chat.username or str(event.chat_id)
|
||
except Exception:
|
||
source = str(event.chat_id)
|
||
|
||
chat_id = event.chat_id
|
||
msg_pw = extract_password(msg)
|
||
if msg_pw:
|
||
_channel_passwords[chat_id] = msg_pw
|
||
password = msg_pw or _channel_passwords.get(chat_id)
|
||
|
||
live_patterns = compile_patterns(config.TARGET_KEYWORDS)
|
||
|
||
if msg.media and isinstance(msg.media, MessageMediaDocument):
|
||
await handle_message(
|
||
user_client, bot_client, msg,
|
||
source, live_patterns, password=password,
|
||
)
|
||
elif msg.buttons and has_download_button(msg):
|
||
await handle_bot_download_message(
|
||
user_client, bot_client, msg,
|
||
source, live_patterns, password=password,
|
||
)
|
||
|
||
_current_handler[0] = _handler
|
||
log.info(f"[bot] Handler registered for {len(channels)} channel(s)")
|
||
bus.post(bus.EvStatus(f"Watching {len(channels)} channel(s)"))
|
||
|
||
# Channel-change event — lives on this (bot) loop.
|
||
# Textual signals it thread-safely via _signal_channel_changed().
|
||
_ch_changed = asyncio.Event()
|
||
self._bot_loop_channel_event = _ch_changed
|
||
|
||
_make_handler(list(self._live_channels))
|
||
bus.post(bus.EvStatus("Live listener active"))
|
||
|
||
await backfill_all(user_client, bot_client, patterns)
|
||
bus.post(bus.EvStatus("Backfill complete — monitoring live"))
|
||
|
||
async def _watch_channels():
|
||
while True:
|
||
await _ch_changed.wait()
|
||
_ch_changed.clear()
|
||
new_channels = list(self._live_channels)
|
||
log.info(f"[bot] Channel list changed → {new_channels}")
|
||
_make_handler(new_channels)
|
||
|
||
await asyncio.gather(
|
||
user_client.run_until_disconnected(),
|
||
_watch_channels(),
|
||
)
|
||
|
||
except Exception as e:
|
||
bus.post(bus.EvStatus(f"Bot error: {e}", level="error"))
|
||
log.error("[bot] Bot main crashed", exc_info=True)
|
||
finally:
|
||
log.info("[bot] Disconnecting clients...")
|
||
await user_client.disconnect()
|
||
await bot_client.disconnect()
|
||
|
||
except Exception as e:
|
||
bus.post(bus.EvStatus(f"Bot connect error: {e}", level="error"))
|
||
log.error("[bot] Connection failed", exc_info=True)
|
||
finally:
|
||
if config.TEMP_DIR.exists():
|
||
_shutil.rmtree(config.TEMP_DIR, ignore_errors=True)
|
||
config.TEMP_DIR.mkdir(exist_ok=True)
|
||
|
||
def action_quit(self) -> None:
|
||
self.exit()
|
||
|
||
|
||
# ─── Entry point ──────────────────────────────────────────────────────────────
|
||
|
||
def run_tui() -> None:
|
||
# Do NOT call bus.init_bus() here — the Queue must be created inside
|
||
# Textual's event loop (see MonitorApp.on_mount). Calling it here
|
||
# would bind the Queue to the outer loop which is discarded when
|
||
# App.run() creates a new one.
|
||
MonitorApp().run()
|