fix: remove event-loop-blocking cold start; unify profiler to cursor-based incremental
Cold start fetched all logs in one bulk query then processed them in a tight synchronous loop with no yields, blocking the asyncio event loop for seconds on datasets of 30K+ rows. This stalled every concurrent await — including the SSE stream generator's initial DB calls — causing the dashboard to show INITIALIZING SENSORS indefinitely. Changes: - Drop _cold_start() and get_all_logs_raw(); uninitialized state now runs the same cursor loop as incremental, starting from last_log_id=0 - Yield to the event loop after every _BATCH_SIZE rows (asyncio.sleep(0)) - Add SSE keepalive comment as first yield so the connection flushes before any DB work begins - Add Cache-Control/X-Accel-Buffering headers to StreamingResponse
This commit is contained in:
@@ -27,7 +27,6 @@ from decnet.profiler.worker import (
|
||||
_STATE_KEY,
|
||||
_WorkerState,
|
||||
_build_record,
|
||||
_cold_start,
|
||||
_extract_commands_from_events,
|
||||
_first_contact_deckies,
|
||||
_incremental_update,
|
||||
@@ -97,11 +96,12 @@ def _make_log_row(
|
||||
|
||||
def _make_repo(logs=None, bounties=None, bounties_for_ips=None, max_log_id=0, saved_state=None):
|
||||
repo = MagicMock()
|
||||
repo.get_all_logs_raw = AsyncMock(return_value=logs or [])
|
||||
repo.get_all_bounties_by_ip = AsyncMock(return_value=bounties or {})
|
||||
repo.get_bounties_for_ips = AsyncMock(return_value=bounties_for_ips or {})
|
||||
repo.get_max_log_id = AsyncMock(return_value=max_log_id)
|
||||
repo.get_logs_after_id = AsyncMock(return_value=[])
|
||||
# Return provided logs on first call (simulating a single page < BATCH_SIZE), then [] to end loop
|
||||
_log_pages = [logs or [], []]
|
||||
repo.get_logs_after_id = AsyncMock(side_effect=_log_pages)
|
||||
repo.get_state = AsyncMock(return_value=saved_state)
|
||||
repo.set_state = AsyncMock()
|
||||
repo.upsert_attacker = AsyncMock(return_value="mock-uuid")
|
||||
@@ -283,7 +283,7 @@ class TestBuildRecord:
|
||||
assert record["updated_at"].tzinfo is not None
|
||||
|
||||
|
||||
# ─── _cold_start ─────────────────────────────────────────────────────────────
|
||||
# ─── cold start via _incremental_update (uninitialized state) ────────────────
|
||||
|
||||
class TestColdStart:
|
||||
@pytest.mark.asyncio
|
||||
@@ -299,7 +299,7 @@ class TestColdStart:
|
||||
repo = _make_repo(logs=rows, max_log_id=3)
|
||||
state = _WorkerState()
|
||||
|
||||
await _cold_start(repo, state)
|
||||
await _incremental_update(repo, state)
|
||||
|
||||
assert state.initialized is True
|
||||
assert state.last_log_id == 3
|
||||
@@ -313,7 +313,7 @@ class TestColdStart:
|
||||
repo = _make_repo(logs=[], max_log_id=0)
|
||||
state = _WorkerState()
|
||||
|
||||
await _cold_start(repo, state)
|
||||
await _incremental_update(repo, state)
|
||||
|
||||
assert state.initialized is True
|
||||
assert state.last_log_id == 0
|
||||
@@ -337,7 +337,7 @@ class TestColdStart:
|
||||
repo = _make_repo(logs=rows, max_log_id=2)
|
||||
state = _WorkerState()
|
||||
|
||||
await _cold_start(repo, state)
|
||||
await _incremental_update(repo, state)
|
||||
|
||||
record = repo.upsert_attacker.call_args[0][0]
|
||||
assert record["is_traversal"] is True
|
||||
@@ -357,7 +357,7 @@ class TestColdStart:
|
||||
)
|
||||
state = _WorkerState()
|
||||
|
||||
await _cold_start(repo, state)
|
||||
await _incremental_update(repo, state)
|
||||
|
||||
record = repo.upsert_attacker.call_args[0][0]
|
||||
assert record["bounty_count"] == 2
|
||||
@@ -376,7 +376,7 @@ class TestColdStart:
|
||||
repo = _make_repo(logs=[row], max_log_id=1)
|
||||
state = _WorkerState()
|
||||
|
||||
await _cold_start(repo, state)
|
||||
await _incremental_update(repo, state)
|
||||
|
||||
record = repo.upsert_attacker.call_args[0][0]
|
||||
commands = json.loads(record["commands"])
|
||||
@@ -542,7 +542,7 @@ class TestIncrementalUpdate:
|
||||
assert called_ips == {"1.1.1.1", "2.2.2.2"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_uninitialized_state_triggers_cold_start(self):
|
||||
async def test_uninitialized_state_runs_full_cursor_sweep(self):
|
||||
rows = [
|
||||
_make_log_row(
|
||||
row_id=1,
|
||||
@@ -556,7 +556,8 @@ class TestIncrementalUpdate:
|
||||
await _incremental_update(repo, state)
|
||||
|
||||
assert state.initialized is True
|
||||
repo.get_all_logs_raw.assert_awaited_once()
|
||||
assert state.last_log_id == 1
|
||||
repo.upsert_attacker.assert_awaited_once()
|
||||
|
||||
|
||||
# ─── attacker_profile_worker ────────────────────────────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user