fix: persist ingester position and profiler cursor across restarts

- Ingester now loads byte-offset from DB on startup (key: ingest_worker_position)
  and saves it after each batch — prevents full re-read on every API restart
- On file truncation/rotation the saved offset is reset to 0
- Profiler worker now loads last_log_id from DB on startup — every restart
  becomes an incremental update instead of a full cold rebuild
- Updated all affected tests to mock get_state/set_state; added new tests
  covering position restore, set_state call, truncation reset, and cursor
  restore/cold-start paths
This commit is contained in:
2026-04-15 13:58:12 -04:00
parent 314e6c6388
commit 63efe6c7ba
5 changed files with 203 additions and 2 deletions

View File

@@ -614,6 +614,60 @@ class TestAttackerProfileWorker:
assert len(update_calls) >= 1
@pytest.mark.asyncio
async def test_cursor_restored_from_db_on_startup(self):
"""Worker loads saved last_log_id from DB and passes it to _incremental_update."""
repo = _make_repo(saved_state={"last_log_id": 99})
_call_count = 0
async def fake_sleep(secs):
nonlocal _call_count
_call_count += 1
if _call_count >= 2:
raise asyncio.CancelledError()
captured_states = []
async def mock_update(_repo, state):
captured_states.append((state.last_log_id, state.initialized))
with patch("decnet.profiler.worker.asyncio.sleep", side_effect=fake_sleep):
with patch("decnet.profiler.worker._incremental_update", side_effect=mock_update):
with pytest.raises(asyncio.CancelledError):
await attacker_profile_worker(repo)
assert captured_states, "_incremental_update never called"
restored_id, initialized = captured_states[0]
assert restored_id == 99
assert initialized is True
@pytest.mark.asyncio
async def test_no_saved_cursor_starts_from_zero(self):
"""When get_state returns None, worker starts fresh from log ID 0."""
repo = _make_repo(saved_state=None)
_call_count = 0
async def fake_sleep(secs):
nonlocal _call_count
_call_count += 1
if _call_count >= 2:
raise asyncio.CancelledError()
captured_states = []
async def mock_update(_repo, state):
captured_states.append((state.last_log_id, state.initialized))
with patch("decnet.profiler.worker.asyncio.sleep", side_effect=fake_sleep):
with patch("decnet.profiler.worker._incremental_update", side_effect=mock_update):
with pytest.raises(asyncio.CancelledError):
await attacker_profile_worker(repo)
assert captured_states, "_incremental_update never called"
restored_id, initialized = captured_states[0]
assert restored_id == 0
assert initialized is False
# ─── JA3 bounty extraction from ingester ─────────────────────────────────────