db: switch MySQL driver to asyncmy, env-tune pool, serialize DDL
- aiomysql → asyncmy on both sides of the URL/import (faster, maintained). - Pool sizing now reads DECNET_DB_POOL_SIZE / MAX_OVERFLOW / RECYCLE / PRE_PING for both SQLite and MySQL engines so stress runs can bump without code edits. - MySQL initialize() now wraps schema DDL in a GET_LOCK advisory lock so concurrent uvicorn workers racing create_all() don't hit 'Table was skipped since its definition is being modified by concurrent DDL'. - sqlite & mysql repo get_log_histogram use the shared _session() helper instead of session_factory() for consistency with the rest of the repo. - SSE stream_events docstring updated to asyncmy.
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
MySQL async engine factory.
|
||||
|
||||
Builds a SQLAlchemy AsyncEngine against MySQL using the ``aiomysql`` driver.
|
||||
Builds a SQLAlchemy AsyncEngine against MySQL using the ``asyncmy`` driver.
|
||||
|
||||
Connection info is resolved (in order of precedence):
|
||||
|
||||
@@ -23,10 +23,10 @@ from urllib.parse import quote_plus
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
|
||||
|
||||
DEFAULT_POOL_SIZE = 10
|
||||
DEFAULT_MAX_OVERFLOW = 20
|
||||
DEFAULT_POOL_RECYCLE = 3600 # seconds — avoid MySQL ``wait_timeout`` disconnects
|
||||
DEFAULT_POOL_PRE_PING = True
|
||||
DEFAULT_POOL_SIZE = int(os.environ.get("DECNET_DB_POOL_SIZE", "20"))
|
||||
DEFAULT_MAX_OVERFLOW = int(os.environ.get("DECNET_DB_MAX_OVERFLOW", "40"))
|
||||
DEFAULT_POOL_RECYCLE = int(os.environ.get("DECNET_DB_POOL_RECYCLE", "3600"))
|
||||
DEFAULT_POOL_PRE_PING = os.environ.get("DECNET_DB_POOL_PRE_PING", "true").lower() == "true"
|
||||
|
||||
|
||||
def build_mysql_url(
|
||||
@@ -36,7 +36,7 @@ def build_mysql_url(
|
||||
user: Optional[str] = None,
|
||||
password: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Compose an async SQLAlchemy URL for MySQL using the aiomysql driver.
|
||||
"""Compose an async SQLAlchemy URL for MySQL using the asyncmy driver.
|
||||
|
||||
Component args override env vars. Password is percent-encoded so special
|
||||
characters (``@``, ``:``, ``/``…) don't break URL parsing.
|
||||
@@ -59,7 +59,7 @@ def build_mysql_url(
|
||||
|
||||
pw_enc = quote_plus(password)
|
||||
user_enc = quote_plus(user)
|
||||
return f"mysql+aiomysql://{user_enc}:{pw_enc}@{host}:{port}/{database}"
|
||||
return f"mysql+asyncmy://{user_enc}:{pw_enc}@{host}:{port}/{database}"
|
||||
|
||||
|
||||
def resolve_url(url: Optional[str] = None) -> str:
|
||||
|
||||
@@ -24,7 +24,7 @@ from decnet.web.db.sqlmodel_repo import SQLModelRepository
|
||||
|
||||
|
||||
class MySQLRepository(SQLModelRepository):
|
||||
"""MySQL backend — uses ``aiomysql``."""
|
||||
"""MySQL backend — uses ``asyncmy``."""
|
||||
|
||||
def __init__(self, url: Optional[str] = None, **engine_kwargs) -> None:
|
||||
self.engine = get_async_engine(url=url, **engine_kwargs)
|
||||
@@ -81,13 +81,24 @@ class MySQLRepository(SQLModelRepository):
|
||||
))
|
||||
|
||||
async def initialize(self) -> None:
|
||||
"""Create tables and run all MySQL-specific migrations."""
|
||||
"""Create tables and run all MySQL-specific migrations.
|
||||
|
||||
Uses a MySQL advisory lock to serialize DDL across concurrent
|
||||
uvicorn workers — prevents the 'Table was skipped since its
|
||||
definition is being modified by concurrent DDL' race.
|
||||
"""
|
||||
from sqlmodel import SQLModel
|
||||
await self._migrate_attackers_table()
|
||||
await self._migrate_column_types()
|
||||
async with self.engine.begin() as conn:
|
||||
await conn.run_sync(SQLModel.metadata.create_all)
|
||||
await self._ensure_admin_user()
|
||||
async with self.engine.connect() as lock_conn:
|
||||
await lock_conn.execute(text("SELECT GET_LOCK('decnet_schema_init', 30)"))
|
||||
try:
|
||||
await self._migrate_attackers_table()
|
||||
await self._migrate_column_types()
|
||||
async with self.engine.begin() as conn:
|
||||
await conn.run_sync(SQLModel.metadata.create_all)
|
||||
await self._ensure_admin_user()
|
||||
finally:
|
||||
await lock_conn.execute(text("SELECT RELEASE_LOCK('decnet_schema_init')"))
|
||||
await lock_conn.close()
|
||||
|
||||
def _json_field_equals(self, key: str):
|
||||
# MySQL 5.7+ exposes JSON_EXTRACT; quoted string result returned for
|
||||
@@ -115,7 +126,7 @@ class MySQLRepository(SQLModelRepository):
|
||||
literal_column("bucket_time")
|
||||
)
|
||||
|
||||
async with self.session_factory() as session:
|
||||
async with self._session() as session:
|
||||
results = await session.execute(statement)
|
||||
# Normalize to ISO string for API parity with the SQLite backend
|
||||
# (SQLite's datetime() returns a string already; FROM_UNIXTIME
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import os
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy import create_engine, Engine, event
|
||||
from sqlmodel import SQLModel
|
||||
@@ -11,9 +13,20 @@ def get_async_engine(db_path: str) -> AsyncEngine:
|
||||
prefix = "sqlite+aiosqlite:///"
|
||||
if db_path.startswith(":memory:"):
|
||||
prefix = "sqlite+aiosqlite://"
|
||||
|
||||
pool_size = int(os.environ.get("DECNET_DB_POOL_SIZE", "20"))
|
||||
max_overflow = int(os.environ.get("DECNET_DB_MAX_OVERFLOW", "40"))
|
||||
|
||||
pool_recycle = int(os.environ.get("DECNET_DB_POOL_RECYCLE", "3600"))
|
||||
pool_pre_ping = os.environ.get("DECNET_DB_POOL_PRE_PING", "true").lower() == "true"
|
||||
|
||||
engine = create_async_engine(
|
||||
f"{prefix}{db_path}",
|
||||
echo=False,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_recycle=pool_recycle,
|
||||
pool_pre_ping=pool_pre_ping,
|
||||
connect_args={"uri": True, "timeout": 30},
|
||||
)
|
||||
|
||||
|
||||
@@ -54,6 +54,6 @@ class SQLiteRepository(SQLModelRepository):
|
||||
literal_column("bucket_time")
|
||||
)
|
||||
|
||||
async with self.session_factory() as session:
|
||||
async with self._session() as session:
|
||||
results = await session.execute(statement)
|
||||
return [{"time": r[0], "count": r[1]} for r in results.all()]
|
||||
|
||||
Reference in New Issue
Block a user