bugfix:修复sqlite部分场景下锁竞争问题；新增插件恶意触发配置文件 (#2144)

* bugfix:修复sqlite部分场景下锁竞争问题；新增插件恶意触发配置文件

* 文件没同步完

zhenxun/services/db_context/__init__.py

@@ -18,6 +18,7 @@ from zhenxun.configs.config import BotConfig
 from zhenxun.services.log import logger
 from zhenxun.utils.manager.priority_manager import PriorityLifecycle
 
+from . import watchdog as _watchdog  # noqa: F401
 from .base_model import Model
 from .config import (
     DB_TIMEOUT_SECONDS,

zhenxun/services/db_context/base_model.py

@@ -27,7 +27,7 @@ class Model(TortoiseModel):
     """
 
     sem_data: ClassVar[dict[str, dict[str, asyncio.Semaphore]]] = {}
-    _current_locks: ClassVar[dict[int, DbLockType]] = {}  # 跟踪当前协程持有的锁
+    _current_locks: ClassVar[dict[tuple[str, int], DbLockType]] = {}
 
     def __init_subclass__(cls, **kwargs):
         super().__init_subclass__(**kwargs)
@@ -95,21 +95,21 @@ class Model(TortoiseModel):
     @classmethod
     def _require_lock(cls, lock_type: DbLockType) -> bool:
         """检查是否需要真正加锁"""
-        task_id = id(asyncio.current_task())
-        return cls._current_locks.get(task_id) != lock_type
+        lock_key = (cls.__name__, id(asyncio.current_task()))
+        return cls._current_locks.get(lock_key) != lock_type
 
     @classmethod
     @contextlib.asynccontextmanager
     async def _lock_context(cls, lock_type: DbLockType):
         """带重入检查的锁上下文"""
-        task_id = id(asyncio.current_task())
+        lock_key = (cls.__name__, id(asyncio.current_task()))
         need_lock = cls._require_lock(lock_type)
 
         if need_lock and (sem := cls.get_semaphore(lock_type)):
-            cls._current_locks[task_id] = lock_type
+            cls._current_locks[lock_key] = lock_type
             async with sem:
                 yield
-            cls._current_locks.pop(task_id, None)
+            cls._current_locks.pop(lock_key, None)
         else:
             yield
 
@@ -171,8 +171,8 @@ class Model(TortoiseModel):
                         await obj.save()
                         result = (obj, False)
                     else:
-                        # 创建时不重复加锁
-                        result = await cls.create(**kwargs, **(defaults or {})), True
+                        obj = await super().create(**kwargs, **(defaults or {}))
+                        result = (obj, True)
 
                 if cache_type := cls.get_cache_type():
                     await CacheRoot.invalidate_cache(

zhenxun/services/db_context/config.py

@@ -27,7 +27,9 @@ MYSQL_CONFIG = {
 
 SQLITE_CONFIG = {
     "journal_mode": "WAL",  # 提高并发写入性能
-    "busy_timeout": 30000,  # SQLite 锁等待超时，单位毫秒
+    # SQLite 的底层锁等待应接近业务超时，避免上层放弃后 worker 仍长时间占线。
+    # Windows bind mount / Docker 场景下 SQLite 不适合高写并发。
+    "busy_timeout": 5000,
     "foreign_keys": "ON",
 }
 

zhenxun/services/db_context/utils.py

@@ -1,7 +1,9 @@
 import asyncio
+import contextlib
 import time
 
 from zhenxun.services.log import logger
+from zhenxun.services.message_load import signal_db_unhealthy
 
 from .config import (
     DB_TIMEOUT_SECONDS,
@@ -9,6 +11,40 @@ from .config import (
     SLOW_QUERY_THRESHOLD,
 )
 
+_SQLITE_STALL_UNTIL = 0.0
+_SQLITE_STALL_REASON = ""
+_DB_UNHEALTHY_TIMEOUT_SECONDS = 30.0
+_SQLITE_STALL_TIMEOUT_SECONDS = 60.0
+
+
+def _is_sqlite_connection() -> bool:
+    with contextlib.suppress(Exception):
+        from tortoise import Tortoise
+
+        connection = Tortoise.get_connection("default")
+        capabilities = getattr(connection, "capabilities", None)
+        dialect = str(getattr(capabilities, "dialect", "") or "").lower()
+        return dialect.startswith("sqlite")
+    return False
+
+
+def _mark_sqlite_stall(reason: str, duration: float) -> None:
+    global _SQLITE_STALL_REASON, _SQLITE_STALL_UNTIL
+    until = time.monotonic() + max(duration, 0.0)
+    if until > _SQLITE_STALL_UNTIL:
+        _SQLITE_STALL_UNTIL = until
+        _SQLITE_STALL_REASON = str(reason or "")[:200]
+
+
+def is_sqlite_stall_suspected() -> bool:
+    return time.monotonic() < _SQLITE_STALL_UNTIL
+
+
+def sqlite_stall_reason() -> str:
+    if not is_sqlite_stall_suspected():
+        return ""
+    return _SQLITE_STALL_REASON
+
 
 async def with_db_timeout(
     coro,
@@ -19,13 +55,23 @@ async def with_db_timeout(
     """带超时控制的数据库操作"""
     start_time = time.time()
     try:
-        logger.debug(f"开始执行数据库操作: {operation} 来源: {source}")
         result = await asyncio.wait_for(coro, timeout=timeout)
         elapsed = time.time() - start_time
         if elapsed > SLOW_QUERY_THRESHOLD and operation:
             logger.warning(f"慢查询: {operation} 耗时 {elapsed:.3f}s", LOG_COMMAND)
         return result
     except asyncio.TimeoutError:
+        timeout_reason = f"{operation or 'database_operation'} from {source or '-'}"
+        unhealthy_duration = _DB_UNHEALTHY_TIMEOUT_SECONDS
+        if _is_sqlite_connection():
+            unhealthy_duration = _SQLITE_STALL_TIMEOUT_SECONDS
+            _mark_sqlite_stall(timeout_reason, unhealthy_duration)
+            logger.warning(
+                "SQLite 数据库操作超时，疑似 aiosqlite worker/连接被锁等待卡住；"
+                "已暂停低优先级数据库任务",
+                LOG_COMMAND,
+            )
+        signal_db_unhealthy(unhealthy_duration, reason=timeout_reason)
         if operation:
             logger.error(
                 f"数据库操作超时: {operation} (>{timeout}s) 来源: {source}",

zhenxun/services/db_context/watchdog.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import time
+
+from tortoise import Tortoise
+from tortoise.connection import connections
+
+from zhenxun.services.log import logger
+from zhenxun.services.low_priority_writer import low_priority_writer_active_count
+from zhenxun.services.message_load import (
+    signal_db_unhealthy,
+)
+from zhenxun.utils.manager.priority_manager import PriorityLifecycle
+
+from .config import LOG_COMMAND
+
+_CHECK_INTERVAL_SECONDS = 15.0
+_CHECK_TIMEOUT_SECONDS = 2.0
+_FAIL_THRESHOLD = 3
+_UNHEALTHY_SECONDS = 60.0
+_RECONNECT_COOLDOWN_SECONDS = 60.0
+_RECONNECT_WAIT_IDLE_SECONDS = 3.0
+
+_WATCHDOG_TASK: asyncio.Task[None] | None = None
+_RECONNECT_LOCK = asyncio.Lock()
+_LAST_RECONNECT_AT = 0.0
+
+
+def _is_sqlite_connection() -> bool:
+    with contextlib.suppress(Exception):
+        connection = Tortoise.get_connection("default")
+        capabilities = getattr(connection, "capabilities", None)
+        dialect = str(getattr(capabilities, "dialect", "") or "").lower()
+        return dialect.startswith("sqlite")
+    return False
+
+
+async def _select_one() -> None:
+    connection = Tortoise.get_connection("default")
+    await connection.execute_query("SELECT 1")
+
+
+async def _try_reconnect(reason: str) -> None:
+    global _LAST_RECONNECT_AT
+    now = time.monotonic()
+    if now - _LAST_RECONNECT_AT < _RECONNECT_COOLDOWN_SECONDS:
+        return
+    if low_priority_writer_active_count() > 0:
+        return
+    async with _RECONNECT_LOCK:
+        now = time.monotonic()
+        if now - _LAST_RECONNECT_AT < _RECONNECT_COOLDOWN_SECONDS:
+            return
+        await asyncio.sleep(_RECONNECT_WAIT_IDLE_SECONDS)
+        if low_priority_writer_active_count() > 0:
+            return
+        try:
+            await connections.close_all(discard=True)
+            # ConnectionHandler lazily recreates default connection from db_config.
+            Tortoise.get_connection("default")
+            _LAST_RECONNECT_AT = time.monotonic()
+            logger.warning(
+                f"SQLite watchdog rebuilt default connection: {reason}",
+                LOG_COMMAND,
+            )
+        except Exception as exc:
+            _LAST_RECONNECT_AT = time.monotonic()
+            signal_db_unhealthy(
+                _UNHEALTHY_SECONDS,
+                reason=f"watchdog reconnect:{reason}",
+            )
+            logger.warning("SQLite watchdog reconnect failed", LOG_COMMAND, e=exc)
+
+
+async def _watchdog_loop() -> None:
+    failures = 0
+    while True:
+        await asyncio.sleep(_CHECK_INTERVAL_SECONDS)
+        if not _is_sqlite_connection():
+            failures = 0
+            continue
+        try:
+            await asyncio.wait_for(_select_one(), timeout=_CHECK_TIMEOUT_SECONDS)
+            failures = 0
+        except asyncio.CancelledError:
+            raise
+        except Exception as exc:
+            failures += 1
+            reason = f"sqlite watchdog SELECT 1 failed x{failures}: {exc}"
+            signal_db_unhealthy(_UNHEALTHY_SECONDS, reason=reason)
+            logger.warning(reason, LOG_COMMAND)
+            if failures >= _FAIL_THRESHOLD:
+                await _try_reconnect(reason)
+
+
+def start_db_watchdog() -> None:
+    global _WATCHDOG_TASK
+    if _WATCHDOG_TASK is not None and not _WATCHDOG_TASK.done():
+        return
+    _WATCHDOG_TASK = asyncio.create_task(_watchdog_loop())
+
+
+async def stop_db_watchdog() -> None:
+    global _WATCHDOG_TASK
+    task = _WATCHDOG_TASK
+    _WATCHDOG_TASK = None
+    if task is None:
+        return
+    task.cancel()
+    with contextlib.suppress(BaseException):
+        await task
+
+
+@PriorityLifecycle.on_startup(priority=8)
+async def _start_db_watchdog() -> None:
+    start_db_watchdog()
+
+
+@PriorityLifecycle.on_shutdown(priority=10)
+async def _stop_db_watchdog() -> None:
+    await stop_db_watchdog()