收窄熔断判断范围,修改熔断逻辑 (#2145)

This commit is contained in:
Copaan
2026-06-28 22:12:07 +08:00
committed by GitHub
parent f4d2342693
commit bdc1374848
6 changed files with 59 additions and 14 deletions
@@ -39,14 +39,8 @@ class HandleEventSelectorDependencies:
_HANDLE_EVENT_PATCHED = False
_ORIGINAL_HANDLE_EVENT: Callable[..., Awaitable[None]] | None = None
_ORIGINAL_ADAPTER_HANDLE_EVENTS: dict[object, object] = {}
_MATCHER_DEADLINE_BY_LANE = {
"system": 15.0,
"command": 12.0,
"temp": 15.0,
"passive_light": 3.0,
"fallback_ai": 5.0,
}
_DEFAULT_MATCHER_DEADLINE = 5.0
_DEFAULT_MATCHER_DEADLINE = 21600.0
_MATCHER_DEADLINE_BY_LANE: dict[str, float] = {}
def _matcher_deadline_for_lane(lane: str) -> float:
@@ -76,7 +70,6 @@ async def _run_matcher_with_deadline(
with anyio_mod.fail_after(timeout):
await coro
except TimeoutError:
signal_overload(20.0)
logger.warning(
"matcher dispatch timeout: "
f"matcher={_matcher_name(matcher)}, lane={lane}, timeout={timeout:.1f}s",
@@ -2,6 +2,7 @@ from __future__ import annotations
from dataclasses import dataclass, fields
import os
from urllib.parse import urlparse
@dataclass(frozen=True, slots=True)
@@ -101,6 +102,22 @@ def _warn_invalid_env(env_name: str, raw: str, exc: Exception) -> None:
return
def _default_passive_db_limit() -> int:
try:
from zhenxun.configs.config import BotConfig
scheme = urlparse(BotConfig.db_url or "").scheme.lower()
except Exception:
scheme = ""
if scheme == "sqlite":
return 1
if scheme.startswith("postgres"):
return 6
if scheme == "mysql":
return 4
return 2
def _load_config(cls: type, prefix: str):
values = {}
default_obj = cls()
@@ -115,6 +132,9 @@ def _load_config(cls: type, prefix: str):
raw = candidate_value
break
if raw is None or not raw.strip():
if cls is AuthDispatchRuntimeConfig and item.name == "passive_db_limit":
values[item.name] = _default_passive_db_limit()
continue
values[item.name] = default
continue
try:
+2 -2
View File
@@ -27,9 +27,9 @@ MYSQL_CONFIG = {
SQLITE_CONFIG = {
"journal_mode": "WAL", # 提高并发写入性能
# SQLite 的底层锁等待应接近业务超时,避免上层放弃后 worker 仍长时间占线。
# SQLite 底层锁等待不应超过上层 DB_TIMEOUT_SECONDS=3.0。
# Windows bind mount / Docker 场景下 SQLite 不适合高写并发。
"busy_timeout": 5000,
"busy_timeout": 3000,
"foreign_keys": "ON",
}
+28
View File
@@ -15,6 +15,12 @@ _SQLITE_STALL_UNTIL = 0.0
_SQLITE_STALL_REASON = ""
_DB_UNHEALTHY_TIMEOUT_SECONDS = 30.0
_SQLITE_STALL_TIMEOUT_SECONDS = 60.0
_SQLITE_LOCK_PATTERNS = (
"database is locked",
"database is busy",
"database table is locked",
"database table is busy",
)
def _is_sqlite_connection() -> bool:
@@ -46,6 +52,24 @@ def sqlite_stall_reason() -> str:
return _SQLITE_STALL_REASON
def _is_sqlite_lock_error(exc: BaseException) -> bool:
if not _is_sqlite_connection():
return False
message = str(exc).casefold()
return any(pattern in message for pattern in _SQLITE_LOCK_PATTERNS)
def _mark_sqlite_lock_unhealthy(exc: BaseException, operation: str | None) -> None:
reason = f"{operation or 'database_operation'} sqlite lock: {exc}"
_mark_sqlite_stall(reason, _SQLITE_STALL_TIMEOUT_SECONDS)
signal_db_unhealthy(_SQLITE_STALL_TIMEOUT_SECONDS, reason=reason)
logger.warning(
"SQLite 数据库锁等待失败,已暂停低优先级数据库任务",
LOG_COMMAND,
e=exc if isinstance(exc, Exception) else None,
)
async def with_db_timeout(
coro,
timeout: float = DB_TIMEOUT_SECONDS,
@@ -78,3 +102,7 @@ async def with_db_timeout(
LOG_COMMAND,
)
raise
except Exception as exc:
if _is_sqlite_lock_error(exc):
_mark_sqlite_lock_unhealthy(exc, operation)
raise
+3 -3
View File
@@ -8,7 +8,7 @@ import time
from typing import Any
from zhenxun.services.log import logger
from zhenxun.services.message_load import should_pause_tasks, signal_db_unhealthy
from zhenxun.services.message_load import should_pause_db_tasks, signal_db_unhealthy
from zhenxun.utils.manager.priority_manager import PriorityLifecycle
LOG_COMMAND = "LowPriorityWriter"
@@ -159,7 +159,7 @@ async def _worker_loop() -> None:
except asyncio.TimeoutError:
pass
event.clear()
if should_pause_tasks():
if should_pause_db_tasks():
continue
async with _FLUSH_LOCK:
for state in list(_WRITERS.values()):
@@ -186,7 +186,7 @@ async def _flush_state(
force: bool = False,
) -> int:
if not force:
if should_pause_tasks():
if should_pause_db_tasks():
return 0
if time.monotonic() < state.backoff_until:
return 0
+4
View File
@@ -57,3 +57,7 @@ def db_unhealthy_reason() -> str:
def should_pause_tasks() -> bool:
return is_overloaded() or is_db_unhealthy()
def should_pause_db_tasks() -> bool:
return is_db_unhealthy()