mirror of
https://github.com/zhenxun-org/zhenxun_bot.git
synced 2026-09-28 16:20:56 +08:00
收窄熔断判断范围,修改熔断逻辑 (#2145)
This commit is contained in:
@@ -39,14 +39,8 @@ class HandleEventSelectorDependencies:
|
|||||||
_HANDLE_EVENT_PATCHED = False
|
_HANDLE_EVENT_PATCHED = False
|
||||||
_ORIGINAL_HANDLE_EVENT: Callable[..., Awaitable[None]] | None = None
|
_ORIGINAL_HANDLE_EVENT: Callable[..., Awaitable[None]] | None = None
|
||||||
_ORIGINAL_ADAPTER_HANDLE_EVENTS: dict[object, object] = {}
|
_ORIGINAL_ADAPTER_HANDLE_EVENTS: dict[object, object] = {}
|
||||||
_MATCHER_DEADLINE_BY_LANE = {
|
_DEFAULT_MATCHER_DEADLINE = 21600.0
|
||||||
"system": 15.0,
|
_MATCHER_DEADLINE_BY_LANE: dict[str, float] = {}
|
||||||
"command": 12.0,
|
|
||||||
"temp": 15.0,
|
|
||||||
"passive_light": 3.0,
|
|
||||||
"fallback_ai": 5.0,
|
|
||||||
}
|
|
||||||
_DEFAULT_MATCHER_DEADLINE = 5.0
|
|
||||||
|
|
||||||
|
|
||||||
def _matcher_deadline_for_lane(lane: str) -> float:
|
def _matcher_deadline_for_lane(lane: str) -> float:
|
||||||
@@ -76,7 +70,6 @@ async def _run_matcher_with_deadline(
|
|||||||
with anyio_mod.fail_after(timeout):
|
with anyio_mod.fail_after(timeout):
|
||||||
await coro
|
await coro
|
||||||
except TimeoutError:
|
except TimeoutError:
|
||||||
signal_overload(20.0)
|
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"matcher dispatch timeout: "
|
"matcher dispatch timeout: "
|
||||||
f"matcher={_matcher_name(matcher)}, lane={lane}, timeout={timeout:.1f}s",
|
f"matcher={_matcher_name(matcher)}, lane={lane}, timeout={timeout:.1f}s",
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from dataclasses import dataclass, fields
|
from dataclasses import dataclass, fields
|
||||||
import os
|
import os
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
@@ -101,6 +102,22 @@ def _warn_invalid_env(env_name: str, raw: str, exc: Exception) -> None:
|
|||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
|
def _default_passive_db_limit() -> int:
|
||||||
|
try:
|
||||||
|
from zhenxun.configs.config import BotConfig
|
||||||
|
|
||||||
|
scheme = urlparse(BotConfig.db_url or "").scheme.lower()
|
||||||
|
except Exception:
|
||||||
|
scheme = ""
|
||||||
|
if scheme == "sqlite":
|
||||||
|
return 1
|
||||||
|
if scheme.startswith("postgres"):
|
||||||
|
return 6
|
||||||
|
if scheme == "mysql":
|
||||||
|
return 4
|
||||||
|
return 2
|
||||||
|
|
||||||
|
|
||||||
def _load_config(cls: type, prefix: str):
|
def _load_config(cls: type, prefix: str):
|
||||||
values = {}
|
values = {}
|
||||||
default_obj = cls()
|
default_obj = cls()
|
||||||
@@ -115,6 +132,9 @@ def _load_config(cls: type, prefix: str):
|
|||||||
raw = candidate_value
|
raw = candidate_value
|
||||||
break
|
break
|
||||||
if raw is None or not raw.strip():
|
if raw is None or not raw.strip():
|
||||||
|
if cls is AuthDispatchRuntimeConfig and item.name == "passive_db_limit":
|
||||||
|
values[item.name] = _default_passive_db_limit()
|
||||||
|
continue
|
||||||
values[item.name] = default
|
values[item.name] = default
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -27,9 +27,9 @@ MYSQL_CONFIG = {
|
|||||||
|
|
||||||
SQLITE_CONFIG = {
|
SQLITE_CONFIG = {
|
||||||
"journal_mode": "WAL", # 提高并发写入性能
|
"journal_mode": "WAL", # 提高并发写入性能
|
||||||
# SQLite 的底层锁等待应接近业务超时,避免上层放弃后 worker 仍长时间占线。
|
# SQLite 底层锁等待不应超过上层 DB_TIMEOUT_SECONDS=3.0。
|
||||||
# Windows bind mount / Docker 场景下 SQLite 不适合高写并发。
|
# Windows bind mount / Docker 场景下 SQLite 不适合高写并发。
|
||||||
"busy_timeout": 5000,
|
"busy_timeout": 3000,
|
||||||
"foreign_keys": "ON",
|
"foreign_keys": "ON",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,12 @@ _SQLITE_STALL_UNTIL = 0.0
|
|||||||
_SQLITE_STALL_REASON = ""
|
_SQLITE_STALL_REASON = ""
|
||||||
_DB_UNHEALTHY_TIMEOUT_SECONDS = 30.0
|
_DB_UNHEALTHY_TIMEOUT_SECONDS = 30.0
|
||||||
_SQLITE_STALL_TIMEOUT_SECONDS = 60.0
|
_SQLITE_STALL_TIMEOUT_SECONDS = 60.0
|
||||||
|
_SQLITE_LOCK_PATTERNS = (
|
||||||
|
"database is locked",
|
||||||
|
"database is busy",
|
||||||
|
"database table is locked",
|
||||||
|
"database table is busy",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _is_sqlite_connection() -> bool:
|
def _is_sqlite_connection() -> bool:
|
||||||
@@ -46,6 +52,24 @@ def sqlite_stall_reason() -> str:
|
|||||||
return _SQLITE_STALL_REASON
|
return _SQLITE_STALL_REASON
|
||||||
|
|
||||||
|
|
||||||
|
def _is_sqlite_lock_error(exc: BaseException) -> bool:
|
||||||
|
if not _is_sqlite_connection():
|
||||||
|
return False
|
||||||
|
message = str(exc).casefold()
|
||||||
|
return any(pattern in message for pattern in _SQLITE_LOCK_PATTERNS)
|
||||||
|
|
||||||
|
|
||||||
|
def _mark_sqlite_lock_unhealthy(exc: BaseException, operation: str | None) -> None:
|
||||||
|
reason = f"{operation or 'database_operation'} sqlite lock: {exc}"
|
||||||
|
_mark_sqlite_stall(reason, _SQLITE_STALL_TIMEOUT_SECONDS)
|
||||||
|
signal_db_unhealthy(_SQLITE_STALL_TIMEOUT_SECONDS, reason=reason)
|
||||||
|
logger.warning(
|
||||||
|
"SQLite 数据库锁等待失败,已暂停低优先级数据库任务",
|
||||||
|
LOG_COMMAND,
|
||||||
|
e=exc if isinstance(exc, Exception) else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def with_db_timeout(
|
async def with_db_timeout(
|
||||||
coro,
|
coro,
|
||||||
timeout: float = DB_TIMEOUT_SECONDS,
|
timeout: float = DB_TIMEOUT_SECONDS,
|
||||||
@@ -78,3 +102,7 @@ async def with_db_timeout(
|
|||||||
LOG_COMMAND,
|
LOG_COMMAND,
|
||||||
)
|
)
|
||||||
raise
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
if _is_sqlite_lock_error(exc):
|
||||||
|
_mark_sqlite_lock_unhealthy(exc, operation)
|
||||||
|
raise
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import time
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from zhenxun.services.log import logger
|
from zhenxun.services.log import logger
|
||||||
from zhenxun.services.message_load import should_pause_tasks, signal_db_unhealthy
|
from zhenxun.services.message_load import should_pause_db_tasks, signal_db_unhealthy
|
||||||
from zhenxun.utils.manager.priority_manager import PriorityLifecycle
|
from zhenxun.utils.manager.priority_manager import PriorityLifecycle
|
||||||
|
|
||||||
LOG_COMMAND = "LowPriorityWriter"
|
LOG_COMMAND = "LowPriorityWriter"
|
||||||
@@ -159,7 +159,7 @@ async def _worker_loop() -> None:
|
|||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
pass
|
pass
|
||||||
event.clear()
|
event.clear()
|
||||||
if should_pause_tasks():
|
if should_pause_db_tasks():
|
||||||
continue
|
continue
|
||||||
async with _FLUSH_LOCK:
|
async with _FLUSH_LOCK:
|
||||||
for state in list(_WRITERS.values()):
|
for state in list(_WRITERS.values()):
|
||||||
@@ -186,7 +186,7 @@ async def _flush_state(
|
|||||||
force: bool = False,
|
force: bool = False,
|
||||||
) -> int:
|
) -> int:
|
||||||
if not force:
|
if not force:
|
||||||
if should_pause_tasks():
|
if should_pause_db_tasks():
|
||||||
return 0
|
return 0
|
||||||
if time.monotonic() < state.backoff_until:
|
if time.monotonic() < state.backoff_until:
|
||||||
return 0
|
return 0
|
||||||
|
|||||||
@@ -57,3 +57,7 @@ def db_unhealthy_reason() -> str:
|
|||||||
|
|
||||||
def should_pause_tasks() -> bool:
|
def should_pause_tasks() -> bool:
|
||||||
return is_overloaded() or is_db_unhealthy()
|
return is_overloaded() or is_db_unhealthy()
|
||||||
|
|
||||||
|
|
||||||
|
def should_pause_db_tasks() -> bool:
|
||||||
|
return is_db_unhealthy()
|
||||||
|
|||||||
Reference in New Issue
Block a user