bugfix:修复notice事件扩散问题 (#2132)

* bugfix:修复notice事件扩散问题

* 优化并发调度

* bugfix:修复签到样式

* bugfix：功能调用统计修复

* bugfix:修复私聊时功能调用统计显示已退群问题

* 提高插件适配兼容性

* 优化发送队列

* 修改权限检查设计

* 继续修改权限检查设计

* 完善权限检查设计

* 优化sqlite配置

* 优化数据库初始化

* 代码整理，无用代码清理

* bugfix:修复启动时数据库校验问题

* bugfix:修复预算裁剪过于激进问题

zhenxun/services/auth_observability.py

@@ -0,0 +1,547 @@
+from __future__ import annotations
+
+import asyncio
+from collections import deque
+import contextlib
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+import json
+import random
+import time
+from typing import Any, TypeVar
+
+from tortoise import Tortoise
+
+from zhenxun.builtin_plugins.hooks.auth_runtime_config import (
+    AUTH_OBSERVABILITY_RUNTIME_CONFIG,
+)
+from zhenxun.models.auth_decision_log import AuthDecisionLog
+from zhenxun.models.runtime_backpressure_log import RuntimeBackpressureLog
+from zhenxun.services.log import logger
+from zhenxun.utils.manager.priority_manager import PriorityLifecycle
+
+LOG_COMMAND = "AuthObservability"
+
+_BUFFER_MAX_RETAIN = AUTH_OBSERVABILITY_RUNTIME_CONFIG.buffer_max_retain
+_FLUSH_TRIGGER_SIZE = AUTH_OBSERVABILITY_RUNTIME_CONFIG.flush_trigger_size
+_FLUSH_BATCH_SIZE = AUTH_OBSERVABILITY_RUNTIME_CONFIG.flush_batch_size
+_FLUSH_INTERVAL_SECONDS = AUTH_OBSERVABILITY_RUNTIME_CONFIG.flush_interval_seconds
+_DROP_LOG_INTERVAL_SECONDS = AUTH_OBSERVABILITY_RUNTIME_CONFIG.drop_log_interval_seconds
+_ALLOW_SAMPLE_RATE = AUTH_OBSERVABILITY_RUNTIME_CONFIG.allow_sample_rate
+_OVERLOADED_ALLOW_SAMPLE_RATE = (
+    AUTH_OBSERVABILITY_RUNTIME_CONFIG.overloaded_allow_sample_rate
+)
+_NON_ALLOW_SAMPLE_RATE = AUTH_OBSERVABILITY_RUNTIME_CONFIG.non_allow_sample_rate
+_BACKPRESSURE_SAMPLE_RATE = AUTH_OBSERVABILITY_RUNTIME_CONFIG.backpressure_sample_rate
+_BACKPRESSURE_SEVERE_ACTIVE_THRESHOLD = (
+    AUTH_OBSERVABILITY_RUNTIME_CONFIG.backpressure_severe_active_threshold
+)
+
+
+@dataclass(slots=True)
+class AuthDecisionLogRecord:
+    bot_id: str | None
+    platform: str | None
+    group_id: str | None
+    user_id: str | None
+    module: str | None
+    effect: str
+    reason: str | None = None
+    shadow_effect: str | None = None
+    shadow_reason: str | None = None
+    side_effect_state: dict[str, Any] | None = None
+    latency_ms: float = 0.0
+    overloaded: bool = False
+
+    def to_model(self) -> AuthDecisionLog:
+        return AuthDecisionLog(
+            bot_id=self.bot_id,
+            platform=self.platform,
+            group_id=self.group_id,
+            user_id=self.user_id,
+            module=self.module,
+            effect=self.effect,
+            reason=self.reason,
+            shadow_effect=self.shadow_effect,
+            shadow_reason=self.shadow_reason,
+            side_effect_state=json.dumps(
+                self.side_effect_state,
+                ensure_ascii=False,
+                separators=(",", ":"),
+            )[:4000]
+            if self.side_effect_state
+            else None,
+            latency_ms=self.latency_ms,
+            overloaded=self.overloaded,
+        )
+
+
+@dataclass(slots=True)
+class RuntimeBackpressureLogRecord:
+    scope_key: str | None
+    reason: str | None
+    lane: str | None
+    action: str
+    queue_size: int = 0
+    active_count: int = 0
+    duration_ms: float = 0.0
+
+    def to_model(self) -> RuntimeBackpressureLog:
+        return RuntimeBackpressureLog(
+            scope_key=self.scope_key,
+            reason=self.reason,
+            lane=self.lane,
+            action=self.action,
+            queue_size=self.queue_size,
+            active_count=self.active_count,
+            duration_ms=self.duration_ms,
+        )
+
+
+_auth_decision_buffer: deque[AuthDecisionLogRecord] = deque()
+_backpressure_buffer: deque[RuntimeBackpressureLogRecord] = deque()
+_buffer_lock = asyncio.Lock()
+_flush_lock = asyncio.Lock()
+_flush_task: asyncio.Task[None] | None = None
+_dropped = 0
+_last_drop_log_at = 0.0
+_last_schema_repair_at = 0.0
+_SCHEMA_REPAIR_INTERVAL_SECONDS = 300.0
+
+T = TypeVar("T")
+
+
+def _ensure_flush_task() -> None:
+    global _flush_task
+    if _flush_task is not None and not _flush_task.done():
+        return
+    _flush_task = asyncio.create_task(_flush_loop())
+
+
+def _record_drop() -> None:
+    global _dropped, _last_drop_log_at
+    _dropped += 1
+    now = time.monotonic()
+    if now - _last_drop_log_at < _DROP_LOG_INTERVAL_SECONDS:
+        return
+    _last_drop_log_at = now
+    logger.warning(
+        "auth observability buffer full, dropped "
+        f"{_dropped} records, auth_backlog={len(_auth_decision_buffer)}, "
+        f"backpressure_backlog={len(_backpressure_buffer)}",
+        LOG_COMMAND,
+    )
+
+
+def _sample(rate: float) -> bool:
+    if rate >= 1:
+        return True
+    if rate <= 0:
+        return False
+    return random.random() < rate
+
+
+def _auth_decision_sample_rate(effect: str, overloaded: bool) -> float:
+    if effect != "allow":
+        return _NON_ALLOW_SAMPLE_RATE
+    if overloaded:
+        return _OVERLOADED_ALLOW_SAMPLE_RATE
+    return _ALLOW_SAMPLE_RATE
+
+
+def _backpressure_sample_rate(record: RuntimeBackpressureLogRecord) -> float:
+    if record.reason and record.reason.startswith("hooks_"):
+        return 1.0
+    if record.active_count >= _BACKPRESSURE_SEVERE_ACTIVE_THRESHOLD:
+        return 1.0
+    if record.action in {"skip", "defer"}:
+        return _BACKPRESSURE_SAMPLE_RATE
+    return min(_BACKPRESSURE_SAMPLE_RATE, 0.02)
+
+
+async def _append_auth_decision_record(record: AuthDecisionLogRecord) -> None:
+    _ensure_flush_task()
+    async with _buffer_lock:
+        total = len(_auth_decision_buffer) + len(_backpressure_buffer)
+        if total >= _BUFFER_MAX_RETAIN:
+            if len(_auth_decision_buffer) >= len(_backpressure_buffer):
+                with contextlib.suppress(IndexError):
+                    _auth_decision_buffer.popleft()
+            else:
+                with contextlib.suppress(IndexError):
+                    _backpressure_buffer.popleft()
+            _record_drop()
+        _auth_decision_buffer.append(record)
+        should_flush = (
+            len(_auth_decision_buffer) + len(_backpressure_buffer)
+            >= _FLUSH_TRIGGER_SIZE
+            and not _flush_lock.locked()
+        )
+    if should_flush:
+        # Fire-and-forget keeps auth hot path independent of database stalls.
+        asyncio.create_task(flush_auth_observability_buffer("缓冲区触发"))  # noqa: RUF006
+
+
+async def _append_backpressure_record(record: RuntimeBackpressureLogRecord) -> None:
+    _ensure_flush_task()
+    async with _buffer_lock:
+        total = len(_auth_decision_buffer) + len(_backpressure_buffer)
+        if total >= _BUFFER_MAX_RETAIN:
+            if len(_auth_decision_buffer) >= len(_backpressure_buffer):
+                with contextlib.suppress(IndexError):
+                    _auth_decision_buffer.popleft()
+            else:
+                with contextlib.suppress(IndexError):
+                    _backpressure_buffer.popleft()
+            _record_drop()
+        _backpressure_buffer.append(record)
+        should_flush = (
+            len(_auth_decision_buffer) + len(_backpressure_buffer)
+            >= _FLUSH_TRIGGER_SIZE
+            and not _flush_lock.locked()
+        )
+    if should_flush:
+        # Fire-and-forget keeps auth hot path independent of database stalls.
+        asyncio.create_task(flush_auth_observability_buffer("缓冲区触发"))  # noqa: RUF006
+
+
+async def append_auth_decision_log(
+    *,
+    bot_id: str | None,
+    platform: str | None,
+    group_id: str | None,
+    user_id: str | None,
+    module: str | None,
+    effect: str,
+    reason: str | None = None,
+    shadow_effect: str | None = None,
+    shadow_reason: str | None = None,
+    side_effect_state: dict[str, Any] | None = None,
+    latency_ms: float = 0.0,
+    overloaded: bool = False,
+) -> None:
+    if shadow_effect is None and not _sample(
+        _auth_decision_sample_rate(effect, overloaded)
+    ):
+        return
+    record = AuthDecisionLogRecord(
+        bot_id=bot_id,
+        platform=platform,
+        group_id=group_id,
+        user_id=user_id,
+        module=module,
+        effect=effect,
+        reason=(reason or "")[:255] or None,
+        shadow_effect=(shadow_effect or "")[:32] or None,
+        shadow_reason=(shadow_reason or "")[:255] or None,
+        side_effect_state=side_effect_state,
+        latency_ms=latency_ms,
+        overloaded=overloaded,
+    )
+    await _append_auth_decision_record(record)
+
+
+async def append_runtime_backpressure_log(
+    *,
+    scope_key: str | None,
+    reason: str | None,
+    lane: str | None,
+    action: str,
+    queue_size: int = 0,
+    active_count: int = 0,
+    duration_ms: float = 0.0,
+) -> None:
+    record = RuntimeBackpressureLogRecord(
+        scope_key=(scope_key or "")[:255] or None,
+        reason=(reason or "")[:255] or None,
+        lane=(lane or "")[:64] or None,
+        action=action,
+        queue_size=queue_size,
+        active_count=active_count,
+        duration_ms=duration_ms,
+    )
+    if not _sample(_backpressure_sample_rate(record)):
+        return
+    await _append_backpressure_record(record)
+
+
+async def _flush_loop() -> None:
+    while True:
+        await asyncio.sleep(_FLUSH_INTERVAL_SECONDS)
+        try:
+            await flush_auth_observability_buffer("定时")
+        except asyncio.CancelledError:
+            raise
+        except Exception as exc:
+            logger.warning("定时批量写入权限观测日志失败", LOG_COMMAND, e=exc)
+
+
+async def _drain_batch(buffer: deque[T]) -> list[T]:
+    batch: list[T] = []
+    async with _buffer_lock:
+        while buffer and len(batch) < _FLUSH_BATCH_SIZE:
+            batch.append(buffer.popleft())
+    return batch
+
+
+async def _restore_batch(buffer: deque[T], batch: list[T]) -> None:
+    async with _buffer_lock:
+        retain_count = max(_BUFFER_MAX_RETAIN - len(buffer), 0)
+        for record in reversed(batch[-retain_count:]):
+            buffer.appendleft(record)
+
+
+def _is_schema_mismatch_error(exc: Exception) -> bool:
+    message = str(exc).lower()
+    return any(
+        marker in message
+        for marker in (
+            "no column named",
+            "unknown column",
+            "column does not exist",
+            "no such column",
+        )
+    )
+
+
+async def _try_repair_auth_schema_once() -> bool:
+    global _last_schema_repair_at
+    now = time.monotonic()
+    if now - _last_schema_repair_at < _SCHEMA_REPAIR_INTERVAL_SECONDS:
+        return False
+    _last_schema_repair_at = now
+    try:
+        from zhenxun.services.db_context.schema_guard import repair_table_schema
+
+        await repair_table_schema("auth_decision_log")
+        await repair_table_schema("runtime_backpressure_log")
+        return True
+    except Exception as exc:
+        logger.warning("权限观测日志表结构自修复失败", LOG_COMMAND, e=exc)
+        return False
+
+
+async def flush_auth_observability_buffer(reason: str) -> int:
+    async with _flush_lock:
+        written = 0
+        while True:
+            auth_batch = await _drain_batch(_auth_decision_buffer)
+            backpressure_batch = await _drain_batch(_backpressure_buffer)
+            if not auth_batch and not backpressure_batch:
+                break
+            try:
+                if auth_batch:
+                    await AuthDecisionLog.bulk_create(
+                        [record.to_model() for record in auth_batch],
+                        _FLUSH_BATCH_SIZE,
+                    )
+                    written += len(auth_batch)
+                if backpressure_batch:
+                    await RuntimeBackpressureLog.bulk_create(
+                        [record.to_model() for record in backpressure_batch],
+                        _FLUSH_BATCH_SIZE,
+                    )
+                    written += len(backpressure_batch)
+            except Exception as exc:
+                if _is_schema_mismatch_error(exc):
+                    if await _try_repair_auth_schema_once():
+                        try:
+                            if auth_batch:
+                                await AuthDecisionLog.bulk_create(
+                                    [record.to_model() for record in auth_batch],
+                                    _FLUSH_BATCH_SIZE,
+                                )
+                                written += len(auth_batch)
+                            if backpressure_batch:
+                                await RuntimeBackpressureLog.bulk_create(
+                                    [
+                                        record.to_model()
+                                        for record in backpressure_batch
+                                    ],
+                                    _FLUSH_BATCH_SIZE,
+                                )
+                                written += len(backpressure_batch)
+                            continue
+                        except Exception as retry_exc:
+                            exc = retry_exc
+                    dropped = len(auth_batch) + len(backpressure_batch)
+                    logger.warning(
+                        f"{reason}批量写入权限观测日志遇到表结构不匹配，"
+                        f"已丢弃低优先级观测日志 {dropped} 条，等待下次启动修复",
+                        LOG_COMMAND,
+                        e=exc,
+                    )
+                    return written
+                await _restore_batch(_auth_decision_buffer, auth_batch)
+                await _restore_batch(_backpressure_buffer, backpressure_batch)
+                logger.error(f"{reason}批量写入权限观测日志失败", LOG_COMMAND, e=exc)
+                return written
+        if written:
+            logger.debug(f"{reason}批量写入权限观测日志 {written} 条", LOG_COMMAND)
+        return written
+
+
+async def stop_auth_observability_buffer() -> int:
+    global _flush_task
+    task = _flush_task
+    _flush_task = None
+    if task is not None:
+        task.cancel()
+        with contextlib.suppress(BaseException):
+            await task
+    return await flush_auth_observability_buffer("关闭")
+
+
+def _percentile(values: list[float], ratio: float) -> float:
+    if not values:
+        return 0.0
+    ordered = sorted(values)
+    index = min(max(round((len(ordered) - 1) * ratio), 0), len(ordered) - 1)
+    return round(ordered[index], 3)
+
+
+def _bucket_counts(rows: list[dict[str, Any]], field: str) -> dict[str, int]:
+    counts: dict[str, int] = {}
+    for row in rows:
+        key = str(row.get(field) or "<none>")
+        counts[key] = counts.get(key, 0) + 1
+    return counts
+
+
+def _lane_budget_advice(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
+    buckets: dict[str, list[dict[str, Any]]] = {}
+    for row in rows:
+        lane = str(row.get("lane") or "<unknown>")
+        buckets.setdefault(lane, []).append(row)
+    advice: dict[str, dict[str, Any]] = {}
+    for lane, items in buckets.items():
+        if lane == "<unknown>":
+            continue
+        durations = [float(item.get("duration_ms") or 0.0) for item in items]
+        slow_waits = sum(1 for value in durations if value >= 200.0)
+        active_max = max(
+            (int(item.get("active_count") or 0) for item in items), default=0
+        )
+        total = len(items)
+        if not total:
+            continue
+        pressure_ratio = slow_waits / total
+        if pressure_ratio >= 0.2 or active_max >= 5:
+            action = "increase_or_split"
+        elif pressure_ratio == 0 and active_max <= 1 and total >= 20:
+            action = "can_reduce"
+        else:
+            action = "keep"
+        advice[lane] = {
+            "samples": total,
+            "slow_waits": slow_waits,
+            "pressure_ratio": round(pressure_ratio, 3),
+            "active_max": active_max,
+            "p95_duration_ms": _percentile(durations, 0.95),
+            "action": action,
+        }
+    return advice
+
+
+def _query_placeholder() -> str:
+    try:
+        connection = Tortoise.get_connection("default")
+        if (
+            getattr(connection, "capabilities", None)
+            and getattr(
+                connection.capabilities,
+                "dialect",
+                "",
+            )
+            == "postgres"
+        ):
+            return "$1"
+    except Exception:
+        return "?"
+    return "?"
+
+
+async def build_auth_observability_report(*, hours: float = 24.0) -> dict[str, Any]:
+    since = datetime.now() - timedelta(hours=hours)
+    db = Tortoise.get_connection("default")
+    placeholder = _query_placeholder()
+    auth_rows = await db.execute_query_dict(
+        "SELECT module, effect, reason, shadow_effect, shadow_reason, latency_ms, "
+        f"overloaded FROM auth_decision_log WHERE create_time >= {placeholder} "
+        "ORDER BY create_time DESC LIMIT 100000",
+        [since],
+    )
+    backpressure_rows = await db.execute_query_dict(
+        "SELECT scope_key, lane, reason, action, queue_size, active_count, duration_ms "
+        f"FROM runtime_backpressure_log WHERE create_time >= {placeholder} "
+        "ORDER BY create_time DESC LIMIT 100000",
+        [since],
+    )
+
+    module_buckets: dict[str, list[dict[str, Any]]] = {}
+    for row in auth_rows:
+        module_buckets.setdefault(str(row.get("module") or "<unknown>"), []).append(row)
+    module_stats: list[dict[str, Any]] = []
+    for module, items in module_buckets.items():
+        latencies = [float(item.get("latency_ms") or 0.0) for item in items]
+        module_stats.append(
+            {
+                "module": module,
+                "total": len(items),
+                "effects": _bucket_counts(items, "effect"),
+                "shadow_effects": _bucket_counts(items, "shadow_effect"),
+                "avg_latency_ms": round(sum(latencies) / len(latencies), 3)
+                if latencies
+                else 0.0,
+                "p95_latency_ms": _percentile(latencies, 0.95),
+                "overloaded": sum(1 for item in items if bool(item.get("overloaded"))),
+            }
+        )
+
+    backpressure_buckets: dict[str, list[dict[str, Any]]] = {}
+    for row in backpressure_rows:
+        key = f"{row.get('lane') or '<unknown>'}:{row.get('reason') or '<none>'}"
+        backpressure_buckets.setdefault(key, []).append(row)
+    backpressure_stats: list[dict[str, Any]] = []
+    for key, items in backpressure_buckets.items():
+        durations = [float(item.get("duration_ms") or 0.0) for item in items]
+        backpressure_stats.append(
+            {
+                "key": key,
+                "total": len(items),
+                "actions": _bucket_counts(items, "action"),
+                "avg_duration_ms": round(sum(durations) / len(durations), 3)
+                if durations
+                else 0.0,
+                "p95_duration_ms": _percentile(durations, 0.95),
+            }
+        )
+
+    return {
+        "created_at": datetime.now().isoformat(timespec="seconds"),
+        "window_hours": hours,
+        "auth_decisions": {
+            "total": len(auth_rows),
+            "effects": _bucket_counts(auth_rows, "effect"),
+            "shadow_effects": _bucket_counts(auth_rows, "shadow_effect"),
+            "top_modules_by_p95": sorted(
+                module_stats,
+                key=lambda item: (item["p95_latency_ms"], item["total"]),
+                reverse=True,
+            )[:30],
+        },
+        "backpressure": {
+            "total": len(backpressure_rows),
+            "lane_budget_advice": _lane_budget_advice(backpressure_rows),
+            "top_reasons": sorted(
+                backpressure_stats,
+                key=lambda item: (item["total"], item["p95_duration_ms"]),
+                reverse=True,
+            )[:30],
+        },
+    }
+
+
+@PriorityLifecycle.on_shutdown(priority=90)
+async def _flush_auth_observability_buffer_on_shutdown() -> None:
+    await stop_auth_observability_buffer()

zhenxun/services/cache/runtime_cache.py

@@ -715,12 +715,22 @@ class PluginInfoMemoryCache:
             return
         await cls.refresh()
 
+    @classmethod
+    def is_loaded(cls) -> bool:
+        return cls._loaded
+
     @classmethod
     async def get_by_module(cls, module: str) -> "PluginInfo | None":
         if not cls._loaded:
             await cls.ensure_loaded()
         return cls._to_model(cls._by_module.get(module))
 
+    @classmethod
+    def get_by_module_if_ready(cls, module: str) -> "PluginInfo | None":
+        if not cls._loaded:
+            return None
+        return cls._to_model(cls._by_module.get(module))
+
     @classmethod
     async def get_all(cls) -> dict[str, "PluginInfo"]:
         if not cls._loaded:
@@ -851,6 +861,10 @@ class BotMemoryCache:
             return
         await cls.refresh()
 
+    @classmethod
+    def is_loaded(cls) -> bool:
+        return cls._loaded
+
     @classmethod
     async def get(cls, bot_id: str | None) -> BotSnapshot | None:
         bot_id = cls._normalize(bot_id)
@@ -866,6 +880,19 @@ class BotMemoryCache:
         cls._mark_negative(bot_id)
         return None
 
+    @classmethod
+    def get_if_ready(cls, bot_id: str | None) -> BotSnapshot | None:
+        bot_id = cls._normalize(bot_id)
+        if not bot_id or not cls._loaded:
+            return None
+        entry = cls._by_id.get(bot_id)
+        if entry:
+            return entry
+        if cls._is_negative(bot_id):
+            return None
+        cls._mark_negative(bot_id)
+        return None
+
     @classmethod
     async def get_all(cls) -> dict[str, BotSnapshot]:
         if not cls._loaded:
@@ -1199,6 +1226,10 @@ class LevelUserMemoryCache:
             return
         await cls.refresh()
 
+    @classmethod
+    def is_loaded(cls) -> bool:
+        return cls._loaded
+
     @classmethod
     async def ensure_fresh(cls) -> None:
         interval = LEVEL_MEM_REFRESH_INTERVAL
@@ -1244,6 +1275,23 @@ class LevelUserMemoryCache:
                 group_user = cls._by_key.get(group_key)
         return global_user, group_user
 
+    @classmethod
+    def get_levels_if_ready(
+        cls, user_id: str | None, group_id: str | None
+    ) -> tuple[LevelUserSnapshot | None, LevelUserSnapshot | None] | None:
+        if not cls._loaded:
+            return None
+        global_user = None
+        group_user = None
+        global_key = cls._key(user_id, "")
+        if global_key:
+            global_user = cls._by_key.get(global_key)
+        if group_id:
+            group_key = cls._key(user_id, group_id)
+            if group_key:
+                group_user = cls._by_key.get(group_key)
+        return global_user, group_user
+
     @classmethod
     async def get_max_level(cls, user_id: str | None) -> int:
         user_id = cls._normalize(user_id)
@@ -1576,6 +1624,10 @@ class PluginLimitMemoryCache:
             return
         await cls.refresh()
 
+    @classmethod
+    def is_loaded(cls) -> bool:
+        return cls._loaded
+
     @classmethod
     async def get_limits(cls, module: str) -> list[PluginLimitSnapshot]:
         normalized = cls._normalize(module)
@@ -1592,6 +1644,22 @@ class PluginLimitMemoryCache:
         cls._mark_negative(module)
         return []
 
+    @classmethod
+    def get_limits_if_ready(cls, module: str) -> list[PluginLimitSnapshot] | None:
+        normalized = cls._normalize(module)
+        if not normalized:
+            return []
+        module = normalized
+        if not cls._loaded:
+            return None
+        limits = cls._by_module.get(module)
+        if limits is not None:
+            return limits
+        if cls._is_negative(module):
+            return []
+        cls._mark_negative(module)
+        return []
+
     @classmethod
     def get_all_limits(cls) -> list[PluginLimitSnapshot]:
         return list(cls._by_id.values())

zhenxun/services/db_context/__init__.py

@@ -1,8 +1,10 @@
 import asyncio
 import hashlib
 import json
+import os
 from pathlib import Path
 import re
+from typing import Literal
 from urllib.parse import urlparse
 
 import aiofiles
@@ -27,8 +29,12 @@ from .config import (
     prompt,
 )
 from .exceptions import DbConnectError, DbUrlIsNode
+from .schema_guard import repair_safe_schema_drift
+from .schema_ops import SchemaOpRisk, normalize_schema_ops
 from .utils import with_db_timeout
 
+Dialect = Literal["sqlite", "postgres", "mysql", "unknown"]
+
 MODELS = db_model.models
 SCRIPT_METHOD = db_model.script_method
 
@@ -47,7 +53,66 @@ __all__ = [
 
 driver = nonebot.get_driver()
 
-_SCRIPT_HASH_FILE = Path() / "data" / ".db_script_hash"
+_SCRIPT_HASH_DIR = Path() / "data" / ".db_script_hashes"
+_TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+def _connection_dialect() -> Dialect:
+    try:
+        connection = Tortoise.get_connection("default")
+        capabilities = getattr(connection, "capabilities", None)
+        raw = str(getattr(capabilities, "dialect", "") or "").lower()
+        if raw.startswith("sqlite"):
+            return "sqlite"
+        if raw.startswith("postgres"):
+            return "postgres"
+        if raw.startswith("mysql"):
+            return "mysql"
+    except Exception:
+        pass
+    return "unknown"
+
+
+def _allow_guarded_schema_ops() -> bool:
+    """Whether startup may run guarded SchemaOp migrations.
+
+    Safe SchemaOps are limited to non-destructive changes such as adding nullable
+    columns and non-unique indexes. Guarded operations may rename, drop, or alter
+    columns, so keep them opt-in to avoid damaging existing databases during
+    normal startup.
+    """
+    return os.getenv("DB_SCHEMA_RUN_GUARDED_OPS", "").strip().lower() in _TRUE_VALUES
+
+
+def _extract_alter_table_name(sql: str) -> str | None:
+    match = re.match(r"ALTER\s+TABLE\s+[`\"]?(\w+)[`\"]?", sql, re.IGNORECASE)
+    return match.group(1) if match else None
+
+
+def _extract_create_index_table_name(sql: str) -> str | None:
+    match = re.search(r"\bON\s+[`\"]?(\w+)[`\"]?\s*\(", sql, re.IGNORECASE)
+    return match.group(1) if match else None
+
+
+def _db_script_hash_file(script_fingerprint: str) -> Path:
+    parsed = urlparse(BotConfig.db_url or "")
+    dialect = parsed.scheme or "unknown"
+    if dialect == "sqlite":
+        db_identity = str(Path(parsed.path).resolve())
+    else:
+        db_identity = f"{parsed.hostname or ''}:{parsed.port or ''}{parsed.path}"
+    db_hash = hashlib.md5(
+        json.dumps(
+            {
+                "dialect": dialect,
+                "db": db_identity,
+                "script": script_fingerprint,
+            },
+            ensure_ascii=False,
+            sort_keys=True,
+        ).encode()
+    ).hexdigest()
+    return _SCRIPT_HASH_DIR / f"{db_hash}.json"
 
 
 def get_config() -> dict:
@@ -131,20 +196,39 @@ async def init():
                 f"<u><y>{len(db_model.script_method)}</y></u> 个..."
             )
             sql_list = []
+            allow_guarded_ops = _allow_guarded_schema_ops()
             for module, func in db_model.script_method:
                 try:
-                    sql = await func() if is_coroutine_callable(func) else func()
-                    if sql:
-                        sql_list += sql
+                    items = await func() if is_coroutine_callable(func) else func()
+                    if not items:
+                        continue
+                    for item in items:
+                        if not isinstance(item, str):
+                            if item.risk == SchemaOpRisk.MANUAL:
+                                logger.debug(f"{module} 跳过手动迁移动作: {item}")
+                                continue
+                            if (
+                                item.risk == SchemaOpRisk.GUARDED
+                                and not allow_guarded_ops
+                            ):
+                                logger.debug(f"{module} 跳过受保护迁移动作: {item}")
+                                continue
+                            if item.risk != SchemaOpRisk.SAFE and not allow_guarded_ops:
+                                logger.debug(f"{module} 跳过未知风险迁移动作: {item}")
+                                continue
+                        sql_list += normalize_schema_ops([item], _connection_dialect())
                 except Exception as e:
                     logger.debug(f"{module} 执行SCRIPT_METHOD方法出错...", e=e)
             if sql_list:
                 fingerprint = hashlib.md5(
                     json.dumps(sorted(sql_list), ensure_ascii=False).encode()
                 ).hexdigest()
+                script_hash_file = _db_script_hash_file(fingerprint)
                 need_run = not (
-                    _SCRIPT_HASH_FILE.exists()
-                    and _SCRIPT_HASH_FILE.read_text(encoding="utf-8").strip()
+                    script_hash_file.exists()
+                    and json.loads(script_hash_file.read_text(encoding="utf-8")).get(
+                        "script_fingerprint"
+                    )
                     == fingerprint
                 )
                 if need_run:
@@ -186,12 +270,16 @@ async def init():
 
                     for sql in sql_list:
                         # 对于 ALTER TABLE 操作，先检查表是否存在
-                        if sql.strip().upper().startswith("ALTER TABLE"):
-                            match = re.match(
-                                r"ALTER\s+TABLE\s+(\w+)", sql, re.IGNORECASE
-                            )
-                            if match:
-                                table_name = match.group(1)
+                        sql_upper = sql.strip().upper()
+                        if sql_upper.startswith("ALTER TABLE"):
+                            table_name = _extract_alter_table_name(sql)
+                            if table_name:
+                                if not await table_exists(table_name):
+                                    logger.debug(f"跳过SQL（表不存在）: {sql}")
+                                    continue
+                        elif sql_upper.startswith("CREATE INDEX"):
+                            table_name = _extract_create_index_table_name(sql)
+                            if table_name:
                                 if not await table_exists(table_name):
                                     logger.debug(f"跳过SQL（表不存在）: {sql}")
                                     continue
@@ -236,13 +324,31 @@ async def init():
                         except Exception as e:
                             logger.debug(f"执行SQL: {sql} 错误...", e=e)
                     logger.debug("SCRIPT_METHOD方法执行完毕!")
-                    _SCRIPT_HASH_FILE.parent.mkdir(parents=True, exist_ok=True)
-                    _SCRIPT_HASH_FILE.write_text(fingerprint, encoding="utf-8")
+                    script_hash_file.parent.mkdir(parents=True, exist_ok=True)
+                    script_hash_file.write_text(
+                        json.dumps(
+                            {
+                                "dialect": urlparse(BotConfig.db_url or "").scheme,
+                                "db_url_hash": hashlib.md5(
+                                    (BotConfig.db_url or "").encode()
+                                ).hexdigest(),
+                                "script_fingerprint": fingerprint,
+                            },
+                            ensure_ascii=False,
+                            indent=2,
+                        ),
+                        encoding="utf-8",
+                    )
                 else:
                     logger.debug("迁移脚本无变化，跳过执行")
+        # Tortoise may emit column comments/index SQL during generate_schemas().
+        # On existing databases with newly added nullable fields, PostgreSQL can
+        # fail before the post-generate SchemaGuard gets a chance to repair drift.
+        await repair_safe_schema_drift()
         logger.debug("开始生成数据库表结构...")
         await Tortoise.generate_schemas()
         logger.debug("数据库表结构生成完毕!")
+        await repair_safe_schema_drift()
         logger.info("Database loaded successfully!")
     except Exception as e:
         raise DbConnectError(f"数据库连接错误... e:{e}") from e

zhenxun/services/db_context/config.py

@@ -24,7 +24,8 @@ MYSQL_CONFIG = {
 
 SQLITE_CONFIG = {
     "journal_mode": "WAL",  # 提高并发写入性能
-    "timeout": 30,  # 锁等待超时（可选）
+    "busy_timeout": 30000,  # SQLite 锁等待超时，单位毫秒
+    "foreign_keys": "ON",
 }
 
 

zhenxun/services/db_context/schema_guard.py

@@ -0,0 +1,593 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from dataclasses import dataclass
+from datetime import datetime
+import json
+from pathlib import Path
+from typing import Any, Literal
+
+from tortoise import Tortoise
+from tortoise.exceptions import OperationalError
+
+from zhenxun.services.log import logger
+
+from .config import DB_TIMEOUT_SECONDS, LOG_COMMAND
+
+Dialect = Literal["sqlite", "postgres", "mysql", "unknown"]
+
+
+@dataclass(slots=True)
+class SchemaGuardResult:
+    checked_tables: int = 0
+    repaired_columns: int = 0
+    repaired_indexes: int = 0
+    skipped_columns: int = 0
+    skipped_indexes: int = 0
+    type_mismatches: int = 0
+    warnings: int = 0
+    drift: list[dict[str, Any]] | None = None
+
+
+@dataclass(slots=True)
+class ColumnInfo:
+    name: str
+    data_type: str
+    nullable: bool | None = None
+    default: str | None = None
+
+
+def _quote_identifier(identifier: str, dialect: Dialect) -> str:
+    escaped = identifier.replace('"', '""')
+    if dialect == "mysql":
+        return f"`{identifier.replace('`', '``')}`"
+    return f'"{escaped}"'
+
+
+def _connection_dialect(connection: Any) -> Dialect:
+    capabilities = getattr(connection, "capabilities", None)
+    raw = str(getattr(capabilities, "dialect", "") or "").lower()
+    if raw.startswith("sqlite"):
+        return "sqlite"
+    if raw.startswith("postgres"):
+        return "postgres"
+    if raw.startswith("mysql"):
+        return "mysql"
+    return "unknown"
+
+
+def _is_safe_missing_field(field: Any) -> bool:
+    if getattr(field, "pk", False):
+        return False
+    if getattr(field, "generated", False):
+        return False
+    if bool(getattr(field, "null", False)):
+        return True
+    default = getattr(field, "default", None)
+    return default is not None
+
+
+def _field_sql_type(field: Any, dialect: Dialect) -> str | None:
+    if dialect != "unknown" and hasattr(field, "get_for_dialect"):
+        with contextlib.suppress(Exception):
+            value = field.get_for_dialect(dialect, "SQL_TYPE")
+            if value:
+                return str(value)
+    value = getattr(field, "SQL_TYPE", None)
+    return str(value) if value else None
+
+
+def _is_db_field(field: Any, dialect: Dialect) -> bool:
+    if getattr(field, "virtual", False):
+        return False
+    return _field_sql_type(field, dialect) is not None
+
+
+def _field_default_sql(field: Any, dialect: Dialect) -> str:
+    default = getattr(field, "default", None)
+    if default is None or callable(default):
+        return ""
+    if isinstance(default, bool):
+        if dialect == "postgres":
+            return " DEFAULT TRUE" if default else " DEFAULT FALSE"
+        return " DEFAULT 1" if default else " DEFAULT 0"
+    if isinstance(default, int | float):
+        return f" DEFAULT {default}"
+    if isinstance(default, str):
+        escaped = default.replace("'", "''")
+        return f" DEFAULT '{escaped}'"
+    return ""
+
+
+def _missing_column_sql(
+    table: str,
+    column: str,
+    field: Any,
+    dialect: Dialect,
+) -> str | None:
+    sql_type = _field_sql_type(field, dialect)
+    if not sql_type:
+        return None
+    table_sql = _quote_identifier(table, dialect)
+    column_sql = _quote_identifier(column, dialect)
+    null_sql = "" if bool(getattr(field, "null", False)) else " NOT NULL"
+    default_sql = _field_default_sql(field, dialect)
+    if not bool(getattr(field, "null", False)) and not default_sql:
+        return None
+    if dialect == "postgres":
+        return (
+            f"ALTER TABLE {table_sql} ADD COLUMN IF NOT EXISTS "
+            f"{column_sql} {sql_type}{default_sql}{null_sql}"
+        )
+    return (
+        f"ALTER TABLE {table_sql} ADD COLUMN "
+        f"{column_sql} {sql_type}{default_sql}{null_sql}"
+    )
+
+
+async def _table_columns(
+    connection: Any, table: str, dialect: Dialect
+) -> dict[str, ColumnInfo] | None:
+    if dialect == "sqlite":
+        rows = await connection.execute_query_dict(
+            f"PRAGMA table_xinfo({_quote_identifier(table, dialect)})"
+        )
+        if not rows:
+            return None
+        return {
+            str(row.get("name")): ColumnInfo(
+                name=str(row.get("name")),
+                data_type=str(row.get("type") or ""),
+                nullable=not bool(row.get("notnull")),
+                default=str(row.get("dflt_value"))
+                if row.get("dflt_value") is not None
+                else None,
+            )
+            for row in rows
+            if row.get("name")
+        }
+    if dialect == "postgres":
+        rows = await connection.execute_query_dict(
+            "SELECT column_name, data_type, is_nullable, column_default "
+            "FROM information_schema.columns "
+            "WHERE table_schema = current_schema() AND table_name = $1",
+            [table],
+        )
+        if not rows:
+            return None
+        return {
+            str(row.get("column_name")): ColumnInfo(
+                name=str(row.get("column_name")),
+                data_type=str(row.get("data_type") or ""),
+                nullable=str(row.get("is_nullable") or "").upper() == "YES",
+                default=str(row.get("column_default"))
+                if row.get("column_default") is not None
+                else None,
+            )
+            for row in rows
+            if row.get("column_name")
+        }
+    if dialect == "mysql":
+        rows = await connection.execute_query_dict(
+            "SELECT COLUMN_NAME AS column_name, COLUMN_TYPE AS column_type, "
+            "IS_NULLABLE AS is_nullable, COLUMN_DEFAULT AS column_default "
+            "FROM INFORMATION_SCHEMA.COLUMNS "
+            "WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = %s",
+            [table],
+        )
+        if not rows:
+            return None
+        return {
+            str(row.get("column_name")): ColumnInfo(
+                name=str(row.get("column_name")),
+                data_type=str(row.get("column_type") or ""),
+                nullable=str(row.get("is_nullable") or "").upper() == "YES",
+                default=str(row.get("column_default"))
+                if row.get("column_default") is not None
+                else None,
+            )
+            for row in rows
+            if row.get("column_name")
+        }
+    return None
+
+
+def _normalize_type(value: str, dialect: Dialect) -> str:
+    text = value.lower().strip()
+    if not text:
+        return ""
+    if dialect == "sqlite":
+        if "int" in text:
+            return "integer"
+        if any(part in text for part in ("char", "clob", "text", "varchar")):
+            return "text"
+        if any(part in text for part in ("real", "floa", "doub")):
+            return "real"
+        if "blob" in text:
+            return "blob"
+        if "bool" in text:
+            return "integer"
+        if "timestamp" in text or "datetime" in text:
+            return "text"
+        return text
+    if dialect == "postgres":
+        text = text.replace("character varying", "varchar")
+        if text.startswith("timestamp"):
+            return "timestamp"
+        if text in {"boolean", "bool"}:
+            return "boolean"
+        return text
+    if dialect == "mysql":
+        if text.startswith("tinyint(1)") or text == "bool" or text == "boolean":
+            return "boolean"
+        if text.startswith("datetime") or text.startswith("timestamp"):
+            return "datetime"
+        return text
+    return text
+
+
+def _field_type_compatible(field: Any, column: ColumnInfo, dialect: Dialect) -> bool:
+    expected = _field_sql_type(field, dialect)
+    if not expected:
+        return True
+    expected_norm = _normalize_type(expected, dialect)
+    actual_norm = _normalize_type(column.data_type, dialect)
+    if not expected_norm or not actual_norm:
+        return True
+    if expected_norm == actual_norm:
+        return True
+    if dialect == "sqlite":
+        # SQLite affinity is intentionally loose; varchar/text and bool/integer are
+        # compatible enough for startup validation.
+        compatible = {
+            ("varchar", "text"),
+            ("text", "varchar"),
+            ("boolean", "integer"),
+            ("integer", "boolean"),
+        }
+        return (expected_norm, actual_norm) in compatible
+    return False
+
+
+def _index_name(table: str, columns: tuple[str, ...]) -> str:
+    raw = f"idx_{table}_{'_'.join(columns)}"
+    return raw[:62]
+
+
+async def _table_indexes(
+    connection: Any, table: str, dialect: Dialect
+) -> set[tuple[str, ...]]:
+    indexes: set[tuple[str, ...]] = set()
+    if dialect == "sqlite":
+        rows = await connection.execute_query_dict(
+            f"PRAGMA index_list({_quote_identifier(table, dialect)})"
+        )
+        for row in rows:
+            if bool(row.get("unique")):
+                continue
+            index_name = row.get("name")
+            if not index_name:
+                continue
+            info = await connection.execute_query_dict(
+                f"PRAGMA index_info({_quote_identifier(str(index_name), dialect)})"
+            )
+            columns = tuple(
+                str(item.get("name"))
+                for item in sorted(info, key=lambda item: int(item.get("seqno") or 0))
+                if item.get("name")
+            )
+            if columns:
+                indexes.add(columns)
+        return indexes
+    if dialect == "postgres":
+        rows = await connection.execute_query_dict(
+            "SELECT indexname, indexdef FROM pg_indexes "
+            "WHERE schemaname = current_schema() AND tablename = $1",
+            [table],
+        )
+        for row in rows:
+            indexdef = str(row.get("indexdef") or "")
+            if " UNIQUE INDEX " in indexdef.upper():
+                continue
+            start = indexdef.rfind("(")
+            end = indexdef.rfind(")")
+            if start < 0 or end <= start:
+                continue
+            columns = tuple(
+                part.strip().strip('"')
+                for part in indexdef[start + 1 : end].split(",")
+                if part.strip()
+            )
+            if columns:
+                indexes.add(columns)
+        return indexes
+    if dialect == "mysql":
+        rows = await connection.execute_query_dict(
+            "SELECT INDEX_NAME, COLUMN_NAME, SEQ_IN_INDEX, NON_UNIQUE "
+            "FROM INFORMATION_SCHEMA.STATISTICS "
+            "WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = %s",
+            [table],
+        )
+        grouped: dict[str, list[tuple[int, str]]] = {}
+        unique_names: set[str] = set()
+        for row in rows:
+            name = str(row.get("INDEX_NAME") or "")
+            column = str(row.get("COLUMN_NAME") or "")
+            if not name or not column:
+                continue
+            if int(row.get("NON_UNIQUE") or 0) == 0:
+                unique_names.add(name)
+                continue
+            grouped.setdefault(name, []).append(
+                (int(row.get("SEQ_IN_INDEX") or 0), column)
+            )
+        for name, items in grouped.items():
+            if name in unique_names:
+                continue
+            columns = tuple(column for _, column in sorted(items))
+            if columns:
+                indexes.add(columns)
+        return indexes
+    return indexes
+
+
+def _index_sql(table: str, columns: tuple[str, ...], dialect: Dialect) -> str | None:
+    if not columns:
+        return None
+    table_sql = _quote_identifier(table, dialect)
+    index_sql = _quote_identifier(_index_name(table, columns), dialect)
+    columns_sql = ", ".join(_quote_identifier(column, dialect) for column in columns)
+    if dialect in {"sqlite", "postgres"}:
+        return (
+            f"CREATE INDEX IF NOT EXISTS {index_sql} " f"ON {table_sql}({columns_sql})"
+        )
+    if dialect == "mysql":
+        return f"CREATE INDEX {index_sql} ON {table_sql}({columns_sql})"
+    return None
+
+
+def _meta_index_columns(index: Any) -> tuple[str, ...]:
+    if isinstance(index, str):
+        return (index,)
+    if isinstance(index, list | tuple):
+        return tuple(str(column) for column in index if column)
+    fields = getattr(index, "fields", None)
+    if fields:
+        return tuple(str(column) for column in fields if column)
+    return ()
+
+
+def _empty_drift(table: str) -> dict[str, Any]:
+    return {
+        "table": table,
+        "missing_columns": [],
+        "extra_columns": [],
+        "type_mismatches": [],
+        "missing_indexes": [],
+        "unsafe_changes": [],
+    }
+
+
+async def _write_schema_report(result: SchemaGuardResult) -> None:
+    report = {
+        "created_at": datetime.now().isoformat(timespec="seconds"),
+        "summary": {
+            "checked_tables": result.checked_tables,
+            "repaired_columns": result.repaired_columns,
+            "repaired_indexes": result.repaired_indexes,
+            "skipped_columns": result.skipped_columns,
+            "skipped_indexes": result.skipped_indexes,
+            "type_mismatches": result.type_mismatches,
+            "warnings": result.warnings,
+        },
+        "tables": result.drift or [],
+    }
+    path = Path() / "data" / "db" / "schema_report.json"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    await asyncio.to_thread(
+        path.write_text,
+        json.dumps(report, ensure_ascii=False, indent=2),
+        "utf-8",
+    )
+
+
+async def repair_safe_schema_drift() -> SchemaGuardResult:
+    """Repair low-risk missing columns after Tortoise schema creation.
+
+    This guard intentionally avoids type changes, constraints, unique indexes, foreign
+    keys, and SQLite table rebuilds. It is a startup-only safety net for model/schema
+    drift caused by copied databases or skipped legacy script hashes.
+    """
+    result = SchemaGuardResult()
+    result.drift = []
+    connection = Tortoise.get_connection("default")
+    dialect = _connection_dialect(connection)
+    if dialect == "unknown":
+        logger.debug("SchemaGuard 跳过未知数据库方言", LOG_COMMAND)
+        return result
+
+    app = Tortoise.apps.get("models", {})
+    for model in app.values():
+        meta = getattr(model, "_meta", None)
+        table = getattr(meta, "db_table", None) or getattr(meta, "table", None)
+        if not table:
+            continue
+        try:
+            columns = await _table_columns(connection, table, dialect)
+        except Exception as exc:
+            result.warnings += 1
+            logger.debug(f"SchemaGuard 检查表 {table} 失败", LOG_COMMAND, e=exc)
+            continue
+        if columns is None:
+            continue
+        result.checked_tables += 1
+        drift = _empty_drift(table)
+        fields_map = getattr(meta, "fields_map", {}) or {}
+        expected_sources: set[str] = set()
+        for field_name, field in fields_map.items():
+            if not _is_db_field(field, dialect):
+                continue
+            source = str(getattr(field, "source_field", None) or field_name)
+            expected_sources.add(source)
+            if source in columns:
+                if not _field_type_compatible(field, columns[source], dialect):
+                    result.type_mismatches += 1
+                    drift["type_mismatches"].append(
+                        {
+                            "column": source,
+                            "expected": _field_sql_type(field, dialect),
+                            "actual": columns[source].data_type,
+                        }
+                    )
+                continue
+            drift["missing_columns"].append(source)
+            if not _is_safe_missing_field(field):
+                result.skipped_columns += 1
+                drift["unsafe_changes"].append(
+                    {
+                        "kind": "missing_required_column",
+                        "column": source,
+                    }
+                )
+                logger.debug(
+                    f"SchemaGuard 跳过非低风险缺字段: {table}.{source}",
+                    LOG_COMMAND,
+                )
+                continue
+            sql = _missing_column_sql(table, source, field, dialect)
+            if not sql:
+                result.skipped_columns += 1
+                logger.debug(
+                    f"SchemaGuard 无法生成补字段 SQL: {table}.{source}",
+                    LOG_COMMAND,
+                )
+                continue
+            try:
+                await asyncio.wait_for(
+                    connection.execute_query_dict(sql),
+                    timeout=DB_TIMEOUT_SECONDS,
+                )
+                columns[source] = ColumnInfo(name=source, data_type="")
+                result.repaired_columns += 1
+                logger.info(f"SchemaGuard 已补齐字段: {table}.{source}", LOG_COMMAND)
+            except OperationalError as exc:
+                err = str(exc).lower()
+                if any(
+                    text in err
+                    for text in ("duplicate column", "already exists", "已存在")
+                ):
+                    columns[source] = ColumnInfo(name=source, data_type="")
+                    continue
+                result.warnings += 1
+                logger.warning(
+                    f"SchemaGuard 补齐字段失败: {table}.{source}",
+                    LOG_COMMAND,
+                    e=exc,
+                )
+            except Exception as exc:
+                result.warnings += 1
+                logger.warning(
+                    f"SchemaGuard 补齐字段失败: {table}.{source}",
+                    LOG_COMMAND,
+                    e=exc,
+                )
+        drift["extra_columns"] = sorted(
+            column for column in columns.keys() if column not in expected_sources
+        )
+        try:
+            existing_indexes = await _table_indexes(connection, table, dialect)
+        except Exception as exc:
+            result.warnings += 1
+            logger.debug(f"SchemaGuard 检查索引 {table} 失败", LOG_COMMAND, e=exc)
+            existing_indexes = set()
+        indexes = getattr(meta, "indexes", ()) or ()
+        for index in indexes:
+            index_columns = _meta_index_columns(index)
+            if not index_columns or index_columns in existing_indexes:
+                continue
+            drift["missing_indexes"].append(list(index_columns))
+            if not all(column in columns for column in index_columns):
+                result.skipped_indexes += 1
+                logger.debug(
+                    f"SchemaGuard 跳过缺字段索引: {table}.{index_columns}",
+                    LOG_COMMAND,
+                )
+                continue
+            sql = _index_sql(table, index_columns, dialect)
+            if not sql:
+                result.skipped_indexes += 1
+                continue
+            try:
+                await asyncio.wait_for(
+                    connection.execute_query_dict(sql),
+                    timeout=DB_TIMEOUT_SECONDS,
+                )
+                existing_indexes.add(index_columns)
+                result.repaired_indexes += 1
+                logger.debug(
+                    f"SchemaGuard 已补齐索引: {table}.{index_columns}",
+                    LOG_COMMAND,
+                )
+            except OperationalError as exc:
+                err = str(exc).lower()
+                if any(
+                    text in err for text in ("already exists", "duplicate", "已存在")
+                ):
+                    existing_indexes.add(index_columns)
+                    continue
+                result.warnings += 1
+                logger.warning(
+                    f"SchemaGuard 补齐索引失败: {table}.{index_columns}",
+                    LOG_COMMAND,
+                    e=exc,
+                )
+        if any(
+            drift[key]
+            for key in (
+                "missing_columns",
+                "extra_columns",
+                "type_mismatches",
+                "missing_indexes",
+                "unsafe_changes",
+            )
+        ):
+            result.drift.append(drift)
+
+    logger.info(
+        "SchemaGuard 完成: "
+        f"checked={result.checked_tables}, "
+        f"repaired_columns={result.repaired_columns}, "
+        f"repaired_indexes={result.repaired_indexes}, "
+        f"skipped_columns={result.skipped_columns}, "
+        f"skipped_indexes={result.skipped_indexes}, "
+        f"type_mismatches={result.type_mismatches}, "
+        f"warnings={result.warnings}",
+        LOG_COMMAND,
+    )
+    with contextlib.suppress(Exception):
+        await _write_schema_report(result)
+    return result
+
+
+async def repair_table_schema(table_name: str) -> SchemaGuardResult:
+    """Repair one table by reusing the startup guard path.
+
+    The current guard is cheap enough and already table-scoped internally by model
+    metadata, so this helper keeps write-path recovery simple and conservative.
+    """
+    result = SchemaGuardResult()
+    result.drift = []
+    connection = Tortoise.get_connection("default")
+    dialect = _connection_dialect(connection)
+    if dialect == "unknown":
+        return result
+    app = Tortoise.apps.get("models", {})
+    for model in app.values():
+        meta = getattr(model, "_meta", None)
+        table = getattr(meta, "db_table", None) or getattr(meta, "table", None)
+        if table == table_name:
+            # Keep the implementation conservative: repairing all tables is still
+            # startup-style work and avoids a second partial code path.
+            return await repair_safe_schema_drift()
+    return result

zhenxun/services/db_context/schema_ops.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Literal, Protocol
+
+Dialect = Literal["sqlite", "postgres", "mysql", "unknown"]
+
+
+class SchemaOpRisk(str, Enum):
+    SAFE = "safe"
+    GUARDED = "guarded"
+    MANUAL = "manual"
+
+
+class SchemaOp(Protocol):
+    risk: SchemaOpRisk
+
+    def to_sql(self, dialect: Dialect) -> list[str]: ...
+
+
+def quote_identifier(identifier: str, dialect: Dialect) -> str:
+    if dialect == "mysql":
+        return f"`{identifier.replace('`', '``')}`"
+    return f'"{identifier.replace(chr(34), chr(34) + chr(34))}"'
+
+
+def _column_type(column_type: str | dict[str, str], dialect: Dialect) -> str:
+    if isinstance(column_type, dict):
+        return column_type.get(dialect) or column_type.get("default") or "TEXT"
+    return column_type
+
+
+def _default_sql(default: str | float | bool | None, dialect: Dialect) -> str:
+    if default is None:
+        return ""
+    if isinstance(default, bool):
+        if dialect == "postgres":
+            return " DEFAULT TRUE" if default else " DEFAULT FALSE"
+        return " DEFAULT 1" if default else " DEFAULT 0"
+    if isinstance(default, int | float):
+        return f" DEFAULT {default}"
+    escaped = default.replace("'", "''")
+    return f" DEFAULT '{escaped}'"
+
+
+@dataclass(frozen=True, slots=True)
+class AddColumn:
+    table: str
+    column: str
+    column_type: str | dict[str, str]
+    nullable: bool = True
+    default: str | float | bool | None = None
+    risk: SchemaOpRisk = SchemaOpRisk.SAFE
+
+    def to_sql(self, dialect: Dialect) -> list[str]:
+        table = quote_identifier(self.table, dialect)
+        column = quote_identifier(self.column, dialect)
+        column_type = _column_type(self.column_type, dialect)
+        null_sql = "" if self.nullable else " NOT NULL"
+        default_sql = _default_sql(self.default, dialect)
+        if not self.nullable and not default_sql:
+            return []
+        if dialect == "postgres":
+            return [
+                "ALTER TABLE "
+                f"{table} ADD COLUMN IF NOT EXISTS {column} "
+                f"{column_type}{default_sql}{null_sql}"
+            ]
+        return [
+            "ALTER TABLE "
+            f"{table} ADD COLUMN {column} {column_type}{default_sql}{null_sql}"
+        ]
+
+
+@dataclass(frozen=True, slots=True)
+class CreateIndex:
+    table: str
+    columns: tuple[str, ...]
+    name: str | None = None
+    if_not_exists: bool = True
+    unique: bool = False
+    where: str | None = None
+    risk: SchemaOpRisk = SchemaOpRisk.SAFE
+
+    def __init__(
+        self,
+        table: str,
+        columns: tuple[str, ...] | list[str],
+        name: str | None = None,
+        if_not_exists: bool = True,
+        unique: bool = False,
+        where: str | None = None,
+        risk: SchemaOpRisk | None = None,
+    ) -> None:
+        object.__setattr__(self, "table", table)
+        object.__setattr__(self, "columns", tuple(columns))
+        object.__setattr__(self, "name", name)
+        object.__setattr__(self, "if_not_exists", if_not_exists)
+        object.__setattr__(self, "unique", unique)
+        object.__setattr__(self, "where", where)
+        resolved_risk = risk or (SchemaOpRisk.MANUAL if unique else SchemaOpRisk.SAFE)
+        object.__setattr__(self, "risk", resolved_risk)
+
+    def to_sql(self, dialect: Dialect) -> list[str]:
+        if self.unique or not self.columns:
+            return []
+        name = self.name or f"idx_{self.table}_{'_'.join(self.columns)}"[:62]
+        if dialect == "mysql" and self.where:
+            return []
+        exists_sql = (
+            "IF NOT EXISTS " if self.if_not_exists and dialect != "mysql" else ""
+        )
+        table = quote_identifier(self.table, dialect)
+        index = quote_identifier(name, dialect)
+        columns = ", ".join(
+            quote_identifier(column, dialect) for column in self.columns
+        )
+        where = f" WHERE {self.where}" if self.where and dialect != "mysql" else ""
+        return [f"CREATE INDEX {exists_sql}{index} ON {table}({columns}){where}"]
+
+
+@dataclass(frozen=True, slots=True)
+class RenameColumn:
+    table: str
+    old: str
+    new: str
+    risk: SchemaOpRisk = SchemaOpRisk.GUARDED
+
+    def to_sql(self, dialect: Dialect) -> list[str]:
+        table = quote_identifier(self.table, dialect)
+        old = quote_identifier(self.old, dialect)
+        new = quote_identifier(self.new, dialect)
+        return [f"ALTER TABLE {table} RENAME COLUMN {old} TO {new}"]
+
+
+@dataclass(frozen=True, slots=True)
+class DropColumn:
+    table: str
+    column: str
+    risk: SchemaOpRisk = SchemaOpRisk.GUARDED
+
+    def to_sql(self, dialect: Dialect) -> list[str]:
+        table = quote_identifier(self.table, dialect)
+        column = quote_identifier(self.column, dialect)
+        return [f"ALTER TABLE {table} DROP COLUMN {column}"]
+
+
+@dataclass(frozen=True, slots=True)
+class AlterColumnType:
+    table: str
+    column: str
+    column_type: str | dict[str, str]
+    nullable: bool | None = None
+    risk: SchemaOpRisk = SchemaOpRisk.GUARDED
+
+    def to_sql(self, dialect: Dialect) -> list[str]:
+        column_type = _column_type(self.column_type, dialect)
+        table = quote_identifier(self.table, dialect)
+        column = quote_identifier(self.column, dialect)
+        if dialect == "postgres":
+            return [f"ALTER TABLE {table} ALTER COLUMN {column} TYPE {column_type}"]
+        if dialect == "mysql":
+            null_sql = " NULL" if self.nullable else " NOT NULL"
+            if self.nullable is None:
+                null_sql = ""
+            return [
+                f"ALTER TABLE {table} MODIFY COLUMN {column} {column_type}{null_sql}"
+            ]
+        return []
+
+
+def normalize_schema_ops(items: list[str | SchemaOp], dialect: Dialect) -> list[str]:
+    sql_list: list[str] = []
+    for item in items:
+        if isinstance(item, str):
+            sql_list.append(item)
+        else:
+            sql_list.extend(item.to_sql(dialect))
+    return sql_list

zhenxun/services/hot_query_cache.py

@@ -0,0 +1,482 @@
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Literal
+
+from tortoise.functions import Count
+
+from zhenxun.services.cache.bounded_ttl import BoundedTTLCache
+
+
+@dataclass(frozen=True, slots=True)
+class GroupMemberSnapshot:
+    id: int
+    user_id: str
+    user_name: str
+    group_id: str
+    user_join_time: datetime | None
+    uid: int | None
+    platform: str | None
+
+
+def _member_cache_sizeof(members: tuple[GroupMemberSnapshot, ...]) -> int:
+    size = 0
+    for member in members:
+        size += 96
+        size += len(member.user_id) + len(member.user_name) + len(member.group_id)
+        size += len(member.platform or "")
+    return size
+
+
+_GROUP_MEMBER_CACHE = BoundedTTLCache[str, tuple[GroupMemberSnapshot, ...]](
+    "hot_group_info_users",
+    ttl_seconds=45,
+    max_items=512,
+    max_total_bytes=32 * 1024 * 1024,
+    sizeof=_member_cache_sizeof,
+)
+_GROUP_USER_IDS_CACHE = BoundedTTLCache[str, tuple[str, ...]](
+    "hot_group_info_user_ids",
+    ttl_seconds=45,
+    max_items=2048,
+)
+_GROUP_MEMBER_BY_ID_CACHE = BoundedTTLCache[str, tuple[GroupMemberSnapshot | None]](
+    "hot_group_info_user_by_id",
+    ttl_seconds=45,
+    max_items=50000,
+)
+_USER_GROUP_CACHE = BoundedTTLCache[str, tuple[str, ...]](
+    "hot_group_info_user_groups",
+    ttl_seconds=45,
+    max_items=4096,
+)
+_USER_NAME_CACHE = BoundedTTLCache[str, str](
+    "hot_group_info_user_names",
+    ttl_seconds=45,
+    max_items=20000,
+)
+_CHAT_RANK_CACHE = BoundedTTLCache[str, tuple[tuple[str, int], ...]](
+    "hot_chat_history_rank",
+    ttl_seconds=20,
+    max_items=512,
+)
+_CHAT_FIRST_MSG_CACHE = BoundedTTLCache[str, tuple[datetime | None]](
+    "hot_chat_history_first_msg",
+    ttl_seconds=300,
+    max_items=2048,
+)
+_STATISTICS_COUNT_CACHE = BoundedTTLCache[str, tuple[tuple[str, int], ...]](
+    "hot_statistics_plugin_counts",
+    ttl_seconds=20,
+    max_items=512,
+)
+
+_GROUP_MEMBER_LOCKS: dict[str, asyncio.Lock] = {}
+_GROUP_USER_IDS_LOCKS: dict[str, asyncio.Lock] = {}
+_USER_GROUP_LOCKS: dict[str, asyncio.Lock] = {}
+_CHAT_RANK_LOCKS: dict[str, asyncio.Lock] = {}
+_CHAT_FIRST_MSG_LOCKS: dict[str, asyncio.Lock] = {}
+_STATISTICS_LOCKS: dict[str, asyncio.Lock] = {}
+_MAX_LOCK_POOL_SIZE = 4096
+
+
+def _get_lock(pool: dict[str, asyncio.Lock], key: str) -> asyncio.Lock:
+    lock = pool.get(key)
+    if lock is None:
+        if len(pool) >= _MAX_LOCK_POOL_SIZE:
+            for old_key, old_lock in list(pool.items()):
+                if not old_lock.locked():
+                    pool.pop(old_key, None)
+                    break
+        lock = asyncio.Lock()
+        pool[key] = lock
+    return lock
+
+
+def _normalize_id(value: object) -> str:
+    return str(value or "")
+
+
+def _normalize_ids(values: Iterable[object] | None) -> tuple[str, ...] | None:
+    if values is None:
+        return None
+    return tuple(dict.fromkeys(v for value in values if (v := _normalize_id(value))))
+
+
+async def get_group_members(
+    group_id: str | int | None,
+) -> tuple[GroupMemberSnapshot, ...]:
+    """Return lightweight group-member snapshots with a short runtime TTL."""
+    group_key = _normalize_id(group_id)
+    if not group_key:
+        return ()
+
+    cached = await _GROUP_MEMBER_CACHE.get(group_key)
+    if cached is not None:
+        return cached
+
+    lock = _get_lock(_GROUP_MEMBER_LOCKS, group_key)
+    async with lock:
+        cached = await _GROUP_MEMBER_CACHE.get(group_key)
+        if cached is not None:
+            return cached
+
+        from zhenxun.models.group_member_info import GroupInfoUser
+
+        rows = await GroupInfoUser.filter(group_id=group_key).values_list(
+            "id",
+            "user_id",
+            "user_name",
+            "user_join_time",
+            "uid",
+            "platform",
+        )
+        members = tuple(
+            GroupMemberSnapshot(
+                id=int(row[0] or 0),
+                user_id=str(row[1] or ""),
+                user_name=str(row[2] or ""),
+                group_id=group_key,
+                user_join_time=row[3],
+                uid=int(row[4]) if row[4] is not None else None,
+                platform=str(row[5]) if row[5] else None,
+            )
+            for row in rows
+            if row[1]
+        )
+        await _GROUP_MEMBER_CACHE.set(group_key, members)
+        await _GROUP_USER_IDS_CACHE.set(
+            group_key, tuple(member.user_id for member in members)
+        )
+        return members
+
+
+async def get_group_member_map(
+    group_id: str | int | None,
+    user_ids: Iterable[object] | None = None,
+) -> dict[str, GroupMemberSnapshot]:
+    group_key = _normalize_id(group_id)
+    if not group_key:
+        return {}
+    wanted = _normalize_ids(user_ids)
+    if wanted is not None and not wanted:
+        return {}
+    if wanted is None:
+        members = await get_group_members(group_key)
+        return {member.user_id: member for member in members}
+
+    cached_members = await _GROUP_MEMBER_CACHE.get(group_key)
+    if cached_members is not None:
+        wanted_set = set(wanted)
+        return {
+            member.user_id: member
+            for member in cached_members
+            if member.user_id in wanted_set
+        }
+
+    result: dict[str, GroupMemberSnapshot] = {}
+    missing: list[str] = []
+    for user_id in wanted:
+        cache_key = f"{group_key}:{user_id}"
+        cached = await _GROUP_MEMBER_BY_ID_CACHE.get(cache_key)
+        if cached is None:
+            missing.append(user_id)
+        else:
+            member = cached[0]
+            if member is not None:
+                result[user_id] = member
+
+    if missing:
+        from zhenxun.models.group_member_info import GroupInfoUser
+
+        rows = await GroupInfoUser.filter(
+            group_id=group_key, user_id__in=missing
+        ).values_list(
+            "id",
+            "user_id",
+            "user_name",
+            "user_join_time",
+            "uid",
+            "platform",
+        )
+        found: set[str] = set()
+        for row in rows:
+            if not row[1]:
+                continue
+            member = GroupMemberSnapshot(
+                id=int(row[0] or 0),
+                user_id=str(row[1] or ""),
+                user_name=str(row[2] or ""),
+                group_id=group_key,
+                user_join_time=row[3],
+                uid=int(row[4]) if row[4] is not None else None,
+                platform=str(row[5]) if row[5] else None,
+            )
+            result[member.user_id] = member
+            found.add(member.user_id)
+            await _GROUP_MEMBER_BY_ID_CACHE.set(
+                f"{group_key}:{member.user_id}", (member,)
+            )
+        for user_id in missing:
+            if user_id not in found:
+                await _GROUP_MEMBER_BY_ID_CACHE.set(f"{group_key}:{user_id}", (None,))
+    return result
+
+
+async def get_group_member(
+    group_id: str | int | None,
+    user_id: str | int | None,
+) -> GroupMemberSnapshot | None:
+    user_key = _normalize_id(user_id)
+    if not user_key:
+        return None
+    return (await get_group_member_map(group_id, [user_key])).get(user_key)
+
+
+async def get_group_user_ids(group_id: str | int | None) -> set[str]:
+    group_key = _normalize_id(group_id)
+    if not group_key:
+        return set()
+    cached = await _GROUP_USER_IDS_CACHE.get(group_key)
+    if cached is not None:
+        return set(cached)
+
+    cached_members = await _GROUP_MEMBER_CACHE.get(group_key)
+    if cached_members is not None:
+        user_ids = tuple(member.user_id for member in cached_members)
+        await _GROUP_USER_IDS_CACHE.set(group_key, user_ids)
+        return set(user_ids)
+
+    lock = _get_lock(_GROUP_USER_IDS_LOCKS, group_key)
+    async with lock:
+        cached = await _GROUP_USER_IDS_CACHE.get(group_key)
+        if cached is not None:
+            return set(cached)
+
+        from zhenxun.models.group_member_info import GroupInfoUser
+
+        rows = await GroupInfoUser.filter(group_id=group_key).values_list(
+            "user_id", flat=True
+        )
+        user_ids = tuple(str(user_id) for user_id in rows if user_id)
+        await _GROUP_USER_IDS_CACHE.set(group_key, user_ids)
+        return set(user_ids)
+
+
+async def get_user_group_ids(user_id: str | int | None) -> list[str]:
+    user_key = _normalize_id(user_id)
+    if not user_key:
+        return []
+
+    cached = await _USER_GROUP_CACHE.get(user_key)
+    if cached is not None:
+        return list(cached)
+
+    lock = _get_lock(_USER_GROUP_LOCKS, user_key)
+    async with lock:
+        cached = await _USER_GROUP_CACHE.get(user_key)
+        if cached is not None:
+            return list(cached)
+
+        from zhenxun.models.group_member_info import GroupInfoUser
+
+        rows = await GroupInfoUser.filter(user_id=user_key).values_list(
+            "group_id", flat=True
+        )
+        group_ids = tuple(str(group_id) for group_id in rows if group_id)
+        await _USER_GROUP_CACHE.set(user_key, group_ids)
+        return list(group_ids)
+
+
+async def get_member_names(
+    user_ids: Iterable[object],
+    group_id: str | int | None = None,
+) -> dict[str, str]:
+    user_keys = _normalize_ids(user_ids) or ()
+    if not user_keys:
+        return {}
+    if group_id:
+        members = await get_group_member_map(group_id, user_keys)
+        return {user_id: members[user_id].user_name for user_id in members}
+
+    result: dict[str, str] = {}
+    missing: list[str] = []
+    for user_id in user_keys:
+        cached = await _USER_NAME_CACHE.get(user_id)
+        if cached is None:
+            missing.append(user_id)
+        else:
+            result[user_id] = cached
+
+    if missing:
+        from zhenxun.models.group_member_info import GroupInfoUser
+
+        rows = await GroupInfoUser.filter(user_id__in=missing).values_list(
+            "user_id", "user_name"
+        )
+        for user_id, user_name in rows:
+            user_key = str(user_id)
+            if user_key not in result:
+                result[user_key] = str(user_name or "")
+        for user_id in missing:
+            await _USER_NAME_CACHE.set(user_id, result.get(user_id, ""))
+    return result
+
+
+async def get_member_name(
+    user_id: str | int | None,
+    group_id: str | int | None = None,
+) -> str | None:
+    user_key = _normalize_id(user_id)
+    if not user_key:
+        return None
+    return (await get_member_names([user_key], group_id)).get(user_key) or None
+
+
+async def invalidate_group_members(
+    group_id: str | int | None = None,
+    user_ids: Iterable[object] | None = None,
+) -> None:
+    if group_id is None:
+        await _GROUP_MEMBER_CACHE.clear()
+        await _GROUP_USER_IDS_CACHE.clear()
+        await _GROUP_MEMBER_BY_ID_CACHE.clear()
+        return
+    group_key = _normalize_id(group_id)
+    await _GROUP_MEMBER_CACHE.delete(group_key)
+    await _GROUP_USER_IDS_CACHE.delete(group_key)
+    normalized_ids = _normalize_ids(user_ids)
+    if normalized_ids is None:
+        await _GROUP_MEMBER_BY_ID_CACHE.clear()
+        return
+    for user_id in normalized_ids:
+        await _GROUP_MEMBER_BY_ID_CACHE.delete(f"{group_key}:{user_id}")
+
+
+async def invalidate_member_names(user_ids: Iterable[object] | None = None) -> None:
+    if user_ids is None:
+        await _USER_NAME_CACHE.clear()
+        await _USER_GROUP_CACHE.clear()
+        return
+    for user_id in _normalize_ids(user_ids) or ():
+        await _USER_NAME_CACHE.delete(user_id)
+        await _USER_GROUP_CACHE.delete(user_id)
+
+
+def _datetime_key(value: datetime | None) -> str:
+    return value.isoformat(" ", timespec="seconds") if value else ""
+
+
+def _date_scope_key(date_scope: tuple[datetime, datetime] | None) -> str:
+    if not date_scope:
+        return ""
+    end_bucket = int(date_scope[1].timestamp() // 20)
+    return f"{_datetime_key(date_scope[0])}..bucket:{end_bucket}"
+
+
+async def get_chat_history_rank_cached(
+    model: Any,
+    gid: str | None,
+    limit: int = 10,
+    order: str = "DESC",
+    date_scope: tuple[datetime, datetime] | None = None,
+) -> list[tuple[str, int]]:
+    key = f"{gid or '*'}:{limit}:{order}:{_date_scope_key(date_scope)}"
+    cached = await _CHAT_RANK_CACHE.get(key)
+    if cached is not None:
+        return list(cached)
+
+    lock = _get_lock(_CHAT_RANK_LOCKS, key)
+    async with lock:
+        cached = await _CHAT_RANK_CACHE.get(key)
+        if cached is not None:
+            return list(cached)
+
+        order_prefix = "-" if order == "DESC" else ""
+        query: Any = model.filter(group_id=gid) if gid else model
+        if date_scope:
+            filter_scope = (
+                date_scope[0].isoformat(" "),
+                date_scope[1].isoformat(" "),
+            )
+            query = query.filter(create_time__range=filter_scope)
+        rows = (
+            await query.annotate(count=Count("user_id"))
+            .order_by(f"{order_prefix}count")
+            .group_by("user_id")
+            .limit(limit)
+            .values_list("user_id", "count")
+        )
+        result = tuple((str(user_id), int(count)) for user_id, count in rows)
+        await _CHAT_RANK_CACHE.set(key, result)
+        return list(result)
+
+
+async def get_chat_history_first_msg_datetime_cached(
+    model: Any,
+    group_id: str | None,
+) -> datetime | None:
+    key = group_id or "*"
+    cached = await _CHAT_FIRST_MSG_CACHE.get(key)
+    if cached is not None:
+        return cached[0]
+
+    lock = _get_lock(_CHAT_FIRST_MSG_LOCKS, key)
+    async with lock:
+        cached = await _CHAT_FIRST_MSG_CACHE.get(key)
+        if cached is not None:
+            return cached[0]
+
+        query: Any = model.filter(group_id=group_id) if group_id else model.all()
+        message = await query.order_by("create_time").first()
+        result = getattr(message, "create_time", None) if message else None
+        await _CHAT_FIRST_MSG_CACHE.set(key, (result,))
+        return result
+
+
+async def get_statistics_plugin_counts_cached(
+    scope: Literal["global", "user", "group"],
+    *,
+    plugin_name: str | None,
+    start_time: datetime | None,
+    user_id: str | None = None,
+    group_id: str | None = None,
+) -> list[tuple[str, int]]:
+    key = (
+        f"{scope}:{plugin_name or ''}:{_datetime_key(start_time)}:"
+        f"{user_id or ''}:{group_id or ''}"
+    )
+    cached = await _STATISTICS_COUNT_CACHE.get(key)
+    if cached is not None:
+        return list(cached)
+
+    lock = _get_lock(_STATISTICS_LOCKS, key)
+    async with lock:
+        cached = await _STATISTICS_COUNT_CACHE.get(key)
+        if cached is not None:
+            return list(cached)
+
+        from zhenxun.models.statistics import Statistics
+
+        query: Any = Statistics
+        if scope == "user":
+            query = Statistics.filter(user_id=user_id)
+            if group_id:
+                query = query.filter(group_id=group_id)
+        elif scope == "group":
+            query = Statistics.filter(group_id=group_id)
+        if plugin_name:
+            query = query.filter(plugin_name=plugin_name)
+        if start_time:
+            query = query.filter(create_time__gte=start_time)
+        rows = (
+            await query.annotate(count=Count("id"))
+            .group_by("plugin_name")
+            .values_list("plugin_name", "count")
+        )
+        result = tuple((str(plugin), int(count)) for plugin, count in rows)
+        await _STATISTICS_COUNT_CACHE.set(key, result)
+        return list(result)

zhenxun/services/runtime_bootstrap.py

@@ -140,6 +140,9 @@ def register_runtime_bootstrap(_driver) -> None:
         global _thread_executor
         await _stop_launcher_watchdog()
         await stop_send_queue()
+        from zhenxun.models._bot_message_buffer import stop_bot_message_store_buffer
+
+        await stop_bot_message_store_buffer()
         await stop_memory_governor()
         executor = _thread_executor
         _thread_executor = None

zhenxun/services/send_queue.py

@@ -1,30 +1,124 @@
 import asyncio
+from collections import defaultdict
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass
 import time
-from typing import Any
+from typing import Any, ClassVar, cast
 
 from nonebot.adapters import Bot
+from nonebot.adapters.onebot.v11 import Adapter as OneBotV11Adapter
+from nonebot.adapters.onebot.v11 import Bot as OneBotV11Bot
 
 from zhenxun.services.log import logger
 
 _SEND_APIS = {"send_msg", "send_group_msg", "send_private_msg", "send_like"}
+_OBSERVED_SEND_APIS = {"send_msg", "send_group_msg", "send_private_msg"}
 _WORKERS = 3
 _MIN_INTERVAL = 0.05
 _QUEUE_MAXSIZE = 2000
 _SHUTDOWN_DRAIN_TIMEOUT_SECONDS = 3.0
 _QUEUE_PRESSURE_LOG_INTERVAL = 10.0
-_QUEUE: asyncio.Queue[tuple[Bot, str, dict[str, Any], asyncio.Future[Any]]] = (
-    asyncio.Queue(maxsize=_QUEUE_MAXSIZE)
-)
+_QUEUE: asyncio.Queue[
+    tuple[Bot, str, dict[str, Any], asyncio.Future[Any], str | None]
+] = asyncio.Queue(maxsize=_QUEUE_MAXSIZE)
 _SEND_LOCK = asyncio.Lock()
 _LAST_SEND_TS = 0.0
 _API_SEMAPHORE = asyncio.Semaphore(3)
-_ORIG_CALL_API = Bot.call_api
+_ORIG_CALL_API = OneBotV11Adapter._call_api
 _PATCHED = False
 _WORKER_TASKS: list[asyncio.Task] = []
 _QUEUE_TIMEOUT_COUNT = 0
 _SEND_LIKE_DROP_COUNT = 0
 _LAST_QUEUE_PRESSURE_LOG = 0.0
 _STOPPING = False
+_CURRENT_SEND_TRACE_ID: ContextVar[str | None] = ContextVar(
+    "zhenxun_send_trace_id",
+    default=None,
+)
+_MAX_OBSERVED_RECORDS_PER_TRACE = 12
+_MAX_OBSERVED_TEXT_LEN = 900
+
+
+def _send_platform_scope(adapter: Any) -> str:
+    if adapter is None:
+        return "unknown"
+    if isinstance(adapter, OneBotV11Adapter):
+        return "qq_client"
+    get_name = getattr(adapter, "get_name", None)
+    if callable(get_name):
+        try:
+            name = str(get_name()).lower()
+        except Exception:
+            name = ""
+    else:
+        name = adapter.__class__.__name__.lower()
+    if name == "qq" or "qq" in name:
+        return "qq_api"
+    return name or "unknown"
+
+
+@dataclass(frozen=True)
+class SendObservation:
+    trace_id: str
+    api: str
+    text: str
+    raw_message: str
+    result: Any
+    timestamp: float
+
+
+class SendObserver:
+    _records: ClassVar[dict[str, list[SendObservation]]] = defaultdict(list)
+
+    @classmethod
+    @contextmanager
+    def activate(cls, trace_id: str):
+        trace_key = str(trace_id or "").strip()
+        token = _CURRENT_SEND_TRACE_ID.set(trace_key or None)
+        try:
+            yield
+        finally:
+            _CURRENT_SEND_TRACE_ID.reset(token)
+
+    @classmethod
+    def record(
+        cls,
+        *,
+        trace_id: str | None,
+        api: str,
+        data: dict[str, Any],
+        result: Any,
+    ) -> None:
+        trace_key = str(trace_id or "").strip()
+        if not trace_key or api not in _OBSERVED_SEND_APIS:
+            return
+        target = cls._records[trace_key]
+        if len(target) >= _MAX_OBSERVED_RECORDS_PER_TRACE:
+            return
+        raw_message = _message_to_text(data.get("message"))
+        target.append(
+            SendObservation(
+                trace_id=trace_key,
+                api=api,
+                text=_compact_text(raw_message),
+                raw_message=raw_message[:_MAX_OBSERVED_TEXT_LEN],
+                result=result,
+                timestamp=time.time(),
+            )
+        )
+
+    @classmethod
+    def pop(cls, trace_id: str) -> list[SendObservation]:
+        return cls._records.pop(str(trace_id or "").strip(), [])
+
+
+def observe_send_trace(trace_id: str):
+    return SendObserver.activate(trace_id)
+
+
+def pop_send_observations(trace_id: str) -> list[SendObservation]:
+    return SendObserver.pop(trace_id)
 
 
 async def _rate_limit():
@@ -50,17 +144,45 @@ def _log_queue_pressure(reason: str) -> None:
     )
 
 
-async def _direct_call_api(bot: Bot, api: str, data: dict[str, Any]) -> Any:
+async def _direct_call_api(
+    adapter: OneBotV11Adapter,
+    bot: Bot,
+    api: str,
+    data: dict[str, Any],
+    trace_id: str | None = None,
+) -> Any:
     await _rate_limit()
     async with _API_SEMAPHORE:
-        return await _ORIG_CALL_API(bot, api, **data)
+        try:
+            result = await _ORIG_CALL_API(
+                adapter,
+                cast(OneBotV11Bot, bot),
+                api,
+                **data,
+            )
+        except Exception as exc:
+            SendObserver.record(
+                trace_id=trace_id,
+                api=api,
+                data=data,
+                result={"ok": False, "error": str(exc)},
+            )
+            raise
+        SendObserver.record(trace_id=trace_id, api=api, data=data, result=result)
+        return result
 
 
 async def _worker(worker_id: int):
     while True:
-        bot, api, data, future = await _QUEUE.get()
+        bot, api, data, future, trace_id = await _QUEUE.get()
         try:
-            result = await _direct_call_api(bot, api, data)
+            result = await _direct_call_api(
+                cast(OneBotV11Adapter, bot.adapter),
+                bot,
+                api,
+                data,
+                trace_id=trace_id,
+            )
             if not future.done():
                 future.set_result(result)
         except asyncio.CancelledError:
@@ -80,15 +202,28 @@ async def _worker(worker_id: int):
             _QUEUE.task_done()
 
 
-async def _queued_call_api(self: Bot, api: str, **data: Any):
+async def _queued_call_api(
+    adapter: OneBotV11Adapter,
+    bot: Bot,
+    api: str,
+    **data: Any,
+):
+    if _send_platform_scope(adapter) != "qq_client":
+        return await _ORIG_CALL_API(adapter, cast(OneBotV11Bot, bot), api, **data)
     if api not in _SEND_APIS:
-        return await _ORIG_CALL_API(self, api, **data)
+        return await _ORIG_CALL_API(adapter, cast(OneBotV11Bot, bot), api, **data)
     if _STOPPING:
-        return await _direct_call_api(self, api, data)
+        return await _direct_call_api(
+            adapter,
+            bot,
+            api,
+            data,
+            trace_id=_CURRENT_SEND_TRACE_ID.get(),
+        )
 
     loop = asyncio.get_running_loop()
     future: asyncio.Future[Any] = loop.create_future()
-    queue_item = (self, api, data, future)
+    queue_item = (bot, api, data, future, _CURRENT_SEND_TRACE_ID.get())
     try:
         _QUEUE.put_nowait(queue_item)
     except asyncio.QueueFull:
@@ -100,7 +235,13 @@ async def _queued_call_api(self: Bot, api: str, **data: Any):
         global _QUEUE_TIMEOUT_COUNT
         _QUEUE_TIMEOUT_COUNT += 1
         _log_queue_pressure(f"{api} fallback to direct send because queue is full")
-        return await _direct_call_api(self, api, data)
+        return await _direct_call_api(
+            adapter,
+            bot,
+            api,
+            data,
+            trace_id=_CURRENT_SEND_TRACE_ID.get(),
+        )
     return await future
 
 
@@ -108,7 +249,7 @@ def _drain_pending_futures(reason: str) -> int:
     drained = 0
     while True:
         try:
-            _, _, _, future = _QUEUE.get_nowait()
+            _, _, _, future, _ = _QUEUE.get_nowait()
         except asyncio.QueueEmpty:
             break
         if not future.done():
@@ -122,7 +263,7 @@ def patch_send_queue() -> None:
     global _PATCHED
     if _PATCHED:
         return
-    Bot.call_api = _queued_call_api  # type: ignore[assignment]
+    OneBotV11Adapter._call_api = _queued_call_api  # type: ignore[assignment]
     _PATCHED = True
 
 
@@ -130,7 +271,7 @@ def unpatch_send_queue() -> None:
     global _PATCHED
     if not _PATCHED:
         return
-    Bot.call_api = _ORIG_CALL_API  # type: ignore[assignment]
+    OneBotV11Adapter._call_api = _ORIG_CALL_API  # type: ignore[assignment]
     _PATCHED = False
 
 
@@ -138,12 +279,37 @@ async def start_send_queue() -> None:
     global _STOPPING
     patch_send_queue()
     _STOPPING = False
+    _WORKER_TASKS[:] = [task for task in _WORKER_TASKS if not task.done()]
     if _WORKER_TASKS:
         return
     for idx in range(_WORKERS):
         _WORKER_TASKS.append(asyncio.create_task(_worker(idx)))
 
 
+def _message_to_text(message: Any) -> str:
+    if message is None:
+        return ""
+    if hasattr(message, "extract_plain_text"):
+        try:
+            text = str(message.extract_plain_text())
+            if text.strip():
+                return text
+        except Exception:
+            pass
+    try:
+        return str(message)
+    except Exception as exc:
+        logger.debug(f"send observation stringify failed: {exc}")
+        return ""
+
+
+def _compact_text(text: str) -> str:
+    normalized = " ".join(str(text or "").split())
+    if len(normalized) <= _MAX_OBSERVED_TEXT_LEN:
+        return normalized
+    return normalized[: _MAX_OBSERVED_TEXT_LEN - 1].rstrip() + "…"
+
+
 async def stop_send_queue() -> None:
     global _STOPPING
     _STOPPING = True