♻️ refactor(core): 重构 AI 编排框架与记忆及 RAG 子系统 (#2149)

* ♻️ refactor(core): 重构 AI 编排框架与记忆及 RAG 子系统

- 【重构】重构 `BaseRunnable` 并引入统一的 `RunIntent` 意图载体，规范 Agent、Team 和 Workflow 的执行流
- 【解耦】将中期记忆槽和长期向量记忆从 `MemoryConfig` 中解耦，转为独立的能力组件与工具箱进行管理
- 【记忆】移除 `MemoryReader` 和 `MemoryWriter`，统一封装为 `SessionMemoryContext` 会话记忆门面
- 【RAG】重构检索器与存储后端接口，统一采用 `QueryRequest` 进行多维度联合检索，并引入 `InMemoryScorer` 提升打分性能
- 【事件】优化 `EventBus` 异步事件分发机制，引入队列机制确保事件按序处理，避免并发竞态问题
- 【依赖注入】移除 `memory` 注入项，优化 `DependencyInjector` 的签名解析缓存以提升性能

* 🚨 auto fix by pre-commit hooks

---------

Co-authored-by: webjoin111 <455457521@qq.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

zhenxun/services/ai/context/rag/backends/embedders.py

@@ -4,6 +4,7 @@ import threading
 from typing import Any, Literal, Protocol, runtime_checkable
 
 from zhenxun.services.ai.core.messages import EmbedBatch
+from zhenxun.services.ai.core.options import LLMEmbeddingConfig
 from zhenxun.services.ai.llm.api import embed as api_embed
 from zhenxun.services.ai.message_builder import MessageBuilder
 from zhenxun.services.ai.utils.logger import log_rag as logger
@@ -16,7 +17,7 @@ EmbedTaskType = Literal[
 @runtime_checkable
 class Embedder(Protocol):
     """
-    向量化引擎协议 (Callable Protocol)。
+    向量化引擎协议。
     任何实现了异步 __call__ 的对象或闭包函数均可作为 Embedder。
     """
 
@@ -32,7 +33,9 @@ class Embedder(Protocol):
 class DefaultEmbedder(Embedder):
     """系统默认的向量化引擎，调用大模型底座 API"""
 
-    def __init__(self, model_name: str | None = None, config: Any = None):
+    def __init__(
+        self, model_name: str | None = None, config: LLMEmbeddingConfig | None = None
+    ):
         self.model_name = model_name
         self.config = config
 
@@ -57,10 +60,28 @@ class BaseLocalEmbedder(Embedder, ABC):
     def __init__(self, model_name: str):
         self.model_name = model_name
         self._model_lock = threading.Lock()
+        self._model: Any | None = None
+
+    def _ensure_model_loaded(self) -> None:
+        """线程安全的懒加载机制"""
+        if self._model is None:
+            with self._model_lock:
+                if self._model is None:
+                    logger.info(
+                        f"正在后台加载本地向量模型: {self.model_name} ... "
+                        "(首次加载可能需要较长时间下载)"
+                    )
+                    self._model = self._load_model_impl()
+                    logger.info(f"本地向量模型 {self.model_name} 加载完毕！")
+
+    @abstractmethod
+    def _load_model_impl(self) -> Any:
+        """子类实现：执行具体的依赖导入与模型实例化，并返回模型对象。"""
+        pass
 
     @abstractmethod
     def _encode_texts(self, texts: list[str]) -> list[list[float]]:
-        """子类只需实现此同步的批量文本向量化方法即可。"""
+        """子类实现：执行同步的批量文本向量化方法。"""
         pass
 
     async def __call__(
@@ -93,7 +114,6 @@ class FastEmbedder(BaseLocalEmbedder):
 
     def __init__(self, model_name: str | None = None):
         super().__init__(model_name or "BAAI/bge-small-zh-v1.5")
-        self.model = None
 
         import importlib.util
 
@@ -102,29 +122,19 @@ class FastEmbedder(BaseLocalEmbedder):
                 "⚠️ 使用 FastEmbed 需要额外依赖，请在终端执行: pip install fastembed"
             )
 
-    def _ensure_model_loaded(self):
-        """线程安全的懒加载机制"""
-        if self.model is None:
-            with self._model_lock:
-                if self.model is None:
-                    try:
-                        from fastembed import TextEmbedding
-                    except ImportError:
-                        raise ImportError(
-                            "⚠️ 使用 FastEmbed 需要额外依赖，"
-                            "请在终端执行: pip install fastembed"
-                        )
-                    logger.info(
-                        f"正在后台加载 FastEmbed 本地模型: {self.model_name} ... "
-                        "(首次加载可能需要极长时间下载)"
-                    )
-                    self.model = TextEmbedding(model_name=self.model_name)
-                    logger.info(f"FastEmbed 模型 {self.model_name} 加载完毕！")
+    def _load_model_impl(self) -> Any:
+        try:
+            from fastembed import TextEmbedding
+        except ImportError:
+            raise ImportError(
+                "⚠️ 使用 FastEmbed 需要额外依赖，请在终端执行: pip install fastembed"
+            )
+        return TextEmbedding(model_name=self.model_name)
 
     def _encode_texts(self, texts: list[str]) -> list[list[float]]:
         self._ensure_model_loaded()
-        assert self.model is not None
-        return [vec.tolist() for vec in self.model.embed(texts)]
+        assert self._model is not None
+        return [vec.tolist() for vec in self._model.embed(texts)]
 
 
 class SentenceTransformerEmbedder(BaseLocalEmbedder):
@@ -135,7 +145,6 @@ class SentenceTransformerEmbedder(BaseLocalEmbedder):
 
     def __init__(self, model_name: str | None = None):
         super().__init__(model_name or "BAAI/bge-small-zh-v1.5")
-        self.model = None
 
         import importlib.util
 
@@ -145,32 +154,18 @@ class SentenceTransformerEmbedder(BaseLocalEmbedder):
                 "请在终端执行: pip install sentence-transformers"
             )
 
-    def _ensure_model_loaded(self):
-        """线程安全的懒加载机制"""
-        if self.model is None:
-            with self._model_lock:
-                if self.model is None:
-                    try:
-                        from sentence_transformers import (
-                            SentenceTransformer,
-                        )
-                    except ImportError:
-                        raise ImportError(
-                            "⚠️ 使用 SentenceTransformers 需要额外依赖，"
-                            "请在终端执行: pip install sentence-transformers"
-                        )
-                    logger.info(
-                        "正在后台加载 SentenceTransformer "
-                        f"本地模型: {self.model_name} ... "
-                        "(首次加载可能需要极长时间下载)"
-                    )
-                    self.model = SentenceTransformer(self.model_name)
-                    logger.info(
-                        f"SentenceTransformer 模型 {self.model_name} 加载完毕！"
-                    )
+    def _load_model_impl(self) -> Any:
+        try:
+            from sentence_transformers import SentenceTransformer
+        except ImportError:
+            raise ImportError(
+                "⚠️ 使用 SentenceTransformers 需要额外依赖，"
+                "请在终端执行: pip install sentence-transformers"
+            )
+        return SentenceTransformer(self.model_name)
 
     def _encode_texts(self, texts: list[str]) -> list[list[float]]:
         self._ensure_model_loaded()
-        assert self.model is not None
-        embeddings = self.model.encode(texts)
+        assert self._model is not None
+        embeddings = self._model.encode(texts)
         return embeddings.tolist()

zhenxun/services/ai/context/rag/backends/storages.py

@@ -11,6 +11,10 @@ from zhenxun.services.ai.context.rag.models import (
     SearchResult,
 )
 from zhenxun.services.ai.context.rag.retrieval import FilterEvaluator
+from zhenxun.services.ai.context.rag.utils import (
+    InMemoryScorer,
+    normalize_vector,
+)
 from zhenxun.services.ai.utils.logger import log_rag as logger
 from zhenxun.services.ai.utils.scope import ScopeSelector
 from zhenxun.services.db_context import Model
@@ -24,9 +28,7 @@ class StorageBackend(Protocol):
         """保存或更新数据块"""
         ...
 
-    async def search(
-        self, query: QueryRequest, scopes: list[str] | None = None
-    ) -> list[SearchResult]:
+    async def search(self, request: QueryRequest) -> list[SearchResult]:
         """按向量和前缀检索数据块"""
         ...
 
@@ -49,22 +51,6 @@ class StorageBackend(Protocol):
         ...
 
 
-def normalize_vector(vec: list[float] | np.ndarray) -> np.ndarray:
-    """将一维向量转化为 float32 数组并进行 L2 归一化"""
-    v = np.array(vec, dtype=np.float32)
-    norm = np.linalg.norm(v)
-    if norm == 0:
-        return v
-    return v / norm
-
-
-def normalize_matrix(mat: np.ndarray) -> np.ndarray:
-    """将二维矩阵的每一行进行 L2 归一化"""
-    norms = np.linalg.norm(mat, axis=1, keepdims=True)
-    norms[norms == 0] = 1.0
-    return mat / norms
-
-
 class DictStorageBackend(StorageBackend):
     """基于内存字典的轻量级纯净 RAG 存储实现"""
 
@@ -83,62 +69,36 @@ class DictStorageBackend(StorageBackend):
             else:
                 self._vectors.pop(r.id, None)
 
-    async def search(
-        self, query: QueryRequest, scopes: list[str] | None = None
-    ) -> list[SearchResult]:
+    async def search(self, request: QueryRequest) -> list[SearchResult]:
         candidate_ids = []
         for record in self._records.values():
-            if scopes is not None:
-                if record.metadata.get("scope", "/") not in scopes:
+            if request.scopes is not None:
+                if record.metadata.get("scope", "/") not in request.scopes:
                     continue
-            if not FilterEvaluator.evaluate(record.metadata, query.metadata_filters):
+            if not FilterEvaluator.evaluate(record.metadata, request.metadata_filters):
                 continue
-            if not query.embedding and query.text and query.text not in record.content:
+            if (
+                not request.embedding
+                and request.text
+                and request.text not in record.content
+            ):
                 continue
             candidate_ids.append(record.id)
 
         if not candidate_ids:
             return []
 
-        results = []
-        if query.search_type == "sparse":
-            import jieba
+        records = [self._records[r_id] for r_id in candidate_ids]
 
-            tokens = set(jieba.lcut_for_search(query.text.lower()))
-            for r_id in candidate_ids:
-                record = self._records[r_id]
-                content = record.content.lower()
-                matched_count = sum(1 for t in tokens if t in content)
-                if matched_count > 0:
-                    score = matched_count / len(tokens)
-                    results.append(SearchResult(record=record, score=score))
-        elif query.search_type == "dense" and query.embedding:
-            q_vec = normalize_vector(query.embedding)
-            valid_ids = [r_id for r_id in candidate_ids if r_id in self._vectors]
-
-            if valid_ids:
-                try:
-                    mat = np.array([self._vectors[r_id] for r_id in valid_ids])
-                    scores = mat @ q_vec
-                    for r_id, score in zip(valid_ids, scores):
-                        results.append(
-                            SearchResult(record=self._records[r_id], score=float(score))
-                        )
-                except ValueError as e:
-                    logger.warning(
-                        "⚠️ DictStorage 中缓存的向量维度与当前查询维度不匹配，"
-                        f"跳过向量检索。原因: {e}"
-                    )
-
-            missing_ids = [r_id for r_id in candidate_ids if r_id not in self._vectors]
-            for r_id in missing_ids:
-                results.append(SearchResult(record=self._records[r_id], score=0.1))
+        if request.search_type == "sparse":
+            results = InMemoryScorer.calculate_sparse_scores(request.text, records)
+        elif request.search_type == "dense" and request.embedding:
+            results = InMemoryScorer.calculate_dense_scores(request.embedding, records)
         else:
-            for r_id in candidate_ids:
-                results.append(SearchResult(record=self._records[r_id], score=0.1))
+            results = [SearchResult(record=r, score=0.1) for r in records]
 
         results.sort(key=lambda x: x.score, reverse=True)
-        return results[: query.limit]
+        return results[: request.limit]
 
     async def update(self, record: BaseRecord) -> None:
         if record.id in self._records:
@@ -212,94 +172,48 @@ class TortoiseStorageBackend(StorageBackend):
                 },
             )
 
-    async def search(
-        self, query: QueryRequest, scopes: list[str] | None = None
-    ) -> list[SearchResult]:
+    async def search(self, request: QueryRequest) -> list[SearchResult]:
         query_orm = self.model_class.all()
-        if scopes is not None:
-            query_orm = query_orm.filter(scope__in=scopes)
+        if request.scopes is not None:
+            query_orm = query_orm.filter(scope__in=request.scopes)
 
-        if query.search_type == "sparse" and query.text:
+        if request.search_type == "sparse" and request.text:
             import jieba
             from tortoise.expressions import Q
 
             tokens = [
-                t for t in jieba.lcut_for_search(query.text) if len(t.strip()) > 1
-            ] or [query.text]
+                t for t in jieba.lcut_for_search(request.text) if len(t.strip()) > 1
+            ] or [request.text]
             q_expr = Q()
             for token in tokens:
                 q_expr |= Q(content__icontains=token)
             query_orm = query_orm.filter(q_expr)
-        elif query.search_type == "dense" and not query.embedding and query.text:
-            query_orm = query_orm.filter(content__icontains=query.text)
+        elif request.search_type == "dense" and not request.embedding and request.text:
+            query_orm = query_orm.filter(content__icontains=request.text)
 
         rows = await query_orm
 
         valid_rows = []
         for row in rows:
             row_meta = row.meta_data if isinstance(row.meta_data, dict) else {}
-            if not FilterEvaluator.evaluate(row_meta, query.metadata_filters):
+            if not FilterEvaluator.evaluate(row_meta, request.metadata_filters):
                 continue
             valid_rows.append(row)
 
         if not valid_rows:
             return []
 
-        results = []
-        if query.search_type == "sparse":
-            import jieba
+        records = [self._to_base_record(row) for row in valid_rows]
 
-            tokens = set(jieba.lcut_for_search(query.text.lower()))
-            for row in valid_rows:
-                content = row.content.lower()
-                matched_count = sum(1 for t in tokens if t in content)
-                score = matched_count / len(tokens) if tokens else 0.1
-                results.append(
-                    SearchResult(record=self._to_base_record(row), score=score)
-                )
-        elif query.search_type == "dense" and query.embedding:
-            q_vec = normalize_vector(query.embedding)
-            vec_rows = []
-            missing_rows = []
-
-            for row in valid_rows:
-                if isinstance(row.embedding, list):
-                    vec_rows.append(row)
-                else:
-                    missing_rows.append(row)
-
-            if vec_rows:
-                try:
-                    raw_mat = np.array(
-                        [r.embedding for r in vec_rows], dtype=np.float32
-                    )
-                    norm_mat = normalize_matrix(raw_mat)
-                    scores = norm_mat @ q_vec
-
-                    for row, score in zip(vec_rows, scores):
-                        results.append(
-                            SearchResult(
-                                record=self._to_base_record(row), score=float(score)
-                            )
-                        )
-                except ValueError as e:
-                    logger.warning(
-                        "⚠️ 数据库中缓存的向量维度与当前模型查询维度不匹配，"
-                        f"已安全跳过向量检索(降级为稀疏匹配)。原因: {e}"
-                    )
-
-            for row in missing_rows:
-                results.append(
-                    SearchResult(record=self._to_base_record(row), score=0.1)
-                )
+        if request.search_type == "sparse":
+            results = InMemoryScorer.calculate_sparse_scores(request.text, records)
+        elif request.search_type == "dense" and request.embedding:
+            results = InMemoryScorer.calculate_dense_scores(request.embedding, records)
         else:
-            for row in valid_rows:
-                results.append(
-                    SearchResult(record=self._to_base_record(row), score=0.1)
-                )
+            results = [SearchResult(record=r, score=0.1) for r in records]
 
         results.sort(key=lambda x: x.score, reverse=True)
-        return results[: query.limit]
+        return results[: request.limit]
 
     async def update(self, record: BaseRecord) -> None:
         await self.model_class.filter(id=record.id).update(
@@ -386,50 +300,50 @@ class QdrantStorageBackend(StorageBackend):
             )
         await self.client.upsert(collection_name=self.collection_name, points=points)
 
-    async def search(
-        self, query: QueryRequest, scopes: list[str] | None = None
-    ) -> list[SearchResult]:
-        if query.search_type == "dense" and not query.embedding:
+    async def search(self, request: QueryRequest) -> list[SearchResult]:
+        if request.search_type == "dense" and not request.embedding:
             return []
-        if query.embedding:
-            await self._ensure_collection(len(query.embedding))
+        if request.embedding:
+            await self._ensure_collection(len(request.embedding))
 
         from qdrant_client.models import FieldCondition, Filter, MatchText, MatchValue
 
         must_conditions = []
 
-        if scopes is not None:
+        if request.scopes is not None:
             try:
                 from qdrant_client.models import MatchAny
 
                 must_conditions.append(
-                    FieldCondition(key="metadata.scope", match=MatchAny(any=scopes))
+                    FieldCondition(
+                        key="metadata.scope", match=MatchAny(any=request.scopes)
+                    )
                 )
             except ImportError:
                 scope_conditions = [
                     FieldCondition(key="metadata.scope", match=MatchValue(value=s))
-                    for s in scopes
+                    for s in request.scopes
                 ]
                 must_conditions.append(Filter(should=scope_conditions))
 
-        if query.metadata_filters:
-            for k, v in query.metadata_filters.items():
+        if request.metadata_filters:
+            for k, v in request.metadata_filters.items():
                 must_conditions.append(
                     FieldCondition(key=f"metadata.{k}", match=MatchValue(value=v))
                 )
 
-        if query.search_type == "sparse":
+        if request.search_type == "sparse":
             must_conditions.append(
-                FieldCondition(key="content", match=MatchText(text=query.text))
+                FieldCondition(key="content", match=MatchText(text=request.text))
             )
 
         query_filter = Filter(must=must_conditions) if must_conditions else None
 
-        if query.search_type == "sparse":
+        if request.search_type == "sparse":
             results = await self.client.scroll(
                 collection_name=self.collection_name,
                 scroll_filter=query_filter,
-                limit=query.limit,
+                limit=request.limit,
                 with_payload=True,
             )
             return [
@@ -446,8 +360,8 @@ class QdrantStorageBackend(StorageBackend):
 
         results = await self.client.search(  # type: ignore
             collection_name=self.collection_name,
-            query_vector=query.embedding,
-            limit=query.limit,
+            query_vector=request.embedding,
+            limit=request.limit,
             query_filter=query_filter,
         )
 
@@ -559,27 +473,25 @@ class LanceDBStorageBackend(StorageBackend):
         else:
             self.db.open_table(self.table_name).add(data)
 
-    async def search(
-        self, query: QueryRequest, scopes: list[str] | None = None
-    ) -> list[SearchResult]:
+    async def search(self, request: QueryRequest) -> list[SearchResult]:
         if self.table_name not in self.db.table_names():
             return []
-        if query.search_type == "dense" and not query.embedding:
+        if request.search_type == "dense" and not request.embedding:
             return []
 
         tbl = self.db.open_table(self.table_name)
-        if query.search_type == "sparse":
+        if request.search_type == "sparse":
             try:
                 results = (
-                    tbl.search(query.text, query_type="fts")
-                    .limit(query.limit)
+                    tbl.search(request.text, query_type="fts")
+                    .limit(request.limit)
                     .to_list()
                 )
             except Exception as e:
                 logger.warning(f"LanceDB FTS 检索失败(可能是由于尚未创建FTS索引): {e}")
                 return []
         else:
-            results = tbl.search(query.embedding).limit(query.limit).to_list()
+            results = tbl.search(request.embedding).limit(request.limit).to_list()
 
         import ast