mirror of
https://github.com/zhenxun-org/zhenxun_bot.git
synced 2026-10-23 13:10:04 +08:00
✨ feat!(llm): 重构并升级大语言模型服务为全新 AI 智能体框架 (#2146)
* ✨ feat!(llm): 重构并升级大语言模型服务为全新 AI 智能体框架 - 【重构】将原 services/llm 重构并迁移至全新的 services/ai 架构，提供向下兼容垫片 - 【新增】引入 Agent、Team、Workflow 三大智能体与工作流编排范式 - 【新增】引入基于 RAG 的长期向量记忆与中期槽位记忆系统 - 【新增】引入基于 Docker 的安全代码执行沙箱环境 - 【新增】支持 MCP 协议，允许动态管理和调用 MCP 服务 - 【新增】引入输入输出安全合规护栏与自愈反思机制 - 【优化】重构并优化多厂商 API 适配器 (Gemini, OpenAI, DeepSeek, GLM 等) - 【优化】优化日志脱敏与 Token 预估机制 - 【移除】移除旧版 llm default 和 llm reset-key 命令，新增 llm mcp 管理命令 * 🔧 chore(deps): 更新项目依赖与配置 - 添加 mcp、jieba 和 aiodocker 依赖到配置文件及 requirements.txt - 在 pyright 配置中设置 reportMissingImports 为 none - 调整 .gitignore 中 resources 目录的忽略规则 * ♻️ refactor(tools): 重构工具终止机制并清理知识库日志输出 - 统一使用 `context.state["__end_run__"]` 替代 `EndRunResult` 控制任务结束 - 移除文件系统和向量知识库检索工具中 `ToolResult` 的 `.with_log` 调用 - 调整指令处理器（Directive）的返回值为 `tool_res.output` - 修复部分类型检查警告并优化联合类型判断语法 * ♻️ refactor(tools): 重构工具副作用指令与控制流熔断机制 - 引入 `DirectivePayload` 及 `ToolResult` 的子类以结构化表达工具副作用 - 移除通过 `context.state` 传递魔术变量的隐式控制流设计 - 重构 `DirectiveManager` 处理器接口，直接在处理器中修改 `AgentState` 并构建 `AgentRunResult` - 在 `StandardAgentExecutor` 中统一通过 `directive_manager` 调度工具返回的副作用指令 - 补全 `MessageBuilder` 中部分核心方法的文档注释 * 🐛 fix(sandbox): 修复 Docker 沙箱容器状态检测与会话清理逻辑 -【修复】修正 `is_alive` 中直接读取私有属性的问题，改用 `show()` 返回值 -【修复】解决 `execute_code` 中缓存的执行器与当前会话不一致的问题 -【优化】在清理工作区前增加容器存活检测，避免向已死容器发送请求 -【优化】创建容器时增加运行状态校验，若已停止则自动从缓存中移除并重建 -【优化】优化容器销毁和清理逻辑，静默处理容器不存在 (404) 的异常 * 📝 docs(core): 补充核心模块初始化方法的文档注释 * 🚨 auto fix by pre-commit hooks --------- Co-authored-by: webjoin111 <455457521@qq.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Rumio
co-authored by
webjoin111
pre-commit-ci[bot]
webjoin111
pre-commit-ci[bot]
parent
bdc1374848
commit
80fc5b86a7
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from zhenxun.services.ai.core.messages import (
|
||||
AudioResponse,
|
||||
ChatRequest,
|
||||
EmbeddingRequest,
|
||||
ImageRequest,
|
||||
LLMMessage,
|
||||
RerankRequest,
|
||||
RerankResult,
|
||||
SpeechRequest,
|
||||
)
|
||||
from zhenxun.services.ai.core.models import (
|
||||
ModelCapabilities,
|
||||
ModelDetail,
|
||||
ModelIdentity,
|
||||
ToolDefinition,
|
||||
)
|
||||
from zhenxun.services.ai.core.options import (
|
||||
GenerationConfig,
|
||||
)
|
||||
from zhenxun.services.ai.llm.adapters.base import BaseAdapter, RequestData, ResponseData
|
||||
|
||||
|
||||
class ConfigMapper(ABC):
|
||||
@abstractmethod
|
||||
def map_config(
|
||||
self,
|
||||
config: GenerationConfig,
|
||||
model_detail: ModelDetail | None = None,
|
||||
capabilities: ModelCapabilities | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""将通用生成配置转换为特定 API 的参数字典"""
|
||||
...
|
||||
|
||||
|
||||
class MessageConverter(ABC):
|
||||
@abstractmethod
|
||||
async def convert_messages_async(
|
||||
self, messages: list[LLMMessage]
|
||||
) -> list[dict[str, Any]] | dict[str, Any]:
|
||||
"""将通用消息列表异步转换为特定 API 的消息格式"""
|
||||
...
|
||||
|
||||
|
||||
class ToolSerializer(ABC):
|
||||
def serialize_tools(
|
||||
self, tools: list[ToolDefinition]
|
||||
) -> list[dict[str, Any]] | None:
|
||||
"""将通用工具定义转换为特定 API 的工具格式 (模板方法)"""
|
||||
if not tools:
|
||||
return None
|
||||
|
||||
serialized_tools = []
|
||||
for tool in tools:
|
||||
raw_schema = tool.parameters.copy() if tool.parameters else {}
|
||||
sanitized_schema = self.sanitize_schema(raw_schema)
|
||||
tool_payload = self.format_tool_payload(
|
||||
tool_name=tool.name,
|
||||
tool_description=tool.description or "",
|
||||
sanitized_schema=sanitized_schema,
|
||||
)
|
||||
serialized_tools.append(tool_payload)
|
||||
return serialized_tools
|
||||
|
||||
@abstractmethod
|
||||
def format_tool_payload(
|
||||
self, tool_name: str, tool_description: str, sanitized_schema: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
"""由子类实现：格式化单一工具的 Payload"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def sanitize_schema(self, schema: dict[str, Any]) -> dict[str, Any]:
|
||||
"""对 JSON Schema 进行特定 API 的清洗和格式化"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def serialize_server_tools(
|
||||
self, tools: list[Any], capabilities: ModelCapabilities
|
||||
) -> list[dict[str, Any]]:
|
||||
"""将系统内置的原生云端工具转译为底层 API 载荷（执行双重能力校验）"""
|
||||
...
|
||||
|
||||
|
||||
class ResponseParser(ABC):
|
||||
@abstractmethod
|
||||
def parse(self, response_json: dict[str, Any]) -> ResponseData:
|
||||
"""将特定 API 的响应解析为通用响应数据"""
|
||||
...
|
||||
|
||||
|
||||
class BaseTextHandler(ABC):
|
||||
"""
|
||||
文本对话生成处理器接口。
|
||||
负责将真寻的通用消息与工具列表转换为底层 API 的请求格式，并解析响应。
|
||||
"""
|
||||
|
||||
async def _resolve_and_split_tools(
|
||||
self, tools: list[Any] | None
|
||||
) -> tuple[list[Any], list[Any], list[Any]]:
|
||||
"""
|
||||
统一的工具解析逻辑：分离客户端工具与服务端内置工具，并发获取 Schema。
|
||||
返回: (tool_defs, client_executables, server_tools)
|
||||
"""
|
||||
client_executables, server_tools, tool_defs = [], [], []
|
||||
if tools:
|
||||
raw_tools = list(tools.values()) if isinstance(tools, dict) else tools
|
||||
for tool in raw_tools:
|
||||
if getattr(tool, "execution_side", "client") == "server":
|
||||
server_tools.append(tool)
|
||||
elif hasattr(tool, "get_definition"):
|
||||
client_executables.append(tool)
|
||||
if definition_tasks := [t.get_definition() for t in client_executables]:
|
||||
tool_defs = [td for td in await asyncio.gather(*definition_tasks) if td]
|
||||
return tool_defs, client_executables, server_tools
|
||||
|
||||
@abstractmethod
|
||||
async def prepare_text_request(
|
||||
self,
|
||||
adapter: BaseAdapter,
|
||||
identity: ModelIdentity,
|
||||
api_key: str,
|
||||
request: ChatRequest,
|
||||
) -> RequestData: ...
|
||||
|
||||
@abstractmethod
|
||||
def parse_text_response(
|
||||
self,
|
||||
adapter: BaseAdapter,
|
||||
identity: ModelIdentity,
|
||||
response_json: dict[str, Any],
|
||||
is_advanced: bool = False,
|
||||
) -> ResponseData: ...
|
||||
|
||||
|
||||
class BaseEmbeddingHandler(ABC):
|
||||
"""
|
||||
文本嵌入向量处理器接口。
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def prepare_embedding_request(
|
||||
self,
|
||||
adapter: BaseAdapter,
|
||||
identity: ModelIdentity,
|
||||
api_key: str,
|
||||
request: EmbeddingRequest,
|
||||
) -> RequestData: ...
|
||||
|
||||
@abstractmethod
|
||||
def parse_embedding_response(
|
||||
self, adapter: BaseAdapter, response_json: dict[str, Any]
|
||||
) -> list[list[float]]: ...
|
||||
|
||||
|
||||
class BaseImageHandler(ABC):
|
||||
"""
|
||||
图像生成/编辑处理器接口。
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def prepare_image_request(
|
||||
self,
|
||||
adapter: BaseAdapter,
|
||||
identity: ModelIdentity,
|
||||
api_key: str,
|
||||
request: ImageRequest,
|
||||
) -> RequestData: ...
|
||||
|
||||
@abstractmethod
|
||||
def parse_image_response(
|
||||
self, adapter: BaseAdapter, response_json: dict[str, Any]
|
||||
) -> ResponseData: ...
|
||||
|
||||
|
||||
class BaseRerankHandler(ABC):
|
||||
"""
|
||||
文本重排处理器接口。
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def prepare_rerank_request(
|
||||
self,
|
||||
adapter: BaseAdapter,
|
||||
identity: ModelIdentity,
|
||||
api_key: str,
|
||||
request: RerankRequest,
|
||||
) -> RequestData: ...
|
||||
|
||||
@abstractmethod
|
||||
def parse_rerank_response(
|
||||
self, adapter: BaseAdapter, response_json: dict[str, Any]
|
||||
) -> list[RerankResult]: ...
|
||||
|
||||
|
||||
class BaseAudioHandler(ABC):
|
||||
"""
|
||||
文本转语音 (TTS) 处理器接口。
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def prepare_speech_request(
|
||||
self,
|
||||
adapter: BaseAdapter,
|
||||
identity: ModelIdentity,
|
||||
api_key: str,
|
||||
request: SpeechRequest,
|
||||
) -> RequestData: ...
|
||||
|
||||
@abstractmethod
|
||||
async def parse_speech_response(
|
||||
self,
|
||||
adapter: BaseAdapter,
|
||||
identity: ModelIdentity,
|
||||
raw_response: httpx.Response,
|
||||
) -> AudioResponse: ...
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,199 @@
|
||||
import base64
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from zhenxun.services.ai.core.messages import (
|
||||
AudioPart,
|
||||
AudioResponse,
|
||||
ImagePart,
|
||||
LLMMessage,
|
||||
SpeechRequest,
|
||||
TextPart,
|
||||
UsageInfo,
|
||||
VideoPart,
|
||||
)
|
||||
from zhenxun.services.ai.core.models import (
|
||||
ModelCapabilities,
|
||||
ModelDetail,
|
||||
ModelIdentity,
|
||||
)
|
||||
from zhenxun.services.ai.core.options import GenerationConfig, TTSConfig
|
||||
from zhenxun.services.ai.llm.adapters.base import BaseAdapter, RequestData
|
||||
from zhenxun.services.ai.llm.adapters.handlers.base import BaseAudioHandler
|
||||
from zhenxun.services.ai.llm.adapters.handlers.openai_handlers import (
|
||||
OpenAIConfigMapper,
|
||||
OpenAIMessageConverter,
|
||||
OpenAITextHandler,
|
||||
OpenAIToolSerializer,
|
||||
)
|
||||
|
||||
|
||||
class MiMoToolSerializer(OpenAIToolSerializer):
|
||||
"""MiMo 工具序列化器，负责拦截并构造独有的 web_search 工具"""
|
||||
|
||||
def serialize_server_tools(
|
||||
self, tools: list[Any], capabilities: ModelCapabilities
|
||||
) -> list[dict[str, Any]]:
|
||||
res = []
|
||||
for t in tools:
|
||||
type_id = getattr(t, "type_id", "unknown")
|
||||
if type_id not in capabilities.supported_native_tools:
|
||||
continue
|
||||
if type_id == "web_search":
|
||||
res.append(
|
||||
{
|
||||
"type": "web_search",
|
||||
"max_keyword": getattr(t, "max_keyword", 3),
|
||||
"force_search": getattr(t, "force_search", True),
|
||||
"limit": getattr(t, "limit", 1),
|
||||
}
|
||||
)
|
||||
return res
|
||||
|
||||
|
||||
class MiMoConfigMapper(OpenAIConfigMapper):
|
||||
"""MiMo 配置映射器，处理深度思考参数差异"""
|
||||
|
||||
def map_config(
|
||||
self,
|
||||
config: GenerationConfig,
|
||||
model_detail: ModelDetail | None = None,
|
||||
capabilities: ModelCapabilities | None = None,
|
||||
) -> dict[str, Any]:
|
||||
params = super().map_config(config, model_detail, capabilities)
|
||||
|
||||
if config.common.reasoning_effort:
|
||||
effort = str(config.common.reasoning_effort).lower()
|
||||
if effort == "none":
|
||||
params["thinking"] = {"type": "disabled"}
|
||||
else:
|
||||
params["thinking"] = {"type": "enabled"}
|
||||
elif (
|
||||
hasattr(config, "deepseek_options")
|
||||
and config.deepseek_options.thinking is not None
|
||||
):
|
||||
if config.deepseek_options.thinking is True:
|
||||
params["thinking"] = {"type": "enabled"}
|
||||
elif config.deepseek_options.thinking is False:
|
||||
params["thinking"] = {"type": "disabled"}
|
||||
|
||||
return params
|
||||
|
||||
|
||||
class MiMoMessageConverter(OpenAIMessageConverter):
|
||||
"""MiMo 消息转换器，拦截处理特有的音视频多模态结构"""
|
||||
|
||||
async def convert_messages_async(
|
||||
self, messages: list[LLMMessage]
|
||||
) -> list[dict[str, Any]]:
|
||||
openai_messages = await super().convert_messages_async(messages)
|
||||
|
||||
for o_msg, o_orig in zip(openai_messages, messages):
|
||||
if o_msg["role"] == "user":
|
||||
content_parts = []
|
||||
for part in o_orig.content:
|
||||
if isinstance(part, TextPart):
|
||||
content_parts.append({"type": "text", "text": part.text})
|
||||
elif isinstance(part, ImagePart):
|
||||
src = (
|
||||
part.url
|
||||
if part.url
|
||||
else await part.get_data_uri(part.mime_type or "image/jpeg")
|
||||
)
|
||||
content_parts.append(
|
||||
{"type": "image_url", "image_url": {"url": src}}
|
||||
)
|
||||
elif isinstance(part, VideoPart):
|
||||
src = (
|
||||
part.url
|
||||
if part.url
|
||||
else await part.get_data_uri(part.mime_type or "video/mp4")
|
||||
)
|
||||
content_parts.append(
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {"url": src},
|
||||
"fps": getattr(part, "fps", 2),
|
||||
"media_resolution": getattr(
|
||||
part, "media_resolution", "default"
|
||||
),
|
||||
}
|
||||
)
|
||||
elif isinstance(part, AudioPart):
|
||||
src = (
|
||||
part.url
|
||||
if part.url
|
||||
else await part.get_data_uri(part.mime_type or "audio/mp3")
|
||||
)
|
||||
content_parts.append(
|
||||
{"type": "input_audio", "input_audio": {"data": src}}
|
||||
)
|
||||
o_msg["content"] = content_parts
|
||||
return openai_messages
|
||||
|
||||
|
||||
class MiMoTextHandler(OpenAITextHandler):
|
||||
"""MiMo 文本对话处理器集成"""
|
||||
|
||||
def __init__(self, api_type: str = "mimo"):
|
||||
super().__init__(api_type=api_type)
|
||||
self.converter = MiMoMessageConverter(api_type=api_type)
|
||||
self.serializer = MiMoToolSerializer(api_type=api_type)
|
||||
self.mapper = MiMoConfigMapper(api_type=api_type)
|
||||
|
||||
|
||||
class MiMoAudioHandler(BaseAudioHandler):
|
||||
"""MiMo TTS 接口实现：挂载在 Chat Completions 端点上"""
|
||||
|
||||
def prepare_speech_request(
|
||||
self,
|
||||
adapter: BaseAdapter,
|
||||
identity: ModelIdentity,
|
||||
api_key: str,
|
||||
request: SpeechRequest,
|
||||
) -> RequestData:
|
||||
input_text = request.input_text
|
||||
config = request.config or TTSConfig()
|
||||
|
||||
config_voice = (
|
||||
config.mimo_options.voice_id if hasattr(config, "mimo_options") else None
|
||||
)
|
||||
voice = (
|
||||
config_voice
|
||||
or request.voice
|
||||
or identity.capabilities.default_voice_id
|
||||
or "mimo_default"
|
||||
)
|
||||
|
||||
url = adapter.get_api_url(identity, "/v1/chat/completions")
|
||||
headers = adapter.get_base_headers(api_key)
|
||||
|
||||
body = {
|
||||
"model": identity.model_name,
|
||||
"messages": [{"role": "assistant", "content": input_text}],
|
||||
"audio": {
|
||||
"format": config.response_format
|
||||
if config.response_format in ("wav", "pcm16")
|
||||
else "wav",
|
||||
"voice": voice,
|
||||
},
|
||||
}
|
||||
return RequestData(url=url, headers=headers, body=body)
|
||||
|
||||
async def parse_speech_response(
|
||||
self,
|
||||
adapter: BaseAdapter,
|
||||
identity: ModelIdentity,
|
||||
raw_response: httpx.Response,
|
||||
) -> AudioResponse:
|
||||
data = json.loads(await raw_response.aread())
|
||||
adapter.validate_response(data)
|
||||
audio_b64 = data["choices"][0]["message"]["audio"]["data"]
|
||||
return AudioResponse(
|
||||
audio_bytes=base64.b64decode(audio_b64),
|
||||
audio_format="wav",
|
||||
usage=UsageInfo(),
|
||||
model_name=identity.model_name,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user