zhenxun_bot/zhenxun/services/ai/llm/adapters/handlers/mimo_handlers.py

import base64
import json
from typing import Any

import httpx

from zhenxun.services.ai.core.messages import (
    AudioPart,
    AudioResponse,
    ImagePart,
    LLMMessage,
    SpeechRequest,
    TextPart,
    UsageInfo,
    VideoPart,
)
from zhenxun.services.ai.core.models import (
    ModelCapabilities,
    ModelIdentity,
)
from zhenxun.services.ai.core.options import TTSConfig
from zhenxun.services.ai.llm.adapters.base import BaseAdapter, RequestData

from .base import BaseAudioHandler
from .openai_handlers import (
    OpenAIConfigMapper,
    OpenAIMessageConverter,
    OpenAITextHandler,
    OpenAIToolSerializer,
)


class MiMoToolSerializer(OpenAIToolSerializer):
    """MiMo 工具序列化器，负责拦截并构造独有的 web_search 工具"""

    def serialize_server_tools(
        self, tools: list[Any], capabilities: ModelCapabilities
    ) -> list[dict[str, Any]]:
        res = []
        for t in tools:
            type_id = getattr(t, "type_id", "unknown")
            if type_id not in capabilities.supported_native_tools:
                continue
            if type_id == "web_search":
                res.append(
                    {
                        "type": "web_search",
                        "max_keyword": getattr(t, "max_keyword", 3),
                        "force_search": getattr(t, "force_search", True),
                        "limit": getattr(t, "limit", 1),
                    }
                )
        return res


class MiMoMessageConverter(OpenAIMessageConverter):
    """MiMo 消息转换器，拦截处理特有的音视频多模态结构"""

    async def convert_messages_async(
        self, messages: list[LLMMessage]
    ) -> list[dict[str, Any]]:
        openai_messages = await super().convert_messages_async(messages)

        for o_msg, o_orig in zip(openai_messages, messages):
            if o_msg["role"] == "user":
                content_parts = []
                for part in o_orig.content:
                    if isinstance(part, TextPart):
                        content_parts.append({"type": "text", "text": part.text})
                    elif isinstance(part, ImagePart):
                        src = (
                            part.url
                            if part.url
                            else await part.get_data_uri(part.mime_type or "image/jpeg")
                        )
                        content_parts.append(
                            {"type": "image_url", "image_url": {"url": src}}
                        )
                    elif isinstance(part, VideoPart):
                        src = (
                            part.url
                            if part.url
                            else await part.get_data_uri(part.mime_type or "video/mp4")
                        )
                        content_parts.append(
                            {
                                "type": "video_url",
                                "video_url": {"url": src},
                                "fps": getattr(part, "fps", 2),
                                "media_resolution": getattr(
                                    part, "media_resolution", "default"
                                ),
                            }
                        )
                    elif isinstance(part, AudioPart):
                        src = (
                            part.url
                            if part.url
                            else await part.get_data_uri(part.mime_type or "audio/mp3")
                        )
                        content_parts.append(
                            {"type": "input_audio", "input_audio": {"data": src}}
                        )
                o_msg["content"] = content_parts
        return openai_messages


class MiMoTextHandler(OpenAITextHandler):
    """MiMo 文本对话处理器集成"""

    def __init__(self, api_type: str = "mimo"):
        super().__init__(api_type=api_type)
        self.converter = MiMoMessageConverter(api_type=api_type)
        self.serializer = MiMoToolSerializer(api_type=api_type)
        self.mapper = OpenAIConfigMapper(api_type=api_type)


class MiMoAudioHandler(BaseAudioHandler):
    """MiMo TTS 接口实现：挂载在 Chat Completions 端点上"""

    def prepare_speech_request(
        self,
        adapter: BaseAdapter,
        identity: ModelIdentity,
        api_key: str,
        request: SpeechRequest,
    ) -> RequestData:
        input_text = request.input_text
        config = request.config or TTSConfig()

        config_voice = (
            config.mimo_options.voice_id if hasattr(config, "mimo_options") else None
        )
        voice = (
            config_voice
            or request.voice
            or identity.capabilities.default_voice_id
            or "mimo_default"
        )

        url = adapter.get_api_url(identity, "/v1/chat/completions")
        headers = adapter.get_base_headers(api_key)

        body = {
            "model": identity.model_name,
            "messages": [{"role": "assistant", "content": input_text}],
            "audio": {
                "format": config.response_format
                if config.response_format in ("wav", "pcm16")
                else "wav",
                "voice": voice,
            },
        }
        return RequestData(url=url, headers=headers, body=body)

    async def parse_speech_response(
        self,
        adapter: BaseAdapter,
        identity: ModelIdentity,
        raw_response: httpx.Response,
    ) -> AudioResponse:
        data = json.loads(await raw_response.aread())
        adapter.validate_response(data)
        audio_b64 = data["choices"][0]["message"]["audio"]["data"]
        return AudioResponse(
            audio_bytes=base64.b64decode(audio_b64),
            audio_format="wav",
            usage=UsageInfo(),
            model_name=identity.model_name,
        )