From 5c067bcf040466f4d10f6f9735915325f12d4910 Mon Sep 17 00:00:00 2001 From: HibiKier <45528451+HibiKier@users.noreply.github.com> Date: Fri, 6 Feb 2026 16:45:48 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(http=5Futils):=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E5=8F=AF=E9=80=89=E7=9A=84=20accept=5Fstatus=5Fcodes?= =?UTF-8?q?=20=E5=8F=82=E6=95=B0=E4=BB=A5=E6=94=AF=E6=8C=81=E8=87=AA?= =?UTF-8?q?=E5=AE=9A=E4=B9=89=E7=8A=B6=E6=80=81=E7=A0=81=E5=A4=84=E7=90=86?= =?UTF-8?q?=20(#2096)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在 AsyncHttpx 类中，新增 accept_status_codes 参数，允许用户指定哪些 HTTP 状态码不视为错误。 - 更新了单次请求的逻辑，以便在响应状态码符合 accept_status_codes 时不调用 raise_for_status 方法。 ✨ feat(message): 增加随机失败提示消息功能 - 在 MessageUtils 类中，添加可爱风格的失败提示消息列表和获取随机消息的方法。 ✨ feat(github_utils): 引入动态查询外部插件分组的功能 - 在 AliyunFileInfo 类中，新增 list_group_repositories 方法以列出分组下的所有仓库，并实现通过仓库名称获取仓库ID的逻辑。 ✨ feat(virtual_env_package_manager): 增加清理 requirements 文件的功能 - 新增 _clean_requirements_file 方法，清理 requirements 文件中的非ASCII注释，避免 Windows 上 pip 使用 GBK 编码读取 UTF-8 文件时出错。 ✨ feat(repo_utils): 优化阿里云仓库 URL 处理逻辑 - 更新 prepare_aliyun_url 方法以支持指定分组名称，并新增 get_aliyun_group_for_repo 方法以获取仓库所属的阿里云分组名。 --- zhenxun/utils/github_utils/const.py | 3 + zhenxun/utils/github_utils/models.py | 196 ++++++++++++++++-- zhenxun/utils/http_utils.py | 28 ++- .../manager/virtual_env_package_manager.py | 36 ++++ zhenxun/utils/message.py | 39 ++++ zhenxun/utils/repo_utils/file_manager.py | 47 ++++- zhenxun/utils/repo_utils/utils.py | 41 +++- 7 files changed, 368 insertions(+), 22 deletions(-) diff --git a/zhenxun/utils/github_utils/const.py b/zhenxun/utils/github_utils/const.py index 102e6f19..582332eb 100644 --- a/zhenxun/utils/github_utils/const.py +++ b/zhenxun/utils/github_utils/const.py @@ -74,3 +74,6 @@ ALIYUN_REPO_MAPPING = { "zhenxun_bot": "4957428", } """阿里云仓库ID映射""" + +ALIYUN_EXTERNAL_PLUGIN_GROUPS: list[str] = ["zhenxun_plugins"] +"""阿里云外部插件分组列表，用于动态查询第三方插件仓库""" diff --git a/zhenxun/utils/github_utils/models.py b/zhenxun/utils/github_utils/models.py index ae4ab2d3..9c5c3fef 100644 --- a/zhenxun/utils/github_utils/models.py +++ b/zhenxun/utils/github_utils/models.py @@ -1,7 +1,7 @@ import base64 import contextlib import sys -from typing import Protocol +from typing import ClassVar, Protocol from aiocache import cached from alibabacloud_devops20210625 import models as devops_20210625_models @@ -20,6 +20,7 @@ else: from .const import ( ALIYUN_ENDPOINT, + ALIYUN_EXTERNAL_PLUGIN_GROUPS, ALIYUN_ORG_ID, ALIYUN_REGION, ALIYUN_REPO_MAPPING, @@ -317,6 +318,9 @@ class AliyunFileInfo: repository_id: str """仓库ID""" + # 动态仓库ID缓存: {repo_name: repository_id} + _dynamic_repo_cache: ClassVar[dict[str, str]] = {} + @classmethod async def get_client(cls) -> devops20210625Client: """获取阿里云客户端""" @@ -331,6 +335,179 @@ class AliyunFileInfo: return devops20210625Client(config) + @classmethod + @cached(ttl=CACHED_API_TTL) + async def list_group_repositories(cls, group_path: str) -> list[dict]: + """列出分组下的所有仓库 + + 参数: + group_path: 分组路径，如 "zhenxun_plugins" + + 返回: + list[dict]: 仓库信息列表，每个元素包含 id, name, path 等 + """ + try: + client = await cls.get_client() + result = [] + page = 1 + per_page = 100 + + while True: + request = devops_20210625_models.ListRepositoriesRequest( + organization_id=ALIYUN_ORG_ID, + access_token=base64.b64decode( + RDC_access_token_encrypted.encode() + ).decode(), + page=page, + per_page=per_page, + ) + + runtime = util_models.RuntimeOptions() + headers = {} + + response = await client.list_repositories_with_options_async( + request, headers, runtime + ) + + if response and response.body: + if not response.body.success: + raise ValueError( + f"阿里云请求失败: {response.body.error_code} - " + f"{response.body.error_message}" + ) + + repos = response.body.result or [] + if not repos: + break + + for repo in repos: + repo_dict = repo.to_map() + # 尝试多种可能的字段名 + path_with_ns = ( + repo_dict.get("pathWithNamespace") + or repo_dict.get("path_with_namespace") + or repo_dict.get("path") + or "" + ) + # pathWithNamespace 格式: {org_id}/{group_path}/{repo_name} + # 只保留属于该分组的仓库 + if f"/{group_path}/" in path_with_ns: + result.append(repo_dict) + # 同时更新缓存 + repo_name = repo_dict.get("name", "") + # 注意：阿里云API返回的ID字段是 "Id" (大写I) + repo_id = str( + repo_dict.get("Id") or repo_dict.get("id") or "" + ) + if repo_name and repo_id: + cls._dynamic_repo_cache[repo_name] = repo_id + + # 如果返回的数量小于请求的数量，说明已经没有更多数据 + if len(repos) < per_page: + break + page += 1 + else: + break + + return result + except Exception as e: + raise ValueError(f"获取仓库列表失败: {e}") + + @classmethod + async def get_repository_id(cls, repo_name: str) -> str | None: + """通过仓库名称获取仓库ID + + 优先从静态映射中查找，然后从缓存中查找，最后从API动态查询 + + 参数: + repo_name: 仓库名称 + + 返回: + str | None: 仓库ID，未找到返回 None + """ + # 1. 先从静态映射中查找 + if repo_id := ALIYUN_REPO_MAPPING.get(repo_name): + return repo_id + + # 2. 从动态缓存中查找 + if repo_id := cls._dynamic_repo_cache.get(repo_name): + return repo_id + + # 3. 从外部插件分组中动态查询 + for group_path in ALIYUN_EXTERNAL_PLUGIN_GROUPS: + try: + repos = await cls.list_group_repositories(group_path) + for repo in repos: + if repo.get("name") == repo_name: + # 注意：阿里云API返回的ID字段是 "Id" (大写I) + repo_id = str(repo.get("Id") or repo.get("id") or "") + if repo_id: + cls._dynamic_repo_cache[repo_name] = repo_id + return repo_id + except Exception: + continue + + return None + + @classmethod + async def debug_list_all_repositories(cls) -> list[dict]: + """调试方法：列出组织下的所有仓库信息 + + 返回: + list[dict]: 所有仓库信息列表 + """ + try: + client = await cls.get_client() + result = [] + + request = devops_20210625_models.ListRepositoriesRequest( + organization_id=ALIYUN_ORG_ID, + access_token=base64.b64decode( + RDC_access_token_encrypted.encode() + ).decode(), + page=1, + per_page=100, + ) + + runtime = util_models.RuntimeOptions() + headers = {} + + response = await client.list_repositories_with_options_async( + request, headers, runtime + ) + + if response and response.body and response.body.result: + for repo in response.body.result: + repo_dict = repo.to_map() + result.append(repo_dict) + # 同时填充缓存 + repo_name = repo_dict.get("name", "") + repo_id = str(repo_dict.get("Id") or repo_dict.get("id") or "") + if repo_name and repo_id: + cls._dynamic_repo_cache[repo_name] = repo_id + + return result + except Exception as e: + raise ValueError(f"列出仓库失败: {e}") + + @classmethod + async def _get_repository_id_or_raise(cls, repo: str) -> str: + """获取仓库ID，如果未找到则抛出异常 + + 参数: + repo: 仓库名称 + + 返回: + str: 仓库ID + + 异常: + ValueError: 未找到仓库 + """ + repository_id = await cls.get_repository_id(repo) + if not repository_id: + raise ValueError(f"未找到仓库 {repo} 对应的阿里云仓库ID") + return repository_id + @classmethod async def get_file_content( cls, file_path: str, repo: str, ref: str = "main" @@ -346,9 +523,7 @@ class AliyunFileInfo: str: 文件内容 """ try: - repository_id = ALIYUN_REPO_MAPPING.get(repo) - if not repository_id: - raise ValueError(f"未找到仓库 {repo} 对应的阿里云仓库ID") + repository_id = await cls._get_repository_id_or_raise(repo) client = await cls.get_client() @@ -406,9 +581,7 @@ class AliyunFileInfo: list[AliyunTree]: 仓库树信息列表 """ try: - repository_id = ALIYUN_REPO_MAPPING.get(repo) - if not repository_id: - raise ValueError(f"未找到仓库 {repo} 对应的阿里云仓库ID") + repository_id = await cls._get_repository_id_or_raise(repo) client = await cls.get_client() @@ -452,9 +625,7 @@ class AliyunFileInfo: commit: 最新提交信息 """ try: - repository_id = ALIYUN_REPO_MAPPING.get(repo) - if not repository_id: - raise ValueError(f"未找到仓库 {repo} 对应的阿里云仓库ID") + repository_id = await cls._get_repository_id_or_raise(repo) client = await cls.get_client() @@ -498,9 +669,8 @@ class AliyunFileInfo: @classmethod async def parse_repo_info(cls, repo: str) -> list[str]: """解析仓库信息获取仓库树""" - repository_id = ALIYUN_REPO_MAPPING.get(repo) - if not repository_id: - raise ValueError(f"未找到仓库 {repo} 对应的阿里云仓库ID") + # 验证仓库存在 + await cls._get_repository_id_or_raise(repo) tree_list = await cls.get_repository_tree( repo=repo, diff --git a/zhenxun/utils/http_utils.py b/zhenxun/utils/http_utils.py index 8c8e97eb..71b2a9e0 100644 --- a/zhenxun/utils/http_utils.py +++ b/zhenxun/utils/http_utils.py @@ -219,8 +219,12 @@ class AsyncHttpx: ) -> Response: """ 执行单次HTTP请求的私有方法，内置了默认的重试逻辑。 + accept_status_codes: 若提供，这些状态码不视为错误，不调用 raise_for_status。 """ client_kwargs, request_kwargs = cls._split_kwargs(kwargs) + accept_status_codes: Sequence[int] | None = request_kwargs.pop( + "accept_status_codes", None + ) async with cls._get_active_client_context( client=client, **client_kwargs @@ -228,7 +232,11 @@ class AsyncHttpx: response = await cls()._execute_request_inner( active_client, method, url, **request_kwargs ) - response.raise_for_status() + if ( + accept_status_codes is None + or response.status_code not in accept_status_codes + ): + response.raise_for_status() return response @classmethod @@ -292,6 +300,7 @@ class AsyncHttpx: *, follow_redirects: bool = True, check_status_code: int | None = None, + accept_status_codes: Sequence[int] | None = None, client: AsyncClient | None = None, **kwargs, ) -> Response: @@ -301,6 +310,8 @@ class AsyncHttpx: url: 单个请求 URL 或一个 URL 列表。 follow_redirects: 是否跟随重定向。 check_status_code: (可选) 若提供，将检查响应状态码是否匹配，否则抛出异常。 + accept_status_codes: (可选) 这些状态码不视为错误， + 如 (302,) 用于接受重定向响应。 client: (可选) 指定一个活动的HTTP客户端实例。若提供，则忽略 `**kwargs`中的客户端配置。 **kwargs: 其他所有传递给 httpx.get 的参数 (如 `params`, `headers`, @@ -316,6 +327,8 @@ class AsyncHttpx: async def worker(current_url: str, **worker_kwargs) -> Response: logger.info(f"开始获取 {current_url}..", "AsyncHttpx:get") + if accept_status_codes is not None: + worker_kwargs["accept_status_codes"] = accept_status_codes response = await cls._single_request( "GET", current_url, follow_redirects=follow_redirects, **worker_kwargs ) @@ -342,11 +355,20 @@ class AsyncHttpx: @classmethod async def post( - cls, url: str | list[str], *, client: AsyncClient | None = None, **kwargs + cls, + url: str | list[str], + *, + accept_status_codes: Sequence[int] | None = None, + client: AsyncClient | None = None, + **kwargs, ) -> Response: - """发送 POST 请求，并返回第一个成功的响应。""" + """发送 POST 请求，并返回第一个成功的响应。 + accept_status_codes: (可选) 这些状态码不视为错误，如 (302,) 用于接受重定向响应。 + """ async def worker(current_url: str, **worker_kwargs) -> Response: + if accept_status_codes is not None: + worker_kwargs["accept_status_codes"] = accept_status_codes return await cls._single_request("POST", current_url, **worker_kwargs) return await cls._execute_with_fallbacks(url, worker, client=client, **kwargs) diff --git a/zhenxun/utils/manager/virtual_env_package_manager.py b/zhenxun/utils/manager/virtual_env_package_manager.py index 7f938e0a..c25143fb 100644 --- a/zhenxun/utils/manager/virtual_env_package_manager.py +++ b/zhenxun/utils/manager/virtual_env_package_manager.py @@ -125,6 +125,40 @@ class VirtualEnvPackageManager: logger.error(f"更新虚拟环境包指令执行失败: {e.stderr}.", LOG_COMMAND) return e.stderr + @staticmethod + def _clean_requirements_file(file_path: Path) -> None: + """清理 requirements 文件中的非ASCII注释 + + 防止 Windows 上 pip 使用 GBK 编码读取 UTF-8 文件时出错 + + 参数: + file_path: requirements 文件路径 + """ + try: + content = file_path.read_text(encoding="utf-8") + lines = content.splitlines() + cleaned_lines = [] + for line in lines: + stripped = line.strip() + # 跳过空行 + if not stripped: + continue + # 如果是注释行且包含非ASCII字符，跳过 + if stripped.startswith("#"): + try: + stripped.encode("ascii") + except UnicodeEncodeError: + continue + cleaned_lines.append(line) + # 写回文件 + file_path.write_text( + "\n".join(cleaned_lines) + "\n" if cleaned_lines else "", + encoding="utf-8", + ) + except Exception: + # 如果清理失败，忽略错误继续安装 + pass + @classmethod async def install_requirement(cls, requirement_file: Path): """安装依赖文件 @@ -137,6 +171,8 @@ class VirtualEnvPackageManager: """ if not requirement_file.exists(): raise FileNotFoundError(f"依赖文件 {requirement_file} 不存在", LOG_COMMAND) + # 清理 requirements 文件中的非ASCII注释，防止 Windows GBK 编码问题 + cls._clean_requirements_file(requirement_file) try: command = cls.__get_command() command.append("install") diff --git a/zhenxun/utils/message.py b/zhenxun/utils/message.py index 5fec2213..27e7b8f2 100644 --- a/zhenxun/utils/message.py +++ b/zhenxun/utils/message.py @@ -1,6 +1,8 @@ import base64 from io import BytesIO from pathlib import Path +import random +from typing import ClassVar import nonebot from nonebot.adapters.onebot.v11 import Message, MessageSegment @@ -48,6 +50,43 @@ class Config(BaseModel): class MessageUtils: + # 可爱风格的失败提示消息列表 + FAILURE_MESSAGES: ClassVar[list[str]] = [ + "出了点小问题，待会再试试吧~ (´・ω・`)", + "哎呀，失败了呢 QAQ", + "好像哪里不对劲... (・∀・；)", + "emm...出错了 (；′⌒`)", + "失败了，但问题不大！╮(╯▽╰)╭", + "搞砸了...下次一定行 (๑•̀ㅂ•́)و✧", + "这次没成功，再来一次？", + "出错啦，让我缓缓... (´-ω-`)", + "翻车了，容我想想 ( ˘•ω•˘ )", + "不太顺利呢，稍后再试吧 (´;ω;`)", + "失败了...不过没关系啦 (・ω<)☆", + "呃，出了点状况 Σ(っ°Д°;)っ", + "没搞定，但别灰心~", + "坏掉了...等等再试试看 (>_<)", + "这次不太行，下次加油！(ง •_•)ง", + ] + + @classmethod + def get_failure_message(cls) -> str: + """获取随机失败提示消息 + + 返回: + str: 随机的可爱失败提示消息 + """ + return random.choice(cls.FAILURE_MESSAGES) + + @classmethod + def build_failure_message(cls) -> UniMessage: + """构造随机失败提示消息 + + 返回: + UniMessage: 构造完成的失败提示消息 + """ + return cls.build_message(cls.get_failure_message()) + @classmethod def __build_message( cls, msg_list: list[MESSAGE_TYPE], format_args: dict | None = None diff --git a/zhenxun/utils/repo_utils/file_manager.py b/zhenxun/utils/repo_utils/file_manager.py index 6aca3120..2fd3abcf 100644 --- a/zhenxun/utils/repo_utils/file_manager.py +++ b/zhenxun/utils/repo_utils/file_manager.py @@ -23,7 +23,7 @@ from .exceptions import ( RepoManagerError, ) from .models import FileDownloadResult, RepoFileInfo, RepoType -from .utils import prepare_aliyun_url, sparse_checkout_clone +from .utils import get_aliyun_group_for_repo, prepare_aliyun_url, sparse_checkout_clone class RepoFileManager: @@ -469,6 +469,35 @@ class RepoFileManager: if all(f.path != file.path for f in file_list if f != file) ] + def _clean_requirements_content(self, content: str) -> str: + """ + 清理 requirements.txt 内容，移除包含非ASCII字符的注释行 + + 这是为了防止 Windows 上 pip 使用 GBK 编码读取 UTF-8 文件时出错 + + 参数: + content: requirements.txt 文件内容 + + 返回: + str: 清理后的内容 + """ + lines = content.splitlines() + cleaned_lines = [] + for line in lines: + stripped = line.strip() + # 跳过空行 + if not stripped: + continue + # 如果是注释行且包含非ASCII字符，跳过 + if stripped.startswith("#"): + try: + stripped.encode("ascii") + except UnicodeEncodeError: + # 包含非ASCII字符的注释行，跳过 + continue + cleaned_lines.append(line) + return "\n".join(cleaned_lines) + "\n" if cleaned_lines else "" + async def download_files( self, repo_url: str, @@ -576,6 +605,10 @@ class RepoFileManager: local_path = file_path_mapping[repo_file_path] local_path.parent.mkdir(parents=True, exist_ok=True) if isinstance(content, str): + # 对 requirements 文件特殊处理：移除包含非ASCII字符的注释行 + # 防止 Windows GBK 编码问题 + if repo_file_path.endswith(("requirements.txt", "requirement.txt")): + content = self._clean_requirements_content(content) content_bytes = content.encode("utf-8") else: content_bytes = content @@ -604,8 +637,16 @@ class RepoFileManager: result: FileDownloadResult, ) -> FileDownloadResult: try: + # 获取仓库所属的分组名（外部插件仓库可能在不同分组下） + repo_name = ( + repo_url.split("/tree/")[0].split("/")[-1].replace(".git", "").strip() + ) + group_name = await get_aliyun_group_for_repo(repo_name) + + aliyun_repo_url = prepare_aliyun_url(repo_url, group_name) + await sparse_checkout_clone( - repo_url=prepare_aliyun_url(repo_url), + repo_url=aliyun_repo_url, branch=branch, sparse_path=sparse_path, target_dir=target_dir, @@ -618,7 +659,7 @@ class RepoFileManager: total_size += f.stat().st_size result.success = True result.file_size = total_size - logger.info(f"sparse-checkout 克隆成功: {target_dir}") + logger.info(f"sparse-checkout 克隆成功: {target_dir}:{aliyun_repo_url}") return result except GitUnavailableError as e: logger.error(f"Git不可用: {e}") diff --git a/zhenxun/utils/repo_utils/utils.py b/zhenxun/utils/repo_utils/utils.py index 62cb1cec..b54a83a2 100644 --- a/zhenxun/utils/repo_utils/utils.py +++ b/zhenxun/utils/repo_utils/utils.py @@ -227,11 +227,12 @@ async def sparse_checkout_clone( shutil.move(str(source_path), str(target_path)) -def prepare_aliyun_url(repo_url: str) -> str: +def prepare_aliyun_url(repo_url: str, group_name: str | None = None) -> str: """解析阿里云CodeUp的仓库URL 参数: repo_url: 仓库URL + group_name: 分组名称，如果为None则使用默认组织名称 返回: str: 解析后的仓库URL @@ -239,10 +240,12 @@ def prepare_aliyun_url(repo_url: str) -> str: config = RepoConfig.get_instance() repo_name = repo_url.split("/tree/")[0].split("/")[-1].replace(".git", "") + # 使用指定的分组名或默认组织名称 + group = group_name or config.aliyun_codeup.organization_name # 构建仓库URL # 阿里云CodeUp的仓库URL格式通常为： - # https://codeup.aliyun.com/{organization_id}/{organization_name}/{repo_name}.git - url = f"https://codeup.aliyun.com/{config.aliyun_codeup.organization_id}/{config.aliyun_codeup.organization_name}/{repo_name}.git" + # https://codeup.aliyun.com/{organization_id}/{group_name}/{repo_name}.git + url = f"https://codeup.aliyun.com/{config.aliyun_codeup.organization_id}/{group}/{repo_name}.git" # 添加访问令牌 - 使用base64解码后的令牌 if config.aliyun_codeup.rdc_access_token_encrypted: @@ -258,3 +261,35 @@ def prepare_aliyun_url(repo_url: str) -> str: logger.error(f"解码RDC令牌失败: {e}") return url + + +async def get_aliyun_group_for_repo(repo_name: str) -> str | None: + """获取仓库所属的阿里云分组名 + + 参数: + repo_name: 仓库名称 + + 返回: + str | None: 分组名称，如果在核心映射中则返回None（使用默认组织名） + """ + from zhenxun.utils.github_utils.const import ( + ALIYUN_EXTERNAL_PLUGIN_GROUPS, + ALIYUN_REPO_MAPPING, + ) + from zhenxun.utils.github_utils.models import AliyunFileInfo + + # 如果在核心映射中，使用默认组织名 + if repo_name in ALIYUN_REPO_MAPPING: + return None + + # 尝试从外部插件分组中查找 + for group_path in ALIYUN_EXTERNAL_PLUGIN_GROUPS: + try: + repos = await AliyunFileInfo.list_group_repositories(group_path) + for repo in repos: + if repo.get("name") == repo_name: + return group_path + except Exception: + continue + + return None