Diuse/zhenxun_bot
1
0
Fork 0
mirror of https://github.com/zhenxun-org/zhenxun_bot.git synced 2026-10-07 04:40:00 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Bugfix/fix aliyun (#2036)

Browse Source 
* 🐛 修复数据库超时问题

* 🔧 移除帮助图片清理功能.

* ✨ 更新插件商店功能，允许在添加插件时指定源类型为 None。优化插件 ID 查找逻辑，增强代码可读性。新增 zhenxun/ui 模块导入。

* 🔧 优化数据访问和数据库上下文逻辑，移除不必要的全局变量和日志信息，调整日志级别为调试，提升代码可读性和性能。

* ✨ 增强插件商店功能，支持在下载文件时指定稀疏检出路径和目标目录。优化二进制文件处理逻辑，提升文件下载的准确性和效率。

* ✨ 增强阿里云和GitHub的文件管理功能，新增Git不可用异常处理，优化稀疏检出逻辑，提升代码可读性和稳定性。

* ✨ 增强插件下载功能，新增对下载结果的异常处理，确保在Git不可用时抛出相应异常信息。优化错误提示，提升用户体验。

* ✨ 增强插件商店功能，优化添加插件时的提示信息，明确区分插件模块和名称。新增 Windows 下删除只读文件的处理逻辑，提升插件管理的稳定性和用户体验。

* ✨ 优化文件内容获取逻辑，新增对非二进制文件的UTF-8解码处理，提升文件读取的稳定性和准确性。
This commit is contained in:
HibiKier
2025-08-29 14:57:08 +08:00
committed by GitHub
parent 7472cabd48
commit 1e2aa99207
7 changed files with 313 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
zhenxun/utils/repo_utils/aliyun_manager.py
+1 -26
View File
@@ -11,6 +11,7 @@ from aiocache import cached
from zhenxun.services.log import logger
from zhenxun.utils.github_utils.models import AliyunFileInfo
from zhenxun.utils.repo_utils.utils import prepare_aliyun_url
from .base_manager import BaseRepoManager
from .config import LOG_COMMAND, RepoConfig
@@ -445,32 +446,6 @@ class AliyunCodeupManager(BaseRepoManager):
返回:
RepoUpdateResult: 更新结果
"""
# 定义预处理函数，构建阿里云CodeUp的URL
def prepare_aliyun_url(repo_url: str) -> str:
import base64
repo_name = repo_url.split("/tree/")[0].split("/")[-1].replace(".git", "")
# 构建仓库URL
# 阿里云CodeUp的仓库URL格式通常为：
# https://codeup.aliyun.com/{organization_id}/{organization_name}/{repo_name}.git
url = f"https://codeup.aliyun.com/{self.config.aliyun_codeup.organization_id}/{self.config.aliyun_codeup.organization_name}/{repo_name}.git"
# 添加访问令牌 - 使用base64解码后的令牌
if self.config.aliyun_codeup.rdc_access_token_encrypted:
try:
# 解码RDC访问令牌
token = base64.b64decode(
self.config.aliyun_codeup.rdc_access_token_encrypted.encode()
).decode()
# 阿里云CodeUp使用oauth2:token的格式进行身份验证
url = url.replace("https://", f"https://oauth2:{token}@")
logger.debug(f"使用RDC令牌构建阿里云URL: {url.split('@')[0]}@***")
except Exception as e:
logger.error(f"解码RDC令牌失败: {e}")
return url
# 调用基类的update_via_git方法
return await super().update_via_git(
repo_url=repo_url,
zhenxun/utils/repo_utils/exceptions.py
+7
View File
@@ -66,3 +66,10 @@ class ConfigError(RepoManagerError):
def __init__(self, message: str):
super().__init__(f"配置错误: {message}")
class GitUnavailableError(RepoManagerError):
"""Git不可用异常"""
def __init__(self, message: str = "Git命令不可用"):
super().__init__(message)
zhenxun/utils/repo_utils/file_manager.py
+111 -23
View File
@@ -2,6 +2,7 @@
仓库文件管理器，用于从GitHub和阿里云CodeUp获取指定文件内容
"""
import contextlib
from pathlib import Path
from typing import cast, overload
@@ -12,10 +13,17 @@ from zhenxun.services.log import logger
from zhenxun.utils.github_utils import GithubUtils
from zhenxun.utils.github_utils.models import AliyunTreeType, GitHubStrategy, TreeType
from zhenxun.utils.http_utils import AsyncHttpx
from zhenxun.utils.utils import is_binary_file
from .config import LOG_COMMAND, RepoConfig
from .exceptions import FileNotFoundError, NetworkError, RepoManagerError
from .exceptions import (
FileNotFoundError,
GitUnavailableError,
NetworkError,
RepoManagerError,
)
from .models import FileDownloadResult, RepoFileInfo, RepoType
from .utils import prepare_aliyun_url, sparse_checkout_clone
class RepoFileManager:
@@ -74,18 +82,20 @@ class RepoFileManager:
)
if response.status_code == 200:
logger.info(f"获取github文件内容成功: {f}", LOG_COMMAND)
text_content = response.content
# 确保使用UTF-8编码解析响应内容
try:
text_content = response.content.decode("utf-8")
except UnicodeDecodeError:
# 如果UTF-8解码失败，尝试其他编码
text_content = response.content.decode(
"utf-8", errors="ignore"
)
logger.warning(
f"解码文件内容时出现错误，使用忽略错误模式: {f}",
LOG_COMMAND,
)
if not is_binary_file(f):
try:
text_content = response.content.decode("utf-8")
except UnicodeDecodeError:
# 如果UTF-8解码失败，尝试其他编码
text_content = response.content.decode(
"utf-8", errors="ignore"
)
logger.warning(
f"解码文件内容时出现错误，使用忽略错误模式:{f}",
LOG_COMMAND,
)
results.append((f, text_content))
break
else:
@@ -262,15 +272,15 @@ class RepoFileManager:
if repo_type is None:
# 尝试GitHub，失败则尝试阿里云
try:
return await self._list_github_directory_files(
repo_url, directory_path, branch, recursive
return await self._list_aliyun_directory_files(
repo_name, directory_path, branch, recursive
)
except Exception as e:
logger.warning(
"获取GitHub目录文件失败，尝试阿里云", LOG_COMMAND, e=e
"获取阿里云目录文件失败，尝试GitHub", LOG_COMMAND, e=e
)
return await self._list_aliyun_directory_files(
repo_name, directory_path, branch, recursive
return await self._list_github_directory_files(
repo_url, directory_path, branch, recursive
)
if repo_type == RepoType.GITHUB:
return await self._list_github_directory_files(
@@ -466,6 +476,8 @@ class RepoFileManager:
branch: str = "main",
repo_type: RepoType | None = None,
ignore_error: bool = False,
sparse_path: str | None = None,
target_dir: Path | None = None,
) -> FileDownloadResult:
"""
下载单个文件
@@ -476,10 +488,19 @@ class RepoFileManager:
branch: 分支名称
repo_type: 仓库类型，如果为None则自动判断
ignore_error: 是否忽略错误
sparse_path: 稀疏检出路径
target_dir: 稀疏目标目录
返回:
FileDownloadResult: 下载结果
"""
# 参数一致性校验：sparse_path 与 target_dir 必须同时有值或同时为 None
if (sparse_path is None) ^ (target_dir is None):
raise RepoManagerError(
"参数错误: sparse_path 与 target_dir 必须同时提供或同时为 None"
)
# 确定仓库类型和所有者
repo_name = (
repo_url.split("/tree/")[0].split("/")[-1].replace(".git", "").strip()
@@ -497,12 +518,43 @@ class RepoFileManager:
file_path=file_path,
version=branch,
)
if (
any(is_binary_file(file_name) for file_name in file_path_mapping)
and repo_type != RepoType.GITHUB
and sparse_path
and target_dir
):
return await self._handle_binary_with_sparse_checkout(
repo_url=repo_url,
branch=branch,
sparse_path=sparse_path,
target_dir=target_dir,
result=result,
)
else:
# 不包含二进制时
return await self._download_and_write_files(
repo_url=repo_url,
file_paths=[f[0] for f in file_path],
file_path_mapping=file_path_mapping,
branch=branch,
repo_type=repo_type,
ignore_error=ignore_error,
result=result,
)
async def _download_and_write_files(
self,
repo_url: str,
file_paths: list[str],
file_path_mapping: dict[str, Path],
branch: str,
repo_type: RepoType | None,
ignore_error: bool,
result: FileDownloadResult,
) -> FileDownloadResult:
try:
# 由于我们传入的是列表，所以这里一定返回列表
file_paths = [f[0] for f in file_path]
if len(file_paths) == 1:
# 如果只有一个文件，可能返回单个元组
file_contents_result = await self.get_file_content(
repo_url, file_paths[0], branch, repo_type, ignore_error
)
@@ -513,7 +565,6 @@ class RepoFileManager:
else:
file_contents = cast(list[tuple[str, str]], file_contents_result)
else:
# 多个文件一定返回列表
file_contents = cast(
list[tuple[str, str]],
await self.get_file_content(
@@ -524,7 +575,6 @@ class RepoFileManager:
for repo_file_path, content in file_contents:
local_path = file_path_mapping[repo_file_path]
local_path.parent.mkdir(parents=True, exist_ok=True)
# 使用二进制模式写入文件，避免编码问题
if isinstance(content, str):
content_bytes = content.encode("utf-8")
else:
@@ -533,7 +583,6 @@ class RepoFileManager:
async with aiofiles.open(local_path, "wb") as f:
await f.write(content_bytes)
result.success = True
# 计算文件大小
result.file_size = sum(
len(content.encode("utf-8") if isinstance(content, str) else content)
for _, content in file_contents
@@ -545,3 +594,42 @@ class RepoFileManager:
result.success = False
result.error_message = str(e)
return result
async def _handle_binary_with_sparse_checkout(
self,
repo_url: str,
branch: str,
sparse_path: str,
target_dir: Path,
result: FileDownloadResult,
) -> FileDownloadResult:
try:
await sparse_checkout_clone(
repo_url=prepare_aliyun_url(repo_url),
branch=branch,
sparse_path=sparse_path,
target_dir=target_dir,
)
total_size = 0
if target_dir.exists():
for f in target_dir.rglob("*"):
if f.is_file():
with contextlib.suppress(Exception):
total_size += f.stat().st_size
result.success = True
result.file_size = total_size
logger.info(f"sparse-checkout 克隆成功: {target_dir}")
return result
except GitUnavailableError as e:
logger.error(f"Git不可用: {e}")
result.success = False
result.error_message = (
"当前插件包含二进制文件，因ali限制需要使用git，"
"当前Git不可用，请尝试添加参数 -s git 或 安装 git"
)
return result
except Exception as e:
logger.error(f"sparse-checkout 克隆失败: {e}")
result.success = False
result.error_message = str(e)
return result
zhenxun/utils/repo_utils/utils.py
+120 -1
View File
@@ -3,12 +3,15 @@
"""
import asyncio
import base64
from pathlib import Path
import re
import shutil
from zhenxun.services.log import logger
from .config import LOG_COMMAND
from .config import LOG_COMMAND, RepoConfig
from .exceptions import GitUnavailableError
async def check_git() -> bool:
@@ -133,3 +136,119 @@ def filter_files(
result = [file for file in result if not re.match(regex_pattern, file)]
return result
async def sparse_checkout_clone(
repo_url: str,
branch: str,
sparse_path: str,
target_dir: Path,
) -> None:
"""
使用 git 稀疏检出克隆指定路径到目标目录（完全独立于主项目 git）。
关键保障:
- 在 target_dir 下检测/初始化 .git，所有 git 操作均以 cwd=target_dir 执行
- 强制拉取与工作区覆盖: fetch --force、checkout -B、reset --hard、clean -xdf
- 反复设置 sparse-checkout 路径，确保路径更新生效
"""
target_dir.mkdir(parents=True, exist_ok=True)
if not await check_git():
raise GitUnavailableError()
git_dir = target_dir / ".git"
if not git_dir.exists():
success, out, err = await run_git_command("init", target_dir)
if not success:
raise RuntimeError(f"git init 失败: {err or out}")
success, out, err = await run_git_command(
f"remote add origin {repo_url}", target_dir
)
if not success:
raise RuntimeError(f"添加远程失败: {err or out}")
else:
success, out, err = await run_git_command(
f"remote set-url origin {repo_url}", target_dir
)
if not success:
# 兜底尝试添加
await run_git_command(f"remote add origin {repo_url}", target_dir)
# 启用稀疏检出（使用 --no-cone 模式以获得更精确的控制）
await run_git_command("config core.sparseCheckout true", target_dir)
await run_git_command("sparse-checkout init --no-cone", target_dir)
# 设置需要检出的路径（每次都覆盖配置）
if not sparse_path:
raise RuntimeError("sparse-checkout 路径不能为空")
# 使用 --no-cone 模式，直接指定要检出的具体路径
# 例如：sparse_path="plugins/mahiro" -> 只检出 plugins/mahiro/ 下的内容
success, out, err = await run_git_command(
f"sparse-checkout set {sparse_path}/", target_dir
)
if not success:
raise RuntimeError(f"配置稀疏路径失败: {err or out}")
# 强制拉取并同步到远端
success, out, err = await run_git_command(
f"fetch --force --depth 1 origin {branch}", target_dir
)
if not success:
raise RuntimeError(f"fetch 失败: {err or out}")
# 使用远端强制更新本地分支并覆盖工作区
success, out, err = await run_git_command(
f"checkout -B {branch} origin/{branch}", target_dir
)
if not success:
# 回退方案
success2, out2, err2 = await run_git_command(f"checkout {branch}", target_dir)
if not success2:
raise RuntimeError(f"checkout 失败: {(err or out) or (err2 or out2)}")
# 强制对齐工作区
await run_git_command(f"reset --hard origin/{branch}", target_dir)
await run_git_command("clean -xdf", target_dir)
dir_path = target_dir / Path(sparse_path)
for f in dir_path.iterdir():
shutil.move(f, target_dir / f.name)
dir_name = sparse_path.split("/")[0]
rm_path = target_dir / dir_name
if rm_path.exists():
shutil.rmtree(rm_path)
def prepare_aliyun_url(repo_url: str) -> str:
"""解析阿里云CodeUp的仓库URL
参数:
repo_url: 仓库URL
返回:
str: 解析后的仓库URL
"""
config = RepoConfig.get_instance()
repo_name = repo_url.split("/tree/")[0].split("/")[-1].replace(".git", "")
# 构建仓库URL
# 阿里云CodeUp的仓库URL格式通常为：
# https://codeup.aliyun.com/{organization_id}/{organization_name}/{repo_name}.git
url = f"https://codeup.aliyun.com/{config.aliyun_codeup.organization_id}/{config.aliyun_codeup.organization_name}/{repo_name}.git"
# 添加访问令牌 - 使用base64解码后的令牌
if config.aliyun_codeup.rdc_access_token_encrypted:
try:
# 解码RDC访问令牌
token = base64.b64decode(
config.aliyun_codeup.rdc_access_token_encrypted.encode()
).decode()
# 阿里云CodeUp使用oauth2:token的格式进行身份验证
url = url.replace("https://", f"https://oauth2:{token}@")
logger.debug(f"使用RDC令牌构建阿里云URL: {url.split('@')[0]}@***")
except Exception as e:
logger.error(f"解码RDC令牌失败: {e}")
return url