mirror of
https://github.com/zhenxun-org/zhenxun_bot.git
synced 2026-09-28 16:20:56 +08:00
fix(plugin-store): 优化下载逻辑及回退 (#2153)
* feat: 添加插件下载源管理与二进制文件处理逻辑 * fix(test): 修复强制阿里云测试中的参数断言,确保 repo_type 正确 --------- Co-authored-by: ATTomatoo <1126160939@qq.com>
This commit is contained in:
co-authored by
ATTomatoo
parent
9f202666aa
commit
e5fa0f0335
@@ -0,0 +1,437 @@
|
|||||||
|
import asyncio
|
||||||
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from pytest_mock import MockerFixture
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_git(*args: str) -> None:
|
||||||
|
process = await asyncio.create_subprocess_exec(
|
||||||
|
"git",
|
||||||
|
*args,
|
||||||
|
stdout=asyncio.subprocess.PIPE,
|
||||||
|
stderr=asyncio.subprocess.PIPE,
|
||||||
|
)
|
||||||
|
_, stderr = await process.communicate()
|
||||||
|
assert process.returncode == 0, stderr.decode(errors="replace")
|
||||||
|
|
||||||
|
|
||||||
|
def _plugin_info(*, ali_url: str | None = None):
|
||||||
|
from zhenxun.builtin_plugins.plugin_store.models import StorePluginInfo
|
||||||
|
from zhenxun.utils.enum import PluginType
|
||||||
|
|
||||||
|
return StorePluginInfo(
|
||||||
|
name="测试插件",
|
||||||
|
module="demo",
|
||||||
|
module_path="demo",
|
||||||
|
description="",
|
||||||
|
usage="",
|
||||||
|
author="tester",
|
||||||
|
version="1.0.0",
|
||||||
|
plugin_type=PluginType.NORMAL,
|
||||||
|
is_dir=True,
|
||||||
|
github_url="https://github.com/example/demo",
|
||||||
|
ali_url=ali_url,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_order() -> None:
|
||||||
|
from zhenxun.builtin_plugins.plugin_store.data_source import StoreManager
|
||||||
|
from zhenxun.utils.repo_utils.models import RepoType
|
||||||
|
|
||||||
|
assert StoreManager._get_source_order(None) == (
|
||||||
|
RepoType.ALIYUN,
|
||||||
|
RepoType.GITHUB,
|
||||||
|
)
|
||||||
|
assert StoreManager._get_source_order("ali") == (RepoType.ALIYUN,)
|
||||||
|
assert StoreManager._get_source_order("git") == (RepoType.GITHUB,)
|
||||||
|
|
||||||
|
|
||||||
|
def test_repository_branch_is_resolved_per_source() -> None:
|
||||||
|
from zhenxun.builtin_plugins.plugin_store.data_source import StoreManager
|
||||||
|
from zhenxun.utils.repo_utils.models import RepoType
|
||||||
|
|
||||||
|
plugin_info = _plugin_info()
|
||||||
|
plugin_info.github_url = "https://github.com/example/demo/tree/master"
|
||||||
|
|
||||||
|
assert (
|
||||||
|
StoreManager._get_plugin_repository_branch(plugin_info, RepoType.ALIYUN, "main")
|
||||||
|
== "main"
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
StoreManager._get_plugin_repository_branch(plugin_info, RepoType.GITHUB, "main")
|
||||||
|
== "master"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("is_external", [False, True])
|
||||||
|
async def test_default_source_falls_back_to_github_for_all_plugins(
|
||||||
|
mocker: MockerFixture,
|
||||||
|
tmp_path: Path,
|
||||||
|
is_external: bool,
|
||||||
|
) -> None:
|
||||||
|
from zhenxun.builtin_plugins.plugin_store.data_source import StoreManager
|
||||||
|
from zhenxun.utils.repo_utils.models import (
|
||||||
|
FileDownloadResult,
|
||||||
|
RepoFileInfo,
|
||||||
|
RepoType,
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_base_path = mocker.patch(
|
||||||
|
"zhenxun.builtin_plugins.plugin_store.data_source.BASE_PATH",
|
||||||
|
new=tmp_path / "zhenxun",
|
||||||
|
)
|
||||||
|
source_calls: list[RepoType] = []
|
||||||
|
download_calls: list[tuple[RepoType, list[str]]] = []
|
||||||
|
files = [
|
||||||
|
RepoFileInfo(path="demo/__init__.py", is_dir=False),
|
||||||
|
RepoFileInfo(path="demo/assets/icon.png", is_dir=False),
|
||||||
|
RepoFileInfo(path="demo/requirements.txt", is_dir=False),
|
||||||
|
]
|
||||||
|
|
||||||
|
async def list_directory_files(
|
||||||
|
repo_url: str,
|
||||||
|
directory_path: str,
|
||||||
|
branch: str,
|
||||||
|
repo_type: RepoType,
|
||||||
|
) -> list[RepoFileInfo]:
|
||||||
|
source_calls.append(repo_type)
|
||||||
|
if repo_type == RepoType.ALIYUN:
|
||||||
|
raise RuntimeError("aliyun unavailable")
|
||||||
|
return files
|
||||||
|
|
||||||
|
async def download_files(
|
||||||
|
repo_url: str,
|
||||||
|
file_path: list[tuple[str, Path]],
|
||||||
|
branch: str,
|
||||||
|
repo_type: RepoType,
|
||||||
|
ignore_error: bool = False,
|
||||||
|
) -> FileDownloadResult:
|
||||||
|
download_calls.append((repo_type, [path for path, _ in file_path]))
|
||||||
|
for source_path, destination_path in file_path:
|
||||||
|
destination_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
if source_path.endswith(".png"):
|
||||||
|
destination_path.write_bytes(b"\x89PNG\r\n\x1a\n")
|
||||||
|
else:
|
||||||
|
destination_path.write_text(source_path, encoding="utf-8")
|
||||||
|
return FileDownloadResult(
|
||||||
|
repo_type=repo_type,
|
||||||
|
repo_name="demo",
|
||||||
|
file_path=file_path,
|
||||||
|
version=branch,
|
||||||
|
success=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
mocker.patch(
|
||||||
|
"zhenxun.builtin_plugins.plugin_store.data_source."
|
||||||
|
"RepoFileManager.list_directory_files",
|
||||||
|
side_effect=list_directory_files,
|
||||||
|
)
|
||||||
|
mocker.patch(
|
||||||
|
"zhenxun.builtin_plugins.plugin_store.data_source."
|
||||||
|
"RepoFileManager.download_files",
|
||||||
|
side_effect=download_files,
|
||||||
|
)
|
||||||
|
install_requirement = mocker.patch(
|
||||||
|
"zhenxun.builtin_plugins.plugin_store.data_source."
|
||||||
|
"VirtualEnvPackageManager.install_requirement",
|
||||||
|
)
|
||||||
|
|
||||||
|
await StoreManager.install_plugin_with_repo(
|
||||||
|
_plugin_info(),
|
||||||
|
is_external=is_external,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert source_calls == [RepoType.ALIYUN, RepoType.GITHUB]
|
||||||
|
assert download_calls == [
|
||||||
|
(
|
||||||
|
RepoType.GITHUB,
|
||||||
|
[
|
||||||
|
"demo/__init__.py",
|
||||||
|
"demo/assets/icon.png",
|
||||||
|
"demo/requirements.txt",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
]
|
||||||
|
assert (
|
||||||
|
mock_base_path / "plugins" / "demo" / "assets" / "icon.png"
|
||||||
|
).read_bytes() == b"\x89PNG\r\n\x1a\n"
|
||||||
|
install_requirement.assert_awaited_once()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_zero_byte_aliyun_binary_falls_back_to_github(
|
||||||
|
mocker: MockerFixture,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
from zhenxun.builtin_plugins.plugin_store.data_source import StoreManager
|
||||||
|
from zhenxun.utils.repo_utils.models import (
|
||||||
|
FileDownloadResult,
|
||||||
|
RepoFileInfo,
|
||||||
|
RepoType,
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_base_path = mocker.patch(
|
||||||
|
"zhenxun.builtin_plugins.plugin_store.data_source.BASE_PATH",
|
||||||
|
new=tmp_path / "zhenxun",
|
||||||
|
)
|
||||||
|
calls: list[RepoType] = []
|
||||||
|
files = [
|
||||||
|
RepoFileInfo(path="demo/__init__.py", is_dir=False),
|
||||||
|
RepoFileInfo(path="demo/icon.png", is_dir=False),
|
||||||
|
]
|
||||||
|
|
||||||
|
mocker.patch(
|
||||||
|
"zhenxun.builtin_plugins.plugin_store.data_source."
|
||||||
|
"RepoFileManager.list_directory_files",
|
||||||
|
return_value=files,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def download_files(
|
||||||
|
repo_url: str,
|
||||||
|
file_path: list[tuple[str, Path]],
|
||||||
|
branch: str,
|
||||||
|
repo_type: RepoType,
|
||||||
|
ignore_error: bool = False,
|
||||||
|
) -> FileDownloadResult:
|
||||||
|
calls.append(repo_type)
|
||||||
|
for source_path, destination_path in file_path:
|
||||||
|
destination_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
if source_path.endswith(".png"):
|
||||||
|
content = b"" if repo_type == RepoType.ALIYUN else b"image"
|
||||||
|
destination_path.write_bytes(content)
|
||||||
|
else:
|
||||||
|
destination_path.write_text("", encoding="utf-8")
|
||||||
|
return FileDownloadResult(
|
||||||
|
repo_type=repo_type,
|
||||||
|
repo_name="demo",
|
||||||
|
file_path=file_path,
|
||||||
|
version=branch,
|
||||||
|
success=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
mocker.patch(
|
||||||
|
"zhenxun.builtin_plugins.plugin_store.data_source."
|
||||||
|
"RepoFileManager.download_files",
|
||||||
|
side_effect=download_files,
|
||||||
|
)
|
||||||
|
mocker.patch(
|
||||||
|
"zhenxun.builtin_plugins.plugin_store.data_source."
|
||||||
|
"VirtualEnvPackageManager.install_requirement",
|
||||||
|
)
|
||||||
|
|
||||||
|
await StoreManager.install_plugin_with_repo(_plugin_info(), is_external=True)
|
||||||
|
|
||||||
|
assert calls[:2] == [RepoType.ALIYUN, RepoType.GITHUB]
|
||||||
|
assert (mock_base_path / "plugins" / "demo" / "icon.png").read_bytes() == b"image"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_forced_aliyun_does_not_fall_back(
|
||||||
|
mocker: MockerFixture,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
from zhenxun.builtin_plugins.plugin_store.data_source import StoreManager
|
||||||
|
from zhenxun.builtin_plugins.plugin_store.exceptions import PluginStoreException
|
||||||
|
from zhenxun.utils.repo_utils.models import RepoType
|
||||||
|
|
||||||
|
mocker.patch(
|
||||||
|
"zhenxun.builtin_plugins.plugin_store.data_source.BASE_PATH",
|
||||||
|
new=tmp_path / "zhenxun",
|
||||||
|
)
|
||||||
|
list_directory_files = mocker.patch(
|
||||||
|
"zhenxun.builtin_plugins.plugin_store.data_source."
|
||||||
|
"RepoFileManager.list_directory_files",
|
||||||
|
side_effect=RuntimeError("aliyun unavailable"),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(PluginStoreException, match="阿里云"):
|
||||||
|
await StoreManager.install_plugin_with_repo(
|
||||||
|
_plugin_info(),
|
||||||
|
source="ali",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert list_directory_files.await_count == 1
|
||||||
|
await_args = list_directory_files.await_args
|
||||||
|
assert await_args is not None
|
||||||
|
assert await_args.kwargs["repo_type"] == RepoType.ALIYUN
|
||||||
|
|
||||||
|
|
||||||
|
async def test_root_plugin_uses_exact_sparse_paths(
|
||||||
|
mocker: MockerFixture,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
from zhenxun.builtin_plugins.plugin_store.data_source import StoreManager
|
||||||
|
from zhenxun.utils.repo_utils.models import (
|
||||||
|
FileDownloadResult,
|
||||||
|
RepoFileInfo,
|
||||||
|
RepoType,
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_base_path = mocker.patch(
|
||||||
|
"zhenxun.builtin_plugins.plugin_store.data_source.BASE_PATH",
|
||||||
|
new=tmp_path / "zhenxun",
|
||||||
|
)
|
||||||
|
plugin_info = _plugin_info(
|
||||||
|
ali_url="https://codeup.aliyun.com/organization/group/demo-mirror"
|
||||||
|
)
|
||||||
|
plugin_info.module_path = "."
|
||||||
|
files = [
|
||||||
|
RepoFileInfo(path="__init__.py", is_dir=False),
|
||||||
|
RepoFileInfo(path="assets/icon.png", is_dir=False),
|
||||||
|
RepoFileInfo(path="requirements.txt", is_dir=False),
|
||||||
|
]
|
||||||
|
mocker.patch(
|
||||||
|
"zhenxun.builtin_plugins.plugin_store.data_source."
|
||||||
|
"RepoFileManager.list_directory_files",
|
||||||
|
return_value=files,
|
||||||
|
)
|
||||||
|
downloaded_paths: list[str] = []
|
||||||
|
|
||||||
|
async def download_files(
|
||||||
|
repo_url: str,
|
||||||
|
file_path: list[tuple[str, Path]],
|
||||||
|
branch: str,
|
||||||
|
repo_type: RepoType,
|
||||||
|
ignore_error: bool = False,
|
||||||
|
) -> FileDownloadResult:
|
||||||
|
downloaded_paths.extend(path for path, _ in file_path)
|
||||||
|
for source_path, destination_path in file_path:
|
||||||
|
destination_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
content = b"image" if source_path.endswith(".png") else b""
|
||||||
|
destination_path.write_bytes(content)
|
||||||
|
return FileDownloadResult(
|
||||||
|
repo_type=repo_type,
|
||||||
|
repo_name="demo-mirror",
|
||||||
|
file_path=file_path,
|
||||||
|
version=branch,
|
||||||
|
success=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
mocker.patch(
|
||||||
|
"zhenxun.builtin_plugins.plugin_store.data_source."
|
||||||
|
"RepoFileManager.download_files",
|
||||||
|
side_effect=download_files,
|
||||||
|
)
|
||||||
|
mocker.patch(
|
||||||
|
"zhenxun.builtin_plugins.plugin_store.data_source."
|
||||||
|
"VirtualEnvPackageManager.install_requirement",
|
||||||
|
)
|
||||||
|
|
||||||
|
await StoreManager.install_plugin_with_repo(plugin_info, source="ali")
|
||||||
|
|
||||||
|
assert downloaded_paths == [
|
||||||
|
"__init__.py",
|
||||||
|
"assets/icon.png",
|
||||||
|
"requirements.txt",
|
||||||
|
]
|
||||||
|
assert (
|
||||||
|
mock_base_path / "plugins" / "demo" / "assets" / "icon.png"
|
||||||
|
).read_bytes() == b"image"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_repo_manager_sparse_checkout_preserves_exact_paths(
|
||||||
|
mocker: MockerFixture,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
from zhenxun.utils.repo_utils import RepoFileManager
|
||||||
|
from zhenxun.utils.repo_utils.models import RepoType
|
||||||
|
|
||||||
|
async def sparse_checkout_clone(
|
||||||
|
repo_url: str,
|
||||||
|
branch: str,
|
||||||
|
sparse_path: list[str],
|
||||||
|
target_dir: Path,
|
||||||
|
) -> list[str]:
|
||||||
|
assert repo_url == "https://github.com/example/demo.git"
|
||||||
|
assert branch == "master"
|
||||||
|
assert sparse_path == ["demo/assets/icon.png"]
|
||||||
|
source = target_dir / sparse_path[0]
|
||||||
|
source.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
source.write_bytes(b"image")
|
||||||
|
return sparse_path
|
||||||
|
|
||||||
|
sparse_checkout = mocker.patch(
|
||||||
|
"zhenxun.utils.repo_utils.file_manager.sparse_checkout_clone",
|
||||||
|
side_effect=sparse_checkout_clone,
|
||||||
|
)
|
||||||
|
target = tmp_path / "target" / "icon.png"
|
||||||
|
result = await RepoFileManager.download_files(
|
||||||
|
"https://github.com/example/demo/tree/master",
|
||||||
|
[("demo/assets/icon.png", target)],
|
||||||
|
"master",
|
||||||
|
repo_type=RepoType.GITHUB,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.success
|
||||||
|
assert target.read_bytes() == b"image"
|
||||||
|
sparse_checkout.assert_awaited_once()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_sparse_checkout_retries_git_fetch(
|
||||||
|
mocker: MockerFixture,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
from zhenxun.utils.repo_utils.utils import sparse_checkout_clone
|
||||||
|
|
||||||
|
mocker.patch("zhenxun.utils.repo_utils.utils.check_git", return_value=True)
|
||||||
|
sleep = mocker.patch("zhenxun.utils.repo_utils.utils.asyncio.sleep")
|
||||||
|
fetch_attempts = 0
|
||||||
|
fetch_timeouts: list[float | None] = []
|
||||||
|
|
||||||
|
async def run_git_command(
|
||||||
|
command: str | list[str],
|
||||||
|
cwd: Path | None = None,
|
||||||
|
timeout_seconds: float | None = None,
|
||||||
|
) -> tuple[bool, str, str]:
|
||||||
|
nonlocal fetch_attempts
|
||||||
|
if isinstance(command, list) and "fetch" in command:
|
||||||
|
fetch_attempts += 1
|
||||||
|
fetch_timeouts.append(timeout_seconds)
|
||||||
|
if fetch_attempts < 3:
|
||||||
|
return False, "", "connection reset"
|
||||||
|
return True, "", ""
|
||||||
|
|
||||||
|
mocker.patch(
|
||||||
|
"zhenxun.utils.repo_utils.utils.run_git_command",
|
||||||
|
side_effect=run_git_command,
|
||||||
|
)
|
||||||
|
|
||||||
|
downloaded = await sparse_checkout_clone(
|
||||||
|
repo_url="https://github.com/example/demo",
|
||||||
|
branch="main",
|
||||||
|
sparse_path=["demo/__init__.py"],
|
||||||
|
target_dir=tmp_path / "target",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert downloaded == []
|
||||||
|
assert fetch_attempts == 3
|
||||||
|
assert fetch_timeouts == [60, 60, 60]
|
||||||
|
assert sleep.await_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(shutil.which("git") is None, reason="git is not installed")
|
||||||
|
async def test_git_checkout_preserves_binary(tmp_path: Path) -> None:
|
||||||
|
from zhenxun.utils.repo_utils.utils import sparse_checkout_clone
|
||||||
|
|
||||||
|
source_repo = tmp_path / "source"
|
||||||
|
source_repo.mkdir()
|
||||||
|
await _run_git("init", "-b", "main", str(source_repo))
|
||||||
|
await _run_git("-C", str(source_repo), "config", "user.name", "test")
|
||||||
|
await _run_git("-C", str(source_repo), "config", "user.email", "test@example.com")
|
||||||
|
binary_content = b"\x89PNG\r\n\x1a\n\x00\x01\xffbinary"
|
||||||
|
(source_repo / "icon.png").write_bytes(binary_content)
|
||||||
|
(source_repo / "__init__.py").write_text("", encoding="utf-8")
|
||||||
|
await _run_git("-C", str(source_repo), "add", ".")
|
||||||
|
await _run_git("-C", str(source_repo), "commit", "-m", "test")
|
||||||
|
|
||||||
|
target_dir = tmp_path / "target"
|
||||||
|
downloaded = await sparse_checkout_clone(
|
||||||
|
repo_url=source_repo.as_uri(),
|
||||||
|
branch="main",
|
||||||
|
sparse_path=["icon.png", "__init__.py"],
|
||||||
|
target_dir=target_dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert downloaded == ["icon.png", "__init__.py"]
|
||||||
|
assert (target_dir / "icon.png").read_bytes() == binary_content
|
||||||
|
assert not (target_dir / ".git").exists()
|
||||||
@@ -1,12 +1,12 @@
|
|||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import random
|
|
||||||
import shutil
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
from typing import ClassVar
|
||||||
|
|
||||||
import ujson as json
|
import ujson as json
|
||||||
|
|
||||||
from zhenxun.builtin_plugins.plugin_store.models import StorePluginInfo
|
from zhenxun.builtin_plugins.plugin_store.models import StorePluginInfo
|
||||||
from zhenxun.configs.path_config import TEMP_PATH
|
|
||||||
from zhenxun.models.plugin_info import PluginInfo
|
from zhenxun.models.plugin_info import PluginInfo
|
||||||
from zhenxun.services.cache.bounded_ttl import BoundedTTLCache
|
from zhenxun.services.cache.bounded_ttl import BoundedTTLCache
|
||||||
from zhenxun.services.log import logger
|
from zhenxun.services.log import logger
|
||||||
@@ -52,6 +52,58 @@ def row_style(column: str, text: str) -> RowStyle:
|
|||||||
|
|
||||||
|
|
||||||
class StoreManager:
|
class StoreManager:
|
||||||
|
_SOURCE_NAMES: ClassVar[dict[RepoType, str]] = {
|
||||||
|
RepoType.ALIYUN: "阿里云",
|
||||||
|
RepoType.GITHUB: "GitHub",
|
||||||
|
}
|
||||||
|
_BINARY_EXTENSIONS: ClassVar[frozenset[str]] = frozenset(
|
||||||
|
{
|
||||||
|
".7z",
|
||||||
|
".avi",
|
||||||
|
".bin",
|
||||||
|
".bmp",
|
||||||
|
".class",
|
||||||
|
".dat",
|
||||||
|
".db",
|
||||||
|
".dll",
|
||||||
|
".doc",
|
||||||
|
".docx",
|
||||||
|
".dylib",
|
||||||
|
".eot",
|
||||||
|
".exe",
|
||||||
|
".flv",
|
||||||
|
".gif",
|
||||||
|
".gz",
|
||||||
|
".ico",
|
||||||
|
".jpeg",
|
||||||
|
".jpg",
|
||||||
|
".mov",
|
||||||
|
".mp3",
|
||||||
|
".mp4",
|
||||||
|
".otf",
|
||||||
|
".pdf",
|
||||||
|
".png",
|
||||||
|
".ppt",
|
||||||
|
".pptx",
|
||||||
|
".pyc",
|
||||||
|
".rar",
|
||||||
|
".so",
|
||||||
|
".svg",
|
||||||
|
".tar",
|
||||||
|
".tif",
|
||||||
|
".tiff",
|
||||||
|
".ttf",
|
||||||
|
".webp",
|
||||||
|
".wmv",
|
||||||
|
".woff",
|
||||||
|
".woff2",
|
||||||
|
".xls",
|
||||||
|
".xlsx",
|
||||||
|
".xz",
|
||||||
|
".zip",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _resolve_local_plugin_path(
|
def _resolve_local_plugin_path(
|
||||||
cls, plugin_info: StorePluginInfo, *, is_external: bool
|
cls, plugin_info: StorePluginInfo, *, is_external: bool
|
||||||
@@ -333,87 +385,214 @@ class StoreManager:
|
|||||||
|
|
||||||
参数:
|
参数:
|
||||||
plugin_info: 插件信息
|
plugin_info: 插件信息
|
||||||
is_external: 是否是外部仓库
|
is_external: 是否是外部仓库(保留用于兼容旧调用)
|
||||||
source: 源
|
source: 强制使用的源,ali 为阿里云,git 为 GitHub;
|
||||||
|
不指定时优先阿里云,失败后回退 GitHub
|
||||||
"""
|
"""
|
||||||
repo_type = RepoType.GITHUB if is_external else None
|
source_order = cls._get_source_order(source)
|
||||||
if not is_external:
|
errors: list[str] = []
|
||||||
repo_type = RepoType.ALIYUN
|
|
||||||
elif (source is None and plugin_info.ali_url) or source == "ali":
|
with tempfile.TemporaryDirectory(prefix="zhenxun_plugin_store_") as temp_dir:
|
||||||
repo_type = RepoType.ALIYUN
|
staged_result: tuple[list[tuple[Path, Path]], list[Path]] | None = None
|
||||||
elif source == "git":
|
selected_source: RepoType | None = None
|
||||||
repo_type = RepoType.GITHUB
|
|
||||||
|
for repo_type in source_order:
|
||||||
|
source_name = cls._SOURCE_NAMES[repo_type]
|
||||||
|
staging_root = Path(temp_dir) / repo_type.value
|
||||||
|
try:
|
||||||
|
staged_result = await cls._download_plugin_to_staging(
|
||||||
|
plugin_info,
|
||||||
|
repo_type,
|
||||||
|
branch,
|
||||||
|
staging_root,
|
||||||
|
)
|
||||||
|
selected_source = repo_type
|
||||||
|
logger.info(
|
||||||
|
f"插件 {plugin_info.name} 使用{source_name}下载成功",
|
||||||
|
LOG_COMMAND,
|
||||||
|
)
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
errors.append(f"{source_name}: {e}")
|
||||||
|
if repo_type != source_order[-1]:
|
||||||
|
logger.warning(
|
||||||
|
f"插件 {plugin_info.name} 使用{source_name}下载失败,"
|
||||||
|
"尝试 GitHub",
|
||||||
|
LOG_COMMAND,
|
||||||
|
e=e,
|
||||||
|
)
|
||||||
|
|
||||||
|
if staged_result is None or selected_source is None:
|
||||||
|
raise PluginStoreException(
|
||||||
|
f"插件 {plugin_info.name} 下载失败({';'.join(errors)})"
|
||||||
|
)
|
||||||
|
|
||||||
|
deploy_files, requirement_files = staged_result
|
||||||
|
for requirement_file in requirement_files:
|
||||||
|
logger.info(
|
||||||
|
f"开始安装插件 {plugin_info.module_path} "
|
||||||
|
f"依赖文件: {requirement_file}",
|
||||||
|
LOG_COMMAND,
|
||||||
|
)
|
||||||
|
await VirtualEnvPackageManager.install_requirement(requirement_file)
|
||||||
|
|
||||||
|
for staged_path, destination_path in deploy_files:
|
||||||
|
destination_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copy2(staged_path, destination_path)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_source_order(source: str | None) -> tuple[RepoType, ...]:
|
||||||
|
"""解析插件下载源。"""
|
||||||
|
if source is None:
|
||||||
|
return (RepoType.ALIYUN, RepoType.GITHUB)
|
||||||
|
if source == "ali":
|
||||||
|
return (RepoType.ALIYUN,)
|
||||||
|
if source == "git":
|
||||||
|
return (RepoType.GITHUB,)
|
||||||
|
raise PluginStoreException(f"源类型错误: {source},请使用 ali 或 git")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_repository_url(plugin_info: StorePluginInfo, repo_type: RepoType) -> str:
|
||||||
|
"""获取指定下载源对应的仓库地址。"""
|
||||||
|
if repo_type == RepoType.ALIYUN and plugin_info.ali_url:
|
||||||
|
return plugin_info.ali_url
|
||||||
|
if plugin_info.github_url:
|
||||||
|
return plugin_info.github_url
|
||||||
|
raise PluginStoreException(f"插件 {plugin_info.name} 缺少仓库地址")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_repository_branch(repo_url: str | None, default_branch: str) -> str:
|
||||||
|
"""优先使用仓库 URL 中显式指定的分支、标签或提交。"""
|
||||||
|
if repo_url and "/tree/" in repo_url:
|
||||||
|
_, _, ref = repo_url.partition("/tree/")
|
||||||
|
if ref := ref.strip("/"):
|
||||||
|
return ref
|
||||||
|
return default_branch
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _get_plugin_repository_branch(
|
||||||
|
cls,
|
||||||
|
plugin_info: StorePluginInfo,
|
||||||
|
repo_type: RepoType,
|
||||||
|
default_branch: str,
|
||||||
|
) -> str:
|
||||||
|
"""按下载源独立解析分支,避免把 GitHub 分支套到阿里云镜像。"""
|
||||||
|
branch_source_url = (
|
||||||
|
plugin_info.ali_url
|
||||||
|
if repo_type == RepoType.ALIYUN
|
||||||
|
else plugin_info.github_url
|
||||||
|
)
|
||||||
|
return cls._get_repository_branch(branch_source_url, default_branch)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
async def _download_plugin_to_staging(
|
||||||
|
cls,
|
||||||
|
plugin_info: StorePluginInfo,
|
||||||
|
repo_type: RepoType,
|
||||||
|
default_branch: str,
|
||||||
|
staging_root: Path,
|
||||||
|
) -> tuple[list[tuple[Path, Path]], list[Path]]:
|
||||||
|
"""从单一仓库源完整下载插件到临时目录。"""
|
||||||
|
repo_url = cls._get_repository_url(plugin_info, repo_type)
|
||||||
|
branch = cls._get_plugin_repository_branch(
|
||||||
|
plugin_info,
|
||||||
|
repo_type,
|
||||||
|
default_branch,
|
||||||
|
)
|
||||||
module_path = plugin_info.module_path
|
module_path = plugin_info.module_path
|
||||||
is_dir = plugin_info.is_dir
|
repository_plugin_path = module_path.replace(".", "/").strip("/")
|
||||||
github_url = plugin_info.github_url
|
|
||||||
assert github_url
|
if plugin_info.is_dir:
|
||||||
replace_module_path = module_path.replace(".", "/").lstrip("/")
|
|
||||||
plugin_module = plugin_info.module
|
|
||||||
if is_dir:
|
|
||||||
files = await RepoFileManager.list_directory_files(
|
files = await RepoFileManager.list_directory_files(
|
||||||
github_url, replace_module_path, branch, repo_type=repo_type
|
repo_url,
|
||||||
|
repository_plugin_path,
|
||||||
|
branch,
|
||||||
|
repo_type=repo_type,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
files = [RepoFileInfo(path=f"{replace_module_path}.py", is_dir=False)]
|
if not repository_plugin_path:
|
||||||
if is_dir:
|
raise PluginStoreException(
|
||||||
target_dir = BASE_PATH / "plugins" / plugin_module
|
f"插件 {plugin_info.name} 的模块路径不能为空"
|
||||||
else:
|
)
|
||||||
target_dir = BASE_PATH / "plugins"
|
files = [RepoFileInfo(path=f"{repository_plugin_path}.py", is_dir=False)]
|
||||||
|
|
||||||
files = [file for file in files if not file.is_dir]
|
files = [file for file in files if not file.is_dir]
|
||||||
|
if not files:
|
||||||
|
raise PluginStoreException(
|
||||||
|
f"仓库中未找到插件目录: {plugin_info.module_path}"
|
||||||
|
)
|
||||||
|
|
||||||
|
target_root = (
|
||||||
|
BASE_PATH / "plugins" / plugin_info.module
|
||||||
|
if plugin_info.is_dir
|
||||||
|
else BASE_PATH / "plugins"
|
||||||
|
)
|
||||||
download_files: list[tuple[str, Path]] = []
|
download_files: list[tuple[str, Path]] = []
|
||||||
|
deploy_files: list[tuple[Path, Path]] = []
|
||||||
|
|
||||||
for file in files:
|
for file in files:
|
||||||
src_path = file.path
|
source_path = Path(file.path)
|
||||||
if is_dir:
|
if source_path.is_absolute() or ".." in source_path.parts:
|
||||||
dst_path = target_dir / Path(src_path).relative_to(replace_module_path)
|
raise PluginStoreException(f"仓库包含不安全的文件路径: {file.path}")
|
||||||
else:
|
|
||||||
dst_path = target_dir / f"{plugin_module}.py"
|
staged_path = staging_root / source_path
|
||||||
|
if plugin_info.is_dir:
|
||||||
|
plugin_root = (
|
||||||
|
Path(repository_plugin_path) if repository_plugin_path else Path()
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
relative_path = source_path.relative_to(plugin_root)
|
||||||
|
except ValueError as e:
|
||||||
|
raise PluginStoreException(
|
||||||
|
f"插件文件不在模块目录内: {file.path}"
|
||||||
|
) from e
|
||||||
|
destination_path = target_root / relative_path
|
||||||
|
else:
|
||||||
|
destination_path = target_root / f"{plugin_info.module}.py"
|
||||||
|
|
||||||
|
download_files.append((file.path, staged_path))
|
||||||
|
deploy_files.append((staged_path, destination_path))
|
||||||
|
|
||||||
|
required_download_files = download_files.copy()
|
||||||
|
requirement_files = [
|
||||||
|
staging_root / Path(file.path)
|
||||||
|
for file in files
|
||||||
|
if Path(file.path).name in {"requirement.txt", "requirements.txt"}
|
||||||
|
]
|
||||||
|
root_requirements: list[tuple[str, Path]] = []
|
||||||
|
if not requirement_files:
|
||||||
|
root_requirements = [
|
||||||
|
("requirement.txt", staging_root / "requirement.txt"),
|
||||||
|
("requirements.txt", staging_root / "requirements.txt"),
|
||||||
|
]
|
||||||
|
download_files.extend(root_requirements)
|
||||||
|
|
||||||
download_files.append((src_path, dst_path))
|
|
||||||
rand = random.randint(1, 10000)
|
|
||||||
requirement_path_ = TEMP_PATH / f"plugin_store_{rand}_req.txt"
|
|
||||||
requirements_path_ = TEMP_PATH / f"plugin_store_{rand}_reqs.txt"
|
|
||||||
download_files.append(("requirement.txt", requirement_path_))
|
|
||||||
download_files.append(("requirements.txt", requirements_path_))
|
|
||||||
result = await RepoFileManager.download_files(
|
result = await RepoFileManager.download_files(
|
||||||
github_url,
|
repo_url,
|
||||||
download_files,
|
download_files,
|
||||||
branch,
|
branch,
|
||||||
repo_type=repo_type,
|
repo_type=repo_type,
|
||||||
ignore_error=True,
|
ignore_error=bool(root_requirements),
|
||||||
)
|
)
|
||||||
if not result.success:
|
if not result.success:
|
||||||
raise PluginStoreException(result.error_message)
|
raise PluginStoreException(result.error_message or "未知下载错误")
|
||||||
|
|
||||||
requirement_paths = [
|
for source_path, staged_path in required_download_files:
|
||||||
file
|
if not staged_path.is_file():
|
||||||
for file in files
|
raise PluginStoreException(f"插件文件下载不完整: {source_path}")
|
||||||
if file.path.endswith("requirement.txt")
|
if (
|
||||||
or file.path.endswith("requirements.txt")
|
Path(source_path).suffix.lower() in cls._BINARY_EXTENSIONS
|
||||||
]
|
and staged_path.stat().st_size == 0
|
||||||
|
):
|
||||||
|
raise PluginStoreException(f"二进制文件下载为空: {source_path}")
|
||||||
|
|
||||||
is_install_req = False
|
requirement_files = [path for path in requirement_files if path.is_file()]
|
||||||
for requirement_path in requirement_paths:
|
if root_requirements:
|
||||||
requirement_file = target_dir / Path(requirement_path.path).relative_to(
|
requirement_files = [
|
||||||
replace_module_path
|
path for _, path in root_requirements if path.is_file()
|
||||||
)
|
]
|
||||||
if requirement_file.exists():
|
|
||||||
is_install_req = True
|
|
||||||
await VirtualEnvPackageManager.install_requirement(requirement_file)
|
|
||||||
|
|
||||||
if not is_install_req:
|
return deploy_files, requirement_files
|
||||||
if requirement_path_.exists():
|
|
||||||
logger.info(
|
|
||||||
f"开始安装插件 {module_path} 依赖文件: {requirement_path_}",
|
|
||||||
LOG_COMMAND,
|
|
||||||
)
|
|
||||||
await VirtualEnvPackageManager.install_requirement(requirement_path_)
|
|
||||||
if requirements_path_.exists():
|
|
||||||
logger.info(
|
|
||||||
f"开始安装插件 {module_path} 依赖文件: {requirements_path_}",
|
|
||||||
LOG_COMMAND,
|
|
||||||
)
|
|
||||||
await VirtualEnvPackageManager.install_requirement(requirements_path_)
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
async def remove_plugin(cls, index_or_module: str) -> str:
|
async def remove_plugin(cls, index_or_module: str) -> str:
|
||||||
|
|||||||
@@ -4,11 +4,14 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import base64
|
import base64
|
||||||
|
import contextlib
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
|
import psutil
|
||||||
|
|
||||||
from zhenxun.services.log import logger
|
from zhenxun.services.log import logger
|
||||||
|
|
||||||
from .config import LOG_COMMAND, RepoConfig
|
from .config import LOG_COMMAND, RepoConfig
|
||||||
@@ -48,7 +51,9 @@ async def clean_git(cwd: Path):
|
|||||||
|
|
||||||
|
|
||||||
async def run_git_command(
|
async def run_git_command(
|
||||||
command: str | list[str], cwd: Path | None = None
|
command: str | list[str],
|
||||||
|
cwd: Path | None = None,
|
||||||
|
timeout_seconds: float | None = None,
|
||||||
) -> tuple[bool, str, str]:
|
) -> tuple[bool, str, str]:
|
||||||
"""
|
"""
|
||||||
运行git命令,实时输出 stderr 进度信息(如 git clone --progress)。
|
运行git命令,实时输出 stderr 进度信息(如 git clone --progress)。
|
||||||
@@ -56,6 +61,7 @@ async def run_git_command(
|
|||||||
参数:
|
参数:
|
||||||
command: 命令字符串或参数列表
|
command: 命令字符串或参数列表
|
||||||
cwd: 工作目录
|
cwd: 工作目录
|
||||||
|
timeout_seconds: 硬超时时间,None 表示不限制
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
tuple[bool, str, str]: (是否成功, 标准输出, 标准错误)
|
tuple[bool, str, str]: (是否成功, 标准输出, 标准错误)
|
||||||
@@ -104,7 +110,18 @@ async def run_git_command(
|
|||||||
stderr_lines.append(text)
|
stderr_lines.append(text)
|
||||||
logger.debug(text, LOG_COMMAND)
|
logger.debug(text, LOG_COMMAND)
|
||||||
|
|
||||||
stdout_bytes, _ = await asyncio.gather(_collect_stdout(process), _read_stderr())
|
output_tasks = asyncio.gather(_collect_stdout(process), _read_stderr())
|
||||||
|
try:
|
||||||
|
if timeout_seconds is None:
|
||||||
|
stdout_bytes, _ = await output_tasks
|
||||||
|
else:
|
||||||
|
stdout_bytes, _ = await asyncio.wait_for(
|
||||||
|
output_tasks,
|
||||||
|
timeout=timeout_seconds,
|
||||||
|
)
|
||||||
|
except TimeoutError:
|
||||||
|
await _kill_process_tree(process)
|
||||||
|
return False, "", f"命令执行超时({timeout_seconds:g} 秒)"
|
||||||
|
|
||||||
await process.wait()
|
await process.wait()
|
||||||
stdout = (stdout_bytes or b"").decode("utf-8").strip()
|
stdout = (stdout_bytes or b"").decode("utf-8").strip()
|
||||||
@@ -116,6 +133,24 @@ async def run_git_command(
|
|||||||
return False, "", str(e)
|
return False, "", str(e)
|
||||||
|
|
||||||
|
|
||||||
|
async def _kill_process_tree(process: asyncio.subprocess.Process) -> None:
|
||||||
|
"""终止超时的 Git 进程及其派生的认证、网络辅助进程。"""
|
||||||
|
with contextlib.suppress(psutil.Error):
|
||||||
|
parent = psutil.Process(process.pid)
|
||||||
|
children = parent.children(recursive=True)
|
||||||
|
for child in reversed(children):
|
||||||
|
with contextlib.suppress(psutil.Error):
|
||||||
|
child.kill()
|
||||||
|
with contextlib.suppress(psutil.Error):
|
||||||
|
parent.kill()
|
||||||
|
|
||||||
|
if process.returncode is None:
|
||||||
|
with contextlib.suppress(ProcessLookupError):
|
||||||
|
process.kill()
|
||||||
|
with contextlib.suppress(TimeoutError):
|
||||||
|
await asyncio.wait_for(process.wait(), timeout=5)
|
||||||
|
|
||||||
|
|
||||||
async def _collect_stdout(process: asyncio.subprocess.Process) -> bytes:
|
async def _collect_stdout(process: asyncio.subprocess.Process) -> bytes:
|
||||||
"""收集子进程的全部 stdout 输出。"""
|
"""收集子进程的全部 stdout 输出。"""
|
||||||
assert process.stdout is not None
|
assert process.stdout is not None
|
||||||
@@ -238,12 +273,40 @@ async def sparse_checkout_clone(
|
|||||||
if not success:
|
if not success:
|
||||||
raise RuntimeError(f"配置稀疏路径失败: {err or out}")
|
raise RuntimeError(f"配置稀疏路径失败: {err or out}")
|
||||||
|
|
||||||
# 强制拉取并同步到远端
|
# 强制拉取并同步到远端。弱网或远端连接卡住时有限重试,
|
||||||
success, out, err = await run_git_command(
|
# 防止一次下载永久占住插件商店任务。
|
||||||
["fetch", "--force", "--depth", "1", "origin", branch], temp_path
|
fetch_error = ""
|
||||||
)
|
for attempt in range(3):
|
||||||
if not success:
|
success, out, err = await run_git_command(
|
||||||
raise RuntimeError(f"fetch 失败: {err or out}")
|
[
|
||||||
|
"-c",
|
||||||
|
"http.lowSpeedLimit=1024",
|
||||||
|
"-c",
|
||||||
|
"http.lowSpeedTime=30",
|
||||||
|
"fetch",
|
||||||
|
"--force",
|
||||||
|
"--depth",
|
||||||
|
"1",
|
||||||
|
"origin",
|
||||||
|
branch,
|
||||||
|
],
|
||||||
|
temp_path,
|
||||||
|
timeout_seconds=60,
|
||||||
|
)
|
||||||
|
if success:
|
||||||
|
break
|
||||||
|
fetch_error = err or out
|
||||||
|
if attempt < 2:
|
||||||
|
for lock_name in ("shallow.lock", "index.lock"):
|
||||||
|
with contextlib.suppress(FileNotFoundError, PermissionError):
|
||||||
|
(temp_path / ".git" / lock_name).unlink()
|
||||||
|
logger.warning(
|
||||||
|
f"git fetch 失败,将重试({attempt + 1}/3): {fetch_error}",
|
||||||
|
LOG_COMMAND,
|
||||||
|
)
|
||||||
|
await asyncio.sleep(2)
|
||||||
|
else:
|
||||||
|
raise RuntimeError(f"fetch 失败: {fetch_error}")
|
||||||
|
|
||||||
# 使用远端强制更新本地分支并覆盖工作区
|
# 使用远端强制更新本地分支并覆盖工作区
|
||||||
success, out, err = await run_git_command(
|
success, out, err = await run_git_command(
|
||||||
|
|||||||
Reference in New Issue
Block a user