🔨 提取GitHub相关操作 (#1609)

* 🔨 提取GitHub相关操作

* 🔨 重构API策略

zhenxun/utils/github_utils/__init__.py

@@ -0,0 +1,23 @@
+from .consts import GITHUB_REPO_URL_PATTERN
+from .func import get_fastest_raw_format, get_fastest_archive_format
+from .models import RepoAPI, RepoInfo, GitHubStrategy, JsdelivrStrategy
+
+__all__ = [
+    "parse_github_url",
+    "get_fastest_raw_format",
+    "get_fastest_archive_format",
+    "api_strategy",
+]
+
+
+def parse_github_url(github_url: str) -> "RepoInfo":
+    if matched := GITHUB_REPO_URL_PATTERN.match(github_url):
+        return RepoInfo(**{k: v for k, v in matched.groupdict().items() if v})
+    raise ValueError("github地址格式错误")
+
+
+# 使用
+jsdelivr_api = RepoAPI(JsdelivrStrategy())  # type: ignore
+github_api = RepoAPI(GitHubStrategy())  # type: ignore
+
+api_strategy = [jsdelivr_api, github_api]

zhenxun/utils/github_utils/consts.py

@@ -0,0 +1,35 @@
+import re
+
+GITHUB_REPO_URL_PATTERN = re.compile(
+    r"^https://github.com/(?P<owner>[^/]+)/(?P<repo>[^/]+)(/tree/(?P<branch>[^/]+))?$"
+)
+"""github仓库地址正则"""
+
+JSD_PACKAGE_API_FORMAT = (
+    "https://data.jsdelivr.com/v1/packages/gh/{owner}/{repo}@{branch}"
+)
+"""jsdelivr包地址格式"""
+
+GIT_API_TREES_FORMAT = (
+    "https://api.github.com/repos/{owner}/{repo}/git/trees/{branch}?recursive=1"
+)
+"""git api trees地址格式"""
+
+CACHED_API_TTL = 300
+"""缓存api ttl"""
+
+RAW_CONTENT_FORMAT = "https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{path}"
+"""raw content格式"""
+
+ARCHIVE_URL_FORMAT = "https://github.com/{owner}/{repo}/archive/refs/heads/{branch}.zip"
+"""archive url格式"""
+
+RELEASE_ASSETS_FORMAT = (
+    "https://github.com/{owner}/{repo}/releases/download/{version}/{filename}"
+)
+"""release assets格式"""
+
+RELEASE_SOURCE_FORMAT = (
+    "https://codeload.github.com/{owner}/{repo}/legacy.{compress}/refs/tags/{version}"
+)
+"""release 源码格式"""

zhenxun/utils/github_utils/func.py

@@ -0,0 +1,63 @@
+from aiocache import cached
+
+from ..http_utils import AsyncHttpx
+from .consts import (
+    ARCHIVE_URL_FORMAT,
+    RAW_CONTENT_FORMAT,
+    RELEASE_ASSETS_FORMAT,
+    RELEASE_SOURCE_FORMAT,
+)
+
+
+async def __get_fastest_format(formats: dict[str, str]) -> str:
+    sorted_urls = await AsyncHttpx.get_fastest_mirror(list(formats.keys()))
+    if not sorted_urls:
+        raise Exception("无法获取任意GitHub资源加速地址，请检查网络")
+    return formats[sorted_urls[0]]
+
+
+@cached()
+async def get_fastest_raw_format() -> str:
+    """获取最快的raw下载地址格式"""
+    formats: dict[str, str] = {
+        "https://raw.githubusercontent.com/": RAW_CONTENT_FORMAT,
+        "https://ghproxy.cc/": f"https://ghproxy.cc/{RAW_CONTENT_FORMAT}",
+        "https://mirror.ghproxy.com/": f"https://mirror.ghproxy.com/{RAW_CONTENT_FORMAT}",
+        "https://gh-proxy.com/": f"https://gh-proxy.com/{RAW_CONTENT_FORMAT}",
+        "https://cdn.jsdelivr.net/": "https://cdn.jsdelivr.net/gh/{owner}/{repo}@{branch}/{path}",
+    }
+    return await __get_fastest_format(formats)
+
+
+@cached()
+async def get_fastest_archive_format() -> str:
+    """获取最快的归档下载地址格式"""
+    formats: dict[str, str] = {
+        "https://github.com/": ARCHIVE_URL_FORMAT,
+        "https://ghproxy.cc/": f"https://ghproxy.cc/{ARCHIVE_URL_FORMAT}",
+        "https://mirror.ghproxy.com/": f"https://mirror.ghproxy.com/{ARCHIVE_URL_FORMAT}",
+        "https://gh-proxy.com/": f"https://gh-proxy.com/{ARCHIVE_URL_FORMAT}",
+    }
+    return await __get_fastest_format(formats)
+
+
+@cached()
+async def get_fastest_release_format() -> str:
+    """获取最快的发行版资源下载地址格式"""
+    formats: dict[str, str] = {
+        "https://objects.githubusercontent.com/": RELEASE_ASSETS_FORMAT,
+        "https://ghproxy.cc/": f"https://ghproxy.cc/{RELEASE_ASSETS_FORMAT}",
+        "https://mirror.ghproxy.com/": f"https://mirror.ghproxy.com/{RELEASE_ASSETS_FORMAT}",
+        "https://gh-proxy.com/": f"https://gh-proxy.com/{RELEASE_ASSETS_FORMAT}",
+    }
+    return await __get_fastest_format(formats)
+
+
+@cached()
+async def get_fastest_release_source_format() -> str:
+    """获取最快的发行版源码下载地址格式"""
+    formats: dict[str, str] = {
+        "https://codeload.github.com/": RELEASE_SOURCE_FORMAT,
+        "https://p.102333.xyz/": f"https://p.102333.xyz/{RELEASE_SOURCE_FORMAT}",
+    }
+    return await __get_fastest_format(formats)

zhenxun/utils/github_utils/models.py

@@ -0,0 +1,212 @@
+from typing import Protocol
+
+from aiocache import cached
+from strenum import StrEnum
+from pydantic import BaseModel
+
+from ..http_utils import AsyncHttpx
+from .consts import CACHED_API_TTL, GIT_API_TREES_FORMAT, JSD_PACKAGE_API_FORMAT
+from .func import (
+    get_fastest_raw_format,
+    get_fastest_archive_format,
+    get_fastest_release_source_format,
+)
+
+
+class RepoInfo(BaseModel):
+    """仓库信息"""
+
+    owner: str
+    repo: str
+    branch: str = "main"
+
+    async def get_raw_download_url(self, path: str):
+        url_format = await get_fastest_raw_format()
+        return url_format.format(**self.dict(), path=path)
+
+    async def get_archive_download_url(self):
+        url_format = await get_fastest_archive_format()
+        return url_format.format(**self.dict())
+
+    async def get_release_source_download_url_tgz(self, version: str):
+        url_format = await get_fastest_release_source_format()
+        return url_format.format(**self.dict(), version=version, compress="tar.gz")
+
+    async def get_release_source_download_url_zip(self, version: str):
+        url_format = await get_fastest_release_source_format()
+        return url_format.format(**self.dict(), version=version, compress="zip")
+
+
+class APIStrategy(Protocol):
+    """API策略"""
+
+    body: BaseModel
+
+    async def parse_repo_info(self, repo_info: RepoInfo) -> BaseModel: ...
+
+    def get_files(self, module_path: str, is_dir: bool) -> list[str]: ...
+
+
+class RepoAPI:
+    """基础接口"""
+
+    def __init__(self, strategy: APIStrategy):
+        self.strategy = strategy
+
+    async def parse_repo_info(self, repo_info: RepoInfo):
+        body = await self.strategy.parse_repo_info(repo_info)
+        self.strategy.body = body
+
+    def get_files(self, module_path: str, is_dir: bool) -> list[str]:
+        return self.strategy.get_files(module_path, is_dir)
+
+
+class FileType(StrEnum):
+    """文件类型"""
+
+    FILE = "file"
+    DIR = "directory"
+    PACKAGE = "gh"
+
+
+class FileInfo(BaseModel):
+    """文件信息"""
+
+    type: FileType
+    name: str
+    files: list["FileInfo"] = []
+
+
+class JsdelivrStrategy:
+    """Jsdelivr策略"""
+
+    body: FileInfo
+
+    def get_file_paths(self, module_path: str, is_dir: bool = True) -> list[str]:
+        """获取文件路径"""
+        paths = module_path.split("/")
+        filename = "" if is_dir else paths[-1]
+        paths = paths if is_dir else paths[:-1]
+        cur_file = self.body
+        for path in paths:  # 导航到正确的目录
+            cur_file = next(
+                (
+                    f
+                    for f in cur_file.files
+                    if f.type == FileType.DIR and f.name == path
+                ),
+                None,
+            )
+            if not cur_file:
+                raise ValueError(f"模块路径{module_path}不存在")
+
+        def collect_files(file: FileInfo, current_path: str, filename: str):
+            """收集文件"""
+            if file.type == FileType.FILE and (not filename or file.name == filename):
+                return [f"{current_path}/{file.name}"]
+            elif file.type == FileType.DIR and file.files:
+                return [
+                    path
+                    for f in file.files
+                    for path in collect_files(
+                        f,
+                        (
+                            f"{current_path}/{f.name}"
+                            if f.type == FileType.DIR
+                            else current_path
+                        ),
+                        filename,
+                    )
+                ]
+            return []
+
+        return collect_files(cur_file, "/".join(paths), filename)
+
+    @classmethod
+    @cached(ttl=CACHED_API_TTL)
+    async def parse_repo_info(cls, repo_info: RepoInfo) -> "FileInfo":
+        """解析仓库信息"""
+
+        """获取插件包信息
+
+        参数:
+            repo_info: 仓库信息
+
+        返回:
+            FileInfo: 插件包信息
+        """
+        jsd_package_url: str = JSD_PACKAGE_API_FORMAT.format(
+            owner=repo_info.owner, repo=repo_info.repo, branch=repo_info.branch
+        )
+        res = await AsyncHttpx.get(url=jsd_package_url)
+        if res.status_code != 200:
+            raise ValueError(f"下载错误, code: {res.status_code}")
+        return FileInfo(**res.json())
+
+    def get_files(self, module_path: str, is_dir: bool = True) -> list[str]:
+        """获取文件路径"""
+        return self.get_file_paths(module_path, is_dir)
+
+
+class TreeType(StrEnum):
+    """树类型"""
+
+    FILE = "blob"
+    DIR = "tree"
+
+
+class Tree(BaseModel):
+    """树"""
+
+    path: str
+    mode: str
+    type: TreeType
+    sha: str
+    size: int | None
+    url: str
+
+
+class TreeInfo(BaseModel):
+    """树信息"""
+
+    sha: str
+    url: str
+    tree: list[Tree]
+
+
+class GitHubStrategy:
+    """GitHub策略"""
+
+    body: TreeInfo
+
+    def export_files(self, module_path: str) -> list[str]:
+        """导出文件路径"""
+        tree_info = self.body
+        return [
+            file.path
+            for file in tree_info.tree
+            if file.type == TreeType.FILE and file.path.startswith(module_path)
+        ]
+
+    @classmethod
+    @cached(ttl=CACHED_API_TTL)
+    async def parse_repo_info(cls, repo_info: RepoInfo) -> "TreeInfo":
+        """获取仓库树
+
+        参数:
+            repo_info: 仓库信息
+
+        返回:
+            TreesInfo: 仓库树信息
+        """
+        git_tree_url: str = GIT_API_TREES_FORMAT.format(
+            owner=repo_info.owner, repo=repo_info.repo, branch=repo_info.branch
+        )
+        res = await AsyncHttpx.get(url=git_tree_url)
+        if res.status_code != 200:
+            raise ValueError(f"下载错误, code: {res.status_code}")
+        return TreeInfo(**res.json())
+
+    def get_files(self, module_path: str, is_dir: bool = True) -> list[str]:
+        """获取文件路径"""
+        return self.export_files(module_path)