From ce94f63d9ac6573224899f0feffbae886e0ad025 Mon Sep 17 00:00:00 2001 From: ManyManyTomato <93612024+ATTomatoo@users.noreply.github.com> Date: Tue, 3 Mar 2026 15:54:47 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(hook):=20=E5=A2=9E=E5=BC=BA?= =?UTF-8?q?=E8=AE=A4=E8=AF=81=E9=92=A9=E5=AD=90=E5=92=8C=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E6=97=B6=E7=BC=93=E5=AD=98=E7=AE=A1=E7=90=86=20(#2106)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ feat(hook): 增强认证钩子和运行时缓存管理 ``` ♻ refactor(hook): 移除未使用的配置项并优化缓存设置 移除 AUTH_HOOKS_CONCURRENCY_LIMIT 配置项，该配置项不再使用 ✨feat(auth_ban): 简化缓存配置并添加实体参数支持 将 BAN_CACHE_TTL 相关配置从动态配置改为常量定义， 移除复杂的 TTL 值转换逻辑，并为 auth_ban 函数添加可选的 entity 参数以支持外部传入实体信息 ♻ refactor(auth_limit): 移除未使用的配置依赖 移除 AUTH_LIMIT_NOTICE_CD 配置项，直接使用常量值 2 作为限制通知冷却时间 📦 依赖更新: update playwright dependency to version 1.57.0 in pyproject.toml and requirements.txt ``` * ✨ feat(auth_checker): 增强插件模块处理和预过滤逻辑，支持用户插件兼容性 * ✨ feat(http_utils): 添加内容缓存机制以优化HTTP响应处理 * ✨ feat(cache): 添加群组插件设置视图缓存类型并更新相关逻辑 * ✨ feat(renderer): 优化渲染引擎，增加内存缓存管理和HTML文档处理逻辑 * ✨ feat(sign_in): 添加好感度排行和好感度总排行命令 * ✨ feat(renderer): 增强浏览器实例管理和模板预处理，支持历史 include 语法兼容 * ✨ feat(renderer): 优化 Playwright 环境检查逻辑，增加结果缓存以提高性能 * ✨ feat(renderer): 增强模板文件渲染策略，优化资产加载路径处理 * :rotating_light: auto fix by pre-commit hooks * ✨ feat(log): 增加日志内容安全序列化，避免超长 base64 等污染日志 ✨ feat(log_sanitizer): 添加对嵌入超长 base64/data URI 的清理功能 ✨ feat(auth_checker): 添加 Alconna 快捷方式缓存检查，优化路由匹配逻辑 * feat(renderer): 添加渲染结果内存缓存功能以优化性能 feat(help): 实现帮助菜单图像缓存机制 feat(sign_in): 更新HTML卡片生成以支持动画禁用和剪裁 feat(superuser): 在启动时预热超级用户帮助缓存 feat(theme): 优化主题管理器的资源解析缓存 * :rotating_light: auto fix by pre-commit hooks * feat(renderer): 增加全页面视口最大宽度限制并优化内容尺寸计算 --------- Co-authored-by: ATTomatoo <1126160939@qq.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: HibiKier <45528451+HibiKier@users.noreply.github.com> --- poetry.lock | 254 +---- pyproject.toml | 4 +- requirements.txt | 2 +- zhenxun/builtin_plugins/help/_data_source.py | 26 +- zhenxun/builtin_plugins/hooks/__init__.py | 9 - .../builtin_plugins/hooks/auth/auth_ban.py | 58 +- .../builtin_plugins/hooks/auth/auth_limit.py | 9 +- zhenxun/builtin_plugins/hooks/auth_checker.py | 805 +++++++++++++--- zhenxun/builtin_plugins/hooks/auth_hook.py | 133 ++- zhenxun/builtin_plugins/init/__init_cache.py | 1 + zhenxun/builtin_plugins/sign_in/__init__.py | 2 + zhenxun/builtin_plugins/sign_in/utils.py | 55 +- .../builtin_plugins/superuser/super_help.py | 9 + .../web_ui/api/tabs/main/__init__.py | 2 +- zhenxun/models/ban_console.py | 39 +- zhenxun/services/avatar_service.py | 34 +- zhenxun/services/cache/runtime_cache.py | 179 +--- zhenxun/services/group_settings_service.py | 16 +- zhenxun/services/help_service.py | 28 +- zhenxun/services/log.py | 25 +- zhenxun/services/renderer/__init__.py | 7 + zhenxun/services/renderer/engine.py | 883 +++++++++++++++++- zhenxun/services/renderer/result_cache.py | 91 ++ zhenxun/services/renderer/template.py | 60 +- zhenxun/services/renderer/theme.py | 103 +- zhenxun/services/runtime_bootstrap.py | 85 ++ zhenxun/ui/__init__.py | 2 +- zhenxun/utils/enum.py | 2 + zhenxun/utils/http_utils.py | 91 +- zhenxun/utils/log_sanitizer.py | 16 + 30 files changed, 2392 insertions(+), 638 deletions(-) create mode 100644 zhenxun/services/renderer/result_cache.py create mode 100644 zhenxun/services/runtime_bootstrap.py diff --git a/poetry.lock b/poetry.lock index 3783c41d..b4bfe52d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. [[package]] name = "aiocache" @@ -727,33 +727,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "bilireq" -version = "0.2.13" -description = "又一个哔哩哔哩请求库" -optional = false -python-versions = "<4.0,>=3.8" -groups = ["main"] -files = [ - {file = "bilireq-0.2.13-py3-none-any.whl", hash = "sha256:7f76f55bb4b8bd5eb7ba920271dcab46f3e0544d23f75b9bdc6c2c547ab1fdca"}, - {file = "bilireq-0.2.13.tar.gz", hash = "sha256:f36451d523d9c9564e32b97f38e0df030727dfeba97652ebc6669b47a24a6fa2"}, -] - -[package.dependencies] -grpcio = ">=1.56.2" -httpx = "*" -protobuf = ">=4.23.4" -pycryptodome = ">=3.19.0" -qrcode = ">=7.4.2" - -[package.extras] -web-qrcode = ["lxml"] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "binaryornot" version = "0.4.4" @@ -1653,6 +1626,8 @@ files = [ {file = "greenlet-3.2.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c2ca18a03a8cfb5b25bc1cbe20f3d9a4c80d8c3b13ba3df49ac3961af0b1018d"}, {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9fe0a28a7b952a21e2c062cd5756d34354117796c6d9215a87f55e38d15402c5"}, {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8854167e06950ca75b898b104b63cc646573aa5fef1353d4508ecdd1ee76254f"}, + {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f47617f698838ba98f4ff4189aef02e7343952df3a615f847bb575c3feb177a7"}, + {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af41be48a4f60429d5cad9d22175217805098a9ef7c40bfef44f7669fb9d74d8"}, {file = "greenlet-3.2.4-cp310-cp310-win_amd64.whl", hash = "sha256:73f49b5368b5359d04e18d15828eecc1806033db5233397748f4ca813ff1056c"}, {file = "greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2"}, {file = "greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246"}, @@ -1662,6 +1637,8 @@ files = [ {file = "greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8"}, {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52"}, {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:55e9c5affaa6775e2c6b67659f3a71684de4c549b3dd9afca3bc773533d284fa"}, + {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c9c6de1940a7d828635fbd254d69db79e54619f165ee7ce32fda763a9cb6a58c"}, + {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03c5136e7be905045160b1b9fdca93dd6727b180feeafda6818e6496434ed8c5"}, {file = "greenlet-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:9c40adce87eaa9ddb593ccb0fa6a07caf34015a29bf8d344811665b573138db9"}, {file = "greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd"}, {file = "greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb"}, @@ -1671,6 +1648,8 @@ files = [ {file = "greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0"}, {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0"}, {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f"}, + {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee7a6ec486883397d70eec05059353b8e83eca9168b9f3f9a361971e77e0bcd0"}, + {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:326d234cbf337c9c3def0676412eb7040a35a768efc92504b947b3e9cfc7543d"}, {file = "greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02"}, {file = "greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31"}, {file = "greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945"}, @@ -1680,6 +1659,8 @@ files = [ {file = "greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671"}, {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b"}, {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d25c5091190f2dc0eaa3f950252122edbbadbb682aa7b1ef2f8af0f8c0afefae"}, + {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6e343822feb58ac4d0a1211bd9399de2b3a04963ddeec21530fc426cc121f19b"}, + {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca7f6f1f2649b89ce02f6f229d7c19f680a6238af656f61e0115b24857917929"}, {file = "greenlet-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:554b03b6e73aaabec3745364d6239e9e012d64c68ccd0b8430c64ccc14939a8b"}, {file = "greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0"}, {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f"}, @@ -1687,6 +1668,8 @@ files = [ {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1"}, {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735"}, {file = "greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337"}, + {file = "greenlet-3.2.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2917bdf657f5859fbf3386b12d68ede4cf1f04c90c3a6bc1f013dd68a22e2269"}, + {file = "greenlet-3.2.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:015d48959d4add5d6c9f6c5210ee3803a830dce46356e3bc326d6776bde54681"}, {file = "greenlet-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:e37ab26028f12dbb0ff65f29a8d3d44a765c61e729647bf2ddfbbed621726f01"}, {file = "greenlet-3.2.4-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:b6a7c19cf0d2742d0809a4c05975db036fdff50cd294a93632d6a310bf9ac02c"}, {file = "greenlet-3.2.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:27890167f55d2387576d1f41d9487ef171849ea0359ce1510ca6e06c8bece11d"}, @@ -1696,6 +1679,8 @@ files = [ {file = "greenlet-3.2.4-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9913f1a30e4526f432991f89ae263459b1c64d1608c0d22a5c79c287b3c70df"}, {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b90654e092f928f110e0007f572007c9727b5265f7632c2fa7415b4689351594"}, {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:81701fd84f26330f0d5f4944d4e92e61afe6319dcd9775e39396e39d7c3e5f98"}, + {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:28a3c6b7cd72a96f61b0e4b2a36f681025b60ae4779cc73c1535eb5f29560b10"}, + {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:52206cd642670b0b320a1fd1cbfd95bca0e043179c1d8a045f2c6109dfe973be"}, {file = "greenlet-3.2.4-cp39-cp39-win32.whl", hash = "sha256:65458b409c1ed459ea899e939f0e1cdb14f58dbc803f2f93c5eab5694d32671b"}, {file = "greenlet-3.2.4-cp39-cp39-win_amd64.whl", hash = "sha256:d2e685ade4dafd447ede19c31277a224a239a0a1a4eca4e6390efedf20260cfb"}, {file = "greenlet-3.2.4.tar.gz", hash = "sha256:0dca0d95ff849f9a364385f36ab49f50065d76964944638be9691e1832e9f86d"}, @@ -1710,75 +1695,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "grpcio" -version = "1.74.0" -description = "HTTP/2-based RPC framework" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "grpcio-1.74.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907"}, - {file = "grpcio-1.74.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb"}, - {file = "grpcio-1.74.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486"}, - {file = "grpcio-1.74.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11"}, - {file = "grpcio-1.74.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9"}, - {file = "grpcio-1.74.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc"}, - {file = "grpcio-1.74.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e"}, - {file = "grpcio-1.74.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82"}, - {file = "grpcio-1.74.0-cp310-cp310-win32.whl", hash = "sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7"}, - {file = "grpcio-1.74.0-cp310-cp310-win_amd64.whl", hash = "sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5"}, - {file = "grpcio-1.74.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31"}, - {file = "grpcio-1.74.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4"}, - {file = "grpcio-1.74.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce"}, - {file = "grpcio-1.74.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3"}, - {file = "grpcio-1.74.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182"}, - {file = "grpcio-1.74.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d"}, - {file = "grpcio-1.74.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f"}, - {file = "grpcio-1.74.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4"}, - {file = "grpcio-1.74.0-cp311-cp311-win32.whl", hash = "sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b"}, - {file = "grpcio-1.74.0-cp311-cp311-win_amd64.whl", hash = "sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11"}, - {file = "grpcio-1.74.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8"}, - {file = "grpcio-1.74.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6"}, - {file = "grpcio-1.74.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5"}, - {file = "grpcio-1.74.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49"}, - {file = "grpcio-1.74.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7"}, - {file = "grpcio-1.74.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3"}, - {file = "grpcio-1.74.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707"}, - {file = "grpcio-1.74.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b"}, - {file = "grpcio-1.74.0-cp312-cp312-win32.whl", hash = "sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c"}, - {file = "grpcio-1.74.0-cp312-cp312-win_amd64.whl", hash = "sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc"}, - {file = "grpcio-1.74.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89"}, - {file = "grpcio-1.74.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01"}, - {file = "grpcio-1.74.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e"}, - {file = "grpcio-1.74.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91"}, - {file = "grpcio-1.74.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249"}, - {file = "grpcio-1.74.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362"}, - {file = "grpcio-1.74.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f"}, - {file = "grpcio-1.74.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20"}, - {file = "grpcio-1.74.0-cp313-cp313-win32.whl", hash = "sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa"}, - {file = "grpcio-1.74.0-cp313-cp313-win_amd64.whl", hash = "sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24"}, - {file = "grpcio-1.74.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae"}, - {file = "grpcio-1.74.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b"}, - {file = "grpcio-1.74.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a"}, - {file = "grpcio-1.74.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a"}, - {file = "grpcio-1.74.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9"}, - {file = "grpcio-1.74.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7"}, - {file = "grpcio-1.74.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176"}, - {file = "grpcio-1.74.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac"}, - {file = "grpcio-1.74.0-cp39-cp39-win32.whl", hash = "sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854"}, - {file = "grpcio-1.74.0-cp39-cp39-win_amd64.whl", hash = "sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa"}, - {file = "grpcio-1.74.0.tar.gz", hash = "sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1"}, -] - -[package.extras] -protobuf = ["grpcio-tools (>=1.74.0)"] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "h11" version = "0.16.0" @@ -2063,6 +1979,23 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" +[[package]] +name = "json-repair" +version = "0.54.3" +description = "A package to repair broken json strings" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "json_repair-0.54.3-py3-none-any.whl", hash = "sha256:4cdc132ee27d4780576f71bf27a113877046224a808bfc17392e079cb344fb81"}, + {file = "json_repair-0.54.3.tar.gz", hash = "sha256:e50feec9725e52ac91f12184609754684ac1656119dfbd31de09bdaf9a1d8bf6"}, +] + +[package.source] +type = "legacy" +url = "https://mirrors.aliyun.com/pypi/simple" +reference = "aliyun" + [[package]] name = "loguru" version = "0.7.3" @@ -2080,7 +2013,7 @@ colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} [package.extras] -dev = ["Sphinx (==8.1.3) ; python_version >= \"3.11\"", "build (==1.2.2) ; python_version >= \"3.11\"", "colorama (==0.4.5) ; python_version < \"3.8\"", "colorama (==0.4.6) ; python_version >= \"3.8\"", "exceptiongroup (==1.1.3) ; python_version >= \"3.7\" and python_version < \"3.11\"", "freezegun (==1.1.0) ; python_version < \"3.8\"", "freezegun (==1.5.0) ; python_version >= \"3.8\"", "mypy (==v0.910) ; python_version < \"3.6\"", "mypy (==v0.971) ; python_version == \"3.6\"", "mypy (==v1.13.0) ; python_version >= \"3.8\"", "mypy (==v1.4.1) ; python_version == \"3.7\"", "myst-parser (==4.0.0) ; python_version >= \"3.11\"", "pre-commit (==4.0.1) ; python_version >= \"3.9\"", "pytest (==6.1.2) ; python_version < \"3.8\"", "pytest (==8.3.2) ; python_version >= \"3.8\"", "pytest-cov (==2.12.1) ; python_version < \"3.8\"", "pytest-cov (==5.0.0) ; python_version == \"3.8\"", "pytest-cov (==6.0.0) ; python_version >= \"3.9\"", "pytest-mypy-plugins (==1.9.3) ; python_version >= \"3.6\" and python_version < \"3.8\"", "pytest-mypy-plugins (==3.1.0) ; python_version >= \"3.8\"", "sphinx-rtd-theme (==3.0.2) ; python_version >= \"3.11\"", "tox (==3.27.1) ; python_version < \"3.8\"", "tox (==4.23.2) ; python_version >= \"3.8\"", "twine (==6.0.1) ; python_version >= \"3.11\""] +dev = ["Sphinx (==8.1.3) ; python_version >= \"3.11\"", "build (==1.2.2) ; python_version >= \"3.11\"", "colorama (==0.4.5) ; python_version < \"3.8\"", "colorama (==0.4.6) ; python_version >= \"3.8\"", "exceptiongroup (==1.1.3) ; python_version >= \"3.7\" and python_version < \"3.11\"", "freezegun (==1.1.0) ; python_version < \"3.8\"", "freezegun (==1.5.0) ; python_version >= \"3.8\"", "mypy (==0.910) ; python_version < \"3.6\"", "mypy (==0.971) ; python_version == \"3.6\"", "mypy (==1.13.0) ; python_version >= \"3.8\"", "mypy (==1.4.1) ; python_version == \"3.7\"", "myst-parser (==4.0.0) ; python_version >= \"3.11\"", "pre-commit (==4.0.1) ; python_version >= \"3.9\"", "pytest (==6.1.2) ; python_version < \"3.8\"", "pytest (==8.3.2) ; python_version >= \"3.8\"", "pytest-cov (==2.12.1) ; python_version < \"3.8\"", "pytest-cov (==5.0.0) ; python_version == \"3.8\"", "pytest-cov (==6.0.0) ; python_version >= \"3.9\"", "pytest-mypy-plugins (==1.9.3) ; python_version >= \"3.6\" and python_version < \"3.8\"", "pytest-mypy-plugins (==3.1.0) ; python_version >= \"3.8\"", "sphinx-rtd-theme (==3.0.2) ; python_version >= \"3.11\"", "tox (==3.27.1) ; python_version < \"3.8\"", "tox (==4.23.2) ; python_version >= \"3.8\"", "twine (==6.0.1) ; python_version >= \"3.11\""] [package.source] type = "legacy" @@ -3116,20 +3049,20 @@ reference = "aliyun" [[package]] name = "playwright" -version = "1.54.0" +version = "1.57.0" description = "A high-level API to automate web browsers" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "playwright-1.54.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:bf3b845af744370f1bd2286c2a9536f474cc8a88dc995b72ea9a5be714c9a77d"}, - {file = "playwright-1.54.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:780928b3ca2077aea90414b37e54edd0c4bbb57d1aafc42f7aa0b3fd2c2fac02"}, - {file = "playwright-1.54.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:81d0b6f28843b27f288cfe438af0a12a4851de57998009a519ea84cee6fbbfb9"}, - {file = "playwright-1.54.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:09919f45cc74c64afb5432646d7fef0d19fff50990c862cb8d9b0577093f40cc"}, - {file = "playwright-1.54.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13ae206c55737e8e3eae51fb385d61c0312eeef31535643bb6232741b41b6fdc"}, - {file = "playwright-1.54.0-py3-none-win32.whl", hash = "sha256:0b108622ffb6906e28566f3f31721cd57dda637d7e41c430287804ac01911f56"}, - {file = "playwright-1.54.0-py3-none-win_amd64.whl", hash = "sha256:9e5aee9ae5ab1fdd44cd64153313a2045b136fcbcfb2541cc0a3d909132671a2"}, - {file = "playwright-1.54.0-py3-none-win_arm64.whl", hash = "sha256:a975815971f7b8dca505c441a4c56de1aeb56a211290f8cc214eeef5524e8d75"}, + {file = "playwright-1.57.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:9351c1ac3dfd9b3820fe7fc4340d96c0d3736bb68097b9b7a69bd45d25e9370c"}, + {file = "playwright-1.57.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a4a9d65027bce48eeba842408bcc1421502dfd7e41e28d207e94260fa93ca67e"}, + {file = "playwright-1.57.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:99104771abc4eafee48f47dac2369e0015516dc1ce8c409807d2dd440828b9a4"}, + {file = "playwright-1.57.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:284ed5a706b7c389a06caa431b2f0ba9ac4130113c3a779767dda758c2497bb1"}, + {file = "playwright-1.57.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a1bae6c0a07839cdeaddbc0756b3b2b85e476c07945f64ece08f1f956a86f1"}, + {file = "playwright-1.57.0-py3-none-win32.whl", hash = "sha256:1dd93b265688da46e91ecb0606d36f777f8eadcf7fbef12f6426b20bf0c9137c"}, + {file = "playwright-1.57.0-py3-none-win_amd64.whl", hash = "sha256:6caefb08ed2c6f29d33b8088d05d09376946e49a73be19271c8cd5384b82b14c"}, + {file = "playwright-1.57.0-py3-none-win_arm64.whl", hash = "sha256:5f065f5a133dbc15e6e7c71e7bc04f258195755b1c32a432b792e28338c8335e"}, ] [package.dependencies] @@ -3336,30 +3269,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "protobuf" -version = "6.32.0" -description = "" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "protobuf-6.32.0-cp310-abi3-win32.whl", hash = "sha256:84f9e3c1ff6fb0308dbacb0950d8aa90694b0d0ee68e75719cb044b7078fe741"}, - {file = "protobuf-6.32.0-cp310-abi3-win_amd64.whl", hash = "sha256:a8bdbb2f009cfc22a36d031f22a625a38b615b5e19e558a7b756b3279723e68e"}, - {file = "protobuf-6.32.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:d52691e5bee6c860fff9a1c86ad26a13afbeb4b168cd4445c922b7e2cf85aaf0"}, - {file = "protobuf-6.32.0-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:501fe6372fd1c8ea2a30b4d9be8f87955a64d6be9c88a973996cef5ef6f0abf1"}, - {file = "protobuf-6.32.0-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:75a2aab2bd1aeb1f5dc7c5f33bcb11d82ea8c055c9becbb41c26a8c43fd7092c"}, - {file = "protobuf-6.32.0-cp39-cp39-win32.whl", hash = "sha256:7db8ed09024f115ac877a1427557b838705359f047b2ff2f2b2364892d19dacb"}, - {file = "protobuf-6.32.0-cp39-cp39-win_amd64.whl", hash = "sha256:15eba1b86f193a407607112ceb9ea0ba9569aed24f93333fe9a497cf2fda37d3"}, - {file = "protobuf-6.32.0-py3-none-any.whl", hash = "sha256:ba377e5b67b908c8f3072a57b63e2c6a4cbd18aea4ed98d2584350dbf46f2783"}, - {file = "protobuf-6.32.0.tar.gz", hash = "sha256:a81439049127067fc49ec1d36e25c6ee1d1a2b7be930675f919258d03c04e7d2"}, -] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "psutil" version = "5.9.8" @@ -3446,62 +3355,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "pycryptodome" -version = "3.23.0" -description = "Cryptographic library for Python" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" -groups = ["main"] -files = [ - {file = "pycryptodome-3.23.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a176b79c49af27d7f6c12e4b178b0824626f40a7b9fed08f712291b6d54bf566"}, - {file = "pycryptodome-3.23.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:573a0b3017e06f2cffd27d92ef22e46aa3be87a2d317a5abf7cc0e84e321bd75"}, - {file = "pycryptodome-3.23.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:63dad881b99ca653302b2c7191998dd677226222a3f2ea79999aa51ce695f720"}, - {file = "pycryptodome-3.23.0-cp27-cp27m-win32.whl", hash = "sha256:b34e8e11d97889df57166eda1e1ddd7676da5fcd4d71a0062a760e75060514b4"}, - {file = "pycryptodome-3.23.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:7ac1080a8da569bde76c0a104589c4f414b8ba296c0b3738cf39a466a9fb1818"}, - {file = "pycryptodome-3.23.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:6fe8258e2039eceb74dfec66b3672552b6b7d2c235b2dfecc05d16b8921649a8"}, - {file = "pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4"}, - {file = "pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae"}, - {file = "pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477"}, - {file = "pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7"}, - {file = "pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446"}, - {file = "pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265"}, - {file = "pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b"}, - {file = "pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d"}, - {file = "pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a"}, - {file = "pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625"}, - {file = "pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39"}, - {file = "pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27"}, - {file = "pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843"}, - {file = "pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490"}, - {file = "pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575"}, - {file = "pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b"}, - {file = "pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a"}, - {file = "pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f"}, - {file = "pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa"}, - {file = "pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886"}, - {file = "pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2"}, - {file = "pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c"}, - {file = "pycryptodome-3.23.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:350ebc1eba1da729b35ab7627a833a1a355ee4e852d8ba0447fafe7b14504d56"}, - {file = "pycryptodome-3.23.0-pp27-pypy_73-win32.whl", hash = "sha256:93837e379a3e5fd2bb00302a47aee9fdf7940d83595be3915752c74033d17ca7"}, - {file = "pycryptodome-3.23.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ddb95b49df036ddd264a0ad246d1be5b672000f12d6961ea2c267083a5e19379"}, - {file = "pycryptodome-3.23.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e95564beb8782abfd9e431c974e14563a794a4944c29d6d3b7b5ea042110b4"}, - {file = "pycryptodome-3.23.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14e15c081e912c4b0d75632acd8382dfce45b258667aa3c67caf7a4d4c13f630"}, - {file = "pycryptodome-3.23.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fc76bf273353dc7e5207d172b83f569540fc9a28d63171061c42e361d22353"}, - {file = "pycryptodome-3.23.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:45c69ad715ca1a94f778215a11e66b7ff989d792a4d63b68dc586a1da1392ff5"}, - {file = "pycryptodome-3.23.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:865d83c906b0fc6a59b510deceee656b6bc1c4fa0d82176e2b77e97a420a996a"}, - {file = "pycryptodome-3.23.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89d4d56153efc4d81defe8b65fd0821ef8b2d5ddf8ed19df31ba2f00872b8002"}, - {file = "pycryptodome-3.23.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3f2d0aaf8080bda0587d58fc9fe4766e012441e2eed4269a77de6aea981c8be"}, - {file = "pycryptodome-3.23.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64093fc334c1eccfd3933c134c4457c34eaca235eeae49d69449dc4728079339"}, - {file = "pycryptodome-3.23.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:ce64e84a962b63a47a592690bdc16a7eaf709d2c2697ababf24a0def566899a6"}, - {file = "pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef"}, -] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "pydantic" version = "2.11.7" @@ -4183,31 +4036,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "qrcode" -version = "8.2" -description = "QR Code image generator" -optional = false -python-versions = ">=3.9,<4.0" -groups = ["main"] -files = [ - {file = "qrcode-8.2-py3-none-any.whl", hash = "sha256:16e64e0716c14960108e85d853062c9e8bba5ca8252c0b4d0231b9df4060ff4f"}, - {file = "qrcode-8.2.tar.gz", hash = "sha256:35c3f2a4172b33136ab9f6b3ef1c00260dd2f66f858f24d88418a015f446506c"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} - -[package.extras] -all = ["pillow (>=9.1.0)", "pypng"] -pil = ["pillow (>=9.1.0)"] -png = ["pypng"] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "redis" version = "6.4.0" @@ -5690,4 +5518,4 @@ redis = ["redis"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "f4deabfa0c32fbce9779e5898e45a8a5398839311ee90be8a5200f5417a58422" +content-hash = "c52d7551bdd10fa81aef3fc2aed18b09f76827a238e1825c605e8a1d042100ea" diff --git a/pyproject.toml b/pyproject.toml index 615cad87..d1ef7dc3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ priority = "primary" [tool.poetry.dependencies] python = "^3.10" -playwright = "^1.41.1" +playwright = "1.57.0" nonebot-adapter-onebot = ">=2.3.1" nonebot-plugin-apscheduler = "^0.5" tortoise-orm = "^0.20.0" @@ -145,4 +145,4 @@ asyncio_default_fixture_loop_scope = "session" [build-system] requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" \ No newline at end of file +build-backend = "poetry.core.masonry.api" diff --git a/requirements.txt b/requirements.txt index 26a01df4..967390c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -playwright>=1.41.1,<2.0.0 +playwright==1.57.0 nonebot-adapter-onebot>=2.3.1 nonebot-plugin-apscheduler>=0.5,<0.6 tortoise-orm>=0.20.0,<0.21.0 diff --git a/zhenxun/builtin_plugins/help/_data_source.py b/zhenxun/builtin_plugins/help/_data_source.py index 3e836e8b..44315e20 100644 --- a/zhenxun/builtin_plugins/help/_data_source.py +++ b/zhenxun/builtin_plugins/help/_data_source.py @@ -17,6 +17,7 @@ from zhenxun.services import ( generate, ) from zhenxun.services.log import logger +from zhenxun.services.renderer.result_cache import RenderResultMemoryCache from zhenxun.ui.models import PluginMenuCategory, PluginMenuData from zhenxun.utils.common_utils import format_usage_for_markdown from zhenxun.utils.enum import BlockType, PluginType @@ -28,6 +29,11 @@ random_bk_path = IMAGE_PATH / "background" / "help" / "simple_help" background = IMAGE_PATH / "background" / "0.png" driver = nonebot.get_driver() +_HELP_MENU_IMAGE_CACHE = RenderResultMemoryCache( + ttl_seconds=300, + max_items=64, + max_total_bytes=64 * 1024 * 1024, +) def _create_plugin_menu_item( @@ -121,7 +127,25 @@ async def create_help_img( categories=categories_objects, ) - return await ui.render(menu_data) + cache_payload = { + "self_id": session.self_id, + "group_id": group_id, + "is_detail": is_detail, + "theme": Config.get_config("UI", "THEME", "default"), + "menu_data": menu_data, + } + cache_key = RenderResultMemoryCache.build_key(cache_payload) + if cached_image := await _HELP_MENU_IMAGE_CACHE.get(cache_key): + return cached_image + + image_bytes = await ui.render( + menu_data, + clip_selector=".wrapper", + clip_padding=20, + disable_animations=True, + ) + await _HELP_MENU_IMAGE_CACHE.set(cache_key, image_bytes) + return image_bytes async def get_user_allow_help(user_id: str) -> list[PluginType]: diff --git a/zhenxun/builtin_plugins/hooks/__init__.py b/zhenxun/builtin_plugins/hooks/__init__.py index e61ec71d..2f8c79de 100644 --- a/zhenxun/builtin_plugins/hooks/__init__.py +++ b/zhenxun/builtin_plugins/hooks/__init__.py @@ -58,14 +58,5 @@ Config.add_plugin_config( type=bool, ) -Config.add_plugin_config( - "hook", - "AUTH_HOOKS_CONCURRENCY_LIMIT", - 5, - help="同步进入权限钩子最大并发数", - default_value=5, - type=int, -) - nonebot.load_plugins(str(Path(__file__).parent.resolve())) diff --git a/zhenxun/builtin_plugins/hooks/auth/auth_ban.py b/zhenxun/builtin_plugins/hooks/auth/auth_ban.py index b877da83..6aef96a2 100644 --- a/zhenxun/builtin_plugins/hooks/auth/auth_ban.py +++ b/zhenxun/builtin_plugins/hooks/auth/auth_ban.py @@ -26,52 +26,13 @@ Config.add_plugin_config( "才不会给你发消息.", help="对被ban用户发送的消息", ) -Config.add_plugin_config( - "hook", - "BAN_CACHE_TTL", - 2, - help="ban cache ttl seconds", -) -Config.add_plugin_config( - "hook", - "BAN_CACHE_TTL_POSITIVE", - 30, - help="ban cache ttl seconds for banned users", -) -Config.add_plugin_config( - "hook", - "BAN_CACHE_TTL_NEGATIVE", - 5, - help="ban cache ttl seconds for non-banned users", -) - - -def _coerce_ttl(value, default): - try: - value_int = int(value) - except (TypeError, ValueError): - return default - return value_int if value_int >= 0 else default - - -_ban_cache_ttl_value = Config.get_config("hook", "BAN_CACHE_TTL", 2) -try: - _ban_cache_ttl_value = int(_ban_cache_ttl_value) -except (TypeError, ValueError): - _ban_cache_ttl_value = 2 - -_ban_cache_ttl_positive = _coerce_ttl( - Config.get_config("hook", "BAN_CACHE_TTL_POSITIVE", _ban_cache_ttl_value), - _ban_cache_ttl_value, -) -_ban_cache_ttl_negative = _coerce_ttl( - Config.get_config("hook", "BAN_CACHE_TTL_NEGATIVE", _ban_cache_ttl_value), - _ban_cache_ttl_value, -) +BAN_CACHE_TTL = 2 +BAN_CACHE_TTL_POSITIVE = 30 +BAN_CACHE_TTL_NEGATIVE = 5 BAN_CACHE = ( CacheDict("AUTH_BAN_CACHE", expire=0) - if max(_ban_cache_ttl_positive, _ban_cache_ttl_negative) > 0 + if max(BAN_CACHE_TTL_POSITIVE, BAN_CACHE_TTL_NEGATIVE) > 0 else None ) @@ -92,7 +53,7 @@ def _ban_cache_get(key: str) -> int | None: def _ban_cache_set(key: str, value: int) -> None: if not BAN_CACHE: return - ttl = _ban_cache_ttl_positive if value else _ban_cache_ttl_negative + ttl = BAN_CACHE_TTL_POSITIVE if value else BAN_CACHE_TTL_NEGATIVE if ttl <= 0: return BAN_CACHE.set(key, value, expire=ttl) @@ -255,7 +216,11 @@ async def user_handle(plugin: PluginInfo, entity: EntityIDs, session: Uninfo) -> async def auth_ban( - matcher: Matcher, bot: Bot, session: Uninfo, plugin: PluginInfo + matcher: Matcher, + bot: Bot, + session: Uninfo, + plugin: PluginInfo, + entity: EntityIDs | None = None, ) -> None: """权限检查 - ban 检查 @@ -270,7 +235,8 @@ async def auth_ban( return if not matcher.plugin_name: return - entity = get_entity_ids(session) + if entity is None: + entity = get_entity_ids(session) if entity.user_id in bot.config.superusers: return if entity.group_id: diff --git a/zhenxun/builtin_plugins/hooks/auth/auth_limit.py b/zhenxun/builtin_plugins/hooks/auth/auth_limit.py index 0a915765..e8201001 100644 --- a/zhenxun/builtin_plugins/hooks/auth/auth_limit.py +++ b/zhenxun/builtin_plugins/hooks/auth/auth_limit.py @@ -6,7 +6,6 @@ import nonebot from nonebot_plugin_uninfo import Uninfo from pydantic import BaseModel -from zhenxun.configs.config import Config from zhenxun.models.plugin_info import PluginInfo from zhenxun.models.plugin_limit import PluginLimit from zhenxun.services.cache.runtime_cache import ( @@ -27,13 +26,7 @@ from .exception import SkipPluginException driver = nonebot.get_driver() -Config.add_plugin_config( - "hook", - "AUTH_LIMIT_NOTICE_CD", - 2, - help="auth limit notice cooldown seconds", -) -_LIMIT_NOTICE_CD = int(Config.get_config("hook", "AUTH_LIMIT_NOTICE_CD", 2) or 2) +_LIMIT_NOTICE_CD = 2 _LIMIT_NOTICE_LIMITER = FreqLimiter(_LIMIT_NOTICE_CD) _LIMIT_NOTICE_TASKS: set[asyncio.Task] = set() diff --git a/zhenxun/builtin_plugins/hooks/auth_checker.py b/zhenxun/builtin_plugins/hooks/auth_checker.py index 9f77bf27..03900194 100644 --- a/zhenxun/builtin_plugins/hooks/auth_checker.py +++ b/zhenxun/builtin_plugins/hooks/auth_checker.py @@ -1,5 +1,8 @@ import asyncio +from collections.abc import Awaitable, Callable import contextlib +import os +import re import time from typing import cast @@ -7,10 +10,10 @@ from nonebot import get_loaded_plugins from nonebot.adapters import Bot, Event from nonebot.exception import IgnoredException from nonebot.matcher import Matcher +import nonebot.message as nb_message from nonebot_plugin_alconna import UniMsg from nonebot_plugin_uninfo import Uninfo -from zhenxun.configs.config import Config from zhenxun.configs.utils import PluginExtraData from zhenxun.models.plugin_info import PluginInfo from zhenxun.models.user_console import UserConsole @@ -46,54 +49,12 @@ from .auth.exception import ( PermissionExemption, SkipPluginException, ) -from .auth.utils import base_config -Config.add_plugin_config( - "hook", - "AUTH_HOOKS_CONCURRENCY_LIMIT", - 6, - help="auth hooks concurrency limit", -) -Config.add_plugin_config( - "hook", - "AUTH_DB_CONCURRENCY_LIMIT", - 6, - help="auth db concurrency limit", -) -Config.add_plugin_config( - "hook", - "AUTH_PLUGIN_CACHE_TTL", - 30, - help="plugin info cache ttl seconds", -) -Config.add_plugin_config( - "hook", - "AUTH_USER_CACHE_TTL", - 5, - help="user cache ttl seconds", -) -Config.add_plugin_config( - "hook", - "AUTH_EVENT_CACHE_TTL", - 2, - help="event auth cache ttl seconds", -) - - -def _coerce_positive_int(value, default): - try: - value_int = int(value) - except (TypeError, ValueError): - return default - return value_int if value_int > 0 else default - - -def _coerce_cache_ttl(value, default): - try: - value_int = int(value) - except (TypeError, ValueError): - return default - return value_int if value_int >= 0 else default +AUTH_HOOKS_CONCURRENCY_LIMIT = 5 +AUTH_DB_CONCURRENCY_LIMIT = 6 +AUTH_PLUGIN_CACHE_TTL = 30 +AUTH_USER_CACHE_TTL = 5 +AUTH_EVENT_CACHE_TTL = 2 # 超时设置（秒） @@ -111,18 +72,11 @@ CIRCUIT_BREAKERS = { CIRCUIT_RESET_TIME = 300 # 5分钟 # 并发控制：限制同时进入 hooks 并行检查的协程数 +HOOKS_CONCURRENCY_LIMIT = AUTH_HOOKS_CONCURRENCY_LIMIT +DB_CONCURRENCY_LIMIT = AUTH_DB_CONCURRENCY_LIMIT -# 默认为 6，可通过环境变量 AUTH_HOOKS_CONCURRENCY_LIMIT 调整 -HOOKS_CONCURRENCY_LIMIT = _coerce_positive_int( - base_config.get("AUTH_HOOKS_CONCURRENCY_LIMIT", 6), 6 -) -DB_CONCURRENCY_LIMIT = _coerce_positive_int( - base_config.get("AUTH_DB_CONCURRENCY_LIMIT", HOOKS_CONCURRENCY_LIMIT), - HOOKS_CONCURRENCY_LIMIT, -) - -PLUGIN_CACHE_TTL = _coerce_cache_ttl(base_config.get("AUTH_PLUGIN_CACHE_TTL", 30), 30) -USER_CACHE_TTL = _coerce_cache_ttl(base_config.get("AUTH_USER_CACHE_TTL", 5), 5) +PLUGIN_CACHE_TTL = AUTH_PLUGIN_CACHE_TTL +USER_CACHE_TTL = AUTH_USER_CACHE_TTL PLUGIN_CACHE = ( CacheDict("AUTH_PLUGIN_CACHE", expire=PLUGIN_CACHE_TTL) @@ -132,7 +86,7 @@ PLUGIN_CACHE = ( USER_CACHE = ( CacheDict("AUTH_USER_CACHE", expire=USER_CACHE_TTL) if USER_CACHE_TTL > 0 else None ) -EVENT_CACHE_TTL = _coerce_cache_ttl(base_config.get("AUTH_EVENT_CACHE_TTL", 2), 2) +EVENT_CACHE_TTL = AUTH_EVENT_CACHE_TTL EVENT_CACHE = ( CacheDict("AUTH_EVENT_CACHE", expire=EVENT_CACHE_TTL) if EVENT_CACHE_TTL > 0 @@ -145,15 +99,72 @@ _ROUTE_INDEX_READY = False _ROUTE_COMMAND_MAP: dict[str, set[str]] = {} _ROUTE_PREFIX_MAP: dict[str, set[str]] = {} _ROUTE_MODULES_WITH_COMMANDS: set[str] = set() +MATCHER_ROUTE_PREFILTER_TTL = 2 +PREFILTER_STATS_LOG_INTERVAL = 10.0 +CACHE_SWEEP_INTERVAL = 1.0 + +CPU_COUNT = os.cpu_count() or 4 +COMMAND_MATCHER_CONCURRENCY = max(8, min(48, CPU_COUNT * 4)) +HEAVY_COMMAND_CONCURRENCY = max(1, min(3, CPU_COUNT // 2)) +HEAVY_COMMAND_MODULES = frozenset({"shop", "sign_in"}) # 全局信号量与计数器 -HOOKS_SEMAPHORE = asyncio.Semaphore(HOOKS_CONCURRENCY_LIMIT) HOOKS_ACTIVE_COUNT = 0 HOOKS_ACTIVE_LOCK = asyncio.Lock() +HOOKS_SEMAPHORE = asyncio.Semaphore(HOOKS_CONCURRENCY_LIMIT) +COMMAND_MATCHER_SEMAPHORE = asyncio.Semaphore(COMMAND_MATCHER_CONCURRENCY) +HEAVY_COMMAND_SEMAPHORE = asyncio.Semaphore(HEAVY_COMMAND_CONCURRENCY) DB_SEMAPHORE = asyncio.Semaphore(DB_CONCURRENCY_LIMIT) DB_ACTIVE_COUNT = 0 DB_ACTIVE_LOCK = asyncio.Lock() +_CHECK_MATCHER_PATCHED = False +_ORIGINAL_CHECK_AND_RUN_MATCHER: Callable[..., Awaitable[None]] | None = None +_MATCHER_COMMAND_TYPE_CACHE: dict[type[Matcher], bool] = {} +_MATCHER_COMMAND_LITERAL_CACHE: dict[type[Matcher], tuple[str, ...] | None] = {} +_MATCHER_ALCONNA_SHORTCUT_CACHE: dict[type[Matcher], bool] = {} +_CHECK_MATCHER_ROUTE_CACHE = CacheDict( + "AUTH_MATCHER_ROUTE_CACHE", expire=MATCHER_ROUTE_PREFILTER_TTL +) +_PREFILTER_STATS = { + "checked": 0, + "skipped": 0, + "type_miss": 0, + "route_miss": 0, + "command_miss": 0, + "empty_text": 0, +} +_PREFILTER_LAST_LOG = 0.0 +_CACHE_SWEEP_TASK: asyncio.Task | None = None + + +class HookTraceRecorder: + def __init__(self, start_time: float) -> None: + self._start_time = start_time + self._enabled = False + self._data: dict[str, str] = {} + + def _ensure_enabled(self) -> bool: + if self._enabled: + return True + if time.time() - self._start_time <= WARNING_THRESHOLD: + return False + self._enabled = True + return True + + def set(self, key: str, value: str) -> None: + if self._ensure_enabled(): + self._data[key] = value + + def setdefault(self, key: str, value: str) -> None: + if self._ensure_enabled(): + self._data.setdefault(key, value) + + def contains(self, key: str) -> bool: + return key in self._data + + def snapshot(self) -> dict[str, str]: + return self._data if self._enabled else {} def _cache_get(cache: CacheDict | None, key: str): @@ -204,15 +215,69 @@ def _get_event_cache(event: Event, session: Uninfo, entity): def _normalize_command(command: str) -> str: - return command.strip() + text = command.strip() + if not text: + return "" + + # strip leading placeholders like "[引用消息] 撤回" + text = re.sub(r"^(?:\s*(?:\[[^\]]*]|\<[^>]*>))+\s*", "", text) + + # keep command head: "点歌 [歌名]" -> "点歌", "foo " -> "foo" + cut_points = [idx for idx in (text.find("["), text.find("<")) if idx >= 0] + if cut_points: + text = text[: min(cut_points)] + + # normalize spacing after trimming placeholders + text = re.sub(r"\s+", " ", text).strip() + # remove trailing template markers left by forms like "foo ?[arg]" / "foo ?*[tags]" + text = re.sub(r"(?:\s+[?*]+|[?*]+)$", "", text).strip() + return text -def _extract_commands(extra: PluginExtraData | None) -> set[str]: +def _split_command_variants(command: str) -> tuple[str, ...]: + text = command.strip() + if not text: + return () + # Keep slash-prefixed commands like "/info" as-is. + if text.startswith("/"): + return (text,) + # "今日运势/抽签/运势" => ("今日运势", "抽签", "运势") + if "/" in text and " " not in text: + parts = tuple(part.strip() for part in text.split("/") if part.strip()) + if parts: + return parts + return (text,) + + +def _is_ambiguous_route_command(command: str) -> bool: + text = command.strip() + if not text: + return True + # Keep route-index strict only for literal, deterministic command heads. + if any(token in text for token in ("?", "*", "|", "(", ")", "^", "$", "re:")): + return True + if "xx" in text.lower(): + return True + return False + + +def _extract_commands(extra: PluginExtraData | None) -> tuple[set[str], bool]: if not extra: - return set() + return set(), False commands = {c.command for c in extra.commands if c.command} commands.update(extra.aliases or set()) - return {cmd.strip() for cmd in commands if cmd and cmd.strip()} + normalized_commands: set[str] = set() + has_ambiguous = False + for command in commands: + normalized = _normalize_command(command) + if not normalized: + continue + for variant in _split_command_variants(normalized): + if _is_ambiguous_route_command(variant): + has_ambiguous = True + continue + normalized_commands.add(variant) + return normalized_commands, has_ambiguous async def _ensure_route_index(): @@ -233,15 +298,14 @@ async def _ensure_route_index(): extra_data = PluginExtraData(**extra) except Exception: continue - command_set = _extract_commands(extra_data) + command_set, has_ambiguous = _extract_commands(extra_data) if not command_set: continue + if has_ambiguous: + continue module = plugin.name _ROUTE_MODULES_WITH_COMMANDS.add(module) - for command in command_set: - normalized = _normalize_command(command) - if not normalized: - continue + for normalized in command_set: _ROUTE_COMMAND_MAP.setdefault(normalized, set()).add(module) _ROUTE_PREFIX_MAP.setdefault(normalized[0], set()).add(normalized) _ROUTE_INDEX_READY = True @@ -276,14 +340,368 @@ def _match_route_modules(text: str) -> set[str]: return matched_modules -def _get_message_text(message: UniMsg, event_cache: dict | None) -> str: - if event_cache is None: - return message.extract_plain_text() - cached = event_cache.get("plain_text") - if cached is None: - cached = message.extract_plain_text() - event_cache["plain_text"] = cached - return cached +def _matcher_module_name(matcher_cls: type[Matcher]) -> str: + module = getattr(matcher_cls, "plugin_name", "") or "" + if module: + return module + plugin = getattr(matcher_cls, "plugin", None) + if not plugin: + return "" + return (getattr(plugin, "name", "") or "").strip() + + +def _is_command_matcher_class(matcher_cls: type[Matcher]) -> bool: + if matcher_cls in _MATCHER_COMMAND_TYPE_CACHE: + return _MATCHER_COMMAND_TYPE_CACHE[matcher_cls] + if hasattr(matcher_cls, "command"): + _MATCHER_COMMAND_TYPE_CACHE[matcher_cls] = True + return True + rule = getattr(matcher_cls, "rule", None) + checkers = getattr(rule, "checkers", ()) or () + for checker in checkers: + call = getattr(checker, "call", None) + if call is None: + continue + call_type = call.__class__ + call_module = getattr(call_type, "__module__", "") + call_name = getattr(call_type, "__name__", "") + if call_module.startswith("nonebot.rule") and call_name in { + "CommandRule", + "ShellCommandRule", + "Command", + "ShellCommand", + }: + _MATCHER_COMMAND_TYPE_CACHE[matcher_cls] = True + return True + if ( + call_module.startswith("nonebot_plugin_alconna.rule") + and call_name == "AlconnaRule" + ): + _MATCHER_COMMAND_TYPE_CACHE[matcher_cls] = True + return True + _MATCHER_COMMAND_TYPE_CACHE[matcher_cls] = False + return False + + +def _matcher_route_cache_key(event: Event) -> str: + msg_id = getattr(event, "message_id", None) + if msg_id is None: + msg_id = getattr(event, "id", None) + if msg_id is None: + msg_id = id(event) + user_id = getattr(event, "user_id", "") + group_id = getattr(event, "group_id", "") + channel_id = getattr(event, "channel_id", "") + return f"{msg_id}:{user_id}:{group_id}:{channel_id}" + + +def _event_plain_text(event: Event) -> str: + with contextlib.suppress(Exception): + return (event.get_plaintext() or "").strip() + return "" + + +def _state_plain_text(state: dict | None) -> str: + if state is None: + return "" + text = state.get("_zx_plain_text") + if isinstance(text, str): + return text.strip() + return "" + + +def _get_route_modules_for_event(event: Event, state: dict | None = None) -> set[str]: + if state is not None: + route_modules = state.get("_zx_route_modules") + if isinstance(route_modules, set): + return route_modules + key = _matcher_route_cache_key(event) + try: + route_modules = _CHECK_MATCHER_ROUTE_CACHE[key] + except KeyError: + route_modules = _match_route_modules(_event_plain_text(event)) + _CHECK_MATCHER_ROUTE_CACHE[key] = route_modules + if state is not None: + state["_zx_route_modules"] = route_modules + return route_modules + + +def _record_prefilter_stats(skipped: bool, reason: str | None) -> None: + global _PREFILTER_LAST_LOG + _PREFILTER_STATS["checked"] += 1 + if skipped: + _PREFILTER_STATS["skipped"] += 1 + if reason == "type_miss": + _PREFILTER_STATS["type_miss"] += 1 + elif reason == "route_miss": + _PREFILTER_STATS["route_miss"] += 1 + elif reason == "command_miss": + _PREFILTER_STATS["command_miss"] += 1 + elif reason == "empty_text": + _PREFILTER_STATS["empty_text"] += 1 + + if _PREFILTER_STATS["checked"] % 1024 == 0: + with contextlib.suppress(Exception): + _ = len(_CHECK_MATCHER_ROUTE_CACHE) + + now = time.monotonic() + if now - _PREFILTER_LAST_LOG < PREFILTER_STATS_LOG_INTERVAL or is_overloaded(): + return + _PREFILTER_LAST_LOG = now + _debug_log( + ( + "matcher prefilter stats: " + f"checked={_PREFILTER_STATS['checked']} " + f"skipped={_PREFILTER_STATS['skipped']} " + f"type_miss={_PREFILTER_STATS['type_miss']} " + f"route_miss={_PREFILTER_STATS['route_miss']} " + f"command_miss={_PREFILTER_STATS['command_miss']} " + f"empty_text={_PREFILTER_STATS['empty_text']}" + ), + LOGGER_COMMAND, + ) + + +def _collect_command_literals(value, target: set[str], depth: int = 0) -> None: + if depth > 3 or value is None: + return + if isinstance(value, str): + text = value.strip() + if text: + target.add(text) + return + if isinstance(value, list | tuple | set | frozenset): + for item in value: + _collect_command_literals(item, target, depth + 1) + return + for attr in ("command", "commands", "cmd", "cmds"): + nested = getattr(value, attr, None) + if nested is not None and nested is not value: + _collect_command_literals(nested, target, depth + 1) + + +def _extract_matcher_command_literals( + matcher_cls: type[Matcher], +) -> tuple[str, ...] | None: + if matcher_cls in _MATCHER_COMMAND_LITERAL_CACHE: + return _MATCHER_COMMAND_LITERAL_CACHE[matcher_cls] + + commands: set[str] = set() + _collect_command_literals(getattr(matcher_cls, "command", None), commands) + + rule = getattr(matcher_cls, "rule", None) + checkers = getattr(rule, "checkers", ()) or () + for checker in checkers: + call = getattr(checker, "call", None) + if call is None: + continue + for attr in ("cmds", "command", "commands", "cmd"): + _collect_command_literals(getattr(call, attr, None), commands) + + if not commands: + _MATCHER_COMMAND_LITERAL_CACHE[matcher_cls] = None + return None + + sorted_commands = tuple(sorted(commands, key=len, reverse=True)) + _MATCHER_COMMAND_LITERAL_CACHE[matcher_cls] = sorted_commands + return sorted_commands + + +def _matcher_has_alconna_shortcuts(matcher_cls: type[Matcher]) -> bool: + cached = _MATCHER_ALCONNA_SHORTCUT_CACHE.get(matcher_cls) + if cached is not None: + return cached + + has_shortcuts = False + rule = getattr(matcher_cls, "rule", None) + checkers = getattr(rule, "checkers", ()) or () + for checker in checkers: + call = getattr(checker, "call", None) + if call is None: + continue + call_type = call.__class__ + call_module = getattr(call_type, "__module__", "") + call_name = getattr(call_type, "__name__", "") + if not ( + call_module.startswith("nonebot_plugin_alconna.rule") + and call_name == "AlconnaRule" + ): + continue + + # Alconna matcher supports shortcut-based parsing (regex/fuzzy expansion). + # Route prefilter only knows literal command heads, so shortcut matchers + # must bypass strict route miss to avoid false negative skips. + command_ref = getattr(call, "command", None) + command = None + if callable(command_ref): + with contextlib.suppress(Exception): + command = command_ref() + if command is not None: + get_shortcuts = getattr(command, "get_shortcuts", None) + if callable(get_shortcuts): + shortcuts = get_shortcuts() + if shortcuts: + has_shortcuts = True + break + formatter = getattr(command, "formatter", None) + if formatter is not None: + with contextlib.suppress(Exception): + data = getattr(formatter, "data", None) + if isinstance(data, dict): + for trace in data.values(): + if getattr(trace, "shortcuts", None): + has_shortcuts = True + break + if has_shortcuts: + break + + _MATCHER_ALCONNA_SHORTCUT_CACHE[matcher_cls] = has_shortcuts + return has_shortcuts + + +def _is_heavy_command_module(module: str) -> bool: + normalized = module.strip().lower() + if not normalized: + return False + if normalized in HEAVY_COMMAND_MODULES: + return True + return any(normalized.endswith(f".{name}") for name in HEAVY_COMMAND_MODULES) + + +async def _check_matcher_prefilter( + matcher_cls: type[Matcher], event: Event, state: dict | None = None +) -> tuple[bool, str | None]: + event_type = event.get_type() + matcher_type = getattr(matcher_cls, "type", "") or "" + if isinstance(matcher_type, str) and matcher_type and matcher_type != event_type: + # Explicit matcher type mismatch cannot match this event. + return True, "type_miss" + + if event_type != "message": + return False, None + + # Session continuation matchers generated by pause/reject are temp=True. + # They must bypass command-route prefilter, otherwise follow-up messages + # (e.g. got_path waiting for plain text) will be dropped. + if getattr(matcher_cls, "temp", False): + return False, None + + is_command_matcher = _is_command_matcher_class(matcher_cls) + if not is_command_matcher: + return False, None + + text = _state_plain_text(state) + if is_command_matcher and not text: + text = _event_plain_text(event) + if state is not None and text: + state["_zx_plain_text"] = text + if is_command_matcher and not text: + return True, "empty_text" + + module = _matcher_module_name(matcher_cls) + if not module: + return False, None + + if not _ROUTE_INDEX_READY: + await _ensure_route_index() + + if module not in _ROUTE_MODULES_WITH_COMMANDS: + matcher_commands = _extract_matcher_command_literals(matcher_cls) + if matcher_commands: + for command in matcher_commands: + if _command_matches(text, command): + return False, None + if _matcher_has_alconna_shortcuts(matcher_cls): + return False, None + return True, "command_miss" + return False, None + + route_modules = _get_route_modules_for_event(event, state) + if module not in route_modules: + if _matcher_has_alconna_shortcuts(matcher_cls): + return False, None + return True, "route_miss" + return False, None + + +async def _patched_check_and_run_matcher( + Matcher: type[Matcher], + bot: Bot, + event: Event, + state: dict, + stack=None, + dependency_cache=None, +) -> None: + skip, reason = await _check_matcher_prefilter( + Matcher, event, state if isinstance(state, dict) else None + ) + _record_prefilter_stats(skip, reason) + if skip: + return + + original = _ORIGINAL_CHECK_AND_RUN_MATCHER + if not original: + return + kwargs = { + "Matcher": Matcher, + "bot": bot, + "event": event, + "state": state, + "stack": stack, + "dependency_cache": dependency_cache, + } + if _is_command_matcher_class(Matcher): + module = _matcher_module_name(Matcher) + if _is_heavy_command_module(module): + async with HEAVY_COMMAND_SEMAPHORE: + await original(**kwargs) + return + async with COMMAND_MATCHER_SEMAPHORE: + await original(**kwargs) + return + await original(**kwargs) + + +def _install_matcher_prefilter() -> None: + global _CHECK_MATCHER_PATCHED, _ORIGINAL_CHECK_AND_RUN_MATCHER + if _CHECK_MATCHER_PATCHED: + return + _ORIGINAL_CHECK_AND_RUN_MATCHER = nb_message.check_and_run_matcher + nb_message.check_and_run_matcher = _patched_check_and_run_matcher # type: ignore[assignment] + _CHECK_MATCHER_PATCHED = True + + +def _uninstall_matcher_prefilter() -> None: + global _CHECK_MATCHER_PATCHED, _ORIGINAL_CHECK_AND_RUN_MATCHER + if not _CHECK_MATCHER_PATCHED: + return + if _ORIGINAL_CHECK_AND_RUN_MATCHER is not None: + nb_message.check_and_run_matcher = _ORIGINAL_CHECK_AND_RUN_MATCHER # type: ignore[assignment] + _CHECK_MATCHER_PATCHED = False + _ORIGINAL_CHECK_AND_RUN_MATCHER = None + + +def _get_message_text( + message: UniMsg | None, + event_cache: dict | None, + event: Event | None = None, +) -> str: + if event_cache is not None: + cached = event_cache.get("plain_text") + if isinstance(cached, str): + return cached + + text = "" + if message is not None: + with contextlib.suppress(Exception): + text = message.extract_plain_text() + if not text and event is not None: + with contextlib.suppress(Exception): + text = (event.get_plaintext() or "").strip() + + if event_cache is not None: + event_cache["plain_text"] = text + return text async def _get_route_context(text: str, event_cache: dict | None) -> set[str]: @@ -298,6 +716,34 @@ async def _get_route_context(text: str, event_cache: dict | None) -> set[str]: return matched +async def _cache_sweep_loop() -> None: + while True: + await asyncio.sleep(CACHE_SWEEP_INTERVAL) + with contextlib.suppress(Exception): + if EVENT_CACHE is not None: + _ = len(EVENT_CACHE) + _ = len(_CHECK_MATCHER_ROUTE_CACHE) + + +async def start_auth_runtime_tasks() -> None: + global _CACHE_SWEEP_TASK + await _ensure_route_index() + _install_matcher_prefilter() + if _CACHE_SWEEP_TASK is None or _CACHE_SWEEP_TASK.done(): + _CACHE_SWEEP_TASK = asyncio.create_task(_cache_sweep_loop()) + + +async def stop_auth_runtime_tasks() -> None: + global _CACHE_SWEEP_TASK + _uninstall_matcher_prefilter() + task = _CACHE_SWEEP_TASK + _CACHE_SWEEP_TASK = None + if task is not None: + task.cancel() + with contextlib.suppress(BaseException): + await task + + async def _has_limits_cached(module: str, event_cache: dict | None) -> bool: module_limit_cache: dict[str, bool] = {} if event_cache is not None: @@ -601,46 +1047,55 @@ async def reduce_gold(user_id: str, module: str, cost_gold: int, session: Uninfo # 辅助函数，用于记录每个 hook 的执行时间 -async def time_hook(coro, name, time_dict): +async def time_hook(coro, name, recorder: HookTraceRecorder | None = None): start = time.time() try: # 检查熔断状态 if check_circuit_breaker(name): logger.info(f"{name} 熔断器激活中，跳过执行", LOGGER_COMMAND) - time_dict[name] = "熔断跳过" + if recorder is not None: + recorder.set(name, "熔断跳过") return # 添加超时控制 return await with_timeout(coro, name=name) except asyncio.TimeoutError: - time_dict[name] = f"超时 (>{TIMEOUT_SECONDS}s)" + if recorder is not None: + recorder.set(name, f"超时 (>{TIMEOUT_SECONDS}s)") finally: - if name not in time_dict: - time_dict[name] = f"{time.time() - start:.3f}s" + if recorder is not None and not recorder.contains(name): + recorder.set(name, f"{time.time() - start:.3f}s") async def _enter_hooks_section(): """尝试获取全局信号量并更新计数器，超时则抛出 PermissionExemption。""" global HOOKS_ACTIVE_COUNT - # 队列模式：如果达到上限，协程将排队等待直到获取到信号量 await HOOKS_SEMAPHORE.acquire() async with HOOKS_ACTIVE_LOCK: HOOKS_ACTIVE_COUNT += 1 - _debug_log(f"当前并发权限检查数量: {HOOKS_ACTIVE_COUNT}", LOGGER_COMMAND) + _debug_log( + ( + "当前并发权限检查数量: " + f"{HOOKS_ACTIVE_COUNT}, limit={HOOKS_CONCURRENCY_LIMIT}" + ), + LOGGER_COMMAND, + ) async def _leave_hooks_section(): """释放信号量并更新计数器。""" global HOOKS_ACTIVE_COUNT - from contextlib import suppress - - with suppress(Exception): + with contextlib.suppress(Exception): HOOKS_SEMAPHORE.release() async with HOOKS_ACTIVE_LOCK: - HOOKS_ACTIVE_COUNT -= 1 - # 保证计数不为负 - HOOKS_ACTIVE_COUNT = max(HOOKS_ACTIVE_COUNT, 0) - _debug_log(f"当前并发权限检查数量: {HOOKS_ACTIVE_COUNT}", LOGGER_COMMAND) + HOOKS_ACTIVE_COUNT = max(HOOKS_ACTIVE_COUNT - 1, 0) + _debug_log( + ( + "当前并发权限检查数量: " + f"{HOOKS_ACTIVE_COUNT}, limit={HOOKS_CONCURRENCY_LIMIT}" + ), + LOGGER_COMMAND, + ) async def auth_ban_fast( @@ -671,19 +1126,31 @@ async def route_precheck( matcher: Matcher, event: Event, session: Uninfo, - message: UniMsg, + message: UniMsg | None, + *, + entity=None, + event_cache: dict | None = None, + text: str | None = None, + route_modules: set[str] | None = None, ) -> bool: module = matcher.plugin_name or "" if not module: return False if _is_hidden_plugin(matcher): return False - entity = get_entity_ids(session) - event_cache = _get_event_cache(event, session, entity) - text = _get_message_text(message, event_cache) - route_modules = await _get_route_context(text, event_cache) - await _ensure_route_index() + if not _is_command_matcher_class(type(matcher)): + return False + if entity is None: + entity = get_entity_ids(session) + if event_cache is None: + event_cache = _get_event_cache(event, session, entity) + if text is None: + text = _get_message_text(message, event_cache, event) + if route_modules is None: + route_modules = await _get_route_context(text, event_cache) if module in _ROUTE_MODULES_WITH_COMMANDS and module not in route_modules: + if _matcher_has_alconna_shortcuts(type(matcher)): + return False if event_cache is not None: event_cache["route_skip"] = True return True @@ -721,14 +1188,36 @@ async def auth_precheck( await auth_admin(plugin, session, cached_levels=levels) +async def _call_auth_ban_compat( + matcher: Matcher, + bot: Bot, + session: Uninfo, + plugin: PluginInfo, + *, + entity, +) -> None: + """兼容旧签名 auth_ban(matcher, bot, session, plugin)。""" + try: + await auth_ban(matcher, bot, session, plugin, entity=entity) + except TypeError as exc: + if "unexpected keyword argument 'entity'" not in str(exc): + raise + await auth_ban(matcher, bot, session, plugin) + + async def auth( matcher: Matcher, event: Event, bot: Bot, session: Uninfo, - message: UniMsg, + message: UniMsg | None, *, skip_ban: bool = False, + entity=None, + event_cache: dict | None = None, + text: str | None = None, + route_modules: set[str] | None = None, + is_superuser: bool | None = None, ): """权限检查 @@ -742,15 +1231,20 @@ async def auth( start_time = time.time() cost_gold = 0 ignore_flag = False - entity = get_entity_ids(session) + if entity is None: + entity = get_entity_ids(session) + if is_superuser is None: + is_superuser = session.user.id in bot.config.superusers module = matcher.plugin_name or "" - event_cache = _get_event_cache(event, session, entity) + is_command_matcher = _is_command_matcher_class(type(matcher)) + if event_cache is None: + event_cache = _get_event_cache(event, session, entity) auth_allowed = None auth_result_cache = None admin_checked_pre = False - # 用于记录各个 hook 的执行时间 - hook_times = {} + # 仅在慢请求时记录 hook 明细，避免热路径高频构造字符串 + hook_recorder = HookTraceRecorder(start_time) hooks_time = 0 # 初始化 hooks_time 变量 # 记录是否已进入 hooks 区域（用于 finally 中释放） @@ -758,7 +1252,8 @@ async def auth( try: if not module: - raise PermissionExemption("Matcher插件名称不存在...") + auth_allowed = True + return if event_cache is not None: auth_result_cache = event_cache.setdefault("auth_result", {}) @@ -770,20 +1265,25 @@ async def auth( return if _is_hidden_plugin(matcher): - raise PermissionExemption(f"plugin {module} hidden, skip") + auth_allowed = True + return if event_cache is not None and event_cache.get("ban_state") is True: raise SkipPluginException("user or group banned (cached)") - text = _get_message_text(message, event_cache) - route_modules = await _get_route_context(text, event_cache) - await _ensure_route_index() + if text is None: + text = _get_message_text(message, event_cache, event) + if route_modules is None: + route_modules = await _get_route_context(text, event_cache) route_skip_checks = ( - module in _ROUTE_MODULES_WITH_COMMANDS and module not in route_modules + is_command_matcher + and module in _ROUTE_MODULES_WITH_COMMANDS + and module not in route_modules + and not _matcher_has_alconna_shortcuts(type(matcher)) ) if route_skip_checks: if event_cache is not None: event_cache["route_skip"] = True - hook_times["route"] = "miss" + hook_recorder.set("route", "miss") auth_allowed = True return @@ -801,22 +1301,25 @@ async def auth( ), name="get_plugin_and_user", ) - hook_times["get_plugin_user"] = f"{time.time() - plugin_user_start:.3f}s" + hook_recorder.set( + "get_plugin_user", f"{time.time() - plugin_user_start:.3f}s" + ) except asyncio.TimeoutError: logger.error( f"获取插件和用户数据超时，模块: {module}", LOGGER_COMMAND, session=session, ) - raise PermissionExemption("获取插件和用户数据超时，请稍后再试...") + auth_allowed = True + return if not route_skip_checks and _needs_admin_check(plugin): if plugin.plugin_type in { PluginType.SUPERUSER, PluginType.SUPER_AND_ADMIN, }: - if session.user.id in bot.config.superusers: - hook_times["auth_admin"] = "superuser" + if is_superuser: + hook_recorder.set("auth_admin", "superuser") admin_checked_pre = True elif plugin.plugin_type == PluginType.SUPERUSER: raise SkipPluginException("超级管理员权限不足...") @@ -829,11 +1332,13 @@ async def auth( session, entity, event_cache ) if admin_timeout: - hook_times["auth_admin"] = "timeout" + hook_recorder.set("auth_admin", "timeout") else: admin_start = time.time() await auth_admin(plugin, session, cached_levels=admin_levels) - hook_times["auth_admin"] = f"{time.time() - admin_start:.3f}s(pre)" + hook_recorder.set( + "auth_admin", f"{time.time() - admin_start:.3f}s(pre)" + ) admin_checked_pre = True ban_cache_state = None @@ -841,40 +1346,44 @@ async def auth( ban_cache_state = event_cache.get("ban_state") if skip_ban: if ban_cache_state is True: - hook_times["auth_ban"] = "cached" + hook_recorder.set("auth_ban", "cached") raise SkipPluginException("user or group banned (cached)") if ban_cache_state is None: ban_start = time.time() try: - await auth_ban(matcher, bot, session, plugin) - hook_times["auth_ban"] = f"{time.time() - ban_start:.3f}s" + await _call_auth_ban_compat( + matcher, bot, session, plugin, entity=entity + ) + hook_recorder.set("auth_ban", f"{time.time() - ban_start:.3f}s") if event_cache is not None: event_cache["ban_state"] = False except SkipPluginException: - hook_times["auth_ban"] = f"{time.time() - ban_start:.3f}s" + hook_recorder.set("auth_ban", f"{time.time() - ban_start:.3f}s") if event_cache is not None: event_cache["ban_state"] = True raise else: - hook_times["auth_ban"] = "skipped" + hook_recorder.set("auth_ban", "skipped") else: if ban_cache_state is True: - hook_times["auth_ban"] = "cached" + hook_recorder.set("auth_ban", "cached") raise SkipPluginException("user or group banned (cached)") if ban_cache_state is None: ban_start = time.time() try: - await auth_ban(matcher, bot, session, plugin) - hook_times["auth_ban"] = f"{time.time() - ban_start:.3f}s" + await _call_auth_ban_compat( + matcher, bot, session, plugin, entity=entity + ) + hook_recorder.set("auth_ban", f"{time.time() - ban_start:.3f}s") if event_cache is not None: event_cache["ban_state"] = False except SkipPluginException: - hook_times["auth_ban"] = f"{time.time() - ban_start:.3f}s" + hook_recorder.set("auth_ban", f"{time.time() - ban_start:.3f}s") if event_cache is not None: event_cache["ban_state"] = True raise else: - hook_times["auth_ban"] = "cached" + hook_recorder.set("auth_ban", "cached") # 获取插件费用 if not route_skip_checks and plugin.cost_gold > 0: @@ -883,14 +1392,14 @@ async def auth( cost_gold = await with_timeout( get_plugin_cost(bot, user, plugin, session), name="get_plugin_cost" ) - hook_times["cost_gold"] = f"{time.time() - cost_start:.3f}s" + hook_recorder.set("cost_gold", f"{time.time() - cost_start:.3f}s") except asyncio.TimeoutError: logger.error( f"获取插件费用超时，模块: {module}", LOGGER_COMMAND, session=session ) # 继续执行，不阻止权限检查 else: - hook_times["cost_gold"] = "skipped" + hook_recorder.set("cost_gold", "skipped") # 执行 bot_filter bot_filter(session) @@ -921,11 +1430,11 @@ async def auth( hook_tasks = [] if event_cache is None: hook_tasks.append( - time_hook(auth_bot(plugin, bot.self_id), "auth_bot", hook_times) + time_hook(auth_bot(plugin, bot.self_id), "auth_bot", hook_recorder) ) else: if bot_timeout: - hook_times["auth_bot"] = "timeout" + hook_recorder.set("auth_bot", "timeout") else: hook_tasks.append( time_hook( @@ -936,42 +1445,42 @@ async def auth( skip_fetch=True, ), "auth_bot", - hook_times, + hook_recorder, ) ) - if session.user.id in bot.config.superusers: - hook_times["auth_group"] = "superuser" + if is_superuser: + hook_recorder.set("auth_group", "superuser") else: hook_tasks.append( time_hook( auth_group(plugin, group, text, entity.group_id), "auth_group", - hook_times, + hook_recorder, ) ) if not route_skip_checks and plugin.admin_level and not admin_checked_pre: if event_cache is None: hook_tasks.append( - time_hook(auth_admin(plugin, session), "auth_admin", hook_times) + time_hook(auth_admin(plugin, session), "auth_admin", hook_recorder) ) else: if admin_timeout: - hook_times["auth_admin"] = "timeout" + hook_recorder.set("auth_admin", "timeout") else: hook_tasks.append( time_hook( auth_admin(plugin, session, cached_levels=admin_levels), "auth_admin", - hook_times, + hook_recorder, ) ) else: - hook_times.setdefault("auth_admin", "skipped") + hook_recorder.setdefault("auth_admin", "skipped") - if session.user.id in bot.config.superusers: - hook_times["auth_plugin"] = "superuser" + if is_superuser: + hook_recorder.set("auth_plugin", "superuser") elif not route_skip_checks and _needs_auth_plugin(plugin, group, entity): hook_tasks.append( time_hook( @@ -980,25 +1489,25 @@ async def auth( group, session, event, - skip_group_block=session.user.id in bot.config.superusers, + skip_group_block=is_superuser, ), "auth_plugin", - hook_times, + hook_recorder, ) ) else: - hook_times["auth_plugin"] = "skipped" + hook_recorder.set("auth_plugin", "skipped") if not route_skip_checks: has_limits = await _has_limits_cached(module, event_cache) if has_limits: hook_tasks.append( - time_hook(auth_limit(plugin, session), "auth_limit", hook_times) + time_hook(auth_limit(plugin, session), "auth_limit", hook_recorder) ) else: - hook_times["auth_limit"] = "skipped" + hook_recorder.set("auth_limit", "skipped") else: - hook_times["auth_limit"] = "skipped" + hook_recorder.set("auth_limit", "skipped") if hook_tasks: # 进入 hooks 并行检查区域（会在高并发时排队） @@ -1055,7 +1564,7 @@ async def auth( reduce_gold(entity.user_id, module, cost_gold, session), name="reduce_gold", ) - hook_times["reduce_gold"] = f"{time.time() - gold_start:.3f}s" + hook_recorder.set("reduce_gold", f"{time.time() - gold_start:.3f}s") except asyncio.TimeoutError: logger.error( f"扣除金币超时，模块: {module}", LOGGER_COMMAND, session=session @@ -1067,7 +1576,7 @@ async def auth( logger.warning( f"权限检查耗时过长: {total_time:.3f}s, 模块: {module}, " f"hooks时间: {hooks_time:.3f}s, " - f"详情: {hook_times}", + f"详情: {hook_recorder.snapshot()}", LOGGER_COMMAND, session=session, ) diff --git a/zhenxun/builtin_plugins/hooks/auth_hook.py b/zhenxun/builtin_plugins/hooks/auth_hook.py index b0458aea..911da775 100644 --- a/zhenxun/builtin_plugins/hooks/auth_hook.py +++ b/zhenxun/builtin_plugins/hooks/auth_hook.py @@ -1,4 +1,5 @@ import asyncio +import contextlib import time from nonebot import get_driver @@ -6,33 +7,39 @@ from nonebot.adapters import Bot, Event from nonebot.exception import IgnoredException from nonebot.matcher import Matcher from nonebot.message import event_preprocessor, run_postprocessor, run_preprocessor +from nonebot.typing import T_State from nonebot_plugin_alconna import UniMsg from nonebot_plugin_uninfo import Uninfo from zhenxun.services.cache.runtime_cache import is_cache_ready from zhenxun.services.log import logger from zhenxun.services.message_load import is_overloaded +from zhenxun.services.runtime_bootstrap import register_runtime_bootstrap from zhenxun.utils.utils import get_entity_ids from .auth.config import LOGGER_COMMAND from .auth_checker import ( LimitManager, _get_event_cache, + _get_route_context, auth, route_precheck, + start_auth_runtime_tasks, + stop_auth_runtime_tasks, ) _SKIP_AUTH_PLUGINS = {"chat_history", "chat_message"} _BOT_CONNECT_TS: float | None = None _AUTH_QUEUE_MAXSIZE = 200 -_AUTH_QUEUE: asyncio.Queue[tuple[Matcher, Event, Bot, Uninfo, UniMsg]] = asyncio.Queue( - maxsize=_AUTH_QUEUE_MAXSIZE +_AUTH_QUEUE: asyncio.Queue[tuple[Matcher, Event, Bot, Uninfo, UniMsg | None]] = ( + asyncio.Queue(maxsize=_AUTH_QUEUE_MAXSIZE) ) _AUTH_QUEUE_STARTED = False _AUTH_WORKERS: list[asyncio.Task] = [] _LAST_DROP_LOG = 0.0 driver = get_driver() +register_runtime_bootstrap(driver) @driver.on_bot_connect @@ -63,6 +70,17 @@ async def _auth_worker(worker_id: int) -> None: _AUTH_QUEUE.task_done() +def _extract_plain_text(message: UniMsg | None, event: Event) -> str: + if message is not None: + with contextlib.suppress(Exception): + return message.extract_plain_text() + with contextlib.suppress(Exception): + plain = event.get_plaintext() + if plain: + return plain.strip() + return "" + + @driver.on_startup async def _start_auth_queue(): global _AUTH_QUEUE_STARTED @@ -72,6 +90,26 @@ async def _start_auth_queue(): worker_count = max(1, min(6, _AUTH_QUEUE_MAXSIZE // 50)) for idx in range(worker_count): _AUTH_WORKERS.append(asyncio.create_task(_auth_worker(idx))) + await start_auth_runtime_tasks() + + +@driver.on_shutdown +async def _stop_auth_queue(): + global _AUTH_QUEUE_STARTED + _AUTH_QUEUE_STARTED = False + workers = _AUTH_WORKERS.copy() + _AUTH_WORKERS.clear() + for task in workers: + task.cancel() + if workers: + await asyncio.gather(*workers, return_exceptions=True) + + while not _AUTH_QUEUE.empty(): + with contextlib.suppress(Exception): + _AUTH_QUEUE.get_nowait() + _AUTH_QUEUE.task_done() + + await stop_auth_runtime_tasks() def _skip_auth_for_plugin(matcher: Matcher) -> bool: @@ -84,6 +122,35 @@ def _skip_auth_for_plugin(matcher: Matcher) -> bool: return "chat_history" in module_name +def _resolve_actor_user_id(event: Event, fallback_user_id: str) -> str: + """优先使用事件发起者ID，避免 notice 场景 session.user 指向 bot 自身。""" + event_user_id = getattr(event, "user_id", None) + if event_user_id is None: + return fallback_user_id + event_user_id = str(event_user_id) + return event_user_id or fallback_user_id + + +def _resolve_event_group_id(event: Event, fallback_group_id: str | None) -> str | None: + """notice 场景 session.group 可能缺失，回退到事件上的 group_id。""" + event_group_id = getattr(event, "group_id", None) + if event_group_id is None: + return fallback_group_id + resolved = str(event_group_id) + return resolved or fallback_group_id + + +def _resolve_event_channel_id( + event: Event, fallback_channel_id: str | None +) -> str | None: + """频道场景回退到事件上的 channel_id。""" + event_channel_id = getattr(event, "channel_id", None) + if event_channel_id is None: + return fallback_channel_id + resolved = str(event_channel_id) + return resolved or fallback_channel_id + + @event_preprocessor async def _drop_message_before_cache_ready(event: Event): if event.get_type() != "message": @@ -98,15 +165,56 @@ async def _drop_message_before_cache_ready(event: Event): @run_preprocessor async def _auth_preprocessor( - matcher: Matcher, event: Event, bot: Bot, session: Uninfo, message: UniMsg + matcher: Matcher, + event: Event, + bot: Bot, + session: Uninfo, + state: T_State, + message: UniMsg | None = None, ): if event.get_type() == "message" and not is_cache_ready(): raise IgnoredException("cache not ready ignore") start_time = time.time() - entity = get_entity_ids(session) - _get_event_cache(event, session, entity) + entity = state.get("_zx_entity") + if entity is None: + entity = get_entity_ids(session) + entity.user_id = _resolve_actor_user_id(event, entity.user_id) + entity.group_id = _resolve_event_group_id(event, entity.group_id) + entity.channel_id = _resolve_event_channel_id(event, entity.channel_id) + state["_zx_entity"] = entity - if await route_precheck(matcher, event, session, message): + event_cache = state.get("_zx_event_cache") + if event_cache is None: + event_cache = _get_event_cache(event, session, entity) + state["_zx_event_cache"] = event_cache + + text = state.get("_zx_plain_text") + if text is None: + text = _extract_plain_text(message, event) + state["_zx_plain_text"] = text + if event_cache is not None: + event_cache["plain_text"] = text + + route_modules = state.get("_zx_route_modules") + if route_modules is None: + route_modules = await _get_route_context(text, event_cache) + state["_zx_route_modules"] = route_modules + + is_superuser = state.get("_zx_is_superuser") + if is_superuser is None: + is_superuser = entity.user_id in bot.config.superusers + state["_zx_is_superuser"] = is_superuser + + if await route_precheck( + matcher, + event, + session, + message, + entity=entity, + event_cache=event_cache, + text=text, + route_modules=route_modules, + ): return if _skip_auth_for_plugin(matcher): return @@ -119,6 +227,11 @@ async def _auth_preprocessor( session, message, skip_ban=False, + entity=entity, + event_cache=event_cache, + text=text, + route_modules=route_modules, + is_superuser=is_superuser, ) except IgnoredException: raise @@ -137,10 +250,10 @@ async def _auth_preprocessor( @run_postprocessor -async def _unblock_after_matcher(matcher: Matcher, session: Uninfo): - user_id = session.user.id - group_id = None - channel_id = None +async def _unblock_after_matcher(matcher: Matcher, session: Uninfo, event: Event): + user_id = _resolve_actor_user_id(event, session.user.id) + group_id = _resolve_event_group_id(event, None) + channel_id = _resolve_event_channel_id(event, None) if session.group: if session.group.parent: group_id = session.group.parent.id diff --git a/zhenxun/builtin_plugins/init/__init_cache.py b/zhenxun/builtin_plugins/init/__init_cache.py index 8938c0a0..5608d00e 100644 --- a/zhenxun/builtin_plugins/init/__init_cache.py +++ b/zhenxun/builtin_plugins/init/__init_cache.py @@ -29,6 +29,7 @@ def register_cache_types(): GroupPluginSetting, key_format="{group_id}_{plugin_name}_{key}", ) + CacheRegistry.register(CacheType.GROUP_PLUGIN_SETTINGS_VIEW, dict) CacheRegistry.register( CacheType.LEVEL, LevelUser, key_format="{user_id}_{group_id}" ) diff --git a/zhenxun/builtin_plugins/sign_in/__init__.py b/zhenxun/builtin_plugins/sign_in/__init__.py index 589b2312..3dd8acdb 100644 --- a/zhenxun/builtin_plugins/sign_in/__init__.py +++ b/zhenxun/builtin_plugins/sign_in/__init__.py @@ -44,6 +44,8 @@ __plugin_meta__ = PluginMetadata( commands=[ Command(command="签到"), Command(command="我的签到"), + Command(command="好感度排行"), + Command(command="好感度总排行"), Command(command="签到排行"), Command(command="签到总排行"), ], diff --git a/zhenxun/builtin_plugins/sign_in/utils.py b/zhenxun/builtin_plugins/sign_in/utils.py index 047e8ab3..4b611b7e 100644 --- a/zhenxun/builtin_plugins/sign_in/utils.py +++ b/zhenxun/builtin_plugins/sign_in/utils.py @@ -10,6 +10,7 @@ from nonebot_plugin_uninfo import Uninfo from zhenxun import ui from zhenxun.configs.config import BotConfig, Config +from zhenxun.configs.path_config import THEMES_PATH from zhenxun.models.sign_user import SignUser from zhenxun.services import avatar_service from zhenxun.utils.manager.priority_manager import PriorityLifecycle @@ -48,6 +49,39 @@ LG_MESSAGE = [ ] +def _get_sign_template_files() -> list[Path]: + theme_name = str(Config.get_config("UI", "THEME", "default") or "default") + files: list[Path] = [] + theme_candidates = [theme_name] + if theme_name != "default": + theme_candidates.append("default") + + for candidate in theme_candidates: + base = THEMES_PATH / candidate / "pages" / "builtin" / "sign" + for file_name in ("main.html", "style.css", "manifest.json"): + file_path = base / file_name + if file_path.exists(): + files.append(file_path) + return files + + +def _is_sign_card_cache_stale(card_file: Path) -> bool: + if not card_file.exists(): + return False + try: + card_mtime = card_file.stat().st_mtime + except OSError: + return True + + for template_file in _get_sign_template_files(): + try: + if template_file.stat().st_mtime > card_mtime: + return True + except OSError: + continue + return False + + @PriorityLifecycle.on_startup(priority=5) async def init_image(): SIGN_TODAY_CARD_PATH.mkdir(exist_ok=True, parents=True) @@ -86,13 +120,17 @@ async def get_card( card_file = SIGN_TODAY_CARD_PATH / file_name if card_file.exists(): - return card_file + if not _is_sign_card_cache_stale(card_file): + return card_file + card_file.unlink(missing_ok=True) if add_impression == -1: view_name = f"{user_id}_view_{date}.png" view_card_file = SIGN_TODAY_CARD_PATH / view_name if view_card_file.exists(): - return view_card_file + if not _is_sign_card_cache_stale(view_card_file): + return view_card_file + view_card_file.unlink(missing_ok=True) is_card_view = True return await _generate_html_card( @@ -171,7 +209,9 @@ async def _generate_html_card( card_file = SIGN_TODAY_CARD_PATH / file_name if card_file.exists(): - return card_file + if not _is_sign_card_cache_stale(card_file): + return card_file + card_file.unlink(missing_ok=True) impression = float(user.impression) user_console = await user.user_console @@ -283,7 +323,14 @@ async def _generate_html_card( "total_gold": total_gold, } - image_bytes = await ui.render_template("pages/builtin/sign", data=card_data) + image_bytes = await ui.render_template( + "pages/builtin/sign", + data=card_data, + clip_selector=".wrapper", + clip_padding=8, + disable_animations=True, + screenshot_scale="css", + ) async with aiofiles.open(card_file, "wb") as f: await f.write(image_bytes) diff --git a/zhenxun/builtin_plugins/superuser/super_help.py b/zhenxun/builtin_plugins/superuser/super_help.py index 2136790e..17a2af01 100644 --- a/zhenxun/builtin_plugins/superuser/super_help.py +++ b/zhenxun/builtin_plugins/superuser/super_help.py @@ -8,6 +8,7 @@ from zhenxun.services.help_service import create_plugin_help_image from zhenxun.services.log import logger from zhenxun.utils.enum import PluginType from zhenxun.utils.exception import EmptyError +from zhenxun.utils.manager.priority_manager import PriorityLifecycle from zhenxun.utils.message import MessageUtils __plugin_meta__ = PluginMetadata( @@ -32,6 +33,14 @@ async def build_html_help() -> bytes: ) +@PriorityLifecycle.on_startup(priority=15) +async def _prewarm_super_help_cache() -> None: + try: + await build_html_help() + except Exception as e: + logger.warning("预热超级用户帮助缓存失败", "超级用户帮助", e=e) + + _matcher = on_alconna( Alconna("超级用户帮助"), permission=SUPERUSER, diff --git a/zhenxun/builtin_plugins/web_ui/api/tabs/main/__init__.py b/zhenxun/builtin_plugins/web_ui/api/tabs/main/__init__.py index 36059101..e1963a5c 100644 --- a/zhenxun/builtin_plugins/web_ui/api/tabs/main/__init__.py +++ b/zhenxun/builtin_plugins/web_ui/api/tabs/main/__init__.py @@ -93,7 +93,7 @@ async def _(bot_id: str | None = None) -> Result[QueryCount]: @router.get( - "get_fg_count", + "/get_fg_count", dependencies=[authentication()], response_model=Result[dict[str, int]], response_class=JSONResponse, diff --git a/zhenxun/models/ban_console.py b/zhenxun/models/ban_console.py index 612d6660..f20d1bc2 100644 --- a/zhenxun/models/ban_console.py +++ b/zhenxun/models/ban_console.py @@ -4,6 +4,7 @@ from typing_extensions import Self from tortoise import fields +from zhenxun.services.cache import CacheException, CacheRegistry, CacheRoot from zhenxun.services.cache.runtime_cache import BanMemoryCache from zhenxun.services.data_access import DataAccess from zhenxun.services.db_context import Model @@ -42,6 +43,22 @@ class BanConsole(Model): """缓存键字段""" enable_lock: ClassVar[list[DbLockType]] = [DbLockType.CREATE, DbLockType.UPSERT] """开启锁""" + _cache_checked: ClassVar[bool] = False + + @classmethod + def _ensure_cache_registered(cls): + """兜底注册 BAN 缓存，避免启动时序导致的未注册问题。""" + if cls._cache_checked: + return + try: + CacheRoot.get_model(CacheType.BAN) + except CacheException: + CacheRegistry.register( + CacheType.BAN, + cls, + key_format="{user_id}_{group_id}", + ) + cls._cache_checked = True @classmethod async def create(cls, *args, **kwargs) -> Self: @@ -69,6 +86,7 @@ class BanConsole(Model): 返回: Self | None: Self """ + cls._ensure_cache_registered() if not user_id and not group_id: raise UserAndGroupIsNone() dao = DataAccess(cls) @@ -153,18 +171,21 @@ class BanConsole(Model): f"封禁用户/群组，等级:{ban_level}，时长: {duration}", target=f"{group_id}:{user_id}", ) - target = await cls._get_data(user_id, group_id) - if target: - await cls.unban(user_id, group_id) - await cls.create( + if not user_id and not group_id: + raise UserAndGroupIsNone() + cls._ensure_cache_registered() + target, _ = await cls.update_or_create( user_id=user_id, group_id=group_id, - ban_level=ban_level, - ban_time=int(time.time()), - ban_reason=reason, - duration=duration, - operator=operator or 0, + defaults={ + "ban_level": ban_level, + "ban_time": int(time.time()), + "ban_reason": reason, + "duration": duration, + "operator": operator or 0, + }, ) + await BanMemoryCache.upsert_from_model(target) @classmethod async def unban(cls, user_id: str | None, group_id: str | None = None) -> bool: diff --git a/zhenxun/services/avatar_service.py b/zhenxun/services/avatar_service.py index 3ced2bfe..8e46ac67 100644 --- a/zhenxun/services/avatar_service.py +++ b/zhenxun/services/avatar_service.py @@ -4,6 +4,7 @@ 提供一个统一的、带缓存的头像获取服务，支持多平台和可配置的过期策略。 """ +from collections import OrderedDict import os from pathlib import Path import time @@ -47,10 +48,12 @@ class AvatarService: 一个集中式的头像缓存服务，提供L1（内存）和L2（文件）两级缓存。 """ + _MEMORY_CACHE_MAX_ITEMS = 2000 + def __init__(self): self.cache_path = (DATA_PATH / "cache" / "avatars").resolve() self.cache_path.mkdir(parents=True, exist_ok=True) - self._memory_cache: dict[str, Path] = {} + self._memory_cache: OrderedDict[str, Path] = OrderedDict() def _get_cache_path(self, platform: str, identifier: str) -> Path: """ @@ -79,8 +82,11 @@ class AvatarService: cache_key = f"{platform}-{identifier}" if not force_refresh and cache_key in self._memory_cache: - if self._memory_cache[cache_key].exists(): - return self._memory_cache[cache_key] + cached_path = self._memory_cache[cache_key] + if cached_path.exists(): + self._memory_cache.move_to_end(cache_key) + return cached_path + self._memory_cache.pop(cache_key, None) local_path = self._get_cache_path(platform, identifier) ttl_seconds = Config.get_config("avatar_cache", "TTL_DAYS", 7) * 86400 @@ -90,6 +96,9 @@ class AvatarService: file_mtime = os.path.getmtime(local_path) if time.time() - file_mtime < ttl_seconds: self._memory_cache[cache_key] = local_path + self._memory_cache.move_to_end(cache_key) + while len(self._memory_cache) > self._MEMORY_CACHE_MAX_ITEMS: + self._memory_cache.popitem(last=False) return local_path except FileNotFoundError: pass @@ -102,6 +111,9 @@ class AvatarService: if await AsyncHttpx.download_file(avatar_url, local_path): self._memory_cache[cache_key] = local_path + self._memory_cache.move_to_end(cache_key) + while len(self._memory_cache) > self._MEMORY_CACHE_MAX_ITEMS: + self._memory_cache.popitem(last=False) return local_path else: logger.warning(f"下载头像失败: {avatar_url}", "AvatarService") @@ -126,6 +138,22 @@ class AvatarService: except FileNotFoundError: continue + if self._memory_cache: + stale_keys = [] + for key, cached_path in self._memory_cache.items(): + if not cached_path.exists(): + stale_keys.append(key) + continue + try: + if now - os.path.getmtime(cached_path) > ttl_seconds: + stale_keys.append(key) + except OSError: + stale_keys.append(key) + for key in stale_keys: + self._memory_cache.pop(key, None) + while len(self._memory_cache) > self._MEMORY_CACHE_MAX_ITEMS: + self._memory_cache.popitem(last=False) + logger.info( f"头像缓存清理完成，共删除 {deleted_count} 个过期文件。", "AvatarService" ) diff --git a/zhenxun/services/cache/runtime_cache.py b/zhenxun/services/cache/runtime_cache.py index 9ce9d68f..6655e3f0 100644 --- a/zhenxun/services/cache/runtime_cache.py +++ b/zhenxun/services/cache/runtime_cache.py @@ -8,7 +8,6 @@ import time from typing import TYPE_CHECKING, Any, ClassVar import uuid -from zhenxun.configs.config import Config from zhenxun.services.cache.config import CacheMode from zhenxun.services.log import logger from zhenxun.utils.enum import LimitCheckType, LimitWatchType, PluginLimitType @@ -19,108 +18,23 @@ if TYPE_CHECKING: LOG_COMMAND = "RuntimeCache" -Config.add_plugin_config( - "hook", - "PLUGININFO_MEM_REFRESH_INTERVAL", - 1800, - help="plugin info memory cache refresh seconds", -) -Config.add_plugin_config( - "hook", - "BAN_MEM_REFRESH_INTERVAL", - 900, - help="ban memory cache full refresh seconds", -) -Config.add_plugin_config( - "hook", - "BAN_MEM_CLEAN_INTERVAL", - 900, - help="ban memory cache cleanup seconds", -) -Config.add_plugin_config( - "hook", - "BAN_MEM_CLEANUP_DB", - True, - help="delete expired ban records from database", -) -Config.add_plugin_config( - "hook", - "BAN_MEM_NEGATIVE_TTL", - 5, - help="ban memory negative cache ttl seconds", -) -Config.add_plugin_config( - "hook", - "BOT_MEM_REFRESH_INTERVAL", - 900, - help="bot memory cache refresh seconds", -) -Config.add_plugin_config( - "hook", - "BOT_MEM_NEGATIVE_TTL", - 60, - help="bot memory negative cache ttl seconds", -) -Config.add_plugin_config( - "hook", - "GROUP_MEM_REFRESH_INTERVAL", - 900, - help="group memory cache refresh seconds", -) -Config.add_plugin_config( - "hook", - "GROUP_MEM_NEGATIVE_TTL", - 60, - help="group memory negative cache ttl seconds", -) -Config.add_plugin_config( - "hook", - "LEVEL_MEM_REFRESH_INTERVAL", - 900, - help="level memory cache refresh seconds", -) -Config.add_plugin_config( - "hook", - "LEVEL_MEM_NEGATIVE_TTL", - 60, - help="level memory negative cache ttl seconds", -) -Config.add_plugin_config( - "hook", - "TASK_MEM_REFRESH_INTERVAL", - 900, - help="task info memory cache refresh seconds", -) -Config.add_plugin_config( - "hook", - "TASK_MEM_NEGATIVE_TTL", - 60, - help="task info negative cache ttl seconds", -) -Config.add_plugin_config( - "hook", - "LIMIT_MEM_REFRESH_INTERVAL", - 900, - help="plugin limit memory cache refresh seconds", -) -Config.add_plugin_config( - "hook", - "LIMIT_MEM_NEGATIVE_TTL", - 30, - help="plugin limit negative cache ttl seconds", -) -Config.add_plugin_config( - "hook", - "RUNTIME_CACHE_SYNC_ENABLED", - True, - help="enable redis pubsub runtime cache sync", -) -Config.add_plugin_config( - "hook", - "RUNTIME_CACHE_SYNC_CHANNEL", - "ZHENXUN_RUNTIME_CACHE_SYNC", - help="redis pubsub channel for runtime cache sync", -) +PLUGININFO_MEM_REFRESH_INTERVAL = 300 +BAN_MEM_REFRESH_INTERVAL = 60 +BAN_MEM_CLEAN_INTERVAL = 60 +BAN_MEM_CLEANUP_DB = True +BAN_MEM_NEGATIVE_TTL = 5 +BOT_MEM_REFRESH_INTERVAL = 60 +BOT_MEM_NEGATIVE_TTL = 60 +GROUP_MEM_REFRESH_INTERVAL = 60 +GROUP_MEM_NEGATIVE_TTL = 60 +LEVEL_MEM_REFRESH_INTERVAL = 120 +LEVEL_MEM_NEGATIVE_TTL = 60 +TASK_MEM_REFRESH_INTERVAL = 900 +TASK_MEM_NEGATIVE_TTL = 60 +LIMIT_MEM_REFRESH_INTERVAL = 60 +LIMIT_MEM_NEGATIVE_TTL = 30 +RUNTIME_CACHE_SYNC_ENABLED = True +RUNTIME_CACHE_SYNC_CHANNEL = "ZHENXUN_RUNTIME_CACHE_SYNC" def _coerce_int(value, default: int) -> int: @@ -496,7 +410,7 @@ class RuntimeCacheSync: @classmethod def _sync_enabled(cls) -> bool: - enabled = bool(Config.get_config("hook", "RUNTIME_CACHE_SYNC_ENABLED", True)) + enabled = RUNTIME_CACHE_SYNC_ENABLED return enabled and _redis_enabled() @classmethod @@ -518,11 +432,7 @@ class RuntimeCacheSync: return port = _coerce_int(_env_get("REDIS_PORT"), 6379) password = _env_get("REDIS_PASSWORD") - cls._channel = str( - Config.get_config( - "hook", "RUNTIME_CACHE_SYNC_CHANNEL", "ZHENXUN_RUNTIME_CACHE_SYNC" - ) - ) + cls._channel = RUNTIME_CACHE_SYNC_CHANNEL try: cls._redis = redis_async.Redis( host=host, @@ -730,10 +640,7 @@ class PluginInfoMemoryCache: @classmethod def start_refresh_task(cls) -> None: - interval = _coerce_int( - Config.get_config("hook", "PLUGININFO_MEM_REFRESH_INTERVAL", 300), - 300, - ) + interval = PLUGININFO_MEM_REFRESH_INTERVAL if interval <= 0: return if cls._refresh_task and not cls._refresh_task.done(): @@ -763,7 +670,7 @@ class BotMemoryCache: @classmethod def _negative_ttl(cls) -> int: - return _coerce_int(Config.get_config("hook", "BOT_MEM_NEGATIVE_TTL", 60), 60) + return BOT_MEM_NEGATIVE_TTL @classmethod def _is_negative(cls, bot_id: str) -> bool: @@ -887,9 +794,7 @@ class BotMemoryCache: @classmethod def start_tasks(cls) -> None: - interval = _coerce_int( - Config.get_config("hook", "BOT_MEM_REFRESH_INTERVAL", 60), 60 - ) + interval = BOT_MEM_REFRESH_INTERVAL if interval <= 0: return if cls._refresh_task and not cls._refresh_task.done(): @@ -929,7 +834,7 @@ class GroupMemoryCache: @classmethod def _negative_ttl(cls) -> int: - return _coerce_int(Config.get_config("hook", "GROUP_MEM_NEGATIVE_TTL", 60), 60) + return GROUP_MEM_NEGATIVE_TTL @classmethod def _is_negative(cls, key: tuple[str, str]) -> bool: @@ -1061,9 +966,7 @@ class GroupMemoryCache: @classmethod def start_tasks(cls) -> None: - interval = _coerce_int( - Config.get_config("hook", "GROUP_MEM_REFRESH_INTERVAL", 60), 60 - ) + interval = GROUP_MEM_REFRESH_INTERVAL if interval <= 0: return if cls._refresh_task and not cls._refresh_task.done(): @@ -1103,7 +1006,7 @@ class LevelUserMemoryCache: @classmethod def _negative_ttl(cls) -> int: - return _coerce_int(Config.get_config("hook", "LEVEL_MEM_NEGATIVE_TTL", 60), 60) + return LEVEL_MEM_NEGATIVE_TTL @classmethod def _is_negative(cls, key: tuple[str, str]) -> bool: @@ -1153,9 +1056,7 @@ class LevelUserMemoryCache: @classmethod async def ensure_fresh(cls) -> None: - interval = _coerce_int( - Config.get_config("hook", "LEVEL_MEM_REFRESH_INTERVAL", 120), 120 - ) + interval = LEVEL_MEM_REFRESH_INTERVAL if not cls._loaded: await cls.refresh() return @@ -1284,9 +1185,7 @@ class LevelUserMemoryCache: @classmethod def start_tasks(cls) -> None: - interval = _coerce_int( - Config.get_config("hook", "LEVEL_MEM_REFRESH_INTERVAL", 120), 120 - ) + interval = LEVEL_MEM_REFRESH_INTERVAL if interval <= 0: return if cls._refresh_task and not cls._refresh_task.done(): @@ -1316,7 +1215,7 @@ class TaskInfoMemoryCache: @classmethod def _negative_ttl(cls) -> int: - return _coerce_int(Config.get_config("hook", "TASK_MEM_NEGATIVE_TTL", 60), 60) + return TASK_MEM_NEGATIVE_TTL @classmethod def _is_negative(cls, module: str) -> bool: @@ -1423,9 +1322,7 @@ class TaskInfoMemoryCache: @classmethod def start_tasks(cls) -> None: - interval = _coerce_int( - Config.get_config("hook", "TASK_MEM_REFRESH_INTERVAL", 300), 300 - ) + interval = TASK_MEM_REFRESH_INTERVAL if interval <= 0: return if cls._refresh_task and not cls._refresh_task.done(): @@ -1456,7 +1353,7 @@ class PluginLimitMemoryCache: @classmethod def _negative_ttl(cls) -> int: - return _coerce_int(Config.get_config("hook", "LIMIT_MEM_NEGATIVE_TTL", 30), 30) + return LIMIT_MEM_NEGATIVE_TTL @classmethod def _is_negative(cls, module: str) -> bool: @@ -1598,9 +1495,7 @@ class PluginLimitMemoryCache: @classmethod def start_tasks(cls) -> None: - interval = _coerce_int( - Config.get_config("hook", "LIMIT_MEM_REFRESH_INTERVAL", 60), 60 - ) + interval = LIMIT_MEM_REFRESH_INTERVAL if interval <= 0: return if cls._refresh_task and not cls._refresh_task.done(): @@ -1634,7 +1529,7 @@ class BanMemoryCache: @classmethod def _neg_ttl(cls) -> int: - return _coerce_int(Config.get_config("hook", "BAN_MEM_NEGATIVE_TTL", 5), 5) + return BAN_MEM_NEGATIVE_TTL @classmethod def _neg_key( @@ -1894,13 +1789,9 @@ class BanMemoryCache: @classmethod def start_tasks(cls) -> None: - refresh_interval = _coerce_int( - Config.get_config("hook", "BAN_MEM_REFRESH_INTERVAL", 60), 60 - ) - clean_interval = _coerce_int( - Config.get_config("hook", "BAN_MEM_CLEAN_INTERVAL", 60), 60 - ) - cleanup_db = bool(Config.get_config("hook", "BAN_MEM_CLEANUP_DB", True)) + refresh_interval = BAN_MEM_REFRESH_INTERVAL + clean_interval = BAN_MEM_CLEAN_INTERVAL + cleanup_db = BAN_MEM_CLEANUP_DB if refresh_interval > 0 and (not cls._refresh_task or cls._refresh_task.done()): cls._refresh_task = asyncio.create_task(cls._refresh_loop(refresh_interval)) diff --git a/zhenxun/services/group_settings_service.py b/zhenxun/services/group_settings_service.py index 4de0a55b..49cb2f34 100644 --- a/zhenxun/services/group_settings_service.py +++ b/zhenxun/services/group_settings_service.py @@ -8,6 +8,7 @@ from zhenxun.models.group_plugin_setting import GroupPluginSetting from zhenxun.services.cache import Cache from zhenxun.services.data_access import DataAccess from zhenxun.services.log import logger +from zhenxun.utils.enum import CacheType from zhenxun.utils.pydantic_compat import model_dump, model_validate, parse_as T = TypeVar("T", bound=BaseModel) @@ -21,7 +22,14 @@ class GroupSettingsService: def __init__(self): self.dao = DataAccess(GroupPluginSetting) - self._cache = Cache[dict]("group_plugin_settings") + self._cache = Cache[dict[str, Any]](CacheType.GROUP_PLUGIN_SETTINGS_VIEW) + + @staticmethod + def _build_cache_key(group_id: str, plugin_name: str) -> str: + return f"{group_id}:{plugin_name}" + + async def _clear_merged_cache(self, group_id: str, plugin_name: str) -> None: + await self._cache.delete(self._build_cache_key(group_id, plugin_name)) async def set( self, group_id: str, plugin_name: str, settings_model: BaseModel @@ -44,6 +52,7 @@ class GroupSettingsService: ) await self.dao.clear_cache(group_id=group_id, plugin_name=plugin_name) + await self._clear_merged_cache(group_id, plugin_name) async def set_key_value( self, group_id: str, plugin_name: str, key: str, value: Any @@ -61,6 +70,7 @@ class GroupSettingsService: setting_entry.settings[key] = value await setting_entry.save(update_fields=["settings"]) await self.dao.clear_cache(group_id=group_id, plugin_name=plugin_name) + await self._clear_merged_cache(group_id, plugin_name) async def reset_key(self, group_id: str, plugin_name: str, key: str) -> bool: """重置单个配置项""" @@ -72,6 +82,7 @@ class GroupSettingsService: else: await setting.save(update_fields=["settings"]) await self.dao.clear_cache(group_id=group_id, plugin_name=plugin_name) + await self._clear_merged_cache(group_id, plugin_name) return True return False @@ -111,6 +122,7 @@ class GroupSettingsService: if deleted_count > 0: await self.dao.clear_cache(group_id=group_id, plugin_name=plugin_name) + await self._clear_merged_cache(group_id, plugin_name) logger.debug(f"已重置插件 '{plugin_name}' 在群组 '{group_id}' 的配置。") return True @@ -138,7 +150,7 @@ class GroupSettingsService: plugin_name: 插件的模块名。 parse_model: (可选) Pydantic模型，用于解析和验证配置。 """ - cache_key = f"{group_id}:{plugin_name}" + cache_key = self._build_cache_key(group_id, plugin_name) cached_settings = await self._cache.get(cache_key) if cached_settings is not None: logger.debug(f"缓存命中: {cache_key}") diff --git a/zhenxun/services/help_service.py b/zhenxun/services/help_service.py index e4454308..ea1ac5c1 100644 --- a/zhenxun/services/help_service.py +++ b/zhenxun/services/help_service.py @@ -5,13 +5,20 @@ from nonebot.plugin import PluginMetadata from pydantic import BaseModel from zhenxun import ui -from zhenxun.configs.config import BotConfig +from zhenxun.configs.config import BotConfig, Config from zhenxun.models.plugin_info import PluginInfo from zhenxun.models.task_info import TaskInfo +from zhenxun.services.renderer.result_cache import RenderResultMemoryCache from zhenxun.ui.models import HelpCategory, HelpItem, PluginHelpPageData from zhenxun.utils.common_utils import format_usage_for_markdown from zhenxun.utils.enum import PluginType +_PLUGIN_HELP_IMAGE_CACHE = RenderResultMemoryCache( + ttl_seconds=300, + max_items=48, + max_total_bytes=48 * 1024 * 1024, +) + class PluginData(BaseModel): plugin: PluginInfo @@ -111,6 +118,23 @@ async def create_plugin_help_image( categories=categories, ) - image_bytes = await ui.render(page_data, use_cache=True) + cache_payload = { + "plugin_types": sorted([plugin_type.value for plugin_type in plugin_types]), + "page_title": page_title, + "theme": Config.get_config("UI", "THEME", "default"), + "page_data": page_data, + } + cache_key = RenderResultMemoryCache.build_key(cache_payload) + if cached_image := await _PLUGIN_HELP_IMAGE_CACHE.get(cache_key): + return cached_image + + image_bytes = await ui.render( + page_data, + use_cache=True, + clip_selector=".container", + clip_padding=20, + disable_animations=True, + ) + await _PLUGIN_HELP_IMAGE_CACHE.set(cache_key, image_bytes) return image_bytes diff --git a/zhenxun/services/log.py b/zhenxun/services/log.py index beb2b9c0..74a1a03a 100644 --- a/zhenxun/services/log.py +++ b/zhenxun/services/log.py @@ -11,6 +11,7 @@ from nonebot_plugin_session import Session from nonebot_plugin_uninfo import Session as uninfoSession from zhenxun.configs.path_config import LOG_PATH +from zhenxun.utils.log_sanitizer import sanitize_for_logging driver = nonebot.get_driver() @@ -48,6 +49,18 @@ class logger: TEMPLATE_TARGET = "[Target]([{}])" SUCCESS_TEMPLATE = "[{}]: {} | 参数[{}] 返回: [{}]" + @staticmethod + def _to_safe_text(value: Any) -> str: + """对日志内容做安全序列化，避免超长 base64 等污染日志。""" + try: + value = sanitize_for_logging(value) + except Exception: + pass + try: + return str(value) + except Exception: + return repr(value) + @classmethod def __parser_template( cls, @@ -110,11 +123,19 @@ class logger: platform = session.basic.get("scope") template = cls.__parser_template( - info, command, user_id, group_id, adapter, target, platform + cls._to_safe_text(info), + cls._to_safe_text(command) if command is not None else None, + user_id, + group_id, + cls._to_safe_text(adapter) if adapter is not None else None, + cls._to_safe_text(target) if target is not None else None, + cls._to_safe_text(platform) if platform is not None else None, ) if e: - template += f" || 错误 {type(e).__name__}: {e}" + err_type = type(e).__name__ + err_msg = cls._to_safe_text(e) + template += f" || 错误 {err_type}: {err_msg}" try: log_func = getattr(logger_.opt(colors=True), level) diff --git a/zhenxun/services/renderer/__init__.py b/zhenxun/services/renderer/__init__.py index 41f935df..6bc55787 100644 --- a/zhenxun/services/renderer/__init__.py +++ b/zhenxun/services/renderer/__init__.py @@ -1,5 +1,6 @@ from zhenxun.utils.manager.priority_manager import PriorityLifecycle +from .engine import engine_manager from .service import RendererService from .types import Renderable, RenderResult @@ -12,4 +13,10 @@ async def _init_renderer_service(): await renderer_service.initialize() +@PriorityLifecycle.on_shutdown(priority=10) +async def _shutdown_renderer_service(): + """在Bot关闭时回收截图引擎资源。""" + await engine_manager.close() + + __all__ = ["RenderResult", "Renderable", "renderer_service"] diff --git a/zhenxun/services/renderer/engine.py b/zhenxun/services/renderer/engine.py index 7278c766..fe7aae12 100644 --- a/zhenxun/services/renderer/engine.py +++ b/zhenxun/services/renderer/engine.py @@ -1,17 +1,856 @@ +import asyncio +from collections import OrderedDict +from collections.abc import Awaitable, Callable +import contextlib +import hashlib +import inspect +import json from pathlib import Path +import time +from typing import Any, ClassVar, cast -from nonebot_plugin_htmlrender import html_to_pic +import nonebot_plugin_htmlrender.browser as htmlrender_browser +import psutil + +from zhenxun.configs.config import Config +from zhenxun.services.log import logger from .types import BaseScreenshotEngine +async def _await_if_needed(value: Any) -> Any: + if inspect.isawaitable(value): + return await cast(Awaitable[Any], value) + return value + + +async def _get_browser_instance() -> Any: + for attr_name in ("get_browser", "get_new_browser"): + browser_getter = getattr(htmlrender_browser, attr_name, None) + if callable(browser_getter): + return await _await_if_needed(browser_getter()) + raise RuntimeError("nonebot_plugin_htmlrender.browser 未提供可用浏览器获取函数。") + + +async def _shutdown_browser_instance() -> None: + for attr_name in ( + "shutdown_htmlrender", + "shutdown_browser", + "close_browser", + "close_htmlrender", + ): + shutdown_func = getattr(htmlrender_browser, attr_name, None) + if callable(shutdown_func): + try: + await _await_if_needed(shutdown_func()) + finally: + with contextlib.suppress(Exception): + setattr(htmlrender_browser, "_browser", None) + with contextlib.suppress(Exception): + setattr(htmlrender_browser, "_playwright", None) + return + + browser_obj = getattr(htmlrender_browser, "_browser", None) + close_func = getattr(browser_obj, "close", None) if browser_obj else None + if callable(close_func): + with contextlib.suppress(Exception): + await _await_if_needed(close_func()) + + playwright_obj = getattr(htmlrender_browser, "_playwright", None) + stop_func = getattr(playwright_obj, "stop", None) if playwright_obj else None + if callable(stop_func): + with contextlib.suppress(Exception): + await _await_if_needed(stop_func()) + + with contextlib.suppress(Exception): + setattr(htmlrender_browser, "_browser", None) + with contextlib.suppress(Exception): + setattr(htmlrender_browser, "_playwright", None) + + if callable(close_func) or callable(stop_func): + return + + logger.debug( + "未找到 htmlrender 浏览器关闭函数，跳过 shutdown。", + "PlaywrightEngine", + ) + + +def _patch_playwright_env_check_once() -> None: + if getattr(htmlrender_browser, "_zhenxun_check_once_patched", False): + return + + original_check: Callable[..., Awaitable[Any]] | None = None + check_attr_name = "" + for attr_name in ("check_playwright_env", "check_browser_env"): + candidate = getattr(htmlrender_browser, attr_name, None) + if callable(candidate): + original_check = cast(Callable[..., Awaitable[Any]], candidate) + check_attr_name = attr_name + break + + if original_check is None: + logger.debug( + "未找到 htmlrender 环境检查函数，跳过 check_once 补丁。", + "PlaywrightEngine", + ) + setattr(htmlrender_browser, "_zhenxun_check_once_patched", True) + return + + check_func = original_check + state: dict[str, Any] = {"checked": False, "result": None} + check_lock: asyncio.Lock | None = None + + def _is_browser_usable(browser_obj: Any) -> bool: + if browser_obj is None: + return False + is_connected = getattr(browser_obj, "is_connected", None) + if callable(is_connected): + with contextlib.suppress(Exception): + return bool(is_connected()) + # 无法判断连接状态时，保守认为可用 + return True + + def _get_current_browser_candidate() -> Any: + current = state["result"] + if _is_browser_usable(current): + return current + fallback = getattr(htmlrender_browser, "_browser", None) + if _is_browser_usable(fallback): + return fallback + return None + + async def _check_once(**kwargs: Any) -> Any: + nonlocal check_lock + if state["checked"]: + cached_browser = _get_current_browser_candidate() + if cached_browser is not None: + return cached_browser + state["checked"] = False + state["result"] = None + + if check_lock is None: + check_lock = asyncio.Lock() + async with check_lock: + if state["checked"]: + cached_browser = _get_current_browser_candidate() + if cached_browser is not None: + return cached_browser + state["checked"] = False + state["result"] = None + + result = await check_func(**kwargs) + state["checked"] = True + state["result"] = result + + browser = _get_current_browser_candidate() + if browser is not None: + return browser + return result + + setattr(htmlrender_browser, check_attr_name, _check_once) + setattr(htmlrender_browser, "_zhenxun_check_once_patched", True) + + class PlaywrightEngine(BaseScreenshotEngine): """使用 nonebot-plugin-htmlrender 实现的截图引擎。""" + _MAX_CONCURRENT_RENDER = 2 + _CONTEXT_POOL_SIZE = 2 + _PREWARM_CONTEXT_COUNT = 1 + _SET_CONTENT_WAIT_UNTIL = "domcontentloaded" + _READY_STATE_TIMEOUT_MS = 2_000 + _IMAGE_READY_TIMEOUT_MS = 1_800 + _FONT_READY_TIMEOUT_MS = 1_200 + _FULL_PAGE_VIEWPORT_MAX_HEIGHT = 4_096 + _FULL_PAGE_VIEWPORT_MAX_WIDTH = 4_096 + _CLIP_PADDING_DEFAULT = 0 + _DISABLE_ANIMATIONS_STYLE = """ + *, *::before, *::after { + animation: none !important; + transition: none !important; + caret-color: transparent !important; + scroll-behavior: auto !important; + } + """ + _RECENT_RESULT_TTL_SECONDS = 1.5 + _RECENT_RESULT_MAX_ITEMS = 64 + _RSS_RECYCLE_MIN_THRESHOLD_BYTES = 700 * 1024 * 1024 + _RSS_RECYCLE_MAX_THRESHOLD_BYTES = 1200 * 1024 * 1024 + _RSS_RECYCLE_HEADROOM_BYTES = 224 * 1024 * 1024 + _RECYCLE_COOLDOWN_SECONDS = 300 + _RECYCLE_CHECK_EVERY = 8 + _IDLE_CHECK_INTERVAL_SECONDS = 15 + _IDLE_RECYCLE_SECONDS = 180 + _POOL_UNSAFE_OPTION_KEYS: ClassVar[set[str]] = { + "device_scale_factor", + "color_scheme", + "extra_http_headers", + "forced_colors", + "geolocation", + "has_touch", + "http_credentials", + "ignore_https_errors", + "is_mobile", + "java_script_enabled", + "locale", + "permissions", + "proxy", + "record_har_content", + "record_har_mode", + "record_har_omit_content", + "record_har_path", + "record_video_dir", + "record_video_size", + "reduced_motion", + "screen", + "service_workers", + "storage_state", + "timezone_id", + "user_agent", + } + + def __init__(self): + _patch_playwright_env_check_once() + self._render_semaphore = asyncio.Semaphore(self._MAX_CONCURRENT_RENDER) + self._debug_console_log = bool(Config.get_config("UI", "DEBUG_MODE", False)) + self._state_lock = asyncio.Lock() + self._recycle_lock = asyncio.Lock() + self._active_renders = 0 + self._render_count = 0 + self._recycle_pending = False + self._last_recycle_at = 0.0 + self._last_render_finished_at = time.monotonic() + self._rss_baseline_bytes: int | None = None + self._recent_results: OrderedDict[str, tuple[float, bytes]] = OrderedDict() + self._inflight_tasks: dict[str, asyncio.Task[bytes]] = {} + self._context_pool: asyncio.LifoQueue[Any] = asyncio.LifoQueue() + self._all_contexts: set[Any] = set() + self._idle_recycle_task: asyncio.Task[None] | None = None + self._closing = False + self._process = psutil.Process() + + @staticmethod + def _normalize_base_url(path: Path) -> str: + base_url = path.absolute().as_uri() + if not base_url.endswith("/"): + base_url += "/" + return base_url + + @staticmethod + def _build_render_key( + html: str, template_path: str, render_options: dict[str, Any] + ) -> str: + options_json = json.dumps(render_options, sort_keys=True, default=str) + hasher = hashlib.sha256() + hasher.update(template_path.encode("utf-8")) + hasher.update(b"\0") + hasher.update(options_json.encode("utf-8")) + hasher.update(b"\0") + hasher.update(html.encode("utf-8", errors="ignore")) + return hasher.hexdigest() + + def _cleanup_recent_results_nolock(self, now: float) -> None: + while self._recent_results: + expire_at, _ = next(iter(self._recent_results.values())) + if expire_at > now: + break + self._recent_results.popitem(last=False) + while len(self._recent_results) > self._RECENT_RESULT_MAX_ITEMS: + self._recent_results.popitem(last=False) + + def _get_recent_result_nolock(self, key: str, now: float) -> bytes | None: + entry = self._recent_results.get(key) + if not entry: + return None + expire_at, result = entry + if expire_at <= now: + self._recent_results.pop(key, None) + return None + self._recent_results.move_to_end(key) + return result + + def _get_total_rss(self) -> int | None: + try: + total_rss = self._process.memory_info().rss + for child in self._process.children(recursive=True): + with contextlib.suppress(Exception): + total_rss += child.memory_info().rss + return total_rss + except Exception: + return None + + def _update_rss_baseline_nolock(self, current_rss: int) -> None: + if self._rss_baseline_bytes is None or current_rss < self._rss_baseline_bytes: + self._rss_baseline_bytes = current_rss + return + + threshold = self._rss_baseline_bytes + self._RSS_RECYCLE_HEADROOM_BYTES * 2 + if current_rss >= threshold: + self._rss_baseline_bytes = int( + self._rss_baseline_bytes * 0.9 + current_rss * 0.1 + ) + + def _get_dynamic_threshold_nolock(self, current_rss: int) -> int: + self._update_rss_baseline_nolock(current_rss) + baseline = self._rss_baseline_bytes or current_rss + dynamic = baseline + self._RSS_RECYCLE_HEADROOM_BYTES + dynamic = max(dynamic, self._RSS_RECYCLE_MIN_THRESHOLD_BYTES) + dynamic = min(dynamic, self._RSS_RECYCLE_MAX_THRESHOLD_BYTES) + return dynamic + + def _mark_recycle_if_needed_nolock(self, now: float) -> None: + if self._render_count % self._RECYCLE_CHECK_EVERY != 0: + return + if now - self._last_recycle_at < self._RECYCLE_COOLDOWN_SECONDS: + return + current_rss = self._get_total_rss() + if current_rss is None: + return + threshold = self._get_dynamic_threshold_nolock(current_rss) + if current_rss >= threshold: + self._recycle_pending = True + + async def initialize(self) -> None: + async with self._state_lock: + if self._idle_recycle_task and not self._idle_recycle_task.done(): + return + self._closing = False + self._last_render_finished_at = time.monotonic() + if current_rss := self._get_total_rss(): + self._rss_baseline_bytes = current_rss + self._idle_recycle_task = asyncio.create_task(self._idle_recycle_loop()) + await self._prewarm_browser_and_pool() + + async def close(self) -> None: + idle_task: asyncio.Task[None] | None = None + async with self._state_lock: + self._closing = True + idle_task = self._idle_recycle_task + self._idle_recycle_task = None + for task in self._inflight_tasks.values(): + task.cancel() + self._inflight_tasks.clear() + self._recent_results.clear() + self._recycle_pending = False + + if idle_task: + idle_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await idle_task + + await self._dispose_context_pool() + await _shutdown_browser_instance() + + async def _on_render_begin(self) -> None: + async with self._state_lock: + self._active_renders += 1 + + async def _on_render_end(self) -> None: + should_recycle = False + async with self._state_lock: + self._active_renders = max(0, self._active_renders - 1) + self._render_count += 1 + now = time.monotonic() + self._last_render_finished_at = now + self._mark_recycle_if_needed_nolock(now) + if self._recycle_pending and self._active_renders == 0: + self._recycle_pending = False + self._last_recycle_at = now + should_recycle = True + if should_recycle: + await self._recycle_browser("active") + + @staticmethod + def _build_page_options( + render_options: dict[str, Any], *, pooled: bool + ) -> dict[str, Any]: + options = render_options.copy() + options.pop("wait", None) + options.pop("type", None) + options.pop("quality", None) + options.pop("scale", None) + options.pop("screenshot_scale", None) + options.pop("screenshot_timeout", None) + options.pop("full_page", None) + options.pop("clip_selector", None) + options.pop("clip_padding", None) + options.pop("disable_animations", None) + if pooled: + options.pop("base_url", None) + return options + + @staticmethod + def _build_screenshot_options(render_options: dict[str, Any]) -> dict[str, Any]: + scale = render_options.get("screenshot_scale", render_options.get("scale")) + if scale not in ("css", "device"): + scale = None + return { + "full_page": bool(render_options.get("full_page", True)), + "type": render_options.get("type", "png"), + "quality": render_options.get("quality"), + "scale": scale, + "timeout": render_options.get("screenshot_timeout", 30_000), + } + + @staticmethod + def _get_wait_timeout(render_options: dict[str, Any]) -> int: + wait = render_options.get("wait", 0) + if isinstance(wait, int): + return max(wait, 0) + return 0 + + @staticmethod + def _coerce_non_negative_int(value: Any, default: int = 0) -> int: + try: + value_int = int(value) + except (TypeError, ValueError): + return default + return value_int if value_int >= 0 else default + + @classmethod + def _should_use_context_pool(cls, render_options: dict[str, Any]) -> bool: + for key in cls._POOL_UNSAFE_OPTION_KEYS: + if key in render_options: + return False + return True + + async def _render_with_page( + self, + page: Any, + html: str, + template_path: str, + render_options: dict[str, Any], + ) -> bytes: + if self._debug_console_log: + page.on("console", lambda msg: logger.debug(f"浏览器控制台: {msg.text}")) + await page.goto(template_path, wait_until="domcontentloaded") + await page.set_content(html, wait_until=self._SET_CONTENT_WAIT_UNTIL) + if bool(render_options.get("disable_animations", False)): + await self._disable_page_animations(page) + await self._wait_for_visual_stability(page) + if wait_ms := self._get_wait_timeout(render_options): + await page.wait_for_timeout(wait_ms) + screenshot_options = self._build_screenshot_options(render_options) + clip_selector = render_options.get("clip_selector") + if isinstance(clip_selector, str) and clip_selector.strip(): + if image_bytes := await self._capture_by_selector( + page, + selector=clip_selector.strip(), + screenshot_options=screenshot_options, + clip_padding=self._coerce_non_negative_int( + render_options.get("clip_padding"), + self._CLIP_PADDING_DEFAULT, + ), + ): + return image_bytes + await self._optimize_full_page_capture(page, screenshot_options) + return await page.screenshot(**screenshot_options) + + async def _disable_page_animations(self, page: Any) -> None: + with contextlib.suppress(Exception): + await page.add_style_tag(content=self._DISABLE_ANIMATIONS_STYLE) + + async def _capture_by_selector( + self, + page: Any, + selector: str, + screenshot_options: dict[str, Any], + clip_padding: int, + ) -> bytes | None: + element = await page.query_selector(selector) + if element is None: + return None + + element_screenshot_options = { + "type": screenshot_options.get("type", "png"), + "quality": screenshot_options.get("quality"), + "timeout": screenshot_options.get("timeout", 30_000), + } + with contextlib.suppress(Exception): + box = await element.bounding_box() + if box and clip_padding > 0: + viewport = page.viewport_size or {} + width = int(viewport.get("width") or 0) + if width > 0: + target_height = int(box["y"] + box["height"] + clip_padding) + current_height = int(viewport.get("height") or 0) + if target_height > current_height: + await page.set_viewport_size( + { + "width": width, + "height": min( + target_height, + self._FULL_PAGE_VIEWPORT_MAX_HEIGHT, + ), + } + ) + + if clip_padding <= 0: + return await element.screenshot(**element_screenshot_options) + + with contextlib.suppress(Exception): + clip_box = await element.bounding_box() + if clip_box is None: + return await element.screenshot(**element_screenshot_options) + clip = { + "x": max(clip_box["x"] - clip_padding, 0), + "y": max(clip_box["y"] - clip_padding, 0), + "width": clip_box["width"] + clip_padding * 2, + "height": clip_box["height"] + clip_padding * 2, + } + page_options = { + "type": screenshot_options.get("type", "png"), + "quality": screenshot_options.get("quality"), + "timeout": screenshot_options.get("timeout", 30_000), + "clip": clip, + } + return await page.screenshot(**page_options) + + return await element.screenshot(**element_screenshot_options) + + async def _wait_for_visual_stability(self, page: Any) -> None: + with contextlib.suppress(Exception): + await page.wait_for_function( + "() => document.readyState === 'complete'", + timeout=self._READY_STATE_TIMEOUT_MS, + ) + + with contextlib.suppress(Exception): + await page.wait_for_function( + "() => Array.from(document.images || []).every(img => img.complete)", + timeout=self._IMAGE_READY_TIMEOUT_MS, + ) + + with contextlib.suppress(Exception): + await page.evaluate( + """ + async (timeoutMs) => { + if (!document.fonts || !document.fonts.ready) return; + await Promise.race([ + document.fonts.ready, + new Promise(resolve => setTimeout(resolve, timeoutMs)), + ]); + } + """, + self._FONT_READY_TIMEOUT_MS, + ) + + async def _optimize_full_page_capture( + self, page: Any, screenshot_options: dict[str, Any] + ) -> None: + if not bool(screenshot_options.get("full_page")): + return + + viewport = page.viewport_size or {} + width = viewport.get("width") + height = viewport.get("height") + if ( + not isinstance(width, int) + or width <= 0 + or not isinstance(height, int) + or height <= 0 + ): + return + + with contextlib.suppress(Exception): + content_size = await page.evaluate( + """ + () => { + const body = document.body; + const doc = document.documentElement; + const bodyWidth = body ? Math.max( + body.scrollWidth, + body.offsetWidth, + body.clientWidth + ) : 0; + const bodyHeight = body ? Math.max( + body.scrollHeight, + body.offsetHeight, + body.clientHeight + ) : 0; + const docWidth = doc ? Math.max( + doc.scrollWidth, + doc.offsetWidth, + doc.clientWidth + ) : 0; + const docHeight = doc ? Math.max( + doc.scrollHeight, + doc.offsetHeight, + doc.clientHeight + ) : 0; + return { + width: Math.ceil(Math.max(bodyWidth, docWidth, 10)), + height: Math.ceil(Math.max(bodyHeight, docHeight, 10)), + }; + } + """ + ) + if not isinstance(content_size, dict): + return + + content_width = content_size.get("width") + content_height = content_size.get("height") + if not isinstance(content_width, int) or not isinstance( + content_height, int + ): + return + if ( + content_width < 10 + or content_height < 10 + or content_width > self._FULL_PAGE_VIEWPORT_MAX_WIDTH + or content_height > self._FULL_PAGE_VIEWPORT_MAX_HEIGHT + ): + return + + target_width = max(width, content_width) + target_height = max(height, content_height) + await page.set_viewport_size( + {"width": target_width, "height": target_height} + ) + screenshot_options["full_page"] = False + + async def _render_with_oneoff_page( + self, + html: str, + template_path: str, + render_options: dict[str, Any], + ) -> bytes: + browser = await _get_browser_instance() + page_options = self._build_page_options(render_options, pooled=False) + page = await browser.new_page(**page_options) + try: + return await self._render_with_page( + page, html, template_path, render_options + ) + finally: + with contextlib.suppress(Exception): + await page.close() + + async def _acquire_context(self) -> Any: + try: + return self._context_pool.get_nowait() + except asyncio.QueueEmpty: + pass + + async with self._state_lock: + if len(self._all_contexts) < self._CONTEXT_POOL_SIZE: + create_new = True + else: + create_new = False + + if create_new: + browser = await _get_browser_instance() + context = await browser.new_context( + viewport={"width": 800, "height": 10}, + device_scale_factor=2, + ) + async with self._state_lock: + self._all_contexts.add(context) + return context + return await self._context_pool.get() + + async def _release_context(self, context: Any, broken: bool = False) -> None: + if broken: + await self._discard_context(context) + return + + async with self._state_lock: + if self._closing: + broken = True + elif context not in self._all_contexts: + broken = True + else: + self._context_pool.put_nowait(context) + return + + if broken: + await self._discard_context(context) + + async def _discard_context(self, context: Any) -> None: + async with self._state_lock: + existed = context in self._all_contexts + if existed: + self._all_contexts.remove(context) + if existed: + with contextlib.suppress(Exception): + await context.close() + + async def _dispose_context_pool(self) -> None: + async with self._state_lock: + contexts = list(self._all_contexts) + self._all_contexts.clear() + while True: + try: + self._context_pool.get_nowait() + except asyncio.QueueEmpty: + break + + for context in contexts: + with contextlib.suppress(Exception): + await context.close() + + async def _render_with_context_pool( + self, + html: str, + template_path: str, + render_options: dict[str, Any], + ) -> bytes: + context = await self._acquire_context() + page = None + broken = False + try: + page = await context.new_page() + page_options = self._build_page_options(render_options, pooled=True) + viewport = page_options.get("viewport") + if isinstance(viewport, dict): + width = viewport.get("width") + height = viewport.get("height") + if isinstance(width, int) and isinstance(height, int): + await page.set_viewport_size({"width": width, "height": height}) + return await self._render_with_page( + page, html, template_path, render_options + ) + except Exception: + broken = True + raise + finally: + if page is not None: + with contextlib.suppress(Exception): + await page.close() + await self._release_context(context, broken=broken) + + async def _render_html( + self, + html: str, + template_path: str, + render_options: dict[str, Any], + ) -> bytes: + if self._should_use_context_pool(render_options): + return await self._render_with_context_pool( + html, template_path, render_options + ) + return await self._render_with_oneoff_page(html, template_path, render_options) + + async def _recycle_browser(self, reason: str) -> None: + async with self._recycle_lock: + try: + await self._dispose_context_pool() + await _shutdown_browser_instance() + current_rss = self._get_total_rss() + if current_rss is not None: + self._update_rss_baseline_nolock(current_rss) + await self._prewarm_browser_and_pool() + logger.debug( + f"截图引擎触发回收({reason})，已重建浏览器实例。", + "PlaywrightEngine", + ) + except Exception as e: + logger.warning("浏览器实例重建失败。", "PlaywrightEngine", e=e) + + async def _prewarm_browser_and_pool(self) -> None: + if self._closing: + return + try: + browser = await _get_browser_instance() + except Exception as e: + logger.warning("截图引擎浏览器预热失败。", "PlaywrightEngine", e=e) + return + + for _ in range(self._PREWARM_CONTEXT_COUNT): + async with self._state_lock: + if self._closing: + return + if len(self._all_contexts) >= self._CONTEXT_POOL_SIZE: + return + if self._context_pool.qsize() >= self._PREWARM_CONTEXT_COUNT: + return + + context = None + try: + context = await browser.new_context( + viewport={"width": 800, "height": 10}, + device_scale_factor=2, + ) + page = await context.new_page() + await page.goto("about:blank", wait_until="domcontentloaded") + await page.set_content( + "", + wait_until="domcontentloaded", + ) + await page.close() + except Exception as e: + logger.warning("截图引擎上下文预热失败。", "PlaywrightEngine", e=e) + if context is not None: + with contextlib.suppress(Exception): + await context.close() + return + + async with self._state_lock: + if self._closing: + with contextlib.suppress(Exception): + await context.close() + return + if context in self._all_contexts: + continue + self._all_contexts.add(context) + self._context_pool.put_nowait(context) + + async def _idle_recycle_loop(self) -> None: + while True: + await asyncio.sleep(self._IDLE_CHECK_INTERVAL_SECONDS) + should_recycle = False + async with self._state_lock: + if self._closing: + return + now = time.monotonic() + if self._active_renders > 0: + continue + if now - self._last_recycle_at < self._RECYCLE_COOLDOWN_SECONDS: + continue + idle_for = now - self._last_render_finished_at + if idle_for < self._IDLE_RECYCLE_SECONDS: + continue + current_rss = self._get_total_rss() + if current_rss is None: + continue + threshold = self._get_dynamic_threshold_nolock(current_rss) + if current_rss >= threshold: + self._last_recycle_at = now + should_recycle = True + if should_recycle: + await self._recycle_browser("idle") + + async def _render_and_store_result( + self, + key: str, + html: str, + base_url_for_browser: str, + render_options: dict[str, Any], + ) -> bytes: + async with self._render_semaphore: + await self._on_render_begin() + try: + result = await self._render_html( + html, + base_url_for_browser, + render_options, + ) + finally: + await self._on_render_end() + + async with self._state_lock: + now = time.monotonic() + self._recent_results[key] = ( + now + self._RECENT_RESULT_TTL_SECONDS, + result, + ) + self._recent_results.move_to_end(key) + self._cleanup_recent_results_nolock(now) + return result + async def render(self, html: str, base_url_path: Path, **render_options) -> bytes: - base_url_for_browser = base_url_path.absolute().as_uri() - if not base_url_for_browser.endswith("/"): - base_url_for_browser += "/" + base_url_for_browser = self._normalize_base_url(base_url_path) final_render_options = { "viewport": {"width": 800, "height": 10}, @@ -19,12 +858,40 @@ class PlaywrightEngine(BaseScreenshotEngine): "base_url": base_url_for_browser, } - return await html_to_pic( - html=html, - template_path=base_url_for_browser, - **final_render_options, + dedupe_key = self._build_render_key( + html, + base_url_for_browser, + final_render_options, ) + owner = False + async with self._state_lock: + now = time.monotonic() + self._cleanup_recent_results_nolock(now) + if cached_result := self._get_recent_result_nolock(dedupe_key, now): + return cached_result + + task = self._inflight_tasks.get(dedupe_key) + if task is None: + task = asyncio.create_task( + self._render_and_store_result( + dedupe_key, + html, + base_url_for_browser, + final_render_options, + ) + ) + self._inflight_tasks[dedupe_key] = task + owner = True + + try: + return await task + finally: + if owner: + async with self._state_lock: + if self._inflight_tasks.get(dedupe_key) is task: + self._inflight_tasks.pop(dedupe_key, None) + class EngineManager: """ diff --git a/zhenxun/services/renderer/result_cache.py b/zhenxun/services/renderer/result_cache.py new file mode 100644 index 00000000..75fa5a08 --- /dev/null +++ b/zhenxun/services/renderer/result_cache.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import asyncio +from collections import OrderedDict +import hashlib +import time +from typing import Any + +from zhenxun.utils.pydantic_compat import dump_json_safely + + +class RenderResultMemoryCache: + def __init__( + self, + ttl_seconds: float, + max_items: int, + max_total_bytes: int | None = None, + ): + self._ttl_seconds = max(ttl_seconds, 0.0) + self._max_items = max(max_items, 1) + self._max_total_bytes = ( + max_total_bytes + if isinstance(max_total_bytes, int) and max_total_bytes > 0 + else None + ) + self._cache: OrderedDict[str, tuple[float, bytes]] = OrderedDict() + self._total_bytes = 0 + self._lock = asyncio.Lock() + + @staticmethod + def build_key(payload: Any) -> str: + payload_text = dump_json_safely( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(payload_text.encode("utf-8")).hexdigest() + + def _pop_oldest(self) -> None: + if not self._cache: + return + _, (_, value) = self._cache.popitem(last=False) + self._total_bytes -= len(value) + if self._total_bytes < 0: + self._total_bytes = 0 + + def _cleanup(self, now: float) -> None: + while self._cache: + expire_at, _ = next(iter(self._cache.values())) + if expire_at > now: + break + self._pop_oldest() + while len(self._cache) > self._max_items: + self._pop_oldest() + if self._max_total_bytes is not None: + while self._total_bytes > self._max_total_bytes and self._cache: + self._pop_oldest() + + async def get(self, key: str) -> bytes | None: + now = time.monotonic() + async with self._lock: + self._cleanup(now) + item = self._cache.get(key) + if item is None: + return None + expire_at, value = item + if expire_at <= now: + removed = self._cache.pop(key, None) + if removed: + self._total_bytes -= len(removed[1]) + if self._total_bytes < 0: + self._total_bytes = 0 + return None + self._cache.move_to_end(key) + return value + + async def set(self, key: str, value: bytes) -> None: + value_size = len(value) + if self._max_total_bytes is not None and value_size > self._max_total_bytes: + return + now = time.monotonic() + async with self._lock: + if old := self._cache.pop(key, None): + self._total_bytes -= len(old[1]) + if self._total_bytes < 0: + self._total_bytes = 0 + self._cache[key] = (now + self._ttl_seconds, value) + self._total_bytes += value_size + self._cache.move_to_end(key) + self._cleanup(now) diff --git a/zhenxun/services/renderer/template.py b/zhenxun/services/renderer/template.py index 2032453a..2ccde094 100644 --- a/zhenxun/services/renderer/template.py +++ b/zhenxun/services/renderer/template.py @@ -1,6 +1,7 @@ from collections.abc import Callable import os from pathlib import Path +import re from typing import TYPE_CHECKING, Any from jinja2 import ( @@ -11,6 +12,7 @@ from jinja2 import ( select_autoescape, ) +from zhenxun.configs.config import Config from zhenxun.configs.path_config import THEMES_PATH from zhenxun.services.log import logger from zhenxun.services.renderer.theme import DependencyCollector @@ -25,6 +27,11 @@ from zhenxun.utils.exception import RenderingError if TYPE_CHECKING: from .types import RenderContext +_LEGACY_INCLUDE_TAG_PATTERN = re.compile( + r"\{\s*%\s*include\s+([\"'][^\"']+[\"'])\s*%\s*\}", + flags=re.IGNORECASE, +) + class RelativePathEnvironment(Environment): """ @@ -37,6 +44,22 @@ class RelativePathEnvironment(Environment): return path.replace(os.path.sep, "/") return super().join_path(template, parent) + def preprocess( + self, + source: str, + name: str | None = None, + filename: str | None = None, + ) -> str: + """ + 兼容历史模板里的错误 include 语法: + { + % include './style.css' % + } + 自动归一化为标准 Jinja 语法，避免页面样式丢失。 + """ + normalized_source = _LEGACY_INCLUDE_TAG_PATTERN.sub(r"{% include \1 %}", source) + return super().preprocess(normalized_source, name, filename) + class JinjaTemplateEngine: """ @@ -143,6 +166,11 @@ class JinjaTemplateEngine: } return await base_template.render_async(**page_context) else: + # 对于本身已是完整 HTML 文档的模板，直接返回，避免重复拼接大段 CSS。 + html_head = html_fragment.lstrip()[:32].lower() + if html_head.startswith(" str: hot_reload = Config.get_config("UI", "HOT_RELOAD", False) cache_key = (asset_path, current_template_name) - if not hot_reload and cache_key in self.theme_manager._asset_resolution_cache: - return self.theme_manager._asset_resolution_cache[cache_key] + if not hot_reload: + if cached_uri := self.theme_manager._get_lru_entry( + self.theme_manager._asset_resolution_cache, + cache_key, + ): + return cached_uri request = AssetRequest( asset_path=asset_path, template_name=current_template_name, @@ -324,7 +329,7 @@ class AssetResolutionService: if result_path := resolver.resolve(request): uri = result_path.absolute().as_uri() if not hot_reload: - self.theme_manager._asset_resolution_cache[cache_key] = uri + self.theme_manager._set_asset_resolution_cache(cache_key, uri) return uri logger.warning( f"资源文件未找到: '{asset_path}' (在 '{current_template_name}' 中)" @@ -333,6 +338,10 @@ class AssetResolutionService: class ThemeManager: + _ASSET_RESOLUTION_CACHE_MAX = 2048 + _GLOBAL_TEMPLATE_CACHE_MAX = 512 + _COMPONENT_DEP_CACHE_MAX = 512 + def __init__(self): """ 主题管理器，负责UI主题的加载、解析和模板渲染。 @@ -347,11 +356,58 @@ class ThemeManager: self.current_theme_context: dict[str, Any] = {} self.current_default_palette: dict[str, Any] = {} - self._asset_resolution_cache: dict[tuple[str, str], str] = {} - self._global_template_cache: dict[str, str] = {} - self._component_dependency_cache: dict[ + self._asset_resolution_cache: OrderedDict[tuple[str, str], str] = OrderedDict() + self._global_template_cache: OrderedDict[str, str] = OrderedDict() + self._component_dependency_cache: OrderedDict[ tuple[type, str, str | None], ComponentDependency - ] = {} + ] = OrderedDict() + + @staticmethod + def _get_lru_entry(cache: OrderedDict, key: Any) -> Any: + value = cache.get(key) + if value is not None: + cache.move_to_end(key) + return value + + @staticmethod + def _set_lru_entry( + cache: OrderedDict, + key: Any, + value: Any, + max_items: int, + ) -> None: + cache[key] = value + cache.move_to_end(key) + while len(cache) > max_items: + cache.popitem(last=False) + + def _set_asset_resolution_cache(self, key: tuple[str, str], value: str) -> None: + self._set_lru_entry( + self._asset_resolution_cache, + key, + value, + self._ASSET_RESOLUTION_CACHE_MAX, + ) + + def _set_global_template_cache(self, key: str, value: str) -> None: + self._set_lru_entry( + self._global_template_cache, + key, + value, + self._GLOBAL_TEMPLATE_CACHE_MAX, + ) + + def _set_component_dependency_cache( + self, + key: tuple[type, str, str | None], + value: ComponentDependency, + ) -> None: + self._set_lru_entry( + self._component_dependency_cache, + key, + value, + self._COMPONENT_DEP_CACHE_MAX, + ) def bind_template_engine(self, env: Environment): """绑定模板引擎环境，用于Manifest加载和asset解析""" @@ -429,9 +485,15 @@ class ThemeManager: """为独立模板创建一个专用的 asset loader。""" def asset_loader(asset_path: str) -> str: - full_path = local_base_path / asset_path - if full_path.exists(): - return full_path.absolute().as_uri() + clean_path = asset_path[2:] if asset_path.startswith("./") else asset_path + candidate_paths = [ + local_base_path / asset_path, + local_base_path / clean_path, + local_base_path / "assets" / clean_path, + ] + for full_path in candidate_paths: + if full_path.exists(): + return full_path.absolute().as_uri() return "" return asset_loader @@ -575,10 +637,12 @@ class ThemeManager: variant = getattr(component, "variant", None) cache_key = f"{component_path_base}::{variant or 'default'}" - if not hot_reload and ( - cached_path := self._global_template_cache.get(cache_key) - ): - return cached_path + if not hot_reload: + if cached_path := self._get_lru_entry( + self._global_template_cache, + cache_key, + ): + return cached_path if not hot_reload and ( cached_path := context.resolved_template_paths.get(cache_key) @@ -630,7 +694,7 @@ class ThemeManager: logger.debug(f"解析到模板路径: '{path}'") if not hot_reload: context.resolved_template_paths[cache_key] = path - self._global_template_cache[cache_key] = path + self._set_global_template_cache(cache_key, path) return path except TemplateNotFound: continue @@ -731,8 +795,8 @@ class DependencyCollector: cached_dep = None if not hot_reload: - cached_dep = context.theme_manager._component_dependency_cache.get( - cache_key + cached_dep = context.theme_manager._get_lru_entry( + context.theme_manager._component_dependency_cache, cache_key ) if cached_dep: @@ -798,7 +862,10 @@ class DependencyCollector: new_dep.asset_styles = component.get_required_styles() if not hot_reload: - context.theme_manager._component_dependency_cache[cache_key] = new_dep + context.theme_manager._set_component_dependency_cache( + cache_key, + new_dep, + ) context.collected_inline_css.extend(cached_css_results) context.collected_scripts.update(new_dep.scripts) diff --git a/zhenxun/services/runtime_bootstrap.py b/zhenxun/services/runtime_bootstrap.py new file mode 100644 index 00000000..1ae1da7e --- /dev/null +++ b/zhenxun/services/runtime_bootstrap.py @@ -0,0 +1,85 @@ +import asyncio +from concurrent.futures import ThreadPoolExecutor +import contextlib +import os + +import anyio.to_thread +from nonebot.drivers import Driver + +DEFAULT_EXECUTOR_MIN_WORKERS = 16 +DEFAULT_EXECUTOR_MAX_WORKERS = 64 +DEFAULT_ANYIO_MIN_TOKENS = 32 +DEFAULT_ANYIO_MAX_TOKENS = 128 + +_thread_executor: ThreadPoolExecutor | None = None +_runtime_hooks_registered = False +_alconna_patch_applied = False + + +def _clamp(value: int, minimum: int, maximum: int) -> int: + return max(minimum, min(value, maximum)) + + +def _get_executor_workers() -> int: + cpu = os.cpu_count() or 4 + return _clamp(cpu * 4, DEFAULT_EXECUTOR_MIN_WORKERS, DEFAULT_EXECUTOR_MAX_WORKERS) + + +def _get_anyio_tokens(executor_workers: int) -> int: + return _clamp( + executor_workers * 2, DEFAULT_ANYIO_MIN_TOKENS, DEFAULT_ANYIO_MAX_TOKENS + ) + + +def _apply_alconna_conflict_patch() -> None: + global _alconna_patch_applied + if _alconna_patch_applied: + return + with contextlib.suppress(Exception): + from arclet.alconna import formatter as alconna_formatter + + text_formatter = getattr(alconna_formatter, "TextFormatter", None) + if text_formatter is None: + return + original_remove = getattr(text_formatter, "remove", None) + if getattr(original_remove, "__zhenxun_safe_remove__", False): + _alconna_patch_applied = True + return + + def _safe_remove(self, base): + # Tolerate duplicate command cleanup when formatter hash is absent. + self.data.pop(base._hash, None) + + setattr(_safe_remove, "__zhenxun_safe_remove__", True) + setattr(text_formatter, "remove", _safe_remove) + _alconna_patch_applied = True + + +def register_runtime_bootstrap(driver: Driver) -> None: + _apply_alconna_conflict_patch() + global _runtime_hooks_registered + if _runtime_hooks_registered: + return + _runtime_hooks_registered = True + + @driver.on_startup + async def _setup_runtime_concurrency() -> None: + global _thread_executor + workers = _get_executor_workers() + loop = asyncio.get_running_loop() + if _thread_executor is None: + _thread_executor = ThreadPoolExecutor( + max_workers=workers, thread_name_prefix="zhenxun-worker" + ) + loop.set_default_executor(_thread_executor) + with contextlib.suppress(Exception): + limiter = anyio.to_thread.current_default_thread_limiter() + limiter.total_tokens = _get_anyio_tokens(workers) + + @driver.on_shutdown + async def _shutdown_runtime_concurrency() -> None: + global _thread_executor + executor = _thread_executor + _thread_executor = None + if executor is not None: + executor.shutdown(wait=False, cancel_futures=True) diff --git a/zhenxun/ui/__init__.py b/zhenxun/ui/__init__.py index 914978bc..c5b39f7d 100644 --- a/zhenxun/ui/__init__.py +++ b/zhenxun/ui/__init__.py @@ -415,7 +415,7 @@ async def render_full_result( screenshot_engine=renderer_service._screenshot_engine, component=component, use_cache=use_cache, - render_options=kwargs, + render_options={**kwargs, "_keep_html_content": True}, ) return await renderer_service._render_component(context) diff --git a/zhenxun/utils/enum.py b/zhenxun/utils/enum.py index 0412ee4a..2fc94e11 100644 --- a/zhenxun/utils/enum.py +++ b/zhenxun/utils/enum.py @@ -55,6 +55,8 @@ class CacheType(StrEnum): """全局全部群组""" GROUP_PLUGIN_SETTINGS = "GROUP_PLUGIN_SETTINGS" """插件分群配置""" + GROUP_PLUGIN_SETTINGS_VIEW = "GROUP_PLUGIN_SETTINGS_VIEW" + """插件分群配置视图缓存（聚合 dict）""" USERS = "GLOBAL_ALL_USERS" """全部用户""" BAN = "GLOBAL_ALL_BAN" diff --git a/zhenxun/utils/http_utils.py b/zhenxun/utils/http_utils.py index 71b2a9e0..c2f4b0f1 100644 --- a/zhenxun/utils/http_utils.py +++ b/zhenxun/utils/http_utils.py @@ -1,4 +1,5 @@ import asyncio +from collections import OrderedDict from collections.abc import AsyncGenerator, Awaitable, Callable, Sequence from contextlib import asynccontextmanager import os @@ -164,6 +165,72 @@ class AsyncHttpx: else None ) + _CONTENT_CACHE_TTL: ClassVar[float] = 3.0 + _CONTENT_CACHE_MAX_ITEMS: ClassVar[int] = 256 + _CONTENT_CACHE_MAX_BYTES: ClassVar[int] = 2 * 1024 * 1024 + _content_cache: ClassVar[OrderedDict[str, tuple[float, bytes]]] = OrderedDict() + _content_inflight: ClassVar[dict[str, asyncio.Task[Response]]] = {} + _content_cache_lock: ClassVar[asyncio.Lock] = asyncio.Lock() + + @classmethod + def _is_probably_image_url(cls, url: str) -> bool: + lower_url = url.lower() + if any( + ext in lower_url + for ext in ( + ".jpg", + ".jpeg", + ".png", + ".webp", + ".gif", + ".bmp", + ".avif", + ".heic", + ) + ): + return True + return "qpic.cn" in lower_url or "qlogo.cn" in lower_url + + @classmethod + def _get_cached_content_nolock(cls, key: str) -> bytes | None: + entry = cls._content_cache.get(key) + if not entry: + return None + expire_at, content = entry + if expire_at <= time.monotonic(): + cls._content_cache.pop(key, None) + return None + cls._content_cache.move_to_end(key) + return content + + @classmethod + def _cleanup_content_cache_nolock(cls) -> None: + now = time.monotonic() + while cls._content_cache: + expire_at, _ = next(iter(cls._content_cache.values())) + if expire_at > now: + break + cls._content_cache.popitem(last=False) + while len(cls._content_cache) > cls._CONTENT_CACHE_MAX_ITEMS: + cls._content_cache.popitem(last=False) + + @classmethod + async def _try_cache_content(cls, key: str, response: Response) -> None: + content = response.content + if not content or len(content) > cls._CONTENT_CACHE_MAX_BYTES: + return + content_type = response.headers.get("content-type", "").lower() + is_image = content_type.startswith("image/") or cls._is_probably_image_url(key) + if not is_image: + return + async with cls._content_cache_lock: + cls._content_cache[key] = ( + time.monotonic() + cls._CONTENT_CACHE_TTL, + content, + ) + cls._content_cache.move_to_end(key) + cls._cleanup_content_cache_nolock() + @classmethod def _prepare_temporary_client_config(cls, client_kwargs: dict) -> dict: """ @@ -378,7 +445,29 @@ class AsyncHttpx: cls, url: str | list[str], *, client: AsyncClient | None = None, **kwargs ) -> bytes: """获取指定 URL 的二进制内容。""" - res = await cls.get(url, client=client, **kwargs) + if not isinstance(url, str): + res = await cls.get(url, client=client, **kwargs) + return res.content + + cache_key = url + async with cls._content_cache_lock: + cached = cls._get_cached_content_nolock(cache_key) + if cached is not None: + return cached + task = cls._content_inflight.get(cache_key) + if task is None: + task = asyncio.create_task(cls.get(url, client=client, **kwargs)) + cls._content_inflight[cache_key] = task + + def _cleanup_inflight( + _: asyncio.Task[Response], key: str = cache_key + ) -> None: + cls._content_inflight.pop(key, None) + + task.add_done_callback(_cleanup_inflight) + + res = await task + await cls._try_cache_content(cache_key, res) return res.content @classmethod diff --git a/zhenxun/utils/log_sanitizer.py b/zhenxun/utils/log_sanitizer.py index 2938d65c..29900d6a 100644 --- a/zhenxun/utils/log_sanitizer.py +++ b/zhenxun/utils/log_sanitizer.py @@ -15,6 +15,22 @@ def _truncate_base64_string(value: str, threshold: int = 256) -> str: prefix = next((p for p in prefixes if value.startswith(p)), "base64") return f"[{prefix}_data_omitted_len={len(value)}]" + # 清理嵌入在普通文本中的超长 base64/data URI， + # 例如: "声音 -> base64://AAAA..." + embedded_patterns = ( + (re.compile(r"base64://[A-Za-z0-9+/=\s]{80,}"), "base64"), + ( + re.compile(r"data:(?:image|video|audio)[^,]*,[A-Za-z0-9+/=\s]{80,}"), + "data_uri", + ), + ) + for pattern, tag in embedded_patterns: + if pattern.search(value): + value = pattern.sub( + lambda m: f"[{tag}_data_omitted_len={len(m.group(0))}]", + value, + ) + if len(value) > 1000: return f"[long_string_omitted_len={len(value)}] {value[:20]}...{value[-20:]}"