mirror of
https://github.com/zhenxun-org/zhenxun_bot.git
synced 2026-09-29 16:50:02 +08:00
gold_readbag and russian
This commit is contained in:
hibiki
@@ -8,6 +8,8 @@ from util.utils import get_bot, get_local_proxy
|
||||
from asyncio.exceptions import TimeoutError
|
||||
import aiofiles
|
||||
import aiohttp
|
||||
from aiohttp.client_exceptions import ClientConnectorError
|
||||
|
||||
try:
|
||||
import ujson as json
|
||||
except ModuleNotFoundError:
|
||||
@@ -16,71 +18,97 @@ except ModuleNotFoundError:
|
||||
|
||||
async def update_setu_img():
|
||||
async with aiohttp.ClientSession(headers=get_user_agent()) as session:
|
||||
for file_name in ['setu_url.txt', 'setu_r18_url.txt']:
|
||||
if file_name == 'setu_url.txt':
|
||||
json_name = 'setu_img_hash.json'
|
||||
path = 'setu/'
|
||||
for file_name in ['setu_url.json', 'setu_r18_url.json']:
|
||||
if file_name == 'setu_url.json':
|
||||
json_name = 'setu_data.json'
|
||||
path = '_setu/'
|
||||
rar_path = 'setu_rar/'
|
||||
else:
|
||||
json_name = 'r18_setu_img_hash.json'
|
||||
path = 'r18/'
|
||||
json_name = 'r18_setu_data.json'
|
||||
path = '_r18/'
|
||||
rar_path = 'r18_rar/'
|
||||
if not os.path.exists(IMAGE_PATH + path):
|
||||
os.mkdir(IMAGE_PATH + path)
|
||||
if not os.path.exists(IMAGE_PATH + rar_path):
|
||||
os.mkdir(IMAGE_PATH + rar_path)
|
||||
try:
|
||||
data = json.load(open(TXT_PATH + json_name))
|
||||
data = json.load(open(TXT_PATH + json_name, encoding='utf8'))
|
||||
if not data:
|
||||
continue
|
||||
data = {}
|
||||
except (FileNotFoundError, TypeError):
|
||||
continue
|
||||
data = {}
|
||||
_success = 0
|
||||
_similar = 0
|
||||
try:
|
||||
with open(TXT_PATH + file_name, 'r') as f:
|
||||
txt_data = f.read()
|
||||
with open(TXT_PATH + file_name, 'r', encoding='utf8') as f:
|
||||
txt_data = json.load(f)
|
||||
if not txt_data:
|
||||
continue
|
||||
except FileNotFoundError:
|
||||
except (FileNotFoundError, ValueError):
|
||||
continue
|
||||
urls = list(set(txt_data[:-1].split(',')))
|
||||
total = len(urls)
|
||||
for url in urls:
|
||||
total = len(txt_data)
|
||||
urls = [data[x]['img_url'] for x in data.keys()]
|
||||
for pid in txt_data.keys():
|
||||
index = str(len(os.listdir(IMAGE_PATH + path)))
|
||||
url = txt_data[pid]["img_url"].replace('img-master', 'img-original').replace('_master1200', '')
|
||||
if url in urls or txt_data[pid]["img_url"] in urls:
|
||||
continue
|
||||
logger.info(f'开始更新 index:{index} --> {url}')
|
||||
for _ in range(3):
|
||||
try:
|
||||
async with session.get(url, proxy=get_local_proxy(), timeout=15) as response:
|
||||
if response.status == 200:
|
||||
async with aiofiles.open(IMAGE_PATH + 'rar/' + index + ".jpg", 'wb') as f:
|
||||
async with aiofiles.open(IMAGE_PATH + rar_path + index + ".jpg", 'wb') as f:
|
||||
await f.write(await response.read())
|
||||
_success += 1
|
||||
else:
|
||||
logger.info(f'{url} 不存在，跳过更新')
|
||||
break
|
||||
if os.path.getsize(IMAGE_PATH + 'rar/' + str(index) + ".jpg") > 1024 * 1024 * 1.5:
|
||||
logger.info(f'{url} 不存在，使用更新原url')
|
||||
url = txt_data[pid]["img_url"]
|
||||
async with session.get(txt_data[pid]["img_url"], proxy=get_local_proxy(),
|
||||
timeout=15) as response:
|
||||
if response.status == 200:
|
||||
async with aiofiles.open(IMAGE_PATH + rar_path + index + ".jpg", 'wb') as f:
|
||||
await f.write(await response.read())
|
||||
_success += 1
|
||||
if os.path.getsize(IMAGE_PATH + rar_path + str(index) + ".jpg") > 1024 * 1024 * 1.5:
|
||||
rar_imgs(
|
||||
'rar/',
|
||||
rar_path,
|
||||
path,
|
||||
in_file_name=index,
|
||||
out_file_name=index
|
||||
)
|
||||
else:
|
||||
logger.info('不需要压缩，移动图片 ' + IMAGE_PATH + 'rar/' + index + ".jpg --> "
|
||||
logger.info('不需要压缩，移动图片 ' + IMAGE_PATH + rar_path + index + ".jpg --> "
|
||||
+ IMAGE_PATH + path + index + ".jpg")
|
||||
os.rename(IMAGE_PATH + 'rar/' + index + ".jpg",
|
||||
os.rename(IMAGE_PATH + rar_path + index + ".jpg",
|
||||
IMAGE_PATH + path + index + ".jpg")
|
||||
img_hash = str(get_img_hash(f'{IMAGE_PATH}{path}{index}.jpg'))
|
||||
if img_hash in data.values():
|
||||
if img_hash in [data[x]['img_hash'] for x in data.keys()]:
|
||||
logger.info(f'index:{index} 与 '
|
||||
f'{list(data.keys())[list(data.values()).index(img_hash)]} 存在重复，删除')
|
||||
f'{[data[x]["img_hash"] for x in data.keys()].index(img_hash)} 存在重复，删除')
|
||||
os.remove(IMAGE_PATH + path + index + ".jpg")
|
||||
_similar += 1
|
||||
data[index] = img_hash
|
||||
else:
|
||||
data[index] = {
|
||||
'title': txt_data[pid]['title'],
|
||||
'author': txt_data[pid]['author'],
|
||||
'pid': txt_data[pid]['pid'],
|
||||
'img_hash': img_hash,
|
||||
'img_url': url,
|
||||
'tags': txt_data[pid]['tags'],
|
||||
}
|
||||
break
|
||||
except TimeoutError:
|
||||
except (TimeoutError, ClientConnectorError) as e:
|
||||
logger.warning(f'{url} 更新失败 ..{type(e)}：{e}')
|
||||
continue
|
||||
with open(TXT_PATH + json_name, 'w', encoding='utf-8') as f:
|
||||
json.dump(data, f, indent=4)
|
||||
json.dump(data, f, indent=4, ensure_ascii=False)
|
||||
open(TXT_PATH + file_name, 'w')
|
||||
logger.info(
|
||||
f'{str(datetime.now()).split(".")[0]} 更新 {file_name.split(".")[0]}完成，预计更新 {total} 张，实际更新 {_success} 张，相似 {_similar} 张，实际存入 {_success - _similar} 张')
|
||||
f'{str(datetime.now()).split(".")[0]} 更新 {file_name.split(".")[0]}完成，预计更新 {total} 张，'
|
||||
f'实际更新 {_success} 张，相似 {_similar} 张，实际存入 {_success - _similar} 张')
|
||||
await get_bot().send_private_msg(
|
||||
user_id=775757368,
|
||||
message=f'{str(datetime.now()).split(".")[0]} 更新{file_name.split(".")[0]}完成，预计更新 {total} 张，实际更新 {_success} 张，相似 {_similar} 张，实际存入 {_success - _similar} 张'
|
||||
user_id=list(get_bot().config.superusers)[0],
|
||||
message=f'{str(datetime.now()).split(".")[0]} 更新{file_name.split(".")[0]}完成，预计更新 {total} 张，'
|
||||
f'实际更新 {_success} 张，相似 {_similar} 张，实际存入 {_success - _similar} 张'
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user