Diuse/zhenxun_bot
1
0
Fork 0
mirror of https://github.com/zhenxun-org/zhenxun_bot.git synced 2026-10-07 12:50:03 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
hibiki
2021-06-04 18:01:33 +08:00
parent 9e401e7a83
commit 8c5e0bd381
43 changed files with 1605 additions and 426 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/draw_card/update_game_info.py
+76 -38
View File
@@ -3,11 +3,13 @@ import aiohttp
from .config import DRAW_PATH
from asyncio.exceptions import TimeoutError
from bs4 import BeautifulSoup
import asyncio
from .util import download_img
from urllib.parse import unquote
from services.log import logger
from .util import remove_prohibited_str
import bs4
import re
from util.utils import get_local_proxy
try:
import ujson as json
except ModuleNotFoundError:
@@ -25,13 +27,15 @@ async def update_info(url: str, game_name: str, info_list: list = None) -> 'dict
data = {}
try:
async with aiohttp.ClientSession(headers=headers) as session:
async with session.get(url, proxy=get_local_proxy(), timeout=7) as response:
async with session.get(url, timeout=7) as response:
soup = BeautifulSoup(await response.text(), 'lxml')
_tbody = get_tbody(soup, game_name, url)
trs = _tbody.find_all('tr')
att_dict, start_index, index = init_attr(game_name)
if game_name == 'guardian':
start_index = 1
if game_name == 'azur':
start_index = 0
for th in trs[0].find_all('th')[start_index:]:
text = th.text
if text[-1] == '\n':
@@ -51,12 +55,13 @@ async def update_info(url: str, game_name: str, info_list: list = None) -> 'dict
member_dict = intermediate_check(member_dict, key, game_name, td)
avatar_img = await _modify_avatar_url(session, game_name, member_dict["名称"])
member_dict['头像'] = avatar_img if avatar_img else member_dict['头像']
name = replace_name(member_dict, game_name)
member_dict, name = replace_update_name(member_dict, game_name)
await download_img(member_dict['头像'], game_name, name)
data[name] = member_dict
print(f'{name} is update...')
logger.info(f'{name} is update...')
data = await _last_check(data, game_name, session)
except TimeoutError:
logger.warning(f'更新 {game_name} 超时...')
return {}, 999
with open(DRAW_PATH + f'{game_name}.json', 'w', encoding='utf8') as wf:
wf.write(json.dumps(data, ensure_ascii=False, indent=4))
@@ -94,7 +99,7 @@ def _find_last_tag(element: bs4.element.Tag, attr: str, game_name: str) -> str:
# 获取大图（小图快爬）
async def _modify_avatar_url(session: aiohttp.ClientSession, game_name: str, char_name: str):
if game_name == 'prts':
async with session.get(f'https://wiki.biligame.com/arknights/{char_name}', proxy=get_local_proxy(), timeout=7) as res:
async with session.get(f'https://wiki.biligame.com/arknights/{char_name}', timeout=7) as res:
soup = BeautifulSoup(await res.text(), 'lxml')
try:
img_url = str(soup.find('img', {'class': 'img-bg'})['srcset']).split(' ')[-2]
@@ -104,7 +109,7 @@ async def _modify_avatar_url(session: aiohttp.ClientSession, game_name: str, cha
if game_name == 'genshin':
return None
if game_name == 'pretty_card':
async with session.get(f'https://wiki.biligame.com/umamusume/{char_name}', proxy=get_local_proxy(), timeout=7) as res:
async with session.get(f'https://wiki.biligame.com/umamusume/{char_name}', timeout=7) as res:
soup = BeautifulSoup(await res.text(), 'lxml')
img_url = soup.find('div', {'class': 'support_card-left'}).find('div').find('img').get('src')
return img_url
@@ -118,7 +123,7 @@ async def _modify_avatar_url(session: aiohttp.ClientSession, game_name: str, cha
# except KeyError:
# img_url = str(soup.find('img', {'class': 'img-kk'})['src'])
# except TypeError:
# print(f'{char_name} 图片还未上传，跳过...')
# logger.info(f'{char_name} 图片还未上传，跳过...')
# img_url = ''
# return img_url
return None
@@ -127,49 +132,46 @@ async def _modify_avatar_url(session: aiohttp.ClientSession, game_name: str, cha
# 数据最后处理（是否需要额外数据或处理数据）
async def _last_check(data: dict, game_name: str, session: aiohttp.ClientSession):
if game_name == 'prts':
url = 'https://wiki.biligame.com/arknights/'
tasks = []
for key in data.keys():
async with session.get(f'https://wiki.biligame.com/arknights/{key}', proxy=get_local_proxy(), timeout=7) as res:
tasks.append(asyncio.ensure_future(_async_update_prts_extra_info(url, key, session)))
asyResult = await asyncio.gather(*tasks)
for x in asyResult:
for key in x.keys():
data[key]['获取途径'] = x[key]['获取途径']
if game_name == 'genshin':
for key in data.keys():
async with session.get(f'https://wiki.biligame.com/ys/{key}', timeout=7) as res:
soup = BeautifulSoup(await res.text(), 'lxml')
obtain = str(soup.find('table', {'class': 'wikitable'}).find('tbody').find_all('td')[-1])
obtain = re.search(r'<td.*?>([\s\S]*)</.*?', obtain).group(1)
obtain = obtain[:-1] if obtain[-1] == '\n' else obtain
if obtain.find('<br/>'):
obtain = obtain.split('<br/>')
elif obtain.find('<br>'):
obtain = obtain.split('<br>')
for i in range(len(obtain)):
if obtain[i].find('<a') != -1:
text = ''
for msg in obtain[i].split('</a>'):
r = re.search('>(.*)', msg)
if r:
text += r.group(1) + ' '
obtain[i] = obtain[i].split('<a')[0] + text[:-1] + obtain[i].split('</a>')[-1]
print(f'明日方舟获取额外信息....{obtain}')
data[key]['获取途径'] = obtain
# if game_name == 'genshin':
# for key in data.keys():
# async with session.get(f'https://wiki.biligame.com/ys/{key}', timeout=7) as res:
# soup = BeautifulSoup(await res.text(), 'lxml')
# trs = soup.find('div', {'class': 'poke-bg'}).find('table').find('tbody').find_all('tr')
# for tr in trs:
# if tr.find('th').text.find('常驻/限定') != -1:
# data[key]['常驻/限定'] = tr.find('td').text
# break
_trs = ''
for table in soup.find_all('table', {'class': 'wikitable'}):
if str(table).find('常驻/限定') != -1:
_trs = table.find('tbody').find_all('tr')
break
for tr in _trs:
if str(tr).find('限定UP') != -1:
data[key]['常驻/限定'] = '限定UP'
logger.info(f'原神获取额外数据 {key}...{data[key]["常驻/限定"]}')
break
elif str(tr).find('常驻UP') != -1:
data[key]['常驻/限定'] = '常驻UP'
logger.info(f'原神获取额外数据 {key}...{data[key]["常驻/限定"]}')
break
if game_name == 'pretty':
for keys in data.keys():
for key in data[keys].keys():
r = re.search(r'.*?40px-(.*)图标.png', str(data[keys][key]))
if r:
data[keys][key] = r.group(1)
print(f'赛马娘额外修改数据....{keys}[{key}]=> {r.group(1)}')
logger.info(f'赛马娘额外修改数据...{keys}[{key}]=> {r.group(1)}')
if game_name == 'guardian':
for keys in data.keys():
for key in data[keys].keys():
r = re.search(r'.*?-star_(.*).png', str(data[keys][key]))
if r:
data[keys][key] = r.group(1)
print(f'坎公骑士剑额外修改数据...{keys}[{key}] => {r.group(1)}')
logger.info(f'坎公骑士剑额外修改数据...{keys}[{key}] => {r.group(1)}')
return data
@@ -199,6 +201,7 @@ def init_attr(game_name: str):
return att_dict, start_index, index
# 解析key
def parse_key(key: str, game_name):
attr = ''
if game_name == 'genshin_arms':
@@ -209,11 +212,17 @@ def parse_key(key: str, game_name):
return key, attr
def replace_name(member_dict: dict, game_name: str):
# 拿到名称
def replace_update_name(member_dict: dict, game_name: str):
name = member_dict['名称']
if game_name == 'pretty_card':
name = member_dict['中文名']
return name
name = remove_prohibited_str(name)
member_dict['中文名'] = name
else:
name = remove_prohibited_str(name)
member_dict['名称'] = name
return member_dict, name
# 拿到tbody，不同游戏tbody可能不同
@@ -234,3 +243,32 @@ def get_tbody(soup: bs4.BeautifulSoup, game_name: str, url: str):
max_count = len(tbody.find_all('tr'))
return _tbody
async def _async_update_prts_extra_info(url: str, key: str, session: aiohttp.ClientSession):
for i in range(10):
try:
async with session.get(f'https://wiki.biligame.com/arknights/{key}', timeout=7) as res:
soup = BeautifulSoup(await res.text(), 'lxml')
obtain = str(soup.find('table', {'class': 'wikitable'}).find('tbody').find_all('td')[-1])
obtain = re.search(r'<td.*?>([\s\S]*)</.*?', obtain).group(1)
obtain = obtain[:-1] if obtain[-1] == '\n' else obtain
if obtain.find('<br/>'):
obtain = obtain.split('<br/>')
elif obtain.find('<br>'):
obtain = obtain.split('<br>')
for i in range(len(obtain)):
if obtain[i].find('<a') != -1:
text = ''
for msg in obtain[i].split('</a>'):
r = re.search('>(.*)', msg)
if r:
text += r.group(1) + ' '
obtain[i] = obtain[i].split('<a')[0] + text[:-1] + obtain[i].split('</a>')[-1]
logger.info(f'明日方舟获取额外信息 {key}...{obtain}')
x = {key: {}}
x[key]['获取途径'] = obtain
return x
except TimeoutError:
logger.warning(f'访问{url}{key} 第 {i}次 超时...已再次访问')
return {}