提供全局字典GDict，适配omega 13w张图的数据结构表

plugins/pix_gallery/__init__.py

@@ -1,4 +1,5 @@
 from configs.config import Config
+from utils.manager import GDict
 import nonebot
 
 
@@ -60,6 +61,8 @@ Config.add_plugin_config(
     default_value=True
 )
 
+GDict['run_sql'].append("ALTER TABLE omega_pixiv_illusts ADD classified Integer;")
+
 nonebot.load_plugins("plugins/pix_gallery")
 
 

plugins/pix_gallery/_model/omega_pixiv_illusts.py

@@ -1,5 +1,4 @@
 from typing import Optional, List, Tuple
-from datetime import datetime
 from services.db_context import db
 
 
@@ -12,13 +11,12 @@ class OmegaPixivIllusts(db.Model):
     uid = db.Column(db.BigInteger(), nullable=False)
     title = db.Column(db.String(), nullable=False)
     uname = db.Column(db.String(), nullable=False)
+    classified = db.Column(db.Integer(), nullable=False)
     nsfw_tag = db.Column(db.Integer(), nullable=False)
     width = db.Column(db.Integer(), nullable=False)
     height = db.Column(db.Integer(), nullable=False)
     tags = db.Column(db.String(), nullable=False)
     url = db.Column(db.String(), nullable=False)
-    created_at = db.Column(db.DateTime(timezone=True))
-    updated_at = db.Column(db.DateTime(timezone=True))
 
     _idx1 = db.Index("omega_pixiv_illusts_idx1", "pid", "url", unique=True)
 
@@ -32,10 +30,9 @@ class OmegaPixivIllusts(db.Model):
             url: str,
             uid: int,
             uname: str,
+            classified: int,
             nsfw_tag: int,
             tags: str,
-            created_at: datetime,
-            updated_at: datetime,
     ):
         """
         说明:
@@ -48,10 +45,9 @@ class OmegaPixivIllusts(db.Model):
             :param url: url链接
             :param uid: 作者uid
             :param uname: 作者名称
-            :param nsfw_tag: nsfw标签, 0=safe, 1=setu. 2=r18
+            :param classified: 标记标签, 0=未标记, 1=已人工标记或从可信已标记来源获取
+            :param nsfw_tag: nsfw标签,-1=未标记, 0=safe, 1=setu. 2=r18
             :param tags: 相关tag
-            :param created_at: 创建日期
-            :param updated_at: 更新日期
         """
         if not await cls.check_exists(pid):
             await cls.create(
@@ -62,6 +58,7 @@ class OmegaPixivIllusts(db.Model):
                 url=url,
                 uid=uid,
                 uname=uname,
+                classified=classified,
                 nsfw_tag=nsfw_tag,
                 tags=tags,
             )

plugins/pix_gallery/pix_update.py

@@ -1,3 +1,5 @@
+import re
+
 from nonebot import on_command
 from utils.utils import is_number
 from nonebot.permission import SUPERUSER
@@ -11,7 +13,6 @@ import time
 from services.log import logger
 from pathlib import Path
 from typing import List
-from datetime import datetime
 import asyncio
 import os
 
@@ -144,38 +145,36 @@ async def _(arg: Message = CommandArg()):
 @check_omega.handle()
 async def _():
     async def _tasks(line: str, all_pid: List[int], length: int, index: int):
-        data = line.split("VALUES", maxsplit=1)[-1].strip()
-        if data.startswith("("):
-            data = data[1:]
-        if data.endswith(");"):
-            data = data[:-2]
-        x = data.split(maxsplit=3)
-        pid = int(x[1][:-1].strip())
+        data = line.split("VALUES", maxsplit=1)[-1].strip()[1:-2]
+        num_list = re.findall(r'(\d+)', data)
+        pid = int(num_list[1])
+        uid = int(num_list[2])
+        id_ = 3
+        while num_list[id_] not in ['0', '1']:
+            id_ += 1
+        classified = int(num_list[id_])
+        nsfw_tag = int(num_list[id_ + 1])
+        width = int(num_list[id_ + 2])
+        height = int(num_list[id_ + 3])
+        str_list = re.findall(r"'(.*?)',", data)
+        title = str_list[0]
+        uname = str_list[1]
+        tags = str_list[2]
+        url = str_list[3]
         if pid in all_pid:
             logger.info(f"添加OmegaPixivIllusts图库数据已存在 ---> pid：{pid}")
             return
-        uid = int(x[2][:-1].strip())
-        x = x[3].split(", '")
-        title = x[0].strip()[1:-1]
-        tmp = x[1].split(", ")
-        author = tmp[0].strip()[:-1]
-        nsfw_tag = int(tmp[1])
-        width = int(tmp[2])
-        height = int(tmp[3])
-        tags = x[2][:-1]
-        url = x[3][:-1]
         if await OmegaPixivIllusts.add_image_data(
-                pid,
-                title,
-                width,
-                height,
-                url,
-                uid,
-                author,
-                nsfw_tag,
-                tags,
-                datetime.min,
-                datetime.min,
+                pid=pid,
+                title=title,
+                width=width,
+                height=height,
+                url=url,
+                uid=uid,
+                nsfw_tag=nsfw_tag,
+                tags=tags,
+                uname=uname,
+                classified=classified
         ):
             logger.info(
                 f"成功添加OmegaPixivIllusts图库数据 pid：{pid} 本次预计存储 {length} 张，已更新第 {index} 张"
@@ -197,6 +196,7 @@ async def _():
         for line in lines:
             if "INSERT INTO" in line.upper():
                 index += 1
+                logger.info(f'line: {line} 加入更新计划')
                 tasks.append(
                     asyncio.ensure_future(_tasks(line, all_pid, length, index))
                 )