Merge-build: v2.7.0

Author: jasoneri

.gitignore

@@ -174,6 +174,7 @@ analyze/
 *.d
 *.zip
 *.psd
+*.clip
 *demo*.py
 __temp
 test/*

ComicSpider/middlewares.py

@@ -129,13 +129,17 @@ def process_request(self, request, spider):
 
 
 class ComicDlProxyMiddleware(ComicspiderDownloaderMiddleware):
-    """使用情况是“通常页需要over wall访问”，“图源cn就能访问”... 因此domain的都使用代理"""
+    """使用情况是"通常页需要over wall访问"，"图源cn就能访问"... 因此domain的都使用代理
+    Spider 可定义 proxy_domains 列表指定需要代理的域名，未定义则回退到 domain
+    """
     domain_regex: re.Pattern = None
 
     @classmethod
     def from_crawler(cls, crawler):
         _ = super(ComicDlProxyMiddleware, cls).from_crawler(crawler)
-        _.domain_regex = re.compile(crawler.spider.domain)
+        spider = crawler.spider
+        domains = getattr(spider, 'proxy_domains', None) or [spider.domain]
+        _.domain_regex = re.compile('|'.join(re.escape(d) for d in domains))
         return _
 
     def process_request(self, request, spider):

ComicSpider/pipelines.py

@@ -71,11 +71,8 @@ def file_folder(self, basepath, section, spider, title, item):
 
         os.makedirs(path, exist_ok=True)
         tasks_obj = spider.tasks.get(uuid_md5)
-        if tasks_obj and getattr(tasks_obj, 'comic_info', None):
-            comic_info_path = path.joinpath("ComicInfo.xml")
-            if not comic_info_path.exists():
-                with open(comic_info_path, 'w', encoding='utf-8') as f:
-                    f.write(tasks_obj.comic_info.out)
+        if tasks_obj and getattr(tasks_obj, 'meta_info', None):
+            tasks_obj.meta_info.sv_meta_in(path)
 
         spider.tasks_path[uuid_md5] = path
         return path
@@ -98,6 +95,9 @@ def handle_task(spider, stats, task_obj):
         _tasks.downloaded.append(task_obj)
         curr_progress = int(len(_tasks.downloaded) / _tasks.tasks_count * 100)
         if conf.isDeduplicate and curr_progress >= 100:
+            tasks_obj = spider.tasks[task_obj.taskid]
+            if getattr(tasks_obj, 'meta_info', None):
+                tasks_obj.meta_info.fin_callback(spider.tasks_path[tasks_obj.taskid])
             spider.sql_handler.add(task_obj.taskid)
         spider.Q('TasksQueue').send(task_obj, wait=True)
         stats.inc_value('image/downloaded')

ComicSpider/spiders/basecomicspider.py

@@ -233,7 +233,7 @@ def parse_section(self, response):
         for ep in book.episodes:
             url_list = self.mk_page_tasks(url=ep.url)
             now_start_crawl_desc = self.res.parse_sec_now_start_crawl_desc % book.name
-            self.say(font_color(f"📢	{now_start_crawl_desc}：{ep}", cls='theme-tip', size=5))
+            self.say(font_color(f"📢\t{now_start_crawl_desc}：{ep}", cls='theme-tip', size=5))
             for url in url_list:
                 yield scrapy.Request(url=url, callback=self.parse_fin_page, meta={'ep': ep})
 
@@ -258,9 +258,8 @@ def mk_page_tasks(self, *arg, **kw) -> iter:
         return [kw['url']]
 
     def set_task(self, task_info: t.Union[BookInfo, Episode]):
-        comic_info_obj = self.mr.out(task_info)
         tasks_obj = task_info.to_tasks_obj()
-        tasks_obj.comic_info = comic_info_obj
+        tasks_obj.meta_info = self.mr.toMetaInfo(task_info)
         self.tasks[tasks_obj.taskid] = tasks_obj
         self.Q('TasksQueue').send(tasks_obj, wait=True)
 
@@ -298,7 +297,7 @@ def from_crawler(cls, crawler, *args, **kwargs):
         spider.say = SayToGui(spider, q, spider.text_browser_state)
         spider.sql_handler = SqlUtils()
         spider.ut = spider_utils_map[spider.name]
-        spider.mr = MetaRecorder()
+        spider.mr = MetaRecorder(conf)
         return spider
 
     def _remove_cache(self):
@@ -329,7 +328,7 @@ def close(self, reason):
 
     def _handle_finished_status(self, stats):
         if 'init' in self.process_state.process:
-            self.say(font_color('unknown init end,<br>if error occur, please contact maintainer with operation-process', cls='theme-err', size=6))
+            self.say(font_color('unknown init end, if cgs not work, please contact maintainer with log', cls='theme-tip'))
             return
         downloaded_count = stats.get_value('image/downloaded', 0)
         exception_count = stats.get_value('process_exception/count', 0)

ComicSpider/spiders/kaobei.py

@@ -53,7 +53,7 @@ class KaobeiSpider(BaseComicSpider):
         'Accept': '*/*',
         'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
         'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
-        'dnts': '2',
+        'dnts': '3',
         'Connection': 'keep-alive',
         'Sec-Fetch-Dest': 'empty',
         'Sec-Fetch-Mode': 'cors',
@@ -75,6 +75,7 @@ class KaobeiSpider(BaseComicSpider):
     }
     domain = domain
     pc_domain = pc_domain
+    proxy_domains = [domain, pc_domain]  # 需要代理的域名列表
     custom_settings = {
         "DOWNLOADER_MIDDLEWARES": {'ComicSpider.middlewares.UAKaobeiMiddleware': 5,
                                    'ComicSpider.middlewares.ComicDlProxyMiddleware': 6},

GUI/browser_window.py

@@ -84,7 +84,15 @@ def setupUi(self, _window):
         self.set_btn()
         self.set_html()
         self.patch_tip()
-        FluentMonkeyPatch.rbutton_menu_WebEngine(self)
+        if self.is_publish_page():
+            FluentMonkeyPatch.rbutton_menu_PulishPage(self)
+        else:
+            FluentMonkeyPatch.rbutton_menu_WebEngine(self)
+
+    def is_publish_page(self) -> bool:
+        if hasattr(self.gui, 'tf') and self.gui.tf:
+            return 'publish' in str(self.gui.tf).lower()
+        return False
 
     def patch_tip(self):
         for button in (self.topHintBox, self.homeBtn, self.backBtn, self.forwardBtn, self.refreshBtn, self.copyBtn, self.ensureBtn):
@@ -135,8 +143,9 @@ def second_init(self):
         self.home_url = QUrl.fromLocalFile(self.gui.tf)
         self.load_home()
 
-    def keep_top_hint(self):
-        if self.topHintBox.isChecked():
+    def keep_top_hint(self, _flag: bool = None):
+        flag = _flag if _flag is not None else self.topHintBox.isChecked()
+        if flag:
             self.setWindowFlags(Qt.WindowStaysOnTopHint)
         else:
             self.setWindowFlags(Qt.Widget)

GUI/conf_dialog.py

@@ -13,7 +13,7 @@
 from PyQt5.QtCore import QUrl, Qt
 from qfluentwidgets import (
     FluentIcon as FIF, PushButton, PrimaryPushButton, TransparentPushButton, 
-    PushSettingCard, InfoBarPosition, TransparentToggleToolButton, InfoBar
+    PushSettingCard, InfoBarPosition, TransparentToggleToolButton, InfoBar, ComboBox
 )
 import uncurl
 
@@ -69,12 +69,18 @@ def setupUi(self, Dialog):
         self._repaint_textEdit()
 
     def retranslateUiAgain(self, Dialog):
+        self.langBox = ComboBox(Dialog)
+        self.langBox.setObjectName("langBox")
+        self.langBox.setCurrentIndex(-1)
+        self.pypiSourceBox = ComboBox(Dialog)
+        self.pypiSourceBox.setObjectName("pypiSourceBox")
         _translate = QtCore.QCoreApplication.translate
         self.label_2.setText(_translate("Dialog", res.GUI.Uic.confDia_labelLogLevel))
         self.label_4.setText(_translate("Dialog", res.GUI.Uic.confDia_labelProxy))
         self.label_completer.setText(_translate("Dialog", res.GUI.Uic.confDia_labelPreset))
         self.label_6.setText(_translate("Dialog", res.GUI.Uic.confDia_labelClipDb))
         self.label_7.setText(_translate("Dialog", res.GUI.Uic.confDia_labelClipNum))
+        self.metaTypeLabel.setText(_translate("Dialog", res.GUI.Uic.confDia_metaType))
         self.concurr_numLabel.setText(_translate("Dialog", res.GUI.Uic.confDia_labelConcurrNum))
         # 添加cookie类型选项
         support = list(COOKIES_PLACEHOLDER.keys())
@@ -88,6 +94,11 @@ def retranslateUiAgain(self, Dialog):
         self.pypiSourceBox.setItemText(1, _translate("Dialog", "清华源"))
         self.pypiSourceBox.setItemText(2, _translate("Dialog", "阿里源"))
         self.pypiSourceBox.setItemText(3, _translate("Dialog", "华为源"))
+        self.metaTypeBox.addItem("")
+        self.metaTypeBox.addItem("")
+        self.metaTypeBox.setItemText(0, _translate("Dialog", "-"))
+        self.metaTypeBox.setItemText(1, _translate("Dialog", "ComicInfo.xml"))
+        self.metaTypeBox.setCurrentIndex(0)
         self.cookiesBox.setCurrentText(support[0])
 
         for k, ui_key in LANG.items():
@@ -182,6 +193,19 @@ def _tip_lang_change(idx):
                     duration=5000, parent=self
                 )
         self.langBox.activated.connect(_tip_lang_change)
+        def _tip_meta_change(idx):
+            match self.metaTypeBox.itemText(idx):
+                case "ComicInfo.xml":
+                    _meta_tip = "适配平台例如：ComicRack, Komga, kavita"
+                case _:
+                    _meta_tip = ""
+            if _meta_tip:
+                InfoBar.success(
+                    title="", content=_meta_tip,
+                    orient=Qt.Horizontal, isClosable=True, position=InfoBarPosition.TOP,
+                    duration=5000, parent=self
+                )
+        self.metaTypeBox.activated.connect(_tip_meta_change)
         self.cookiesBox.currentTextChanged.connect(self._on_cookie_type_changed)
         def _tip_on(is_checked: bool, tip_content=None):
             if is_checked:
@@ -213,6 +237,7 @@ def show_self(self):  # can't naming `show`. If done, just run code once
         self.logLevelComboBox.setCurrentIndex(self.logLevelComboBox.findText(getattr(conf, "log_level")))
         self.pypiSourceBox.setCurrentIndex(getattr(conf, "pypi_source"))
         self.langBox.setCurrentIndex(self.langBox.findData(getattr(conf, "lang")))
+        self.metaTypeBox.setCurrentIndex(self.metaTypeBox.findText(getattr(conf, "meta_type")))
         # 仅当 初次confdia ui创建 & conf值设入ui后，才绑定槽函数
         if self._init_flag:
             self.bind_logic()
@@ -254,6 +279,7 @@ def save_conf(self):
             "sv_path": sv_path,
             "log_level": getattr(self, "logLevelComboBox").currentText(),
             "lang": getattr(self, "langBox").currentData(),
+            "meta_type": getattr(self, "metaTypeBox").currentText(),
             "concurr_num": getattr(self, "concurr_numEdit").value(),
             "isDeduplicate": getattr(self, "isDeduplicate").isChecked(),
             "addUuid": getattr(self, "addUuid").isChecked(),

GUI/gui.py

@@ -5,7 +5,7 @@
 import traceback
 from multiprocessing import Process
 import multiprocessing.managers as m
-from PyQt5.QtGui import QKeySequence
+from PyQt5.QtGui import QKeySequence, QGuiApplication
 from PyQt5.QtCore import QThread, Qt, QCoreApplication, QRect, QTimer
 from PyQt5.QtWidgets import QMainWindow, QCompleter, QShortcut
 
@@ -18,16 +18,17 @@
 from GUI.conf_dialog import ConfDialog
 from GUI.browser_window import BrowserWindow as BrowserWindowCls
 from GUI.thread import WorkThread, QueueInitThread
-from GUI.tools import ToolWindow, TextUtils
+from GUI.tools import ToolWindow, TextUtils, DomainToolView
 from GUI.manager import TaskProgressManager, ClipGUIManager, AggrSearchManager
 from GUI.manager.preprocess import PreprocessManager
+from GUI.uic.qfluent import CustomFlyout
 from variables import *
 from assets import res
-from utils import Queues, QueuesManager, conf, p, curr_os, select
+from utils import Queues, QueuesManager, conf, p, curr_os, select, ori_path, bs_theme
 from utils.processed_class import (
     InputFieldState, TextBrowserState, ProcessState,
     GuiQueuesManger, refresh_state, crawl_what,
-    PreviewHtml
+    PreviewHtml, TmpFormatHtml
 )
 from utils.website import spider_utils_map, InfoMinix
 from utils.sql import SqlUtils
@@ -584,3 +585,24 @@ def hook_exception(self, exc_type, exc_value, exc_traceback):
         self.log.error(exception)
         self.say(font_color(rf"{type(exc_value)}{exc_value}", cls='theme-err', size=4), ignore_http=True)
         self.say(font_color(rf"<br>{self.res.global_err_hook} <br>[{conf.log_path}\GUI.log]<br>", cls='theme-err', size=5))
+
+    def do_publish(self):
+        with open(ori_path.joinpath('assets/pubilsh_helper.html')) as f:
+            format_text = f.read()
+            html = format_text.replace("{bs_theme}", bs_theme()) \
+                    .replace("{publish_url}", self.spiderUtils.publish_url)
+            self.tf = TmpFormatHtml.created_temp_html(html, flag="publish")
+        self.set_preview()
+        screen_width = QGuiApplication.primaryScreen().availableGeometry().width()
+        o_w, o_h = self.BrowserWindow.width(), self.BrowserWindow.height()
+        if o_w < screen_width * 0.75:
+            o_w = int(screen_width * 0.75)
+        self.BrowserWindow.resize(o_w, o_h+150)
+        self.BrowserWindow.show()
+
+    def tpd(self, texts):
+        self.BrowserWindow.domain_v = DomainToolView(self)
+        CustomFlyout.make(
+            view=self.BrowserWindow.domain_v, target=self.BrowserWindow, parent=self.BrowserWindow
+        )
+        self.BrowserWindow.domain_v.handle(texts)

GUI/manager/preprocess.py

@@ -39,7 +39,6 @@ def handle_choosebox_changed(self, index: int):
                     self._preprocess_jm()
                 else:
                     self.gui.say("🔔 已设置代理，跳过域名缓存处理")
-                    self.gui.toolWin.addDomainTool()
                     self.gui.toolWin.addAggrSearchView()
             case 4:
                 self._preprocess_ehentai()
@@ -88,14 +87,14 @@ def on_success(_):
 
         def on_error(_):
             self.gui.disable_start()
-            self.gui.say("<br>❌ 域名获取/测试失效，点击 rV按钮 > domainTool, 按指示操作")
+            self.gui.say("<br>❌ 域名获取/测试失效，按内置浏览器引导操作")
+            self.gui.do_publish()
 
         self.task_manager.execute_simple_task(
             task_func=task,
             success_callback=on_success, show_error_info=self.show_err, error_callback=on_error,
             tooltip_title="更新域名缓存", task_id="domain_preprocess"
         )
-        self.gui.toolWin.addDomainTool()
         self.gui.toolWin.addAggrSearchView()
 
     def _preprocess_ehentai(self):