v2.3.16: 修复zip插件在album模式下的一些bug; 调整获取APP_COOKIES的位置; 一些代码优化。 (#160)

Author: hect0x7

.github/release.py

@@ -18,7 +18,7 @@ def parse_body(body):
         e: str = e.strip()
         if e == '':
             continue
-        points.append(f'{i}. {e}')
+        points.append(f'{i + 1}. {e}')
 
     return '\n'.join(points)
 

src/jmcomic/__init__.py

@@ -2,7 +2,7 @@
 # 被依赖方 <--- 使用方
 # config <--- entity <--- toolkit <--- client <--- option <--- downloader
 
-__version__ = '2.3.15'
+__version__ = '2.3.16'
 
 from .api import *
 from .jm_plugin import *

src/jmcomic/jm_client_impl.py

@@ -493,13 +493,13 @@ def fetch_scramble_id(self, photo_id):
             }
         )
 
-        match = JmcomicText.pattern_html_album_scramble_id.search(resp.text)
-
-        if match is not None:
-            scramble_id = match[1]
-        else:
-            jm_debug('api.scramble', '未从响应中匹配到scramble_id，返回默认值220980')
-            scramble_id = '220980'
+        scramble_id = PatternTool.match_or_default(resp.text,
+                                                   JmcomicText.pattern_html_album_scramble_id,
+                                                   None,
+                                                   )
+        if scramble_id is None:
+            jm_debug('api.scramble', f'未匹配到scramble_id，响应文本：{resp.text}')
+            scramble_id = str(JmModuleConfig.SCRAMBLE_220980)
 
         return scramble_id
 
@@ -610,15 +610,12 @@ def require_resp_success(cls, resp: JmApiResp, orig_req_url: str):
         # 2. 是否是特殊的内容
         # 暂无
 
-    @classmethod
-    @field_cache('__init_cookies__')
-    def fetch_init_cookies(cls, client: 'JmApiClient'):
-        resp = client.setting()
-        return dict(resp.resp.cookies)
-
     def after_init(self):
-        cookies = self.__class__.fetch_init_cookies(self)
-        self.get_root_postman().get_meta_data()['cookies'] = cookies
+        # cookies = self.__class__.fetch_init_cookies(self)
+        # self.get_root_postman().get_meta_data()['cookies'] = cookies
+
+        self.get_root_postman().get_meta_data()['cookies'] = JmModuleConfig.get_cookies(self)
+        pass
 
 
 class FutureClientProxy(JmcomicClient):

src/jmcomic/jm_config.py

@@ -3,7 +3,7 @@ def field_cache(*args, **kwargs):
     return field_cache(*args, **kwargs)
 
 
-def default_jm_debug(topic: str, msg: str):
+def default_jm_debug_logging(topic: str, msg: str):
     from common import format_ts
     print(f'{format_ts()}:【{topic}】{msg}')
 
@@ -62,13 +62,14 @@ class JmModuleConfig:
     SCRAMBLE_421926 = 421926  # 2023-02-08后改了图片切割算法
     SCRAMBLE_CACHE = {}
 
-    # 移动端API的相关配置
-    # API密钥
-    APP_SECRET = '18comicAPP'
+    # 移动端API密钥
+    APP_SECRET = '18comicAPPContent'
 
-    # 域名配置 - 移动端
-    # 图片域名
-    DOMAIN_API_IMAGE_LIST = str_to_list('''
+    # cookies，目前只在移动端使用，因为移动端请求接口须携带，但不会校验cookies的内容。
+    APP_COOKIES = None
+
+    # 移动端图片域名
+    DOMAIN_IMAGE_LIST = str_to_list('''
     cdn-msp.jmapiproxy1.monster
     cdn-msp2.jmapiproxy1.monster
     cdn-msp.jmapiproxy1.cc
@@ -78,7 +79,7 @@ class JmModuleConfig:
 
     ''')
 
-    # API域名
+    # 移动端API域名
     DOMAIN_API_LIST = str_to_list('''
     www.jmapinode1.top
     www.jmapinode2.top
@@ -88,8 +89,11 @@ class JmModuleConfig:
     
     ''')
 
-    # 域名配置 - 网页端
+    # 网页端域名配置
     # 无需配置，默认为None，需要的时候会发起请求获得
+    # 使用优先级:
+    # 1. DOMAIN_HTML_LIST
+    # 2. [DOMAIN_HTML]
     DOMAIN_HTML = None
     DOMAIN_HTML_LIST = None
 
@@ -106,7 +110,7 @@ class JmModuleConfig:
     REGISTRY_PLUGIN = {}
 
     # 执行debug的函数
-    debug_executor = default_jm_debug
+    debug_executor = default_jm_debug_logging
     # postman构造函数
     postman_constructor = default_postman_constructor
     # 网页正则表达式解析失败时，执行抛出异常的函数，可以替换掉用于debug
@@ -190,7 +194,7 @@ def get_html_url(cls, postman=None):
         postman = postman or cls.new_postman(session=True)
 
         url = postman.with_redirect_catching().get(cls.JM_REDIRECT_URL)
-        cls.jm_debug('获取禁漫网页URL', f'[{cls.JM_REDIRECT_URL}] → [{url}]')
+        cls.jm_debug('module.html_url', f'获取禁漫网页URL: [{cls.JM_REDIRECT_URL}] → [{url}]')
         return url
 
     @classmethod
@@ -211,9 +215,22 @@ def get_html_domain_all(cls, postman=None):
         from .jm_toolkit import JmcomicText
         domain_list = JmcomicText.analyse_jm_pub_html(resp.text)
 
-        cls.jm_debug('获取禁漫网页全部域名', f'[{resp.url}] → {domain_list}')
+        cls.jm_debug('module.html_domain_all', f'获取禁漫网页全部域名: [{resp.url}] → {domain_list}')
         return domain_list
 
+    @classmethod
+    @field_cache("APP_COOKIES")
+    def get_cookies(cls, postman=None):
+        from .jm_toolkit import JmcomicText
+        url = JmcomicText.format_url('/setting', cls.DOMAIN_API_LIST[0])
+        postman = postman or cls.new_postman()
+
+        resp = postman.get(url)
+        cookies = dict(resp.cookies)
+
+        cls.jm_debug('module.cookies', f'获取cookies: [{url}] → {cookies}')
+        return cookies
+
     @classmethod
     def new_html_headers(cls, domain='18comic.vip'):
         """
@@ -247,7 +264,7 @@ def new_api_headers(cls, key_ts):
             key_ts = time_stamp()
 
         import hashlib
-        token = hashlib.md5(f"{key_ts}{cls.APP_SECRET}".encode()).hexdigest()
+        token = hashlib.md5(f"{key_ts}{cls.APP_SECRET}".encode("utf-8")).hexdigest()
 
         return {
             'token': token,

src/jmcomic/jm_option.py

@@ -314,6 +314,7 @@ def build_jm_client(self, **kwargs):
     def new_jm_client(self, domain=None, impl=None, cache=None, **kwargs) -> JmcomicClient:
         # 所有需要用到的 self.client 配置项如下
         postman_conf: dict = self.client.postman.src_dict  # postman dsl 配置
+        meta_data: dict = postman_conf['meta_data']  # 请求元信息
         impl: str = impl or self.client.impl  # client_key
         retry_times: int = self.client.retry_times  # 重试次数
         cache: str = cache or self.client.cache  # 启用缓存
@@ -335,15 +336,11 @@ def decide_domain():
 
         # support kwargs overwrite meta_data
         if len(kwargs) != 0:
-            postman_conf['meta_data'].update(kwargs)
+            meta_data.update(kwargs)
 
         # headers
-        meta_data = postman_conf['meta_data']
         if meta_data['headers'] is None:
-            headers = self.decide_postman_headers(impl, domain[0])
-            # if headers is None:
-            #     postman_conf['type'] = 'requests'
-            meta_data['headers'] = headers
+            meta_data['headers'] = self.decide_postman_headers(impl, domain[0])
 
         # postman
         postman = Postmans.create(data=postman_conf)
@@ -375,6 +372,9 @@ def decide_client_domain(self, client_key: str) -> List[str]:
 
         if is_client_type(JmHtmlClient):
             # 网页端
+            domain_list = JmModuleConfig.DOMAIN_HTML_LIST
+            if domain_list is not None:
+                return domain_list
             return [JmModuleConfig.get_html_domain()]
 
         ExceptionTool.raises(f'没有配置域名，且是无法识别的client类型: {client_key}')

src/jmcomic/jm_plugin.py

@@ -249,26 +249,32 @@ def invoke(self,
         mkdir_if_not_exists(zip_dir)
 
         # 原文件夹 -> zip文件
-        dir_zip_dict = {}
+        dir_zip_dict: Dict[str, Optional[str]] = {}
         photo_dict = downloader.all_downloaded[album]
 
         if level == 'album':
             zip_path = self.get_zip_path(album, None, filename_rule, suffix, zip_dir)
             dir_path = self.zip_album(album, photo_dict, zip_path)
-            dir_zip_dict[dir_path] = zip_path
+            if dir_path is not None:
+                # 要删除这个album文件夹
+                dir_zip_dict[dir_path] = zip_path
+                # 也要删除album下的photo文件夹
+                for d in files_of_dir(dir_path):
+                    dir_zip_dict[d] = None
 
         elif level == 'photo':
             for photo, image_list in photo_dict.items():
                 zip_path = self.get_zip_path(None, photo, filename_rule, suffix, zip_dir)
                 dir_path = self.zip_photo(photo, image_list, zip_path)
-                dir_zip_dict[dir_path] = zip_path
+                if dir_path is not None:
+                    dir_zip_dict[dir_path] = zip_path
 
         else:
             ExceptionTool.raises(f'Not Implemented Zip Level: {level}')
 
         self.after_zip(dir_zip_dict)
 
-    def zip_photo(self, photo, image_list: list, zip_path: str):
+    def zip_photo(self, photo, image_list: list, zip_path: str) -> Optional[str]:
         """
         压缩photo文件夹
         :returns: photo文件夹路径
@@ -277,46 +283,54 @@ def zip_photo(self, photo, image_list: list, zip_path: str):
             if len(image_list) == 0 \
             else os.path.dirname(image_list[0][0])
 
-        all_filepath = set(map(lambda t: t[0], image_list))
+        all_filepath = set(map(lambda t: self.unified_path(t[0]), image_list))
 
-        if len(all_filepath) == 0:
-            self.debug('无下载文件，无需压缩', 'skip')
-            return
-
-        from common import backup_dir_to_zip
-        backup_dir_to_zip(photo_dir, zip_path, acceptor=lambda f: f in all_filepath)
-        self.debug(f'压缩章节[{photo.photo_id}]成功 → {zip_path}', 'finish')
+        return self.do_zip(photo_dir,
+                           zip_path,
+                           all_filepath,
+                           f'压缩章节[{photo.photo_id}]成功 → {zip_path}',
+                           )
 
-        return photo_dir
+    @staticmethod
+    def unified_path(f):
+        return fix_filepath(f, os.path.isdir(f))
 
-    def zip_album(self, album, photo_dict: dict, zip_path):
+    def zip_album(self, album, photo_dict: dict, zip_path) -> Optional[str]:
         """
         压缩album文件夹
         :returns: album文件夹路径
         """
-        album_dir = self.option.decide_album_dir(album)
         all_filepath: Set[str] = set()
 
-        for image_list in photo_dict.values():
-            image_list: List[Tuple[str, JmImageDetail]]
-            for path, _ in image_list:
-                all_filepath.add(path)
+        def addpath(f):
+            all_filepath.update(set(f))
+
+        album_dir = self.option.decide_album_dir(album)
+        # addpath(self.option.decide_image_save_dir(photo) for photo in photo_dict.keys())
+        addpath(path for ls in photo_dict.values() for path, _ in ls)
+
+        return self.do_zip(album_dir,
+                           zip_path,
+                           all_filepath,
+                           msg=f'压缩本子[{album.album_id}]成功 → {zip_path}',
+                           )
 
+    def do_zip(self, source_dir, zip_path, all_filepath, msg):
         if len(all_filepath) == 0:
             self.debug('无下载文件，无需压缩', 'skip')
-            return
+            return None
 
         from common import backup_dir_to_zip
         backup_dir_to_zip(
-            album_dir,
+            source_dir,
             zip_path,
-            acceptor=lambda f: f in all_filepath
-        )
+            acceptor=lambda f: os.path.isdir(f) or self.unified_path(f) in all_filepath
+        ).close()
 
-        self.debug(f'压缩本子[{album.album_id}]成功 → {zip_path}', 'finish')
-        return album_dir
+        self.debug(msg, 'finish')
+        return self.unified_path(source_dir)
 
-    def after_zip(self, dir_zip_dict: Dict[str, str]):
+    def after_zip(self, dir_zip_dict: Dict[str, Optional[str]]):
         # 是否要删除所有原文件
         if self.delete_original_file is True:
             self.delete_all_files_and_empty_dir(
@@ -352,10 +366,10 @@ def delete_all_files_and_empty_dir(self, all_downloaded: dict, dir_list: List[st
                     os.remove(f)
                     self.debug(f'删除原文件: {f}', 'remove')
 
-        for d in dir_list:
+        for d in sorted(dir_list, reverse=True):
             # check exist
-            if file_exists(d) and len(os.listdir(d)) == 0:
-                os.removedirs(d)
+            if file_exists(d):
+                os.rmdir(d)
                 self.debug(f'删除文件夹: {d}', 'remove')
 
 

src/jmcomic/jm_toolkit.py

@@ -489,7 +489,7 @@ def post_adapt_photo(cls, data: dict, _clazz: type, fields: dict):
 
         fields['sort'] = sort
         import random
-        fields['data_original_domain'] = random.choice(JmModuleConfig.DOMAIN_API_IMAGE_LIST)
+        fields['data_original_domain'] = random.choice(JmModuleConfig.DOMAIN_IMAGE_LIST)
 
 
 class JmImageTool:

tests/test_jmcomic/test_jm_custom.py

@@ -55,8 +55,10 @@ class MyClient(JmHtmlClient):
         JmModuleConfig.register_client(MyClient)
 
         html_domain = self.client.get_html_domain()
+        JmModuleConfig.DOMAIN_HTML_LIST = [html_domain]
+
         self.assertListEqual(
-            [html_domain],
+            JmModuleConfig.DOMAIN_HTML_LIST,
             self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list()
         )