v2.5.9: 新增插件【跳过下载图片数少的章节】，可过滤一些公告章节; 新增Downloadable基类，对插件扩展更友好; 修复本子名过长超出导致oserror的错误; 升级curl_cffi. (#222)

Author: hect0x7

assets/docs/sources/option_file_syntax.md

@@ -198,6 +198,11 @@ plugins:
         zip_enable: true # 对收藏夹进行压缩
         zip_filepath: ${JM_DOWNLOAD_DIR}/export.zip # 压缩文件路径
         zip_password: ${ZIP_PASSWORD} # 压缩密码
+  
+  before_photo:
+    - plugin: skip_photo_with_few_images # 跳过下载章节图片数量过少的章节。一些韩漫的章节是公告，没有实际内容，就可以用该插件来跳过下载这些章节。
+      kwargs:
+        at_least_image_count: 3
 
   after_photo:
     - plugin: j2p # jpg图片合成为一个pdf插件

requirements-dev.txt

@@ -1,5 +1,5 @@
-commonX
 curl_cffi
+commonX
 PyYAML
 Pillow
 psutil

setup.py

@@ -27,8 +27,8 @@
     package_dir={"": "src"},
     python_requires=">=3.7",
     install_requires=[
-        'commonX>=0.6.4',
         'curl_cffi',
+        'commonX',
         'PyYAML',
         'Pillow',
         'pycryptodome',

src/jmcomic/__init__.py

@@ -2,7 +2,7 @@
 # 被依赖方 <--- 使用方
 # config <--- entity <--- toolkit <--- client <--- option <--- downloader
 
-__version__ = '2.5.8'
+__version__ = '2.5.9'
 
 from .api import *
 from .jm_plugin import *

src/jmcomic/jm_client_impl.py

@@ -224,7 +224,7 @@ def append_params_to_url(self, url, params):
 
     # noinspection PyMethodMayBeStatic
     def decode(self, url: str):
-        if not JmModuleConfig.flag_decode_url_when_logging or '/search/' not in url:
+        if not JmModuleConfig.FLAG_DECODE_URL_WHEN_LOGGING or '/search/' not in url:
             return url
 
         from urllib.parse import unquote
@@ -767,7 +767,7 @@ def setting(self) -> JmApiResp:
         # 检查禁漫最新的版本号
         setting_ver = str(resp.model_data.version)
         # 禁漫接口的版本 > jmcomic库内置版本
-        if setting_ver > JmMagicConstants.APP_VERSION and JmModuleConfig.flag_use_version_newer_if_behind:
+        if setting_ver > JmMagicConstants.APP_VERSION and JmModuleConfig.FLAG_USE_VERSION_NEWER_IF_BEHIND:
             jm_log('api.setting', f'change APP_VERSION from [{JmMagicConstants.APP_VERSION}] to [{setting_ver}]')
             JmMagicConstants.APP_VERSION = setting_ver
 
@@ -883,7 +883,7 @@ def decide_headers_and_ts(self, kwargs, url):
             ts = time_stamp()
             token, tokenparam = JmCryptoTool.token_and_tokenparam(ts, secret=JmMagicConstants.APP_TOKEN_SECRET_2)
 
-        elif JmModuleConfig.flag_use_fix_timestamp:
+        elif JmModuleConfig.FLAG_USE_FIX_TIMESTAMP:
             ts, token, tokenparam = JmModuleConfig.get_fix_ts_token_tokenparam()
 
         else:
@@ -954,7 +954,7 @@ def raise_if_resp_should_retry(self, resp):
 
     def after_init(self):
         # 保证拥有cookies，因为移动端要求必须携带cookies，否则会直接跳转同一本子【禁漫娘】
-        if JmModuleConfig.flag_api_client_require_cookies:
+        if JmModuleConfig.FLAG_API_CLIENT_REQUIRE_COOKIES:
             self.ensure_have_cookies()
 
     client_init_cookies_lock = Lock()

src/jmcomic/jm_config.py

@@ -146,25 +146,29 @@ class JmModuleConfig:
     REGISTRY_EXCEPTION_LISTENER = {}
 
     # 执行log的函数
-    executor_log = default_jm_logging
+    EXECUTOR_LOG = default_jm_logging
 
     # 使用固定时间戳
-    flag_use_fix_timestamp = True
+    FLAG_USE_FIX_TIMESTAMP = True
     # 移动端Client初始化cookies
-    flag_api_client_require_cookies = True
+    FLAG_API_CLIENT_REQUIRE_COOKIES = True
     # log开关标记
-    flag_enable_jm_log = True
+    FLAG_ENABLE_JM_LOG = True
     # log时解码url
-    flag_decode_url_when_logging = True
+    FLAG_DECODE_URL_WHEN_LOGGING = True
     # 当内置的版本号落后时，使用最新的禁漫app版本号
-    flag_use_version_newer_if_behind = True
+    FLAG_USE_VERSION_NEWER_IF_BEHIND = True
 
     # 关联dir_rule的自定义字段与对应的处理函数
     # 例如:
     # Amyname -> JmModuleConfig.AFIELD_ADVICE['myname'] = lambda album: "自定义名称"
     AFIELD_ADVICE = dict()
     PFIELD_ADVICE = dict()
 
+    # 当发生 oserror: [Errno 36] File name too long 时，
+    # 把文件名限制在指定个字符以内
+    VAR_FILE_NAME_LENGTH_LIMIT = 100
+
     @classmethod
     def downloader_class(cls):
         if cls.CLASS_DOWNLOADER is not None:
@@ -319,12 +323,12 @@ def get_fix_ts_token_tokenparam(cls):
     # noinspection PyUnusedLocal
     @classmethod
     def jm_log(cls, topic: str, msg: str):
-        if cls.flag_enable_jm_log is True:
-            cls.executor_log(topic, msg)
+        if cls.FLAG_ENABLE_JM_LOG is True:
+            cls.EXECUTOR_LOG(topic, msg)
 
     @classmethod
     def disable_jm_log(cls):
-        cls.flag_enable_jm_log = False
+        cls.FLAG_ENABLE_JM_LOG = False
 
     @classmethod
     def new_postman(cls, session=False, **kwargs):
@@ -347,7 +351,7 @@ def new_postman(cls, session=False, **kwargs):
     DEFAULT_CLIENT_CACHE = None  # 默认关闭Client缓存。缓存的配置详见 CacheRegistry
     DEFAULT_PROXIES = ProxyBuilder.system_proxy()  # 默认使用系统代理
 
-    default_option_dict: dict = {
+    DEFAULT_OPTION_DICT: dict = {
         'log': None,
         'dir_rule': {'rule': 'Bd_Pname', 'base_dir': None},
         'download': {
@@ -364,7 +368,7 @@ def new_postman(cls, session=False, **kwargs):
             'postman': {
                 'type': 'cffi',
                 'meta_data': {
-                    'impersonate': 'chrome110',
+                    'impersonate': 'chrome',
                     'headers': None,
                     'proxies': None,
                 }
@@ -387,11 +391,11 @@ def option_default_dict(cls) -> dict:
         """
         from copy import deepcopy
 
-        option_dict = deepcopy(cls.default_option_dict)
+        option_dict = deepcopy(cls.DEFAULT_OPTION_DICT)
 
         # log
         if option_dict['log'] is None:
-            option_dict['log'] = cls.flag_enable_jm_log
+            option_dict['log'] = cls.FLAG_ENABLE_JM_LOG
 
         # dir_rule.base_dir
         dir_rule = option_dict['dir_rule']

src/jmcomic/jm_downloader.py

@@ -29,7 +29,7 @@ def after_photo(self, photo: JmPhotoDetail):
                f'章节下载完成: [{photo.id}] ({photo.album_id}[{photo.index}/{len(photo.from_album)}])')
 
     def before_image(self, image: JmImageDetail, img_save_path):
-        if image.is_exists:
+        if image.exists:
             jm_log('image.before',
                    f'图片已存在: {image.tag} ← [{img_save_path}]'
                    )
@@ -63,6 +63,8 @@ def download_album(self, album_id):
 
     def download_by_album_detail(self, album: JmAlbumDetail, client: JmcomicClient):
         self.before_album(album)
+        if album.skip:
+            return
         self.execute_by_condition(
             iter_objs=album,
             apply=lambda photo: self.download_by_photo_detail(photo, client),
@@ -80,6 +82,8 @@ def download_by_photo_detail(self, photo: JmPhotoDetail, client: JmcomicClient):
         client.check_photo(photo)
 
         self.before_photo(photo)
+        if photo.skip:
+            return
         self.execute_by_condition(
             iter_objs=photo,
             apply=lambda image: self.download_by_image_detail(image, client),
@@ -91,16 +95,19 @@ def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
         img_save_path = self.option.decide_image_filepath(image)
 
         image.save_path = img_save_path
-        image.is_exists = file_exists(img_save_path)
+        image.exists = file_exists(img_save_path)
 
         self.before_image(image, img_save_path)
 
+        if image.skip:
+            return
+
         # let option decide use_cache and decode_image
         use_cache = self.option.decide_download_cache(image)
         decode_image = self.option.decide_download_image_decode(image)
 
         # skip download
-        if use_cache is True and image.is_exists:
+        if use_cache is True and image.exists:
             return
 
         e = None

src/jmcomic/jm_entity.py

@@ -3,6 +3,14 @@
 from .jm_config import *
 
 
+class Downloadable:
+
+    def __init__(self):
+        self.save_path: str = ''
+        self.exists: bool = False
+        self.skip = False
+
+
 class JmBaseEntity:
 
     def to_file(self, filepath):
@@ -117,7 +125,7 @@ def idoname(self):
     def __str__(self):
         return f'{self.__class__.__name__}' \
                '{' \
-               f'{self.id}: {self.title}'\
+               f'{self.id}: {self.title}' \
                '}'
 
     @classmethod
@@ -156,7 +164,7 @@ def get_dirname(cls, detail: 'DetailEntity', ref: str) -> str:
         return getattr(detail, ref)
 
 
-class JmImageDetail(JmBaseEntity):
+class JmImageDetail(JmBaseEntity, Downloadable):
 
     def __init__(self,
                  aid,
@@ -167,7 +175,8 @@ def __init__(self,
                  from_photo=None,
                  query_params=None,
                  index=-1,
-                 ) -> None:
+                 ):
+        super().__init__()
         if scramble_id is None or (isinstance(scramble_id, str) and scramble_id == ''):
             from .jm_toolkit import ExceptionTool
             ExceptionTool.raises(f'图片的scramble_id不能为空')
@@ -182,10 +191,6 @@ def __init__(self,
         self.query_params: Optional[str] = query_params
         self.index = index  # 从1开始
 
-        # temp fields, in order to simplify passing parameter
-        self.save_path: str = ''
-        self.is_exists: bool = False
-
     @property
     def filename_without_suffix(self):
         return self.img_file_name
@@ -252,7 +257,7 @@ def is_image(cls):
         return True
 
 
-class JmPhotoDetail(DetailEntity):
+class JmPhotoDetail(DetailEntity, Downloadable):
 
     def __init__(self,
                  photo_id,
@@ -267,6 +272,7 @@ def __init__(self,
                  author=None,
                  from_album=None,
                  ):
+        super().__init__()
         self.photo_id: str = str(photo_id)
         self.scramble_id: str = str(scramble_id)
         self.name: str = str(name).strip()
@@ -411,7 +417,7 @@ def is_photo(cls):
         return True
 
 
-class JmAlbumDetail(DetailEntity):
+class JmAlbumDetail(DetailEntity, Downloadable):
 
     def __init__(self,
                  album_id,
@@ -430,6 +436,7 @@ def __init__(self,
                  tags,
                  related_list=None,
                  ):
+        super().__init__()
         self.album_id: str = str(album_id)
         self.scramble_id: str = str(scramble_id)
         self.name: str = name

src/jmcomic/jm_exception.py

@@ -72,13 +72,6 @@ class ExceptionTool:
     CONTEXT_KEY_RE_PATTERN = 'pattern'
     CONTEXT_KEY_MISSING_JM_ID = 'missing_jm_id'
 
-    # 兼容旧版本
-
-    EXTRA_KEY_RESP = 'resp'
-    EXTRA_KEY_HTML = 'html'
-    EXTRA_KEY_RE_PATTERN = 'pattern'
-    EXTRA_KEY_MISSING_JM_ID = 'missing_jm_id'
-
     @classmethod
     def raises(cls,
                msg: str,
@@ -144,7 +137,8 @@ def raise_missing(cls,
         :param resp: 响应对象
         :param jmid: 禁漫本子/章节id
         """
-        url = resp.url
+        from .jm_toolkit import JmcomicText
+        url = JmcomicText.format_album_url(jmid)
 
         req_type = "本子" if "album" in url else "章节"
         cls.raises(

src/jmcomic/jm_option.py

@@ -285,7 +285,15 @@ def decide_image_save_dir(self, photo, ensure_exists=True) -> str:
         )
 
         if ensure_exists:
-            mkdir_if_not_exists(save_dir)
+            try:
+                mkdir_if_not_exists(save_dir)
+            except OSError as e:
+                if e.errno == 36:
+                    # 目录名过长
+                    limit = JmModuleConfig.VAR_FILE_NAME_LENGTH_LIMIT
+                    jm_log('error', f'目录名过长，无法创建目录，强制缩短到{limit}个字符并重试')
+                    save_dir = save_dir[0:limit]
+                    mkdir_if_not_exists(save_dir)
 
         return save_dir
 
@@ -359,7 +367,7 @@ def compatible_with_old_versions(cls, dic):
     def deconstruct(self) -> Dict:
         return {
             'version': JmModuleConfig.JM_OPTION_VER,
-            'log': JmModuleConfig.flag_enable_jm_log,
+            'log': JmModuleConfig.FLAG_ENABLE_JM_LOG,
             'dir_rule': {
                 'rule': self.dir_rule.rule_dsl,
                 'base_dir': self.dir_rule.base_dir,