v2.4.4: 优化GitHub Actions、文档、注释、代码结构 (#168)

Author: hect0x7

.github/workflows/download_dispatch.yml

@@ -15,8 +15,8 @@ on:
 
       CLIENT_IMPL:
         type: string
-        description: 客户端类型（client.impl），[api]=移动端，[html]=网页端。
-        default: 'api'
+        description: 客户端类型（client.impl），[api]=移动端，[html]=网页端。下载失败时，你可以尝试填入此项重试。
+        default: ''
         required: false
 
       IMAGE_SUFFIX:

assets/docs/sources/images/6.png

@@ -22,11 +22,15 @@
 
 ## 2. 填写你需要下载的本子id
 
+在开始下面的步骤之前，你需要先启用你的repo的Actions，开启方式如下：
+![6](../images/6.png)
+
+
 ### 2.1. 方式一（最新、简单、推荐）
 
 访问下面这个网址：
 
-`https://github.com/你的用户名/JMComic-Crawler-Python/master/workflows/download_dispatch.yml`
+`https://github.com/你的用户名/JMComic-Crawler-Python/actions/workflows/download_dispatch.yml`
 
 按下图步骤进行操作：
 

assets/docs/sources/tutorial/1_github_actions.md

@@ -81,4 +81,37 @@ for aid, atitle, tag_list in page.iter_id_title_tag():  # 使用page的iter_id_t
         aid_list.append(aid)
 
 download_album(aid_list, option)
-```
+```
+
+## 手动创建Client
+
+```python
+# 默认的使用方式是先创建option，option封装了所有配置，然后由option.new_jm_client() 创建客户端client，使用client可以访问禁漫接口
+
+# 下面演示直接构造client的方式
+from jmcomic import *
+
+"""
+创建JM客户端
+
+:param postman: 负责实现HTTP请求的对象，持有cookies、headers、proxies等信息
+:param domain_list: 禁漫域名
+:param retry_times: 重试次数
+"""
+
+# 网页端
+cl = JmHtmlClient(
+    postman=JmModuleConfig.new_postman(),
+    domain_list=['18comic.vip'],
+    retry_times=1
+)
+
+# API端（APP）
+cl = JmApiClient(
+    postman=JmModuleConfig.new_postman(),
+    domain_list=JmModuleConfig.DOMAIN_API_LIST,
+    retry_times=1
+)
+
+
+```

assets/docs/sources/tutorial/3_demo.md

@@ -4,6 +4,7 @@ dir_rule:
   rule: Bd_Aauthor_Atitle_Pindex
 
 client:
+  impl: api # 使用api可免登录下载本子
   domain:
     html: [ jmcomic1.me, jmcomic.me ]
 

assets/option/option_workflow_download.yml

@@ -27,7 +27,7 @@
     package_dir={"": "src"},
     python_requires=">=3.7",
     install_requires=[
-        'commonX>=0.6.3',
+        'commonX>=0.6.4',
         'curl_cffi',
         'PyYAML',
         'Pillow',

setup.py

@@ -2,7 +2,7 @@
 # 被依赖方 <--- 使用方
 # config <--- entity <--- toolkit <--- client <--- option <--- downloader
 
-__version__ = '2.4.3'
+__version__ = '2.4.4'
 
 from .api import *
 from .jm_plugin import *

src/jmcomic/__init__.py

@@ -11,20 +11,19 @@ class AbstractJmClient(
 
     def __init__(self,
                  postman: Postman,
-                 retry_times: int,
-                 domain=None,
-                 fallback_domain_list=None,
+                 domain_list: List[str],
+                 retry_times=0,
                  ):
+        """
+        创建JM客户端
+
+        :param postman: 负责实现HTTP请求的对象，持有cookies、headers、proxies等信息
+        :param domain_list: 禁漫域名
+        :param retry_times: 重试次数
+        """
         super().__init__(postman)
         self.retry_times = retry_times
-
-        if fallback_domain_list is None:
-            fallback_domain_list = []
-
-        if domain is not None:
-            fallback_domain_list.insert(0, domain)
-
-        self.domain_list = fallback_domain_list
+        self.domain_list = domain_list
         self.CLIENT_CACHE = None
         self.enable_cache()
         self.after_init()
@@ -98,6 +97,9 @@ def request_with_retry(self,
             resp = request(url, **kwargs)
             return judge(resp)
         except Exception as e:
+            if self.retry_times == 0:
+                raise e
+
             self.before_retry(e, kwargs, retry_count, url)
 
         if retry_count < self.retry_times:
@@ -190,7 +192,7 @@ def append_params_to_url(self, url, params):
 
     # noinspection PyMethodMayBeStatic
     def decode(self, url: str):
-        if not JmModuleConfig.decode_url_when_logging or '/search/' not in url:
+        if not JmModuleConfig.flag_decode_url_when_logging or '/search/' not in url:
             return url
 
         from urllib.parse import unquote
@@ -371,7 +373,7 @@ def album_comment(self,
                       status='true',
                       comment_id=None,
                       **kwargs,
-                      ) -> JmAcResp:
+                      ) -> JmAlbumCommentResp:
         data = {
             'video_id': video_id,
             'comment': comment,
@@ -396,7 +398,7 @@ def album_comment(self,
                          data=data,
                          )
 
-        ret = JmAcResp(resp)
+        ret = JmAlbumCommentResp(resp)
         jm_log('album.comment', f'{video_id}: [{comment}] ← ({ret.model().cid})')
 
         return ret
@@ -754,15 +756,15 @@ def decide_headers_and_ts(self, kwargs, url):
             ts = time_stamp()
             token, tokenparam = JmCryptoTool.token_and_tokenparam(ts, secret=JmMagicConstants.APP_TOKEN_SECRET_2)
 
-        elif JmModuleConfig.use_fix_timestamp:
+        elif JmModuleConfig.flag_use_fix_timestamp:
             ts, token, tokenparam = JmModuleConfig.get_fix_ts_token_tokenparam()
 
         else:
             ts = time_stamp()
             token, tokenparam = JmCryptoTool.token_and_tokenparam(ts)
 
-        # 计算token，tokenparam
-        headers = kwargs.get('headers', JmMagicConstants.APP_HEADERS_TEMPLATE.copy())
+        # 设置headers
+        headers = kwargs.get('headers', None) or JmMagicConstants.APP_HEADERS_TEMPLATE.copy()
         headers.update({
             'token': token,
             'tokenparam': tokenparam,
@@ -786,7 +788,7 @@ def require_resp_success(cls, resp: JmApiResp, orig_req_url: str):
 
     def after_init(self):
         # 保证拥有cookies，因为移动端要求必须携带cookies，否则会直接跳转同一本子【禁漫娘】
-        if JmModuleConfig.api_client_require_cookies:
+        if JmModuleConfig.flag_api_client_require_cookies:
             self.ensure_have_cookies()
 
     from threading import Lock
@@ -800,7 +802,13 @@ def ensure_have_cookies(self):
             if self.get_meta_data('cookies'):
                 return
 
-            self['cookies'] = JmModuleConfig.get_cookies(self)
+            self['cookies'] = self.get_cookies()
+
+    @field_cache("APP_COOKIES", obj=JmModuleConfig)
+    def get_cookies(self):
+        resp = self.setting()
+        cookies = dict(resp.resp.cookies)
+        return cookies
 
 
 class FutureClientProxy(JmcomicClient):
@@ -823,19 +831,21 @@ class FutureClientProxy(JmcomicClient):
                      'set_cache_dict', 'get_cache_dict', 'set_domain_list', ]
 
     class FutureWrapper:
-        def __init__(self, future):
+        def __init__(self, future, after_done_callback):
             from concurrent.futures import Future
             future: Future
             self.future = future
             self.done = False
             self._result = None
+            self.after_done_callback = after_done_callback
 
         def result(self):
             if not self.done:
                 result = self.future.result()
                 self._result = result
                 self.done = True
                 self.future = None  # help gc
+                self.after_done_callback()
 
             return self._result
 
@@ -871,7 +881,9 @@ def get_future(self, cache_key, task):
             if cache_key in self.future_dict:
                 return self.future_dict[cache_key]
 
-            future = self.FutureWrapper(self.executors.submit(task))
+            future = self.FutureWrapper(self.executors.submit(task),
+                                        after_done_callback=lambda: self.future_dict.pop(cache_key, None)
+                                        )
             self.future_dict[cache_key] = future
             return future
 

src/jmcomic/jm_client_impl.py

@@ -9,30 +9,46 @@
 DictModel = AdvancedEasyAccessDict
 
 
-class JmResp(CommonResp):
+class JmResp:
+
+    def __init__(self, resp):
+        ExceptionTool.require_true(not isinstance(resp, JmResp), f'重复包装: {resp}')
+        self.resp = resp
 
     @property
     def is_success(self) -> bool:
         return self.http_code == 200 and len(self.content) != 0
 
-    def json(self, **kwargs) -> Dict:
-        raise NotImplementedError
+    @property
+    def is_not_success(self) -> bool:
+        return not self.is_success
 
-    def model(self) -> DictModel:
-        return DictModel(self.json())
+    @property
+    def content(self):
+        return self.resp.content
+
+    @property
+    def http_code(self):
+        return self.resp.status_code
+
+    @property
+    def text(self) -> str:
+        return self.resp.text
+
+    @property
+    def url(self) -> str:
+        return self.resp.url
 
     def require_success(self):
         if self.is_not_success:
-            ExceptionTool.raises_resp(self.text, self.resp)
+            ExceptionTool.raises_resp(self.text, self)
 
 
 class JmImageResp(JmResp):
 
-    def json(self, **kwargs) -> Dict:
-        raise NotImplementedError
-
     def require_success(self):
-        ExceptionTool.require_true(self.is_success, self.get_error_msg())
+        if self.is_not_success:
+            ExceptionTool.raises_resp(self.get_error_msg(), self)
 
     def get_error_msg(self):
         msg = f'禁漫图片获取失败: [{self.url}]'
@@ -66,21 +82,27 @@ def transfer_to(self,
             )
 
 
-class JmApiResp(JmResp):
+class JmJsonResp(JmResp):
 
-    def __init__(self, resp, ts: str):
-        ExceptionTool.require_true(not isinstance(resp, JmApiResp), f'重复包装: {resp}')
+    def json(self) -> Dict:
+        return self.resp.json()
+
+    def model(self) -> DictModel:
+        return DictModel(self.json())
+
+
+class JmApiResp(JmJsonResp):
 
+    def __init__(self, resp, ts: str):
         super().__init__(resp)
         self.ts = ts
-        self.cache_decode_data = None
 
     @property
     def is_success(self) -> bool:
         return super().is_success and self.json()['code'] == 200
 
     @property
-    @field_cache('__cache_decoded_data__')
+    @field_cache()
     def decoded_data(self) -> str:
         return JmCryptoTool.decode_resp_data(self.encoded_data, self.ts)
 
@@ -94,23 +116,19 @@ def res_data(self) -> Any:
         from json import loads
         return loads(self.decoded_data)
 
-    @field_cache('__cache_json__')
-    def json(self, **kwargs) -> Dict:
-        return self.resp.json()
-
     @property
     def model_data(self) -> DictModel:
         self.require_success()
         return DictModel(self.res_data)
 
 
 # album-comment
-class JmAcResp(JmResp):
+class JmAlbumCommentResp(JmJsonResp):
 
     def is_success(self) -> bool:
         return super().is_success and self.json()['err'] is False
 
-    def json(self, **kwargs) -> Dict:
+    def json(self) -> Dict:
         return self.resp.json()
 
 
@@ -184,7 +202,7 @@ def album_comment(self,
                       status='true',
                       comment_id=None,
                       **kwargs,
-                      ) -> JmAcResp:
+                      ) -> JmAlbumCommentResp:
         """
         评论漫画/评论回复
         :param video_id: album_id/photo_id