v1.1.0: 优化debug日志，使用线程池调整下载调度。 (#5)

hect0x7 · web-flow · commit 2e29d536315f · 2023-04-04T00:13:49.000+08:00
diff --git a/.github/workflows/action_run_tests.yml b/.github/workflows/action_run_tests.yml
@@ -0,0 +1,53 @@
+name: 跑测试
+
+on:
+  push:
+    branches: [ "dev", "master" ]
+    paths:
+      - 'src/**/*.py' # 源码
+      - 'tests/**/*.py' # 测试代码
+
+jobs:
+  test: # This code is based on https://github.com/gaogaotiantian/viztracer/blob/master/.github/workflows/python-package.yml
+    strategy:
+      matrix:
+        python-version: [ 3.7, 3.8, 3.9, "3.10", "3.11" ]
+        os: [ ubuntu-latest, macos-latest ]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        if: matrix.os != 'windows-latest'
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+      - name: Install dependencies on Windows
+        if: matrix.os == 'windows-latest'
+        run: |
+          python -m pip install --upgrade pip
+          if (Test-Path -Path '.\requirements-dev.txt' -PathType Leaf) {pip install -r requirements-dev.txt}
+
+      - name: Install local
+        run: |
+          pip install -e ./
+
+      - name: 跑测试
+        if: matrix.os != 'windows-latest'
+        run: |
+          cd ./tests/
+          python -m unittest
+
+      - name: 跑测试（Windows系统）
+        if: matrix.os == 'windows-latest'
+        run: |
+          python -m unittest discover -s ./tests -p "test_*.py"
diff --git a/.github/workflows/action_workflow.yml b/.github/workflows/action_workflow.yml
@@ -19,7 +19,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python 3.11
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: "3.11"
 
diff --git a/.github/workflows/action_workflow_local.yml b/.github/workflows/action_workflow_local.yml
@@ -25,7 +25,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python 3.11
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: "3.11"
 
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,25 +1,27 @@
 name: 发布PYPI
 
 on:
-  push:
-    branches: [ "releases/**" ]
+  release:
+    types: [ published ]
 
 jobs:
-  build-and-publish:
+  release:
     name: 构建并发布 jmcomic 到 PyPI
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@main
+      - uses: actions/checkout@v3
+
       - name: Set up Python 3.7
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: "3.7"
+
       - name: 构建模块
         run: |
           python -m pip install build
           python -m build
-      - name: 发布上传
+
+      - name: 发布PYPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          password:
-            ${{ secrets.PYPI_JMCOMIC }}
+          password: ${{ secrets.PYPI_JMCOMIC }}
diff --git a/assets/config/option.yml b/assets/config/option.yml
@@ -28,7 +28,7 @@ client_config:
     headers:
       accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
       accept-language: zh-CN,zh;q=0.9
-      authority: jmcomic1.rocks
+      authority: jmcomic1.group
       sec-ch-ua: '"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"'
       sec-ch-ua-mobile: ?0
       sec-ch-ua-platform: '"Windows"'
diff --git a/assets/config/option_test.yml b/assets/config/option_test.yml
@@ -0,0 +1,41 @@
+!!python/object:jmcomic.jm_option.JmOption
+dir_tree: !!python/object:jmcomic.jm_option.DownloadDirTree
+  Bd: $(workspace}/download/
+  flag: Bd_Id_Image
+
+disable_jm_module_debug: false
+download_convert_image_suffix: null
+download_image_then_decode: true
+download_multi_thread_photo_batch_count: 3
+download_multi_thread_photo_len_limit: 5
+download_use_disk_cache: true
+
+
+client_config:
+  # 请求失败时，重试的次数，配个正数就行
+  retry_times: 10
+  domain: jmcomic1.group
+  postman_type: cffi
+  postman_type_list:
+    - requests
+    - requests_Session
+    - cffi
+    - cffi_Session
+  meta_data:
+    proxies: null
+    allow_redirects: true
+    cookies: null
+    headers:
+      accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
+      accept-language: zh-CN,zh;q=0.9
+      authority: jmcomic1.group
+      sec-ch-ua: '"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"'
+      sec-ch-ua-mobile: ?0
+      sec-ch-ua-platform: '"Windows"'
+      sec-fetch-dest: document
+      sec-fetch-mode: navigate
+      sec-fetch-site: none
+      sec-fetch-user: ?1
+      upgrade-insecure-requests: '1'
+      user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML,
+        like Gecko) Chrome/109.0.0.0 Safari/537.36
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -2,3 +2,6 @@ commonX
 curl_cffi
 PyYAML
 Pillow
+
+# for test
+pyperclip
diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py
@@ -2,6 +2,6 @@
 # 被依赖方 <--- 使用方
 # config <--- entity <--- toolkit <--- client <--- service <--- option
 
-__version__ = '1.0.0'
+__version__ = '1.1.0'
 
 from .api import *
diff --git a/src/jmcomic/api.py b/src/jmcomic/api.py
@@ -19,8 +19,11 @@ def download_album(jm_album_id, option=None):
              f'获得album_detail成功，准备下载。'
              f'本子作者是【{album_detail.author}】，一共有{len(album_detail)}集本子')
 
-    def download_photo(index, photo_detail: JmPhotoDetail, debug_topic='download_album_photo'):
-        jm_client.fill_page_arr(photo_detail)
+    def download_photo(index: int,
+                       photo_detail: JmPhotoDetail,
+                       debug_topic='download_album_photo',
+                       ):
+        jm_client.update(photo_detail)
 
         jm_debug(debug_topic,
                  f"下载第[{index + 1}]集: "
@@ -37,16 +40,16 @@ def download_photo(index, photo_detail: JmPhotoDetail, debug_topic='download_alb
                  f"下载完成：({photo_detail.title}) "
                  f"-- photo {photo_detail.photo_id}")
 
-    multi_thread_launcher(
+    thread_pool_executor(
         iter_objs=enumerate(album_detail),
         apply_each_obj_func=download_photo,
-        wait_finish=True,
     )
 
 
 def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
                          option=None,
-                         wait_finish=True) -> List[Thread]:
+                         wait_finish=True,
+                         ) -> List[Thread]:
     """
     批量下载album，每个album一个线程，使用的是同一个option。
 
@@ -58,7 +61,7 @@ def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
     if option is None:
         option = JmOption.default()
 
-    return multi_thread_launcher(
+    return thread_pool_executor(
         iter_objs=((album_id, option) for album_id in jm_album_id_iter),
         apply_each_obj_func=download_album,
         wait_finish=wait_finish,
@@ -92,18 +95,17 @@ def download_by_photo_detail(photo_detail: JmPhotoDetail,
         jm_client.fill_from_album(photo_detail)
 
     if photo_detail.page_arr is None:
-        jm_client.fill_page_arr(photo_detail)
+        jm_client.update(photo_detail)
 
     # 下载每个图片的函数
-    def download_image(index, debug_topic='download_images_of_photo'):
-        img_detail = photo_detail[index]
+    def download_image(index,img_detail, debug_topic='download_images_of_photo'):
         img_save_path = option.decide_image_filepath(photo_detail, index)
 
         # 已下载过，缓存命中
         if use_cache is True and file_exists(img_save_path):
             jm_debug(debug_topic, f'photo-{img_detail.aid}: '
                                   f'图片{img_detail.filename}已下载过，'
-                                  f'命中磁盘缓存（{img_detail.img_url}）')
+                                  f'命中磁盘缓存（{img_save_path}）')
             return
 
         # 开始下载
@@ -121,26 +123,15 @@ def download_image(index, debug_topic='download_images_of_photo'):
     if length <= option.download_multi_thread_photo_len_limit:
         # 如果图片数小的话，直接使用多线程下载，一张图一个线程。
         multi_thread_launcher(
-            iter_objs=range(len(photo_detail)),
+            iter_objs=enumerate(photo_detail),
             apply_each_obj_func=download_image,
-            wait_finish=True
         )
     else:
         # 如果图片数多的话，还是分批下载。
-        batch_count = option.download_multi_thread_photo_batch_count
-        batch_times = length // batch_count
-
-        for i in range(batch_times):
-            begin = i * batch_count
-            multi_thread_launcher(
-                iter_objs=range(begin, begin + batch_count),
-                apply_each_obj_func=download_image,
-            )
-
-        multi_thread_launcher(
-            iter_objs=range(batch_times * batch_count,
-                            length),
+        multi_task_launcher_batch(
+            iter_objs=enumerate(photo_detail),
             apply_each_obj_func=download_image,
+            batch_size=option.download_multi_thread_photo_batch_count
         )
 
 
diff --git a/src/jmcomic/jm_client.py b/src/jmcomic/jm_client.py
@@ -2,7 +2,6 @@
 
 
 class JmcomicClient(PostmanProxy):
-    debug_from_class = 'jm-client'
     retry_postman_type = RetryPostman
 
     def __init__(self, postman: Postman, domain, retry_times=None):
@@ -98,11 +97,9 @@ def fill_from_album(self, photo_detail: JmPhotoDetail) -> JmAlbumDetail:
         photo_detail.from_album = album_detail
         return album_detail
 
-    def fill_page_arr(self, photo_detail: JmPhotoDetail):
+    def update(self, photo_detail: JmPhotoDetail):
         new = self.get_photo_detail(photo_detail.photo_id)
-        photo_detail.page_arr = new.page_arr
-        photo_detail.data_original_domain = new.data_original_domain
-        return new
+        photo_detail.__dict__.update(new.__dict__)
 
     # -- search --
 
@@ -127,7 +124,7 @@ def jm_get(self, url, is_api=True, require_200=True, **kwargs):
         """
         url = self.of_api_url(url) if is_api is True else url
         if is_api is True:
-            self.debug("api", url)
+            jm_debug("api", url)
 
         resp = self.get(url, **kwargs)
 
@@ -152,10 +149,6 @@ def jm_get(self, url, is_api=True, require_200=True, **kwargs):
     def is_empty_image(cls, resp):
         return resp.status_code != 200 or len(resp.content) == 0
 
-    @staticmethod
-    def debug(topic: str, *args, sep='', end='\n', file=None, from_class=debug_from_class):
-        jm_debug(topic, *args, sep=sep, end=end, file=file, from_class=from_class)
-
     @classmethod
     def img_is_not_need_to_decode(cls, data_original: str, _resp):
         return data_original.endswith('.gif')
diff --git a/src/jmcomic/jm_config.py b/src/jmcomic/jm_config.py
@@ -46,9 +46,9 @@ def default_headers(cls):
 
     # noinspection PyUnusedLocal
     @classmethod
-    def jm_debug(cls, topic: str, *args, sep='', end='\n', file=None, from_class='api'):
+    def jm_debug(cls, topic: str, msg: str, from_class=None):
         if cls.enable_jm_debug is True:
-            cls.debug_printer(f'【{topic}】', *args, sep=sep, end=end, file=file)
+            cls.debug_printer(f'【{topic}】{msg}')
 
     @classmethod
     def disable_jm_debug(cls):
diff --git a/src/jmcomic/jm_option.py b/src/jmcomic/jm_option.py
@@ -85,11 +85,21 @@ class DownloadDirTree:
     AdditionalHandler = Callable[[Optional[JmAlbumDetail], JmPhotoDetail], str]
     additional_tree_flag_handler_mapping: Dict[int, AdditionalHandler] = {}
 
+    dsl_support = {
+        '${workspace}': lambda text: workspace(),
+    }
+
+    def fix_bd(self, Bd):
+        for dsl_k, func in self.dsl_support.items():
+            if dsl_k in Bd:
+                Bd = Bd.replace(dsl_k, func(Bd))
+        return fix_filepath(os.path.abspath(Bd), True)
+
     def __init__(self,
                  Bd: str,
                  flag: Union[str, int],
                  ):
-        self.Bd = fix_filepath(Bd)
+        self.Bd = self.fix_bd(Bd)
         self.flag = self.get_flag_enum(flag)
 
     def get_flag_enum(self, flag):
@@ -119,12 +129,12 @@ def dirpath(album_dir: Optional[str], photo_dir: str):
             @param photo_dir: 章节文件夹名
             """
             from common import fix_windir_name
-            photo_dir = fix_windir_name(photo_dir)
+            photo_dir = fix_windir_name(photo_dir.strip())
 
             if album_dir is None:
                 return f'{self.Bd}{photo_dir}/'
 
-            album_dir = fix_windir_name(album_dir)
+            album_dir = fix_windir_name(album_dir.strip())
             return f'{self.Bd}{album_dir}/{photo_dir}/'
 
         def photo_dir(flag_for_title):
diff --git a/src/jmcomic/jm_service.py b/src/jmcomic/jm_service.py
diff --git a/tests/test_jmcomic/__init__.py b/tests/test_jmcomic/__init__.py
diff --git a/tests/test_jmcomic/test_dir_tree.py b/tests/test_jmcomic/test_dir_tree.py
diff --git a/tests/test_jmcomic/test_jm_api.py b/tests/test_jmcomic/test_jm_api.py
diff --git a/tests/test_jmcomic/test_jm_client.py b/tests/test_jmcomic/test_jm_client.py
diff --git a/tests/test_jmcomic/test_jm_search.py b/tests/test_jmcomic/test_jm_search.py
