split manifest and use asyncio

Author: taooceros

.github/workflows/links.yml

@@ -15,7 +15,7 @@ jobs:
   linkChecker:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Link Checker
         id: lychee

.github/workflows/python-passed.yml

@@ -9,14 +9,14 @@ jobs:
     continue-on-error: true
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         name: Checkout Repository
         with:
           token: ${{ secrets.UPDATER }}
           # otherwise, you will failed to push refs to dest repo
           fetch-depth: 0
       - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.7
       - name: Run script

.github/workflows/test-python.yml

@@ -24,7 +24,7 @@ jobs:
     outputs:
       plugins: ${{ steps.filter.outputs.plugins }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - uses: dorny/paths-filter@v2
       id: filter
       with:
@@ -36,15 +36,15 @@ jobs:
     if: ${{ needs.skip_check_job.outputs.plugins == 'true' }}
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - if: ${{ github.event_name == 'pull_request'}}
         name: "Set up Python 3.11 for pull request test"
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
       - if: ${{ github.event_name == 'workflow_dispatch'}}
         name: "Set up Python ${{ github.event.inputs.pyversion }} for manual test"
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ github.event.inputs.pyversion }}
       - uses: actions/cache@v2

.github/workflows/updater.yaml

@@ -9,13 +9,13 @@ jobs:
   update:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           token: ${{ secrets.UPDATER }}
           # otherwise, you will failed to push refs to dest repo
           fetch-depth: 0
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.x"
 
@@ -29,6 +29,10 @@ jobs:
         run: |
           python ./ci/src/updater.py ${{ secrets.DISCORD_WEBHOOK }}
 
+      - name: Merge Manifest
+        run: |
+          python ./ci/src/merge-manifest.py
+
       - name: Commit & Push changes
         uses: stefanzweifel/git-auto-commit-action@v4
         with:

.github/workflows/validator.yaml

@@ -19,8 +19,8 @@ jobs:
     if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.x"
 

ci/envs/requirements-updater.txt

@@ -1,2 +1,2 @@
-requests
+aiohttp
 tqdm

ci/src/_utils.py

@@ -3,14 +3,15 @@
 from pathlib import Path
 from typing import Dict, List, TypeVar
 import re
+import os
 
 # path
 utils_path = Path(__file__).resolve()
 
 src_dir = utils_path.parent
 ci_dir = src_dir.parent
 base_dir = ci_dir.parent
-plugin_file = base_dir / "plugins.json"
+plugin_dir = base_dir / "plugins/"
 etag_file = base_dir / "etags.json"
 
 # constants
@@ -40,8 +41,16 @@
 
 
 def plugin_reader() -> P:
-    with open(plugin_file, "r", encoding="utf-8") as f:
-        return json.load(f)
+    plugin_files = [os.path.join(plugin_dir, file) for file in os.listdir(plugin_dir)]
+
+    manifests = []
+
+    for plugin in plugin_files:
+        with open(plugin, "r", encoding="utf-8") as f:
+            manifest = json.load(f)
+            manifests.append(manifest)
+
+    return manifests
 
 
 def etag_reader() -> ETagsType:
@@ -50,8 +59,9 @@ def etag_reader() -> ETagsType:
 
 
 def plugin_writer(content: P):
-    with open(plugin_file, "w", encoding="utf-8") as f:
-        json.dump(content, f, indent=4)
+    for plugin in content:
+        with open(plugin_dir / f"{plugin[plugin_name]}-{plugin[id_name]}.json", "w", encoding="utf-8") as f:
+            json.dump(plugin, f, indent=4)
 
 def etags_writer(content: ETagsType):
     with open(etag_file, "w", encoding="utf-8") as f:

ci/src/discord.py

@@ -1,11 +1,12 @@
+import aiohttp
 import requests
 
 from _utils import *
 
 MAX_BODY_LEN = 1024
 
 
-def update_hook(webhook_url: str, info: dict, latest_ver: str, release: dict) -> None:
+async def update_hook(webhook_url: str, info: dict, latest_ver: str, release: dict) -> None:
     embed = {
         "content": None,
         "embeds": [
@@ -41,7 +42,8 @@ def update_hook(webhook_url: str, info: dict, latest_ver: str, release: dict) ->
     release_notes = release.get('body')
     if release_notes and release_notes.strip():
         embed['embeds'][0]['fields'].append({"name": "Release Notes", "value": truncate_release_notes(release['html_url'], release.get('body', ""))})
-    requests.post(webhook_url, json=embed)
+    async with aiohttp.ClientSession() as session:
+        await session.post(webhook_url, json=embed)
 
 def truncate_release_notes(url: str, release_notes: str, length: int = MAX_BODY_LEN) -> str:
     if len(release_notes) <= length:

ci/src/merge-manifest.py

@@ -0,0 +1,17 @@
+import os
+from os import path
+import json
+
+if __name__ == "__main__":
+    plugins = [file for file in os.listdir("plugins") if path.isfile(file) and file.endswith("json")]
+
+    manifests = []
+
+    for plugin in plugins:
+        with open(plugin, "r") as f:
+            manifest = json.load(f)
+            manifests.append(manifest)
+
+    with open("plugins.json", "w") as f:
+        json.dump(manifests, f, indent=4)
+        

ci/src/updater.py

@@ -1,21 +1,24 @@
 # -*-coding: utf-8 -*-
-from http.client import responses
+import asyncio
+import aiohttp
 from typing import List
 from unicodedata import name
 from os import getenv
 from sys import argv
 import traceback
 
 import requests
-from tqdm import tqdm
+from tqdm.asyncio import tqdm
 
 from _utils import *
 from discord import update_hook
 
 
-def batch_github_plugin_info(info: P, tags: ETagsType, webhook_url: str = None) -> P:
+async def batch_github_plugin_info(
+    info: P, tags: ETagsType, github_token=None, webhook_url: str = None
+) -> P:
     try:
-        headers = {"authorization": f"token {getenv('GITHUB_TOKEN','')}"}
+        headers = {"authorization": f"token {github_token}"}
         if "github.com" not in info[url_download]:
             return info
 
@@ -28,69 +31,91 @@ def batch_github_plugin_info(info: P, tags: ETagsType, webhook_url: str = None)
 
         if release_date in info.keys():
             headers["If-None-Match"] = tag
-        res = requests.get(
-            url_release.format(repo=repo),
-            headers=headers,
-        )
-        if res.status_code in (403, 304):
-            return info
 
-        latest_rel = res.json()
-        assets = latest_rel.get("assets")
-        if info.get(release_date, '') != latest_rel.get('published_at'):
-            info[release_date] = latest_rel.get('published_at')
-        if assets:
-            info[url_download] = assets[0]["browser_download_url"]
-            send_notification(info, clean(
-                latest_rel["tag_name"], "v"), latest_rel, webhook_url)
-            info[version] = clean(latest_rel["tag_name"], "v")
+        async with aiohttp.ClientSession() as session:
+            res = await session.get(
+                url_release.format(repo=repo),
+                headers=headers,
+            )
+            if res.status in (403, 304):
+                return info
 
-        tags[info[id_name]] = res.headers.get(etag, "")
+            latest_rel = await res.json()
 
-        return info
+            assets = latest_rel.get("assets")
+
+            if info.get(release_date, "") != latest_rel.get("published_at"):
+                info[release_date] = latest_rel.get("published_at")
+            if assets:
+                info[url_download] = assets[0]["browser_download_url"]
+                await send_notification(
+                    info, clean(latest_rel["tag_name"], "v"), latest_rel, webhook_url
+                )
+                info[version] = clean(latest_rel["tag_name"], "v")
+
+            tags[info[id_name]] = res.headers.get(etag, "")
+
+            return info
     except Exception as e:
         tb = traceback.format_exc()
         print(f"Error when processing plugin {info[plugin_name]}:\n{e} {tb}")
         return info
 
 
-def batch_plugin_infos(plugin_infos: Ps, tags: ETagsType, webhook_url: str = None) -> Ps:
-    return [batch_github_plugin_info(info, tags, webhook_url) for info in tqdm(plugin_infos)]
+async def batch_plugin_infos(
+    plugin_infos: Ps, tags: ETagsType, github_token, webhook_url: str = None
+) -> Ps:
+    return await tqdm.gather(
+        *[
+            batch_github_plugin_info(info, tags, github_token, webhook_url)
+            for info in tqdm(plugin_infos)
+        ]
+    )
 
 
 def remove_unused_etags(plugin_infos: Ps, etags: ETagsType) -> ETagsType:
     etags_updated = {}
     plugin_ids = [info.get("ID") for info in plugin_infos]
-    
+
     for id, tag in etags.items():
-        
+
         if id not in plugin_ids:
-            print(f"Plugin with ID {id} has been removed. The associated ETag will be also removed now.")
+            print(
+                f"Plugin with ID {id} has been removed. The associated ETag will be also removed now."
+            )
             continue
-        
+
         etags_updated[id] = tag
-    
+
     return etags_updated
 
 
-def send_notification(info: P, latest_ver, release, webhook_url: str = None) -> None:
+async def send_notification(
+    info: P, latest_ver, release, webhook_url: str = None
+) -> None:
     if version_tuple(info[version]) != version_tuple(latest_ver):
         tqdm.write(f"Update detected: {info[plugin_name]} {latest_ver}")
         try:
-            update_hook(webhook_url, info, latest_ver, release)
+            await update_hook(webhook_url, info, latest_ver, release)
         except Exception as e:
-            tqdm.write(e)
+            tqdm.write(str(e))
 
 
-if __name__ == "__main__":
+async def main():
     webhook_url = None
     if len(argv) > 1:
         webhook_url = argv[1]
     plugin_infos = plugin_reader()
     etags = etag_reader()
-    
-    plugin_infos_new = batch_plugin_infos(plugin_infos, etags, webhook_url)
+
+    plugin_infos_new = await batch_plugin_infos(
+        plugin_infos, etags, getenv("GITHUB_TOKEN"), webhook_url
+    )
     plugin_writer(plugin_infos_new)
-    
+
     etags_new = remove_unused_etags(plugin_infos_new, etags)
     etags_writer(etags_new)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())