Merge branch 'plugin_api_v2' into plugin_api_v2

Author: cibere

.github/workflows/links.yml

@@ -2,26 +2,33 @@ name: Check Links
 
 on:
   workflow_dispatch:
-  push:
-    paths:
-      - 'plugins.json'
   pull_request:
     paths:
-      - 'plugins.json'
+      - plugins/**
   schedule:
-    - cron: "0 */12 * * *"
+    - cron: "0 1 * * *"
 
 jobs:
   linkChecker:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
+      - name: Create plugins.json For All Plugins
+        if: ${{ github.event_name == 'schedule' }}
+        run: |
+          python ./ci/src/merge-manifest.py
+      
+      - name: Create plugins.json For New Plugins Only
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          python ./ci/src/merge-manifest.py new-only
+
       - name: Link Checker
         id: lychee
-        uses: lycheeverse/lychee-action@v1.8.0
+        uses: lycheeverse/lychee-action@v2
         with:
           args: --verbose --no-progress plugins.json
           fail: true
-        env:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          failIfEmpty: false
+          token: ${{ secrets.LINKCHECKER }}

.github/workflows/python-passed.yml

@@ -19,8 +19,12 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: 3.x
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ./ci/envs/requirements-update-tested.txt
       - name: Run script
-        run: python ./ci/src/update-tested.py
+        run: python ./ci/src/update-tested.py ${{ secrets.DISCORD_WEBHOOK }}
       - name: Update plugin manifest
         if: success()
         uses: stefanzweifel/git-auto-commit-action@v4

.github/workflows/test-python.yml

@@ -47,7 +47,7 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ github.event.inputs.pyversion }}
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         if: startsWith(runner.os, 'Windows')
         with:
           path: ~\AppData\Local\pip\Cache

ci/envs/requirements-update-tested.txt

@@ -0,0 +1,2 @@
+aiohttp
+tqdm

ci/src/_utils.py

@@ -1,9 +1,12 @@
 # -*-coding: utf-8 -*-
 import json
+import os
+import re
 from pathlib import Path
 from typing import Dict, List, TypeVar
-import re
-import os
+
+# If adding a third-party library here, check CI workflows Python files
+# that are dependant on this and require pip install.
 
 # path
 utils_path = Path(__file__).resolve()
@@ -17,7 +20,18 @@
 # constants
 id_name = "ID"
 language_name = "Language"
-language_list = ("csharp", "executable", "fsharp", "python", "javascript", "typescript", "python_v2", "executable_v2", "javascript_v2", "typescript_v2")
+language_list = (
+    "csharp",
+    "executable",
+    "fsharp",
+    "python",
+    "javascript",
+    "typescript",
+    "python_v2",
+    "executable_v2",
+    "javascript_v2",
+    "typescript_v2",
+)
 etag = "ETag"
 version = "Version"
 url_sourcecode = "UrlSourceCode"
@@ -30,6 +44,7 @@
 github_url = "https://github.com"
 release_date = "LatestReleaseDate"
 date_added = "DateAdded"
+website = "Website"
 
 # typing
 PluginType = Dict[str, str]
@@ -51,12 +66,20 @@ def plugin_reader() -> P:
 
     return manifests
 
+
+def save_plugins_json_file(content: list[dict[str]]) -> None:
+    with open("plugins.json", "w", encoding="utf-8") as f:
+        json.dump(content, f, indent=4, ensure_ascii=False)
+
+
 def get_plugin_file_paths() -> list[str]:
     return [os.path.join(plugin_dir, filename) for filename in get_plugin_filenames()]
 
+
 def get_plugin_filenames() -> list[str]:
     return os.listdir(plugin_dir)
 
+
 def etag_reader() -> ETagsType:
     with open(etag_file, "r", encoding="utf-8") as f:
         return json.load(f)
@@ -66,7 +89,8 @@ def plugin_writer(content: P):
     for plugin in content:
         with open(plugin_dir / f"{plugin[plugin_name]}-{plugin[id_name]}.json", "w", encoding="utf-8") as f:
             json.dump(plugin, f, indent=4)
-        
+
+
 def etags_writer(content: ETagsType):
     with open(etag_file, "w", encoding="utf-8") as f:
         json.dump(content, f, indent=4)
@@ -75,10 +99,12 @@ def etags_writer(content: ETagsType):
 def clean(string: str, flag="-") -> str:
     return string.lower().replace(flag, "").strip()
 
+
 def version_tuple(version: str) -> tuple:
     version = clean(version, "v")
     return tuple(version.split("."))
 
+
 def check_url(url: str) -> bool:
     regex = re.compile(
         r"^(?:http|ftp)s?://"  # http:// or https://
@@ -100,3 +126,19 @@ def get_file_plugins_json_info(required_key: str = "") -> list[dict[str, str]]:
         return data
 
     return [{required_key: plugin[required_key]} for plugin in data]
+
+
+def get_new_plugin_submission_ids() -> list[str]:
+    plugins_json_ids = [item["ID"] for item in get_file_plugins_json_info("ID")]
+    existing_plugin_file_ids = [info["ID"] for info in plugin_reader()]
+
+    new_ids = []
+
+    for id in existing_plugin_file_ids:
+        # plugins.json would not contain new submission's ID.
+        if id in plugins_json_ids:
+            continue
+
+        new_ids.append(id)
+
+    return new_ids

ci/src/discord.py

@@ -1,11 +1,11 @@
 import aiohttp
-
+from tqdm.asyncio import tqdm
 from _utils import *
 
 MAX_BODY_LEN = 1024
 
 
-async def update_hook(webhook_url: str, info: dict, latest_ver: str, release: dict) -> None:
+async def update_hook(webhook_url: str, info: PluginType, latest_ver: str, release: dict) -> None:
     embed = {
         "content": None,
         "embeds": [
@@ -43,6 +43,43 @@ async def update_hook(webhook_url: str, info: dict, latest_ver: str, release: di
         embed['embeds'][0]['fields'].append({"name": "Release Notes", "value": truncate_release_notes(release['html_url'], release.get('body', ""))})
     async with aiohttp.ClientSession() as session:
         await session.post(webhook_url, json=embed)
+
+async def release_hook(webhook_url: str, info: PluginType) -> None:
+    embed = {
+        "content": None,
+        "embeds": [
+            {
+            "title": info[plugin_name],
+            "description": f"New Plugin!\nReleased at v{info[version]}.",
+            "url": info[website],
+            "color": 5763719, # green
+            "fields": [
+                {
+                "name": "Plugin Description",
+                "value": info[description]
+                },
+                {
+                "name": "Plugin Language",
+                "value": info[language_name]
+                }
+            ],
+            "author": {
+                "name": info[author]
+            },
+            "thumbnail": {
+                "url": info[icon_path]
+            }
+            }
+        ]
+        }
+    if 'github.com' in info[url_sourcecode].lower():
+        github_username = info[url_sourcecode].split('/')[3]
+        embed['embeds'][0]['author']['name'] = github_username
+        embed['embeds'][0]['author']['url'] = f"{github_url}/{github_username}"
+        embed['embeds'][0]["author"]["icon_url"] = f"{github_url}/{github_username}.png?size=40"
+    
+    async with aiohttp.ClientSession() as session:
+        await session.post(webhook_url, json=embed)
 
 def truncate_release_notes(url: str, release_notes: str, length: int = MAX_BODY_LEN) -> str:
     if len(release_notes) <= length:

ci/src/merge-manifest.py

@@ -1,15 +1,30 @@
-import glob
-import json
+import sys
 
-if __name__ == "__main__":
-    plugins = sorted(glob.glob("plugins/*.json"))
+from _utils import get_new_plugin_submission_ids, plugin_reader, save_plugins_json_file
+
+
+def get_all_plugins() -> list[dict[str]]:
+    return plugin_reader()
+
+
+def get_new_plugins() -> list[dict[str]]:
+    ids = get_new_plugin_submission_ids()
+    plugins_from_plugins_dir = plugin_reader()
 
-    manifests = []
+    new_plugins = []
 
-    for plugin in plugins:
-        with open(plugin, "r", encoding="utf-8") as f:
-            manifest = json.load(f)
-            manifests.append(manifest)
+    for id in ids:
+        for plugin in plugins_from_plugins_dir:
+            if plugin["ID"] == id:
+                new_plugins.append(plugin)
+                break
+
+    return new_plugins
+
+
+if __name__ == "__main__":
 
-    with open("plugins.json", "w", encoding="utf-8") as f:
-        json.dump(manifests, f, indent=4, ensure_ascii=False)
+    if len(sys.argv) > 1 and str(sys.argv[1]) == "new-only":
+        save_plugins_json_file(get_new_plugins())
+    else:
+        save_plugins_json_file(get_all_plugins())

ci/src/update-tested.py

@@ -1,26 +1,32 @@
-import sys
-import json
-import os
-import zipfile
-import io
-from datetime import datetime
+import asyncio
+from datetime import datetime, UTC
+from sys import argv
+from _utils import plugin_reader, plugin_writer, date_added
+from discord import release_hook
 
-from _utils import clean, id_name, language_list, language_name, plugin_reader, plugin_writer, release_date, date_added
 
-if __name__ == "__main__":
+async def update_tested():
+    webhook_url = None
+    if len(argv) > 1:
+        webhook_url = argv[1]
+
     plugin_infos = plugin_reader()
 
     for idx, plugin in enumerate(plugin_infos):
         if plugin["Language"] == "python" and "Tested" not in plugin.keys():
             plugin_infos[idx]["Tested"] = True
+
         # Add date added if field is not present
         if plugin.get(date_added) is None:
-            plugin_infos[idx][date_added] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
-    plugin_writer(plugin_infos)
-
-
-
+            plugin_infos[idx][date_added] = datetime.now(UTC).strftime(
+                "%Y-%m-%dT%H:%M:%SZ"
+            )
 
+            if webhook_url:
+                await release_hook(webhook_url, plugin)
 
+    plugin_writer(plugin_infos)
 
 
+if __name__ == "__main__":
+    asyncio.run(update_tested())