From c5db0deef996176c6964066e7a2de7a4c8e5da9f Mon Sep 17 00:00:00 2001 From: cibere <71997063+cibere@users.noreply.github.com> Date: Wed, 29 Jan 2025 09:52:40 -0800 Subject: [PATCH 1/7] add plugin release webhook notification --- .github/workflows/python-passed.yml | 8 +++- ci/envs/requirements-update-tested.txt | 2 + ci/src/discord.py | 57 +++++++++++++++++++++++++- ci/src/update-tested.py | 27 +++++++++--- ci/src/updater.py | 28 ++++--------- 5 files changed, 93 insertions(+), 29 deletions(-) create mode 100644 ci/envs/requirements-update-tested.txt diff --git a/.github/workflows/python-passed.yml b/.github/workflows/python-passed.yml index 384b841b3..8caf4545a 100644 --- a/.github/workflows/python-passed.yml +++ b/.github/workflows/python-passed.yml @@ -19,8 +19,14 @@ jobs: uses: actions/setup-python@v5 with: python-version: 3.x + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r ./ci/envs/requirements-update-tested.txt - name: Run script - run: python ./ci/src/update-tested.py + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python ./ci/src/update-tested.py ${{ secrets.DISCORD_WEBHOOK }} - name: Update plugin manifest if: success() uses: stefanzweifel/git-auto-commit-action@v4 diff --git a/ci/envs/requirements-update-tested.txt b/ci/envs/requirements-update-tested.txt new file mode 100644 index 000000000..a752e171d --- /dev/null +++ b/ci/envs/requirements-update-tested.txt @@ -0,0 +1,2 @@ +aiohttp +tqdm \ No newline at end of file diff --git a/ci/src/discord.py b/ci/src/discord.py index 5bea291d5..33e2c133a 100644 --- a/ci/src/discord.py +++ b/ci/src/discord.py @@ -1,11 +1,11 @@ import aiohttp - +from tqdm.asyncio import tqdm from _utils import * MAX_BODY_LEN = 1024 -async def update_hook(webhook_url: str, info: dict, latest_ver: str, release: dict) -> None: +async def update_hook(webhook_url: str, info: PluginType, latest_ver: str, release: dict) -> None: embed = { "content": None, "embeds": [ @@ -43,6 +43,45 @@ async def update_hook(webhook_url: str, info: dict, latest_ver: str, release: di embed['embeds'][0]['fields'].append({"name": "Release Notes", "value": truncate_release_notes(release['html_url'], release.get('body', ""))}) async with aiohttp.ClientSession() as session: await session.post(webhook_url, json=embed) + +async def release_hook(webhook_url: str, info: PluginType, latest_ver: str, release: dict) -> None: + embed = { + "content": None, + "embeds": [ + { + "title": info[plugin_name], + "description": f"New Plugin!\nReleased at v{latest_ver}.", + "url": release['html_url'], + "color": 5763719, + "fields": [ + { + "name": "Plugin Description", + "value": info[description] + }, + { + "name": "Plugin Language", + "value": info[language_name] + } + ], + "author": { + "name": info[author] + }, + "thumbnail": { + "url": info[icon_path] + } + } + ] + } + if 'github.com' in info[url_sourcecode].lower(): + github_username = info[url_sourcecode].split('/')[3] + embed['embeds'][0]['author']['name'] = github_username + embed['embeds'][0]['author']['url'] = f"{github_url}/{github_username}" + embed['embeds'][0]["author"]["icon_url"] = f"{github_url}/{github_username}.png?size=40" + release_notes = release.get('body') + if release_notes and release_notes.strip(): + embed['embeds'][0]['fields'].append({"name": "Release Notes", "value": truncate_release_notes(release['html_url'], release.get('body', ""))}) + async with aiohttp.ClientSession() as session: + await session.post(webhook_url, json=embed) def truncate_release_notes(url: str, release_notes: str, length: int = MAX_BODY_LEN) -> str: if len(release_notes) <= length: @@ -59,3 +98,17 @@ def truncate_release_notes(url: str, release_notes: str, length: int = MAX_BODY_ graceful_truncation_index = last_included_newline if last_included_newline != -1 else rough_truncation_index return release_notes[:graceful_truncation_index] + TRUNCATION_MESSAGE + +async def send_notification( + info: PluginType, latest_ver, release, webhook_url: str | None = None, is_release: bool = False +) -> None: + if not webhook_url: + return + + if is_release or version_tuple(info[version]) != version_tuple(latest_ver): + tqdm.write(f"Update detected: {info[plugin_name]} {latest_ver}") + hook = release_hook if is_release else update_hook + try: + await hook(webhook_url, info, latest_ver, release) + except Exception as e: + tqdm.write(str(e)) \ No newline at end of file diff --git a/ci/src/update-tested.py b/ci/src/update-tested.py index 7bc5e24d3..2e5a1d2e7 100644 --- a/ci/src/update-tested.py +++ b/ci/src/update-tested.py @@ -2,23 +2,38 @@ import json import os import zipfile -import io -from datetime import datetime +import io, asyncio, aiohttp +from datetime import datetime, UTC +from sys import argv +from os import getenv +from _utils import clean, id_name, language_list, version, plugin_reader, plugin_writer, release_date, date_added, etag_reader, PluginType, ETagsType +from updater import batch_github_plugin_info +from discord import release_hook + +def update_tested(): + webhook_url = None + if len(argv) > 1: + webhook_url = argv[1] + github_token = getenv("GITHUB_TOKEN") -from _utils import clean, id_name, language_list, language_name, plugin_reader, plugin_writer, release_date, date_added - -if __name__ == "__main__": plugin_infos = plugin_reader() + etags = etag_reader() for idx, plugin in enumerate(plugin_infos): if plugin["Language"] == "python" and "Tested" not in plugin.keys(): plugin_infos[idx]["Tested"] = True # Add date added if field is not present if plugin.get(date_added) is None: - plugin_infos[idx][date_added] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + plugin_infos[idx][date_added] = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + yield batch_github_plugin_info(plugin, etags, github_token, webhook_url, True) + plugin_writer(plugin_infos) +async def main(): + await asyncio.gather(*update_tested()) +if __name__ == '__main__': + asyncio.run(main()) diff --git a/ci/src/updater.py b/ci/src/updater.py index 2e22af25b..ed8d98c09 100644 --- a/ci/src/updater.py +++ b/ci/src/updater.py @@ -10,11 +10,10 @@ from tqdm.asyncio import tqdm from _utils import * -from discord import update_hook - +from discord import send_notification async def batch_github_plugin_info( - info: P, tags: ETagsType, github_token=None, webhook_url: str = None + info: P, tags: ETagsType, github_token=None, webhook_url: str | None = None, is_release_noti: bool = False ) -> P: try: headers = {"authorization": f"token {github_token}"} @@ -47,9 +46,10 @@ async def batch_github_plugin_info( info[release_date] = latest_rel.get("published_at") if assets: info[url_download] = assets[0]["browser_download_url"] - await send_notification( - info, clean(latest_rel["tag_name"], "v"), latest_rel, webhook_url - ) + if webhook_url: + await send_notification( + info, clean(latest_rel["tag_name"], "v"), latest_rel, webhook_url, is_release_noti + ) info[version] = clean(latest_rel["tag_name"], "v") tags[info[id_name]] = res.headers.get(etag, "") @@ -62,7 +62,7 @@ async def batch_github_plugin_info( async def batch_plugin_infos( - plugin_infos: Ps, tags: ETagsType, github_token, webhook_url: str = None + plugin_infos: Ps, tags: ETagsType, github_token, webhook_url: str | None = None ) -> Ps: return await tqdm.gather( *[ @@ -72,7 +72,7 @@ async def batch_plugin_infos( ) -def remove_unused_etags(plugin_infos: Ps, etags: ETagsType) -> ETagsType: +def remove_unused_etags(plugin_infos: PluginsType, etags: ETagsType) -> ETagsType: etags_updated = {} plugin_ids = [info.get("ID") for info in plugin_infos] @@ -88,18 +88,6 @@ def remove_unused_etags(plugin_infos: Ps, etags: ETagsType) -> ETagsType: return etags_updated - -async def send_notification( - info: P, latest_ver, release, webhook_url: str = None -) -> None: - if version_tuple(info[version]) != version_tuple(latest_ver): - tqdm.write(f"Update detected: {info[plugin_name]} {latest_ver}") - try: - await update_hook(webhook_url, info, latest_ver, release) - except Exception as e: - tqdm.write(str(e)) - - async def main(): webhook_url = None if len(argv) > 1: From 1332027109a2616914704c8fefb65ca0b87979b3 Mon Sep 17 00:00:00 2001 From: cibere <71997063+cibere@users.noreply.github.com> Date: Tue, 4 Feb 2025 08:35:34 -0800 Subject: [PATCH 2/7] Update tqdm msg for sending disco notifications Co-authored-by: Jeremy Wu --- ci/src/discord.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/src/discord.py b/ci/src/discord.py index 33e2c133a..10ad925c5 100644 --- a/ci/src/discord.py +++ b/ci/src/discord.py @@ -106,7 +106,7 @@ async def send_notification( return if is_release or version_tuple(info[version]) != version_tuple(latest_ver): - tqdm.write(f"Update detected: {info[plugin_name]} {latest_ver}") + tqdm.write(f"Sending Discord notification for {"new plugin" if is_release else "new plugin update"} : {info[plugin_name]} {latest_ver}") hook = release_hook if is_release else update_hook try: await hook(webhook_url, info, latest_ver, release) From 25d28ad7992c3f8de90b90172a26cacd428478e3 Mon Sep 17 00:00:00 2001 From: cibere <71997063+cibere@users.noreply.github.com> Date: Tue, 4 Feb 2025 08:37:14 -0800 Subject: [PATCH 3/7] don't use asyncio.gather in update-tested --- ci/src/update-tested.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ci/src/update-tested.py b/ci/src/update-tested.py index 2e5a1d2e7..f68444ec0 100644 --- a/ci/src/update-tested.py +++ b/ci/src/update-tested.py @@ -2,7 +2,7 @@ import json import os import zipfile -import io, asyncio, aiohttp +import io, asyncio from datetime import datetime, UTC from sys import argv from os import getenv @@ -10,7 +10,7 @@ from updater import batch_github_plugin_info from discord import release_hook -def update_tested(): +async def update_tested(): webhook_url = None if len(argv) > 1: webhook_url = argv[1] @@ -25,15 +25,12 @@ def update_tested(): # Add date added if field is not present if plugin.get(date_added) is None: plugin_infos[idx][date_added] = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") - yield batch_github_plugin_info(plugin, etags, github_token, webhook_url, True) + await batch_github_plugin_info(plugin, etags, github_token, webhook_url, True) plugin_writer(plugin_infos) -async def main(): - await asyncio.gather(*update_tested()) - if __name__ == '__main__': - asyncio.run(main()) + asyncio.run(update_tested()) From e3a8db991f38f8a359f5545c23b0ca1e0a13e41d Mon Sep 17 00:00:00 2001 From: cibere <71997063+cibere@users.noreply.github.com> Date: Tue, 4 Feb 2025 08:42:45 -0800 Subject: [PATCH 4/7] format update-tested + remove unused imports --- ci/src/update-tested.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/ci/src/update-tested.py b/ci/src/update-tested.py index f68444ec0..29855f96a 100644 --- a/ci/src/update-tested.py +++ b/ci/src/update-tested.py @@ -1,14 +1,10 @@ -import sys -import json -import os -import zipfile -import io, asyncio +import asyncio from datetime import datetime, UTC from sys import argv from os import getenv -from _utils import clean, id_name, language_list, version, plugin_reader, plugin_writer, release_date, date_added, etag_reader, PluginType, ETagsType +from _utils import plugin_reader, plugin_writer, date_added, etag_reader from updater import batch_github_plugin_info -from discord import release_hook + async def update_tested(): webhook_url = None @@ -24,15 +20,15 @@ async def update_tested(): plugin_infos[idx]["Tested"] = True # Add date added if field is not present if plugin.get(date_added) is None: - plugin_infos[idx][date_added] = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") - await batch_github_plugin_info(plugin, etags, github_token, webhook_url, True) - - plugin_writer(plugin_infos) - -if __name__ == '__main__': - asyncio.run(update_tested()) - - + plugin_infos[idx][date_added] = datetime.now(UTC).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + await batch_github_plugin_info( + plugin, etags, github_token, webhook_url, True + ) + plugin_writer(plugin_infos) +if __name__ == "__main__": + asyncio.run(update_tested()) From d3cdc6fe6c9150f24a8a07486d2499f28f94a8be Mon Sep 17 00:00:00 2001 From: cibere <71997063+cibere@users.noreply.github.com> Date: Tue, 4 Feb 2025 15:55:32 -0800 Subject: [PATCH 5/7] dont call github in update-tested --- .github/workflows/python-passed.yml | 2 -- ci/src/_utils.py | 1 + ci/src/discord.py | 21 +++++++++------------ ci/src/update-tested.py | 14 ++++++-------- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/.github/workflows/python-passed.yml b/.github/workflows/python-passed.yml index 8caf4545a..6f31e4574 100644 --- a/.github/workflows/python-passed.yml +++ b/.github/workflows/python-passed.yml @@ -24,8 +24,6 @@ jobs: python -m pip install --upgrade pip pip install -r ./ci/envs/requirements-update-tested.txt - name: Run script - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: python ./ci/src/update-tested.py ${{ secrets.DISCORD_WEBHOOK }} - name: Update plugin manifest if: success() diff --git a/ci/src/_utils.py b/ci/src/_utils.py index b29ec75af..388952ff5 100644 --- a/ci/src/_utils.py +++ b/ci/src/_utils.py @@ -30,6 +30,7 @@ github_url = "https://github.com" release_date = "LatestReleaseDate" date_added = "DateAdded" +website = "Website" # typing PluginType = Dict[str, str] diff --git a/ci/src/discord.py b/ci/src/discord.py index 10ad925c5..fc679804d 100644 --- a/ci/src/discord.py +++ b/ci/src/discord.py @@ -44,15 +44,15 @@ async def update_hook(webhook_url: str, info: PluginType, latest_ver: str, relea async with aiohttp.ClientSession() as session: await session.post(webhook_url, json=embed) -async def release_hook(webhook_url: str, info: PluginType, latest_ver: str, release: dict) -> None: +async def release_hook(webhook_url: str, info: PluginType) -> None: embed = { "content": None, "embeds": [ { "title": info[plugin_name], - "description": f"New Plugin!\nReleased at v{latest_ver}.", - "url": release['html_url'], - "color": 5763719, + "description": f"New Plugin!\nReleased at v{info[version]}.", + "url": info[website], + "color": 5763719, # green "fields": [ { "name": "Plugin Description", @@ -77,9 +77,7 @@ async def release_hook(webhook_url: str, info: PluginType, latest_ver: str, rele embed['embeds'][0]['author']['name'] = github_username embed['embeds'][0]['author']['url'] = f"{github_url}/{github_username}" embed['embeds'][0]["author"]["icon_url"] = f"{github_url}/{github_username}.png?size=40" - release_notes = release.get('body') - if release_notes and release_notes.strip(): - embed['embeds'][0]['fields'].append({"name": "Release Notes", "value": truncate_release_notes(release['html_url'], release.get('body', ""))}) + async with aiohttp.ClientSession() as session: await session.post(webhook_url, json=embed) @@ -100,15 +98,14 @@ def truncate_release_notes(url: str, release_notes: str, length: int = MAX_BODY_ return release_notes[:graceful_truncation_index] + TRUNCATION_MESSAGE async def send_notification( - info: PluginType, latest_ver, release, webhook_url: str | None = None, is_release: bool = False + info: PluginType, latest_ver, release, webhook_url: str | None = None ) -> None: if not webhook_url: return - if is_release or version_tuple(info[version]) != version_tuple(latest_ver): - tqdm.write(f"Sending Discord notification for {"new plugin" if is_release else "new plugin update"} : {info[plugin_name]} {latest_ver}") - hook = release_hook if is_release else update_hook + if version_tuple(info[version]) != version_tuple(latest_ver): + tqdm.write(f"Update detected: {info[plugin_name]} {latest_ver}") try: - await hook(webhook_url, info, latest_ver, release) + await update_hook(webhook_url, info, latest_ver, release) except Exception as e: tqdm.write(str(e)) \ No newline at end of file diff --git a/ci/src/update-tested.py b/ci/src/update-tested.py index 29855f96a..892d1c46e 100644 --- a/ci/src/update-tested.py +++ b/ci/src/update-tested.py @@ -1,31 +1,29 @@ import asyncio from datetime import datetime, UTC from sys import argv -from os import getenv -from _utils import plugin_reader, plugin_writer, date_added, etag_reader -from updater import batch_github_plugin_info +from _utils import plugin_reader, plugin_writer, date_added +from discord import release_hook async def update_tested(): webhook_url = None if len(argv) > 1: webhook_url = argv[1] - github_token = getenv("GITHUB_TOKEN") plugin_infos = plugin_reader() - etags = etag_reader() for idx, plugin in enumerate(plugin_infos): if plugin["Language"] == "python" and "Tested" not in plugin.keys(): plugin_infos[idx]["Tested"] = True + # Add date added if field is not present if plugin.get(date_added) is None: plugin_infos[idx][date_added] = datetime.now(UTC).strftime( "%Y-%m-%dT%H:%M:%SZ" ) - await batch_github_plugin_info( - plugin, etags, github_token, webhook_url, True - ) + + if webhook_url: + await release_hook(webhook_url, plugin) plugin_writer(plugin_infos) From c98c5e84863321a873a4733c04b350d2d5456e4e Mon Sep 17 00:00:00 2001 From: cibere <71997063+cibere@users.noreply.github.com> Date: Tue, 4 Feb 2025 18:09:54 -0800 Subject: [PATCH 6/7] fix bug with `batch_github_plugin_info` attempting to pass `is_release_noti` to `send_notification` --- ci/src/updater.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/src/updater.py b/ci/src/updater.py index ed8d98c09..eede95f34 100644 --- a/ci/src/updater.py +++ b/ci/src/updater.py @@ -13,7 +13,7 @@ from discord import send_notification async def batch_github_plugin_info( - info: P, tags: ETagsType, github_token=None, webhook_url: str | None = None, is_release_noti: bool = False + info: P, tags: ETagsType, github_token=None, webhook_url: str | None = None ) -> P: try: headers = {"authorization": f"token {github_token}"} @@ -48,7 +48,7 @@ async def batch_github_plugin_info( info[url_download] = assets[0]["browser_download_url"] if webhook_url: await send_notification( - info, clean(latest_rel["tag_name"], "v"), latest_rel, webhook_url, is_release_noti + info, clean(latest_rel["tag_name"], "v"), latest_rel, webhook_url ) info[version] = clean(latest_rel["tag_name"], "v") From dae44c8934ef45424a48bea7c1185ec0c5ef5a47 Mon Sep 17 00:00:00 2001 From: cibere <71997063+cibere@users.noreply.github.com> Date: Tue, 4 Feb 2025 18:13:21 -0800 Subject: [PATCH 7/7] move send_notification back to updater.py --- ci/src/discord.py | 13 ------------- ci/src/updater.py | 16 +++++++++++++++- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/ci/src/discord.py b/ci/src/discord.py index fc679804d..ecef35216 100644 --- a/ci/src/discord.py +++ b/ci/src/discord.py @@ -96,16 +96,3 @@ def truncate_release_notes(url: str, release_notes: str, length: int = MAX_BODY_ graceful_truncation_index = last_included_newline if last_included_newline != -1 else rough_truncation_index return release_notes[:graceful_truncation_index] + TRUNCATION_MESSAGE - -async def send_notification( - info: PluginType, latest_ver, release, webhook_url: str | None = None -) -> None: - if not webhook_url: - return - - if version_tuple(info[version]) != version_tuple(latest_ver): - tqdm.write(f"Update detected: {info[plugin_name]} {latest_ver}") - try: - await update_hook(webhook_url, info, latest_ver, release) - except Exception as e: - tqdm.write(str(e)) \ No newline at end of file diff --git a/ci/src/updater.py b/ci/src/updater.py index eede95f34..fc54e40f8 100644 --- a/ci/src/updater.py +++ b/ci/src/updater.py @@ -10,7 +10,7 @@ from tqdm.asyncio import tqdm from _utils import * -from discord import send_notification +from discord import update_hook async def batch_github_plugin_info( info: P, tags: ETagsType, github_token=None, webhook_url: str | None = None @@ -88,6 +88,20 @@ def remove_unused_etags(plugin_infos: PluginsType, etags: ETagsType) -> ETagsTyp return etags_updated + +async def send_notification( + info: PluginType, latest_ver, release, webhook_url: str | None = None +) -> None: + if not webhook_url: + return + + if version_tuple(info[version]) != version_tuple(latest_ver): + tqdm.write(f"Update detected: {info[plugin_name]} {latest_ver}") + try: + await update_hook(webhook_url, info, latest_ver, release) + except Exception as e: + tqdm.write(str(e)) + async def main(): webhook_url = None if len(argv) > 1: