diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index f3c761d2e..95aeb7469 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -14,6 +14,16 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Create plugins.json For All Plugins + if: ${{ github.event_name == 'schedule' }} + run: | + python ./ci/src/merge-manifest.py + + - name: Create plugins.json For New Plugins Only + if: ${{ github.event_name == 'pull_request' }} + run: | + python ./ci/src/merge-manifest.py new-only + - name: Link Checker id: lychee uses: lycheeverse/lychee-action@v2 diff --git a/ci/src/_utils.py b/ci/src/_utils.py index b29ec75af..d6441827f 100644 --- a/ci/src/_utils.py +++ b/ci/src/_utils.py @@ -1,9 +1,12 @@ # -*-coding: utf-8 -*- import json +import os +import re from pathlib import Path from typing import Dict, List, TypeVar -import re -import os + +# If adding a third-party library here, check CI workflows Python files +# that are dependant on this and require pip install. # path utils_path = Path(__file__).resolve() @@ -17,7 +20,18 @@ # constants id_name = "ID" language_name = "Language" -language_list = ("csharp", "executable", "fsharp", "python", "javascript", "typescript", "python_v2", "executable_v2", "javascript_v2", "typescript_v2") +language_list = ( + "csharp", + "executable", + "fsharp", + "python", + "javascript", + "typescript", + "python_v2", + "executable_v2", + "javascript_v2", + "typescript_v2", +) etag = "ETag" version = "Version" url_sourcecode = "UrlSourceCode" @@ -51,12 +65,20 @@ def plugin_reader() -> P: return manifests + +def save_plugins_json_file(content: list[dict[str]]) -> None: + with open("plugins.json", "w", encoding="utf-8") as f: + json.dump(content, f, indent=4, ensure_ascii=False) + + def get_plugin_file_paths() -> list[str]: return [os.path.join(plugin_dir, filename) for filename in get_plugin_filenames()] + def get_plugin_filenames() -> list[str]: return os.listdir(plugin_dir) + def etag_reader() -> ETagsType: with open(etag_file, "r", encoding="utf-8") as f: return json.load(f) @@ -66,7 +88,8 @@ def plugin_writer(content: P): for plugin in content: with open(plugin_dir / f"{plugin[plugin_name]}-{plugin[id_name]}.json", "w", encoding="utf-8") as f: json.dump(plugin, f, indent=4) - + + def etags_writer(content: ETagsType): with open(etag_file, "w", encoding="utf-8") as f: json.dump(content, f, indent=4) @@ -75,10 +98,12 @@ def etags_writer(content: ETagsType): def clean(string: str, flag="-") -> str: return string.lower().replace(flag, "").strip() + def version_tuple(version: str) -> tuple: version = clean(version, "v") return tuple(version.split(".")) + def check_url(url: str) -> bool: regex = re.compile( r"^(?:http|ftp)s?://" # http:// or https:// @@ -100,3 +125,19 @@ def get_file_plugins_json_info(required_key: str = "") -> list[dict[str, str]]: return data return [{required_key: plugin[required_key]} for plugin in data] + + +def get_new_plugin_submission_ids() -> list[str]: + plugins_json_ids = [item["ID"] for item in get_file_plugins_json_info("ID")] + existing_plugin_file_ids = [info["ID"] for info in plugin_reader()] + + new_ids = [] + + for id in existing_plugin_file_ids: + # plugins.json would not contain new submission's ID. + if id in plugins_json_ids: + continue + + new_ids.append(id) + + return new_ids diff --git a/ci/src/merge-manifest.py b/ci/src/merge-manifest.py index da132d4fd..30a2568f6 100644 --- a/ci/src/merge-manifest.py +++ b/ci/src/merge-manifest.py @@ -1,15 +1,30 @@ -import glob -import json +import sys -if __name__ == "__main__": - plugins = sorted(glob.glob("plugins/*.json")) +from _utils import get_new_plugin_submission_ids, plugin_reader, save_plugins_json_file + + +def get_all_plugins() -> list[dict[str]]: + return plugin_reader() + + +def get_new_plugins() -> list[dict[str]]: + ids = get_new_plugin_submission_ids() + plugins_from_plugins_dir = plugin_reader() - manifests = [] + new_plugins = [] - for plugin in plugins: - with open(plugin, "r", encoding="utf-8") as f: - manifest = json.load(f) - manifests.append(manifest) + for id in ids: + for plugin in plugins_from_plugins_dir: + if plugin["ID"] == id: + new_plugins.append(plugin) + break + + return new_plugins + + +if __name__ == "__main__": - with open("plugins.json", "w", encoding="utf-8") as f: - json.dump(manifests, f, indent=4, ensure_ascii=False) + if len(sys.argv) > 1 and str(sys.argv[1]) == "new-only": + save_plugins_json_file(get_new_plugins()) + else: + save_plugins_json_file(get_all_plugins()) diff --git a/ci/src/validator.py b/ci/src/validator.py index 6040a8a64..53469e4e6 100644 --- a/ci/src/validator.py +++ b/ci/src/validator.py @@ -1,7 +1,7 @@ # -*-coding: utf-8 -*- import uuid -from _utils import (check_url, clean, get_file_plugins_json_info, get_plugin_file_paths, get_plugin_filenames, +from _utils import (check_url, clean, get_new_plugin_submission_ids, get_plugin_file_paths, get_plugin_filenames, icon_path, id_name, language_list, language_name, plugin_reader) plugin_infos = plugin_reader() @@ -22,16 +22,19 @@ def test_language_in_list(): msg = f"The '{language_name}' is not in the list of {language_list}" assert set(language_list) >= set(languages), msg + def test_valid_icon_url(): for plugin in plugin_infos: msg = f"The URL in {icon_path} is not a valid URL." assert check_url(plugin[icon_path]), msg + def test_file_type_json(): incorrect_ext_files = [file_path for file_path in get_plugin_file_paths() if not file_path.endswith(".json")] assert len(incorrect_ext_files) == 0, f"Expected the following file to be of .json extension: {incorrect_ext_files}" + def test_file_name_construct(): filenames = get_plugin_filenames() for info in plugin_infos: @@ -39,15 +42,9 @@ def test_file_name_construct(): f"{info['Name']}-{info['ID']}.json" in filenames ), f"Plugin {info['Name']} with ID {info['ID']} does not have the correct filename. Make sure it's name + ID, i.e. {info['Name']}-{info['ID']}.json" -def test_submitted_plugin_id_is_valid_uuid(): - plugins_json_ids = [item["ID"] for item in get_file_plugins_json_info("ID")] - existing_plugin_file_ids = [info["ID"] for info in plugin_infos] - - for id in existing_plugin_file_ids: - # plugins.json would not contain new submission's ID. - if id in plugins_json_ids: - continue +def test_submitted_plugin_id_is_valid_uuid(): + for id in get_new_plugin_submission_ids(): try: uuid.UUID(id, version=4) outcome = True