Enhance Dependency Tracking and Issue Creation Github Action (#444)

* wip: new script and yaml format

* Update variables

* Update scripts

* Update workflow

* Update workflow filename

* Untrack json file

Author: 0xLucca

.github/scripts/check_dependencies.py

@@ -0,0 +1,105 @@
+import requests
+import json
+import sys
+import yaml
+from urllib.parse import urlparse
+
+REGISTRIES = {
+    "repositories": lambda info: get_latest_github_release(info.get("repository_url")),
+    "crates": lambda info: get_latest_crate_version(info.get("name")),
+    "javascript_packages": lambda info: get_latest_npm_version(info.get("name")),
+    "python_packages": lambda info: get_latest_pypi_version(info.get("name")),
+}
+
+def get_latest_github_release(repo_url):
+    try:
+        path = urlparse(repo_url).path.strip("/")
+        owner, repo = path.split("/")[-2:]
+        api_url = f"https://api.github.com/repos/{owner}/{repo}/releases/latest"
+        response = requests.get(api_url)
+        if response.status_code == 200:
+            data = response.json()
+            return data["tag_name"], data["html_url"]
+    except Exception as e:
+        print(f"Error fetching GitHub release for {repo_url}: {e}")
+    return None, None
+
+def get_latest_crate_version(crate_name):
+    try:
+        response = requests.get(f"https://crates.io/api/v1/crates/{crate_name}")
+        if response.status_code == 200:
+            data = response.json()["crate"]
+            latest_version = data["max_stable_version"]
+            return latest_version, f"https://crates.io/crates/{crate_name}/{latest_version}"
+    except Exception as e:
+        print(f"Error fetching crate version for {crate_name}: {e}")
+    return None, None
+
+def get_latest_npm_version(package_name):
+    try:
+        response = requests.get(f"https://registry.npmjs.org/{package_name}")
+        if response.status_code == 200:
+            data = response.json()
+            latest_version = data["dist-tags"]["latest"]
+            return latest_version, f"https://www.npmjs.com/package/{package_name}/v/{latest_version}"
+    except Exception as e:
+        print(f"Error fetching npm package version for {package_name}: {e}")
+    return None, None
+
+def get_latest_pypi_version(package_name):
+    try:
+        response = requests.get(f"https://pypi.org/pypi/{package_name}/json")
+        if response.status_code == 200:
+            data = response.json()["info"]
+            latest_version = data["version"]
+            return latest_version, f"https://pypi.org/project/{package_name}/{latest_version}/"
+    except Exception as e:
+        print(f"Error fetching PyPI package version for {package_name}: {e}")
+    return None, None
+
+def check_releases(releases_source_file):
+    try:
+        with open(releases_source_file, "r") as file:
+            data = yaml.safe_load(file)
+            dependencies = data.get("dependencies", {})
+    except Exception as e:
+        print(f"Error reading releases source file '{releases_source_file}': {e}")
+        sys.exit(1)
+    
+    outdated_dependencies = []
+    
+    for category, items in dependencies.items():
+        if category not in REGISTRIES:
+            continue
+        
+        for name, info in items.items():
+            current_version = info.get("version")
+            latest_version, latest_url = REGISTRIES[category](info)
+            
+            if latest_version and latest_version != current_version:
+                outdated_dependencies.append({
+                    "name": name,
+                    "category": category,
+                    "current_version": current_version,
+                    "latest_version": latest_version,
+                    "latest_release_url": latest_url,
+                })
+    
+    return outdated_dependencies
+
+def main(releases_source_file):
+    try:
+        outdated_dependencies = check_releases(releases_source_file)
+        output = {
+            "outdated_dependencies": outdated_dependencies,
+            "outdated_count": len(outdated_dependencies),
+        }
+        with open("outdated_dependencies.json", "w") as f:
+            json.dump(output, f, indent=2)
+        print(json.dumps(output))
+    except Exception as e:
+        print(f"Error in main execution: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main("variables.yml")

.github/scripts/create_issues.py

@@ -2,25 +2,13 @@
 import json
 import os
 import sys
-from urllib.parse import urlparse
 
 GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN")
 
-
-def parse_github_url(url):
-    """Extract owner and repo from a GitHub URL."""
-    try:
-        path = urlparse(url).path.strip("/")
-        return path.split("/")[-2:]
-    except Exception as e:
-        print(f"Error parsing GitHub URL: {e}")
-        sys.exit(1)
-
-
-def issue_exists(owner, repo, title):
+def issue_exists(title):
     """Check if an issue with the same title already exists."""
     try:
-        url = f"https://api.github.com/repos/polkadot-developers/polkadot-docs/issues"
+        url = "https://api.github.com/repos/polkadot-developers/polkadot-docs/issues"
         headers = {
             "Authorization": f"token {GITHUB_TOKEN}",
             "Accept": "application/vnd.github.v3+json",
@@ -37,15 +25,14 @@ def issue_exists(owner, repo, title):
         print(f"Error checking for existing issues: {e}")
         sys.exit(1)
 
-
-def create_github_issue(owner, repo, title, body):
+def create_github_issue(title, body):
     """Create a GitHub issue using the GitHub API."""
     try:
-        if issue_exists(owner, repo, title):
+        if issue_exists(title):
             print(f"Issue '{title}' already exists. Skipping creation.")
             return
 
-        url = f"https://api.github.com/repos/polkadot-developers/polkadot-docs/issues"
+        url = "https://api.github.com/repos/polkadot-developers/polkadot-docs/issues"
         headers = {
             "Authorization": f"token {GITHUB_TOKEN}",
             "Accept": "application/vnd.github.v3+json",
@@ -57,41 +44,38 @@ def create_github_issue(owner, repo, title, body):
             print(f"Successfully created issue '{title}'")
             return
         else:
-            print(
-                f"Failed to create issue '{title}'. Status code: {response.status_code}"
-            )
+            print(f"Failed to create issue '{title}'. Status code: {response.status_code}")
             print(response.text)
             sys.exit(1)
     except Exception as e:
         print(f"Error creating issue: {e}")
         sys.exit(1)
 
-
 def main():
-    # Check if GITHUB_TOKEN is set
     if not GITHUB_TOKEN:
         print("Error: GITHUB_TOKEN environment variable is not set.")
         sys.exit(1)
 
-    # Read the JSON file with outdated repositories
     try:
-        with open("outdated_repositories.json", "r") as f:
+        with open("outdated_dependencies.json", "r") as f:
             data = json.load(f)
     except Exception as e:
         print(f"Error reading JSON file: {e}")
         sys.exit(1)
 
-    for repo in data.get("outdated_repos", []):
-        owner, repo_name = parse_github_url(repo["repository"])
-        title = f"Update needed: {repo_name} ({repo['current_version']} -> {repo['latest_version']})"
-        body = f"""A new release has been detected for {repo['repository']}.
-Current version: {repo['current_version']}
-Latest version: {repo['latest_version']}
+    for dep in data.get("outdated_dependencies", []):
+        title = f"Update needed: {dep['name']} ({dep['current_version']} -> {dep['latest_version']})"
+        body = f"""A new release has been detected for {dep['name']}.
 
-Please review the change log and update the documentation accordingly."""
+Category: {dep['category']}
+Current version: {dep['current_version']}
+Latest version: {dep['latest_version']}
 
-        create_github_issue(owner, repo_name, title, body)
+Latest release: [View here]({dep['latest_release_url']})
+
+Please review the change log and update the documentation accordingly."""
 
+        create_github_issue(title, body)
 
 if __name__ == "__main__":
-    main()
+    main()

.github/workflows/check-repositories.yaml .github/workflows/check-dependencies.yaml.github/workflows/check-repositories.yaml renamed to .github/workflows/check-dependencies.yaml

@@ -1,12 +1,12 @@
-name: Check Repositories
+name: Check Dependencies
 
 on:
   schedule: # Run every day at midnight.
     - cron: '0 0 * * *'
   workflow_dispatch:  # Allows manual triggering of the workflow.
 
 jobs:
-  check-repositories:
+  check-dependencies:
     runs-on: ubuntu-latest
     permissions:
       issues: write
@@ -24,12 +24,12 @@ jobs:
         python -m pip install --upgrade pip
         pip install requests pyyaml
 
-    - name: Check for Outdated Repositories
-      id: check_repositories
+    - name: Check for Outdated Dependencies
+      id: check_dependencies
       run: |
-        python .github/scripts/check_repositories.py
+        python .github/scripts/check_dependencies.py
 
-    - name: Create GitHub Issues for Outdated Repositories
+    - name: Create GitHub Issues for Outdated Dependencies
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |

.snippets/code/develop/toolkit/interoperability/asset-transfer-api/reference/tx-result.md

@@ -133,6 +133,6 @@ A promise containing the result of constructing the transaction.
 
         The `ConstructedFormat` type is a conditional type that returns a specific type based on the value of the TxResult `format` field.
 
-        - **Payload format** - if the format field is set to `'payload'`, the `ConstructedFormat` type will return a [`GenericExtrinsicPayload`](https://github.com/polkadot-js/api/blob/{{ dependencies.polkadot_js_api.version}}/packages/types/src/extrinsic/ExtrinsicPayload.ts#L83){target=\_blank}
+        - **Payload format** - if the format field is set to `'payload'`, the `ConstructedFormat` type will return a [`GenericExtrinsicPayload`](https://github.com/polkadot-js/api/blob/{{ dependencies.repositories.polkadot_js_api.version}}/packages/types/src/extrinsic/ExtrinsicPayload.ts#L83){target=\_blank}
         - **Call format** - if the format field is set to `'call'`, the `ConstructedFormat` type will return a hexadecimal string (`0x${string}`). This is the encoded representation of the extrinsic call
-        - **Submittable format** - if the format field is set to `'submittable'`, the `ConstructedFormat` type will return a [`SubmittableExtrinsic`](https://github.com/polkadot-js/api/blob/{{ dependencies.polkadot_js_api.version}}/packages/api-base/src/types/submittable.ts#L56){target=\_blank}. This is a Polkadot.js type that represents a transaction that can be submitted to the blockchain
+        - **Submittable format** - if the format field is set to `'submittable'`, the `ConstructedFormat` type will return a [`SubmittableExtrinsic`](https://github.com/polkadot-js/api/blob/{{ dependencies.repositories.polkadot_js_api.version}}/packages/api-base/src/types/submittable.ts#L56){target=\_blank}. This is a Polkadot.js type that represents a transaction that can be submitted to the blockchain

develop/interoperability/test-and-debug.md

@@ -21,7 +21,7 @@ For scenarios where real blockchain state is required, [Chopsticks](/tutorials/p
 
 Setting up a live network with multiple interconnected parachains for XCM testing can be complex and resource-intensive. 
 
-The [`xcm-emulator`](https://github.com/paritytech/polkadot-sdk/tree/{{dependencies.polkadot_sdk.stable_version}}/cumulus/xcm/xcm-emulator){target=\_blank} is a tool designed to simulate the execution of XCM programs using predefined runtime configurations. These configurations include those utilized by live networks like Kusama, Polkadot, and Asset Hub.
+The [`xcm-emulator`](https://github.com/paritytech/polkadot-sdk/tree/{{dependencies.repositories.polkadot_sdk.version}}/cumulus/xcm/xcm-emulator){target=\_blank} is a tool designed to simulate the execution of XCM programs using predefined runtime configurations. These configurations include those utilized by live networks like Kusama, Polkadot, and Asset Hub.
 
 This tool enables testing of cross-chain message passing, providing a way to verify outcomes, weights, and side effects efficiently. It achieves this by utilizing mocked runtimes for both the relay chain and connected parachains, enabling developers to focus on message logic and configuration without needing a live network.
 
@@ -44,27 +44,27 @@ The XCM Emulator provides both advantages and limitations when testing cross-cha
 
 ### How Does It Work?
 
-The `xcm-emulator` provides macros for defining a mocked testing environment. Check all the existing macros and functionality in the [XCM Emulator source code](https://github.com/paritytech/polkadot-sdk/blob/{{dependencies.polkadot_sdk.stable_version}}/cumulus/xcm/xcm-emulator/src/lib.rs){target=\_blank}. The most important macros are:
+The `xcm-emulator` provides macros for defining a mocked testing environment. Check all the existing macros and functionality in the [XCM Emulator source code](https://github.com/paritytech/polkadot-sdk/blob/{{dependencies.repositories.polkadot_sdk.version}}/cumulus/xcm/xcm-emulator/src/lib.rs){target=\_blank}. The most important macros are:
 
-- [**`decl_test_relay_chains`**](https://github.com/paritytech/polkadot-sdk/blob/{{dependencies.polkadot_sdk.stable_version}}/cumulus/xcm/xcm-emulator/src/lib.rs#L355){target=\_blank} - defines runtime and configuration for the relay chains. Example:
+- [**`decl_test_relay_chains`**](https://github.com/paritytech/polkadot-sdk/blob/{{dependencies.repositories.polkadot_sdk.version}}/cumulus/xcm/xcm-emulator/src/lib.rs#L355){target=\_blank} - defines runtime and configuration for the relay chains. Example:
 
     ```rust
     --8<-- 'https://raw.githubusercontent.com/paritytech/polkadot-sdk/refs/tags/polkadot-stable2412/cumulus/parachains/integration-tests/emulated/chains/relays/westend/src/lib.rs:26:47'
     ```
 
-- [**`decl_test_parachains`**](https://github.com/paritytech/polkadot-sdk/blob/{{dependencies.polkadot_sdk.stable_version}}/cumulus/xcm/xcm-emulator/src/lib.rs#L590){target=\_blank} - defines runtime and configuration for the parachains. Example:
+- [**`decl_test_parachains`**](https://github.com/paritytech/polkadot-sdk/blob/{{dependencies.repositories.polkadot_sdk.version}}/cumulus/xcm/xcm-emulator/src/lib.rs#L590){target=\_blank} - defines runtime and configuration for the parachains. Example:
 
     ```rust
     --8<-- 'https://raw.githubusercontent.com/paritytech/polkadot-sdk/refs/tags/polkadot-stable2412/cumulus/parachains/integration-tests/emulated/chains/parachains/assets/asset-hub-westend/src/lib.rs:32:55'
     ```
 
-- [**`decl_test_bridges`**](https://github.com/paritytech/polkadot-sdk/blob/{{dependencies.polkadot_sdk.stable_version}}/cumulus/xcm/xcm-emulator/src/lib.rs#L1178){target=\_blank} - creates bridges between chains, specifying the source, target, and message handler. Example:
+- [**`decl_test_bridges`**](https://github.com/paritytech/polkadot-sdk/blob/{{dependencies.repositories.polkadot_sdk.version}}/cumulus/xcm/xcm-emulator/src/lib.rs#L1178){target=\_blank} - creates bridges between chains, specifying the source, target, and message handler. Example:
 
     ```rust
     --8<-- 'https://raw.githubusercontent.com/paritytech/polkadot-sdk/refs/tags/polkadot-stable2412/cumulus/parachains/integration-tests/emulated/networks/rococo-westend-system/src/lib.rs:63:74'
     ```
 
-- [**`decl_test_networks`**](https://github.com/paritytech/polkadot-sdk/blob/{{dependencies.polkadot_sdk.stable_version}}/cumulus/xcm/xcm-emulator/src/lib.rs#L916){target=\_blank} - defines a testing network with relay chains, parachains, and bridges, implementing message transport and processing logic. Example:
+- [**`decl_test_networks`**](https://github.com/paritytech/polkadot-sdk/blob/{{dependencies.repositories.polkadot_sdk.version}}/cumulus/xcm/xcm-emulator/src/lib.rs#L916){target=\_blank} - defines a testing network with relay chains, parachains, and bridges, implementing message transport and processing logic. Example:
 
     ```rust
     --8<-- 'https://raw.githubusercontent.com/paritytech/polkadot-sdk/refs/tags/polkadot-stable2412/cumulus/parachains/integration-tests/emulated/networks/westend-system/src/lib.rs:38:52'
@@ -74,4 +74,4 @@ By leveraging these macros, developers can customize their testing networks by d
 
 This framework enables thorough testing of runtime and cross-chain interactions, enabling developers to effectively design, test, and optimize cross-chain functionality.
 
-To see a complete example of implementing and executing tests, refer to the [integration tests](https://github.com/paritytech/polkadot-sdk/tree/{{dependencies.polkadot_sdk.stable_version}}/cumulus/parachains/integration-tests/emulated){target=\_blank} in the Polkadot SDK repository.
+To see a complete example of implementing and executing tests, refer to the [integration tests](https://github.com/paritytech/polkadot-sdk/tree/{{dependencies.repositories.polkadot_sdk.version}}/cumulus/parachains/integration-tests/emulated){target=\_blank} in the Polkadot SDK repository.

develop/interoperability/xcm-config.md

@@ -15,7 +15,7 @@ The executor is highly configurable, with the [XCM builder](https://paritytech.g
 
 ## XCM Executor Configuration
 
-The `Config` trait defines the XCM executor’s configuration, which requires several associated types. Each type has specific trait bounds that the concrete implementation must fulfill. Some types, such as `RuntimeCall`, come with a default implementation in most cases, while others use the unit type `()` as the default. For many of these types, selecting the appropriate implementation carefully is crucial. Predefined solutions and building blocks can be adapted to your specific needs. These solutions can be found in the [`xcm-builder`](https://github.com/paritytech/polkadot-sdk/tree/{{dependencies.polkadot_sdk.stable_version}}/polkadot/xcm/xcm-builder){target=\_blank} folder.
+The `Config` trait defines the XCM executor’s configuration, which requires several associated types. Each type has specific trait bounds that the concrete implementation must fulfill. Some types, such as `RuntimeCall`, come with a default implementation in most cases, while others use the unit type `()` as the default. For many of these types, selecting the appropriate implementation carefully is crucial. Predefined solutions and building blocks can be adapted to your specific needs. These solutions can be found in the [`xcm-builder`](https://github.com/paritytech/polkadot-sdk/tree/{{dependencies.repositories.polkadot_sdk.version}}/polkadot/xcm/xcm-builder){target=\_blank} folder.
 
 Each type is explained below, along with an overview of some of its implementations: