Update scripts

0xLucca · 0xLucca · commit 6b1096fba021 · 2025-03-13T11:45:46.000-03:00
diff --git a/.github/scripts/check_snippets.py b/.github/scripts/check_snippets.py
@@ -210,20 +210,12 @@ def compare_code_snippets(repo_references, repo_dependencies):
                 start_line, end_line = None, None
         
         # Fetch both versions of the code
-        print(f"Fetching code snippets for {repo_name}...")
-
-        # Print url with line numbers
-        print(f"Current URL: {current_url}")
         current_code = fetch_code_snippet(raw_current_url, current_version, start_line, end_line)
-        print(current_code)
-
-        print(f"Latest: {latest_url}")
         latest_code = fetch_code_snippet(raw_latest_url, latest_version, start_line, end_line)
-        print(latest_code)
 
         # Check if they match
         match = (current_code == latest_code) if current_code and latest_code else False
-        
+
         results.append({
             'file': ref['file'],
             'line_number': ref['line_number'],
@@ -239,6 +231,32 @@ def compare_code_snippets(repo_references, repo_dependencies):
     
     return results
 
+def convert_raw_to_regular_url(raw_url, line_start=None):
+    """
+    Convert a raw.githubusercontent.com URL back to a regular GitHub URL.
+    
+    Args:
+        raw_url: Raw GitHub URL to convert
+        line_start: Starting line number (optional)
+        
+    Returns:
+        Regular GitHub URL with line number if provided
+    """
+    # Pattern: https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{path}
+    pattern = r'https://raw\.githubusercontent\.com/([^/]+)/([^/]+)/([^/]+)/(.+)'
+    match = re.match(pattern, raw_url)
+    
+    if match:
+        owner, repo, branch, path = match.groups()
+        github_url = f"https://github.com/{owner}/{repo}/blob/{branch}/{path}"
+        
+        # Add line number if provided
+        if line_start:
+            github_url += f"#L{line_start}"
+            
+        return github_url
+    return raw_url
+
 def check_outdated_snippets(json_file_path, codebase_root_dir):
     """
     Complete workflow to check for outdated code snippets in the documentation codebase.
@@ -260,7 +278,6 @@ def check_outdated_snippets(json_file_path, codebase_root_dir):
     
     # Step 3: Compare code snippets between current and latest versions
     comparison_results = compare_code_snippets(repo_references, repo_dependencies)
-    #print(comparison_results)
     
     # Filter for outdated snippets (where match is False)
     outdated_snippets = [result for result in comparison_results if not result['match']]
@@ -276,12 +293,60 @@ def check_outdated_snippets(json_file_path, codebase_root_dir):
         repo_name = snippet['repo_name']
         if repo_name not in repo_to_outdated:
             repo_to_outdated[repo_name] = []
+        
+        # Convert raw URLs to regular GitHub URLs for better clickability
+        current_url = snippet['current_url']
+        latest_url = snippet['latest_url']
+        
+        # Extract line numbers from the URLs if present
+        current_line = None
+        latest_line = None
+        
+        # Check if URLs are in raw format and extract line numbers
+        if "raw.githubusercontent.com" in current_url:
+            base_url, start_line, _ = extract_line_range(current_url)
+            if not start_line and ":" in current_url:
+                # Handle the case where line numbers are in the URL format
+                parts = current_url.split(":")
+                if len(parts) >= 2 and parts[-2].isdigit():
+                    start_line = int(parts[-2])
+            current_line = start_line
+            current_url = base_url
+        else:
+            # Extract line number from regular GitHub URL
+            line_match = re.search(r'#L(\d+)', current_url)
+            if line_match:
+                current_line = int(line_match.group(1))
+        
+        if "raw.githubusercontent.com" in latest_url:
+            base_url, start_line, _ = extract_line_range(latest_url)
+            if not start_line and ":" in latest_url:
+                # Handle the case where line numbers are in the URL format
+                parts = latest_url.split(":")
+                if len(parts) >= 2 and parts[-2].isdigit():
+                    start_line = int(parts[-2])
+            latest_line = start_line
+            latest_url = base_url
+        else:
+            # Extract line number from regular GitHub URL
+            line_match = re.search(r'#L(\d+)', latest_url)
+            if line_match:
+                latest_line = int(line_match.group(1))
+        
+        # Convert raw URLs to regular GitHub URLs
+        if "raw.githubusercontent.com" in current_url:
+            current_url = convert_raw_to_regular_url(current_url, current_line)
+        
+        if "raw.githubusercontent.com" in latest_url:
+            latest_url = convert_raw_to_regular_url(latest_url, latest_line)
             
         repo_to_outdated[repo_name].append({
             'file': snippet['file'],
             'line_number': snippet['line_number'],
-            'current_url': snippet['current_url'],
-            'latest_url': snippet['latest_url']
+            'current_url': current_url,
+            'latest_url': latest_url,
+            'current_code': snippet['current_code'],
+            'latest_code': snippet['latest_code']
         })
     
     # Add outdated_snippets field to each repository in the JSON
diff --git a/.github/scripts/create_issues.py b/.github/scripts/create_issues.py
@@ -4,45 +4,41 @@
 import sys
 
 GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN")
+REPO_API_URL = "https://api.github.com/repos/polkadot-developers/polkadot-docs/issues"
 
-def issue_exists(title):
-    """Check if an issue with the same title already exists."""
+def get_existing_issues():
+    """Retrieve all existing issues from GitHub and return their titles."""
     try:
-        url = "https://api.github.com/repos/polkadot-developers/polkadot-docs/issues"
         headers = {
             "Authorization": f"token {GITHUB_TOKEN}",
             "Accept": "application/vnd.github.v3+json",
         }
-        response = requests.get(url, headers=headers)
+        api_url_with_query = f"{REPO_API_URL}?per_page=100"
+        response = requests.get(api_url_with_query, headers=headers)
         response.raise_for_status()
 
         issues = response.json()
-        for issue in issues:
-            if issue["title"] == title:
-                return True
-        return False
+        return {issue["title"] for issue in issues}
     except Exception as e:
-        print(f"Error checking for existing issues: {e}")
+        print(f"Error retrieving existing issues: {e}")
         sys.exit(1)
 
-def create_github_issue(title, body):
-    """Create a GitHub issue using the GitHub API."""
-    try:
-        if issue_exists(title):
-            print(f"Issue '{title}' already exists. Skipping creation.")
-            return
+def create_github_issue(title, body, existing_issues):
+    """Create a GitHub issue using the GitHub API if it does not already exist."""
+    if title in existing_issues:
+        print(f"Issue '{title}' already exists. Skipping creation.")
+        return
 
-        url = "https://api.github.com/repos/polkadot-developers/polkadot-docs/issues"
+    try:
         headers = {
             "Authorization": f"token {GITHUB_TOKEN}",
             "Accept": "application/vnd.github.v3+json",
         }
         data = {"title": title, "body": body}
-        response = requests.post(url, headers=headers, json=data)
+        response = requests.post(REPO_API_URL, headers=headers, json=data)
 
         if response.status_code == 201:
             print(f"Successfully created issue '{title}'")
-            return
         else:
             print(f"Failed to create issue '{title}'. Status code: {response.status_code}")
             print(response.text)
@@ -51,6 +47,24 @@ def create_github_issue(title, body):
         print(f"Error creating issue: {e}")
         sys.exit(1)
 
+def format_code_diff(current_code, latest_code):
+    """Format the code difference for GitHub markdown."""
+    diff_md = "```diff\n"
+    
+    # Split the code into lines
+    current_lines = current_code.splitlines() if current_code else []
+    latest_lines = latest_code.splitlines() if latest_code else []
+    
+    # Simple diff: prefix removed lines with - and added lines with +
+    for line in current_lines:
+        diff_md += f"- {line}\n"
+    
+    for line in latest_lines:
+        diff_md += f"+ {line}\n"
+    
+    diff_md += "```\n"
+    return diff_md
+
 def main():
     if not GITHUB_TOKEN:
         print("Error: GITHUB_TOKEN environment variable is not set.")
@@ -63,19 +77,45 @@ def main():
         print(f"Error reading JSON file: {e}")
         sys.exit(1)
 
+    existing_issues = get_existing_issues()
+
     for dep in data.get("outdated_dependencies", []):
         title = f"Update needed: {dep['name']} ({dep['current_version']} -> {dep['latest_version']})"
         body = f"""A new release has been detected for {dep['name']}.
 
-Category: {dep['category']}
-Current version: {dep['current_version']}
-Latest version: {dep['latest_version']}
+**Category:** {dep['category']}
+**Current version:** {dep['current_version']}
+**Latest version:** {dep['latest_version']}
 
-Latest release: [View here]({dep['latest_release_url']})
+Please review the [changelog]({dep['latest_release_url']}) and update the documentation accordingly.
+"""
 
-Please review the change log and update the documentation accordingly."""
+        # Add outdated snippets information if available
+        if 'outdated_snippets' in dep and dep['outdated_snippets']:
+            body += "\n## Outdated Code Snippets\n\n"
+            body += "The following code snippets in the documentation need to be updated:\n\n"
+            
+            for i, snippet in enumerate(dep['outdated_snippets'], 1):
+                file_path = snippet['file']
+                line_number = snippet['line_number']
+                current_url = snippet['current_url']
+                latest_url = snippet['latest_url']
+                
+                # Include file path and line number as a link to the repository file if possible
+                repo_file_path = f"https://github.com/polkadot-developers/polkadot-docs/blob/master/{file_path.replace('./', '')}?plain=1#L{line_number}"
+                body += f"### {i}. [{file_path}:{line_number}]({repo_file_path})\n\n"
+                
+                # Add URLs for reference
+                body += f"**Current URL:** {current_url}\n\n"
+                body += f"**Latest URL:** {latest_url}\n\n"
+                
+                # If the comparison results include the actual code snippets, show the diff
+                if 'current_code' in snippet and 'latest_code' in snippet:
+                    body += "**Code Difference:**\n\n"
+                    body += format_code_diff(snippet['current_code'], snippet['latest_code'])
+                    body += "\n"
 
-        create_github_issue(title, body)
+        create_github_issue(title, body, existing_issues)
 
 if __name__ == "__main__":
-    main()
+    main()
diff --git a/.github/workflows/check-dependencies.yaml b/.github/workflows/check-dependencies.yaml
@@ -24,12 +24,17 @@ jobs:
         python -m pip install --upgrade pip
         pip install requests pyyaml
 
-    - name: Check for Outdated Dependencies
+    - name: Check for outdated dependencies
       id: check_dependencies
       run: |
         python .github/scripts/check_dependencies.py
 
-    - name: Create GitHub Issues for Outdated Dependencies
+    - name: Check for snippets that need updating
+      id: check_snippets
+      run: |
+        python .github/scripts/check_snippets.py ./outdated_dependencies.json .
+
+    - name: Create gitHub issues for outdated dependencies
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
