using exisitng libs

Author: Saumya-R

.github/workflows/codeql-multiple-repo-scan.yml

@@ -40,23 +40,19 @@ jobs:
     steps:
       - name: Checkout central repository
         uses: actions/checkout@v4
-      - name: Checkout CodeQL Coding Standards scripts
-        uses: actions/checkout@v4
-        with:
-          repository: github/codeql-coding-standards
-          path: codeql-coding-standards-repo # Klonen in diesen Ordner
-          ref: main # Oder eine spezifische Release-Version, z.B. 'v2.53.0-dev'
       # Add coding standard packages and dependencies
-      - name: Install Python dependencies for Coding Standards scripts
+      - name: Install Python dependencies
         run: |
           python3 -m pip install --upgrade pip
-          pip3 install pyyaml jsonpath-ng jsonschema jsonpatch jsonpointer pytest sarif-tools
+          pip3 install --break-system-packages pyyaml jsonpath-ng jsonschema jsonpatch jsonpointer pytest sarif-tools
       - name: Parse known_good.json and create repos.json
         id: parse-repos
         run: |
           python3 scripts/workflow/parse_repos.py
       - name: Checkout all pinned repositories
         id: checkout-repos
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           python3 scripts/workflow/checkout_repos.py
       - name: List files in repos directory (debug)
@@ -76,6 +72,13 @@ jobs:
           upload-database: false # Don't upload databases for each repo
           output: sarif-results/
           category: "multi-repo-scan"
+      # Checkout CodeQL Coding Standards AFTER analysis for recategorization
+      - name: Checkout CodeQL Coding Standards for recategorization
+        uses: actions/checkout@v4
+        with:
+          repository: github/codeql-coding-standards
+          path: codeql-coding-standards-repo
+          ref: v2.50.0 # Use frozen version instead of main
       - name: Recategorize Guidelines
         if: always()
         run: |

scripts/workflow/checkout_repos.py

@@ -63,7 +63,7 @@ def is_commit_hash(ref):
 
 def checkout_repo(name, url, ref, path):
     """
-    Checkout a single repository.
+    Checkout a single repository using git with GitHub token for authentication.
 
     Args:
         name: Repository name
@@ -80,12 +80,21 @@ def checkout_repo(name, url, ref, path):
         # Create parent directory if needed
         path_obj.parent.mkdir(parents=True, exist_ok=True)
 
+        # Use GitHub token if available to avoid rate limits
+        github_token = os.environ.get("GITHUB_TOKEN", "")
+        auth_url = url
+
+        if github_token and "github.com" in url:
+            # Inject token into URL for authenticated requests
+            # Replace https://github.com/ with https://token@github.com/
+            auth_url = url.replace("https://github.com/", f"https://{github_token}@github.com/")
+
         if is_commit_hash(ref):
             print(f"Checking out {name} ({ref}) to {path}")
             print(f"  Detected commit hash. Cloning and then checking out.")
 
             # Clone the repository
-            subprocess.run(["git", "clone", url, path], check=True, capture_output=True)
+            subprocess.run(["git", "clone", auth_url, path], check=True, capture_output=True)
 
             # Checkout specific commit
             subprocess.run(["git", "-C", path, "checkout", ref], check=True, capture_output=True)
@@ -97,7 +106,9 @@ def checkout_repo(name, url, ref, path):
             # Add 'v' prefix if not already present (common convention)
             branch_ref = ref if ref.startswith("v") else f"v{ref}"
             subprocess.run(
-                ["git", "clone", "--depth", "1", "--branch", branch_ref, url, path], check=True, capture_output=True
+                ["git", "clone", "--depth", "1", "--branch", branch_ref, auth_url, path],
+                check=True,
+                capture_output=True,
             )
 
         return True

scripts/workflow/parse_repos.py

@@ -13,28 +13,27 @@
 # *******************************************************************************
 """
 Parse known_good.json and create repos.json for multi-repository CodeQL analysis.
+
+Uses scripts.tooling.lib.known_good for consistent parsing of known_good.json.
 """
 
 import json
 import os
 import sys
-import subprocess
 from pathlib import Path
 
+# Add scripts directory to path for imports from tooling library
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "scripts"))
 
-def install_dependencies():
-    """Ensure jq is installed (for reference, though we use Python's json)."""
-    try:
-        subprocess.run(["sudo", "apt-get", "update"], check=True, capture_output=True)
-        subprocess.run(["sudo", "apt-get", "install", "-y", "jq"], check=True, capture_output=True)
-    except subprocess.CalledProcessError as e:
-        print(f"Warning: Failed to install jq: {e}", file=sys.stderr)
+from tooling.lib.known_good import load_known_good
 
 
 def parse_known_good(json_file="./known_good.json"):
     """
     Parse known_good.json and transform modules into repository objects.
 
+    Uses the centralized scripts.tooling.lib.known_good library for parsing.
+
     Args:
         json_file: Path to known_good.json file
 
@@ -49,39 +48,34 @@ def parse_known_good(json_file="./known_good.json"):
         sys.exit(1)
 
     try:
-        with open(json_path, "r") as f:
-            data = json.load(f)
-    except json.JSONDecodeError as e:
-        print(f"Error: Failed to parse JSON: {e}", file=sys.stderr)
+        # Use the centralized library to parse known_good.json
+        known_good = load_known_good(json_path)
+    except Exception as e:
+        print(f"Error: Failed to parse known_good.json: {e}", file=sys.stderr)
         sys.exit(1)
 
     # Extract target_sw modules
-    modules = data.get("modules", {}).get("target_sw", {})
+    modules = known_good.modules.get("target_sw", {})
 
     # Transform modules into repository objects
     repos = []
     module_outputs = {}
 
-    for name, config in modules.items():
-        repo_url = config.get("repo", "")
-        version = config.get("version", "")
-        branch = config.get("branch", "")
-        hash_val = config.get("hash", "")
-
-        # Use version, branch, or hash (in that order of preference)
-        ref = version or branch or hash_val
+    for name, module in modules.items():
+        repo_url = module.repo
+        ref = module.version or module.branch or module.commit_hash
 
         repo_obj = {"name": name, "url": repo_url, "version": ref, "path": f"repos/{name}"}
         repos.append(repo_obj)
 
         # Store module outputs for GITHUB_OUTPUT compatibility
         module_outputs[f"{name}_url"] = repo_url
-        if version:
-            module_outputs[f"{name}_version"] = version
-        if branch:
-            module_outputs[f"{name}_branch"] = branch
-        if hash_val:
-            module_outputs[f"{name}_hash"] = hash_val
+        if module.version:
+            module_outputs[f"{name}_version"] = module.version
+        if module.branch:
+            module_outputs[f"{name}_branch"] = module.branch
+        if module.commit_hash:
+            module_outputs[f"{name}_hash"] = module.commit_hash
 
     return repos, len(modules), module_outputs