Use pyyaml to parse testinfo.yml file (#5258)

This allows the CodeQL builder to use pyyaml instead of regex to parse the `testinfo.yml` files.

Author: rzuckerm

.github/workflows/codeql-analysis.yml

@@ -131,6 +131,25 @@ jobs:
       with:
         fetch-depth: 2
 
+    - name: Read repo config
+      uses: pietrobolcato/[email protected]
+      id: repo-config
+      with:
+        config: repo-config.yml
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "${{ steps.repo-config.outputs['python-version'] }}"
+
+    - name: Run Poetry Image
+      uses: abatilo/actions-poetry@v3
+      with:
+        poetry-version: "${{ steps.repo-config.outputs['poetry-version'] }}"
+
+    - name: Install Dependencies
+      run: poetry install
+
     # Set up Rust for scanning.
     - name: Set up Rust
       if: ${{ matrix.language == 'rust' }}
@@ -160,7 +179,7 @@ jobs:
 
     - name: Manual build
       if: ${{ matrix.build-mode == 'manual' }}
-      run: python scripts/build_codeql_language.py ${{ matrix.language }} ${{ matrix.paths }}
+      run: poetry run python scripts/build_codeql_language.py ${{ matrix.language }} ${{ matrix.paths }}
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v4

poetry.lock

@@ -8,7 +8,8 @@ license = "MIT"
 
 dependencies = [
     "ronbun (>=0.10.1,<0.11.0)",
-    "glotter2 (>=0.13.0,<0.14.0)"
+    "glotter2 (>=0.13.0,<0.14.0)",
+    "pyyaml (>=6.0.3,<7.0.0)"
 ]
 
 # Initiator of the collection

pyproject.toml

@@ -1,10 +1,11 @@
 import argparse
 import re
-import shlex
 import subprocess
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Dict, List
+from typing import Any, Dict, List
+
+import yaml
 
 TEST_INFO_DIR: Dict[str, str] = {
     "c": "c",
@@ -14,7 +15,6 @@
     "swift": "swift",
 }
 
-BUILD_PATTERN = re.compile(r"""^\s+build:\s*['"]([^'"]+)""", re.M)
 SOURCE_NAME_PATTERN = re.compile(r"\{\{\s*source\.name\s*\}\}")
 SOURCE_EXTENSION_PATTERN = re.compile(r"\{\{\s*source\.extension\s*\}\}")
 
@@ -23,7 +23,7 @@
 class TestInfoStruct:
     path: Path
     language: str
-    test_info_str: str
+    build: str
 
 
 def main():
@@ -36,22 +36,21 @@ def main():
         path = Path(changed_file)
         print(f"Building {path}")
         command = get_build_command(testinfo_struct, path)
-        subprocess.run(command, cwd=path.parent, check=True)
+        subprocess.run(command, cwd=path.parent, check=True, shell=True)
 
 
 def get_test_info_struct(language: str) -> TestInfoStruct:
     path = Path("archive") / language[0] / TEST_INFO_DIR[language] / "testinfo.yml"
-    testinfo_str = path.read_text(encoding="utf-8")
-    return TestInfoStruct(path=path, language=language, test_info_str=testinfo_str)
+    with path.open() as f:
+        testinfo = yaml.safe_load(f)
 
+    return TestInfoStruct(path=path, language=language, build=testinfo["container"]["build"])
 
-def get_build_command(testinfo_struct: TestInfoStruct, path: Path) -> List[str]:
-    match_str = BUILD_PATTERN.search(testinfo_struct.test_info_str)
-    assert match_str
 
-    build = SOURCE_NAME_PATTERN.sub(path.stem, match_str.group(1))
+def get_build_command(testinfo_struct: TestInfoStruct, path: Path) -> str:
+    build = SOURCE_NAME_PATTERN.sub(path.stem, testinfo_struct.build)
     build = SOURCE_EXTENSION_PATTERN.sub(path.suffix, build)
-    return shlex.split(build)
+    return build
 
 
 if __name__ == "__main__":

scripts/build_codeql_language.py

@@ -13,6 +13,7 @@
 path = "{path}"
 """
 
+
 def main():
     cargo_toml = CARGO_TOML
     for path in Path("archive/r/rust").glob("*.rs"):