From 4431004e4e0cca0d1f281a715c4220dae6bac0c5 Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sun, 19 Jan 2025 17:07:37 -0600 Subject: [PATCH 01/11] Modify CodeQL build script to use testinfo.yml --- .github/workflows/codeql-analysis.yml | 15 +++++- scripts/build_codeql_language.py | 68 ++++++++++++++------------- 2 files changed, 49 insertions(+), 34 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d66b67103..8bd40e240 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -116,6 +116,19 @@ jobs: if: ${{ github.event_name == 'pull_request' }} run: rm -f ${{ matrix.paths-ignore }} + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Run Poetry Image + uses: abatilo/actions-poetry@v3 + with: + poetry-version: "1.8.5" + + - name: Install Dependencies + run: poetry install + # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v3 @@ -138,7 +151,7 @@ jobs: - name: Manual build if: ${{ matrix.build-mode == 'manual' }} - run: python scripts/build_codeql_language.py ${{ matrix.language }} ${{ matrix.paths }} + run: poetry run scripts/build_codeql_language.py ${{ matrix.language }} ${{ matrix.paths }} - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 diff --git a/scripts/build_codeql_language.py b/scripts/build_codeql_language.py index f960b316c..aa6f87888 100644 --- a/scripts/build_codeql_language.py +++ b/scripts/build_codeql_language.py @@ -1,7 +1,26 @@ import argparse import subprocess +from dataclasses import dataclass from pathlib import Path -from typing import Callable, Dict, List +from typing import Dict + +from glotter.source import Source + +TEST_INFO_DIR: Dict[str, str] = { + "c": "c", + "cpp": "c-plus-plus", + "c#": "c-sharp", + "java": "java", + "kotlin": "kotlin", + "swift": "swift", +} + + +@dataclass +class TestInfoStruct: + path: Path + language: str + test_info_str: str def main(): @@ -9,46 +28,29 @@ def main(): parser.add_argument("language", help="language to build") parser.add_argument("files_changed", nargs="*", help="files that have changed") parsed_args = parser.parse_args() - command_func = COMMANDS[parsed_args.language] + testinfo_struct = get_test_info_struct(parsed_args.language) for changed_file in parsed_args.files_changed: path = Path(changed_file) print(f"Building {path}") - command = command_func(path) - subprocess.run(command, cwd=path.parent, check=True) - - -def build_c(path: Path) -> List[str]: - return ["gcc", "-o", path.stem, path.name, "-lm"] - + command = get_build_command(testinfo_struct, path) + subprocess.run(command, cwd=path.parent, check=True, shell=True) -def build_cpp(path: Path) -> List[str]: - return ["g++", "-o", path.stem, path.name] +def get_test_info_struct(language: str) -> TestInfoStruct: + path = Path("archive") / language[0] / TEST_INFO_DIR[language] / "testinfo.yml" + testinfo_str = path.read_text(encoding="utf-8") + return TestInfoStruct(path=path, language=language, test_info_str=testinfo_str) -def build_c_sharp(path: Path) -> List[str]: - return ["mcs", "-reference:System.Numerics", path.name] +def get_build_command(testinfo_struct: TestInfoStruct, path: Path) -> str: + source = Source( + name=path.name, + language=testinfo_struct.language, + path=str(path), + test_info_string=testinfo_struct.test_info_str, + ) + return source.test_info.container_info.build -def build_java(path: Path) -> List[str]: - return ["javac", path.name] - - -def build_kotlin(path: Path) -> List[str]: - return ["kotlinc", path.name, "-include-runtime", "-d", f"{path.stem}.jar"] - - -def build_swift(path: Path) -> List[str]: - return ["swiftc", "-o", path.stem, path.name] - - -COMMANDS: Dict[str, Callable[[Path], List[str]]] = { - "c": build_c, - "cpp": build_cpp, - "c#": build_c_sharp, - "java": build_java, - "kotlin": build_kotlin, - "swift": build_swift, -} if __name__ == "__main__": main() From 1773824121ffc98edbf97451d534d0486fb23024 Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sun, 19 Jan 2025 17:11:57 -0600 Subject: [PATCH 02/11] Forgot to run with python --- .github/workflows/codeql-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 8bd40e240..6f3970950 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -151,7 +151,7 @@ jobs: - name: Manual build if: ${{ matrix.build-mode == 'manual' }} - run: poetry run scripts/build_codeql_language.py ${{ matrix.language }} ${{ matrix.paths }} + run: poetry run python scripts/build_codeql_language.py ${{ matrix.language }} ${{ matrix.paths }} - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 From e90b228308c421956c0f7a141f06b636ef43b0f5 Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sun, 19 Jan 2025 17:30:54 -0600 Subject: [PATCH 03/11] Try splitting the build command --- scripts/build_codeql_language.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/build_codeql_language.py b/scripts/build_codeql_language.py index aa6f87888..1eeae130b 100644 --- a/scripts/build_codeql_language.py +++ b/scripts/build_codeql_language.py @@ -1,8 +1,9 @@ import argparse +import shlex import subprocess from dataclasses import dataclass from pathlib import Path -from typing import Dict +from typing import Dict, List from glotter.source import Source @@ -32,8 +33,9 @@ def main(): for changed_file in parsed_args.files_changed: path = Path(changed_file) print(f"Building {path}") + breakpoint() command = get_build_command(testinfo_struct, path) - subprocess.run(command, cwd=path.parent, check=True, shell=True) + subprocess.run(command, cwd=path.parent, check=True) def get_test_info_struct(language: str) -> TestInfoStruct: @@ -42,14 +44,15 @@ def get_test_info_struct(language: str) -> TestInfoStruct: return TestInfoStruct(path=path, language=language, test_info_str=testinfo_str) -def get_build_command(testinfo_struct: TestInfoStruct, path: Path) -> str: +def get_build_command(testinfo_struct: TestInfoStruct, path: Path) -> List[str]: source = Source( name=path.name, language=testinfo_struct.language, path=str(path), test_info_string=testinfo_struct.test_info_str, ) - return source.test_info.container_info.build + build: str = source.test_info.container_info.build + return shlex.split(build) if __name__ == "__main__": From 99232268522e4d9fce98d6cf89f1081960840872 Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sun, 19 Jan 2025 17:32:15 -0600 Subject: [PATCH 04/11] Get rid of breakpoint --- scripts/build_codeql_language.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build_codeql_language.py b/scripts/build_codeql_language.py index 1eeae130b..6b1fb4559 100644 --- a/scripts/build_codeql_language.py +++ b/scripts/build_codeql_language.py @@ -33,7 +33,6 @@ def main(): for changed_file in parsed_args.files_changed: path = Path(changed_file) print(f"Building {path}") - breakpoint() command = get_build_command(testinfo_struct, path) subprocess.run(command, cwd=path.parent, check=True) From da69b984efcca051d8b024abd0b078b1e29d8b07 Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sun, 19 Jan 2025 17:41:35 -0600 Subject: [PATCH 05/11] Use Windows for C# --- scripts/get_codeql_languages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/get_codeql_languages.py b/scripts/get_codeql_languages.py index 97dbc9c2e..04c87bde3 100644 --- a/scripts/get_codeql_languages.py +++ b/scripts/get_codeql_languages.py @@ -8,6 +8,7 @@ LINUX = "ubuntu-latest" MACOS = "macos-latest" +WINDOWS = "windows-latest" @dataclass(frozen=True) @@ -21,7 +22,7 @@ class LanguageInfo: "scripts/*.py": LanguageInfo(language="python"), "archive/c/c/*.c": LanguageInfo(language="c", build_mode="manual"), "archive/c/c-plus-plus/*.cpp": LanguageInfo(language="cpp", build_mode="manual"), - "archive/c/c-sharp/*.cs": LanguageInfo(language="c#", build_mode="manual"), + "archive/c/c-sharp/*.cs": LanguageInfo(language="c#", build_mode="manual", os=WINDOWS), "archive/g/go/*.go": LanguageInfo(language="go", build_mode="autobuild"), "archive/j/java/*.java": LanguageInfo(language="java", build_mode="manual"), "archive/j/javascript/*.js": LanguageInfo(language="javascript"), From 9819039d21c6721603131cbe57386ced6e3093ff Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sun, 19 Jan 2025 17:47:58 -0600 Subject: [PATCH 06/11] Add platform independent script to remove files --- .github/workflows/codeql-analysis.yml | 2 +- scripts/remove_files.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 scripts/remove_files.py diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6f3970950..f2722a7c5 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -114,7 +114,7 @@ jobs: - name: Remove ignored paths if: ${{ github.event_name == 'pull_request' }} - run: rm -f ${{ matrix.paths-ignore }} + run: python scripts/remove_files.py ${{ matrix.paths-ignore }} - name: Set up Python uses: actions/setup-python@v5 diff --git a/scripts/remove_files.py b/scripts/remove_files.py new file mode 100644 index 000000000..40b66056e --- /dev/null +++ b/scripts/remove_files.py @@ -0,0 +1,13 @@ +import os +import sys +from contextlib import suppress + + +def main(): + for path in sys.argv[1:]: + with suppress(OSError): + os.remove(path) + + +if __name__ == "__main__": + main() From 141ce0e1f850fdca3654261f17ea4a1c34596200 Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sun, 19 Jan 2025 17:57:35 -0600 Subject: [PATCH 07/11] Try MacOS for C# --- scripts/get_codeql_languages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get_codeql_languages.py b/scripts/get_codeql_languages.py index 04c87bde3..0116f1536 100644 --- a/scripts/get_codeql_languages.py +++ b/scripts/get_codeql_languages.py @@ -22,7 +22,7 @@ class LanguageInfo: "scripts/*.py": LanguageInfo(language="python"), "archive/c/c/*.c": LanguageInfo(language="c", build_mode="manual"), "archive/c/c-plus-plus/*.cpp": LanguageInfo(language="cpp", build_mode="manual"), - "archive/c/c-sharp/*.cs": LanguageInfo(language="c#", build_mode="manual", os=WINDOWS), + "archive/c/c-sharp/*.cs": LanguageInfo(language="c#", build_mode="manual", os=MACOS), "archive/g/go/*.go": LanguageInfo(language="go", build_mode="autobuild"), "archive/j/java/*.java": LanguageInfo(language="java", build_mode="manual"), "archive/j/javascript/*.js": LanguageInfo(language="javascript"), From 4a017fe8660191278d069efe6d15b595aca35958 Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sun, 19 Jan 2025 18:07:25 -0600 Subject: [PATCH 08/11] Try installing mono-mcs for C# --- .github/workflows/codeql-analysis.yml | 6 +++++- scripts/get_codeql_languages.py | 2 +- scripts/remove_files.py | 13 ------------- 3 files changed, 6 insertions(+), 15 deletions(-) delete mode 100644 scripts/remove_files.py diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index f2722a7c5..581c311db 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -112,9 +112,13 @@ jobs: distribution: 'oracle' java-version: '23' + - name: Install C# compiler + if: ${{ matrix.language == 'c#' }} + run: apt-get update && apt-get install mono-mcs + - name: Remove ignored paths if: ${{ github.event_name == 'pull_request' }} - run: python scripts/remove_files.py ${{ matrix.paths-ignore }} + run: rm -f ${{ matrix.paths-ignore }} - name: Set up Python uses: actions/setup-python@v5 diff --git a/scripts/get_codeql_languages.py b/scripts/get_codeql_languages.py index 0116f1536..4ee5f17d1 100644 --- a/scripts/get_codeql_languages.py +++ b/scripts/get_codeql_languages.py @@ -22,7 +22,7 @@ class LanguageInfo: "scripts/*.py": LanguageInfo(language="python"), "archive/c/c/*.c": LanguageInfo(language="c", build_mode="manual"), "archive/c/c-plus-plus/*.cpp": LanguageInfo(language="cpp", build_mode="manual"), - "archive/c/c-sharp/*.cs": LanguageInfo(language="c#", build_mode="manual", os=MACOS), + "archive/c/c-sharp/*.cs": LanguageInfo(language="c#", build_mode="manual"), "archive/g/go/*.go": LanguageInfo(language="go", build_mode="autobuild"), "archive/j/java/*.java": LanguageInfo(language="java", build_mode="manual"), "archive/j/javascript/*.js": LanguageInfo(language="javascript"), diff --git a/scripts/remove_files.py b/scripts/remove_files.py deleted file mode 100644 index 40b66056e..000000000 --- a/scripts/remove_files.py +++ /dev/null @@ -1,13 +0,0 @@ -import os -import sys -from contextlib import suppress - - -def main(): - for path in sys.argv[1:]: - with suppress(OSError): - os.remove(path) - - -if __name__ == "__main__": - main() From 7e338713932fd2e5a39b74da3a9276e21bbf178c Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sun, 19 Jan 2025 18:10:22 -0600 Subject: [PATCH 09/11] Try again to install C# compiler --- .github/workflows/codeql-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 581c311db..fb79a6869 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -114,7 +114,7 @@ jobs: - name: Install C# compiler if: ${{ matrix.language == 'c#' }} - run: apt-get update && apt-get install mono-mcs + uses: actions/setup-dotnet@v3 - name: Remove ignored paths if: ${{ github.event_name == 'pull_request' }} From 4aa27152b88d91b7626866ff281c1749d82b1173 Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sun, 19 Jan 2025 18:11:34 -0600 Subject: [PATCH 10/11] Use .NET 6.x --- .github/workflows/codeql-analysis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index fb79a6869..75d656dc7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -115,6 +115,8 @@ jobs: - name: Install C# compiler if: ${{ matrix.language == 'c#' }} uses: actions/setup-dotnet@v3 + with: + dotnet-version: '6.x' - name: Remove ignored paths if: ${{ github.event_name == 'pull_request' }} From 906496e61c5793a18518fb46fcdf625794dad9e5 Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sun, 19 Jan 2025 18:14:43 -0600 Subject: [PATCH 11/11] Try sudo apt-get --- .github/workflows/codeql-analysis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 75d656dc7..39b97529f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -114,9 +114,7 @@ jobs: - name: Install C# compiler if: ${{ matrix.language == 'c#' }} - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '6.x' + run: sudo apt-get update && sudo apt-get install -y mono-mcs - name: Remove ignored paths if: ${{ github.event_name == 'pull_request' }}