From ed74a117abd94632dfd0f8dd6c696faf09b8af4a Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sat, 10 May 2025 06:18:29 -0500 Subject: [PATCH 1/8] Add Swift to CodeQL again --- .github/CONTRIBUTING.md | 2 +- scripts/build_codeql_language.py | 1 + scripts/get_codeql_languages.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 4962e86fa..cc78ed8c7 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -306,7 +306,7 @@ To run the tests locally, **you will need the following dependencies**: - **Python 3.9+**: our build system is constructed with Python given that testing is built with glotter, a Python testing library that leverages docker. - **Poetry**: our build system is managed and versioned using Poetry. Make sure - to use version **2.1.1** or later. If you are using an older version, please + to use version **2.1.3** or later. If you are using an older version, please update to the latest. With all three installed, the remaining dependencies can be installed using diff --git a/scripts/build_codeql_language.py b/scripts/build_codeql_language.py index 5b3306c1e..29d49543d 100644 --- a/scripts/build_codeql_language.py +++ b/scripts/build_codeql_language.py @@ -12,6 +12,7 @@ "cpp": "c-plus-plus", "java": "java", "kotlin": "kotlin", + "switft": "swift", } diff --git a/scripts/get_codeql_languages.py b/scripts/get_codeql_languages.py index 32544d5a1..92af9f0fa 100644 --- a/scripts/get_codeql_languages.py +++ b/scripts/get_codeql_languages.py @@ -30,6 +30,7 @@ class LanguageInfo: "archive/p/python/*.py": LanguageInfo(language="python"), "archive/r/ruby/*.rb": LanguageInfo(language="ruby"), "archive/t/typescript/*.ts": LanguageInfo(language="typescript"), + "archive/s/swift/*.swift": LanguageInfo(language="swift", build_mode="manual", os=MACOS), } ALL_CODEQL_LANGUAGES_FILES = { ".github/workflows/codeql-analysis.yml", From 38ad9379591f37d79ad2064210888fb53a7f8790 Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sat, 10 May 2025 06:33:10 -0500 Subject: [PATCH 2/8] Try MacOS 14 --- scripts/get_codeql_languages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get_codeql_languages.py b/scripts/get_codeql_languages.py index 92af9f0fa..e82c9d4e6 100644 --- a/scripts/get_codeql_languages.py +++ b/scripts/get_codeql_languages.py @@ -7,7 +7,7 @@ from typing import DefaultDict, Dict, Set LINUX = "ubuntu-latest" -MACOS = "macos-latest" +MACOS = "macos-14" WINDOWS = "windows-latest" From b242660f8078d826632dd3dbc21de55603376024 Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sat, 10 May 2025 06:35:18 -0500 Subject: [PATCH 3/8] Try MacOS 13 --- scripts/get_codeql_languages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get_codeql_languages.py b/scripts/get_codeql_languages.py index e82c9d4e6..c56411146 100644 --- a/scripts/get_codeql_languages.py +++ b/scripts/get_codeql_languages.py @@ -7,7 +7,7 @@ from typing import DefaultDict, Dict, Set LINUX = "ubuntu-latest" -MACOS = "macos-14" +MACOS = "macos-13" WINDOWS = "windows-latest" From ffcf7c290fa19e4b5bf527a9b12314b3ca9e7e93 Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sat, 10 May 2025 06:37:38 -0500 Subject: [PATCH 4/8] Fix typo --- scripts/build_codeql_language.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_codeql_language.py b/scripts/build_codeql_language.py index 29d49543d..10d300b3a 100644 --- a/scripts/build_codeql_language.py +++ b/scripts/build_codeql_language.py @@ -12,7 +12,7 @@ "cpp": "c-plus-plus", "java": "java", "kotlin": "kotlin", - "switft": "swift", + "swift": "swift", } From 376317ecb02aa75fece1609c9af1538717980693 Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sat, 10 May 2025 06:59:27 -0500 Subject: [PATCH 5/8] Try to do this without glotter --- .github/workflows/codeql-analysis.yml | 8 -------- scripts/build_codeql_language.py | 21 ++++++++++++--------- scripts/get_codeql_languages.py | 2 +- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index dce907484..a853be4b4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -140,14 +140,6 @@ jobs: with: python-version: "${{ steps.repo-config.outputs['python-version'] }}" - - name: Run Poetry Image - uses: abatilo/actions-poetry@v3 - with: - poetry-version: "${{ steps.repo-config.outputs['poetry-version'] }}" - - - name: Install Dependencies - run: poetry install - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v3 diff --git a/scripts/build_codeql_language.py b/scripts/build_codeql_language.py index 10d300b3a..f1cdb6fe5 100644 --- a/scripts/build_codeql_language.py +++ b/scripts/build_codeql_language.py @@ -1,12 +1,11 @@ import argparse +import re import shlex import subprocess from dataclasses import dataclass from pathlib import Path from typing import Dict, List -from glotter.source import Source - TEST_INFO_DIR: Dict[str, str] = { "c": "c", "cpp": "c-plus-plus", @@ -15,6 +14,11 @@ "swift": "swift", } +EXTENSION_PATTERN = re.compile(r"""^\s+extension:\s*['"](^'"]+)['"]""") +BUILD_PATTERN = re.compile(r"""^\s+build:\s*['"]([^'"]+)['"]""") +SOURCE_NAME_PATTERN = re.compile(r"\{\{\s*source\.name\s*\}\}") +SOURCE_EXTENSION_PATTERN = re.compile(r"\{\{\s*source\.extension\s*\}\}") + @dataclass class TestInfoStruct: @@ -43,13 +47,12 @@ def get_test_info_struct(language: str) -> TestInfoStruct: def get_build_command(testinfo_struct: TestInfoStruct, path: Path) -> List[str]: - source = Source( - name=path.name, - language=testinfo_struct.language, - path=str(path), - test_info_string=testinfo_struct.test_info_str, - ) - build: str = source.test_info.container_info.build + test_info_str = testinfo_struct.test_info_str + extension = EXTENSION_PATTERN.sub(test_info_str, r"\g<1>") + source_name = path.name[: -len(extension)] + build = BUILD_PATTERN.sub(test_info_str, r"\g<1>") + build = SOURCE_NAME_PATTERN.sub(build, source_name) + build = SOURCE_EXTENSION_PATTERN.sub(build, extension) return shlex.split(build) diff --git a/scripts/get_codeql_languages.py b/scripts/get_codeql_languages.py index c56411146..92af9f0fa 100644 --- a/scripts/get_codeql_languages.py +++ b/scripts/get_codeql_languages.py @@ -7,7 +7,7 @@ from typing import DefaultDict, Dict, Set LINUX = "ubuntu-latest" -MACOS = "macos-13" +MACOS = "macos-latest" WINDOWS = "windows-latest" From 3dfd808991b4a1b8b184094cc7d2b2725f577f7d Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sat, 10 May 2025 07:30:12 -0500 Subject: [PATCH 6/8] Change method used to find and replace build command --- scripts/build_codeql_language.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/scripts/build_codeql_language.py b/scripts/build_codeql_language.py index f1cdb6fe5..8988aac01 100644 --- a/scripts/build_codeql_language.py +++ b/scripts/build_codeql_language.py @@ -14,8 +14,7 @@ "swift": "swift", } -EXTENSION_PATTERN = re.compile(r"""^\s+extension:\s*['"](^'"]+)['"]""") -BUILD_PATTERN = re.compile(r"""^\s+build:\s*['"]([^'"]+)['"]""") +BUILD_PATTERN = re.compile(r"""^\s+build:\s*['"]([^'"]+)""", re.M) SOURCE_NAME_PATTERN = re.compile(r"\{\{\s*source\.name\s*\}\}") SOURCE_EXTENSION_PATTERN = re.compile(r"\{\{\s*source\.extension\s*\}\}") @@ -47,12 +46,11 @@ def get_test_info_struct(language: str) -> TestInfoStruct: def get_build_command(testinfo_struct: TestInfoStruct, path: Path) -> List[str]: - test_info_str = testinfo_struct.test_info_str - extension = EXTENSION_PATTERN.sub(test_info_str, r"\g<1>") - source_name = path.name[: -len(extension)] - build = BUILD_PATTERN.sub(test_info_str, r"\g<1>") - build = SOURCE_NAME_PATTERN.sub(build, source_name) - build = SOURCE_EXTENSION_PATTERN.sub(build, extension) + match_str = BUILD_PATTERN.search(testinfo_struct.test_info_str) + assert match_str + + build = SOURCE_NAME_PATTERN.sub(path.stem, match_str.group(1)) + build = SOURCE_EXTENSION_PATTERN.sub(path.suffix, build) return shlex.split(build) From c3d673276331d7e9fd7f2ed2ee4a311cb9095fce Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sat, 10 May 2025 07:33:58 -0500 Subject: [PATCH 7/8] Don't use poetry to run builder --- .github/workflows/codeql-analysis.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index a853be4b4..6166e3a8c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -129,17 +129,6 @@ jobs: with: fetch-depth: 2 - - name: Read repo config - uses: pietrobolcato/action-read-yaml@1.1.0 - id: repo-config - with: - config: repo-config.yml - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "${{ steps.repo-config.outputs['python-version'] }}" - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v3 @@ -162,7 +151,7 @@ jobs: - name: Manual build if: ${{ matrix.build-mode == 'manual' }} - run: poetry run python scripts/build_codeql_language.py ${{ matrix.language }} ${{ matrix.paths }} + run: python scripts/build_codeql_language.py ${{ matrix.language }} ${{ matrix.paths }} - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 From 43e0a3ae31d266a8eaa2485ea1e65eb95969dd00 Mon Sep 17 00:00:00 2001 From: rzuckerm Date: Sat, 10 May 2025 07:48:02 -0500 Subject: [PATCH 8/8] Don't run for changes to poetry or repo config --- .github/workflows/codeql-analysis.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6166e3a8c..7a2d3a7c4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -16,10 +16,7 @@ on: branches: [ main ] paths: - '.github/workflows/codeql-analysis.yml' - - 'repo-config.yml' - 'scripts/*.py' - - 'pyproject.toml' - - 'poetry.lock' - 'archive/c/c/*.c' - 'archive/c/c/testinfo.yml' - 'archive/c/c-plus-plus/*.cpp' @@ -45,10 +42,7 @@ on: - 'main' paths: - '.github/workflows/codeql-analysis.yml' - - 'repo-config.yml' - 'scripts/*.py' - - 'pyproject.toml' - - 'poetry.lock' - 'archive/c/c/*.c' - 'archive/c/c/testinfo.yml' - 'archive/c/c-plus-plus/*.cpp'