From 96b00af92222ef0b3211c04c68ab6bd2bea1388c Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sat, 30 Aug 2025 00:15:43 -0500 Subject: [PATCH 01/11] test: initial commit of sprt workflow --- .github/workflows/README.md | 8 +++-- .github/workflows/docs.yml | 1 + .github/workflows/sprt.yml | 61 +++++++++++++++++++++++++++++++++++++ ben-bot/README.md | 1 - 4 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/sprt.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 5f439639..975c38a6 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -4,13 +4,17 @@ This action runs on every push, and when a PR is opened. It builds & runs tests, submits results to CDash (in the `Experimental` group), and uploads the engine artifacts. Include the string `[skip ci]` in your commit message to prevent this workflow from being triggered. +## `docs.yml` + +This action builds the Doxygen documentation and deploys it to GitHub pages. This action is triggered by every push to `main`. + ## `nightly.yml` Similar to `ci.yml`, except this action runs on a schedule every night, and CDash results are in the `Nightly` group. -## `docs.yml` +## `sprt.yml` -This action builds the Doxygen documentation and deploys it to GitHub pages. This action is triggered by every push to `main`. +Runs an [SPRT test](https://www.chessprogramming.org/Sequential_Probability_Ratio_Test) of the given branch against the latest release using the [fastchess tool](https://github.com/Disservin/fastchess). This action runs on every push, and when a PR is opened. Include the string `[skip ci]` in your commit message to prevent this workflow from being triggered. ## `tag_and_release.yml` diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bc9b84c1..a87fe1d2 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -43,6 +43,7 @@ jobs: name: Build docs timeout-minutes: 10 env: + CMAKE_BUILD_PARALLEL_LEVEL: 8 BUILD_DIR: Builds/clang DEPLOY_DIR: deploy diff --git a/.github/workflows/sprt.yml b/.github/workflows/sprt.yml new file mode 100644 index 00000000..c6714f54 --- /dev/null +++ b/.github/workflows/sprt.yml @@ -0,0 +1,61 @@ +# ====================================================================================== +# +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓███████▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░▒▓████████▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░ ░▒▓█▓▒░ +# +# ====================================================================================== + +name: SPRT + +run-name: Run SPRT test + +on: + workflow_dispatch: + push: + branches: [main] + pull_request: + types: [opened, reopened, synchronize] + +concurrency: + group: ${{ github.workflow }}.${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + + sprt: + if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }} + runs-on: ubuntu-latest + name: Run SPRT test + timeout-minutes: 60 + env: + CMAKE_BUILD_PARALLEL_LEVEL: 8 + BUILD_DIR: Builds/clang + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Download fastchess release + uses: robinraju/release-downloader@v1.12 + id: download-fastchess + with: + repository: Disservin/fastchess + latest: true + preRelease: true + fileName: '*-ubuntu-*' + tarBall: false + zipBall: false + out-file-path: fastchess + extract: true + + - name: ls + working-directory: ${{ steps.download-fastchess.outputs.downloaded_files[0] }} + run: ls diff --git a/ben-bot/README.md b/ben-bot/README.md index 9a6d8286..5c3f00fd 100644 --- a/ben-bot/README.md +++ b/ben-bot/README.md @@ -26,5 +26,4 @@ The engine supports several non-standard UCI commands. Type `help` for a list of . venv/bin/activate python3 lichess-bot.py ``` - This will start the bot and wait for challenges via Lichess. From 8d53a7ffa45184e690930db02343d9e55b8b79de Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sat, 30 Aug 2025 00:18:31 -0500 Subject: [PATCH 02/11] test: updating sprt workflow --- .github/workflows/sprt.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sprt.yml b/.github/workflows/sprt.yml index c6714f54..91b9c4f0 100644 --- a/.github/workflows/sprt.yml +++ b/.github/workflows/sprt.yml @@ -57,5 +57,5 @@ jobs: extract: true - name: ls - working-directory: ${{ steps.download-fastchess.outputs.downloaded_files[0] }} + working-directory: fastchess run: ls From f19d87122897610e8226dfc38647e48ad577682f Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sat, 30 Aug 2025 00:35:23 -0500 Subject: [PATCH 03/11] test: updating sprt workflow --- .github/workflows/sprt.yml | 24 ++++++++++++++++++++---- tests/fastchess/CMakeLists.txt | 15 +++++++++++++++ tests/fastchess/README.md | 4 ++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sprt.yml b/.github/workflows/sprt.yml index 91b9c4f0..b2fc7c0c 100644 --- a/.github/workflows/sprt.yml +++ b/.github/workflows/sprt.yml @@ -45,7 +45,6 @@ jobs: - name: Download fastchess release uses: robinraju/release-downloader@v1.12 - id: download-fastchess with: repository: Disservin/fastchess latest: true @@ -56,6 +55,23 @@ jobs: out-file-path: fastchess extract: true - - name: ls - working-directory: fastchess - run: ls + - name: Download latest BenBot release + uses: robinraju/release-downloader@v1.12 + with: + latest: true + fileName: '*-Linux-Clang' + tarBall: false + zipBall: false + out-file-path: baseline + extract: true + + - name: Configure CMake + run: cmake --preset clang -D BENBOT_DOCS=OFF -D FASTCHESS_PROGRAM=fastchess/fastchess-ubuntu-22.04 -D BENBOT_INTERNAL_SPRT_BASELINE_TO_USE=baseline/ben_bot-1.3.0-Linux-Clang + + - name: Build + run: cmake --build . --target ben_bot --config Release + working-directory: ${{ env.BUILD_DIR }} + + - name: Run SPRT test + run: cmake --build . --target sprt --config Release + working-directory: ${{ env.BUILD_DIR }} diff --git a/tests/fastchess/CMakeLists.txt b/tests/fastchess/CMakeLists.txt index 1a0f34dd..fbd5cedf 100644 --- a/tests/fastchess/CMakeLists.txt +++ b/tests/fastchess/CMakeLists.txt @@ -33,6 +33,21 @@ add_custom_target ( add_dependencies (sprt_set_baseline ben_bot) +if (DEFINED BENBOT_INTERNAL_SPRT_BASELINE_TO_USE) + if (NOT EXISTS "${BENBOT_INTERNAL_SPRT_BASELINE_TO_USE}") + message ( + WARNING + "Injected baseline executable not found at path '${BENBOT_INTERNAL_SPRT_BASELINE_TO_USE}'!" + ) + else () + file (COPY_FILE "${BENBOT_INTERNAL_SPRT_BASELINE_TO_USE}" + "$/last-Release" ONLY_IF_DIFFERENT INPUT_MAY_BE_RECENT + ) + endif () +endif () + +# + cmake_host_system_information (RESULT num_cores QUERY NUMBER_OF_PHYSICAL_CORES) set (openings_file "${BenBot_SOURCE_DIR}/ben-bot/resources/res/book.pgn") diff --git a/tests/fastchess/README.md b/tests/fastchess/README.md index da540f33..409db27f 100644 --- a/tests/fastchess/README.md +++ b/tests/fastchess/README.md @@ -9,3 +9,7 @@ SPRT testing works by playing a match between the previous ("baseline") build an * `sprt`: runs SPRT testing with the newest `ben_bot` build and the last baseline that was saved by building `sprt_set_baseline` Neither of these targets are run by CTest; SPRT testing should be invoked manually. + +### CMake options + +- `BENBOT_INTERNAL_SPRT_BASELINE_TO_USE`: intended for use by GitHub Actions; if defined at CMake configure-time, injects an arbitrary executable path to be used as the baseline for the `sprt` test (until the next time `sprt_set_baseline` is run). This allows an action to download the latest release and use it as the baseline against the tip of `main` or a PR. From a70f879558fd387cf3e8224dd4fcbb8d13955480 Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sat, 30 Aug 2025 00:45:22 -0500 Subject: [PATCH 04/11] test: updating sprt workflow --- tests/fastchess/CMakeLists.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/fastchess/CMakeLists.txt b/tests/fastchess/CMakeLists.txt index fbd5cedf..efd57eb8 100644 --- a/tests/fastchess/CMakeLists.txt +++ b/tests/fastchess/CMakeLists.txt @@ -34,15 +34,16 @@ add_custom_target ( add_dependencies (sprt_set_baseline ben_bot) if (DEFINED BENBOT_INTERNAL_SPRT_BASELINE_TO_USE) - if (NOT EXISTS "${BENBOT_INTERNAL_SPRT_BASELINE_TO_USE}") + if (EXISTS "${BENBOT_INTERNAL_SPRT_BASELINE_TO_USE}") + file (COPY_FILE "${BENBOT_INTERNAL_SPRT_BASELINE_TO_USE}" + # TODO: should be a generator expression, but need to do this at configure time? + "${CMAKE_CURRENT_BINARY_DIR}/Release" ONLY_IF_DIFFERENT INPUT_MAY_BE_RECENT + ) + else () message ( WARNING "Injected baseline executable not found at path '${BENBOT_INTERNAL_SPRT_BASELINE_TO_USE}'!" ) - else () - file (COPY_FILE "${BENBOT_INTERNAL_SPRT_BASELINE_TO_USE}" - "$/last-Release" ONLY_IF_DIFFERENT INPUT_MAY_BE_RECENT - ) endif () endif () From 5f043254874955cbc134d5ed83f7f68f83c9f5f3 Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sat, 30 Aug 2025 00:55:26 -0500 Subject: [PATCH 05/11] test: updating sprt workflow --- .github/workflows/sprt.yml | 6 ++++-- tests/fastchess/CMakeLists.txt | 16 +++------------ tests/fastchess/InjectSPRTBaseline.cmake | 25 ++++++++++++++++++++++++ tests/fastchess/README.md | 4 ---- 4 files changed, 32 insertions(+), 19 deletions(-) create mode 100644 tests/fastchess/InjectSPRTBaseline.cmake diff --git a/.github/workflows/sprt.yml b/.github/workflows/sprt.yml index b2fc7c0c..c2d91113 100644 --- a/.github/workflows/sprt.yml +++ b/.github/workflows/sprt.yml @@ -63,10 +63,12 @@ jobs: tarBall: false zipBall: false out-file-path: baseline - extract: true - name: Configure CMake - run: cmake --preset clang -D BENBOT_DOCS=OFF -D FASTCHESS_PROGRAM=fastchess/fastchess-ubuntu-22.04 -D BENBOT_INTERNAL_SPRT_BASELINE_TO_USE=baseline/ben_bot-1.3.0-Linux-Clang + run: cmake --preset clang -D BENBOT_DOCS=OFF -D FASTCHESS_PROGRAM=fastchess/fastchess-ubuntu-22.04 + + - name: Inject baseline binary + run: cmake -D BASELINE_BINARY=baseline/ben_bot-1.3.0-Linux-Clang -P ${{ env.BUILD_DIR }}/InjectSPRTBaseline-Release.cmake - name: Build run: cmake --build . --target ben_bot --config Release diff --git a/tests/fastchess/CMakeLists.txt b/tests/fastchess/CMakeLists.txt index efd57eb8..5ec79196 100644 --- a/tests/fastchess/CMakeLists.txt +++ b/tests/fastchess/CMakeLists.txt @@ -33,19 +33,9 @@ add_custom_target ( add_dependencies (sprt_set_baseline ben_bot) -if (DEFINED BENBOT_INTERNAL_SPRT_BASELINE_TO_USE) - if (EXISTS "${BENBOT_INTERNAL_SPRT_BASELINE_TO_USE}") - file (COPY_FILE "${BENBOT_INTERNAL_SPRT_BASELINE_TO_USE}" - # TODO: should be a generator expression, but need to do this at configure time? - "${CMAKE_CURRENT_BINARY_DIR}/Release" ONLY_IF_DIFFERENT INPUT_MAY_BE_RECENT - ) - else () - message ( - WARNING - "Injected baseline executable not found at path '${BENBOT_INTERNAL_SPRT_BASELINE_TO_USE}'!" - ) - endif () -endif () +file (GENERATE OUTPUT "${BenBot_BINARY_DIR}/InjectSPRTBaseline-$.cmake" + INPUT InjectSPRTBaseline.cmake TARGET ben_bot NEWLINE_STYLE UNIX +) # diff --git a/tests/fastchess/InjectSPRTBaseline.cmake b/tests/fastchess/InjectSPRTBaseline.cmake new file mode 100644 index 00000000..dbbb63e0 --- /dev/null +++ b/tests/fastchess/InjectSPRTBaseline.cmake @@ -0,0 +1,25 @@ +# ====================================================================================== +# +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓███████▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░▒▓████████▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░ ░▒▓█▓▒░ +# +# ====================================================================================== + +cmake_minimum_required (VERSION 3.30.0 FATAL_ERROR) + +if (NOT DEFINED BASELINE_BINARY) + message (FATAL_ERROR "BASELINE_BINARY must be defined!") +endif () + +if (NOT EXISTS "${BASELINE_BINARY}") + message (FATAL_ERROR "BASELINE_BINARY does not exist at path '${BASELINE_BINARY}'!") +endif () + +file (COPY_FILE "${BASELINE_BINARY}" "$/last-$" ONLY_IF_DIFFERENT + INPUT_MAY_BE_RECENT +) diff --git a/tests/fastchess/README.md b/tests/fastchess/README.md index 409db27f..da540f33 100644 --- a/tests/fastchess/README.md +++ b/tests/fastchess/README.md @@ -9,7 +9,3 @@ SPRT testing works by playing a match between the previous ("baseline") build an * `sprt`: runs SPRT testing with the newest `ben_bot` build and the last baseline that was saved by building `sprt_set_baseline` Neither of these targets are run by CTest; SPRT testing should be invoked manually. - -### CMake options - -- `BENBOT_INTERNAL_SPRT_BASELINE_TO_USE`: intended for use by GitHub Actions; if defined at CMake configure-time, injects an arbitrary executable path to be used as the baseline for the `sprt` test (until the next time `sprt_set_baseline` is run). This allows an action to download the latest release and use it as the baseline against the tip of `main` or a PR. From 163fe576d44f2a01a8509d086bebc2a72eeb16f4 Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sat, 30 Aug 2025 01:00:56 -0500 Subject: [PATCH 06/11] test: updating sprt workflow --- .github/workflows/sprt.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sprt.yml b/.github/workflows/sprt.yml index c2d91113..b97f8e84 100644 --- a/.github/workflows/sprt.yml +++ b/.github/workflows/sprt.yml @@ -38,6 +38,8 @@ jobs: env: CMAKE_BUILD_PARALLEL_LEVEL: 8 BUILD_DIR: Builds/clang + FASTCHESS_PATH: fastchess/fastchess-ubuntu-22.04 + BASELINE_PATH: baseline/ben_bot-1.3.0-Linux-Clang steps: - name: Checkout code @@ -64,11 +66,14 @@ jobs: zipBall: false out-file-path: baseline + - name: Add execution permissions + run: chmod --preserve-root 777 ${{ env.FASTCHESS_PATH }} ${{ env.BASELINE_PATH }} + - name: Configure CMake - run: cmake --preset clang -D BENBOT_DOCS=OFF -D FASTCHESS_PROGRAM=fastchess/fastchess-ubuntu-22.04 + run: cmake --preset clang -D BENBOT_DOCS=OFF -D FASTCHESS_PROGRAM=${{ env.FASTCHESS_PATH }} - name: Inject baseline binary - run: cmake -D BASELINE_BINARY=baseline/ben_bot-1.3.0-Linux-Clang -P ${{ env.BUILD_DIR }}/InjectSPRTBaseline-Release.cmake + run: cmake -D BASELINE_BINARY=${{ env.BASELINE_PATH }} -P ${{ env.BUILD_DIR }}/InjectSPRTBaseline-Release.cmake - name: Build run: cmake --build . --target ben_bot --config Release From 4f583b4ade151b7d599aa74ff5c705c8cd95a909 Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sat, 30 Aug 2025 02:52:54 -0500 Subject: [PATCH 07/11] test: updating sprt workflow --- .github/scripts/InstallPythonDeps.cmake | 2 +- .github/scripts/ReportSPRTResults.py | 187 ++++++++++++++++++++++++ .github/workflows/sprt.yml | 16 +- tests/CMakeLists.txt | 2 +- tests/fastchess/CMakeLists.txt | 17 ++- 5 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 .github/scripts/ReportSPRTResults.py diff --git a/.github/scripts/InstallPythonDeps.cmake b/.github/scripts/InstallPythonDeps.cmake index 479e9108..52bb0d68 100644 --- a/.github/scripts/InstallPythonDeps.cmake +++ b/.github/scripts/InstallPythonDeps.cmake @@ -18,7 +18,7 @@ CMake find module for locating the python executable used to install dependencie cmake_minimum_required (VERSION 3.30.0 FATAL_ERROR) -find_package (Python 3.9 COMPONENTS Interpreter REQUIRED) +find_package (Python 3.10 COMPONENTS Interpreter REQUIRED) # cmake-format: off execute_process ( diff --git a/.github/scripts/ReportSPRTResults.py b/.github/scripts/ReportSPRTResults.py new file mode 100644 index 00000000..6b496160 --- /dev/null +++ b/.github/scripts/ReportSPRTResults.py @@ -0,0 +1,187 @@ +# ====================================================================================== +# +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓███████▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░▒▓████████▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░ ░▒▓█▓▒░ +# +# ====================================================================================== + +from pathlib import Path +import sys +import re +from enum import Enum + +LOG_FILE = Path(sys.argv[1]) + +with open(LOG_FILE, 'r') as file: + SPRT_OUTPUT = file.readlines() + +# finds the indices of the last 2 '-----------------' lines in the output +# the final results report is in the fenced section between these 2 lines +def find_last_two_fences(): + indices = [] + + for i in range(len(SPRT_OUTPUT) - 1, -1, -1): + if re.search(r"(-)+", SPRT_OUTPUT[i]): + indices.append(i) + if len(indices) == 2: + break + + return sorted(indices) + +def get_final_results_report(): + start, end = find_last_two_fences() + + return SPRT_OUTPUT[start+1:end] + +def replace_pairs(old_values, new_values, string): + for old, new in zip(old_values, new_values): + string.replace(old, new) + + return string + +# returns the first string that matches the given regex +def find_first_match(regex, strings): + return next(str for str in strings if re.search(regex, str)) + +class EmojiType(Enum): + NEUTRAL = 1, + POSITIVE = 2, + NEGATIVE = 3 + +def get_emoji(type): + match type: + case EmojiType.NEUTRAL: return '🟰' + case EmojiType.POSITIVE: return '✅' + case EmojiType.NEGATIVE: return '❌' + +RESULTS = get_final_results_report() + +def get_elo(): + elo_line = find_first_match('Elo:', RESULTS) + + first_space_idx = elo_line.find(' ') + second_space_idx = elo_line.find(' ', first_space_idx + 1) + + return float( + elo_line[first_space_idx:second_space_idx] + ) + +def get_elo_emoji(elo): + if abs(elo) <= 1: + return get_emoji(EmojiType.NEUTRAL) + + if elo > 0: + return get_emoji(EmojiType.POSITIVE) + + return get_emoji(EmojiType.NEGATIVE) + +def get_result_breakdown(): + games_line = find_first_match('Games:', RESULTS) + + second_space_idx = games_line.find(' ', + games_line.find(' ') + 1) + + after_second_space = games_line[second_space_idx+1:] + + # string format is: + # Wins: W, Losses: L, Draws: D, Points: 50.5 (50.50 %) + + wins_comma_idx = after_second_space.find(',') + + num_wins = int( + after_second_space[after_second_space.find(' ')+1:wins_comma_idx] + ) + + after_wins = (after_second_space[after_second_space.find(' ', wins_comma_idx + 1):]).lstrip() + + losses_comma_idx = after_wins.find(',') + + num_losses = int( + after_wins[after_wins.find(' ')+1:losses_comma_idx] + ) + + after_losses = (after_wins[after_wins.find(' ', losses_comma_idx + 1):]).lstrip() + + draws_comma_idx = after_losses.find(',') + + num_draws = int( + after_losses[after_losses.find(' ')+1:draws_comma_idx] + ) + + after_draws = (after_losses[after_losses.find(' ', draws_comma_idx+1):]).lstrip() + + pcnt = float( + after_draws[after_draws.find('(')+1:after_draws.find('%')] + ) + + return ( + num_wins, num_losses, num_draws, pcnt + ) + +def get_wins_emoji(wins): + if wins in range(45, 55): + return get_emoji(EmojiType.NEUTRAL) + + if wins > 50: + return get_emoji(EmojiType.POSITIVE) + + return get_emoji(EmojiType.NEGATIVE) + +def get_losses_emoji(losses): + if losses in range(45, 55): + return get_emoji(EmojiType.NEUTRAL) + + if losses > 50: + return get_emoji(EmojiType.NEGATIVE) + + return get_emoji(EmojiType.POSITIVE) + +def get_draws_emoji(draws, losses, wins): + if draws < losses: + return get_emoji(EmojiType.NEGATIVE) + + if draws > wins: + return get_emoji(EmojiType.POSITIVE) + + return get_emoji(EmojiType.NEUTRAL) + +def get_pcnt_emoji(pcnt): + if pcnt in range(0, 65): + return get_emoji(EmojiType.NEGATIVE) + + if pcnt in range(65, 85): + return get_emoji(EmojiType.NEUTRAL) + + return get_emoji(EmojiType.POSITIVE) + +# + +SCRIPT_DIR = Path(__file__).resolve().parent + +with open(f'{SCRIPT_DIR}/sprt-results.md', 'r') as file: + MD_TEMPLATE_TEXT = file.read() + +elo = get_elo() + +num_wins, num_losses, num_draws, pcnt = get_result_breakdown() + +print( + replace_pairs( + ['%ELO%', '%ELO_EMOJI%', + '%WINS%', '%WINS_EMOJI%', + '%LOSSES%', '%LOSSES_EMOJI%', + '%DRAWS%', '%DRAWS_EMOJI%', + '%PCNT%', '%PCNT_EMOJI%'], + [f'{elo}', get_elo_emoji(elo), + f'{num_wins}', get_wins_emoji(num_wins), + f'{num_losses}', get_losses_emoji(num_losses), + f'{num_draws}', get_draws_emoji(num_draws, num_losses, num_wins), + f'{pcnt}%', get_pcnt_emoji(pcnt)], + MD_TEMPLATE_TEXT + ) +) diff --git a/.github/workflows/sprt.yml b/.github/workflows/sprt.yml index b97f8e84..b3dbf1e0 100644 --- a/.github/workflows/sprt.yml +++ b/.github/workflows/sprt.yml @@ -40,6 +40,7 @@ jobs: BUILD_DIR: Builds/clang FASTCHESS_PATH: fastchess/fastchess-ubuntu-22.04 BASELINE_PATH: baseline/ben_bot-1.3.0-Linux-Clang + SPRT_OUTPUT_LOG: ${{ github.workspace }}/logs/sprt/output.log steps: - name: Checkout code @@ -80,5 +81,18 @@ jobs: working-directory: ${{ env.BUILD_DIR }} - name: Run SPRT test - run: cmake --build . --target sprt --config Release + run: cmake --build . --target sprt --config Release | tee ${{ env.SPRT_OUTPUT_LOG }} working-directory: ${{ env.BUILD_DIR }} + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4.6.2 + with: + name: sprt-logs + path: logs/sprt/* + if-no-files-found: error + + - name: Report results + if: always() + run: python3 ReportSPRTResults.py ${{ env.SPRT_OUTPUT_LOG }} >> $GITHUB_STEP_SUMMARY + working-directory: .github/scripts diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c6646411..95bbbf0b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -10,7 +10,7 @@ # # ====================================================================================== -find_package (Python 3.9 COMPONENTS Interpreter) +find_package (Python 3.10 COMPONENTS Interpreter) if (NOT TARGET Python::Interpreter) message (VERBOSE "Python interpreter not found, not adding all test cases") diff --git a/tests/fastchess/CMakeLists.txt b/tests/fastchess/CMakeLists.txt index 5ec79196..df4df5d0 100644 --- a/tests/fastchess/CMakeLists.txt +++ b/tests/fastchess/CMakeLists.txt @@ -43,16 +43,31 @@ cmake_host_system_information (RESULT num_cores QUERY NUMBER_OF_PHYSICAL_CORES) set (openings_file "${BenBot_SOURCE_DIR}/ben-bot/resources/res/book.pgn") +set (logs_dir "${BenBot_SOURCE_DIR}/logs/sprt") + +add_custom_command ( + OUTPUT "${logs_dir}/stamp" + COMMAND "${CMAKE_COMMAND}" -E make_directory "${logs_dir}" + COMMAND "${CMAKE_COMMAND}" -E touch "${logs_dir}/stamp" + COMMENT "Creating SPRT logs directory..." + VERBATIM USES_TERMINAL +) + # cmake-format: off add_custom_target ( sprt COMMAND "${FASTCHESS_PROGRAM}" -engine "cmd=$" name=Refactor -engine "cmd=${baseline_binary}" name=Baseline - -each tc=8+0.08 -rounds 50 -repeat -concurrency "${num_cores}" -recover + -each tc=8+0.08 -rounds 50 -repeat -concurrency "${num_cores}" + -recover -show-latency -openings "file=${openings_file}" format=pgn -sprt elo0=0 elo1=10 alpha=0.05 beta=0.05 + -pgnout "file=${logs_dir}/games.pgn" nodes=true nps=true timeleft=true latency=true + -epdout "file=${logs_dir}/final-positions.epd" + -log "file=${logs_dir}/sprt.log" engine=true WORKING_DIRECTORY "$" + DEPENDS "${logs_dir}/stamp" COMMENT "Running SPRT test..." VERBATIM USES_TERMINAL ) From c8e34500cc9e0bacc57a92b2f4320459e74bbf66 Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sat, 30 Aug 2025 02:53:52 -0500 Subject: [PATCH 08/11] test: updating sprt workflow --- .github/scripts/sprt-results.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/scripts/sprt-results.md diff --git a/.github/scripts/sprt-results.md b/.github/scripts/sprt-results.md new file mode 100644 index 00000000..7966f124 --- /dev/null +++ b/.github/scripts/sprt-results.md @@ -0,0 +1,21 @@ +# SPRT results + +| ELO | | +| :---: | :---------: | +| %ELO% | %ELO_EMOJI% | + +| Wins | | +| :----: | :----------: | +| %WINS% | %WINS_EMOJI% | + +| Losses | | +| :------: | :------------: | +| %LOSSES% | %LOSSES_EMOJI% | + +| Draws. | | +| :-----: | :-----------: | +| %DRAWS% | %DRAWS_EMOJI% | + +| Percent | | +| :-----: | :----------: | +| %PCNT% | %PCNT_EMOJI% | From 0909a71137bb59ea151622b94fa4544f663ececc Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sat, 30 Aug 2025 03:11:13 -0500 Subject: [PATCH 09/11] refactor: cmake --- .github/scripts/ReportSPRTResults.py | 4 ++-- .github/workflows/sprt.yml | 6 ++++-- tests/fastchess/CMakeLists.txt | 10 +++++++++- tests/fastchess/InjectSPRTBaseline.cmake | 4 +--- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/scripts/ReportSPRTResults.py b/.github/scripts/ReportSPRTResults.py index 6b496160..0b8fbdef 100644 --- a/.github/scripts/ReportSPRTResults.py +++ b/.github/scripts/ReportSPRTResults.py @@ -151,10 +151,10 @@ def get_draws_emoji(draws, losses, wins): return get_emoji(EmojiType.NEUTRAL) def get_pcnt_emoji(pcnt): - if pcnt in range(0, 65): + if pcnt in range(0, 45): return get_emoji(EmojiType.NEGATIVE) - if pcnt in range(65, 85): + if pcnt in range(45, 55): return get_emoji(EmojiType.NEUTRAL) return get_emoji(EmojiType.POSITIVE) diff --git a/.github/workflows/sprt.yml b/.github/workflows/sprt.yml index b3dbf1e0..1c4aeaa1 100644 --- a/.github/workflows/sprt.yml +++ b/.github/workflows/sprt.yml @@ -81,7 +81,9 @@ jobs: working-directory: ${{ env.BUILD_DIR }} - name: Run SPRT test - run: cmake --build . --target sprt --config Release | tee ${{ env.SPRT_OUTPUT_LOG }} + run: | + mkdir ${{ github.workspace }}/logs/sprt + cmake --build . --target sprt --config Release | tee ${{ env.SPRT_OUTPUT_LOG }} working-directory: ${{ env.BUILD_DIR }} - name: Upload logs @@ -89,7 +91,7 @@ jobs: uses: actions/upload-artifact@v4.6.2 with: name: sprt-logs - path: logs/sprt/* + path: logs/sprt if-no-files-found: error - name: Report results diff --git a/tests/fastchess/CMakeLists.txt b/tests/fastchess/CMakeLists.txt index df4df5d0..ec7d59a1 100644 --- a/tests/fastchess/CMakeLists.txt +++ b/tests/fastchess/CMakeLists.txt @@ -33,8 +33,16 @@ add_custom_target ( add_dependencies (sprt_set_baseline ben_bot) +# used by the sprt ci workflow to inject the binary from the latest release as the baseline +set (injection_script "${CMAKE_CURRENT_LIST_DIR}/InjectSPRTBaseline.cmake") + +file (READ "${injection_script}" script_content) +list (APPEND CMAKE_CONFIGURE_DEPENDS "${injection_script}") + +string (CONFIGURE "${script_content}" script_content @ONLY) + file (GENERATE OUTPUT "${BenBot_BINARY_DIR}/InjectSPRTBaseline-$.cmake" - INPUT InjectSPRTBaseline.cmake TARGET ben_bot NEWLINE_STYLE UNIX + CONTENT "${script_content}" TARGET ben_bot NEWLINE_STYLE UNIX ) # diff --git a/tests/fastchess/InjectSPRTBaseline.cmake b/tests/fastchess/InjectSPRTBaseline.cmake index dbbb63e0..0ec05f75 100644 --- a/tests/fastchess/InjectSPRTBaseline.cmake +++ b/tests/fastchess/InjectSPRTBaseline.cmake @@ -20,6 +20,4 @@ if (NOT EXISTS "${BASELINE_BINARY}") message (FATAL_ERROR "BASELINE_BINARY does not exist at path '${BASELINE_BINARY}'!") endif () -file (COPY_FILE "${BASELINE_BINARY}" "$/last-$" ONLY_IF_DIFFERENT - INPUT_MAY_BE_RECENT -) +file (COPY_FILE "${BASELINE_BINARY}" "@baseline_binary@" ONLY_IF_DIFFERENT INPUT_MAY_BE_RECENT) From 52626ea6db19186a9b8e4c6adcc7c6e747da6c0a Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sat, 30 Aug 2025 03:14:14 -0500 Subject: [PATCH 10/11] fix: sprt workflow --- .github/workflows/sprt.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sprt.yml b/.github/workflows/sprt.yml index 1c4aeaa1..1d4c4436 100644 --- a/.github/workflows/sprt.yml +++ b/.github/workflows/sprt.yml @@ -82,7 +82,7 @@ jobs: - name: Run SPRT test run: | - mkdir ${{ github.workspace }}/logs/sprt + mkdir -p ${{ github.workspace }}/logs/sprt cmake --build . --target sprt --config Release | tee ${{ env.SPRT_OUTPUT_LOG }} working-directory: ${{ env.BUILD_DIR }} From dacad609493762bec9cace422c573a50ba9ae0f1 Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sat, 30 Aug 2025 03:49:11 -0500 Subject: [PATCH 11/11] refactor: sprt workflow --- .github/scripts/ReportSPRTResults.py | 30 +++++++++++++++++----------- .github/workflows/sprt.yml | 4 ++++ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/.github/scripts/ReportSPRTResults.py b/.github/scripts/ReportSPRTResults.py index 0b8fbdef..65948388 100644 --- a/.github/scripts/ReportSPRTResults.py +++ b/.github/scripts/ReportSPRTResults.py @@ -25,9 +25,9 @@ def find_last_two_fences(): indices = [] - for i in range(len(SPRT_OUTPUT) - 1, -1, -1): - if re.search(r"(-)+", SPRT_OUTPUT[i]): - indices.append(i) + for i, line in enumerate(reversed(SPRT_OUTPUT)): + if re.search(r"(--)+", line): + indices.append(len(SPRT_OUTPUT) - i) if len(indices) == 2: break @@ -36,17 +36,17 @@ def find_last_two_fences(): def get_final_results_report(): start, end = find_last_two_fences() - return SPRT_OUTPUT[start+1:end] + return SPRT_OUTPUT[start + 1 : end] -def replace_pairs(old_values, new_values, string): - for old, new in zip(old_values, new_values): - string.replace(old, new) - - return string +RESULTS = get_final_results_report() # returns the first string that matches the given regex def find_first_match(regex, strings): - return next(str for str in strings if re.search(regex, str)) + for str in strings: + if re.search(regex, str): + return str + + return '' class EmojiType(Enum): NEUTRAL = 1, @@ -59,8 +59,6 @@ def get_emoji(type): case EmojiType.POSITIVE: return '✅' case EmojiType.NEGATIVE: return '❌' -RESULTS = get_final_results_report() - def get_elo(): elo_line = find_first_match('Elo:', RESULTS) @@ -161,6 +159,14 @@ def get_pcnt_emoji(pcnt): # +def replace_pairs(old_values, new_values, string): + text = string + + for old, new in zip(old_values, new_values): + text = text.replace(old, new) + + return text + SCRIPT_DIR = Path(__file__).resolve().parent with open(f'{SCRIPT_DIR}/sprt-results.md', 'r') as file: diff --git a/.github/workflows/sprt.yml b/.github/workflows/sprt.yml index 1d4c4436..bddfd1eb 100644 --- a/.github/workflows/sprt.yml +++ b/.github/workflows/sprt.yml @@ -25,6 +25,10 @@ concurrency: group: ${{ github.workflow }}.${{ github.ref }} cancel-in-progress: true +defaults: + run: + shell: bash + permissions: contents: read