test: running SPRT in CI

.github/scripts/InstallPythonDeps.cmake

@@ -18,7 +18,7 @@ CMake find module for locating the python executable used to install dependencie
 
 cmake_minimum_required (VERSION 3.30.0 FATAL_ERROR)
 
-find_package (Python 3.9 COMPONENTS Interpreter REQUIRED)
+find_package (Python 3.10 COMPONENTS Interpreter REQUIRED)
 
 # cmake-format: off
 execute_process (

.github/scripts/ReportSPRTResults.py

@@ -0,0 +1,193 @@
+# ======================================================================================
+#
+# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓███████▓▒░       ░▒▓███████▓▒░ ░▒▓██████▓▒░▒▓████████▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓███████▓▒░░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓███████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░      ░▒▓███████▓▒░ ░▒▓██████▓▒░  ░▒▓█▓▒░
+#
+# ======================================================================================
+
+from pathlib import Path
+import sys
+import re
+from enum import Enum
+
+LOG_FILE = Path(sys.argv[1])
+
+with open(LOG_FILE, 'r') as file:
+    SPRT_OUTPUT = file.readlines()
+
+# finds the indices of the last 2 '-----------------' lines in the output
+# the final results report is in the fenced section between these 2 lines
+def find_last_two_fences():
+    indices = []
+
+    for i, line in enumerate(reversed(SPRT_OUTPUT)):
+        if re.search(r"(--)+", line):
+            indices.append(len(SPRT_OUTPUT) - i)
+            if len(indices) == 2:
+                break
+
+    return sorted(indices)
+
+def get_final_results_report():
+    start, end = find_last_two_fences()
+
+    return SPRT_OUTPUT[start + 1 : end]
+
+RESULTS = get_final_results_report()
+
+# returns the first string that matches the given regex
+def find_first_match(regex, strings):
+    for str in strings:
+        if re.search(regex, str):
+            return str
+
+    return ''
+
+class EmojiType(Enum):
+    NEUTRAL = 1,
+    POSITIVE = 2,
+    NEGATIVE = 3
+
+def get_emoji(type):
+    match type:
+        case EmojiType.NEUTRAL: return '🟰'
+        case EmojiType.POSITIVE: return '✅'
+        case EmojiType.NEGATIVE: return '❌'
+
+def get_elo():
+    elo_line = find_first_match('Elo:', RESULTS)
+
+    first_space_idx = elo_line.find(' ')
+    second_space_idx = elo_line.find(' ', first_space_idx + 1)
+
+    return float(
+        elo_line[first_space_idx:second_space_idx]
+    )
+
+def get_elo_emoji(elo):
+    if abs(elo) <= 1:
+        return get_emoji(EmojiType.NEUTRAL)
+
+    if elo > 0:
+        return get_emoji(EmojiType.POSITIVE)
+
+    return get_emoji(EmojiType.NEGATIVE)
+
+def get_result_breakdown():
+    games_line = find_first_match('Games:', RESULTS)
+
+    second_space_idx = games_line.find(' ',
+                            games_line.find(' ') + 1)
+
+    after_second_space = games_line[second_space_idx+1:]
+
+    # string format is:
+    # Wins: W, Losses: L, Draws: D, Points: 50.5 (50.50 %)
+
+    wins_comma_idx = after_second_space.find(',')
+
+    num_wins = int(
+        after_second_space[after_second_space.find(' ')+1:wins_comma_idx]
+    )
+
+    after_wins = (after_second_space[after_second_space.find(' ', wins_comma_idx + 1):]).lstrip()
+
+    losses_comma_idx = after_wins.find(',')
+
+    num_losses = int(
+        after_wins[after_wins.find(' ')+1:losses_comma_idx]
+    )
+
+    after_losses = (after_wins[after_wins.find(' ', losses_comma_idx + 1):]).lstrip()
+
+    draws_comma_idx = after_losses.find(',')
+
+    num_draws = int(
+        after_losses[after_losses.find(' ')+1:draws_comma_idx]
+    )
+
+    after_draws = (after_losses[after_losses.find(' ', draws_comma_idx+1):]).lstrip()
+
+    pcnt = float(
+        after_draws[after_draws.find('(')+1:after_draws.find('%')]
+    )
+
+    return (
+        num_wins, num_losses, num_draws, pcnt
+    )
+
+def get_wins_emoji(wins):
+    if wins in range(45, 55):
+        return get_emoji(EmojiType.NEUTRAL)
+
+    if wins > 50:
+        return get_emoji(EmojiType.POSITIVE)
+
+    return get_emoji(EmojiType.NEGATIVE)
+
+def get_losses_emoji(losses):
+    if losses in range(45, 55):
+        return get_emoji(EmojiType.NEUTRAL)
+
+    if losses > 50:
+        return get_emoji(EmojiType.NEGATIVE)
+
+    return get_emoji(EmojiType.POSITIVE)
+
+def get_draws_emoji(draws, losses, wins):
+    if draws < losses:
+        return get_emoji(EmojiType.NEGATIVE)
+
+    if draws > wins:
+        return get_emoji(EmojiType.POSITIVE)
+
+    return get_emoji(EmojiType.NEUTRAL)
+
+def get_pcnt_emoji(pcnt):
+    if pcnt in range(0, 45):
+        return get_emoji(EmojiType.NEGATIVE)
+
+    if pcnt in range(45, 55):
+        return get_emoji(EmojiType.NEUTRAL)
+
+    return get_emoji(EmojiType.POSITIVE)
+
+#
+
+def replace_pairs(old_values, new_values, string):
+    text = string
+
+    for old, new in zip(old_values, new_values):
+        text = text.replace(old, new)
+
+    return text
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+
+with open(f'{SCRIPT_DIR}/sprt-results.md', 'r') as file:
+    MD_TEMPLATE_TEXT = file.read()
+
+elo = get_elo()
+
+num_wins, num_losses, num_draws, pcnt = get_result_breakdown()
+
+print(
+    replace_pairs(
+        ['%ELO%', '%ELO_EMOJI%',
+         '%WINS%', '%WINS_EMOJI%',
+         '%LOSSES%', '%LOSSES_EMOJI%',
+         '%DRAWS%', '%DRAWS_EMOJI%',
+         '%PCNT%', '%PCNT_EMOJI%'],
+        [f'{elo}', get_elo_emoji(elo),
+         f'{num_wins}', get_wins_emoji(num_wins),
+         f'{num_losses}', get_losses_emoji(num_losses),
+         f'{num_draws}', get_draws_emoji(num_draws, num_losses, num_wins),
+         f'{pcnt}%', get_pcnt_emoji(pcnt)],
+        MD_TEMPLATE_TEXT
+    )
+)

.github/scripts/sprt-results.md

@@ -0,0 +1,21 @@
+# SPRT results
+
+| ELO   |             |
+| :---: | :---------: |
+| %ELO% | %ELO_EMOJI% |
+
+| Wins   |              |
+| :----: | :----------: |
+| %WINS% | %WINS_EMOJI% |
+
+| Losses   |                |
+| :------: | :------------: |
+| %LOSSES% | %LOSSES_EMOJI% |
+
+| Draws.  |               |
+| :-----: | :-----------: |
+| %DRAWS% | %DRAWS_EMOJI% |
+
+| Percent |              |
+| :-----: | :----------: |
+| %PCNT%  | %PCNT_EMOJI% |

.github/workflows/README.md

@@ -4,13 +4,17 @@
 
 This action runs on every push, and when a PR is opened. It builds & runs tests, submits results to CDash (in the `Experimental` group), and uploads the engine artifacts. Include the string `[skip ci]` in your commit message to prevent this workflow from being triggered.
 
+## `docs.yml`
+
+This action builds the Doxygen documentation and deploys it to GitHub pages. This action is triggered by every push to `main`.
+
 ## `nightly.yml`
 
 Similar to `ci.yml`, except this action runs on a schedule every night, and CDash results are in the `Nightly` group.
 
-## `docs.yml`
+## `sprt.yml`
 
-This action builds the Doxygen documentation and deploys it to GitHub pages. This action is triggered by every push to `main`.
+Runs an [SPRT test](https://www.chessprogramming.org/Sequential_Probability_Ratio_Test) of the given branch against the latest release using the [fastchess tool](https://github.com/Disservin/fastchess). This action runs on every push, and when a PR is opened. Include the string `[skip ci]` in your commit message to prevent this workflow from being triggered.
 
 ## `tag_and_release.yml`
 

.github/workflows/docs.yml

@@ -43,6 +43,7 @@ jobs:
     name: Build docs
     timeout-minutes: 10
     env:
+      CMAKE_BUILD_PARALLEL_LEVEL: 8
       BUILD_DIR: Builds/clang
       DEPLOY_DIR: deploy
 

.github/workflows/sprt.yml

@@ -0,0 +1,104 @@
+# ======================================================================================
+#
+# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓███████▓▒░       ░▒▓███████▓▒░ ░▒▓██████▓▒░▒▓████████▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓███████▓▒░░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓███████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░      ░▒▓███████▓▒░ ░▒▓██████▓▒░  ░▒▓█▓▒░
+#
+# ======================================================================================
+
+name: SPRT
+
+run-name: Run SPRT test
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+concurrency:
+  group: ${{ github.workflow }}.${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+
+  sprt:
+    if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
+    runs-on: ubuntu-latest
+    name: Run SPRT test
+    timeout-minutes: 60
+    env:
+      CMAKE_BUILD_PARALLEL_LEVEL: 8
+      BUILD_DIR: Builds/clang
+      FASTCHESS_PATH: fastchess/fastchess-ubuntu-22.04
+      BASELINE_PATH: baseline/ben_bot-1.3.0-Linux-Clang
+      SPRT_OUTPUT_LOG: ${{ github.workspace }}/logs/sprt/output.log
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v5
+
+    - name: Download fastchess release
+      uses: robinraju/release-downloader@v1.12
+      with:
+        repository: Disservin/fastchess
+        latest: true
+        preRelease: true
+        fileName: '*-ubuntu-*'
+        tarBall: false
+        zipBall: false
+        out-file-path: fastchess
+        extract: true
+
+    - name: Download latest BenBot release
+      uses: robinraju/release-downloader@v1.12
+      with:
+        latest: true
+        fileName: '*-Linux-Clang'
+        tarBall: false
+        zipBall: false
+        out-file-path: baseline
+
+    - name: Add execution permissions
+      run: chmod --preserve-root 777 ${{ env.FASTCHESS_PATH }} ${{ env.BASELINE_PATH }}
+
+    - name: Configure CMake
+      run: cmake --preset clang -D BENBOT_DOCS=OFF -D FASTCHESS_PROGRAM=${{ env.FASTCHESS_PATH }}
+
+    - name: Inject baseline binary
+      run: cmake -D BASELINE_BINARY=${{ env.BASELINE_PATH }} -P ${{ env.BUILD_DIR }}/InjectSPRTBaseline-Release.cmake
+
+    - name: Build
+      run: cmake --build . --target ben_bot --config Release
+      working-directory: ${{ env.BUILD_DIR }}
+
+    - name: Run SPRT test
+      run: |
+        mkdir -p ${{ github.workspace }}/logs/sprt
+        cmake --build . --target sprt --config Release | tee ${{ env.SPRT_OUTPUT_LOG }}
+      working-directory: ${{ env.BUILD_DIR }}
+
+    - name: Upload logs
+      if: always()
+      uses: actions/upload-artifact@v4.6.2
+      with:
+        name: sprt-logs
+        path: logs/sprt
+        if-no-files-found: error
+
+    - name: Report results
+      if: always()
+      run: python3 ReportSPRTResults.py ${{ env.SPRT_OUTPUT_LOG }} >> $GITHUB_STEP_SUMMARY
+      working-directory: .github/scripts

ben-bot/README.md

@@ -26,5 +26,4 @@ The engine supports several non-standard UCI commands. Type `help` for a list of
 . venv/bin/activate
 python3 lichess-bot.py
 ```
-
 This will start the bot and wait for challenges via Lichess.

tests/CMakeLists.txt

@@ -10,7 +10,7 @@
 #
 # ======================================================================================
 
-find_package (Python 3.9 COMPONENTS Interpreter)
+find_package (Python 3.10 COMPONENTS Interpreter)
 
 if (NOT TARGET Python::Interpreter)
     message (VERBOSE "Python interpreter not found, not adding all test cases")