diff --git a/.github/scripts/InstallPythonDeps.cmake b/.github/scripts/InstallPythonDeps.cmake index 479e9108..52bb0d68 100644 --- a/.github/scripts/InstallPythonDeps.cmake +++ b/.github/scripts/InstallPythonDeps.cmake @@ -18,7 +18,7 @@ CMake find module for locating the python executable used to install dependencie cmake_minimum_required (VERSION 3.30.0 FATAL_ERROR) -find_package (Python 3.9 COMPONENTS Interpreter REQUIRED) +find_package (Python 3.10 COMPONENTS Interpreter REQUIRED) # cmake-format: off execute_process ( diff --git a/.github/scripts/ReportSPRTResults.py b/.github/scripts/ReportSPRTResults.py new file mode 100644 index 00000000..65948388 --- /dev/null +++ b/.github/scripts/ReportSPRTResults.py @@ -0,0 +1,193 @@ +# ====================================================================================== +# +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓███████▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░▒▓████████▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░ ░▒▓█▓▒░ +# +# ====================================================================================== + +from pathlib import Path +import sys +import re +from enum import Enum + +LOG_FILE = Path(sys.argv[1]) + +with open(LOG_FILE, 'r') as file: + SPRT_OUTPUT = file.readlines() + +# finds the indices of the last 2 '-----------------' lines in the output +# the final results report is in the fenced section between these 2 lines +def find_last_two_fences(): + indices = [] + + for i, line in enumerate(reversed(SPRT_OUTPUT)): + if re.search(r"(--)+", line): + indices.append(len(SPRT_OUTPUT) - i) + if len(indices) == 2: + break + + return sorted(indices) + +def get_final_results_report(): + start, end = find_last_two_fences() + + return SPRT_OUTPUT[start + 1 : end] + +RESULTS = get_final_results_report() + +# returns the first string that matches the given regex +def find_first_match(regex, strings): + for str in strings: + if re.search(regex, str): + return str + + return '' + +class EmojiType(Enum): + NEUTRAL = 1, + POSITIVE = 2, + NEGATIVE = 3 + +def get_emoji(type): + match type: + case EmojiType.NEUTRAL: return '🟰' + case EmojiType.POSITIVE: return '✅' + case EmojiType.NEGATIVE: return '❌' + +def get_elo(): + elo_line = find_first_match('Elo:', RESULTS) + + first_space_idx = elo_line.find(' ') + second_space_idx = elo_line.find(' ', first_space_idx + 1) + + return float( + elo_line[first_space_idx:second_space_idx] + ) + +def get_elo_emoji(elo): + if abs(elo) <= 1: + return get_emoji(EmojiType.NEUTRAL) + + if elo > 0: + return get_emoji(EmojiType.POSITIVE) + + return get_emoji(EmojiType.NEGATIVE) + +def get_result_breakdown(): + games_line = find_first_match('Games:', RESULTS) + + second_space_idx = games_line.find(' ', + games_line.find(' ') + 1) + + after_second_space = games_line[second_space_idx+1:] + + # string format is: + # Wins: W, Losses: L, Draws: D, Points: 50.5 (50.50 %) + + wins_comma_idx = after_second_space.find(',') + + num_wins = int( + after_second_space[after_second_space.find(' ')+1:wins_comma_idx] + ) + + after_wins = (after_second_space[after_second_space.find(' ', wins_comma_idx + 1):]).lstrip() + + losses_comma_idx = after_wins.find(',') + + num_losses = int( + after_wins[after_wins.find(' ')+1:losses_comma_idx] + ) + + after_losses = (after_wins[after_wins.find(' ', losses_comma_idx + 1):]).lstrip() + + draws_comma_idx = after_losses.find(',') + + num_draws = int( + after_losses[after_losses.find(' ')+1:draws_comma_idx] + ) + + after_draws = (after_losses[after_losses.find(' ', draws_comma_idx+1):]).lstrip() + + pcnt = float( + after_draws[after_draws.find('(')+1:after_draws.find('%')] + ) + + return ( + num_wins, num_losses, num_draws, pcnt + ) + +def get_wins_emoji(wins): + if wins in range(45, 55): + return get_emoji(EmojiType.NEUTRAL) + + if wins > 50: + return get_emoji(EmojiType.POSITIVE) + + return get_emoji(EmojiType.NEGATIVE) + +def get_losses_emoji(losses): + if losses in range(45, 55): + return get_emoji(EmojiType.NEUTRAL) + + if losses > 50: + return get_emoji(EmojiType.NEGATIVE) + + return get_emoji(EmojiType.POSITIVE) + +def get_draws_emoji(draws, losses, wins): + if draws < losses: + return get_emoji(EmojiType.NEGATIVE) + + if draws > wins: + return get_emoji(EmojiType.POSITIVE) + + return get_emoji(EmojiType.NEUTRAL) + +def get_pcnt_emoji(pcnt): + if pcnt in range(0, 45): + return get_emoji(EmojiType.NEGATIVE) + + if pcnt in range(45, 55): + return get_emoji(EmojiType.NEUTRAL) + + return get_emoji(EmojiType.POSITIVE) + +# + +def replace_pairs(old_values, new_values, string): + text = string + + for old, new in zip(old_values, new_values): + text = text.replace(old, new) + + return text + +SCRIPT_DIR = Path(__file__).resolve().parent + +with open(f'{SCRIPT_DIR}/sprt-results.md', 'r') as file: + MD_TEMPLATE_TEXT = file.read() + +elo = get_elo() + +num_wins, num_losses, num_draws, pcnt = get_result_breakdown() + +print( + replace_pairs( + ['%ELO%', '%ELO_EMOJI%', + '%WINS%', '%WINS_EMOJI%', + '%LOSSES%', '%LOSSES_EMOJI%', + '%DRAWS%', '%DRAWS_EMOJI%', + '%PCNT%', '%PCNT_EMOJI%'], + [f'{elo}', get_elo_emoji(elo), + f'{num_wins}', get_wins_emoji(num_wins), + f'{num_losses}', get_losses_emoji(num_losses), + f'{num_draws}', get_draws_emoji(num_draws, num_losses, num_wins), + f'{pcnt}%', get_pcnt_emoji(pcnt)], + MD_TEMPLATE_TEXT + ) +) diff --git a/.github/scripts/sprt-results.md b/.github/scripts/sprt-results.md new file mode 100644 index 00000000..7966f124 --- /dev/null +++ b/.github/scripts/sprt-results.md @@ -0,0 +1,21 @@ +# SPRT results + +| ELO | | +| :---: | :---------: | +| %ELO% | %ELO_EMOJI% | + +| Wins | | +| :----: | :----------: | +| %WINS% | %WINS_EMOJI% | + +| Losses | | +| :------: | :------------: | +| %LOSSES% | %LOSSES_EMOJI% | + +| Draws. | | +| :-----: | :-----------: | +| %DRAWS% | %DRAWS_EMOJI% | + +| Percent | | +| :-----: | :----------: | +| %PCNT% | %PCNT_EMOJI% | diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 5f439639..975c38a6 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -4,13 +4,17 @@ This action runs on every push, and when a PR is opened. It builds & runs tests, submits results to CDash (in the `Experimental` group), and uploads the engine artifacts. Include the string `[skip ci]` in your commit message to prevent this workflow from being triggered. +## `docs.yml` + +This action builds the Doxygen documentation and deploys it to GitHub pages. This action is triggered by every push to `main`. + ## `nightly.yml` Similar to `ci.yml`, except this action runs on a schedule every night, and CDash results are in the `Nightly` group. -## `docs.yml` +## `sprt.yml` -This action builds the Doxygen documentation and deploys it to GitHub pages. This action is triggered by every push to `main`. +Runs an [SPRT test](https://www.chessprogramming.org/Sequential_Probability_Ratio_Test) of the given branch against the latest release using the [fastchess tool](https://github.com/Disservin/fastchess). This action runs on every push, and when a PR is opened. Include the string `[skip ci]` in your commit message to prevent this workflow from being triggered. ## `tag_and_release.yml` diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bc9b84c1..a87fe1d2 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -43,6 +43,7 @@ jobs: name: Build docs timeout-minutes: 10 env: + CMAKE_BUILD_PARALLEL_LEVEL: 8 BUILD_DIR: Builds/clang DEPLOY_DIR: deploy diff --git a/.github/workflows/sprt.yml b/.github/workflows/sprt.yml new file mode 100644 index 00000000..bddfd1eb --- /dev/null +++ b/.github/workflows/sprt.yml @@ -0,0 +1,104 @@ +# ====================================================================================== +# +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓███████▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░▒▓████████▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░ ░▒▓█▓▒░ +# +# ====================================================================================== + +name: SPRT + +run-name: Run SPRT test + +on: + workflow_dispatch: + push: + branches: [main] + pull_request: + types: [opened, reopened, synchronize] + +concurrency: + group: ${{ github.workflow }}.${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +permissions: + contents: read + +jobs: + + sprt: + if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }} + runs-on: ubuntu-latest + name: Run SPRT test + timeout-minutes: 60 + env: + CMAKE_BUILD_PARALLEL_LEVEL: 8 + BUILD_DIR: Builds/clang + FASTCHESS_PATH: fastchess/fastchess-ubuntu-22.04 + BASELINE_PATH: baseline/ben_bot-1.3.0-Linux-Clang + SPRT_OUTPUT_LOG: ${{ github.workspace }}/logs/sprt/output.log + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Download fastchess release + uses: robinraju/release-downloader@v1.12 + with: + repository: Disservin/fastchess + latest: true + preRelease: true + fileName: '*-ubuntu-*' + tarBall: false + zipBall: false + out-file-path: fastchess + extract: true + + - name: Download latest BenBot release + uses: robinraju/release-downloader@v1.12 + with: + latest: true + fileName: '*-Linux-Clang' + tarBall: false + zipBall: false + out-file-path: baseline + + - name: Add execution permissions + run: chmod --preserve-root 777 ${{ env.FASTCHESS_PATH }} ${{ env.BASELINE_PATH }} + + - name: Configure CMake + run: cmake --preset clang -D BENBOT_DOCS=OFF -D FASTCHESS_PROGRAM=${{ env.FASTCHESS_PATH }} + + - name: Inject baseline binary + run: cmake -D BASELINE_BINARY=${{ env.BASELINE_PATH }} -P ${{ env.BUILD_DIR }}/InjectSPRTBaseline-Release.cmake + + - name: Build + run: cmake --build . --target ben_bot --config Release + working-directory: ${{ env.BUILD_DIR }} + + - name: Run SPRT test + run: | + mkdir -p ${{ github.workspace }}/logs/sprt + cmake --build . --target sprt --config Release | tee ${{ env.SPRT_OUTPUT_LOG }} + working-directory: ${{ env.BUILD_DIR }} + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4.6.2 + with: + name: sprt-logs + path: logs/sprt + if-no-files-found: error + + - name: Report results + if: always() + run: python3 ReportSPRTResults.py ${{ env.SPRT_OUTPUT_LOG }} >> $GITHUB_STEP_SUMMARY + working-directory: .github/scripts diff --git a/ben-bot/README.md b/ben-bot/README.md index 9a6d8286..5c3f00fd 100644 --- a/ben-bot/README.md +++ b/ben-bot/README.md @@ -26,5 +26,4 @@ The engine supports several non-standard UCI commands. Type `help` for a list of . venv/bin/activate python3 lichess-bot.py ``` - This will start the bot and wait for challenges via Lichess. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c6646411..95bbbf0b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -10,7 +10,7 @@ # # ====================================================================================== -find_package (Python 3.9 COMPONENTS Interpreter) +find_package (Python 3.10 COMPONENTS Interpreter) if (NOT TARGET Python::Interpreter) message (VERBOSE "Python interpreter not found, not adding all test cases") diff --git a/tests/fastchess/CMakeLists.txt b/tests/fastchess/CMakeLists.txt index 1a0f34dd..ec7d59a1 100644 --- a/tests/fastchess/CMakeLists.txt +++ b/tests/fastchess/CMakeLists.txt @@ -33,20 +33,49 @@ add_custom_target ( add_dependencies (sprt_set_baseline ben_bot) +# used by the sprt ci workflow to inject the binary from the latest release as the baseline +set (injection_script "${CMAKE_CURRENT_LIST_DIR}/InjectSPRTBaseline.cmake") + +file (READ "${injection_script}" script_content) +list (APPEND CMAKE_CONFIGURE_DEPENDS "${injection_script}") + +string (CONFIGURE "${script_content}" script_content @ONLY) + +file (GENERATE OUTPUT "${BenBot_BINARY_DIR}/InjectSPRTBaseline-$.cmake" + CONTENT "${script_content}" TARGET ben_bot NEWLINE_STYLE UNIX +) + +# + cmake_host_system_information (RESULT num_cores QUERY NUMBER_OF_PHYSICAL_CORES) set (openings_file "${BenBot_SOURCE_DIR}/ben-bot/resources/res/book.pgn") +set (logs_dir "${BenBot_SOURCE_DIR}/logs/sprt") + +add_custom_command ( + OUTPUT "${logs_dir}/stamp" + COMMAND "${CMAKE_COMMAND}" -E make_directory "${logs_dir}" + COMMAND "${CMAKE_COMMAND}" -E touch "${logs_dir}/stamp" + COMMENT "Creating SPRT logs directory..." + VERBATIM USES_TERMINAL +) + # cmake-format: off add_custom_target ( sprt COMMAND "${FASTCHESS_PROGRAM}" -engine "cmd=$" name=Refactor -engine "cmd=${baseline_binary}" name=Baseline - -each tc=8+0.08 -rounds 50 -repeat -concurrency "${num_cores}" -recover + -each tc=8+0.08 -rounds 50 -repeat -concurrency "${num_cores}" + -recover -show-latency -openings "file=${openings_file}" format=pgn -sprt elo0=0 elo1=10 alpha=0.05 beta=0.05 + -pgnout "file=${logs_dir}/games.pgn" nodes=true nps=true timeleft=true latency=true + -epdout "file=${logs_dir}/final-positions.epd" + -log "file=${logs_dir}/sprt.log" engine=true WORKING_DIRECTORY "$" + DEPENDS "${logs_dir}/stamp" COMMENT "Running SPRT test..." VERBATIM USES_TERMINAL ) diff --git a/tests/fastchess/InjectSPRTBaseline.cmake b/tests/fastchess/InjectSPRTBaseline.cmake new file mode 100644 index 00000000..0ec05f75 --- /dev/null +++ b/tests/fastchess/InjectSPRTBaseline.cmake @@ -0,0 +1,23 @@ +# ====================================================================================== +# +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓███████▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░▒▓████████▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░ ░▒▓█▓▒░ +# +# ====================================================================================== + +cmake_minimum_required (VERSION 3.30.0 FATAL_ERROR) + +if (NOT DEFINED BASELINE_BINARY) + message (FATAL_ERROR "BASELINE_BINARY must be defined!") +endif () + +if (NOT EXISTS "${BASELINE_BINARY}") + message (FATAL_ERROR "BASELINE_BINARY does not exist at path '${BASELINE_BINARY}'!") +endif () + +file (COPY_FILE "${BASELINE_BINARY}" "@baseline_binary@" ONLY_IF_DIFFERENT INPUT_MAY_BE_RECENT)