test: registering each perft depth as an individual test case

Author: benthevining

tests/perft/CMakeLists.txt

@@ -23,9 +23,29 @@ file (GLOB testcase_files LIST_DIRECTORIES false CONFIGURE_DEPENDS
 foreach (data_file IN LISTS testcase_files)
     cmake_path (GET data_file STEM filename)
 
-    set (test_name "ben_bot.perft.${filename}")
+    file (READ "${data_file}" file_content)
 
-    add_test (NAME "${test_name}" COMMAND Python::Interpreter "${CMAKE_CURRENT_LIST_DIR}/perft.py"
-                                          "--test=${data_file}" "--engine=$<TARGET_FILE:ben_bot>"
-    )
+    string (JSON num_depths LENGTH "${file_content}" depths)
+
+    # because foreach(RANGE) includes the boundary value
+    math (EXPR num_depths "${num_depths} - 1")
+
+    foreach (depth_idx RANGE 0 "${num_depths}")
+        # cmake-format: off
+        string (
+            JSON
+            depth GET
+            "${file_content}"
+            depths "${depth_idx}" depth
+        )
+        # cmake-format: on
+
+        add_test (
+            NAME "ben_bot.perft.${filename}.${depth}"
+            COMMAND Python::Interpreter "${CMAKE_CURRENT_LIST_DIR}/perft.py" "--test=${data_file}"
+                    "--engine=$<TARGET_FILE:ben_bot>" "--depth=${depth}"
+        )
+    endforeach ()
+
+    set_property (DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${data_file}")
 endforeach ()

tests/perft/perft.py

@@ -13,11 +13,20 @@
 import json
 import subprocess
 from pathlib import Path
-from typing import Tuple
+from typing import Tuple, Any
 
 #
 
 
+def find_correct_result(fileContent: Any, depth: int) -> dict[str, int]:
+    for obj in fileContent["depths"]:
+        if obj["depth"] == depth:
+            return obj["results"]
+
+    print(f"ERROR! Could not find correct result for depth {depth}")
+    exit(1)
+
+
 def check_result(expected: dict[str, int], actual: dict[str, int]) -> bool:
     for key in (
         "totalNodes",
@@ -36,7 +45,7 @@ def check_result(expected: dict[str, int], actual: dict[str, int]) -> bool:
     return True
 
 
-def parse_args() -> Tuple[Path, Path]:
+def parse_args() -> Tuple[Path, Path, int]:
     parser = argparse.ArgumentParser(
         prog="RunPerft",
         description="Run BenBot perft tests",
@@ -49,10 +58,11 @@ def parse_args() -> Tuple[Path, Path]:
     parser.add_argument(
         "-e", "--engine", required=True, help="Path to engine executable"
     )
+    parser.add_argument("-d", "--depth", required=True, help="Perft depth")
 
     parsed = parser.parse_args()
 
-    return Path(parsed.test).resolve(), Path(parsed.engine).resolve()
+    return Path(parsed.test).resolve(), Path(parsed.engine).resolve(), int(parsed.depth)
 
 
 #
@@ -100,34 +110,29 @@ def quit(self):
 
 #
 
-TESTCASE_FILE, ENGINE_PATH = parse_args()
+TESTCASE_FILE, ENGINE_PATH, DEPTH = parse_args()
 
 with open(TESTCASE_FILE) as file:
-    CORRECT_DATA = json.load(file)
+    FILE_DATA = json.load(file)
+
+startingFEN = FILE_DATA["position"]
+
+print(f"FEN: {startingFEN}", flush=True)
 
-startingFEN = CORRECT_DATA["position"]
+correctResult = find_correct_result(FILE_DATA, DEPTH)
 
-print(f"Running tests for position {startingFEN}", flush=True)
+print(f"Running perft depth {DEPTH}...", flush=True)
 
 engine = Engine(engine_path=ENGINE_PATH, pos_fen=startingFEN)
 
 num_failed = 0
 num_passed = 0
 
-for depthObj in CORRECT_DATA["depths"]:
-    depth = depthObj["depth"]
-    print(f"Running perft depth {depth}...", flush=True)
-
-    result = engine.run_perft(depth)
-
-    if check_result(depthObj["results"], result):
-        num_passed += 1
-    else:
-        num_failed += 1
-
-engine.quit()
-
-print(f"{num_passed} depths passed")
-print(f"{num_failed} depths failed")
+result = engine.run_perft(DEPTH)
 
-exit(num_failed)
+if check_result(correctResult, result):
+    print("Succeeded :-)")
+    exit(0)
+else:
+    print("Failed :-(")
+    exit(1)