Skip to content

Commit f09a4b9

Browse files
committed
feat: Add 'build' to development dependencies for improved package management
1 parent fe593e9 commit f09a4b9

File tree

5 files changed

+214
-79
lines changed

5 files changed

+214
-79
lines changed

benchmark_results.csv

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,21 @@
1-
timestamp,snapshot_name,snapshot_branch,snapshot_commit,snapshot_dirty,warmup_rounds,benchmark_rounds,benchmark_iterations,engine_depth,num_games,positions_count,lm_snapshot_median_ms,lm_snapshot_mean_ms,lm_snapshot_min_ms,lm_snapshot_max_ms,lm_snapshot_stdev_ms,lm_current_median_ms,lm_current_mean_ms,lm_current_min_ms,lm_current_max_ms,lm_current_stdev_ms,lm_change_pct,match_snapshot_wins,match_current_wins,match_draws,match_snapshot_avg_nodes,match_current_avg_nodes,match_snapshot_avg_time_ms,match_current_avg_time_ms,match_time_change_pct,match_snapshot_total_nodes,match_current_total_nodes,hw_platform,hw_platform_release,hw_architecture,hw_cpu_brand,hw_cpu_cores,hw_ram_gb,hw_python_version
2-
2025-12-31T14:18:54.255407,snapshot_20251231_132737,main,75919c5,True,5,10,10,2,20,73,7.65,7.798,6.86,9.502,0.734,3.778,3.948,3.412,5.677,0.714,-50.62,10,10,0,16.3,16.3,4.84,2.35,-51.4,16340,16340,Windows,11,AMD64,12th Gen Intel(R) Core(TM) i5-1245U,12,31.7,3.12.10
3-
2025-12-31T14:31:57.164421,snapshot_20251231_142534,main,a3f4c6b,False,5,10,10,2,20,73,4.062,4.175,3.355,6.345,0.879,3.876,4.132,3.364,6.804,0.998,-4.57,10,10,0,16.2,16.3,2.03,1.26,-38.11,20030,20220,Windows,11,AMD64,12th Gen Intel(R) Core(TM) i5-1245U,12,31.7,3.12.10
4-
2025-12-31T14:36:11.096632,snapshot_20251231_142534,main,a3f4c6b,False,5,10,10,3,40,73,3.469,3.975,3.149,7.303,1.241,3.356,3.548,3.151,4.244,0.395,-3.25,20,20,0,150.6,138.4,15.02,7.58,-49.56,241020,221420,Windows,11,AMD64,12th Gen Intel(R) Core(TM) i5-1245U,12,31.7,3.12.10
5-
2025-12-31T14:42:50.445557,snapshot_20251231_143808,main,86da27b,True,5,10,10,3,40,73,3.346,3.631,3.021,5.345,0.689,3.363,3.434,3.063,4.059,0.311,0.51,40,0,0,76.9,60.2,4.18,6.03,44.27,86180,68620,Windows,11,AMD64,12th Gen Intel(R) Core(TM) i5-1245U,12,31.7,3.12.10
6-
2025-12-31T14:46:11.570275,snapshot_20251231_143808,main,86da27b,True,5,10,10,3,40,73,4.608,5.273,3.567,9.344,1.851,4.318,4.648,3.785,6.264,0.915,-6.3,40,0,0,76.9,60.2,12.57,17.93,42.7,86180,68620,Windows,11,AMD64,12th Gen Intel(R) Core(TM) i5-1245U,12,31.7,3.12.10
7-
2025-12-31T14:51:37.432914,snapshot_20251231_143808,main,86da27b,True,5,10,10,3,40,73,3.77,3.822,3.11,4.487,0.488,3.516,3.737,3.139,5.368,0.653,-6.74,0,20,20,122.0,125.2,7.53,6.99,-7.28,392940,403240,Windows,11,AMD64,12th Gen Intel(R) Core(TM) i5-1245U,12,31.7,3.12.10
1+
name,fen
2+
Introduction,B:W28,31,32,34,35,36,37,38,39,40,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,11,12,13,15,16,17,18,19,20,23
3+
Symmetrical Variation,W:W24,27,29,32,34,37,38,39,40,41,42,43,44,47,48,49,50:B1,2,3,4,7,8,9,11,12,13,15,16,17,18,20,23,25
4+
Chefneux Opening,W:W28,31,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,17,18,19,20,23
5+
Chogoliev Opening,W:W31,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,10,12,13,14,15,16,18,19,20,22
6+
Schwarzman Manoeuvring Opening,W:W31,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,18,19,20,21
7+
Eagle Opening,W:W28,31,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,19,20,22
8+
Hedgehog Opening,W:W28,31,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,21
9+
Black Panther Opening,W:W28,31,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,17,18,19,20,21
10+
Russian Bear Opening,W:W28,31,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,25
11+
Cow Opening,W:W28,31,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,24
12+
Diamond Opening,B:W29,31,32,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20
13+
Sijbrands Opening,W:W27,28,32,33,34,35,36,37,38,39,40,41,42,43 ,45,46,47,48,49,50:B1,2,3,4,5,6,8,9,10,11,12,13,14,15,16,17,18,19,23,24
14+
Fork Lock Opening,B:W29,31,32,33,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20
15+
French Opening,B:W30,31,32,33,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20
16+
Polish Opening,B:W27,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20
17+
Edge Opening,B:W26,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20
18+
Wild Horse Opening,B:W30,31,32,33,34,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50:B1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20
19+
Keller Opening,B:W24,26,29,33,34,36,37,38,39,40,41,42,43,44,45,46,47,48,49:B2,3,4,5,6,7,8,9,11,12,13,14,15,17,18,20,21,22,23
20+
Woldouby,W:W25,27,28,30,32,33,34,35,37,38:B12,13,14,16,18,19,21,23,24,26
21+
2025-12-31T17:18:57.958257,snapshot_20251231_155231,main,fe593e9,True,5,10,10,4,64,32,73,3.148,3.228,2.934,3.666,0.27,3.025,3.103,2.862,3.698,0.248,-3.89,31,31,2,140.0,140.0,6.86,6.84,-0.29,174594,174594,Windows,11,AMD64,12th Gen Intel(R) Core(TM) i5-12400F,12,31.8,3.12.10

docs/source/benchmarking.rst

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,55 @@ Results
6363

6464
Benchmark results are automatically appended to ``benchmark_results.csv`` in the project root.
6565

66+
Optimization Workflow
67+
---------------------
68+
69+
Recommended workflow for improving engine performance:
70+
71+
1. **Profile**
72+
73+
Run profiling to identify bottlenecks:
74+
75+
.. code-block:: bash
76+
77+
# For general engine profiling
78+
python tools/profile_engine_detailed.py
79+
80+
# For legal moves generation specifically
81+
python tools/profile_legal_moves.py
82+
83+
2. **Improve**
84+
85+
Make code changes to address identified bottlenecks.
86+
87+
3. **Verify Profile**
88+
89+
Run the profiling script again to verify local improvements.
90+
91+
4. **Test**
92+
93+
Ensure no regressions in functionality:
94+
95+
.. code-block:: bash
96+
97+
pytest .
98+
99+
5. **Benchmark**
100+
101+
Compare against the baseline snapshot to verify performance gains:
102+
103+
.. code-block:: bash
104+
105+
python tools/compare_versions.py
106+
107+
6. **Snapshot**
108+
109+
If satisfied with the results, create a new snapshot to serve as the next baseline:
110+
111+
.. code-block:: bash
112+
113+
python tools/create_snapshot.py
114+
66115
Profiling
67116
---------
68117

pyproject.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ dev = [
4949
"pytest",
5050
"mypy",
5151
"sphinx",
52+
"build",
5253
"rich",
5354
"pytest-benchmark",
5455
]

tools/compare_versions.py

Lines changed: 110 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
import subprocess
1414
import sys
1515
import tempfile
16-
import venv
1716
from datetime import datetime
1817
from pathlib import Path
1918

@@ -26,18 +25,32 @@
2625
WARMUP_ROUNDS = 5
2726
BENCHMARK_ROUNDS = 10
2827
BENCHMARK_ITERATIONS = 10 # Number of times to run the legal moves benchmark
29-
ENGINE_DEPTH = 3
30-
NUM_GAMES = 40
28+
ENGINE_DEPTH = 4
3129

3230
# === Paths ===
3331
PROJECT_ROOT = Path(__file__).parent.parent
3432
POSITIONS_FILE = PROJECT_ROOT / "test" / "games" / "standard" / "random_positions.json"
33+
OPENINGS_FILE = PROJECT_ROOT / "tools" / "openings.csv"
3534
WORKERS_DIR = PROJECT_ROOT / "tools" / "workers"
3635
RESULTS_CSV = PROJECT_ROOT / "benchmark_results.csv"
3736

3837
console = Console()
3938

4039

40+
def load_openings() -> list[dict]:
41+
"""Load openings from CSV file."""
42+
openings = []
43+
with open(OPENINGS_FILE, "r", encoding="utf-8") as f:
44+
reader = csv.DictReader(f)
45+
for row in reader:
46+
openings.append({
47+
"name": row["name"],
48+
"moves": row["moves"],
49+
"fen": row["fen"] if row["fen"].strip() else None,
50+
})
51+
return openings
52+
53+
4154
def get_hardware_info() -> dict:
4255
"""Collect hardware and system information."""
4356
info = {
@@ -81,6 +94,8 @@ def append_results_to_csv(
8194
lm2_medians: list,
8295
match_stats: dict,
8396
positions_count: int,
97+
num_games: int,
98+
num_openings: int,
8499
):
85100
"""Append benchmark results to CSV file."""
86101
from statistics import median, mean, stdev
@@ -111,7 +126,8 @@ def append_results_to_csv(
111126
"benchmark_rounds": BENCHMARK_ROUNDS,
112127
"benchmark_iterations": BENCHMARK_ITERATIONS,
113128
"engine_depth": ENGINE_DEPTH,
114-
"num_games": NUM_GAMES,
129+
"num_games": num_games,
130+
"num_openings": num_openings,
115131
"positions_count": positions_count,
116132

117133
# Legal moves - snapshot
@@ -165,23 +181,18 @@ def append_results_to_csv(
165181

166182
def create_venv(path: Path) -> Path:
167183
"""Create virtualenv and return python executable path."""
168-
venv.create(path, with_pip=True, clear=True)
184+
subprocess.run(["uv", "venv", str(path)], check=True, capture_output=True)
169185
if sys.platform == "win32":
170186
python = path / "Scripts" / "python.exe"
171187
else:
172188
python = path / "bin" / "python"
173-
subprocess.run(
174-
[str(python), "-m", "pip", "install", "--upgrade", "pip", "-q"],
175-
check=True,
176-
capture_output=True,
177-
)
178189
return python
179190

180191

181192
def install_package(python: Path, source: Path):
182193
"""Install py-draughts from wheel or source."""
183194
subprocess.run(
184-
[str(python), "-m", "pip", "install", str(source), "-q"],
195+
["uv", "pip", "install", "--python", str(python), str(source), "-q"],
185196
check=True,
186197
capture_output=True,
187198
)
@@ -265,8 +276,11 @@ def get_engine_move(python: Path, fen: str | None, depth: int) -> dict:
265276
return json.loads(result.stdout)
266277

267278

268-
def play_match(python1: Path, python2: Path, num_games: int, depth: int) -> dict:
269-
"""Play engine vs engine match using persistent workers."""
279+
def play_match(python1: Path, python2: Path, openings: list[dict], depth: int) -> dict:
280+
"""Play engine vs engine match using persistent workers.
281+
282+
Each opening is played twice: once with v1 as white, once with v2 as white.
283+
"""
270284
stats = {
271285
"v1_wins": 0,
272286
"v2_wins": 0,
@@ -279,62 +293,80 @@ def play_match(python1: Path, python2: Path, num_games: int, depth: int) -> dict
279293
"v2_moves": 0,
280294
}
281295

296+
num_games = len(openings) * 2 # Each opening played twice (swap colors)
297+
game_num = 0
298+
282299
# Start persistent workers (avoids ~500ms startup per move!)
283300
with EngineWorker(python1) as worker1, EngineWorker(python2) as worker2:
284301
with console.status("[bold green]Playing match...") as status:
285-
for game_num in range(num_games):
286-
status.update(f"[bold green]Game {game_num + 1}/{num_games}...")
287-
288-
# Alternate who plays white
289-
if game_num % 2 == 0:
290-
white_worker, black_worker = worker1, worker2
291-
white_ver = "v1"
292-
else:
293-
white_worker, black_worker = worker2, worker1
294-
white_ver = "v2"
295-
296-
fen = None
297-
move_count = 0
298-
result = {}
299-
300-
while move_count < 200:
301-
is_white_turn = move_count % 2 == 0
302-
current_worker = white_worker if is_white_turn else black_worker
303-
current_ver = white_ver if is_white_turn else ("v2" if white_ver == "v1" else "v1")
304-
305-
result = current_worker.get_move(fen, depth)
306-
307-
if result.get("error") or result.get("game_over") or not result.get("move"):
308-
break
309-
310-
# Track stats
311-
if current_ver == "v1":
312-
stats["v1_nodes"] += result.get("nodes", 0)
313-
stats["v1_time_ms"] += result.get("time_ms", 0)
314-
stats["v1_moves"] += 1
302+
for opening in openings:
303+
opening_name = opening["name"]
304+
opening_fen = opening["fen"]
305+
306+
# Play each opening twice: v1 as white, then v2 as white
307+
for swap in range(2):
308+
game_num += 1
309+
color_info = "snapshot=W" if swap == 0 else "current=W"
310+
status.update(f"[bold green]Game {game_num}/{num_games}: {opening_name} ({color_info})...")
311+
312+
# Determine who plays white
313+
if swap == 0:
314+
white_worker, black_worker = worker1, worker2
315+
white_ver = "v1"
316+
else:
317+
white_worker, black_worker = worker2, worker1
318+
white_ver = "v2"
319+
320+
# Start from opening position
321+
fen = opening_fen
322+
move_count = 0
323+
result = {}
324+
325+
# Determine whose turn it is from FEN
326+
# FEN starts with 'W:' or 'B:' to indicate turn
327+
if fen and fen.startswith("B:"):
328+
is_white_turn = False
329+
else:
330+
is_white_turn = True # Default to white's turn
331+
332+
while move_count < 200:
333+
current_worker = white_worker if is_white_turn else black_worker
334+
current_ver = white_ver if is_white_turn else ("v2" if white_ver == "v1" else "v1")
335+
336+
result = current_worker.get_move(fen, depth)
337+
338+
if result.get("error") or result.get("game_over") or not result.get("move"):
339+
break
340+
341+
# Track stats
342+
if current_ver == "v1":
343+
stats["v1_nodes"] += result.get("nodes", 0)
344+
stats["v1_time_ms"] += result.get("time_ms", 0)
345+
stats["v1_moves"] += 1
346+
else:
347+
stats["v2_nodes"] += result.get("nodes", 0)
348+
stats["v2_time_ms"] += result.get("time_ms", 0)
349+
stats["v2_moves"] += 1
350+
351+
fen = result["fen"]
352+
move_count += 1
353+
is_white_turn = not is_white_turn
354+
355+
# Determine winner
356+
game_result = result.get("result", "1/2-1/2")
357+
if game_result == "1-0":
358+
winner = white_ver
359+
elif game_result == "0-1":
360+
winner = "v2" if white_ver == "v1" else "v1"
315361
else:
316-
stats["v2_nodes"] += result.get("nodes", 0)
317-
stats["v2_time_ms"] += result.get("time_ms", 0)
318-
stats["v2_moves"] += 1
319-
320-
fen = result["fen"]
321-
move_count += 1
322-
323-
# Determine winner
324-
game_result = result.get("result", "1/2-1/2")
325-
if game_result == "1-0":
326-
winner = white_ver
327-
elif game_result == "0-1":
328-
winner = "v2" if white_ver == "v1" else "v1"
329-
else:
330-
winner = "draw"
331-
332-
if winner == "v1":
333-
stats["v1_wins"] += 1
334-
elif winner == "v2":
335-
stats["v2_wins"] += 1
336-
else:
337-
stats["draws"] += 1
362+
winner = "draw"
363+
364+
if winner == "v1":
365+
stats["v1_wins"] += 1
366+
elif winner == "v2":
367+
stats["v2_wins"] += 1
368+
else:
369+
stats["draws"] += 1
338370

339371
return stats
340372

@@ -485,21 +517,25 @@ def main():
485517
console.print()
486518
console.print(lm_table)
487519

520+
# Load openings
521+
openings = load_openings()
522+
num_games = len(openings) * 2 # Each opening played as both colors
523+
488524
# Engine match
489-
console.print(f"\n[bold]Engine match ({NUM_GAMES} games, depth={ENGINE_DEPTH})...[/]")
490-
match = play_match(py1, py2, NUM_GAMES, ENGINE_DEPTH)
525+
console.print(f"\n[bold]Engine match ({num_games} games from {len(openings)} openings, depth={ENGINE_DEPTH})...[/]")
526+
match = play_match(py1, py2, openings, ENGINE_DEPTH)
491527

492528
# Calculate averages
493529
v1_avg_nodes = match["v1_nodes"] / match["v1_moves"] if match["v1_moves"] else 0
494530
v2_avg_nodes = match["v2_nodes"] / match["v2_moves"] if match["v2_moves"] else 0
495531
v1_avg_time = match["v1_time_ms"] / match["v1_moves"] if match["v1_moves"] else 0
496532
v2_avg_time = match["v2_time_ms"] / match["v2_moves"] if match["v2_moves"] else 0
497533

498-
v1_pct = (match["v1_wins"] / NUM_GAMES) * 100
499-
v2_pct = (match["v2_wins"] / NUM_GAMES) * 100
500-
draw_pct = (match["draws"] / NUM_GAMES) * 100
534+
v1_pct = (match["v1_wins"] / num_games) * 100
535+
v2_pct = (match["v2_wins"] / num_games) * 100
536+
draw_pct = (match["draws"] / num_games) * 100
501537

502-
match_table = Table(title=f"Match Results ({NUM_GAMES} games)", box=box.ROUNDED)
538+
match_table = Table(title=f"Match Results ({num_games} games from {len(openings)} openings)", box=box.ROUNDED)
503539
match_table.add_column("Metric", style="cyan")
504540
match_table.add_column("Snapshot", justify="right")
505541
match_table.add_column("Current", justify="right")
@@ -545,7 +581,7 @@ def main():
545581
else:
546582
console.print("[yellow]⚠ Performance regression detected[/]")
547583

548-
if match["draws"] == NUM_GAMES:
584+
if match["draws"] == num_games:
549585
console.print("[dim]Note: 100% draws expected with identical code (deterministic engine)[/]")
550586

551587
# Append results to CSV
@@ -556,6 +592,8 @@ def main():
556592
lm2_medians=lm2_medians,
557593
match_stats=match,
558594
positions_count=lm1_results[0]["positions_count"],
595+
num_games=num_games,
596+
num_openings=len(openings),
559597
)
560598

561599

0 commit comments

Comments
 (0)