Skip to content

Commit 8ae9904

Browse files
DisservinDisservin
authored
feat: support round-robin with multiple engines (#720)
Preliminary work for supporting round-robins with multiple engines, output looks like the following: ``` Rank Name Elo +/- nElo +/- Games Score Draw Ptnml(0-2) 1 engine2 34.86 67.31 65.66 124.33 30 55.0% 60.0% [0, 2, 9, 3, 1] 2 engine1 -0.00 56.08 0.00 124.33 30 50.0% 80.0% [1, 0, 12, 2, 0] 3 engine3 -11.59 39.08 -37.04 124.33 30 48.3% 80.0% [0, 2, 12, 1, 0] 4 engine4 -23.20 44.30 -65.66 124.33 30 46.7% 73.3% [0, 3, 11, 1, 0] ```
1 parent 55f271f commit 8ae9904

File tree

9 files changed

+201
-104
lines changed

9 files changed

+201
-104
lines changed

‎app/src/config/sanitize.cpp‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -110,10 +110,6 @@ void sanitize(std::vector<EngineConfiguration>& configs) {
110110
throw std::runtime_error("Error: Need at least two engines to start!");
111111
}
112112

113-
if (configs.size() > 2) {
114-
throw std::runtime_error("Error: Exceeded -engine limit! Must be 2!");
115-
}
116-
117113
for (std::size_t i = 0; i < configs.size(); i++) {
118114
#ifdef _WIN64
119115
// add .exe if . is not present

‎app/src/matchmaking/output/output.hpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,10 @@ class IOutput {
2626

2727
// Interval output. Get's displayed every n `ratinginterval`.
2828
virtual void printInterval(const SPRT& sprt, const Stats& stats, const std::string& first,
29-
const std::string& second, const engines& engines, const std::string& book) {
29+
const std::string& second, const engines& engines, const std::string& book,
30+
ScoreBoard& scoreboard) {
3031
std::cout << "--------------------------------------------------\n";
31-
printElo(stats, first, second, engines, book);
32+
printElo(stats, first, second, engines, book, scoreboard);
3233
printSprt(sprt, stats);
3334
std::cout << "--------------------------------------------------\n";
3435
};
@@ -38,7 +39,7 @@ class IOutput {
3839

3940
// Print current H2H elo stats.
4041
virtual std::string printElo(const Stats& stats, const std::string& first, const std::string& second,
41-
const engines& engines, const std::string& book) = 0;
42+
const engines& engines, const std::string& book, ScoreBoard& scoreboard) = 0;
4243

4344
// Print current SPRT stats.
4445
virtual std::string printSprt(const SPRT& sprt, const Stats& stats) = 0;

‎app/src/matchmaking/output/output_cutechess.hpp‎

Lines changed: 44 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
#pragma once
22

3+
#include <algorithm>
4+
#include <cstdint>
5+
#include <iostream>
6+
#include <string>
7+
#include <tuple>
8+
#include <vector>
9+
310
#include <elo/elo_wdl.hpp>
411
#include <matchmaking/output/output.hpp>
512
#include <util/logger/logger.hpp>
@@ -9,10 +16,10 @@ namespace fastchess {
916
class Cutechess : public IOutput {
1017
public:
1118
void printInterval(const SPRT& sprt, const Stats& stats, const std::string& first, const std::string& second,
12-
const engines& engines, const std::string& book) override {
13-
std::cout //
14-
<< printElo(stats, first, second, engines, book) //
15-
<< printSprt(sprt, stats) //
19+
const engines& engines, const std::string& book, ScoreBoard& scoreboard) override {
20+
std::cout //
21+
<< printElo(stats, first, second, engines, book, scoreboard) //
22+
<< printSprt(sprt, stats) //
1623
<< std::flush;
1724
};
1825

@@ -24,12 +31,40 @@ class Cutechess : public IOutput {
2431
<< std::flush;
2532
}
2633

27-
std::string printElo(const Stats& stats, const std::string&, const std::string&, const engines&,
28-
const std::string&) override {
29-
const elo::EloWDL elo(stats);
34+
std::string printElo(const Stats& stats, const std::string&, const std::string&, const engines&, const std::string&,
35+
ScoreBoard& scoreboard) override {
36+
const auto& ecs = config::EngineConfigs.get();
37+
38+
if (ecs.size() == 2) {
39+
const elo::EloWDL elo(stats);
40+
41+
return fmt::format("Elo difference: {}, LOS: {}, DrawRatio: {:.2f}%\n", elo.getElo(), elo.los(),
42+
stats.drawRatio());
43+
}
44+
45+
std::vector<std::tuple<const EngineConfiguration*, elo::EloWDL, Stats>> elos;
46+
47+
for (auto& e : ecs) {
48+
const auto stats = scoreboard.getAllStats(e.name);
49+
elos.emplace_back(&e, elo::EloWDL(stats), stats);
50+
}
51+
52+
// sort by elo diff
53+
54+
std::sort(elos.begin(), elos.end(),
55+
[](const auto& a, const auto& b) { return std::get<1>(a).diff() > std::get<1>(b).diff(); });
56+
57+
int rank = 0;
58+
std::string out = fmt::format("{:<4} {:<25} {:>10} {:>10} {:>10} {:>10} {:>10} {:>10} {:>10}\n", "Rank", "Name",
59+
"Elo", "+/-", "nElo", "+/-", "Games", "Score", "Draw");
60+
61+
for (const auto& [ec, elo, stats] : elos) {
62+
out += fmt::format("{:>4} {:<25} {:>10.2f} {:>10.2f} {:>10.2f} {:>10.2f} {:>10} {:>9.1f}% {:>9.1f}%\n",
63+
++rank, ec->name, elo.diff(), elo.error(), elo.nEloDiff(), elo.nEloError(), stats.sum(),
64+
stats.pointsRatio(), stats.drawRatio());
65+
}
3066

31-
return fmt::format("Elo difference: {}, LOS: {}, DrawRatio: {:.2f}%\n", elo.getElo(), elo.los(),
32-
stats.drawRatio());
67+
return out;
3368
}
3469

3570
std::string printSprt(const SPRT& sprt, const Stats& stats) override {

‎app/src/matchmaking/output/output_fastchess.hpp‎

Lines changed: 56 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,12 @@ class Fastchess : public IOutput {
1919
Fastchess(bool report_penta = true) : report_penta_(report_penta) {}
2020

2121
void printInterval(const SPRT& sprt, const Stats& stats, const std::string& first, const std::string& second,
22-
const engines& engines, const std::string& book) override {
23-
std::cout //
24-
<< "--------------------------------------------------\n" //
25-
<< printElo(stats, first, second, engines, book) //
26-
<< printSprt(sprt, stats) //
27-
<< "--------------------------------------------------\n" //
22+
const engines& engines, const std::string& book, ScoreBoard& scoreboard) override {
23+
std::cout //
24+
<< "--------------------------------------------------\n" //
25+
<< printElo(stats, first, second, engines, book, scoreboard) //
26+
<< printSprt(sprt, stats) //
27+
<< "--------------------------------------------------\n" //
2828
<< std::flush;
2929
};
3030

@@ -33,26 +33,38 @@ class Fastchess : public IOutput {
3333
}
3434

3535
std::string printElo(const Stats& stats, const std::string& first, const std::string& second,
36-
const engines& engines, const std::string& book) override {
37-
auto elo = createElo(stats, report_penta_);
36+
const engines& engines, const std::string& book, ScoreBoard& scoreboard) override {
37+
const auto& ecs = config::EngineConfigs.get();
3838

39-
const auto& first_engine = engines.first.getConfig().name == first ? engines.first : engines.second;
40-
const auto& second_engine = engines.first.getConfig().name == second ? engines.first : engines.second;
39+
if (ecs.size() == 2) {
40+
return printEloH2H(stats, first, second, engines, book);
41+
}
4142

42-
const auto tc = formatTimeControl(first_engine.getConfig(), second_engine.getConfig());
43-
const auto threads = formatThreads(first_engine.uciOptions(), second_engine.uciOptions());
44-
const auto hash = formatHash(first_engine.uciOptions(), second_engine.uciOptions());
45-
const auto bookname = getShortName(book);
43+
std::vector<std::tuple<const EngineConfiguration*, std::unique_ptr<elo::EloBase>, Stats>> elos;
4644

47-
auto result = fmt::format("{}\n{}\n{}\n{}", formatMatchup(first, second, tc, threads, hash, bookname),
48-
formatElo(elo), formatGameStats(*elo, stats, stats.pairsRatio()),
49-
formatGameResults(stats, stats.points(), stats.pointsRatio()));
45+
for (auto& e : ecs) {
46+
const auto stats = scoreboard.getAllStats(e.name);
47+
elos.emplace_back(&e, createElo(stats, report_penta_), stats);
48+
}
5049

51-
if (report_penta_) {
52-
result += fmt::format("\nPtnml(0-2): {}, {}", formatPentaStats(stats), formatwl_dd_Ratio(stats));
50+
// sort by elo diff
51+
52+
std::sort(elos.begin(), elos.end(),
53+
[](const auto& a, const auto& b) { return std::get<1>(a)->diff() > std::get<1>(b)->diff(); });
54+
55+
int rank = 0;
56+
std::string out = fmt::format("{:<4} {:<25} {:>10} {:>10} {:>10} {:>10} {:>10} {:>10} {:>10} {:>20}\n", "Rank",
57+
"Name", "Elo", "+/-", "nElo", "+/-", "Games", "Score", "Draw", "Ptnml(0-2)");
58+
59+
for (const auto& [ec, elo, stats] : elos) {
60+
out +=
61+
fmt::format("{:>4} {:<25} {:>10.2f} {:>10.2f} {:>10.2f} {:>10.2f} {:>10} {:>9.1f}% {:>9.1f}% {:>20}\n",
62+
++rank, ec->name, elo->diff(), elo->error(), elo->nEloDiff(), elo->nEloError(), stats.sum(),
63+
stats.pointsRatio(), report_penta_ ? stats.drawRatioPenta() : stats.drawRatio(),
64+
report_penta_ ? formatPentaStats(stats) : "");
5365
}
5466

55-
return result + "\n";
67+
return out;
5668
}
5769

5870
std::string printSprt(const SPRT& sprt, const Stats& stats) override {
@@ -82,6 +94,29 @@ class Fastchess : public IOutput {
8294
void endTournament() override { std::cout << "Tournament finished" << std::endl; }
8395

8496
private:
97+
std::string printEloH2H(const Stats& stats, const std::string& first, const std::string& second,
98+
const engines& engines, const std::string& book) {
99+
auto elo = createElo(stats, report_penta_);
100+
101+
const auto& first_engine = engines.first.getConfig().name == first ? engines.first : engines.second;
102+
const auto& second_engine = engines.first.getConfig().name == second ? engines.first : engines.second;
103+
104+
const auto tc = formatTimeControl(first_engine.getConfig(), second_engine.getConfig());
105+
const auto threads = formatThreads(first_engine.uciOptions(), second_engine.uciOptions());
106+
const auto hash = formatHash(first_engine.uciOptions(), second_engine.uciOptions());
107+
const auto bookname = getShortName(book);
108+
109+
auto result = fmt::format("{}\n{}\n{}\n{}", formatMatchup(first, second, tc, threads, hash, bookname),
110+
formatElo(elo), formatGameStats(*elo, stats, stats.pairsRatio()),
111+
formatGameResults(stats, stats.points(), stats.pointsRatio()));
112+
113+
if (report_penta_) {
114+
result += fmt::format("\nPtnml(0-2): {}, {}", formatPentaStats(stats), formatWLDDRatio(stats));
115+
}
116+
117+
return result + "\n";
118+
}
119+
85120
std::string getTime(const EngineConfiguration& config) const {
86121
const auto& limit = config.limit;
87122

@@ -163,7 +198,7 @@ class Fastchess : public IOutput {
163198
stats.penta_WD, stats.penta_WW);
164199
}
165200

166-
std::string formatwl_dd_Ratio(const Stats& stats) const {
201+
std::string formatWLDDRatio(const Stats& stats) const {
167202
return fmt::format("WL/DD Ratio: {:.2f}", stats.wl_dd_Ratio());
168203
}
169204

‎app/src/matchmaking/tournament/base/tournament.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,17 @@ extern std::atomic_bool stop;
2222
} // namespace atomic
2323

2424
BaseTournament::BaseTournament(const stats_map &results) {
25-
const auto &config = config::TournamentConfig.get();
26-
const auto total = setResults(results);
25+
const auto &config = config::TournamentConfig.get();
26+
const auto total = setResults(results);
27+
const auto num_players = config::EngineConfigs.get().size();
2728

2829
initial_matchcount_ = total;
2930
match_count_ = total;
3031

3132
output_ = OutputFactory::create(config.output, config.report_penta);
3233
cores_ = std::make_unique<affinity::AffinityManager>(config.affinity, getMaxAffinity(config::EngineConfigs.get()));
3334
book_ = std::make_unique<book::OpeningBook>(config, initial_matchcount_);
34-
generator_ = std::make_unique<MatchGenerator>(book_.get(), initial_matchcount_);
35+
generator_ = std::make_unique<MatchGenerator>(book_.get(), num_players, config.rounds, config.games, total);
3536

3637
if (!config.pgn.file.empty()) file_writer_pgn = std::make_unique<util::FileWriter>(config.pgn.file);
3738
if (!config.epd.file.empty()) file_writer_epd = std::make_unique<util::FileWriter>(config.epd.file);

‎app/src/matchmaking/tournament/base/tournament.hpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,12 +69,12 @@ class BaseTournament {
6969
std::uint64_t initial_matchcount_;
7070

7171
private:
72-
std::uint64_t setResults(const stats_map &results) noexcept {
72+
std::size_t setResults(const stats_map &results) noexcept {
7373
Logger::trace("Setting results...");
7474

7575
scoreboard_.setResults(results);
7676

77-
std::uint64_t total = 0;
77+
std::size_t total = 0;
7878

7979
for (const auto &pair1 : scoreboard_.getResults()) {
8080
const auto &stats = pair1.second;

‎app/src/matchmaking/tournament/roundrobin/match_generator.hpp‎

Lines changed: 73 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -15,59 +15,94 @@ namespace fastchess {
1515
*/
1616
class MatchGenerator {
1717
public:
18-
MatchGenerator(book::OpeningBook* opening_book, int initial_matchcount)
19-
: ecidx_one(0), ecidx_two(1), round_id(0), pair_id(0), engine_configs_size(config::EngineConfigs.get().size()) {
20-
opening_book_ = opening_book;
21-
offset_ = initial_matchcount / games;
22-
round_id = offset_;
18+
struct Pairing {
19+
std::size_t round_id;
20+
std::size_t pairing_id;
21+
std::size_t game_id;
22+
std::optional<std::size_t> opening_id;
23+
std::size_t player1;
24+
std::size_t player2;
25+
};
26+
27+
MatchGenerator(book::OpeningBook* opening_book, std::size_t players, std::size_t rounds, std::size_t games,
28+
std::size_t played_games)
29+
: opening_book_(opening_book),
30+
n_players(players),
31+
n_rounds(rounds),
32+
n_games_per_round(games),
33+
current_round(1),
34+
game_counter(0),
35+
player1(0),
36+
player2(1),
37+
games_per_pair(0),
38+
pair_counter(0) {
39+
current_round = (played_games / games) + 1;
40+
41+
if (n_players < 2 || n_rounds < 1 || n_games_per_round < 1) {
42+
throw std::invalid_argument("Invalid number of players, rounds, or games per round");
43+
}
2344
}
2445

25-
std::optional<std::tuple<std::size_t, std::size_t, std::size_t, int, std::optional<std::size_t>>> next() {
26-
if (ecidx_one >= engine_configs_size || ecidx_two >= engine_configs_size) return std::nullopt;
46+
// Function to generate the next game pairing
47+
std::optional<Pairing> next() {
48+
Pairing next_game;
2749

28-
if (pair_id == 0) {
29-
// Fetch a new opening only once per round
30-
opening = opening_book_->fetchId();
50+
// Check if the current round is beyond the number of rounds
51+
if (current_round > n_rounds) {
52+
// No more rounds left
53+
return std::nullopt;
3154
}
3255

33-
auto match = std::make_tuple(ecidx_one, ecidx_two, round_id, pair_id, opening);
34-
35-
advance();
36-
37-
return match;
38-
}
39-
40-
private:
41-
std::optional<std::size_t> opening;
42-
book::OpeningBook* opening_book_;
56+
if (games_per_pair == 0) {
57+
opening = opening_book_->fetchId();
58+
}
4359

44-
// 1 - 2
45-
const std::size_t games = config::TournamentConfig.get().games;
46-
const std::size_t rounds = config::TournamentConfig.get().rounds;
60+
next_game.round_id = current_round;
61+
next_game.game_id = ++game_counter;
62+
next_game.player1 = player1;
63+
next_game.player2 = player2;
64+
next_game.pairing_id = pair_counter;
65+
next_game.opening_id = opening;
4766

48-
// Engine configuration indices
49-
std::size_t ecidx_one, ecidx_two;
50-
std::size_t round_id;
67+
// Increment the games between the current player1 and player2
68+
games_per_pair++;
5169

52-
// 0 - 1
53-
std::size_t pair_id;
54-
std::size_t engine_configs_size;
70+
// If we've exhausted the games between the current player pair, move to the next pair
71+
if (games_per_pair >= n_games_per_round) {
72+
games_per_pair = 0;
5573

56-
int offset_;
74+
player2++;
75+
pair_counter++;
5776

58-
void advance() {
59-
if (++pair_id >= games) {
60-
pair_id = 0;
77+
if (player2 >= n_players) {
78+
player1++;
79+
player2 = player1 + 1;
80+
}
6181

62-
if (++round_id >= rounds) {
63-
round_id = offset_;
82+
// If we've exhausted all pairs for this round, move to the next round
83+
if (player1 >= n_players - 1) {
84+
current_round++;
6485

65-
if (++ecidx_two >= engine_configs_size) {
66-
ecidx_two = ++ecidx_one + 1;
67-
}
86+
player1 = 0;
87+
player2 = 1;
6888
}
6989
}
90+
91+
return next_game;
7092
}
93+
94+
private:
95+
book::OpeningBook* opening_book_;
96+
std::optional<std::size_t> opening;
97+
std::size_t n_players;
98+
std::size_t n_rounds;
99+
std::size_t n_games_per_round;
100+
std::size_t current_round;
101+
std::size_t game_counter;
102+
std::size_t player1;
103+
std::size_t player2;
104+
std::size_t games_per_pair;
105+
std::size_t pair_counter;
71106
};
72107

73108
} // namespace fastchess

0 commit comments

Comments
 (0)