Skip to content
This repository was archived by the owner on Dec 8, 2021. It is now read-only.

Commit 6e82ef3

Browse files
authored
cleanup: refactor benchmark configuration (#1121)
Make it easier to test the configuration parsing and avoid code duplication.
1 parent 67e1d28 commit 6e82ef3

File tree

9 files changed

+521
-400
lines changed

9 files changed

+521
-400
lines changed

google/cloud/spanner/benchmarks/BUILD

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,35 @@ package(default_visibility = ["//visibility:public"])
1616

1717
licenses(["notice"]) # Apache 2.0
1818

19-
load(":spanner_client_benchmarks.bzl", "spanner_client_benchmarks")
19+
load(":spanner_client_benchmarks.bzl", "spanner_client_benchmarks_hdrs", "spanner_client_benchmarks_srcs")
20+
load(":spanner_client_benchmark_programs.bzl", "spanner_client_benchmark_programs")
21+
22+
cc_library(
23+
name = "spanner_client_benchmarks",
24+
srcs = spanner_client_benchmarks_srcs,
25+
hdrs = spanner_client_benchmarks_hdrs,
26+
deps = [
27+
"//google/cloud/spanner:spanner_client",
28+
"//google/cloud/spanner:spanner_client_mocks",
29+
"//google/cloud/spanner:spanner_client_testing",
30+
"@com_github_googleapis_google_cloud_cpp_common//google/cloud:google_cloud_cpp_common",
31+
"@com_github_googleapis_google_cloud_cpp_common//google/cloud/testing_util:google_cloud_cpp_testing",
32+
"@com_google_googletest//:gtest",
33+
],
34+
)
2035

2136
[cc_test(
22-
name = "spanner_client_" + test.replace("/", "_").replace(".cc", ""),
37+
name = test.replace("/", "_").replace(".cc", ""),
2338
timeout = "long",
2439
srcs = [test],
2540
tags = ["integration-tests"],
2641
deps = [
42+
":spanner_client_benchmarks",
2743
"//google/cloud/spanner:spanner_client",
2844
"//google/cloud/spanner:spanner_client_mocks",
2945
"//google/cloud/spanner:spanner_client_testing",
3046
"@com_github_googleapis_google_cloud_cpp_common//google/cloud:google_cloud_cpp_common",
3147
"@com_github_googleapis_google_cloud_cpp_common//google/cloud/testing_util:google_cloud_cpp_testing",
3248
"@com_google_googletest//:gtest",
3349
],
34-
) for test in spanner_client_benchmarks]
50+
) for test in spanner_client_benchmark_programs]

google/cloud/spanner/benchmarks/CMakeLists.txt

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -35,22 +35,45 @@ function (spanner_client_define_benchmarks)
3535
GOOGLE_CLOUD_CPP_HAVE_RUSAGE_THREAD=$<BOOL:${GOOGLE_CLOUD_CPP_HAVE_RUSAGE_THREAD}>
3636
)
3737

38-
set(spanner_client_benchmarks
38+
add_library(spanner_client_benchmarks benchmarks_config.cc
39+
benchmarks_config.h)
40+
target_link_libraries(
41+
spanner_client_benchmarks
42+
PUBLIC getrusage_flags
43+
spanner_client_mocks
44+
googleapis-c++::spanner_client
45+
google_cloud_cpp_testing
46+
GTest::gmock_main
47+
GTest::gmock
48+
GTest::gtest)
49+
create_bazel_config(spanner_client_benchmarks)
50+
51+
target_include_directories(
52+
spanner_client_benchmarks
53+
PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}>
54+
$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}>
55+
$<INSTALL_INTERFACE:include>)
56+
target_compile_options(spanner_client_benchmarks
57+
PUBLIC ${GOOGLE_CLOUD_CPP_SPANNER_EXCEPTIONS_FLAG})
58+
59+
set(spanner_client_benchmark_programs
3960
# cmake-format: sortable
40-
single_row_throughput_benchmark.cc multiple_rows_cpu_benchmark.cc)
61+
benchmarks_config_test.cc multiple_rows_cpu_benchmark.cc
62+
single_row_throughput_benchmark.cc)
4163

4264
# Export the list of unit tests to a .bzl file so we do not need to maintain
4365
# the list in two places.
44-
export_list_to_bazel("spanner_client_benchmarks.bzl"
45-
"spanner_client_benchmarks")
66+
export_list_to_bazel("spanner_client_benchmark_programs.bzl"
67+
"spanner_client_benchmark_programs")
4668

4769
# Generate a target for each benchmark.
48-
foreach (fname ${spanner_client_benchmarks})
70+
foreach (fname ${spanner_client_benchmark_programs})
4971
google_cloud_cpp_test_name_to_target(target "${fname}")
5072
add_executable(${target} ${fname})
5173
target_link_libraries(
5274
${target}
53-
PRIVATE googleapis-c++::spanner_client
75+
PRIVATE spanner_client_benchmarks
76+
googleapis-c++::spanner_client
5477
getrusage_flags
5578
spanner_client_testing
5679
google_cloud_cpp_testing
@@ -71,11 +94,6 @@ function (spanner_client_define_benchmarks)
7194
# label them as tests.
7295
set_tests_properties(${target} PROPERTIES LABELS "integration-tests")
7396
endforeach ()
74-
75-
if (NOT GOOGLE_CLOUD_CPP_SPANNER_ENABLE_CXX_EXCEPTIONS)
76-
return()
77-
endif ()
78-
7997
endfunction ()
8098

8199
# Only define the benchmarks if testing is enabled. Package maintainers may not
Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
// Copyright 2019 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include "google/cloud/spanner/benchmarks/benchmarks_config.h"
16+
#include "google/cloud/spanner/internal/build_info.h"
17+
#include "google/cloud/spanner/internal/compiler_info.h"
18+
#include "google/cloud/internal/getenv.h"
19+
#include <functional>
20+
#include <sstream>
21+
#if GOOGLE_CLOUD_CPP_HAVE_GETRUSAGE
22+
#include <sys/resource.h>
23+
#endif // GOOGLE_CLOUD_CPP_HAVE_GETRUSAGE
24+
25+
namespace google {
26+
namespace cloud {
27+
namespace spanner_benchmarks {
28+
inline namespace SPANNER_CLIENT_NS {
29+
30+
namespace cs = google::cloud::spanner;
31+
32+
namespace {
33+
bool SupportPerThreadUsage() {
34+
#if GOOGLE_CLOUD_CPP_HAVE_RUSAGE_THREAD
35+
return true;
36+
#else
37+
return false;
38+
#endif // GOOGLE_CLOUD_CPP_HAVE_RUSAGE_THREAD
39+
}
40+
} // namespace
41+
42+
std::ostream& operator<<(std::ostream& os, Config const& config) {
43+
return os << std::boolalpha << "# Experiment: " << config.experiment
44+
<< "\n# Project: " << config.project_id
45+
<< "\n# Instance: " << config.instance_id
46+
<< "\n# Database: " << config.database_id
47+
<< "\n# Samples: " << config.samples
48+
<< "\n# Minimum Threads: " << config.minimum_threads
49+
<< "\n# Maximum Threads: " << config.maximum_threads
50+
<< "\n# Minimum Clients: " << config.minimum_clients
51+
<< "\n# Maximum Clients: " << config.maximum_clients
52+
<< "\n# Iteration Duration: " << config.iteration_duration.count()
53+
<< "s"
54+
<< "\n# Table Size: " << config.table_size
55+
<< "\n# Query Size: " << config.query_size
56+
<< "\n# Use Only Stubs: " << config.use_only_stubs
57+
<< "\n# Use Only Clients: " << config.use_only_clients
58+
<< "\n# Compiler: " << cs::internal::CompilerId() << "-"
59+
<< cs::internal::CompilerVersion()
60+
<< "\n# Build Flags: " << cs::internal::BuildFlags() << "\n";
61+
}
62+
63+
google::cloud::StatusOr<Config> ParseArgs(std::vector<std::string> args) {
64+
Config config;
65+
66+
config.experiment = "run-all";
67+
68+
config.project_id =
69+
google::cloud::internal::GetEnv("GOOGLE_CLOUD_PROJECT").value_or("");
70+
config.instance_id =
71+
google::cloud::internal::GetEnv("GOOGLE_CLOUD_CPP_SPANNER_INSTANCE")
72+
.value_or("");
73+
74+
struct Flag {
75+
std::string flag_name;
76+
std::function<void(Config&, std::string)> parser;
77+
};
78+
79+
// NOLINTNEXTLINE(modernize-avoid-c-arrays)
80+
Flag flags[] = {
81+
{"--experiment=",
82+
[](Config& c, std::string v) { c.experiment = std::move(v); }},
83+
{"--project=",
84+
[](Config& c, std::string v) { c.project_id = std::move(v); }},
85+
{"--instance=",
86+
[](Config& c, std::string v) { c.instance_id = std::move(v); }},
87+
{"--samples=",
88+
[](Config& c, std::string const& v) { c.samples = std::stoi(v); }},
89+
{"--iteration-duration=",
90+
[](Config& c, std::string const& v) {
91+
c.iteration_duration = std::chrono::seconds(std::stoi(v));
92+
}},
93+
{"--minimum-threads=",
94+
[](Config& c, std::string const& v) {
95+
c.minimum_threads = std::stoi(v);
96+
}},
97+
{"--maximum-threads=",
98+
[](Config& c, std::string const& v) {
99+
c.maximum_threads = std::stoi(v);
100+
}},
101+
{"--minimum-clients=",
102+
[](Config& c, std::string const& v) {
103+
c.minimum_clients = std::stoi(v);
104+
}},
105+
{"--maximum-clients=",
106+
[](Config& c, std::string const& v) {
107+
c.maximum_clients = std::stoi(v);
108+
}},
109+
{"--table-size=",
110+
[](Config& c, std::string const& v) { c.table_size = std::stol(v); }},
111+
{"--query-size=",
112+
[](Config& c, std::string const& v) { c.query_size = std::stol(v); }},
113+
114+
{"--use-only-stubs",
115+
[](Config& c, std::string const&) { c.use_only_stubs = true; }},
116+
{"--use-only-clients",
117+
[](Config& c, std::string const&) { c.use_only_clients = true; }},
118+
};
119+
120+
auto invalid_argument = [](std::string msg) {
121+
return google::cloud::Status(google::cloud::StatusCode::kInvalidArgument,
122+
std::move(msg));
123+
};
124+
125+
for (auto i = std::next(args.begin()); i != args.end(); ++i) {
126+
std::string const& arg = *i;
127+
bool found = false;
128+
for (auto const& flag : flags) {
129+
if (arg.rfind(flag.flag_name, 0) != 0) continue;
130+
found = true;
131+
flag.parser(config, arg.substr(flag.flag_name.size()));
132+
133+
break;
134+
}
135+
if (!found && arg.rfind("--", 0) == 0) {
136+
return invalid_argument("Unexpected command-line flag " + arg);
137+
}
138+
}
139+
140+
if (config.experiment.empty()) {
141+
return invalid_argument("Missing value for --experiment flag");
142+
}
143+
144+
if (config.project_id.empty()) {
145+
return invalid_argument(
146+
"The project id is not set, provide a value in the --project flag,"
147+
" or set the GOOGLE_CLOUD_PROJECT environment variable");
148+
}
149+
150+
if (config.minimum_threads <= 0) {
151+
std::ostringstream os;
152+
os << "The minimum number of threads (" << config.minimum_threads << ")"
153+
<< " must be greater than zero";
154+
return invalid_argument(os.str());
155+
}
156+
if (config.maximum_threads < config.minimum_threads) {
157+
std::ostringstream os;
158+
os << "The maximum number of threads (" << config.maximum_threads << ")"
159+
<< " must be greater or equal than the minimum number of threads ("
160+
<< config.minimum_threads << ")";
161+
return invalid_argument(os.str());
162+
}
163+
164+
if (!SupportPerThreadUsage() && config.maximum_threads > 1) {
165+
std::ostringstream os;
166+
os << "Your platform does not support per-thread getrusage() data."
167+
<< " The benchmark cannot run with more than one thread, and you"
168+
<< " set maximum threads to " << config.maximum_threads;
169+
return invalid_argument(os.str());
170+
}
171+
172+
if (config.minimum_clients <= 0) {
173+
std::ostringstream os;
174+
os << "The minimum number of clients (" << config.minimum_clients << ")"
175+
<< " must be greater than zero";
176+
return invalid_argument(os.str());
177+
}
178+
if (config.maximum_clients < config.minimum_clients) {
179+
std::ostringstream os;
180+
os << "The maximum number of clients (" << config.maximum_clients << ")"
181+
<< " must be greater or equal than the minimum number of clients ("
182+
<< config.minimum_clients << ")";
183+
return invalid_argument(os.str());
184+
}
185+
186+
if (config.query_size <= 0) {
187+
std::ostringstream os;
188+
os << "The query size (" << config.query_size << ") should be > 0";
189+
return invalid_argument(os.str());
190+
}
191+
192+
if (config.table_size < config.query_size) {
193+
std::ostringstream os;
194+
os << "The table size (" << config.table_size << ") should be greater"
195+
<< " than the query size (" << config.query_size << ")";
196+
return invalid_argument(os.str());
197+
}
198+
199+
if (config.use_only_stubs && config.use_only_clients) {
200+
std::ostringstream os;
201+
os << "Only one of --use-only-stubs or --use-only-clients can be set";
202+
return invalid_argument(os.str());
203+
}
204+
205+
return config;
206+
}
207+
208+
} // namespace SPANNER_CLIENT_NS
209+
} // namespace spanner_benchmarks
210+
} // namespace cloud
211+
} // namespace google
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Copyright 2019 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#ifndef GOOGLE_CLOUD_CPP_SPANNER_GOOGLE_CLOUD_SPANNER_BENCHMARKS_BENCHMARKS_CONFIG_H_
16+
#define GOOGLE_CLOUD_CPP_SPANNER_GOOGLE_CLOUD_SPANNER_BENCHMARKS_BENCHMARKS_CONFIG_H_
17+
18+
#include "google/cloud/spanner/version.h"
19+
#include "google/cloud/status_or.h"
20+
#include <chrono>
21+
#include <string>
22+
#include <vector>
23+
24+
namespace google {
25+
namespace cloud {
26+
namespace spanner_benchmarks {
27+
inline namespace SPANNER_CLIENT_NS {
28+
29+
struct Config {
30+
std::string experiment;
31+
32+
std::string project_id;
33+
std::string instance_id;
34+
std::string database_id;
35+
36+
int samples = 2;
37+
std::chrono::seconds iteration_duration = std::chrono::seconds(5);
38+
39+
int minimum_threads = 1;
40+
int maximum_threads = 1;
41+
int minimum_clients = 1;
42+
int maximum_clients = 1;
43+
44+
std::int64_t table_size = 1000 * 1000L;
45+
std::int64_t query_size = 1000;
46+
47+
bool use_only_clients = false;
48+
bool use_only_stubs = false;
49+
};
50+
51+
std::ostream& operator<<(std::ostream& os, Config const& config);
52+
53+
google::cloud::StatusOr<Config> ParseArgs(std::vector<std::string> args);
54+
55+
} // namespace SPANNER_CLIENT_NS
56+
} // namespace spanner_benchmarks
57+
} // namespace cloud
58+
} // namespace google
59+
60+
#endif // GOOGLE_CLOUD_CPP_SPANNER_GOOGLE_CLOUD_SPANNER_BENCHMARKS_BENCHMARKS_CONFIG_H_

0 commit comments

Comments
 (0)