Skip to content

Commit 9d8a199

Browse files
filipecosta90filipecosta90
authored
WIP on oss-cluster benchmarks and local runs: enable specify max number of profilers (#154)
* [wip] WIP on oss-cluster benchmarks and local runs
1 parent 4ffccf1 commit 9d8a199

File tree

4 files changed

+135
-68
lines changed

4 files changed

+135
-68
lines changed

‎redisbench_admin/environments/oss_cluster.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ def generate_cluster_redis_server_args(
9999
"--cluster-enabled",
100100
"yes",
101101
"--dbfilename",
102-
"cluster-node-port-{}.rdb".format(port),
102+
get_cluster_dbfilename(port),
103103
"--cluster-config-file",
104104
"cluster-node-port-{}.config".format(port),
105105
"--save",
@@ -134,3 +134,7 @@ def generate_cluster_redis_server_args(
134134
]
135135
)
136136
return command
137+
138+
139+
def get_cluster_dbfilename(port):
140+
return "cluster-node-port-{}.rdb".format(port)

‎redisbench_admin/run_local/args.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,9 @@
1313
)
1414

1515
PUSH_S3 = bool(os.getenv("PUSH_S3", False))
16-
PROFILERS_ENABLED = os.getenv("PROFILE", 0)
16+
PROFILERS_ENABLED = bool(os.getenv("PROFILE", 0))
1717
PROFILERS = os.getenv("PROFILERS", PROFILERS_DEFAULT)
18+
MAX_PROFILERS_PER_TYPE = int(os.getenv("MAX_PROFILERS", 1))
1819
PROFILE_FREQ = os.getenv("PROFILE_FREQ", PROFILE_FREQ_DEFAULT)
1920
ENV = os.getenv("ENV", "oss-standalone,oss-cluster")
2021

‎redisbench_admin/run_local/run_local.py‎

Lines changed: 107 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@
1313
import sys
1414
import tempfile
1515
import datetime
16+
import traceback
1617

1718
import redis
1819
import wget
20+
from rediscluster import RedisCluster
1921

2022
from redisbench_admin.environments.oss_cluster import spin_up_local_redis_cluster
2123
from redisbench_admin.profilers.perf import Perf
@@ -31,7 +33,7 @@
3133
extract_test_feasible_setups,
3234
get_setup_type_and_primaries_count,
3335
)
34-
from redisbench_admin.run_local.args import PROFILE_FREQ
36+
from redisbench_admin.run_local.args import PROFILE_FREQ, MAX_PROFILERS_PER_TYPE
3537
from redisbench_admin.utils.benchmark_config import (
3638
prepare_benchmark_definitions,
3739
extract_benchmark_tool_settings,
@@ -200,56 +202,74 @@ def run_local_command_logic(args):
200202
dbdir_folder, temporary_dir
201203
)
202204
)
203-
204-
check_dataset_local_requirements(
205-
benchmark_config, temporary_dir, dirname
206-
)
207-
208205
(
209206
redis_configuration_parameters,
210207
_,
211208
) = extract_redis_dbconfig_parameters(
212209
benchmark_config, "dbconfig"
213210
)
214211
cluster_api_enabled = False
215-
if setup_type == "oss-standalone":
216-
redis_processes = spin_up_local_redis(
212+
213+
if setup_type == "oss-cluster":
214+
cluster_api_enabled = True
215+
# pass
216+
redis_processes = spin_up_local_redis_cluster(
217217
temporary_dir,
218+
shard_count,
218219
args.port,
219220
local_module_file,
220221
redis_configuration_parameters,
221222
dbdir_folder,
222223
)
223-
224224
for redis_process in redis_processes:
225225
if is_process_alive(redis_process) is False:
226226
raise Exception(
227227
"Redis process is not alive. Failing test."
228228
)
229-
229+
# we use node 0 for the checks
230230
r = redis.StrictRedis(port=args.port)
231+
r_conns = []
232+
for p in range(args.port, args.port + shard_count):
233+
redis.StrictRedis(port=p).execute_command(
234+
"CLUSTER SAVECONFIG"
235+
)
231236

232-
if setup_type == "oss-cluster":
233-
cluster_api_enabled = True
234-
# pass
235-
redis_processes = spin_up_local_redis_cluster(
237+
check_dataset_local_requirements(
238+
benchmark_config,
239+
temporary_dir,
240+
dirname,
241+
"./datasets",
242+
"dbconfig",
243+
shard_count,
244+
cluster_api_enabled,
245+
)
246+
247+
if setup_type == "oss-standalone":
248+
redis_processes = spin_up_local_redis(
236249
temporary_dir,
237-
shard_count,
238250
args.port,
239251
local_module_file,
240252
redis_configuration_parameters,
241253
dbdir_folder,
242254
)
255+
243256
for redis_process in redis_processes:
244257
if is_process_alive(redis_process) is False:
245258
raise Exception(
246259
"Redis process is not alive. Failing test."
247260
)
248-
# we use node 0 for the checks
261+
249262
r = redis.StrictRedis(port=args.port)
250-
r_conns = []
263+
264+
if setup_type == "oss-cluster":
265+
startup_nodes = []
251266
for p in range(args.port, args.port + shard_count):
252-
r_conns.append(redis.StrictRedis(port=p))
267+
primary_conn = redis.StrictRedis(port=p)
268+
primary_conn.execute_command("DEBUG RELOAD NOSAVE")
269+
r_conns.append(primary_conn)
270+
startup_nodes.append(
271+
{"host": "127.0.0.1", "port": "{}".format(p)}
272+
)
253273
if clusterconfig is not None:
254274
if "init_commands" in clusterconfig:
255275
for command_group in clusterconfig["init_commands"]:
@@ -302,6 +322,16 @@ def run_local_command_logic(args):
302322
],
303323
)
304324
)
325+
326+
rc = RedisCluster(
327+
startup_nodes=startup_nodes, decode_responses=True
328+
)
329+
cluster_info = rc.cluster_info()
330+
logging.info(
331+
"Cluster info after initialization: {}.".format(
332+
cluster_info
333+
)
334+
)
305335
stdout = r.execute_command("info modules")
306336
(
307337
module_names,
@@ -365,7 +395,9 @@ def run_local_command_logic(args):
365395
logging.info("Profilers are enabled")
366396
total_involved_processes = len(redis_processes)
367397
_, profilers_map = get_profilers_map(
368-
profilers_list, total_involved_processes
398+
profilers_list,
399+
total_involved_processes,
400+
MAX_PROFILERS_PER_TYPE,
369401
)
370402
for (
371403
profiler_name,
@@ -374,6 +406,10 @@ def run_local_command_logic(args):
374406
for setup_process_number, redis_process in enumerate(
375407
redis_processes
376408
):
409+
if (setup_process_number + 1) > len(
410+
profiler_obj_arr
411+
):
412+
continue
377413
profiler_obj = profiler_obj_arr[
378414
setup_process_number
379415
]
@@ -447,49 +483,47 @@ def run_local_command_logic(args):
447483
profiler_name,
448484
profiler_obj_arr,
449485
) in profilers_map.items():
450-
for setup_process_number, redis_process in enumerate(
451-
redis_processes
486+
for setup_process_number, profiler_obj in enumerate(
487+
profiler_obj_arr
452488
):
453-
profiler_obj = profiler_obj_arr[
454-
setup_process_number
455-
]
456-
setup_process_number = setup_process_number + 1
457489
logging.info(
458490
"Stopping profiler {} for Process {} of {}: pid {}".format(
459491
profiler_name,
460492
setup_process_number,
461493
total_involved_processes,
462-
redis_process.pid,
494+
profiler_obj.pid,
463495
)
464496
)
465497

466-
profiler_obj.stop_profile()
467-
468-
# Generate:
469-
# - artifact with Flame Graph SVG
470-
# - artifact with output graph image in PNG format
471-
# - artifact with top entries in text form
472-
(
473-
profile_res,
474-
profile_res_artifacts_map,
475-
) = profiler_obj.generate_outputs(
476-
test_name,
477-
details=collection_summary_str,
478-
binary=dso,
479-
primary_id=setup_process_number,
480-
total_primaries=total_involved_processes,
481-
)
498+
profile_res = profiler_obj.stop_profile()
482499
if profile_res is True:
483-
logging.info(
484-
"Profiler {} for pid {} ran successfully and generated {} artifacts.".format(
485-
profiler_name,
486-
redis_process.pid,
487-
len(profile_res_artifacts_map.values()),
488-
)
489-
)
490-
overall_artifacts_map.update(
491-
profile_res_artifacts_map
500+
# Generate:
501+
# - artifact with Flame Graph SVG
502+
# - artifact with output graph image in PNG format
503+
# - artifact with top entries in text form
504+
(
505+
profile_res,
506+
profile_res_artifacts_map,
507+
) = profiler_obj.generate_outputs(
508+
test_name,
509+
details=collection_summary_str,
510+
binary=dso,
511+
primary_id=(setup_process_number + 1),
512+
total_primaries=total_involved_processes,
492513
)
514+
if profile_res is True:
515+
logging.info(
516+
"Profiler {} for pid {} ran successfully and generated {} artifacts.".format(
517+
profiler_name,
518+
profiler_obj.pid,
519+
len(
520+
profile_res_artifacts_map.values()
521+
),
522+
)
523+
)
524+
overall_artifacts_map.update(
525+
profile_res_artifacts_map
526+
)
493527

494528
for (
495529
artifact_name,
@@ -537,9 +571,10 @@ def run_local_command_logic(args):
537571
except:
538572
return_code |= 1
539573
logging.critical(
540-
"Some unexpected exception was caught during remote work. Failing test...."
574+
"Some unexpected exception was caught during local work. Failing test...."
541575
)
542576
logging.critical(sys.exc_info())
577+
logging.critical("Traceback: {}".format(traceback.format_exc()))
543578
# tear-down
544579
logging.info("Tearing down setup")
545580
for redis_process in redis_processes:
@@ -598,22 +633,33 @@ def check_compatible_system_and_kernel_and_prepare_profile(args):
598633
return res
599634

600635

601-
def get_profilers_map(profilers_list, total_involved_processes):
636+
def get_profilers_map(profilers_list, total_involved_processes, max_profilers=1):
602637
profilers_map = {}
603638
res = True
604639
for profiler_name in profilers_list:
605640
try:
606641
profilers_map[profiler_name] = []
607642
if "perf:" in profiler_name:
608-
logging.info("Preparing perf (a.k.a. perf_events ) profiler")
609-
for _ in range(total_involved_processes):
610-
perf = Perf()
611-
profilers_map[profiler_name].append(perf)
643+
for profilers_per_type in range(1, total_involved_processes + 1):
644+
if profilers_per_type <= max_profilers:
645+
logging.info(
646+
"Preparing perf (a.k.a. perf_events ) profiler for proc {} of {}".format(
647+
profilers_per_type, total_involved_processes
648+
)
649+
)
650+
perf = Perf()
651+
profilers_map[profiler_name].append(perf)
612652
if "vtune" in profiler_name:
613-
logging.info("Preparing Intel(R) VTune(TM) profiler")
614-
for _ in range(total_involved_processes):
615-
vtune = Vtune()
616-
profilers_map[profiler_name].append(vtune)
653+
654+
for profilers_per_type in range(total_involved_processes):
655+
logging.info(
656+
"Preparing Intel(R) VTune(TM) profiler for proc {} of {}".format(
657+
profilers_per_type, total_involved_processes
658+
)
659+
)
660+
if profilers_per_type <= max_profilers:
661+
vtune = Vtune()
662+
profilers_map[profiler_name].append(vtune)
617663
except Exception as e:
618664
logging.error(
619665
"Detected error while trying to prepare profiler named {}. Error: {}".format(

‎redisbench_admin/utils/local.py‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,17 @@
1010

1111
import wget
1212

13+
from redisbench_admin.environments.oss_cluster import get_cluster_dbfilename
14+
1315

1416
def check_dataset_local_requirements(
1517
benchmark_config,
1618
redis_dbdir,
1719
dirname=None,
1820
datasets_localtemp_dir="./datasets",
1921
dbconfig_keyname="dbconfig",
22+
number_primaries=1,
23+
is_cluster=False,
2024
):
2125
dataset = None
2226
full_path = None
@@ -29,11 +33,23 @@ def check_dataset_local_requirements(
2933
full_path = check_if_needs_remote_fetch(
3034
dataset, datasets_localtemp_dir, dirname
3135
)
32-
tmp_path = "{}/dump.rdb".format(redis_dbdir)
33-
logging.info(
34-
"Copying rdb from {} to {}/dump.rdb".format(full_path, redis_dbdir)
35-
)
36-
copyfile(full_path, tmp_path)
36+
37+
if is_cluster is False:
38+
tmp_path = "{}/dump.rdb".format(redis_dbdir)
39+
logging.info("Copying rdb from {} to {}".format(full_path, tmp_path))
40+
copyfile(full_path, tmp_path)
41+
else:
42+
start_port = 6379
43+
for primary_number in range(number_primaries):
44+
primary_port = start_port + primary_number
45+
tmp_path = "{}/{}".format(
46+
redis_dbdir, get_cluster_dbfilename(primary_port)
47+
)
48+
logging.info(
49+
"Copying rdb from {} to {}".format(full_path, tmp_path)
50+
)
51+
copyfile(full_path, tmp_path)
52+
3753
return dataset, full_path, tmp_path
3854

3955

0 commit comments

Comments
 (0)