[fix] oss-cluster local benchmarks (compatibility confirmed with RSCoordinator) (#177)

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.4.3"
+version = "0.4.4"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>"]
 readme = "README.md"

redisbench_admin/run/cluster.py

@@ -8,11 +8,15 @@
 import redis
 
 
-def cluster_init_steps(args, clusterconfig, local_module_file, r_conns, shard_count):
+def cluster_init_steps(
+    args, clusterconfig, local_module_file, r_conns, shard_count, contains_dataset=True
+):
     startup_nodes = []
     for p in range(args.port, args.port + shard_count):
         primary_conn = redis.StrictRedis(port=p)
-        primary_conn.execute_command("DEBUG RELOAD NOSAVE")
+        if contains_dataset:
+            # force debug reload nosave to replace the current database with the contents of an existing RDB file
+            primary_conn.execute_command("DEBUG RELOAD NOSAVE")
         r_conns.append(primary_conn)
         startup_nodes.append({"host": "127.0.0.1", "port": "{}".format(p)})
     if clusterconfig is not None:

redisbench_admin/run/common.py

@@ -394,13 +394,22 @@ def run_redis_pre_steps(benchmark_config, r, required_modules):
                     fts_indexname = fts_indexname.decode()
                 ft_info = r.execute_command("ft.info {}".format(fts_indexname))
                 is_indexing = None
+                percent_indexed = "0.0"
                 for arraypos, arrayval in enumerate(ft_info, start=0):
+                    if arrayval == b"percent_indexed" or arrayval == "percent_indexed":
+                        percent_indexed = ft_info[arraypos + 1]
                     if arrayval == b"indexing" or arrayval == "indexing":
                         is_indexing = ft_info[arraypos + 1]
-                if is_indexing == "0" or is_indexing == b"0":
+
+                logging.info(
+                    "indexing={} ; percent_indexed={}.".format(
+                        is_indexing, percent_indexed
+                    )
+                )
+                if is_indexing == "0" or is_indexing == b"0" or is_indexing == 0:
                     loading_indices.pop(index_pos)
 
-            time.sleep(1)
+            time.sleep(5)
         logging.info("Loaded all secondary indices.")
 
     return artifact_versions[0]

redisbench_admin/run/ftsb/ftsb.py

@@ -33,7 +33,7 @@ def prepare_ftsb_benchmark_command(
         ["--host", "{}:{}".format(server_private_ip, server_plaintext_port)]
     )
     if cluster_api_enabled is True:
-        command_arr.extend(["--cluster"])
+        command_arr.extend(["--cluster-mode"])
     if "parameters" in benchmark_config:
         for k in benchmark_config["parameters"]:
             if "input" in k:

redisbench_admin/run_local/run_local.py

@@ -193,7 +193,7 @@ def run_local_command_logic(args, project_name, project_version):
                                     "CLUSTER SAVECONFIG"
                                 )
 
-                        check_dataset_local_requirements(
+                        dataset, _, _ = check_dataset_local_requirements(
                             benchmark_config,
                             temporary_dir,
                             dirname,
@@ -223,12 +223,16 @@ def run_local_command_logic(args, project_name, project_version):
                             r = redis.StrictRedis(port=args.port)
 
                         if setup_type == "oss-cluster":
+                            contains_rdb = False
+                            if dataset is not None:
+                                contains_rdb = True
                             startup_nodes = cluster_init_steps(
                                 args,
                                 clusterconfig,
                                 local_module_file,
                                 r_conns,
                                 shard_count,
+                                contains_rdb,
                             )
 
                             rc = RedisCluster(

redisbench_admin/run_remote/run_remote.py

@@ -264,7 +264,7 @@ def run_remote_command_logic(args, project_name, project_version):
                                     )
 
                             if setup_type == "oss-standalone":
-                                full_logfile = spin_up_standalone_remote_redis(
+                                full_logfile, dataset = spin_up_standalone_remote_redis(
                                     benchmark_config,
                                     server_public_ip,
                                     username,
@@ -306,12 +306,16 @@ def run_remote_command_logic(args, project_name, project_version):
                             )
 
                             if setup_type == "oss-cluster":
+                                contains_rdb = False
+                                if dataset is not None:
+                                    contains_rdb = True
                                 startup_nodes = cluster_init_steps(
                                     args,
                                     clusterconfig,
                                     local_module_file,
                                     r_conns,
                                     shard_count,
+                                    contains_rdb,
                                 )
 
                                 rc = RedisCluster(

redisbench_admin/run_remote/standalone.py

@@ -28,7 +28,7 @@ def spin_up_standalone_remote_redis(
     dbdir_folder=None,
 ):
     # copy the rdb to DB machine
-    check_dataset_remote_requirements(
+    _, dataset, _, _ = check_dataset_remote_requirements(
         benchmark_config,
         server_public_ip,
         username,
@@ -80,4 +80,4 @@ def spin_up_standalone_remote_redis(
     # start redis-server
     commands = [initial_redis_cmd]
     execute_remote_commands(server_public_ip, username, private_key, commands)
-    return full_logfile
+    return full_logfile, dataset

redisbench_admin/utils/remote.py

@@ -163,7 +163,7 @@ def check_dataset_remote_requirements(
             remote_dataset_file,
             None,
         )
-    return res
+    return res, dataset, fullpath, tmppath
 
 
 def setup_remote_environment(