Added caching and session-storage benchmarks

Author: fcostaoliveira

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.302"
+version = "0.1.309"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

redis_benchmarks_specification/__compare__/args.py

@@ -77,7 +77,7 @@ def create_compare_arguments(parser):
     parser.add_argument("--comparison_deployment_name", type=str, default="")
     parser.add_argument("--metric_name", type=str, default="ALL_STATS.Totals.Ops/sec")
     parser.add_argument(
-        "--running_platform", type=str, default="intel64-ubuntu22.04-redis-icx1"
+        "--running_platform", type=str, default=None
     )
     parser.add_argument(
         "--running_platform_baseline",

redis_benchmarks_specification/__compare__/compare.py

@@ -1182,7 +1182,7 @@ def from_rts_to_regression_table(
             filters_baseline.append("deployment_name={}".format(baseline_deployment_name))
         if baseline_github_org != "":
             filters_baseline.append(f"github_org={baseline_github_org}")
-        if running_platform_baseline is not None:
+        if running_platform_baseline is not None and running_platform_baseline != "":
             filters_baseline.append(
                 "running_platform={}".format(running_platform_baseline)
             )
@@ -1202,7 +1202,7 @@ def from_rts_to_regression_table(
             filters_baseline.append("hash==")
         if "hash" not in by_str_comparison:
             filters_comparison.append("hash==")
-        if running_platform_comparison is not None:
+        if running_platform_comparison is not None and running_platform_comparison != "":
             filters_comparison.append(
                 "running_platform={}".format(running_platform_comparison)
             )
@@ -1538,12 +1538,18 @@ def from_rts_to_regression_table(
 
 def get_only_Totals(baseline_timeseries):
     logging.warning("\t\tTime-series: {}".format(", ".join(baseline_timeseries)))
-    logging.info("Checking if Totals will reduce timeseries.")
+    logging.info(f"Checking if Totals will reduce timeseries. initial len={len(baseline_timeseries)}")
     new_base = []
     for ts_name in baseline_timeseries:
+        if "io-threads" in ts_name:
+            continue
+        if "oss-cluster" in ts_name:
+            continue
         if "Totals" in ts_name:
             new_base.append(ts_name)
     baseline_timeseries = new_base
+    logging.info(f"                                          final len={len(baseline_timeseries)}")
+ 
     return baseline_timeseries
 
 

redis_benchmarks_specification/__runner__/runner.py

@@ -1393,17 +1393,27 @@ def delete_temporary_files(
                     ssl_cert_reqs = "required"
                     if tls_skip_verify:
                         ssl_cert_reqs = None
-                    r = redis.StrictRedis(
-                        host=host,
-                        port=port,
-                        password=password,
-                        ssl=tls_enabled,
-                        ssl_cert_reqs=ssl_cert_reqs,
-                        ssl_keyfile=tls_key,
-                        ssl_certfile=tls_cert,
-                        ssl_ca_certs=tls_cacert,
-                        ssl_check_hostname=False,
-                    )
+
+                    # Build Redis connection parameters
+                    redis_params = {
+                        "host": host,
+                        "port": port,
+                        "password": password,
+                        "ssl": tls_enabled,
+                        "ssl_cert_reqs": ssl_cert_reqs,
+                        "ssl_check_hostname": False,
+                    }
+
+                    # Only add SSL certificate parameters if they are provided
+                    if tls_enabled:
+                        if tls_key is not None and tls_key != "":
+                            redis_params["ssl_keyfile"] = tls_key
+                        if tls_cert is not None and tls_cert != "":
+                            redis_params["ssl_certfile"] = tls_cert
+                        if tls_cacert is not None and tls_cacert != "":
+                            redis_params["ssl_ca_certs"] = tls_cacert
+
+                    r = redis.StrictRedis(**redis_params)
                     setup_name = "oss-standalone"
                     r.ping()
 
@@ -1456,17 +1466,26 @@ def delete_temporary_files(
                             slot_network_info = slot[2]
                             prefered_endpoint = slot_network_info[0]
                             prefered_port = slot_network_info[1]
-                            shard_conn = redis.StrictRedis(
-                                host=prefered_endpoint,
-                                port=prefered_port,
-                                password=password,
-                                ssl=tls_enabled,
-                                ssl_cert_reqs=ssl_cert_reqs,
-                                ssl_keyfile=tls_key,
-                                ssl_certfile=tls_cert,
-                                ssl_ca_certs=tls_cacert,
-                                ssl_check_hostname=False,
-                            )
+                            # Build shard connection parameters
+                            shard_params = {
+                                "host": prefered_endpoint,
+                                "port": prefered_port,
+                                "password": password,
+                                "ssl": tls_enabled,
+                                "ssl_cert_reqs": ssl_cert_reqs,
+                                "ssl_check_hostname": False,
+                            }
+
+                            # Only add SSL certificate parameters if they are provided
+                            if tls_enabled:
+                                if tls_key is not None and tls_key != "":
+                                    shard_params["ssl_keyfile"] = tls_key
+                                if tls_cert is not None and tls_cert != "":
+                                    shard_params["ssl_certfile"] = tls_cert
+                                if tls_cacert is not None and tls_cacert != "":
+                                    shard_params["ssl_ca_certs"] = tls_cacert
+
+                            shard_conn = redis.StrictRedis(**shard_params)
                             redis_conns.append(shard_conn)
                         logging.info(
                             "There are a total of {} shards".format(len(redis_conns))
@@ -1475,8 +1494,12 @@ def delete_temporary_files(
 
                     redis_pids = []
                     for conn in redis_conns:
-                        redis_pid = conn.info()["process_id"]
-                        redis_pids.append(redis_pid)
+                        redis_info = conn.info()
+                        redis_pid = redis_info.get("process_id")
+                        if redis_pid is not None:
+                            redis_pids.append(redis_pid)
+                        else:
+                            logging.warning("Redis process_id not found in INFO command, skipping PID collection for this connection")
 
                     # Check if all tested commands are supported by this Redis instance
                     supported_commands = get_supported_redis_commands(redis_conns)
@@ -2991,6 +3014,11 @@ def get_supported_redis_commands(redis_conns):
                 f"Retrieved {len(supported_commands)} supported Redis commands"
             )
 
+            # Handle case where COMMAND returns 0 commands (likely not supported)
+            if len(supported_commands) == 0:
+                logging.warning("COMMAND returned 0 commands - likely not supported by this Redis instance")
+                return None
+
             # Log some sample commands for debugging
             if supported_commands:
                 sample_commands = sorted(list(supported_commands))[:10]
@@ -3013,8 +3041,8 @@ def get_supported_redis_commands(redis_conns):
 
 def check_test_command_support(benchmark_config, supported_commands):
     """Check if all tested-commands in the benchmark config are supported"""
-    if supported_commands is None:
-        logging.warning("No supported commands list available, skipping command check")
+    if supported_commands is None or len(supported_commands) == 0:
+        logging.warning("No supported commands list available (COMMAND not supported or returned 0 commands), skipping command check")
         return True, []
 
     if "tested-commands" not in benchmark_config:

redis_benchmarks_specification/__self_contained_coordinator__/clients.py

@@ -1,10 +1,14 @@
+import logging
+
+
 def prepare_memtier_benchmark_parameters(
     clientconfig,
     full_benchmark_path,
     port,
     server,
     local_benchmark_output_filename,
     oss_cluster_api_enabled,
+    password=None,
 ):
     benchmark_command = [
         full_benchmark_path,
@@ -15,6 +19,12 @@ def prepare_memtier_benchmark_parameters(
         "--server",
         "{}".format(server),
     ]
+
+    # Add password authentication if provided
+    if password is not None and password != "":
+        benchmark_command.extend(["--authenticate", password])
+        logging.info("Memtier benchmark will use password authentication")
+
     if oss_cluster_api_enabled is True:
         benchmark_command.append("--cluster-mode")
     benchmark_command_str = " ".join(benchmark_command)

redis_benchmarks_specification/__self_contained_coordinator__/docker.py

@@ -8,9 +8,9 @@
 
 
 def generate_standalone_redis_server_args(
-    binary, port, dbdir, configuration_parameters=None, redis_arguments=""
+    binary, port, dbdir, configuration_parameters=None, redis_arguments="", password=None
 ):
-    added_params = ["port", "protected-mode", "dir"]
+    added_params = ["port", "protected-mode", "dir", "requirepass"]
     # start redis-server
     command = [
         binary,
@@ -19,6 +19,11 @@ def generate_standalone_redis_server_args(
         "--port",
         "{}".format(port),
     ]
+
+    # Add password authentication if provided
+    if password is not None and password != "":
+        command.extend(["--requirepass", password])
+        logging.info("Redis server will be started with password authentication")
     if dbdir != "":
         command.extend(["--dir", dbdir])
     if configuration_parameters is not None:
@@ -59,13 +64,16 @@ def spin_docker_standalone_redis(
     redis_proc_start_port,
     run_image,
     temporary_dir,
+    password=None,
 ):
     mnt_point = "/mnt/redis/"
     command = generate_standalone_redis_server_args(
         "{}redis-server".format(mnt_point),
         redis_proc_start_port,
         mnt_point,
         redis_configuration_parameters,
+        "",
+        password,
     )
     command_str = " ".join(command)
     db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus(

redis_benchmarks_specification/__self_contained_coordinator__/runners.py

@@ -118,6 +118,8 @@ def process_self_contained_coordinator_stream(
     verbose=False,
     run_tests_with_dataset=False,
 ):
+    # Use a default password for coordinator Redis instances
+    redis_password = "redis_coordinator_password_2024"
     stream_id = "n/a"
     overall_result = False
     total_test_suite_runs = 0
@@ -276,6 +278,7 @@ def process_self_contained_coordinator_stream(
                                     redis_proc_start_port,
                                     run_image,
                                     temporary_dir,
+                                    redis_password,
                                 )
                             else:
                                 shard_count = 1
@@ -307,11 +310,15 @@ def process_self_contained_coordinator_stream(
                                         )
                                     )
 
-                            r = redis.StrictRedis(port=redis_proc_start_port)
+                            r = redis.StrictRedis(port=redis_proc_start_port, password=redis_password)
                             r.ping()
                             redis_pids = []
-                            first_redis_pid = r.info()["process_id"]
-                            redis_pids.append(first_redis_pid)
+                            redis_info = r.info()
+                            first_redis_pid = redis_info.get("process_id")
+                            if first_redis_pid is not None:
+                                redis_pids.append(first_redis_pid)
+                            else:
+                                logging.warning("Redis process_id not found in INFO command")
                             ceil_client_cpu_limit = extract_client_cpu_limit(
                                 benchmark_config
                             )
@@ -385,6 +392,7 @@ def process_self_contained_coordinator_stream(
                                     "localhost",
                                     local_benchmark_output_filename,
                                     benchmark_tool_workdir,
+                                    redis_password,
                                 )
                             elif "vector_db_benchmark" in benchmark_tool:
                                 (

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -496,6 +496,9 @@ def process_self_contained_coordinator_stream(
     restore_build_artifacts_default=True,
     args=None,
 ):
+    # Use a default password for coordinator Redis instances
+    redis_password = "redis_coordinator_password_2024"
+
     stream_id = "n/a"
     overall_result = False
     total_test_suite_runs = 0
@@ -861,6 +864,7 @@ def process_self_contained_coordinator_stream(
                                     mnt_point,
                                     redis_configuration_parameters,
                                     redis_arguments,
+                                    redis_password,
                                 )
                                 command_str = " ".join(command)
                                 db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus(
@@ -876,13 +880,16 @@ def process_self_contained_coordinator_stream(
                                     temporary_dir,
                                 )
 
-                                r = redis.StrictRedis(port=redis_proc_start_port)
+                                r = redis.StrictRedis(port=redis_proc_start_port, password=redis_password)
                                 r.ping()
                                 redis_conns = [r]
                                 reset_commandstats(redis_conns)
                                 redis_pids = []
                                 redis_info = r.info()
-                                first_redis_pid = redis_info["process_id"]
+                                first_redis_pid = redis_info.get("process_id")
+                                if first_redis_pid is None:
+                                    logging.warning("Redis process_id not found in INFO command")
+                                    first_redis_pid = "unknown"
                                 if git_hash is None and "redis_git_sha1" in redis_info:
                                     git_hash = redis_info["redis_git_sha1"]
                                     if (
@@ -972,7 +979,7 @@ def process_self_contained_coordinator_stream(
                                         full_benchmark_path,
                                         redis_proc_start_port,
                                         "localhost",
-                                        None,
+                                        redis_password,
                                         local_benchmark_output_filename,
                                         False,
                                         False,
@@ -1718,7 +1725,7 @@ def data_prepopulation_step(
             full_benchmark_path,
             port,
             "localhost",
-            None,
+            redis_password,
             local_benchmark_output_filename,
             False,
         )