diff --git a/commands-priority.json b/commands-priority.json index 2274d198..0ddbb7bc 100644 --- a/commands-priority.json +++ b/commands-priority.json @@ -1,289 +1,155 @@ { - "set": 1, - "get": 2, - "mget": 3, - "hget": 4, - "hmget": 5, - "hgetall": 6, - "hset": 7, - "zrangebyscore": 8, - "multi": 9, - "exec": 10, - "zadd": 11, - "zcard": 12, - "zrevrangebyscore": 13, - "evalsha": 14, - "expire": 15, - "zscore": 16, - "bf.exists": 17, - "hmset": 18, - "exists": 19, - "bitfield": 20, - "hincrby": 21, - "getbit": 22, - "rpoplpush": 23, - "ping": 24, - "ttl": 25, - "sadd": 26, - "smembers": 27, - "spop": 28, - "del": 29, - "bf.insert": 30, - "setex": 31, - "sismember": 32, - "rpush": 33, - "lrange": 34, - "zrem": 35, - "incrby": 36, - "expireat": 37, - "publish": 38, - "lpush": 39, - "brpop": 40, - "eval": 41, - "pexpire": 42, - "lpop": 43, - "scan": 44, - "zrevrank": 45, - "zrange": 46, - "llen": 47, - "hexists": 48, - "lrem": 49, - "brpoplpush": 50, - "hdel": 51, - "zremrangebyscore": 52, - "scard": 53, - "xadd": 54, - "replconf": 55, - "hvals": 56, - "incr": 57, - "zincrby": 58, - "hincrbyfloat": 59, - "zremrangebyrank": 60, - "zrevrange": 61, - "srem": 62, - "time": 63, - "blpop": 64, - "rpop": 65, - "xreadgroup": 66, - "hscan": 67, - "client": 68, - "unlink": 69, - "zrank": 70, - "json.set": 71, - "pfadd": 72, - "hlen": 73, - "type": 74, - "sscan": 75, - "select": 76, - "lindex": 77, - "setnx": 78, - "srandmember": 79, - "ltrim": 80, - "hsetnx": 81, - "setbit": 82, - "pttl": 83, - "pexpireat": 84, - "bf.mexists": 85, - "xack": 86, - "psetex": 87, - "zscan": 88, - "getset": 89, - "xread": 90, - "persist": 91, - "xtrim": 92, - "decrby": 93, - "xdel": 94, - "json.get": 95, - "bzpopmin": 96, - "touch": 97, - "decr": 98, - "zcount": 99, - "xrange": 100, - "ts.add": 101, - "incrbyfloat": 102, - "json.mget": 103, - "ft.search": 104, - "xpending": 105, - "rpushx": 106, - "sunion": 107, - "sinter": 108, - "dbsize": 109, - "mset": 110, - "sinterstore": 111, - "hkeys": 112, - "restore": 113, - "ts.madd": 114, - "smismember": 115, - "geoadd": 116, - "ts.info": 117, - "zpopmin": 118, - "echo": 119, - "lmove": 120, - "xclaim": 121, - "ft.sugadd": 122, - "_ft.sugadd": 123, - "lset": 124, - "xlen": 125, - "randomkey": 126, - "pfcount": 127, - "append": 128, - "graph.query": 129, - "zinterstore": 130, - "rename": 131, - "dump": 132, - "ft.get": 133, - "_ft.get": 134, - "ts.range": 135, - "_ft.search": 136, - "zunionstore": 137, - "json.type": 138, + "get": 1, + "sismember": 2, + "hget": 3, + "mget": 4, + "evalsha": 5, + "expire": 6, + "getbit": 7, + "hgetall": 8, + "hmget": 9, + "hdel": 10, + "hmset": 11, + "set": 12, + "hset": 13, + "exists": 14, + "multi": 15, + "exec": 16, + "smembers": 17, + "zadd": 18, + "del": 19, + "hincrby": 20, + "zscore": 21, + "setex": 22, + "sadd": 23, + "publish": 24, + "hincrbyfloat": 25, + "zrangebyscore": 26, + "zrange": 27, + "incrbyfloat": 28, + "time": 29, + "incr": 30, + "ping": 31, + "lrange": 32, + "zcard": 33, + "hello": 34, + "pexpire": 35, + "ttl": 36, + "rpoplpush": 37, + "llen": 38, + "eval": 39, + "rpush": 40, + "zrem": 41, + "hexists": 42, + "srem": 43, + "psetex": 44, + "incrby": 45, + "zremrangebyscore": 46, + "xadd": 47, + "expireat": 48, + "pfadd": 49, + "touch": 50, + "zrevrank": 51, + "scard": 52, + "lpop": 53, + "hlen": 54, + "getset": 55, + "xreadgroup": 56, + "spop": 57, + "randomkey": 58, + "scan": 59, + "lpush": 60, + "getex": 61, + "xrange": 62, + "zincrby": 63, + "rpop": 64, + "zcount": 65, + "zrank": 66, + "replconf": 67, + "pexpireat": 68, + "xrevrange": 69, + "lrem": 70, + "brpop": 71, + "xread": 72, + "pttl": 73, + "zrevrange": 74, + "zpopmin": 75, + "zrevrangebyscore": 76, + "ltrim": 77, + "hscan": 78, + "xpending": 79, + "smismember": 80, + "lmove": 81, + "hvals": 82, + "xack": 83, + "hsetnx": 84, + "unlink": 85, + "brpoplpush": 86, + "select": 87, + "setnx": 88, + "mset": 89, + "type": 90, + "lindex": 91, + "decr": 92, + "hkeys": 93, + "zremrangebyrank": 94, + "setbit": 95, + "xdel": 96, + "georadius_ro": 97, + "getdel": 98, + "blpop": 99, + "persist": 100, + "rpushx": 101, + "echo": 102, + "blmove": 103, + "discard": 104, + "getrange": 105, + "sinterstore": 106, + "bzpopmin": 107, + "decrby": 108, + "xlen": 109, + "xtrim": 110, + "zscan": 111, + "pfcount": 112, + "client": 113, + "sscan": 114, + "restore": 115, + "srandmember": 116, + "rename": 117, + "pubsub": 118, + "fcall_ro": 119, + "geoadd": 120, + "renamenx": 121, + "bitfield": 122, + "xclaim": 123, + "smove": 124, + "xautoclaim": 125, + "lset": 126, + "append": 127, + "zrangebylex": 128, + "dump": 129, + "dbsize": 130, + "zpopmax": 131, + "lpos": 132, + "zunion": 133, + "sinter": 134, + "sunionstore": 135, + "zmscore": 136, + "zinterstore": 137, + "sunion": 138, "bitpos": 139, - "ts.get": 140, - "getrange": 141, - "sunionstore": 142, - "getex": 143, - "ts.revrange": 144, - "graph.ro_query": 145, - "blmove": 146, - "zrangebylex": 147, - "json.del": 148, - "pubsub": 149, - "sort": 150, - "bitcount": 151, - "getdel": 152, - "ts.mrange": 153, - "setrange": 154, - "msetnx": 155, - "bf.add": 156, - "renamenx": 157, - "linsert": 158, - "sdiff": 159, - "geopos": 160, - "ts.alter": 161, - "georadius": 162, - "xrevrange": 163, - "zlexcount": 164, - "pfmerge": 165, - "strlen": 166, - "json.objkeys": 167, - "zremrangebylex": 168, - "ft.add": 169, - "lpos": 170, - "json.numincrby": 171, - "georadiusbymember": 172, - "ft.aggregate": 173, - "_ft.cursor": 174, - "_ft.aggregate": 175, - "sdiffstore": 176, - "hrandfield": 177, - "ts.incrby": 178, - "ts.queryindex": 179, - "zmscore": 180, - "ft.mget": 181, - "_ft.mget": 182, - "json.arrappend": 183, - "flushdb": 184, - "zpopmax": 185, - "zrevrangebylex": 186, - "ts.create": 187, - "ft.tagvals": 188, - "ts.mget": 189, - "discard": 190, - "_ft.add": 191, - "bf.madd": 192, - "_ft.tagvals": 193, - "ft.del": 194, - "ft.create": 195, - "json.arrpop": 196, - "json.objlen": 197, - "ft.dropindex": 198, - "ts.createrule": 199, - "xautoclaim": 200, - "json.arrinsert": 201, - "cf.mexists": 202, - "psync": 203, - "cf.addnx": 204, - "bzpopmax": 205, - "zdiffstore": 206, - "smove": 207, - "flushall": 208, - "ft.info": 209, - "zrandmember": 210, - "_ft.info": 211, - "georadius_ro": 212, - "ft.alter": 213, - "zrangestore": 214, - "restoremodaux": 215, - "_ft.del": 216, - "cf.del": 217, - "copy": 218, - "json.resp": 219, - "cms.incrby": 220, - "cf.reserve": 221, - "cf.info": 222, - "cf.insert": 223, - "geosearch": 224, - "xsetid": 225, - "graph.delete": 226, - "ft.sugget": 227, - "ts.decrby": 228, - "ft._list": 229, - "bf.reserve": 230, - "json.arrlen": 231, - "zinter": 232, - "json.debug": 233, - "bf.info": 234, - "timeseries.clusterset": 235, - "_ft._list": 236, - "ft.syndump": 237, - "cf.exists": 238, - "cf.insertnx": 239, - "geodist": 240, - "ft.drop": 241, - "zunion": 242, - "graph.list": 243, - "cf.add": 244, - "ts.del": 245, - "ts.mrevrange": 246, - "ft._createifnx": 247, - "ft._dropindexifx": 248, - "search.clusterset": 249, - "ft.cursor": 250, - "acl": 251, - "ft.sugdel": 252, - "ft.aliasupdate": 253, - "cms.initbydim": 254, - "json.strlen": 255, - "ft.explain": 256, - "ft.profile": 257, - "graph.profile": 258, - "lolwut": 259, - "geohash": 260, - "cms.query": 261, - "graph.explain": 262, - "json.strappend": 263, - "ft._alterifnx": 264, - "bf.debug": 265, - "graph.slowlog": 266, - "ft.dictdump": 267, - "bitop": 268, - "timeseries.hello": 269, - "ft.spellcheck": 270, - "ft.aliasdel": 271, - "cf.debug": 272, - "_ft.profile": 273, - "graph.bulk": 274, - "hstrlen": 275, - "cf.count": 276, - "ft.aliasadd": 277, - "topk.reserve": 278, - "topk.query": 279, - "topk.list": 280, - "topk.info": 281, - "topk.incrby": 282, - "topk.add": 283, - "ft.suglen": 284, - "cms.initbyprob": 285, - "cms.info": 286, - "_ft.create": 287 + "linsert": 140, + "strlen": 141, + "sdiffstore": 142, + "hrandfield": 143, + "geopos": 144, + "bitcount": 145, + "zunionstore": 146, + "setrange": 147, + "sdiff": 148, + "georadius": 149, + "msetnx": 150, + "hexpire": 151, + "geosearch": 152, + "expiretime": 153 } \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index f06bd072..8fa7cf05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.256" +version = "0.1.259" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__cli__/stats.py b/redis_benchmarks_specification/__cli__/stats.py index adf6f2e9..dbca65ad 100644 --- a/redis_benchmarks_specification/__cli__/stats.py +++ b/redis_benchmarks_specification/__cli__/stats.py @@ -8,6 +8,7 @@ from redis_benchmarks_specification.__common__.runner import get_benchmark_specs + # logging settings logging.basicConfig( format="%(asctime)s %(levelname)-4s %(message)s", @@ -16,6 +17,42 @@ ) +def clean_number(value): + """Cleans and converts numeric values from CSV, handling B (billion), M (million), K (thousand).""" + try: + value = value.replace(",", "").strip() # Remove commas and spaces + + # Determine the scale factor + multiplier = 1 + if value.endswith("B"): + multiplier = 1_000_000_000 # Billion + value = value[:-1] # Remove "B" + elif value.endswith("M"): + multiplier = 1_000_000 # Million + value = value[:-1] # Remove "M" + elif value.endswith("K"): + multiplier = 1_000 # Thousand + value = value[:-1] # Remove "K" + + return int(float(value) * multiplier) # Convert to full number + except ValueError: + logging.error(f"Skipping invalid count value: {value}") + return 0 # Default to 0 if invalid + + +def get_arg_value(args, flag, default): + """Extract integer values safely from CLI arguments""" + if flag in args: + try: + val = ( + args[args.index(flag) + 1].lstrip("=").strip() + ) # Remove any leading '=' + return int(val) # Convert to integer safely + except (IndexError, ValueError): + logging.error(f"Failed to extract {flag}, using default: {default}") + return default # Return default if not found or invalid + + def generate_stats_cli_command_logic(args, project_name, project_version): logging.info( "Using: {project_name} {project_version}".format( @@ -55,10 +92,14 @@ def generate_stats_cli_command_logic(args, project_name, project_version): ) priority_json = json.load(fd) tracked_groups = [] + tracked_groups_hist = {} override_enabled = args.override_tests fail_on_required_diff = args.fail_on_required_diff overall_result = True test_names = [] + pipelines = {} + connections = {} + data_sizes = {} defaults_filename = args.defaults_filename for test_file in testsuite_spec_files: @@ -83,6 +124,13 @@ def generate_stats_cli_command_logic(args, project_name, project_version): test_names.append(test_name) group = "" is_memtier = False + + ## defaults + pipeline_size = 1 + clients = 50 + threads = 4 + data_size = 32 + if "memtier" in test_name: is_memtier = True tested_groups = [] @@ -101,6 +149,32 @@ def generate_stats_cli_command_logic(args, project_name, project_version): tested_commands.append(tested_command.lower()) if is_memtier: arguments = benchmark_config["clientconfig"]["arguments"] + arg_list = ( + benchmark_config["clientconfig"]["arguments"] + .replace('"', "") + .split() + ) + + data_size = get_arg_value(arg_list, "--data-size", data_size) + data_size = get_arg_value(arg_list, "-d", data_size) + + # Extract values using the safer parsing function + pipeline_size = get_arg_value(arg_list, "--pipeline", pipeline_size) + pipeline_size = get_arg_value( + arg_list, "-P", pipeline_size + ) # Support short form + + # Extract values using the safer parsing function + clients = get_arg_value(arg_list, "--clients", clients) + clients = get_arg_value( + arg_list, "-c", clients + ) # Support short form + + threads = get_arg_value(arg_list, "--threads", threads) + threads = get_arg_value( + arg_list, "-t", threads + ) # Support short form + arguments_split = arguments.split("--command") if len(arguments_split) == 1: @@ -133,9 +207,27 @@ def generate_stats_cli_command_logic(args, project_name, project_version): group = command_json["group"] if group not in tested_groups: + tested_groups.append(group) if group not in tracked_groups: tracked_groups.append(group) + tracked_groups_hist[group] = 0 + tracked_groups_hist[group] = tracked_groups_hist[group] + 1 + + # Calculate total connections + total_connections = clients * threads + + if pipeline_size not in pipelines: + pipelines[pipeline_size] = 0 + pipelines[pipeline_size] = pipelines[pipeline_size] + 1 + + if total_connections not in connections: + connections[total_connections] = 0 + connections[total_connections] = connections[total_connections] + 1 + + if data_size not in data_sizes: + data_sizes[data_size] = 0 + data_sizes[data_size] = data_sizes[data_size] + 1 if tested_commands != origin_tested_commands: requires_override = True @@ -281,10 +373,10 @@ def generate_stats_cli_command_logic(args, project_name, project_version): if "cmdstat_" not in cmdstat: continue cmdstat = cmdstat.replace("cmdstat_", "") - count = int(row[1]) + count = clean_number(row[1]) usecs = None if len(row) > 2: - usecs = int(row[2]) + usecs = clean_number(row[2]) total_usecs += usecs if count == 0: continue @@ -470,11 +562,15 @@ def generate_stats_cli_command_logic(args, project_name, project_version): logging.info("Top 10 fully tracked?: {}".format(len(top_10_missing) == 0)) logging.info("Top 30 fully tracked?: {}".format(len(top_30_missing) == 0)) if len(top_30_missing) > 0: - logging.info("\t\tTotal missing for Top 30: {}".format(len(top_30_missing))) + logging.info( + f"\t\tTotal missing for Top 30: {len(top_30_missing)}. {top_30_missing}" + ) logging.info("Top 50 fully tracked?: {}".format(len(top_50_missing) == 0)) if len(top_50_missing) > 0: - logging.info("\t\tTotal missing for Top 50: {}".format(len(top_50_missing))) + logging.info( + f"\t\tTotal missing for Top 50: {len(top_50_missing)}. {top_50_missing}" + ) if overall_result is False and fail_on_required_diff: logging.error( @@ -500,3 +596,71 @@ def generate_stats_cli_command_logic(args, project_name, project_version): conn.sadd(tested_groups_key, group) for command in list(tracked_commands_json.keys()): conn.sadd(tested_commands_key, command) + + logging.info(f"There is a total of : {len(tracked_groups)} tracked command groups.") + logging.info( + f"There is a total of : {len(list(tracked_commands_json.keys()))} tracked commands." + ) + # Save pipeline count to CSV + csv_filename = "memtier_pipeline_count.csv" + with open(csv_filename, "w", newline="") as csvfile: + fieldnames = ["pipeline", "count"] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + for pipeline_size in sorted(pipelines.keys()): + writer.writerow( + {"pipeline": pipeline_size, "count": pipelines[pipeline_size]} + ) + + logging.info(f"Pipeline count data saved to {csv_filename}") + + csv_filename = "memtier_connection_count.csv" + with open(csv_filename, "w", newline="") as csvfile: + fieldnames = ["connections", "count"] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + + # Sort connections dictionary by keys before writing + for connection_count in sorted(connections.keys()): + writer.writerow( + { + "connections": connection_count, + "count": connections[connection_count], + } + ) + + logging.info(f"Sorted connection count data saved to {csv_filename}") + + csv_filename = "memtier_data_size_histogram.csv" + with open(csv_filename, "w", newline="") as csvfile: + fieldnames = ["data_size", "count"] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + + # Sort connections dictionary by keys before writing + for data_size in sorted(data_sizes.keys()): + writer.writerow( + { + "data_size": data_size, + "count": data_sizes[data_size], + } + ) + + logging.info(f"Sorted data size count data saved to {csv_filename}") + + csv_filename = "memtier_groups_histogram.csv" + with open(csv_filename, "w", newline="") as csvfile: + fieldnames = ["group", "count"] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + + # Sort connections dictionary by keys before writing + for group in sorted(tracked_groups_hist.keys()): + writer.writerow( + { + "group": group, + "count": tracked_groups_hist[group], + } + ) + + logging.info(f"Sorted command groups count data saved to {csv_filename}") diff --git a/redis_benchmarks_specification/__compare__/args.py b/redis_benchmarks_specification/__compare__/args.py index df087fd5..fc5ace11 100644 --- a/redis_benchmarks_specification/__compare__/args.py +++ b/redis_benchmarks_specification/__compare__/args.py @@ -8,6 +8,10 @@ import datetime import os +from redis_benchmarks_specification.__common__.env import ( + SPECS_PATH_TEST_SUITES, +) + def get_start_time_vars(start_time=None): if start_time is None: @@ -30,6 +34,12 @@ def get_start_time_vars(start_time=None): def create_compare_arguments(parser): + parser.add_argument( + "--test-suites-folder", + type=str, + default=SPECS_PATH_TEST_SUITES, + help="Test suites folder, containing the different test variations", + ) parser.add_argument( "--test", type=str, diff --git a/redis_benchmarks_specification/__compare__/compare.py b/redis_benchmarks_specification/__compare__/compare.py index 864e98e3..a8f53231 100644 --- a/redis_benchmarks_specification/__compare__/compare.py +++ b/redis_benchmarks_specification/__compare__/compare.py @@ -27,6 +27,7 @@ ) from redis_benchmarks_specification.__compare__.args import create_compare_arguments +from redis_benchmarks_specification.__common__.runner import get_benchmark_specs from redis_benchmarks_specification.__common__.package import ( get_version_string, @@ -268,6 +269,40 @@ def compare_command_logic(args, project_name, project_version): ) ) + testsuites_folder = os.path.abspath(args.test_suites_folder) + logging.info("Using test-suites folder dir {}".format(testsuites_folder)) + testsuite_spec_files = get_benchmark_specs(testsuites_folder) + logging.info( + "There are a total of {} test-suites being run in folder {}".format( + len(testsuite_spec_files), testsuites_folder + ) + ) + tests_with_config = {} + for test_file in testsuite_spec_files: + if args.defaults_filename in test_file: + continue + benchmark_config = {} + with open(test_file, "r") as stream: + try: + benchmark_config = yaml.safe_load(stream) + test_name = benchmark_config["name"] + tests_with_config[test_name] = benchmark_config + if "tested-groups" in benchmark_config: + origin_tested_groups = benchmark_config["tested-groups"] + else: + logging.warn("dont have test groups in {}".format(test_name)) + if "tested-commands" in benchmark_config: + origin_tested_commands = benchmark_config["tested-commands"] + else: + logging.warn("dont have test commands in {}".format(test_name)) + except Exception as e: + logging.error( + "while loading file {} and error was returned: {}".format( + test_file, e.__str__() + ) + ) + pass + fn = check_regression_comment ( contains_regression_comment, @@ -330,6 +365,7 @@ def compare_command_logic(args, project_name, project_version): comparison_github_org, args.regression_str, args.improvement_str, + tests_with_config, ) total_regressions = len(regressions_list) total_improvements = len(improvements_list) @@ -588,6 +624,7 @@ def compute_regression_table( comparison_github_org="redis", regression_str="REGRESSION", improvement_str="IMPROVEMENT", + tests_with_config={}, ): START_TIME_NOW_UTC, _, _ = get_start_time_vars() START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31) @@ -668,6 +705,8 @@ def compute_regression_table( baseline_only_list, comparison_only_list, no_datapoints_list, + group_change, + command_change, ) = from_rts_to_regression_table( baseline_deployment_name, comparison_deployment_name, @@ -698,6 +737,7 @@ def compute_regression_table( comparison_github_org, regression_str, improvement_str, + tests_with_config, ) logging.info( "Printing differential analysis between {} and {}".format( @@ -712,6 +752,22 @@ def compute_regression_table( baseline_deployment_name, ) + table_output += "
\n By GROUP change csv:\n\n" + table_output += "\ncommand_group,min_change,max_change \n" + for group_name, changes_list in group_change.items(): + max_change = max(changes_list) + min_change = min(changes_list) + table_output += f"{group_name},{min_change:.3f},{max_change:.3f}\n" + table_output += "\n
\n" + table_output += "\n\n" + table_output += "
\n By COMMAND change csv:\n\n" + table_output += "\ncommand,min_change,max_change \n" + for command_name, changes_list in command_change.items(): + max_change = max(changes_list) + min_change = min(changes_list) + table_output += f"{command_name},{min_change:.3f},{max_change:.3f}\n" + table_output += "\n
\n" + if total_unstable > 0: old_stdout = sys.stdout sys.stdout = mystdout = StringIO() @@ -999,6 +1055,7 @@ def from_rts_to_regression_table( comparison_github_org="redis", regression_str="REGRESSION", improvement_str="IMPROVEMENT", + tests_with_config={}, ): print_all = print_regressions_only is False and print_improvements_only is False table_full = [] @@ -1022,8 +1079,20 @@ def from_rts_to_regression_table( no_datapoints_list = [] no_datapoints_baseline_list = [] no_datapoints_comparison_list = [] + group_change = {} + command_change = {} original_metric_mode = metric_mode for test_name in test_names: + tested_groups = [] + tested_commands = [] + if test_name in tests_with_config: + test_spec = tests_with_config[test_name] + if "tested-groups" in test_spec: + tested_groups = test_spec["tested-groups"] + if "tested-commands" in test_spec: + tested_commands = test_spec["tested-commands"] + else: + logging.error(f"Test does not contain spec info: {test_name}") metric_mode = original_metric_mode compare_version = "main" # GE @@ -1257,6 +1326,16 @@ def from_rts_to_regression_table( if simplify_table is False: note = note + " No Change" + for test_group in tested_groups: + if test_group not in group_change: + group_change[test_group] = [] + group_change[test_group].append(percentage_change) + + for test_command in tested_commands: + if test_command not in command_change: + command_change[test_command] = [] + command_change[test_command].append(percentage_change) + if ( detected_improvement is False and detected_regression is False @@ -1345,6 +1424,8 @@ def from_rts_to_regression_table( baseline_only_list, comparison_only_list, no_datapoints_list, + group_change, + command_change, ) diff --git a/redis_benchmarks_specification/__runner__/args.py b/redis_benchmarks_specification/__runner__/args.py index e58f56a8..96f1f0fa 100644 --- a/redis_benchmarks_specification/__runner__/args.py +++ b/redis_benchmarks_specification/__runner__/args.py @@ -146,6 +146,12 @@ def create_client_runner_args(project_name): action="store_true", help="Run tests that contain a dbconfig with dataset", ) + parser.add_argument( + "--skip-tests-with-preload-via-tool", + default=False, + action="store_true", + help="Run tests that contain a dbconfig with dataset", + ) parser.add_argument( "--client_aggregated_results_folder", type=str, diff --git a/redis_benchmarks_specification/__runner__/runner.py b/redis_benchmarks_specification/__runner__/runner.py index 2ba57d7d..127fb381 100644 --- a/redis_benchmarks_specification/__runner__/runner.py +++ b/redis_benchmarks_specification/__runner__/runner.py @@ -9,7 +9,7 @@ import traceback from pathlib import Path import re - +import tqdm import docker import redis from docker.models.containers import Container @@ -391,7 +391,7 @@ def delete_temporary_files( _, ) = get_defaults(defaults_filename) - for test_file in testsuite_spec_files: + for test_file in tqdm.tqdm(testsuite_spec_files): if defaults_filename in test_file: continue client_containers = [] @@ -605,6 +605,19 @@ def delete_temporary_files( benchmark_tool_global=benchmark_tool_global, ) continue + if "preload_tool" in benchmark_config["dbconfig"]: + if args.skip_tests_with_preload_via_tool is False: + logging.warning( + "Skipping test {} giving it implies dataset preload via tool".format( + test_name + ) + ) + delete_temporary_files( + temporary_dir_client=temporary_dir_client, + full_result_path=None, + benchmark_tool_global=benchmark_tool_global, + ) + continue if dry_run is True: dry_run_count = dry_run_count + 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-10B-psetex-expire-use-case.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-10B-psetex-expire-use-case.yml new file mode 100644 index 00000000..5f935955 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-10B-psetex-expire-use-case.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-10B-psetex-expire-use-case +description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 10 Bytes for each key, which 50% of the keys have expiration set during the benchmark. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "10" "--command" "SET __key__ __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + resources: + requests: + memory: 1g +tested-groups: +- string +tested-commands: +- psetex +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "10" --command "PSETEX __key__ 10 __data__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 120' + resources: + requests: + cpus: '3' + memory: 2g + +priority: 33 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-10B-setex-expire-use-case.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-10B-setex-expire-use-case.yml new file mode 100644 index 00000000..4852b9b4 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-10B-setex-expire-use-case.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-10B-setex-expire-use-case +description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 10 Bytes for each key, which 50% of the keys have expiration set during the benchmark. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "10" "--command" "SET __key__ __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + resources: + requests: + memory: 1g +tested-groups: +- string +tested-commands: +- setex +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "10" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 120' + resources: + requests: + cpus: '3' + memory: 2g + +priority: 33 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-exists-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-exists-pipeline-10.yml index 13a4fe37..2e2573e3 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-exists-pipeline-10.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-exists-pipeline-10.yml @@ -9,7 +9,7 @@ dbconfig: preload_tool: run_image: redislabs/memtier_benchmark:edge tool: memtier_benchmark - arguments: '"--data-size" "100" "--command" "HSET __key__ field __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + arguments: '"--pipeline" "100" "--data-size" "100" "--command" "HSET __key__ field __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' resources: requests: memory: 1g diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-expire-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-expire-pipeline-10.yml index 62dc39ae..f2a3acbf 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-expire-pipeline-10.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-expire-pipeline-10.yml @@ -9,7 +9,7 @@ dbconfig: preload_tool: run_image: redislabs/memtier_benchmark:edge tool: memtier_benchmark - arguments: '"--data-size" "100" "--command" "HSET __key__ field __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + arguments: '"--pipeline" "100" "--data-size" "100" "--command" "HSET __key__ field __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' resources: requests: memory: 1g diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-expireat-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-expireat-pipeline-10.yml index 7d9ba6fc..8b4f85c3 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-expireat-pipeline-10.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-expireat-pipeline-10.yml @@ -9,7 +9,7 @@ dbconfig: preload_tool: run_image: redislabs/memtier_benchmark:edge tool: memtier_benchmark - arguments: '"--data-size" "100" "--command" "HSET __key__ field __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + arguments: '"--pipeline" "100" "--data-size" "100" "--command" "HSET __key__ field __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' resources: requests: memory: 1g diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-pexpire-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-pexpire-pipeline-10.yml index b8245bc8..b3c7011d 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-pexpire-pipeline-10.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-pexpire-pipeline-10.yml @@ -9,7 +9,7 @@ dbconfig: preload_tool: run_image: redislabs/memtier_benchmark:edge tool: memtier_benchmark - arguments: '"--data-size" "100" "--command" "HSET __key__ field __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + arguments: '"--pipeline" "100" "--data-size" "100" "--command" "HSET __key__ field __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' resources: requests: memory: 1g diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-scan-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-scan-pipeline-10.yml index f3b79d9c..4625927d 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-scan-pipeline-10.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-generic-scan-pipeline-10.yml @@ -9,7 +9,7 @@ dbconfig: preload_tool: run_image: redislabs/memtier_benchmark:edge tool: memtier_benchmark - arguments: '"--data-size" "100" "--command" "HSET __key__ field __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + arguments: '"--pipeline" "100" "--data-size" "100" "--command" "HSET __key__ field __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' resources: requests: memory: 1g diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-hash-hincrbyfloat.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-hash-hincrbyfloat.yml new file mode 100644 index 00000000..7cb36243 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-hash-hincrbyfloat.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-hash-hincrbyfloat +description: Runs memtier_benchmark, for a keyspace length of 1M keys loading HASHES with 5 fields each. Each field value has a data size of 1000 Bytes. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "1000" --command "HSET __key__ field1 __data__ field2 __data__ field3 __data__ field4 __data__ field5 __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 -n 5000 -c 50 -t 4 --hide-histogram' + resources: + requests: + memory: 6g +tested-groups: +- hash +tested-commands: +- hincrbyfloat +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 180 --command "HINCRBYFLOAT __key__ field1 0.999999" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 21 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-list-rpoplpush-with-10B-values.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-list-rpoplpush-with-10B-values.yml new file mode 100644 index 00000000..36781c7c --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-list-rpoplpush-with-10B-values.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-list-rpoplpush-with-10B-values +description: Runs memtier_benchmark, for a keyspace length of 1M keys pre-loading LISTs in which the value has a data size of 10 Bytes. After pre-loading the data it issues LPOP and RPOP commands. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "10" --command "LPUSH __key__ __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 60 -c 50 -t 4 --hide-histogram' + resources: + requests: + memory: 2g +tested-groups: +- list +tested-commands: +- rpoplpush +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "10" --command "RPOPLPUSH __key__ myotherlist" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 --test-time 120 -c 50 -t 4 --hide-histogram' + resources: + requests: + cpus: '4' + memory: 2g + +priority: 55 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-list-rpush-with-10B-values.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-list-rpush-with-10B-values.yml new file mode 100644 index 00000000..0edb428e --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-list-rpush-with-10B-values.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-load-list-rpush-with-10B-values +description: Runs memtier_benchmark, for a keyspace length of 1M keys loading LISTs in which the value has a data size of 10 Bytes. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 1g +tested-groups: +- list +tested-commands: +- rpush +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "10" --command "RPUSH __key__ __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram' + resources: + requests: + cpus: '4' + memory: 2g + +priority: 38 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-10.yml index 52443b26..49d2bb82 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-10.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-10.yml @@ -19,10 +19,10 @@ build-variants: clientconfig: run_image: redislabs/memtier_benchmark:edge tool: memtier_benchmark - arguments: '"--pipeline" "10" "--data-size" "10" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram' + arguments: '"--pipeline" "10" "--data-size" "10" --distinct-client-seed --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 120 -c 10 -t 10 --hide-histogram' resources: requests: - cpus: '4' + cpus: '10' memory: 2g tested-groups: diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-100.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-100.yml new file mode 100644 index 00000000..07e3bef2 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-100.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-100 +description: Runs memtier_benchmark, for a keyspace length of 1M keys loading STRINGs in which the value has a data size of 10 Bytes. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 1g +tested-commands: +- set +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--pipeline" "100" "--data-size" "10" --distinct-client-seed --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 120 -c 10 -t 10 --hide-histogram' + resources: + requests: + cpus: '10' + memory: 2g + +tested-groups: +- string +priority: 17 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-50.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-50.yml new file mode 100644 index 00000000..98a0c782 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-50.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-50 +description: Runs memtier_benchmark, for a keyspace length of 1M keys loading STRINGs in which the value has a data size of 10 Bytes. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 1g +tested-commands: +- set +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--pipeline" "50" "--data-size" "10" --distinct-client-seed --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 120 -c 10 -t 10 --hide-histogram' + resources: + requests: + cpus: '10' + memory: 2g + +tested-groups: +- string +priority: 17 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-500.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-500.yml new file mode 100644 index 00000000..45e6edd7 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-500.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-500 +description: Runs memtier_benchmark, for a keyspace length of 1M keys loading STRINGs in which the value has a data size of 10 Bytes. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 1g +tested-commands: +- set +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--pipeline" "500" "--data-size" "10" --distinct-client-seed --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 120 -c 10 -t 10 --hide-histogram' + resources: + requests: + cpus: '10' + memory: 2g + +tested-groups: +- string +priority: 17 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-100B-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-100B-pipeline-10.yml index b92ba6d9..0abf4988 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-100B-pipeline-10.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-100B-pipeline-10.yml @@ -23,7 +23,7 @@ build-variants: clientconfig: run_image: redislabs/memtier_benchmark:edge tool: memtier_benchmark - arguments: '"--pipeline" "10" "--data-size" "100" --ratio 0:1 --key-pattern R:R -c 25 -t 4 --hide-histogram --test-time 180' + arguments: '--pipeline 10 --data-size 100 --ratio 0:1 --key-pattern R:R -c 25 -t 4 --hide-histogram --test-time 180' resources: requests: cpus: '4' diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-10.yml index 457ec036..a17577c7 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-10.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-10.yml @@ -23,10 +23,10 @@ build-variants: clientconfig: run_image: redislabs/memtier_benchmark:edge tool: memtier_benchmark - arguments: '"--key-minimum" "1" "--key-maximum" "1000000" "--pipeline" "10" "--data-size" "10" --ratio 0:1 --key-pattern R:R -c 25 -t 4 --hide-histogram --test-time 180' + arguments: '"--pipeline" "10" --distinct-client-seed "--data-size" "10" --ratio 0:1 --key-pattern R:R -c 10 -t 10 --hide-histogram --test-time 120' resources: requests: - cpus: '4' + cpus: '10' memory: 2g tested-groups: diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-100.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-100.yml new file mode 100644 index 00000000..88463051 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-100.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-get-10B-pipeline-100 +description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 10 Bytes for each key. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--key-maximum" "1000000" "-n" "allkeys" "--data-size" "10" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + resources: + requests: + memory: 1g +tested-commands: +- get +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--pipeline" "100" --distinct-client-seed "--data-size" "10" --ratio 0:1 --key-pattern R:R -c 10 -t 10 --hide-histogram --test-time 120' + resources: + requests: + cpus: '10' + memory: 2g + +tested-groups: +- string +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-50.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-50.yml new file mode 100644 index 00000000..315e7c26 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-50.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-get-10B-pipeline-50 +description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 10 Bytes for each key. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--key-maximum" "1000000" "-n" "allkeys" "--data-size" "10" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + resources: + requests: + memory: 1g +tested-commands: +- get +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--pipeline" "50" --distinct-client-seed "--data-size" "10" --ratio 0:1 --key-pattern R:R -c 10 -t 10 --hide-histogram --test-time 120' + resources: + requests: + cpus: '10' + memory: 2g + +tested-groups: +- string +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-500.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-500.yml new file mode 100644 index 00000000..72efc843 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-500.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-get-10B-pipeline-500 +description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 10 Bytes for each key. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--key-maximum" "1000000" "-n" "allkeys" "--data-size" "10" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + resources: + requests: + memory: 1g +tested-commands: +- get +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--pipeline" "500" --distinct-client-seed "--data-size" "10" --ratio 0:1 --key-pattern R:R -c 10 -t 10 --hide-histogram --test-time 120' + resources: + requests: + cpus: '10' + memory: 2g + +tested-groups: +- string +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B.yml index 2568d88b..516c999f 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B.yml @@ -23,10 +23,10 @@ build-variants: clientconfig: run_image: redislabs/memtier_benchmark:edge tool: memtier_benchmark - arguments: '"--data-size" "10" --ratio 0:1 --key-pattern R:R -c 25 -t 4 --hide-histogram --test-time 180' + arguments: '"--data-size" "10" --distinct-client-seed --ratio 0:1 --key-pattern R:R -c 10 -t 10 --hide-histogram --test-time 120' resources: requests: - cpus: '4' + cpus: '10' memory: 2g tested-groups: diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incr-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incr-pipeline-10.yml new file mode 100644 index 00000000..57098f5d --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incr-pipeline-10.yml @@ -0,0 +1,28 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-incr-pipeline-10 +description: Runs memtier_benchmark, for a keyspace length of 1M keys doing incr of 1. +dbconfig: + configuration-parameters: + save: '""' + resources: + requests: + memory: 1g +tested-groups: +- string +tested-commands: +- incr +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --pipeline 10 --command "INCR __key__" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 21 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-int-encoding-strlen-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-int-encoding-strlen-pipeline-10.yml new file mode 100644 index 00000000..1de3e44c --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-int-encoding-strlen-pipeline-10.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-int-encoding-strlen-pipeline-10 +description: Runs memtier_benchmark, for a keyspace length of 1M keys int encoded, checking STRLEN command performance. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 999998 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: ' --command "SET __key__ __key__" --key-prefix "" --command-key-pattern="S" --key-minimum=2 --key-maximum 1000000 -c 1 -t 1 --pipeline 100 --hide-histogram -n allkeys' + resources: + requests: + memory: 1g +tested-commands: +- strlen +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command "STRLEN __key__" --key-prefix "" --command-key-pattern="G" --key-minimum=2 --key-maximum 1000000 -c 50 -t 4 --pipeline 10 --hide-histogram --test-time 60 + resources: + requests: + cpus: '4' + memory: 2g + +tested-groups: +- string +priority: 98 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-100-elements-llen-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-100-elements-llen-pipeline-10.yml new file mode 100644 index 00000000..7f0b7fa4 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-list-100-elements-llen-pipeline-10.yml @@ -0,0 +1,32 @@ +version: 0.4 +name: memtier_benchmark-1key-list-100-elements-llen-pipeline-10 +description: 'Runs memtier_benchmark, for a keyspace length of 1 LIST key. The LIST contains 100 elements in it and we query its size using LLEN. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + init_commands: + - '"LPUSH" "list:100" "vyoomgwuzv" "xamjodnbpf" "ewomnmugfa" "ljcgdooafo" "pcxdhdjwnf" "djetcyfxuc" "licotqplim" "alqlzsvuuz" "ijsmoyesvd" "whmotknaff" "rkaznetutk" "ksqpdywgdd" "gorgpnnqwr" "gekntrykfh" "rjkknoigmu" "luemuetmia" "gxephxbdru" "ncjfckgkcl" "hhjclfbbka" "cgoeihlnei" "zwnitejtpg" "upodnpqenn" "mibvtmqxcy" "htvbwmfyic" "rqvryfvlie" "nxcdcaqgit" "gfdqdrondm" "lysbgqqfqw" "nxzsnkmxvi" "nsxaigrnje" "cwaveajmcz" "xsepfhdizi" "owtkxlzaci" "agsdggdghc" "tcjvjofxtd" "kgqrovsxce" "ouuybhtvyb" "ueyrvldzwl" "vpbkvwgxsf" "pytrnqdhvs" "qbiwbqiubb" "ssjqrsluod" "urvgxwbiiz" "ujrxcmpvsq" "mtccjerdon" "xczfmrxrja" "imyizmhzjk" "oguwnmniig" "mxwgdcutnb" "pqyurbvifk" "ccagtnjilc" "mbxohpancs" "lgrkndhekf" "eqlgkwosie" "jxoxtnzujs" "lbtpbknelm" "ichqzmiyot" "mbgehjiauu" "aovfsvbwjg" "nmgxcctxpn" "vyqqkuszzh" "rojeolnopp" "ibhohmfxzt" "qbyhorvill" "nhfnbxqgol" "wkbasfyzqz" "mjjuylgssm" "imdqxmkzdj" "oapbvnisyq" "bqntlsaqjb" "ocrcszcznp" "hhniikmtsx" "hlpdstpvzw" "wqiwdbncmt" "vymjzlzqcn" "hhjchwjlmc" "ypfeltycpy" "qjyeqcfhjj" "uapsgmizgh" "owbbdezgxn" "qrosceblyo" "sahqeskveq" "dapacykoah" "wvcnqbvlnf" "perfwnpvkl" "ulbrotlhze" "fhuvzpxjbc" "holjcdpijr" "onzjrteqmu" "pquewclxuy" "vpmpffdoqz" "eouliovvra" "vxcbagyymm" "jekkafodvk" "ypekeuutef" "dlbqcynhrn" "erxulvebrj" "qwxrsgafzy" "dlsjwmqzhx" "exvhmqxvvp"' +tested-groups: +- list +tested-commands: +- llen +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --pipeline 10 --command="LLEN list:100" --hide-histogram --test-time 60 + resources: + requests: + cpus: '4' + memory: 2g + +priority: 34 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-srem-50pct-chance.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-srem-50pct-chance.yml new file mode 100644 index 00000000..307d8920 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-srem-50pct-chance.yml @@ -0,0 +1,33 @@ +version: 0.4 +name: memtier_benchmark-1key-set-10M-elements-srem-50pct-chance +description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key with 10M elements. We query it using SISMEMBER in which the value has 50% change of being member. ' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + resources: + requests: + memory: 1g + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SADD set:10M:elements __key__" --command-key-pattern=P --key-maximum 10000000 --key-prefix "" -n 10000000 --hide-histogram -t 1 -c 1 +tested-groups: +- set +tested-commands: +- srem +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="SREM set:10M:elements __key__" --key-maximum 20000000 --key-prefix "" --hide-histogram --test-time 120 + resources: + requests: + cpus: '4' + memory: 2g +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-200K-elements-sadd-constant.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-200K-elements-sadd-constant.yml index 59c8023e..c505ee96 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-200K-elements-sadd-constant.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-200K-elements-sadd-constant.yml @@ -12,7 +12,7 @@ dbconfig: preload_tool: run_image: redislabs/memtier_benchmark:edge tool: memtier_benchmark - arguments: --command="SADD myset __key__" --command-key-pattern=P --key-maximum 200000 --key-prefix "" -n 200000 --hide-histogram -t 1 -c 1 + arguments: --command="SADD myset __key__" --command-key-pattern=P --pipeline 100 --key-maximum 200000 --key-prefix "" -n 200000 --hide-histogram -t 1 -c 1 tested-groups: - set tested-commands: diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-1M-elements-zremrangebyscore-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-1M-elements-zremrangebyscore-pipeline-10.yml new file mode 100644 index 00000000..bd0629a2 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-1M-elements-zremrangebyscore-pipeline-10.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1key-zset-1M-elements-zremrangebyscore-pipeline-10 +description: 'Runs memtier_benchmark, for a keyspace length of 1 SORTED SET key. The SORTED SET contains 1M elements in it and we query it using ZREVRANGE with a range of 5 elements. This benchmarks helps assessing: https://github.com/redis/redis/issues/10310' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --key-maximum 1000000 --key-prefix "" --command="ZADD lb __key__ __key__" --command-key-pattern P --hide-histogram -t 4 -c 100 + resources: + requests: + memory: 1g +tested-groups: +- sorted-set +tested-commands: +- zremrangebyscore +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command="ZREMRANGEBYSCORE lb __key__ __key__" --hide-histogram --test-time 120 --pipeline 10 + resources: + requests: + cpus: '4' + memory: 2g + +priority: 8 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-connection-ping-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-connection-ping-pipeline-10.yml new file mode 100644 index 00000000..9ff4086c --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-connection-ping-pipeline-10.yml @@ -0,0 +1,28 @@ +version: 0.4 +name: memtier_benchmark-nokeys-connection-ping-pipeline-10 +description: Runs memtier_benchmark, for a empty keyspace doing the TIME command. +dbconfig: + configuration-parameters: + save: '""' + resources: + requests: + memory: 1g +tested-groups: +- connection +tested-commands: +- ping +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --pipeline 10 --command "PING" --command-key-pattern="R" -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 21 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-pubsub-publish-1K-channels-10B-no-subscribers.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-pubsub-publish-1K-channels-10B-no-subscribers.yml new file mode 100644 index 00000000..9fe634e8 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-pubsub-publish-1K-channels-10B-no-subscribers.yml @@ -0,0 +1,28 @@ +version: 0.4 +name: memtier_benchmark-nokeys-pubsub-publish-1K-channels-10B-no-subscribers +description: Runs memtier_benchmark, for a empty keyspace doing the PUBSUB's PUBLISH command with no subscribers. +dbconfig: + configuration-parameters: + save: '""' + resources: + requests: + memory: 1g +tested-groups: +- pubsub +tested-commands: +- publish +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --pipeline 10 -d 10 --key-maximum 1000 --command "PUBLISH __key__ __data__" --command-key-pattern="R" -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 21 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-server-time-pipeline-10.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-server-time-pipeline-10.yml new file mode 100644 index 00000000..4fb4c501 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-server-time-pipeline-10.yml @@ -0,0 +1,28 @@ +version: 0.4 +name: memtier_benchmark-nokeys-server-time-pipeline-10 +description: Runs memtier_benchmark, for a empty keyspace doing the TIME command. +dbconfig: + configuration-parameters: + save: '""' + resources: + requests: + memory: 1g +tested-groups: +- server +tested-commands: +- time +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --pipeline 10 --command "TIME" --command-key-pattern="R" -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 21