diff --git a/pyproject.toml b/pyproject.toml index 696d974b..9dab74ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.219" +version = "0.1.222" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__builder__/builder.py b/redis_benchmarks_specification/__builder__/builder.py index 70c0132e..4cfcec13 100644 --- a/redis_benchmarks_specification/__builder__/builder.py +++ b/redis_benchmarks_specification/__builder__/builder.py @@ -644,11 +644,11 @@ def generate_benchmark_stream_request( build_stream_fields["git_version"] = git_version if git_timestamp_ms is not None: build_stream_fields["git_timestamp_ms"] = git_timestamp_ms + + prefix = f"github_org={github_org}/github_repo={github_repo}/git_branch={str(git_branch)}/git_version={str(git_version)}/git_hash={str(git_hash)}" for artifact in build_artifacts: - bin_key = "zipped:artifacts:{}:{}.zip".format(id, artifact) - bin_artifact = open( - "{}src/{}".format(redis_temporary_dir, artifact), "rb" - ).read() + bin_key = f"zipped:artifacts:{prefix}:{id}:{artifact}.zip" + bin_artifact = open(f"{redis_temporary_dir}src/{artifact}", "rb").read() bin_artifact_len = len(bytes(bin_artifact)) assert bin_artifact_len > 0 conn.set(bin_key, bytes(bin_artifact), ex=REDIS_BINS_EXPIRE_SECS) diff --git a/redis_benchmarks_specification/__compare__/args.py b/redis_benchmarks_specification/__compare__/args.py index 3d24498b..dc7b8f46 100644 --- a/redis_benchmarks_specification/__compare__/args.py +++ b/redis_benchmarks_specification/__compare__/args.py @@ -44,6 +44,8 @@ def create_compare_arguments(parser): ) parser.add_argument("--github_repo", type=str, default="redis") parser.add_argument("--github_org", type=str, default="redis") + parser.add_argument("--baseline_github_org", type=str, default="") + parser.add_argument("--comparison_github_org", type=str, default="") parser.add_argument("--triggering_env", type=str, default="ci") parser.add_argument("--github_token", type=str, default=PERFORMANCE_GH_TOKEN) parser.add_argument("--pull-request", type=str, default=None, nargs="?", const="") diff --git a/redis_benchmarks_specification/__compare__/compare.py b/redis_benchmarks_specification/__compare__/compare.py index 13b2ff0f..699006e2 100644 --- a/redis_benchmarks_specification/__compare__/compare.py +++ b/redis_benchmarks_specification/__compare__/compare.py @@ -256,6 +256,8 @@ def compare_command_logic(args, project_name, project_version): comparison_target_branch = args.comparison_target_branch baseline_github_repo = args.baseline_github_repo comparison_github_repo = args.comparison_github_repo + baseline_github_org = args.baseline_github_org + comparison_github_org = args.comparison_github_org baseline_hash = args.baseline_hash comparison_hash = args.comparison_hash @@ -324,6 +326,8 @@ def compare_command_logic(args, project_name, project_version): comparison_github_repo, baseline_target_branch, comparison_target_branch, + baseline_github_org, + comparison_github_org, ) prepare_regression_comment( auto_approve, @@ -555,6 +559,8 @@ def compute_regression_table( comparison_github_repo="redis", baseline_target_branch=None, comparison_target_branch=None, + baseline_github_org="redis", + comparison_github_org="redis", ): START_TIME_NOW_UTC, _, _ = get_start_time_vars() START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31) @@ -657,6 +663,8 @@ def compute_regression_table( running_platform, baseline_github_repo, comparison_github_repo, + baseline_github_org, + comparison_github_org, ) logging.info( "Printing differential analysis between {} and {}".format( @@ -909,6 +917,8 @@ def from_rts_to_regression_table( running_platform=None, baseline_github_repo="redis", comparison_github_repo="redis", + baseline_github_org="redis", + comparison_github_org="redis", ): print_all = print_regressions_only is False and print_improvements_only is False table_full = [] @@ -942,6 +952,8 @@ def from_rts_to_regression_table( "github_repo={}".format(baseline_github_repo), "triggering_env={}".format(tf_triggering_env), ] + if baseline_github_org != "": + filters_baseline.append(f"github_org={baseline_github_org}") if running_platform is not None: filters_baseline.append("running_platform={}".format(running_platform)) filters_comparison = [ @@ -952,6 +964,8 @@ def from_rts_to_regression_table( "github_repo={}".format(comparison_github_repo), "triggering_env={}".format(tf_triggering_env), ] + if comparison_github_org != "": + filters_baseline.append(f"github_org={comparison_github_org}") if "hash" not in by_str_baseline: filters_baseline.append("hash==") if "hash" not in by_str_comparison: diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-append-1-100B.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-append-1-100B.yml new file mode 100644 index 00000000..72c46d28 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-append-1-100B.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-append-1-100B +description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 1 Bytes for each key initially. Then we append to the same keys with values ranging from 1 to 100B. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "1" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + resources: + requests: + memory: 1g +tested-commands: +- append +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --data-size-range=1-100 --command "APPEND __key__ __data__" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +tested-groups: +- string +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrby.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrby.yml new file mode 100644 index 00000000..89498a77 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrby.yml @@ -0,0 +1,28 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-incrby +description: Runs memtier_benchmark, for a keyspace length of 1M keys doing incrby of 1. +dbconfig: + configuration-parameters: + save: '""' + resources: + requests: + memory: 1g +tested-groups: +- string +tested-commands: +- incrby +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --command "INCRBY __key__ 1" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 21 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrbyfloat.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrbyfloat.yml new file mode 100644 index 00000000..02291212 --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrbyfloat.yml @@ -0,0 +1,28 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-incrbyfloat +description: Runs memtier_benchmark, for a keyspace length of 1M keys doing incrby of 0.01. +dbconfig: + configuration-parameters: + save: '""' + resources: + requests: + memory: 1g +tested-groups: +- string +tested-commands: +- incrbyfloat +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --command "INCRBYFLOAT __key__ 0.01" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +priority: 21 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setrange-100B.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setrange-100B.yml new file mode 100644 index 00000000..4200f2ff --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setrange-100B.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-string-setrange-100B +description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 100 Bytes for each key. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "100" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + resources: + requests: + memory: 1g +tested-commands: +- setrange +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --test-time 120 --data-size-range=1-95 --command "SETRANGE __key__ 5 __data__" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram + resources: + requests: + cpus: '4' + memory: 2g + +tested-groups: +- string +priority: 1 diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml index 9db858f5..249ad60c 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml @@ -1,6 +1,14 @@ version: 0.4 name: memtier_benchmark-connection-hello description: 'Runs memtier_benchmark, for no keyspace, benchmarking the connection setup scenario using HELLO command.' +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 1g tested-groups: - connection tested-commands: