diff --git a/pyproject.toml b/pyproject.toml index faa2d994..60211881 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.213" +version = "0.1.214" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__runner__/runner.py b/redis_benchmarks_specification/__runner__/runner.py index d9ce31ec..2ba57d7d 100644 --- a/redis_benchmarks_specification/__runner__/runner.py +++ b/redis_benchmarks_specification/__runner__/runner.py @@ -255,6 +255,7 @@ def prepare_memtier_benchmark_parameters( if oss_cluster_api_enabled is True: benchmark_command.append("--cluster-mode") + logging.info(f"Preparing the benchmark parameters. {benchmark_command}.") benchmark_command_str = " ".join(benchmark_command) if "arguments" in clientconfig: benchmark_command_str = benchmark_command_str + " " + clientconfig["arguments"] diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index eaf5ea5f..696e751e 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -1596,11 +1596,13 @@ def data_prepopulation_step( ( _, preload_command_str, + _, ) = prepare_memtier_benchmark_parameters( benchmark_config["dbconfig"]["preload_tool"], full_benchmark_path, port, "localhost", + None, local_benchmark_output_filename, False, ) diff --git a/utils/tests/test_data/test-suites/generic-touch.yml b/utils/tests/test_data/test-suites/generic-touch.yml new file mode 100644 index 00000000..12f5f94a --- /dev/null +++ b/utils/tests/test_data/test-suites/generic-touch.yml @@ -0,0 +1,34 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-generic-touch-pipeline-10 +description: Runs memtier_benchmark, for a keyspace length of 1M keys focusing on TOUCH performance. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 1000000 + preload_tool: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "100" "--command" "SETEX __key__ 3600 __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + resources: + requests: + memory: 1g +tested-commands: +- touch +tested-groups: +- generic +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: --command "TOUCH __key__" --pipeline 10 --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 180 + resources: + requests: + cpus: '2' + memory: 2g + +priority: 41 diff --git a/utils/tests/test_runner.py b/utils/tests/test_runner.py index 3fcad3a2..833575f7 100644 --- a/utils/tests/test_runner.py +++ b/utils/tests/test_runner.py @@ -258,7 +258,7 @@ def test_extract_testsuites(): ] ) tests = extract_testsuites(args) - assert len(tests) == 6 + assert len(tests) == 7 args = parser.parse_args( args=[ @@ -269,7 +269,7 @@ def test_extract_testsuites(): ] ) tests = extract_testsuites(args) - assert len(tests) == 6 + assert len(tests) == 7 args = parser.parse_args( args=[ diff --git a/utils/tests/test_self_contained_coordinator_memtier.py b/utils/tests/test_self_contained_coordinator_memtier.py index f5015fa1..8a325bb8 100644 --- a/utils/tests/test_self_contained_coordinator_memtier.py +++ b/utils/tests/test_self_contained_coordinator_memtier.py @@ -226,6 +226,132 @@ def run_coordinator_tests_dockerhub(): return run_coordinator +def test_self_contained_coordinator_dockerhub_preload(): + try: + if run_coordinator_tests_dockerhub(): + db_port = int(os.getenv("DATASINK_PORT", "6379")) + conn = redis.StrictRedis(port=db_port) + conn.ping() + conn.flushall() + + id = "dockerhub" + redis_version = "7.4.0" + run_image = f"redis:{redis_version}" + build_arch = "amd64" + testDetails = {} + build_os = "test_build_os" + build_stream_fields, result = generate_benchmark_stream_request( + id, + conn, + run_image, + build_arch, + testDetails, + build_os, + ) + build_stream_fields["mnt_point"] = "" + if result is True: + benchmark_stream_id = conn.xadd( + STREAM_KEYNAME_NEW_BUILD_EVENTS, build_stream_fields + ) + logging.info( + "sucessfully requested a new run {}. Stream id: {}".format( + build_stream_fields, benchmark_stream_id + ) + ) + + build_variant_name = "gcc:8.5.0-amd64-debian-buster-default" + expected_datapoint_ts = None + + assert conn.exists(STREAM_KEYNAME_NEW_BUILD_EVENTS) + assert conn.xlen(STREAM_KEYNAME_NEW_BUILD_EVENTS) > 0 + running_platform = "fco-ThinkPad-T490" + + build_runners_consumer_group_create(conn, running_platform, "0") + datasink_conn = redis.StrictRedis(port=db_port) + docker_client = docker.from_env() + home = str(Path.home()) + stream_id = ">" + topologies_map = get_topologies( + "./redis_benchmarks_specification/setups/topologies/topologies.yml" + ) + # we use a benchmark spec with smaller CPU limit for client given github machines only contain 2 cores + # and we need 1 core for DB and another for CLIENT + testsuite_spec_files = [ + "./utils/tests/test_data/test-suites/generic-touch.yml" + ] + defaults_filename = "./utils/tests/test_data/test-suites/defaults.yml" + ( + _, + _, + default_metrics, + _, + _, + _, + ) = get_defaults(defaults_filename) + + ( + result, + stream_id, + number_processed_streams, + num_process_test_suites, + ) = self_contained_coordinator_blocking_read( + conn, + True, + docker_client, + home, + stream_id, + datasink_conn, + testsuite_spec_files, + topologies_map, + running_platform, + False, + [], + "", + 0, + 6399, + 1, + False, + 5, + default_metrics, + "amd64", + None, + 0, + 10000, + "unstable", + "", + True, + False, + ) + + assert result == True + assert number_processed_streams == 1 + assert num_process_test_suites == 1 + by_version_key = f"ci.benchmarks.redislabs/ci/redis/redis/memtier_benchmark-1Mkeys-generic-touch-pipeline-10/by.version/{redis_version}/benchmark_end/oss-standalone/memory_maxmemory" + assert datasink_conn.exists(by_version_key) + rts = datasink_conn.ts() + # check we have by version metrics + assert "version" in rts.info(by_version_key).labels + assert redis_version == rts.info(by_version_key).labels["version"] + + # get all keys + all_keys = datasink_conn.keys("*") + by_hash_keys = [] + for key in all_keys: + if "/by.hash/" in key.decode(): + by_hash_keys.append(key) + + # ensure we have by hash keys + assert len(by_hash_keys) > 0 + for hash_key in by_hash_keys: + # ensure we have both version and hash info on the key + assert "version" in rts.info(hash_key).labels + assert "hash" in rts.info(hash_key).labels + assert redis_version == rts.info(hash_key).labels["version"] + + except redis.exceptions.ConnectionError: + pass + + def test_self_contained_coordinator_dockerhub(): try: if run_coordinator_tests_dockerhub():