From 7c4876c9e7e9b0c4e7405121a7979a5e24a01026 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Tue, 13 Aug 2024 10:49:26 +0100 Subject: [PATCH 1/6] Added memtier_benchmark-3Mkeys-load-string-with-512B-values: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 512 Bytes, with 650 clients running sequential SET commands --- ...rk-3Mkeys-load-string-with-512B-values.yml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml new file mode 100644 index 00000000..10de3caa --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-3Mkeys-load-string-with-512B-values +description: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 512 Bytes, with 650 clients running sequential SET commands. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 3g +tested-commands: +- set +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "512" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 3000000 --test-time 180 -c 50 -t 13 --hide-histogram' + resources: + requests: + cpus: '13' + memory: 2g + +tested-groups: +- string +priority: 17 From 46ea9173739e94265172215f10b79b2abc06b21b Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Wed, 14 Aug 2024 02:35:17 +0100 Subject: [PATCH 2/6] Added tests for io-threads setups --- .../__common__/spec.py | 52 +++++++ .../__self_contained_coordinator__/docker.py | 6 +- .../__self_contained_coordinator__/runners.py | 2 +- .../self_contained_coordinator.py | 21 ++- .../setups/topologies/topologies.yml | 65 +++++++++ ...rk-3Mkeys-load-string-with-512B-values.yml | 6 + .../test-memtier-dockerhub-iothreads.yml | 30 +++++ .../test_data/test-suites/topologies.yml | 22 +++ .../tests/test_self_contained_coordinator.py | 4 + ...test_self_contained_coordinator_memtier.py | 127 ++++++++++++++++++ utils/tests/test_spec.py | 11 +- utils/tests/test_topologies.py | 15 +++ 12 files changed, 354 insertions(+), 7 deletions(-) create mode 100644 utils/tests/test_data/test-suites/test-memtier-dockerhub-iothreads.yml create mode 100644 utils/tests/test_data/test-suites/topologies.yml diff --git a/redis_benchmarks_specification/__common__/spec.py b/redis_benchmarks_specification/__common__/spec.py index 14d4c0f3..67495a70 100644 --- a/redis_benchmarks_specification/__common__/spec.py +++ b/redis_benchmarks_specification/__common__/spec.py @@ -1,4 +1,56 @@ import math +import logging + + +def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname): + redis_configuration_parameters = {} + modules_configuration_parameters_map = {} + dataset_load_timeout_secs = 120 + dataset_name = None + dbconfig_present = False + if dbconfig_keyname in benchmark_config: + dbconfig_present = True + if type(benchmark_config[dbconfig_keyname]) == list: + for k in benchmark_config[dbconfig_keyname]: + if "configuration-parameters" in k: + cp = k["configuration-parameters"] + for item in cp: + for k, v in item.items(): + redis_configuration_parameters[k] = v + if "dataset_load_timeout_secs" in k: + dataset_load_timeout_secs = k["dataset_load_timeout_secs"] + if "dataset_name" in k: + dataset_name = k["dataset_name"] + if type(benchmark_config[dbconfig_keyname]) == dict: + if "configuration-parameters" in benchmark_config[dbconfig_keyname]: + cp = benchmark_config[dbconfig_keyname]["configuration-parameters"] + for k, v in cp.items(): + redis_configuration_parameters[k] = v + if "dataset_load_timeout_secs" in benchmark_config[dbconfig_keyname]: + dataset_load_timeout_secs = benchmark_config[dbconfig_keyname][ + "dataset_load_timeout_secs" + ] + if "dataset_name" in benchmark_config[dbconfig_keyname]: + dataset_name = benchmark_config[dbconfig_keyname]["dataset_name"] + + return ( + dbconfig_present, + dataset_name, + redis_configuration_parameters, + dataset_load_timeout_secs, + modules_configuration_parameters_map, + ) + + +def extract_redis_configuration_from_topology(topologies_map, topology_spec_name): + redis_arguments = "" + topology_spec = topologies_map[topology_spec_name] + if "redis_arguments" in topology_spec: + redis_arguments = topology_spec["redis_arguments"] + logging.info( + f"extracted redis_arguments: {redis_arguments} from topology: {topology_spec_name}" + ) + return redis_arguments def extract_client_cpu_limit(benchmark_config): diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/docker.py b/redis_benchmarks_specification/__self_contained_coordinator__/docker.py index 96bc5739..ca3e9f67 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/docker.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/docker.py @@ -8,7 +8,7 @@ def generate_standalone_redis_server_args( - binary, port, dbdir, configuration_parameters=None + binary, port, dbdir, configuration_parameters=None, redis_arguments="" ): added_params = ["port", "protected-mode", "dir"] # start redis-server @@ -30,6 +30,10 @@ def generate_standalone_redis_server_args( parameter_value, ] ) + if redis_arguments != "": + redis_arguments_arr = redis_arguments.split(" ") + logging.info(f"adding redis arguments {redis_arguments_arr}") + command.extend(redis_arguments_arr) return command diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/runners.py b/redis_benchmarks_specification/__self_contained_coordinator__/runners.py index cc3f3983..8bb29922 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/runners.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/runners.py @@ -32,7 +32,6 @@ from redisbench_admin.run.run import calculate_client_tool_duration_and_check from redisbench_admin.utils.benchmark_config import ( get_final_benchmark_config, - extract_redis_dbconfig_parameters, ) from redisbench_admin.utils.local import get_local_run_full_filename from redisbench_admin.utils.results import post_process_benchmark_results @@ -47,6 +46,7 @@ extract_client_cpu_limit, extract_client_tool, extract_client_container_image, + extract_redis_dbconfig_parameters, ) from redis_benchmarks_specification.__self_contained_coordinator__.artifacts import ( restore_build_artifacts_from_test_details, diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index 696e751e..3b7f48d8 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -80,7 +80,6 @@ from redisbench_admin.run.run import calculate_client_tool_duration_and_check from redisbench_admin.utils.benchmark_config import ( get_final_benchmark_config, - extract_redis_dbconfig_parameters, get_defaults, ) from redisbench_admin.utils.local import get_local_run_full_filename @@ -95,6 +94,8 @@ extract_client_cpu_limit, extract_client_tool, extract_client_container_image, + extract_redis_dbconfig_parameters, + extract_redis_configuration_from_topology, ) from redis_benchmarks_specification.__self_contained_coordinator__.artifacts import ( restore_build_artifacts_from_test_details, @@ -756,6 +757,14 @@ def process_self_contained_coordinator_stream( ) ) for topology_spec_name in benchmark_config["redis-topologies"]: + setup_name = topology_spec_name + setup_type = "oss-standalone" + if topology_spec_name in topologies_map: + topology_spec = topologies_map[topology_spec_name] + setup_type = topology_spec["type"] + logging.info( + f"Running topology named {topology_spec_name} of type {setup_type}" + ) test_result = False redis_container = None try: @@ -763,6 +772,11 @@ def process_self_contained_coordinator_stream( ceil_db_cpu_limit = extract_db_cpu_limit( topologies_map, topology_spec_name ) + redis_arguments = ( + extract_redis_configuration_from_topology( + topologies_map, topology_spec_name + ) + ) temporary_dir = tempfile.mkdtemp(dir=home) temporary_dir_client = tempfile.mkdtemp(dir=home) logging.info( @@ -776,8 +790,6 @@ def process_self_contained_coordinator_stream( ) ) - setup_name = "oss-standalone" - setup_type = "oss-standalone" tf_triggering_env = "ci" github_actor = "{}-{}".format( tf_triggering_env, running_platform @@ -814,6 +826,7 @@ def process_self_contained_coordinator_stream( redis_proc_start_port, mnt_point, redis_configuration_parameters, + redis_arguments, ) command_str = " ".join(command) db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus( @@ -926,7 +939,7 @@ def process_self_contained_coordinator_stream( start_time_str, git_hash, test_name, - "oss-standalone", + setup_name, ) ) logging.info( diff --git a/redis_benchmarks_specification/setups/topologies/topologies.yml b/redis_benchmarks_specification/setups/topologies/topologies.yml index 3b3dbd84..59e9d610 100644 --- a/redis_benchmarks_specification/setups/topologies/topologies.yml +++ b/redis_benchmarks_specification/setups/topologies/topologies.yml @@ -10,6 +10,71 @@ spec: cpus: "1" memory: "10g" + - name: oss-standalone-02-io-threads + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --io-threads 2 --io-threads-do-reads yes + resources: + requests: + cpus: "3" + memory: "10g" + + - name: oss-standalone-04-io-threads + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --io-threads 4 --io-threads-do-reads yes + resources: + requests: + cpus: "5" + memory: "10g" + + - name: oss-standalone-08-io-threads + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --io-threads 8 --io-threads-do-reads yes + resources: + requests: + cpus: "9" + memory: "10g" + + - name: oss-standalone-16-io-threads + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --io-threads 16 --io-threads-do-reads yes + resources: + requests: + cpus: "17" + memory: "10g" + + - name: oss-standalone-32-io-threads + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --io-threads 32 --io-threads-do-reads yes + resources: + requests: + cpus: "33" + memory: "10g" + + - name: oss-standalone-64-io-threads + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --io-threads 64 --io-threads-do-reads yes + resources: + requests: + cpus: "65" + memory: "10g" - name: oss-standalone-1replica type: oss-standalone redis_topology: diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml index 10de3caa..5f33d54c 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml @@ -13,6 +13,12 @@ tested-commands: - set redis-topologies: - oss-standalone +- oss-standalone-02-io-threads +- oss-standalone-04-io-threads +- oss-standalone-08-io-threads +- oss-standalone-16-io-threads +- oss-standalone-32-io-threads +- oss-standalone-64-io-threads build-variants: - gcc:8.5.0-amd64-debian-buster-default - dockerhub diff --git a/utils/tests/test_data/test-suites/test-memtier-dockerhub-iothreads.yml b/utils/tests/test_data/test-suites/test-memtier-dockerhub-iothreads.yml new file mode 100644 index 00000000..c6d5cf11 --- /dev/null +++ b/utils/tests/test_data/test-suites/test-memtier-dockerhub-iothreads.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-load-string-with-10B-values +description: Runs memtier_benchmark, for a keyspace length of 1M keys loading STRINGs in which the value has a data size of 10 Bytes. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 1g +tested-commands: +- set +redis-topologies: +- oss-standalone-02-io-threads +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "10" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram' + resources: + requests: + cpus: "1" + memory: "2g" + +tested-groups: +- string +priority: 17 diff --git a/utils/tests/test_data/test-suites/topologies.yml b/utils/tests/test_data/test-suites/topologies.yml new file mode 100644 index 00000000..0fd43931 --- /dev/null +++ b/utils/tests/test_data/test-suites/topologies.yml @@ -0,0 +1,22 @@ +spec: + setups: + - name: oss-standalone + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + resources: + requests: + cpus: "1" + memory: "10g" + + - name: oss-standalone-02-io-threads + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --io-threads 2 --io-threads-do-reads yes + resources: + requests: + cpus: "1" + memory: "10g" diff --git a/utils/tests/test_self_contained_coordinator.py b/utils/tests/test_self_contained_coordinator.py index 8855de6d..a3a9f31b 100644 --- a/utils/tests/test_self_contained_coordinator.py +++ b/utils/tests/test_self_contained_coordinator.py @@ -263,3 +263,7 @@ def test_self_contained_coordinator_blocking_read(): except redis.exceptions.ConnectionError: pass + + +def test_extract_redis_configuration_from_topology(): + assert False diff --git a/utils/tests/test_self_contained_coordinator_memtier.py b/utils/tests/test_self_contained_coordinator_memtier.py index 8a325bb8..3f5538c7 100644 --- a/utils/tests/test_self_contained_coordinator_memtier.py +++ b/utils/tests/test_self_contained_coordinator_memtier.py @@ -478,6 +478,133 @@ def test_self_contained_coordinator_dockerhub(): pass +def test_self_contained_coordinator_dockerhub_iothreads(): + try: + if run_coordinator_tests_dockerhub(): + db_port = int(os.getenv("DATASINK_PORT", "6379")) + conn = redis.StrictRedis(port=db_port) + conn.ping() + conn.flushall() + + id = "dockerhub" + redis_version = "7.4.0" + run_image = f"redis:{redis_version}" + build_arch = "amd64" + testDetails = {} + build_os = "test_build_os" + build_stream_fields, result = generate_benchmark_stream_request( + id, + conn, + run_image, + build_arch, + testDetails, + build_os, + ) + build_stream_fields["mnt_point"] = "" + if result is True: + benchmark_stream_id = conn.xadd( + STREAM_KEYNAME_NEW_BUILD_EVENTS, build_stream_fields + ) + logging.info( + "sucessfully requested a new run {}. Stream id: {}".format( + build_stream_fields, benchmark_stream_id + ) + ) + + build_variant_name = "gcc:8.5.0-amd64-debian-buster-default" + expected_datapoint_ts = None + + assert conn.exists(STREAM_KEYNAME_NEW_BUILD_EVENTS) + assert conn.xlen(STREAM_KEYNAME_NEW_BUILD_EVENTS) > 0 + running_platform = "fco-ThinkPad-T490" + + build_runners_consumer_group_create(conn, running_platform, "0") + datasink_conn = redis.StrictRedis(port=db_port) + docker_client = docker.from_env() + home = str(Path.home()) + stream_id = ">" + topologies_map = get_topologies( + "./utils/tests/test_data/test-suites/topologies.yml" + ) + # we use a benchmark spec with smaller CPU limit for client given github machines only contain 2 cores + # and we need 1 core for DB and another for CLIENT + testsuite_spec_files = [ + "./utils/tests/test_data/test-suites/test-memtier-dockerhub-iothreads.yml" + ] + defaults_filename = "./utils/tests/test_data/test-suites/defaults.yml" + ( + _, + _, + default_metrics, + _, + _, + _, + ) = get_defaults(defaults_filename) + + ( + result, + stream_id, + number_processed_streams, + num_process_test_suites, + ) = self_contained_coordinator_blocking_read( + conn, + True, + docker_client, + home, + stream_id, + datasink_conn, + testsuite_spec_files, + topologies_map, + running_platform, + False, + [], + "", + 0, + 6399, + 1, + False, + 5, + default_metrics, + "amd64", + None, + 0, + 10000, + "unstable", + "", + True, + False, + ) + + assert result == True + assert number_processed_streams == 1 + assert num_process_test_suites == 1 + by_version_key = f"ci.benchmarks.redislabs/ci/redis/redis/memtier_benchmark-1Mkeys-load-string-with-10B-values/by.version/{redis_version}/benchmark_end/oss-standalone-02-io-threads/memory_maxmemory" + assert datasink_conn.exists(by_version_key) + rts = datasink_conn.ts() + # check we have by version metrics + assert "version" in rts.info(by_version_key).labels + assert redis_version == rts.info(by_version_key).labels["version"] + + # get all keys + all_keys = datasink_conn.keys("*") + by_hash_keys = [] + for key in all_keys: + if "/by.hash/" in key.decode(): + by_hash_keys.append(key) + + # ensure we have by hash keys + assert len(by_hash_keys) > 0 + for hash_key in by_hash_keys: + # ensure we have both version and hash info on the key + assert "version" in rts.info(hash_key).labels + assert "oss-standalone-02-io-threads" in hash_key.decode() + assert "hash" in rts.info(hash_key).labels + assert redis_version == rts.info(hash_key).labels["version"] + + except redis.exceptions.ConnectionError: + pass + + def test_self_contained_coordinator_dockerhub_valkey(): try: if run_coordinator_tests_dockerhub(): diff --git a/utils/tests/test_spec.py b/utils/tests/test_spec.py index 08860833..8c640afe 100644 --- a/utils/tests/test_spec.py +++ b/utils/tests/test_spec.py @@ -11,7 +11,6 @@ def test_extract_build_variant_variations(): - with open( "./utils/tests/test_data/test-suites/redis-benchmark-full-suite-1Mkeys-100B.yml", "r", @@ -27,3 +26,13 @@ def test_extract_build_variant_variations(): benchmark_config = yaml.safe_load(yml_file) build_variants = extract_build_variant_variations(benchmark_config) assert "gcc:8.5.0-amd64-debian-buster-default" in build_variants + + +def test_extract_redis_dbconfig_parameters(): + with open( + "./redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-100B-expire-use-case.yml", + "r", + ) as yml_file: + benchmark_config = yaml.safe_load(yml_file) + build_variants = extract_build_variant_variations(benchmark_config) + assert "gcc:8.5.0-amd64-debian-buster-default" in build_variants diff --git a/utils/tests/test_topologies.py b/utils/tests/test_topologies.py index 6b4aaef0..9071955c 100644 --- a/utils/tests/test_topologies.py +++ b/utils/tests/test_topologies.py @@ -1,3 +1,6 @@ +from redis_benchmarks_specification.__common__.spec import ( + extract_redis_configuration_from_topology, +) from redis_benchmarks_specification.__setups__.topologies import get_topologies @@ -7,3 +10,15 @@ def test_get_topologies(): ) assert "oss-standalone" in topologies_map assert topologies_map["oss-standalone"]["resources"]["requests"]["cpus"] == "1" + + +def test_extract_redis_configuration_from_topology(): + topologies_map = get_topologies( + "./redis_benchmarks_specification/setups/topologies/topologies.yml" + ) + assert "oss-standalone-04-io-threads" in topologies_map.keys() + res = extract_redis_configuration_from_topology( + topologies_map, "oss-standalone-04-io-threads" + ) + assert res != "" + assert "--io-threads 4 io-threads-do-reads yes" in res From 3e04c2455cc0625843b3caffcbe59e7a8e234ab9 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Wed, 14 Aug 2024 02:39:25 +0100 Subject: [PATCH 3/6] Removed duplicate test --- utils/tests/test_self_contained_coordinator.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/utils/tests/test_self_contained_coordinator.py b/utils/tests/test_self_contained_coordinator.py index a3a9f31b..8855de6d 100644 --- a/utils/tests/test_self_contained_coordinator.py +++ b/utils/tests/test_self_contained_coordinator.py @@ -263,7 +263,3 @@ def test_self_contained_coordinator_blocking_read(): except redis.exceptions.ConnectionError: pass - - -def test_extract_redis_configuration_from_topology(): - assert False From 20d61ea039e25b5c5ca7c0d11fa6e45430075911 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Wed, 14 Aug 2024 02:41:11 +0100 Subject: [PATCH 4/6] Fixed missing update tests --- utils/tests/test_runner.py | 4 ++-- utils/tests/test_topologies.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/tests/test_runner.py b/utils/tests/test_runner.py index 833575f7..4cb7ab7d 100644 --- a/utils/tests/test_runner.py +++ b/utils/tests/test_runner.py @@ -258,7 +258,7 @@ def test_extract_testsuites(): ] ) tests = extract_testsuites(args) - assert len(tests) == 7 + assert len(tests) == 9 args = parser.parse_args( args=[ @@ -269,7 +269,7 @@ def test_extract_testsuites(): ] ) tests = extract_testsuites(args) - assert len(tests) == 7 + assert len(tests) == 9 args = parser.parse_args( args=[ diff --git a/utils/tests/test_topologies.py b/utils/tests/test_topologies.py index 9071955c..4850e849 100644 --- a/utils/tests/test_topologies.py +++ b/utils/tests/test_topologies.py @@ -21,4 +21,4 @@ def test_extract_redis_configuration_from_topology(): topologies_map, "oss-standalone-04-io-threads" ) assert res != "" - assert "--io-threads 4 io-threads-do-reads yes" in res + assert "--io-threads 4 --io-threads-do-reads yes" in res From f5d9bc4e8552fba1ce48c339c71812361d8c7403 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Fri, 16 Aug 2024 17:12:00 -0300 Subject: [PATCH 5/6] Added way of specifying test-regex via dockerhub cli --- redis_benchmarks_specification/__cli__/cli.py | 17 +++++++++++++++++ .../self_contained_coordinator.py | 6 +++++- tox.ini | 4 +++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/redis_benchmarks_specification/__cli__/cli.py b/redis_benchmarks_specification/__cli__/cli.py index 67bf21ee..2baddd4f 100644 --- a/redis_benchmarks_specification/__cli__/cli.py +++ b/redis_benchmarks_specification/__cli__/cli.py @@ -87,6 +87,23 @@ def trigger_tests_dockerhub_cli_command_logic(args, project_name, project_versio args.build_arch, testDetails, "n/a", + [], + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + ".*", + 0, + 10000, + args.tests_regexp, ) build_stream_fields["github_repo"] = args.gh_repo build_stream_fields["github_org"] = args.gh_org diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index 3b7f48d8..122d9d97 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -846,6 +846,9 @@ def process_self_contained_coordinator_stream( "mode": "rw", }, } + logging.info( + f"setting volume as follow: {volumes}. working_dir={mnt_point}" + ) working_dir = mnt_point redis_container = docker_client.containers.run( image=run_image, @@ -1253,7 +1256,7 @@ def process_self_contained_coordinator_stream( ) except docker.errors.NotFound: logging.info( - "When trying to stop DB container with id {} and image {} it was already stopped".format( + "When trying to fetch logs from DB container with id {} and image {} it was already stopped".format( redis_container.id, redis_container.image, ) @@ -1295,6 +1298,7 @@ def process_self_contained_coordinator_stream( temporary_dir, temporary_dir_client ) ) + shutil.rmtree(temporary_dir, ignore_errors=True) shutil.rmtree(temporary_dir_client, ignore_errors=True) diff --git a/tox.ini b/tox.ini index 1c973ba2..229d673a 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,9 @@ docker = deps = -r{toxinidir}/dev_requirements.txt -# passenv = TST_BUILDER_X,TST_RUNNER_X,GH_TOKEN,TST_REDIS_DIR,DOCKER_HOST,DOCKER_TLS_VERIFY,DOCKER_CERT_PATH + + +passenv = TST_BUILDER_X,TST_RUNNER_X,GH_TOKEN,TST_REDIS_DIR,DOCKER_HOST,DOCKER_TLS_VERIFY,DOCKER_CERT_PATH stoponfail = True From c053b2e4a9354ef090663d7b0a83a610b8f46675 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Sun, 18 Aug 2024 20:09:50 +0100 Subject: [PATCH 6/6] Fixed src build and docker logs retrieval in case of error --- pyproject.toml | 2 +- .../__builder__/builder.py | 2 + .../__self_contained_coordinator__/docker.py | 2 +- .../self_contained_coordinator.py | 89 ++++++++++++------- tox.ini | 2 +- .../tests/test_self_contained_coordinator.py | 55 ++++++++++++ ...test_self_contained_coordinator_memtier.py | 2 +- 7 files changed, 116 insertions(+), 38 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e9b62195..39e20f82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.215" +version = "0.1.216" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__builder__/builder.py b/redis_benchmarks_specification/__builder__/builder.py index a38b6352..678a71d3 100644 --- a/redis_benchmarks_specification/__builder__/builder.py +++ b/redis_benchmarks_specification/__builder__/builder.py @@ -343,6 +343,8 @@ def builder_process_stream( "linenoise", "lua", ] + if "fast_float" in deps_dir: + deps_list.append("fast_float") if "hdr_histogram" in deps_dir: deps_list.append("hdr_histogram") if "fpconv" in deps_dir: diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/docker.py b/redis_benchmarks_specification/__self_contained_coordinator__/docker.py index ca3e9f67..ea35d6b6 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/docker.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/docker.py @@ -20,7 +20,7 @@ def generate_standalone_redis_server_args( "{}".format(port), ] if dbdir != "": - command.extend(["--dbdir", dbdir]) + command.extend(["--dir", dbdir]) if configuration_parameters is not None: for parameter, parameter_value in configuration_parameters.items(): if parameter not in added_params: diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index 122d9d97..42d77bf1 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -832,42 +832,16 @@ def process_self_contained_coordinator_stream( db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus( ceil_db_cpu_limit, current_cpu_pos ) - logging.info( - "Running redis-server on docker image {} (cpuset={}) with the following args: {}".format( - run_image, db_cpuset_cpus, command_str - ) - ) - volumes = {} - working_dir = "/" - if mnt_point != "": - volumes = { - temporary_dir: { - "bind": mnt_point, - "mode": "rw", - }, - } - logging.info( - f"setting volume as follow: {volumes}. working_dir={mnt_point}" - ) - working_dir = mnt_point - redis_container = docker_client.containers.run( - image=run_image, - volumes=volumes, - auto_remove=True, - privileged=True, - working_dir=mnt_point, - command=command_str, - network_mode="host", - detach=True, - cpuset_cpus=db_cpuset_cpus, - pid_mode="host", - publish_all_ports=True, + redis_container = start_redis_container( + command_str, + db_cpuset_cpus, + docker_client, + mnt_point, + redis_containers, + run_image, + temporary_dir, ) - time.sleep(5) - - redis_containers.append(redis_container) - r = redis.StrictRedis(port=redis_proc_start_port) r.ping() redis_conns = [r] @@ -1254,6 +1228,7 @@ def process_self_contained_coordinator_stream( stdout=True, stderr=True ) ) + redis_container.remove() except docker.errors.NotFound: logging.info( "When trying to fetch logs from DB container with id {} and image {} it was already stopped".format( @@ -1272,6 +1247,7 @@ def process_self_contained_coordinator_stream( for redis_container in redis_containers: try: redis_container.stop() + redis_container.remove() except docker.errors.NotFound: logging.info( "When trying to stop DB container with id {} and image {} it was already stopped".format( @@ -1285,6 +1261,7 @@ def process_self_contained_coordinator_stream( if type(redis_container) == Container: try: redis_container.stop() + redis_container.remove() except docker.errors.NotFound: logging.info( "When trying to stop Client container with id {} and image {} it was already stopped".format( @@ -1479,6 +1456,50 @@ def process_self_contained_coordinator_stream( return stream_id, overall_result, total_test_suite_runs +def start_redis_container( + command_str, + db_cpuset_cpus, + docker_client, + mnt_point, + redis_containers, + run_image, + temporary_dir, + auto_remove=False, +): + logging.info( + "Running redis-server on docker image {} (cpuset={}) with the following args: {}".format( + run_image, db_cpuset_cpus, command_str + ) + ) + volumes = {} + working_dir = "/" + if mnt_point != "": + volumes = { + temporary_dir: { + "bind": mnt_point, + "mode": "rw", + }, + } + logging.info(f"setting volume as follow: {volumes}. working_dir={mnt_point}") + working_dir = mnt_point + redis_container = docker_client.containers.run( + image=run_image, + volumes=volumes, + auto_remove=auto_remove, + privileged=True, + working_dir=mnt_point, + command=command_str, + network_mode="host", + detach=True, + cpuset_cpus=db_cpuset_cpus, + pid_mode="host", + publish_all_ports=True, + ) + time.sleep(5) + redis_containers.append(redis_container) + return redis_container + + def filter_test_files( defaults_filename, priority_lower_limit, diff --git a/tox.ini b/tox.ini index 229d673a..cce34822 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ deps = -r{toxinidir}/dev_requirements.txt -passenv = TST_BUILDER_X,TST_RUNNER_X,GH_TOKEN,TST_REDIS_DIR,DOCKER_HOST,DOCKER_TLS_VERIFY,DOCKER_CERT_PATH +passenv = TST_BUILDER_X,TST_RUNNER_X,GH_TOKEN,TST_REDIS_DIR,DOCKER_HOST,DOCKER_TLS_VERIFY,DOCKER_CERT_PATH,TST_BINARY_REDIS_DIR stoponfail = True diff --git a/utils/tests/test_self_contained_coordinator.py b/utils/tests/test_self_contained_coordinator.py index 8855de6d..bbb91e58 100644 --- a/utils/tests/test_self_contained_coordinator.py +++ b/utils/tests/test_self_contained_coordinator.py @@ -7,6 +7,7 @@ from redisbench_admin.utils.remote import get_overall_dashboard_keynames from redisbench_admin.utils.utils import get_ts_metric_name +import logging from redis_benchmarks_specification.__common__.env import ( STREAM_KEYNAME_NEW_BUILD_EVENTS, @@ -18,6 +19,7 @@ ) from redis_benchmarks_specification.__self_contained_coordinator__.self_contained_coordinator import ( self_contained_coordinator_blocking_read, + start_redis_container, ) from redis_benchmarks_specification.__self_contained_coordinator__.runners import ( @@ -31,6 +33,11 @@ from utils.tests.test_data.api_builder_common import flow_1_and_2_api_builder_checks +from redis_benchmarks_specification.__self_contained_coordinator__.docker import ( + generate_standalone_redis_server_args, +) + + def test_extract_client_cpu_limit(): with open( "./utils/tests/test_data/test-suites/redis-benchmark-full-suite-1Mkeys-100B.yml", @@ -263,3 +270,51 @@ def test_self_contained_coordinator_blocking_read(): except redis.exceptions.ConnectionError: pass + + +def test_start_redis_container(): + temporary_dir = os.getenv("TST_BINARY_REDIS_DIR", "") + if temporary_dir == "": + return + + mnt_point = "/mnt/redis/" + executable = f"{mnt_point}redis-server" + redis_proc_start_port = 6379 + current_cpu_pos = 0 + ceil_db_cpu_limit = 1 + redis_configuration_parameters = None + redis_arguments = "" + docker_client = docker.from_env() + redis_containers = [] + + command = generate_standalone_redis_server_args( + executable, + redis_proc_start_port, + mnt_point, + redis_configuration_parameters, + redis_arguments, + ) + command_str = " ".join(command) + db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus( + ceil_db_cpu_limit, current_cpu_pos + ) + run_image = "gcc:8.5" + redis_container = start_redis_container( + command_str, + db_cpuset_cpus, + docker_client, + mnt_point, + redis_containers, + run_image, + temporary_dir, + ) + r = redis.StrictRedis(port=redis_proc_start_port) + try: + r.ping() + except redis.exceptions.ConnectionError: + # Access and print the logs + logs = redis_container.logs().decode("utf-8") + logging.error("Container failed. Here are the logs:") + logging.error(logs) + raise + redis_container.remove() diff --git a/utils/tests/test_self_contained_coordinator_memtier.py b/utils/tests/test_self_contained_coordinator_memtier.py index 3f5538c7..ec4794b8 100644 --- a/utils/tests/test_self_contained_coordinator_memtier.py +++ b/utils/tests/test_self_contained_coordinator_memtier.py @@ -204,7 +204,7 @@ def test_self_contained_coordinator_blocking_read(): assert len(datasink_conn.smembers(running_platforms_setname)) == 1 assert len(datasink_conn.smembers(testcases_setname)) == 1 assert len(datasink_conn.smembers(project_branches_setname)) == 1 - assert len(datasink_conn.smembers(project_versions_setname)) == 0 + assert len(datasink_conn.smembers(project_versions_setname)) == 1 except redis.exceptions.ConnectionError: pass