[add] Enabled reusing the same remote env on multiple bench definitions. Fixed semver extraction on remote setups (#91)

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.1.68"
+version = "0.1.69"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>"]
 readme = "README.md"

redisbench_admin/run/common.py

@@ -22,7 +22,6 @@
 from redisbench_admin.utils.remote import (
     execute_remote_commands,
     fetch_file_from_remote_setup,
-    extract_redisgraph_version_from_resultdict,
     extract_perversion_timeseries_from_results,
     push_data_to_redistimeseries,
     extract_perbranch_timeseries_from_results,
@@ -274,17 +273,14 @@ def common_exporter_logic(
     tf_github_org,
     tf_github_repo,
     tf_triggering_env,
+    artifact_version="N/A",
 ):
     if exporter_timemetric_path is not None and len(metrics) > 0:
         # extract timestamp
         datapoints_timestamp = parse_exporter_timemetric(
             exporter_timemetric_path, results_dict
         )
 
-        rg_version = extract_redisgraph_version_from_resultdict(results_dict)
-        if rg_version is None:
-            rg_version = "N/A"
-
         # extract per branch datapoints
         (
             ok,
@@ -293,7 +289,7 @@ def common_exporter_logic(
             datapoints_timestamp,
             metrics,
             results_dict,
-            rg_version,
+            artifact_version,
             tf_github_org,
             tf_github_repo,
             deployment_type,

redisbench_admin/run_local/run_local.py

@@ -177,12 +177,15 @@ def run_local_command_logic(args):
                 with open(local_benchmark_output_filename, "r") as json_file:
                     results_dict = json.load(json_file)
 
-                if "kpis" in benchmark_config:
-                    result = validate_result_expectations(
-                        benchmark_config, results_dict, result, expectations_key="kpis"
-                    )
-                    if result is not True:
-                        return_code |= 1
+                    if "kpis" in benchmark_config:
+                        result = validate_result_expectations(
+                            benchmark_config,
+                            results_dict,
+                            result,
+                            expectations_key="kpis",
+                        )
+                        if result is not True:
+                            return_code |= 1
             except:
                 return_code |= 1
                 logging.critical(

redisbench_admin/run_remote/run_remote.py

@@ -39,6 +39,8 @@
     get_run_full_filename,
     fetch_remote_setup_from_config,
     execute_remote_commands,
+    extract_redisgraph_version_from_resultdict,
+    retrieve_tf_connection_vars,
 )
 
 # internal aux vars
@@ -82,6 +84,39 @@ def setup_remote_benchmark_tool_requirements_tsbs_run_queries_redistimeseries(
     execute_remote_commands(client_public_ip, username, private_key, commands)
 
 
+def extract_artifact_version_remote(
+    server_public_ip, server_public_port, username, private_key
+):
+    commands = [
+        "redis-cli -h {} -p {} info modules".format(
+            server_public_ip, server_public_port
+        ),
+    ]
+    res = execute_remote_commands(server_public_ip, username, private_key, commands)
+    recv_exit_status, stdout, stderr = res[0]
+    print(stdout)
+    version = extract_module_semver_from_info_modules_cmd(stdout)
+    return version
+
+
+def extract_module_semver_from_info_modules_cmd(stdout):
+    version = None
+    if type(stdout) == str:
+        info_modules_output = stdout.split("\n")[1:]
+    else:
+        info_modules_output = stdout[1:]
+    for module_detail_line in info_modules_output:
+        detail_arr = module_detail_line.split(",")
+        if len(detail_arr) > 1:
+            module_name = detail_arr[0].split("=")
+            module_name = module_name[1]
+            version = detail_arr[1].split("=")[1]
+            logging.info(
+                "Detected artifact={}, semver={}.".format(module_name, version)
+            )
+    return version
+
+
 def run_remote_command_logic(args):
     tf_bin_path = args.terraform_bin_path
     tf_github_org = args.github_org
@@ -263,7 +298,7 @@ def run_remote_command_logic(args):
                 exporter_timemetric_path,
                 stream,
             )
-
+    remote_envs = {}
     for usecase_filename in files:
         with open(usecase_filename, "r") as stream:
             dirname = os.path.dirname(os.path.abspath(usecase_filename))
@@ -284,38 +319,62 @@ def run_remote_command_logic(args):
             )
 
             if "remote" in benchmark_config:
-                remote_setup, deployment_type = fetch_remote_setup_from_config(
-                    benchmark_config["remote"]
-                )
+                (
+                    remote_setup,
+                    deployment_type,
+                    remote_id,
+                ) = fetch_remote_setup_from_config(benchmark_config["remote"])
                 logging.info(
                     "Deploying test defined in {} on AWS using {}".format(
                         usecase_filename, remote_setup
                     )
                 )
                 tf_setup_name = "{}{}".format(remote_setup, tf_setup_name_sufix)
                 logging.info("Using full setup name: {}".format(tf_setup_name))
-                # check if terraform is present
-                tf = Terraform(
-                    working_dir=remote_setup,
-                    terraform_bin_path=tf_bin_path,
-                )
-                (
-                    return_code,
-                    username,
-                    server_private_ip,
-                    server_public_ip,
-                    server_plaintext_port,
-                    client_private_ip,
-                    client_public_ip,
-                ) = setup_remote_environment(
-                    tf,
-                    tf_github_sha,
-                    tf_github_actor,
-                    tf_setup_name,
-                    tf_github_org,
-                    tf_github_repo,
-                    tf_triggering_env,
-                )
+                if remote_id not in remote_envs:
+                    # check if terraform is present
+                    tf = Terraform(
+                        working_dir=remote_setup,
+                        terraform_bin_path=tf_bin_path,
+                    )
+                    (
+                        tf_return_code,
+                        username,
+                        server_private_ip,
+                        server_public_ip,
+                        server_plaintext_port,
+                        client_private_ip,
+                        client_public_ip,
+                    ) = setup_remote_environment(
+                        tf,
+                        tf_github_sha,
+                        tf_github_actor,
+                        tf_setup_name,
+                        tf_github_org,
+                        tf_github_repo,
+                        tf_triggering_env,
+                    )
+                    remote_envs[remote_id] = tf
+                else:
+                    logging.info("Reusing remote setup {}".format(remote_id))
+                    tf = remote_envs[remote_id]
+                    (
+                        tf_return_code,
+                        username,
+                        server_private_ip,
+                        server_public_ip,
+                        server_plaintext_port,
+                        client_private_ip,
+                        client_public_ip,
+                    ) = retrieve_tf_connection_vars(None, tf)
+                    commands = [
+                        "redis-cli -h {} -p {} shutdown".format(
+                            server_private_ip, server_plaintext_port
+                        )
+                    ]
+                    execute_remote_commands(
+                        server_public_ip, username, private_key, commands
+                    )
                 # after we've created the env, even on error we should always teardown
                 # in case of some unexpected error we fail the test
                 try:
@@ -330,7 +389,9 @@ def run_remote_command_logic(args):
                         remote_dataset_file,
                         dirname,
                     )
-
+                    artifact_version = extract_artifact_version_remote(
+                        server_public_ip, server_plaintext_port, username, private_key
+                    )
                     (
                         benchmark_min_tool_version,
                         benchmark_min_tool_version_major,
@@ -460,6 +521,19 @@ def run_remote_command_logic(args):
                     with open(local_benchmark_output_filename, "r") as json_file:
                         results_dict = json.load(json_file)
 
+                    # if the benchmark tool is redisgraph-benchmark-go and
+                    # we still dont have the artifact semver we can extract it from the results dict
+                    if (
+                        benchmark_tool == "redisgraph-benchmark-go"
+                        and artifact_version is None
+                    ):
+                        artifact_version = extract_redisgraph_version_from_resultdict(
+                            results_dict
+                        )
+
+                    if artifact_version is None:
+                        artifact_version = "N/A"
+
                     if "kpis" in benchmark_config:
                         result = validate_result_expectations(
                             benchmark_config,
@@ -514,6 +588,7 @@ def run_remote_command_logic(args):
                             tf_github_org,
                             tf_github_repo,
                             tf_triggering_env,
+                            artifact_version,
                         )
                 except:
                     return_code |= 1
@@ -525,11 +600,12 @@ def run_remote_command_logic(args):
                     print("-" * 60)
                     traceback.print_exc(file=sys.stdout)
                     print("-" * 60)
-                finally:
-                    # tear-down
-                    logging.info("Tearing down setup")
-                    tf.destroy()
-                    logging.info("Tear-down completed")
+
+    for remote_setup_name, tf in remote_envs.items():
+        # tear-down
+        logging.info("Tearing down setup {}".format(remote_setup_name))
+        tf.destroy()
+        logging.info("Tear-down completed")
 
     exit(return_code)
 

redisbench_admin/utils/remote.py

@@ -184,6 +184,10 @@ def setup_remote_environment(
             "triggering_env": tf_triggering_env,
         },
     )
+    return retrieve_tf_connection_vars(return_code, tf)
+
+
+def retrieve_tf_connection_vars(return_code, tf):
     tf_output = tf.output()
     server_private_ip = tf_output["server_private_ip"]["value"][0]
     server_public_ip = tf_output["server_public_ip"]["value"][0]
@@ -404,7 +408,7 @@ def fetch_remote_setup_from_config(
     )
     git.Repo.clone_from(repo, temporary_dir, branch=branch, depth=1)
     terraform_working_dir = temporary_dir + path
-    return terraform_working_dir, setup_type
+    return terraform_working_dir, setup_type, setup
 
 
 def push_data_to_redistimeseries(rts: client, branch_time_series_dict: dict):

tests/test_remote.py

@@ -79,7 +79,7 @@ def test_extract_git_vars_passing_repo3():
 
 
 def test_fetch_remote_setup_from_config():
-    terraform_working_dir, type = fetch_remote_setup_from_config(
+    terraform_working_dir, type, _ = fetch_remote_setup_from_config(
         [{"type": "oss-standalone"}, {"setup": "redistimeseries-m5d"}]
     )
     assert type == "oss-standalone"

tests/test_run_remote.py

@@ -0,0 +1,9 @@
+from redisbench_admin.run_remote.run_remote import (
+    extract_module_semver_from_info_modules_cmd,
+)
+
+
+def test_extract_module_semver_from_info_modules_cmd():
+    stdout = b"# Modules\r\nmodule:name=search,ver=999999,api=1,filters=0,usedby=[],using=[],options=[]\r\n".decode()
+    semver = extract_module_semver_from_info_modules_cmd(stdout)
+    assert semver == "999999"