Added pyperf format to redisbench-admin export (#355)

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.8.10"
+version = "0.9.0"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "README.md"

redisbench_admin/deploy/args.py

@@ -34,4 +34,10 @@ def create_deploy_arguments(parser):
     parser.add_argument(
         "--destroy", help="destroy the current env", action="store_true"
     )
+    parser.add_argument(
+        "--skip-env-vars-verify",
+        default=False,
+        action="store_true",
+        help="skip environment variables check",
+    )
     return parser

redisbench_admin/deploy/deploy.py

@@ -12,6 +12,7 @@
 from redisbench_admin.utils.remote import (
     fetch_remote_setup_git_url,
     setup_remote_environment,
+    check_ec2_env,
 )
 from python_terraform import Terraform
 
@@ -44,6 +45,13 @@ def deploy_command_logic(args, project_name, project_version):
 
     private_key = args.private_key
     ssh_pem_check(EC2_PRIVATE_PEM, private_key)
+
+    if args.skip_env_vars_verify is False:
+        env_check_status, failure_reason = check_ec2_env()
+        if env_check_status is False:
+            logging.critical("{}. Exiting right away!".format(failure_reason))
+            exit(1)
+
     inventory_git = args.inventory_git
     inventory_local_dir = args.inventory_local_dir
     destroy = args.destroy

redisbench_admin/export/args.py

@@ -27,13 +27,13 @@ def create_export_arguments(parser):
     parser.add_argument(
         "--deployment-name",
         type=str,
-        required=True,
+        default="oss-standalone",
         help="Deployment name",
     )
     parser.add_argument(
         "--deployment-type",
         type=str,
-        required=True,
+        default="oss-standalone",
         help="Deployment Type",
     )
     parser.add_argument(
@@ -64,7 +64,7 @@ def create_export_arguments(parser):
         type=str,
         default="json",
         help="results format of the the benchmark results files to read "
-        "results from ( either csv, json, redis-benchmark-txt )",
+        "results from ( either pyperf-json, csv, json, redis-benchmark-txt )",
     )
     parser.add_argument(
         "--use-result",

redisbench_admin/export/export.py

@@ -12,6 +12,9 @@
 
 
 from redisbench_admin.export.common.common import split_tags_string
+from redisbench_admin.export.pyperf.pyperf_json_format import (
+    generate_summary_json_pyperf,
+)
 from redisbench_admin.run.git import git_vars_crosscheck
 
 from redisbench_admin.run.redistimeseries import timeseries_test_sucess_flow
@@ -47,10 +50,10 @@ def export_command_logic(args, project_name, project_version):
             "You need to specify at least one (or more) of --deployment-version --github_branch arguments"
         )
         exit(1)
-    if results_format != "csv":
+    if results_format != "csv" and results_format != "pyperf-json":
         if exporter_spec_file is None:
             logging.error(
-                "--exporter-spec-file is required for all formats with exception of csv"
+                "--exporter-spec-file is required for all formats with exception of csv and pyperf-json"
             )
             exit(1)
         else:
@@ -69,6 +72,10 @@ def export_command_logic(args, project_name, project_version):
     if results_format == "json":
         with open(benchmark_file, "r") as json_file:
             results_dict = json.load(json_file)
+    if results_format == "pyperf-json":
+        with open(benchmark_file, "r") as json_file:
+            start_dict = json.load(json_file)
+            results_dict = generate_summary_json_pyperf(start_dict)
     if args.override_test_time:
         datapoints_timestamp = int(args.override_test_time.timestamp() * 1000.0)
         logging.info(
@@ -90,7 +97,7 @@ def export_command_logic(args, project_name, project_version):
                 datetime.datetime.now(datetime.timezone.utc).timestamp() * 1000.0
             )
             logging.warning(
-                "Error while trying to parse datapoints timestamp. Using current system timestamp Error: {}".format(
+                "Error while trying to parse datapoints timestamp. Using current system timestamp: {}".format(
                     datapoints_timestamp
                 )
             )
@@ -113,6 +120,20 @@ def export_command_logic(args, project_name, project_version):
             triggering_env,
         )
         logging.info("Parsed a total of {} metrics".format(len(timeseries_dict.keys())))
+    if results_format == "pyperf-json":
+        logging.info("Parsing pyperf format into timeseries format")
+        timeseries_dict = export_pyperf_json_to_timeseries_dict(
+            results_dict,
+            break_by_dict,
+            datapoints_timestamp,
+            deployment_name,
+            deployment_type,
+            extra_tags_dict,
+            github_org,
+            github_repo,
+            triggering_env,
+        )
+        logging.info("Parsed a total of {} metrics".format(len(timeseries_dict.keys())))
     logging.info(
         "Checking connection to RedisTimeSeries to host: {}:{}".format(
             args.redistimeseries_host, args.redistimeseries_port
@@ -160,6 +181,48 @@ def export_command_logic(args, project_name, project_version):
     )
 
 
+def export_pyperf_json_to_timeseries_dict(
+    benchmark_file,
+    break_by_dict,
+    datapoints_timestamp,
+    deployment_name,
+    deployment_type,
+    extra_tags_dict,
+    tf_github_org,
+    tf_github_repo,
+    triggering_env,
+):
+    results_dict = {}
+    for test_name, d in benchmark_file.items():
+        for metric_name, metric_value in d.items():
+            for break_by_key, break_by_value in break_by_dict.items():
+                break_by_str = "by.{}".format(break_by_key)
+                timeserie_tags, ts_name = get_ts_tags_and_name(
+                    break_by_key,
+                    break_by_str,
+                    break_by_value,
+                    None,
+                    deployment_name,
+                    deployment_type,
+                    extra_tags_dict,
+                    metric_name,
+                    metric_name,
+                    metric_name,
+                    triggering_env,
+                    test_name,
+                    metric_name,
+                    tf_github_org,
+                    tf_github_repo,
+                    triggering_env,
+                    False,
+                )
+                results_dict[ts_name] = {
+                    "labels": timeserie_tags.copy(),
+                    "data": {datapoints_timestamp: metric_value},
+                }
+    return results_dict
+
+
 def export_opereto_csv_to_timeseries_dict(
     benchmark_file,
     break_by_dict,

redisbench_admin/export/pyperf/__init__.py

@@ -0,0 +1,5 @@
+#  Apache License Version 2.0
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#

redisbench_admin/export/pyperf/pyperf_json_format.py

@@ -0,0 +1,51 @@
+#  Apache License Version 2.0
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#
+import logging
+
+import numpy as np
+
+
+def metric_safe_name(row, replace_by="_"):
+    import re
+
+    metric_name = row.strip()
+    metric_name = re.sub(r"\W+", replace_by, metric_name)
+    return metric_name
+
+
+def generate_summary_json_pyperf(input_json):
+    result_json = {}
+    for benchmark in input_json["benchmarks"]:
+        original_name = benchmark["metadata"]["name"]
+        benchmark_name = original_name
+        non_safe_count = len(original_name) - len(metric_safe_name(original_name, ""))
+        if non_safe_count > 0:
+            benchmark_name = metric_safe_name(original_name)
+            while "_" == benchmark_name[len(benchmark_name) - 1]:
+                benchmark_name = benchmark_name[: len(benchmark_name) - 1]
+            logging.warning(
+                "Given the benchmark name {} contains {} non alphanumeric characters, we're replacing it by the safe version {}".format(
+                    original_name, "-", benchmark_name
+                )
+            )
+
+        runs = benchmark["runs"]
+        total_runs = len(runs)
+        results = []
+        for run in runs:
+            if "values" in run:
+                for value in run["values"]:
+                    results.append(value)
+        avg = np.average(results)
+        std = np.std(results)
+        logging.info(
+            "Adding pyperf metric named {}: avg={} stddev={} total_runs={}".format(
+                benchmark_name, avg, std, total_runs
+            )
+        )
+        result_json[benchmark_name] = {"avg": avg, "std": std, "total_runs": total_runs}
+
+    return result_json

redisbench_admin/run/ssh.py

@@ -68,6 +68,7 @@ def check_connection(ssh_conn):
 
 def ssh_pem_check(EC2_PRIVATE_PEM, private_key):
     if EC2_PRIVATE_PEM is not None and EC2_PRIVATE_PEM != "":
+        logging.info("Given env variable EC2_PRIVATE_PEM exists")
         with open(private_key, "w") as tmp_private_key_file:
             pem_str = check_and_fix_pem_str(EC2_PRIVATE_PEM)
             tmp_private_key_file.write(pem_str)

redisbench_admin/run/ycsb/ycsb.py

@@ -97,10 +97,15 @@ def post_process_ycsb_results(stdout, start_time_ms, start_time_str):
     for row in csv_data[start_row:]:
         if len(row) >= 3:
             op_group = row[0].strip()[1:-1]
-            metric_name = row[1].strip()
-            metric_name = re.sub("[^0-9a-zA-Z]+", "_", metric_name)
+            metric_name = metric_safe_name(row)
             value = row[2].strip()
             if op_group not in results_dict["Tests"]:
                 results_dict["Tests"][op_group] = {}
             results_dict["Tests"][op_group][metric_name] = value
     return results_dict
+
+
+def metric_safe_name(row):
+    metric_name = row[1].strip()
+    metric_name = re.sub("[^0-9a-zA-Z]+", "_", metric_name)
+    return metric_name

redisbench_admin/run_remote/run_remote.py

@@ -158,6 +158,7 @@ def run_remote_command_logic(args, project_name, project_version):
                     tf_github_branch,
                     None,
                 )
+            logging.critical("{}. Exiting right away!".format(failure_reason))
             exit(1)
 
     module_check_status, error_message = redis_modules_check(local_module_files)