Fixed run exec regarding multiple tear-down steps (#7)

filipecosta90 · web-flow · commit 3bec8bfdb103 · 2020-09-15T10:48:12.000+01:00
* [fix] fixed static project being redisearch for it to be loaded from the config

* [fix] Fixed run exec regarding multiple teardown steps
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.1.10"
-description = "Redis benchmark run helper. A wrapper around ftsb_redisearch ( future versions will also support redis-benchmark and memtier_benchmark )."
+version = "0.1.11"
+description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>"]
 readme = "README.md"
 
diff --git a/redisbench_admin/cli.py b/redisbench_admin/cli.py
@@ -1,6 +1,7 @@
 import argparse
 import sys
 
+# import toml
 import toml
 
 from redisbench_admin import __version__
diff --git a/redisbench_admin/export/args.py b/redisbench_admin/export/args.py
@@ -3,9 +3,15 @@ def create_export_arguments(parser):
                         help="benchmark results files to read results from. can be a local file, a remote link, or an s3 bucket.")
     parser.add_argument('--steps', type=str, default="setup,benchmark",
                         help="comma separated list of steps to be analyzed given the benchmark result files")
-    parser.add_argument('--exporter', type=str, default="csv",
+    parser.add_argument('--exporter', type=str, default="redistimeseries",
                         help="exporter to be used ( either csv or redistimeseries )")
     parser.add_argument('--use-result', type=str, default="median-result",
                         help="for each key-metric, use either worst-result, best-result, or median-result")
     parser.add_argument('--extra-tags', type=str, default="", help='comma separated extra tags in the format of key1=value,key2=value,...')
+    parser.add_argument('--host', type=str, default="localhost",
+                        help="redistimeseries host")
+    parser.add_argument('--port', type=int, default=6379,
+                        help="redistimeseries port")
+    parser.add_argument('--password', type=str, default=None,
+                        help="redistimeseries password")
     return parser
diff --git a/redisbench_admin/export/export.py b/redisbench_admin/export/export.py
@@ -1,18 +1,22 @@
 import os
 import pandas as pd
+import redis
+
 from redisbench_admin.compare.compare import get_key_results_and_values
 from redisbench_admin.utils.utils import retrieve_local_or_remote_input_json
 import matplotlib.pyplot as plt
 import seaborn as sns
+
 sns.set(style="darkgrid")
 from redistimeseries.client import Client
 
+
 def get_timeserie_name(labels_kv_array):
     name = ""
     for label_kv in labels_kv_array:
         k = list(label_kv.keys())[0]
         v = list(label_kv.values())[0]
-        k= prepare_tags(k)
+        k = prepare_tags(k)
         v = prepare_tags(v)
         if name != "":
             name += ":"
@@ -47,7 +51,7 @@ def extract_benchmark_config_details(benchmark_result):
     specifications_version = benchmark_config["specifications-version"]
     testcase_description = benchmark_config["description"]
     key_metrics_specs = benchmark_config["key-metrics"]
-    return testcase_name, specifications_version, testcase_description,key_metrics_specs
+    return testcase_name, specifications_version, testcase_description, key_metrics_specs
 
 
 def extract_key_configs(benchmark_result):
@@ -66,14 +70,14 @@ def extract_key_configs(benchmark_result):
     return deployment_type, deployment_shards, project, project_version, git_sha
 
 
-def add_datapoint(time_series_dict, broader_ts_name, start_time_ms, metric_value, tags_array ):
+def add_datapoint(time_series_dict, broader_ts_name, start_time_ms, metric_value, tags_array):
     if broader_ts_name not in time_series_dict:
         tags_dict = {}
         for tag_kv in tags_array:
             k = list(tag_kv.keys())[0]
             v = list(tag_kv.values())[0]
-            tags_dict[k]=v
-        time_series_dict[broader_ts_name]={"index":[],"data":[],"tags-array":tags_array, "tags":tags_dict}
+            tags_dict[k] = v
+        time_series_dict[broader_ts_name] = {"index": [], "data": [], "tags-array": tags_array, "tags": tags_dict}
     time_series_dict[broader_ts_name]["index"].append(start_time_ms)
     time_series_dict[broader_ts_name]["data"].append(metric_value)
 
@@ -85,9 +89,9 @@ def split_tags_string(extra_tags):
         for extra_tag in extra_tags:
             kv = extra_tag.split("=")
             if len(kv) == 2:
-                k= prepare_tags (kv[0])
+                k = prepare_tags(kv[0])
                 v = prepare_tags(kv[1])
-                result.append({k:v})
+                result.append({k: v})
     return result
 
 
@@ -99,33 +103,36 @@ def split_key_metrics_by_step(key_metrics_specs):
             step = key_metric_spec["step"]
             metric_name = key_metric_spec["metric-name"]
             if step not in key_metrics_by_step:
-                key_metrics_by_step[step]={}
-            key_metrics_by_step[step][metric_name]=key_metric_spec
+                key_metrics_by_step[step] = {}
+            key_metrics_by_step[step][metric_name] = key_metric_spec
     return key_metrics_by_step
 
-def get_or_None(dict,property):
+
+def get_or_None(dict, property):
     result = None
     if property in dict:
         result = dict[property]
     return result
 
+
 def get_metric_detail(key_metric_spec):
-    metric_step = get_or_None( key_metric_spec, "step")
-    metric_family = get_or_None( key_metric_spec, "metric-family")
+    metric_step = get_or_None(key_metric_spec, "step")
+    metric_family = get_or_None(key_metric_spec, "metric-family")
     metric_json_path = get_or_None(key_metric_spec, "metric-json-path")
-    metric_name = get_or_None( key_metric_spec, "metric-name")
-    metric_unit = get_or_None( key_metric_spec, "unit")
-    metric_type = get_or_None( key_metric_spec,  "metric-type")
-    metric_comparison = get_or_None( key_metric_spec, "comparison")
-    metric_per_step_comparison_priority = get_or_None( key_metric_spec, "per-step-comparison-metric-priority")
-    return metric_step,metric_family,metric_json_path,metric_name,metric_unit,metric_type,metric_comparison,metric_per_step_comparison_priority
+    metric_name = get_or_None(key_metric_spec, "metric-name")
+    metric_unit = get_or_None(key_metric_spec, "unit")
+    metric_type = get_or_None(key_metric_spec, "metric-type")
+    metric_comparison = get_or_None(key_metric_spec, "comparison")
+    metric_per_step_comparison_priority = get_or_None(key_metric_spec, "per-step-comparison-metric-priority")
+    return metric_step, metric_family, metric_json_path, metric_name, metric_unit, metric_type, metric_comparison, metric_per_step_comparison_priority
+
 
 def export_command_logic(args):
     benchmark_files = args.benchmark_result_files
     local_path = os.path.abspath(args.local_dir)
     use_result = args.use_result
     included_steps = args.steps.split(",")
-    
+
     extra_tags_array = split_tags_string(args.extra_tags)
     print(extra_tags_array)
     results_type = "key-results"
@@ -134,10 +141,12 @@ def export_command_logic(args):
     for filename, benchmark_result in benchmark_results.items():
         print(filename)
         key_result_steps = benchmark_result[results_type].keys()
-        testcase_name, specifications_version, testcase_description,key_metrics_specs = extract_benchmark_config_details(benchmark_result)
+        testcase_name, specifications_version, testcase_description, key_metrics_specs = extract_benchmark_config_details(
+            benchmark_result)
         key_metrics_specs_per_step = split_key_metrics_by_step(key_metrics_specs)
         deployment_type, deployment_shards, project, project_version, git_sha = extract_key_configs(benchmark_result)
-        start_time_ms, start_time_humanized, end_time_ms, end_time_humanized, duration_ms, duration_humanized = extract_benchmark_run_info_details(benchmark_result)
+        start_time_ms, start_time_humanized, end_time_ms, end_time_humanized, duration_ms, duration_humanized = extract_benchmark_run_info_details(
+            benchmark_result)
 
         for step in key_result_steps:
             common_broader_kv_tags = [
@@ -168,8 +177,7 @@ def export_command_logic(args):
                     git_sha_ts_name = get_timeserie_name(git_sha_kv)
 
                     key_metric_spec = key_metrics_specs[metric_name]
-                    metric_step, metric_family, _, _, metric_unit, _, _,_ = get_metric_detail(key_metric_spec)
-
+                    metric_step, metric_family, _, _, metric_unit, _, _, _ = get_metric_detail(key_metric_spec)
 
                     # add_datapoint(time_series_dict,broader_ts_name,start_time_ms,metric_value,tags_kv)
                     # add_datapoint(time_series_dict, version_ts_name, start_time_ms, metric_value, tags_kv)
@@ -178,30 +186,18 @@ def export_command_logic(args):
                     git_sha_tags_kv.extend(
                         [{"metric-step": metric_step}, {"metric-family": metric_family}, {"metric-unit": metric_unit}])
                     add_datapoint(time_series_dict, git_sha_ts_name, start_time_ms, metric_value, git_sha_tags_kv)
-                    # print(broader_ts_name)
-                    # print(version_ts_name)
-                    # print(git_sha_ts_name)
-    # print(time_series_dict)
-    rts = Client()
+    rts = Client(host=args.host,port=args.port,password=args.password)
     for timeseries_name, time_series in time_series_dict.items():
-        rts.create(timeseries_name,labels=time_series['tags'])
-        for pos,timestamp in enumerate(time_series['index']):
+        try:
+            rts.create(timeseries_name, labels=time_series['tags'])
+        except redis.exceptions.ResponseError:
+            # if ts already exists continue
+            pass
+        for pos, timestamp in enumerate(time_series['index']):
             value = time_series['data'][pos]
-            rts.add(timeseries_name, timestamp, value)
-
-        #
-        # index = pd.to_datetime(time_series['index'], unit='ms')
-        # df = pd.Series(time_series['data'], index=index)
-        # print(df)
-
-      #   import matplotlib.pyplot as plt
-      #   fig, ax = plt.subplots()
-      # # Plot the responses for different events and regions
-      #   sns.lineplot( x="timepoint", y="signal",
-      #                data=df, ax=ax)
-      #   bottom, top = plt.ylim()
-      #   plt.ylim(0,top)
-      #   plt.savefig('{}.png'.format(timeseries_name))
-
-
+            try:
+                rts.add(timeseries_name, timestamp, value)
+            except redis.exceptions.ResponseError:
+                # if ts already exists continue
+                pass
 
diff --git a/redisbench_admin/run/ftsb_redisearch/ftsb_redisearch.py b/redisbench_admin/run/ftsb_redisearch/ftsb_redisearch.py
@@ -1,3 +1,19 @@
+# Copyright (C) 2020 Redis Labs Ltd.
+#
+# This file is part of redisbench-admin.
+#
+# redisbench-admin is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 2.
+#
+# redisbench-admin is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with redisbench-admin.  If not, see <http://www.gnu.org/licenses/>.
+
 import json
 import os
 import subprocess
@@ -17,7 +33,7 @@ def get_run_options():
 
 
 def run_ftsb_redisearch(redis_url, ftsb_redisearch_path, setup_run_json_output_fullpath, options, input_file,
-                        workers):
+                        workers=1,args=[]):
     ##################
     # Setup commands #
     ##################
diff --git a/redisbench_admin/run/run.py b/redisbench_admin/run/run.py
@@ -97,15 +97,7 @@ def run_command_logic(args):
     benchmark_infra["db-machines"]["db-machine-1"] = db_machine_1
     benchmark_infra["total-db-machines"] += 1
     benchmark_output_dict["key-configs"] = key_configs
-    print("Running setup steps...")
-    for command in benchmark_config["setup"]["commands"]:
-        try:
-            aux_client = redis.from_url(args.redis_url)
-            aux_client.execute_command(" ".join(command))
-        except redis.connection.ConnectionError as e:
-            print('Error while issuing setup command to Redis.Command {}! Error message: {} Exiting..'.format(command,
-                                                                                                              e.__str__()))
-            sys.exit(1)
+
     ###############################
     # Go client stage starts here #
     ###############################
@@ -118,16 +110,17 @@ def run_command_logic(args):
         total_steps += total_steps
     progress = tqdm(unit="bench steps", total=total_steps)
     for repetition in range(1, args.repetitions + 1):
-        if "setup" in run_stages_inputs:
-            if benchmark_repetitions_require_teardown is True or repetition == 1:
+        if benchmark_repetitions_require_teardown is True or repetition == 1:
+            aux_client = run_setup_commands(args, aux_client, benchmark_config)
+            if "setup" in run_stages_inputs:
                 setup_run_key = "setup-run-{}.json".format(repetition)
                 setup_run_json_output_fullpath = "{}/{}".format(local_path, setup_run_key)
                 input_file = run_stages_inputs["setup"]
                 benchmark_output_dict["setup"][setup_run_key] = run_ftsb_redisearch(args.redis_url,
                                                                                     benchmark_tool_path,
                                                                                     setup_run_json_output_fullpath,
                                                                                     options, input_file, workers)
-                progress.update()
+            progress.update()
 
         ######################
         # Benchmark commands #
@@ -196,6 +189,19 @@ def run_command_logic(args):
         upload_artifacts_to_s3(artifacts, s3_bucket_name, s3_bucket_path)
 
 
+def run_setup_commands(args, aux_client, benchmark_config):
+    print("Running setup steps...")
+    for command in benchmark_config["setup"]["commands"]:
+        try:
+            aux_client = redis.from_url(args.redis_url)
+            aux_client.execute_command(" ".join(command))
+        except redis.connection.ConnectionError as e:
+            print('Error while issuing setup command to Redis.Command {}! Error message: {} Exiting..'.format(command,
+                                                                                                              e.__str__()))
+            sys.exit(1)
+    return aux_client
+
+
 def prepare_run_info_metadata_dict(end_time, start_time):
     start_time_str = start_time.strftime("%Y-%m-%d-%H-%M-%S")
     end_time_str = end_time.strftime("%Y-%m-%d-%H-%M-%S")
diff --git a/redisbench_admin/utils/utils.py b/redisbench_admin/utils/utils.py
@@ -73,20 +73,20 @@ def ts_milli(at_dt):
 
 
 def retrieve_local_or_remote_input_json(config_filename, local_path, option_name):
-    benchmark_config = None
+    benchmark_config = {}
 
     if config_filename.startswith("http"):
         print("retrieving benchmark config file from remote url {}".format(config_filename))
         r = requests.get(config_filename)
-        benchmark_config = r.json()
+        benchmark_config[config_filename] = r.json()
         remote_filename = config_filename[config_filename.rfind('/') + 1:]
         local_config_file = "{}/{}".format(local_path, remote_filename)
         open(local_config_file, 'wb').write(r.content)
         print("To avoid fetching again the config file use the option {option_name} {filename}".format(
             option_name=option_name,
             filename=local_config_file))
 
-    elif config_filename.startswith("S3://"):
+    elif config_filename.startswith("S3://") or config_filename.startswith("s3://"):
         print("s3")
         s3 = boto3.resource('s3')
         bucket_str = config_filename[5:].split("/")[0]
