[add] kicked off the redisbench-admin compare <....> tool

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.1.4"
+version = "0.1.5"
 description = "Redis benchmark run helper. A wrapper around ftsb_redisearch ( future versions will also support redis-benchmark and memtier_benchmark )."
 authors = ["filipecosta90 <[email protected]>"]
 

redisbench_admin/cli.py

@@ -3,9 +3,12 @@
 
 import toml
 
-from redisbench_admin.compare.compare import create_compare_arguments, compare_command_logic
-from redisbench_admin.export.export import create_export_arguments, export_command_logic
-from redisbench_admin.run.run import create_run_arguments, run_command_logic
+from redisbench_admin.compare.args import create_compare_arguments
+from redisbench_admin.compare.compare import compare_command_logic
+from redisbench_admin.export.args import create_export_arguments
+from redisbench_admin.export.export import export_command_logic
+from redisbench_admin.run.args import create_run_arguments
+from redisbench_admin.run.run import run_command_logic
 
 
 def populate_with_poetry_data():
@@ -27,7 +30,10 @@ def main():
     parser = argparse.ArgumentParser(
         description=project_description,
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    # common arguments to all tools
     parser.add_argument('--version', default=False, action='store_true', help='print version and exit')
+    parser.add_argument('--local-dir', type=str, default="./", help='local dir to use as storage')
+
     if requested_tool == "run":
         parser = create_run_arguments(parser)
     elif requested_tool == "compare":
@@ -56,13 +62,10 @@ def main():
 
     argv = sys.argv[2:]
     args = parser.parse_args(args=argv)
-    if args.version:
-        print("{project_name} {project_version}".format(project_name=project_name, project_version=project_version))
-        sys.exit(0)
 
     if requested_tool == "run":
         run_command_logic(args)
     if requested_tool == "compare":
-        export_command_logic(args)
-    if requested_tool == "export":
         compare_command_logic(args)
+    if requested_tool == "export":
+        export_command_logic(args)

redisbench_admin/compare/args.py

@@ -0,0 +1,14 @@
+def create_compare_arguments(parser):
+    parser.add_argument('--baseline-file', type=str, required=True,
+                        help="baseline benchmark output file to read results from. can be a local file or a remote link.")
+    parser.add_argument('--comparison-file', type=str, required=True,
+                        help="comparison benchmark output file to read results from. can be a local file or a remote link.")
+    parser.add_argument('--use-result', type=str, default="median-result",
+                        help="for each key-metric, use either worst-result, best-result, or median-result")
+    parser.add_argument('--steps', type=str, default="setup,benchmark",
+                        help="comma separated list of steps to be analyzed given the benchmark result files")
+    parser.add_argument('--enable-fail-above', default=False, action='store_true',
+                        help="enables failing test if percentage of change is above threshold on any of the benchmark steps being analysed")
+    parser.add_argument('--fail-above-pct-change', type=float, default=10.0,
+                        help='Fail above if any of the key-metrics presents an regression in percentage of change (from 0.0-100.0)')
+    return parser

redisbench_admin/compare/compare.py

@@ -1,6 +1,139 @@
+import json
+import os
+import sys
+
+import pandas as pd
+
+from redisbench_admin.utils.utils import retrieve_local_or_remote_input_json
+
+
+def get_key_results_and_values(baseline_json, step, use_result):
+    selected_run = None
+    metrics = {}
+    for name, value in baseline_json["key-results"][step][use_result][0].items():
+        if name == "run-name":
+            selected_run = value
+        else:
+            metrics[name] = value
+    return selected_run, metrics
+
+
 def compare_command_logic(args):
-    pass
+    baseline_file = args.baseline_file
+    comparison_file = args.comparison_file
+    local_path = os.path.abspath(args.local_dir)
+    use_result = args.use_result
+    included_steps = args.steps.split(",")
+    max_pct_change = args.fail_above_pct_change
+    max_negative_pct_change = max_pct_change * -1.0
+    enabled_fail = args.enable_fail_above
+
+    baseline_json = retrieve_local_or_remote_input_json(baseline_file, local_path, "--baseline-file")
+    if baseline_json is None:
+        print('Error while retrieving {}! Exiting..'.format(baseline_file))
+        sys.exit(1)
+
+    comparison_json = retrieve_local_or_remote_input_json(comparison_file, local_path, "--comparison-file")
+    if comparison_json is None:
+        print('Error while retrieving {}! Exiting..'.format(comparison_file))
+        sys.exit(1)
+
+    ##### Comparison starts here #####
+    baseline_key_results_steps = baseline_json["key-results"].keys()
+    comparison_key_results_steps = comparison_json["key-results"].keys()
+    baseline_df_config = generate_comparison_dataframe_configs(baseline_json["benchmark-config"],
+                                                               baseline_key_results_steps)
+    comparison_df_config = generate_comparison_dataframe_configs(comparison_json["benchmark-config"],
+                                                                 comparison_key_results_steps)
+
+    percentange_change_map = {}
+    for step in baseline_key_results_steps:
+        if step in included_steps:
+            df_dict = {}
+            percentange_change_map[step] = {}
+            print("##############################")
+            print("Comparing {} step".format(step))
+            key_result_run_name, baseline_metrics = get_key_results_and_values(baseline_json, step, use_result)
+            key_result_run_name, comparison_metrics = get_key_results_and_values(comparison_json, step, use_result)
+            for baseline_metric_name, baseline_metric_value in baseline_metrics.items():
+                comparison_metric_value = None
+                if baseline_metric_name in comparison_metrics:
+                    comparison_metric_value = comparison_metrics[baseline_metric_name]
+                df_dict[baseline_metric_name] = [baseline_metric_value, comparison_metric_value]
+            df = pd.DataFrame(df_dict, index=["baseline", "comparison"])
+            print("Percentage of change for comparison on {}".format(step))
+            df = df.append(df.pct_change().rename(index={'comparison': 'pct_change'}).loc['pct_change'] * 100.0)
+
+            for metric_name, items in df.iteritems():
+
+                lower_is_better = baseline_df_config[step]["sorting_metric_sorting_direction_map"][metric_name]
+
+                multiplier = 1.0
+                # if lower is better than negative changes are and performance improvement
+                if lower_is_better:
+                    multiplier = -1.0
+
+                pct_change = items.get("pct_change") * multiplier
+                df.at['pct_change', metric_name] = pct_change
+                percentange_change_map[step][metric_name] = pct_change
+
+            print(df)
+            if enabled_fail:
+                failing_metrics_serie = df.loc['pct_change'] <= max_negative_pct_change
+                failing_metrics = df.loc['pct_change'][failing_metrics_serie]
+                ammount_of_failing_metrics = len (failing_metrics)
+                if ammount_of_failing_metrics > 0:
+                    df_keys = df.keys()
+                    # print(df.loc['pct_change'][0])
+                    # print([0])
+                    print( "There was a total of {} metrics that presented a regression above {} %".format(ammount_of_failing_metrics,max_pct_change) )
+                    for pos,failed in enumerate(failing_metrics_serie):
+                        if failed:
+                            print("\tMetric '{}' failed. with an percentage of change of {:.2f} %".format(df_keys[pos],df.loc['pct_change'][pos]))
+                    sys.exit(1)
+        else:
+            print("Skipping step: {} due to command line argument --steps not containing it ({})".format(step, ",".join(
+                included_steps)))
+
+
+def generate_comparison_dataframe_configs(benchmark_config, steps):
+    step_df_dict = {}
+    for step in steps:
+        step_df_dict[step] = {}
+        step_df_dict[step]["df_dict"] = {"run-name": []}
+        step_df_dict[step]["sorting_metric_names"] = []
+        step_df_dict[step]["sorting_metric_sorting_direction"] = []
+        step_df_dict[step]["sorting_metric_sorting_direction_map"] = {}
+        step_df_dict[step]["metric_json_path"] = []
+    for metric in benchmark_config["key-metrics"]:
+        step = metric["step"]
+        metric_name = metric["metric-name"]
+        metric_json_path = metric["metric-json-path"]
+        step_df_dict[step]["sorting_metric_names"].append(metric_name)
+        step_df_dict[step]["metric_json_path"].append(metric_json_path)
+        step_df_dict[step]["df_dict"][metric_name] = []
+        step_df_dict[step]["sorting_metric_sorting_direction"].append(
+            False if metric["comparison"] == "higher-better" else True)
+        step_df_dict[step]["sorting_metric_sorting_direction_map"][metric_name] = False if metric[
+                                                                                               "comparison"] == "higher-better" else True
+    return step_df_dict
 
 
-def create_compare_arguments(parser):
-    return parser
+def from_resultsDF_to_key_results_dict(resultsDataFrame, step, step_df_dict):
+    key_results_dict = {}
+    key_results_dict["table"] = json.loads(resultsDataFrame.to_json(orient='records'))
+    best_result = resultsDataFrame.head(n=1)
+    worst_result = resultsDataFrame.tail(n=1)
+    first_sorting_col = step_df_dict[step]["sorting_metric_names"][0]
+    first_sorting_median = resultsDataFrame[first_sorting_col].median()
+    result_index = resultsDataFrame[first_sorting_col].sub(first_sorting_median).abs().idxmin()
+    median_result = resultsDataFrame.loc[[result_index]]
+    key_results_dict["best-result"] = json.loads(best_result.to_json(orient='records'))
+    key_results_dict["median-result"] = json.loads(
+        median_result.to_json(orient='records'))
+    key_results_dict["worst-result"] = json.loads(worst_result.to_json(orient='records'))
+    key_results_dict["reliability-analysis"] = {
+        'var': json.loads(resultsDataFrame.var().to_json()),
+        'stddev': json.loads(
+            resultsDataFrame.std().to_json())}
+    return key_results_dict

redisbench_admin/export/args.py

@@ -0,0 +1,2 @@
+def create_export_arguments(parser):
+    return parser

redisbench_admin/export/export.py

@@ -1,6 +1,2 @@
 def export_command_logic(args):
     pass
-
-
-def create_export_arguments(parser):
-    return parser

redisbench_admin/run/args.py

@@ -0,0 +1,18 @@
+def create_run_arguments(parser):
+    parser.add_argument('--benchmark-config-file', type=str, required=True,
+                        help="benchmark config file to read instructions from. can be a local file or a remote link")
+    parser.add_argument('--workers', type=str, default=0,
+                        help='number of workers to use during the benchark. If set to 0 it will auto adjust based on the machine number of VCPUs')
+    parser.add_argument('--repetitions', type=int, default=1,
+                        help='number of repetitions to run')
+    parser.add_argument('--benchmark-requests', type=int, default=0,
+                        help='Number of total requests to issue (0 = all of the present in input file)')
+    parser.add_argument('--upload-results-s3', default=False, action='store_true',
+                        help="uploads the result files and configuration file to public benchmarks.redislabs bucket. Proper credentials are required")
+    parser.add_argument('--redis-url', type=str, default="redis://localhost:6379", help='The url for Redis connection')
+    parser.add_argument('--deployment-type', type=str, default="docker-oss",
+                        help='one of docker-oss,docker-oss-cluster,docker-enterprise,oss,oss-cluster,enterprise')
+    parser.add_argument('--deployment-shards', type=int, default=1,
+                        help='number of database shards used in the deployment')
+    parser.add_argument('--output-file-prefix', type=str, default="", help='prefix to quickly tag some files')
+    return parser

redisbench_admin/run/ftsb_redisearch/__init__.py

@@ -0,0 +1,40 @@
+import json
+import os
+import subprocess
+import sys
+
+
+def get_run_options():
+    environ = os.environ.copy()
+    stdoutPipe = subprocess.PIPE
+    stderrPipe = subprocess.STDOUT
+    stdinPipe = subprocess.PIPE
+    options = {
+        'stderr': stderrPipe,
+        'env': environ,
+    }
+    return options
+
+
+def run_ftsb_redisearch(redis_url, ftsb_redisearch_path, setup_run_json_output_fullpath, options, input_file,
+                        workers):
+    ##################
+    # Setup commands #
+    ##################
+    output_json = None
+    ftsb_args = []
+    ftsb_args += [ftsb_redisearch_path, "--host={}".format(redis_url),
+                  "--input={}".format(input_file), "--workers={}".format(workers),
+                  "--json-out-file={}".format(setup_run_json_output_fullpath)]
+    ftsb_process = subprocess.Popen(args=ftsb_args, **options)
+    if ftsb_process.poll() is not None:
+        print('Error while issuing setup commands. FTSB process is not alive. Exiting..')
+        sys.exit(1)
+    output = ftsb_process.communicate()
+    if ftsb_process.returncode != 0:
+        print('FTSB process returned non-zero exit status {}. Exiting..'.format(ftsb_process.returncode))
+        print('catched output:\n\t{}'.format(output))
+        sys.exit(1)
+    with open(setup_run_json_output_fullpath) as json_result:
+        output_json = json.load(json_result)
+    return output_json