Added profile data HASH docs (#376)

* Fixed memtier when DB is password protected

* Fixed pycpuinfo error on arm darwin setups

* Added profile data HASH docs

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.9.19"
+version = "0.9.23"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "README.md"

redisbench_admin/compare/args.py

@@ -47,6 +47,7 @@ def create_compare_arguments(parser):
     parser.add_argument("--baseline_deployment_name", type=str, default="")
     parser.add_argument("--comparison_deployment_name", type=str, default="")
     parser.add_argument("--metric_name", type=str, default="Tests.Overall.rps")
+    parser.add_argument("--extra-filter", type=str, default=None)
     parser.add_argument(
         "--last_n",
         type=int,
@@ -76,6 +77,7 @@ def create_compare_arguments(parser):
     parser.add_argument("--comparison-tag", type=str, default=None, required=False)
     parser.add_argument("--print-regressions-only", type=bool, default=False)
     parser.add_argument("--verbose", type=bool, default=False)
+    parser.add_argument("--simple-table", type=bool, default=False)
     parser.add_argument("--use_metric_context_path", type=bool, default=False)
     parser.add_argument("--testname_regex", type=str, default=".*", required=False)
     parser.add_argument(

redisbench_admin/compare/compare.py

@@ -77,6 +77,7 @@ def compare_command_logic(args, project_name, project_version):
     use_branch = False
     baseline_branch = args.baseline_branch
     comparison_branch = args.comparison_branch
+    simplify_table = args.simple_table
     by_str = ""
     baseline_str = ""
     comparison_str = ""
@@ -161,7 +162,7 @@ def compare_command_logic(args, project_name, project_version):
                 used_key, len(test_names)
             )
         )
-    profilers_artifacts_matrix = []
+    table = []
     detected_regressions = []
     total_improvements = 0
     total_stable = 0
@@ -192,18 +193,53 @@ def compare_command_logic(args, project_name, project_version):
         baseline_timeseries = [x for x in baseline_timeseries if "target" not in x]
         progress.update()
         if args.verbose:
-            logging.info("Baseline timeseries {}".format(len(baseline_timeseries)))
-            logging.info("Comparison timeseries {}".format(len(comparison_timeseries)))
+            logging.info(
+                "Baseline timeseries for {}: {}. test={}".format(
+                    baseline_str, len(baseline_timeseries), test_name
+                )
+            )
+            logging.info(
+                "Comparison timeseries for {}: {}. test={}".format(
+                    comparison_str, len(comparison_timeseries), test_name
+                )
+            )
+        if len(baseline_timeseries) > 1:
+            logging.warning(
+                "\t\tTime-series: {}".format(", ".join(baseline_timeseries))
+            )
+            logging.info("Checking if Totals will reduce timeseries.")
+            new_base = []
+            for ts_name in baseline_timeseries:
+                if "Totals" in ts_name:
+                    new_base.append(ts_name)
+            baseline_timeseries = new_base
+
         if len(baseline_timeseries) != 1:
             if args.verbose:
                 logging.warning(
                     "Skipping this test given the value of timeseries !=1. Baseline timeseries {}".format(
                         len(baseline_timeseries)
                     )
                 )
+                if len(baseline_timeseries) > 1:
+                    logging.warning(
+                        "\t\tTime-series: {}".format(", ".join(baseline_timeseries))
+                    )
+
             continue
         else:
             ts_name_baseline = baseline_timeseries[0]
+
+        if len(comparison_timeseries) > 1:
+            logging.warning(
+                "\t\tTime-series: {}".format(", ".join(comparison_timeseries))
+            )
+            logging.info("Checking if Totals will reduce timeseries.")
+            new_base = []
+            for ts_name in comparison_timeseries:
+                if "Totals" in ts_name:
+                    new_base.append(ts_name)
+            comparison_timeseries = new_base
         if len(comparison_timeseries) != 1:
             if args.verbose:
                 logging.warning(
@@ -236,39 +272,34 @@ def compare_command_logic(args, project_name, project_version):
             baseline_datapoints = rts.ts().revrange(
                 ts_name_baseline, from_ts_ms, to_ts_ms
             )
-            baseline_nsamples = len(baseline_datapoints)
-            if baseline_nsamples > 0:
-                _, baseline_v = baseline_datapoints[0]
-                for tuple in baseline_datapoints:
-                    if args.last_n < 0 or (
-                        args.last_n > 0 and len(baseline_values) < args.last_n
-                    ):
-                        baseline_values.append(tuple[1])
-                baseline_df = pd.DataFrame(baseline_values)
-                baseline_median = float(baseline_df.median())
-                baseline_v = baseline_median
-                baseline_std = float(baseline_df.std())
-                baseline_pct_change = (baseline_std / baseline_median) * 100.0
-                largest_variance = baseline_pct_change
+            (
+                baseline_pct_change,
+                baseline_v,
+                largest_variance,
+            ) = get_v_pct_change_and_largest_var(
+                args,
+                baseline_datapoints,
+                baseline_pct_change,
+                baseline_v,
+                baseline_values,
+                largest_variance,
+            )
 
             comparison_datapoints = rts.ts().revrange(
                 ts_name_comparison, from_ts_ms, to_ts_ms
             )
-            comparison_nsamples = len(comparison_datapoints)
-            if comparison_nsamples > 0:
-                _, comparison_v = comparison_datapoints[0]
-                for tuple in comparison_datapoints:
-                    if args.last_n < 0 or (
-                        args.last_n > 0 and len(comparison_values) < args.last_n
-                    ):
-                        comparison_values.append(tuple[1])
-                comparison_df = pd.DataFrame(comparison_values)
-                comparison_median = float(comparison_df.median())
-                comparison_v = comparison_median
-                comparison_std = float(comparison_df.std())
-                comparison_pct_change = (comparison_std / comparison_median) * 100.0
-                if comparison_pct_change > largest_variance:
-                    largest_variance = comparison_pct_change
+            (
+                comparison_pct_change,
+                comparison_v,
+                largest_variance,
+            ) = get_v_pct_change_and_largest_var(
+                args,
+                comparison_datapoints,
+                comparison_pct_change,
+                comparison_v,
+                comparison_values,
+                largest_variance,
+            )
 
             waterline = args.regressions_percent_lower_limit
             if args.regressions_percent_lower_limit < largest_variance:
@@ -277,6 +308,9 @@ def compare_command_logic(args, project_name, project_version):
 
         except redis.exceptions.ResponseError:
             pass
+        except ZeroDivisionError as e:
+            logging.error("Detected a ZeroDivisionError. {}".format(e.__str__()))
+            pass
         unstable = False
         if baseline_v != "N/A" and comparison_v != "N/A":
             stamp_b = ""
@@ -286,15 +320,21 @@ def compare_command_logic(args, project_name, project_version):
                 unstable = True
             if baseline_pct_change > 10.0:
                 stamp_b = "UNSTABLE"
-            baseline_v_str = " {:.0f} +- {:.1f}% {} ({} datapoints)".format(
-                baseline_v, baseline_pct_change, stamp_b, len(baseline_values)
-            )
+            if simplify_table:
+                baseline_v_str = " {:.0f}".format(baseline_v)
+            else:
+                baseline_v_str = " {:.0f} +- {:.1f}% {} ({} datapoints)".format(
+                    baseline_v, baseline_pct_change, stamp_b, len(baseline_values)
+                )
             stamp_c = ""
             if comparison_pct_change > 10.0:
                 stamp_c = "UNSTABLE"
-            comparison_v_str = " {:.0f} +- {:.1f}% {} ({} datapoints)".format(
-                comparison_v, comparison_pct_change, stamp_c, len(comparison_values)
-            )
+            if simplify_table:
+                comparison_v_str = " {:.0f}".format(comparison_v)
+            else:
+                comparison_v_str = " {:.0f} +- {:.1f}% {} ({} datapoints)".format(
+                    comparison_v, comparison_pct_change, stamp_c, len(comparison_values)
+                )
             if metric_mode == "higher-better":
                 percentage_change = (
                     float(comparison_v) / float(baseline_v) - 1
@@ -340,15 +380,25 @@ def compare_command_logic(args, project_name, project_version):
 
             if args.print_regressions_only is False or detected_regression:
                 percentage_change_str = "{:.1f}% ".format(percentage_change)
-                profilers_artifacts_matrix.append(
-                    [
-                        test_name,
-                        baseline_v_str,
-                        comparison_v_str,
-                        percentage_change_str,
-                        note.strip(),
-                    ]
-                )
+                if simplify_table:
+                    table.append(
+                        [
+                            test_name,
+                            baseline_v_str,
+                            comparison_v_str,
+                            percentage_change_str,
+                        ]
+                    )
+                else:
+                    table.append(
+                        [
+                            test_name,
+                            baseline_v_str,
+                            comparison_v_str,
+                            percentage_change_str,
+                            note.strip(),
+                        ]
+                    )
 
     logging.info("Printing differential analysis between branches")
 
@@ -370,7 +420,7 @@ def compare_command_logic(args, project_name, project_version):
             "% change ({})".format(metric_mode),
             "Note",
         ],
-        value_matrix=profilers_artifacts_matrix,
+        value_matrix=table,
     )
     writer.write_table()
     if total_stable > 0:
@@ -401,3 +451,38 @@ def compare_command_logic(args, project_name, project_version):
                 ",".join(["{}.yml".format(x) for x in detected_regressions])
             )
         )
+
+
+def get_v_pct_change_and_largest_var(
+    args,
+    comparison_datapoints,
+    comparison_pct_change,
+    comparison_v,
+    comparison_values,
+    largest_variance,
+):
+    comparison_nsamples = len(comparison_datapoints)
+    if comparison_nsamples > 0:
+        _, comparison_v = comparison_datapoints[0]
+        for tuple in comparison_datapoints:
+            if args.last_n < 0 or (
+                args.last_n > 0 and len(comparison_values) < args.last_n
+            ):
+                comparison_values.append(tuple[1])
+        comparison_df = pd.DataFrame(comparison_values)
+        comparison_median = float(comparison_df.median())
+        comparison_v = comparison_median
+        comparison_std = float(comparison_df.std())
+        if args.verbose:
+            logging.info(
+                "comparison_datapoints: {} value: {}; std-dev: {}; median: {}".format(
+                    comparison_datapoints,
+                    comparison_v,
+                    comparison_std,
+                    comparison_median,
+                )
+            )
+        comparison_pct_change = (comparison_std / comparison_median) * 100.0
+        if comparison_pct_change > largest_variance:
+            largest_variance = comparison_pct_change
+    return comparison_pct_change, comparison_v, largest_variance

redisbench_admin/profilers/profilers_local.py

@@ -10,7 +10,6 @@
 import logging
 import platform
 
-from cpuinfo import get_cpu_info
 
 from redisbench_admin.profilers.perf import Perf
 from redisbench_admin.profilers.vtune import Vtune
@@ -283,25 +282,32 @@ def get_profilers_map(profilers_list, total_involved_processes, max_profilers=1)
 def local_profilers_platform_checks(
     dso, github_actor, github_branch, github_repo_name, github_sha
 ):
+    collection_summary_str = ""
     logging.info("Using dso for perf analysis {}".format(dso))
-    local_platform_info = get_cpu_info()
-    cpu_brand = local_platform_info["brand"]
-    cpu_core_count = local_platform_info["count"]
-    platform_uname_release = platform.uname().release
-    platform_uname_system = platform.uname().system
-    platform_uname_node = platform.uname().node
-    span_x = 800
-    collection_summary_str = (
-        '<tspan x="{}" dy="1.2em">Collection platform: system=\'{}\''.format(
-            span_x, platform_uname_system
-        )
-        + " release='{}', node='{}', cpu='{}', core-count={}</tspan>".format(
-            platform_uname_release,
-            platform_uname_node,
-            cpu_brand,
-            cpu_core_count,
+    try:
+        from cpuinfo import get_cpu_info
+
+        local_platform_info = get_cpu_info()
+        cpu_brand = local_platform_info["brand"]
+        cpu_core_count = local_platform_info["count"]
+        platform_uname_release = platform.uname().release
+        platform_uname_system = platform.uname().system
+        platform_uname_node = platform.uname().node
+        span_x = 800
+        collection_summary_str = (
+            '<tspan x="{}" dy="1.2em">Collection platform: system=\'{}\''.format(
+                span_x, platform_uname_system
+            )
+            + " release='{}', node='{}', cpu='{}', core-count={}</tspan>".format(
+                platform_uname_release,
+                platform_uname_node,
+                cpu_brand,
+                cpu_core_count,
+            )
         )
-    )
+    except Exception as e:
+        logging.error("Unable to retrive platform info. Error: {}".format(e.__str__()))
+        pass
     collection_summary_str += (
         '<tspan x="{}" dy="1.2em">Collection trigger: github_actor=\'{}\' '.format(
             span_x, github_actor

redisbench_admin/run/args.py

@@ -38,12 +38,7 @@
 MAX_PROFILERS_PER_TYPE = int(os.getenv("MAX_PROFILERS", 1))
 PROFILE_FREQ = os.getenv("PROFILE_FREQ", PROFILE_FREQ_DEFAULT)
 KEEP_ENV = bool(os.getenv("KEEP_ENV", False))
-ALLOWED_TOOLS_DEFAULT = (
-    "memtier_benchmark,redis-benchmark,redisgraph-benchmark-go,ycsb,go-ycsb,"
-    + "tsbs_run_queries_redistimeseries,tsbs_load_redistimeseries,"
-    + "ftsb_redisearch,"
-    + "aibench_run_inference_redisai_vision,ann-benchmarks",
-)
+ALLOWED_TOOLS_DEFAULT = "memtier_benchmark,redis-benchmark,redisgraph-benchmark-go,ycsb,go-ycsb,tsbs_run_queries_redistimeseries,tsbs_load_redistimeseries,ftsb_redisearch,aibench_run_inference_redisai_vision,ann-benchmarks"
 ALLOWED_BENCH_TOOLS = os.getenv("ALLOWED_BENCH_TOOLS", ALLOWED_TOOLS_DEFAULT)
 
 

redisbench_admin/run/common.py

@@ -37,6 +37,7 @@
     prepare_ycsb_benchmark_command,
     prepare_go_ycsb_benchmark_command,
 )
+from redisbench_admin.run_remote.args import OVERRIDE_MODULES
 from redisbench_admin.run_remote.remote_helpers import (
     extract_module_semver_from_info_modules_cmd,
 )
@@ -544,6 +545,8 @@ def run_redis_pre_steps(benchmark_config, r, required_modules):
         module_names,
         artifact_versions,
     ) = extract_module_semver_from_info_modules_cmd(stdout)
+    if OVERRIDE_MODULES is not None:
+        module_names = OVERRIDE_MODULES.split(",")
     if "search" in module_names:
         logging.info(
             "Detected redisearch module. Ensuring all indices are indexed prior benchmark"
@@ -619,7 +622,9 @@ def dso_check(dso, local_module_file):
     return dso
 
 
-def dbconfig_keyspacelen_check(benchmark_config, redis_conns):
+def dbconfig_keyspacelen_check(
+    benchmark_config, redis_conns, ignore_keyspace_errors=False
+):
     result = True
     (
         requires_keyspacelen_check,
@@ -650,11 +655,12 @@ def dbconfig_keyspacelen_check(benchmark_config, redis_conns):
                     keyspacelen, total_keys
                 )
             )
-            raise Exception(
-                "The total numbers of keys in setup does not match the expected spec: {}!={}. Aborting...".format(
-                    keyspacelen, total_keys
+            if ignore_keyspace_errors is False:
+                raise Exception(
+                    "The total numbers of keys in setup does not match the expected spec: {}!={}. Aborting...".format(
+                        keyspacelen, total_keys
+                    )
                 )
-            )
     return result