Further improved compare mode to focus on regressions only (#152)

* [add] Further improved compare mode to focus on regressions only

* Bumping version from 0.2.14 to 0.2.15

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.2.14"
+version = "0.2.15"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>"]
 readme = "README.md"

redisbench_admin/compare/args.py

@@ -42,6 +42,13 @@ def create_compare_arguments(parser):
     parser.add_argument("--metric_mode", type=str, default="higher-better")
     parser.add_argument("--baseline-branch", type=str, default=None, required=True)
     parser.add_argument("--comparison-branch", type=str, default=None, required=True)
+    parser.add_argument("--print-regressions-only", type=bool, default=False)
+    parser.add_argument(
+        "--regressions-percent-lower-limit",
+        type=float,
+        default=5.0,
+        help="Only consider regressions with a percentage over the defined limit. (0-100)",
+    )
     parser.add_argument(
         "--redistimeseries_host", type=str, default=PERFORMANCE_RTS_HOST
     )

redisbench_admin/compare/compare.py

@@ -58,6 +58,7 @@ def compare_command_logic(args):
         )
     )
     profilers_artifacts_matrix = []
+    detected_regressions = []
     for test_name in test_names:
 
         test_name = test_name.decode()
@@ -99,25 +100,35 @@ def compare_command_logic(args):
         except redis.exceptions.ResponseError:
             pass
         percentage_change = "N/A"
+        percentage_change = 0.0
         if baseline_v != "N/A" and comparison_v != "N/A":
             if metric_mode == "higher-better":
-                percentage_change = "{0:.2f} %".format(
-                    (float(comparison_v) / float(baseline_v) - 1) * 100.0
-                )
+                percentage_change = (
+                    float(comparison_v) / float(baseline_v) - 1
+                ) * 100.0
             else:
                 # lower-better
-                percentage_change = "{0:.2f} %".format(
-                    (float(baseline_v) / float(comparison_v) - 1) * 100.0
-                )
+                percentage_change = (
+                    float(baseline_v) / float(comparison_v) - 1
+                ) * 100.0
         if baseline_v != "N/A" or comparison_v != "N/A":
-            profilers_artifacts_matrix.append(
-                [
-                    test_name,
-                    baseline_v,
-                    comparison_v,
-                    percentage_change,
-                ]
-            )
+            detected_regression = False
+            if (
+                percentage_change < 0.0
+                and percentage_change < -args.regressions_percent_lower_limit
+            ):
+                detected_regression = True
+                detected_regressions.append(test_name)
+            if args.print_regressions_only is False or detected_regression:
+                profilers_artifacts_matrix.append(
+                    [
+                        test_name,
+                        baseline_v,
+                        comparison_v,
+                        percentage_change,
+                    ]
+                )
+
     logging.info("Printing differential analysis between branches")
 
     writer = MarkdownTableWriter(
@@ -133,3 +144,15 @@ def compare_command_logic(args):
         value_matrix=profilers_artifacts_matrix,
     )
     writer.write_table()
+    total_regressions = len(detected_regressions)
+    if total_regressions > 0:
+        logging.warning(
+            "Detected a total of {} regressions. Printing BENCHMARK env var compatible list".format(
+                total_regressions
+            )
+        )
+        logging.warning(
+            "BENCHMARK={}".format(
+                ",".join(["{}.yml".format(x) for x in detected_regressions])
+            )
+        )