WIP on result comparison between two branches (#151)

* [add] Enabled logfile on remote setups. Fetching remote logfile in case of errors

* [add] remote results file should not be the same across variations

* [wip] WIP on comparison between two branches

* [add] Improve redisbench-admin compare mode

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.2.13"
+version = "0.2.14"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>"]
 readme = "README.md"

redisbench_admin/cli.py

@@ -12,6 +12,8 @@
 import toml
 
 from redisbench_admin import __version__
+from redisbench_admin.compare.args import create_compare_arguments
+from redisbench_admin.compare.compare import compare_command_logic
 from redisbench_admin.export.args import create_export_arguments
 from redisbench_admin.export.export import export_command_logic
 from redisbench_admin.extract.args import create_extract_arguments
@@ -78,6 +80,8 @@ def main():
         parser = create_extract_arguments(parser)
     elif requested_tool == "export":
         parser = create_export_arguments(parser)
+    elif requested_tool == "compare":
+        parser = create_compare_arguments(parser)
     elif requested_tool == "watchdog":
         parser = create_watchdog_arguments(parser)
     elif requested_tool == "--version":
@@ -88,6 +92,7 @@ def main():
         sys.exit(0)
     else:
         valid_tool_options = [
+            "compare",
             "run-local",
             "run-remote",
             "export",
@@ -127,6 +132,8 @@ def main():
         extract_command_logic(args)
     if requested_tool == "watchdog":
         watchdog_command_logic(args)
+    if requested_tool == "compare":
+        compare_command_logic(args)
 
 
 def print_invalid_tool_option(requested_tool, valid_tool_options):

redisbench_admin/compare/args.py

@@ -0,0 +1,56 @@
+#  Apache License Version 2.0
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#
+
+# environment variables
+from redisbench_admin.run.common import get_start_time_vars
+from redisbench_admin.run_remote.args import TRIGGERING_ENV
+from redisbench_admin.utils.remote import (
+    PERFORMANCE_RTS_HOST,
+    PERFORMANCE_RTS_PORT,
+    PERFORMANCE_RTS_AUTH,
+    extract_git_vars,
+)
+
+(
+    GITHUB_ORG,
+    GITHUB_REPO,
+    _,
+    _,
+    _,
+    _,
+) = extract_git_vars()
+
+_, NOW_UTC, _ = get_start_time_vars()
+LAST_24_HOURS_UTC = NOW_UTC - (24 * 60 * 60 * 1000)
+
+
+def create_compare_arguments(parser):
+    parser.add_argument(
+        "--test",
+        type=str,
+        default="",
+        help="specify a test to use for comparison. If none is specified by default will use all of them.",
+    )
+    parser.add_argument("--github_repo", type=str, default=GITHUB_REPO)
+    parser.add_argument("--github_org", type=str, default=GITHUB_ORG)
+    parser.add_argument("--triggering_env", type=str, default=TRIGGERING_ENV)
+    parser.add_argument("--deployment_type", type=str, default="oss-standalone")
+    parser.add_argument("--metric_name", type=str, default="Tests.Overall.rps")
+    parser.add_argument("--metric_mode", type=str, default="higher-better")
+    parser.add_argument("--baseline-branch", type=str, default=None, required=True)
+    parser.add_argument("--comparison-branch", type=str, default=None, required=True)
+    parser.add_argument(
+        "--redistimeseries_host", type=str, default=PERFORMANCE_RTS_HOST
+    )
+    parser.add_argument(
+        "--redistimeseries_port", type=int, default=PERFORMANCE_RTS_PORT
+    )
+    parser.add_argument(
+        "--redistimeseries_pass", type=str, default=PERFORMANCE_RTS_AUTH
+    )
+    parser.add_argument("--from_timestamp", type=int, default=LAST_24_HOURS_UTC)
+    parser.add_argument("--to_timestamp", type=int, default=NOW_UTC)
+    return parser

redisbench_admin/compare/compare.py

@@ -0,0 +1,135 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#
+import logging
+
+import redis
+from pytablewriter import MarkdownTableWriter
+
+from redisbench_admin.utils.remote import get_overall_dashboard_keynames
+from redistimeseries.client import Client
+
+from redisbench_admin.utils.utils import get_ts_metric_name
+
+
+def compare_command_logic(args):
+    logging.info("Checking connection to RedisTimeSeries.")
+    rts = Client(
+        host=args.redistimeseries_host,
+        port=args.redistimeseries_port,
+        password=args.redistimeseries_pass,
+    )
+    rts.redis.ping()
+
+    tf_github_org = args.github_org
+    tf_github_repo = args.github_repo
+    tf_triggering_env = args.triggering_env
+    deployment_type = args.deployment_type
+    from_ts_ms = args.from_timestamp
+    to_ts_ms = args.to_timestamp
+    baseline_branch = args.baseline_branch
+    comparison_branch = args.comparison_branch
+    metric_name = args.metric_name
+    metric_mode = args.metric_mode
+    (
+        prefix,
+        testcases_setname,
+        tsname_project_total_failures,
+        tsname_project_total_success,
+    ) = get_overall_dashboard_keynames(tf_github_org, tf_github_repo, tf_triggering_env)
+    test_names = []
+    try:
+        test_names = rts.redis.smembers(testcases_setname)
+        test_names = list(test_names)
+        test_names.sort()
+    except redis.exceptions.ResponseError as e:
+        logging.warning(
+            "Error while trying to fetch test cases set (key={}) {}. ".format(
+                testcases_setname, e.__str__()
+            )
+        )
+        pass
+
+    logging.warning(
+        "Based on test-cases set (key={}) we have {} distinct benchmarks. ".format(
+            testcases_setname, len(test_names)
+        )
+    )
+    profilers_artifacts_matrix = []
+    for test_name in test_names:
+
+        test_name = test_name.decode()
+
+        ts_name_baseline = get_ts_metric_name(
+            "by.branch",
+            baseline_branch,
+            tf_github_org,
+            tf_github_repo,
+            deployment_type,
+            test_name,
+            tf_triggering_env,
+            metric_name,
+        )
+        ts_name_comparison = get_ts_metric_name(
+            "by.branch",
+            comparison_branch,
+            tf_github_org,
+            tf_github_repo,
+            deployment_type,
+            test_name,
+            tf_triggering_env,
+            metric_name,
+        )
+        baseline_v = "N/A"
+        comparison_v = "N/A"
+        try:
+
+            baseline_datapoints = rts.revrange(
+                ts_name_baseline, from_ts_ms, to_ts_ms, count=1
+            )
+            if len(baseline_datapoints) > 0:
+                _, baseline_v = baseline_datapoints[0]
+            comparison_datapoints = rts.revrange(
+                ts_name_comparison, from_ts_ms, to_ts_ms, count=1
+            )
+            if len(comparison_datapoints) > 0:
+                _, comparison_v = comparison_datapoints[0]
+        except redis.exceptions.ResponseError:
+            pass
+        percentage_change = "N/A"
+        if baseline_v != "N/A" and comparison_v != "N/A":
+            if metric_mode == "higher-better":
+                percentage_change = "{0:.2f} %".format(
+                    (float(comparison_v) / float(baseline_v) - 1) * 100.0
+                )
+            else:
+                # lower-better
+                percentage_change = "{0:.2f} %".format(
+                    (float(baseline_v) / float(comparison_v) - 1) * 100.0
+                )
+        if baseline_v != "N/A" or comparison_v != "N/A":
+            profilers_artifacts_matrix.append(
+                [
+                    test_name,
+                    baseline_v,
+                    comparison_v,
+                    percentage_change,
+                ]
+            )
+    logging.info("Printing differential analysis between branches")
+
+    writer = MarkdownTableWriter(
+        table_name="Comparison between {} and {} for metric: {}".format(
+            baseline_branch, comparison_branch, metric_name
+        ),
+        headers=[
+            "Test Case",
+            "Baseline value",
+            "Comparison Value",
+            "% change ({})".format(metric_mode),
+        ],
+        value_matrix=profilers_artifacts_matrix,
+    )
+    writer.write_table()

redisbench_admin/utils/benchmark_config.py

@@ -230,8 +230,8 @@ def get_testfiles_to_process(args):
             )
         )
     else:
-        logging.info("Running specific benchmark in file: {}".format(args.test))
-        files = [args.test]
+        files = args.test.split(",")
+        logging.info("Running specific benchmark in file: {}".format(files))
     return defaults_filename, files
 
 

redisbench_admin/watchdog/args.py

@@ -29,6 +29,12 @@ def create_watchdog_arguments(parser):
         default=60,
         help="watchdog update interval in seconds",
     )
+    parser.add_argument(
+        "--terminate-after-secs",
+        type=int,
+        default=4200,
+        help="watchdog will terminate machines running for longer than value (that are associated with ci benchmarks)",
+    )
     parser.add_argument(
         "--redistimeseries_host", type=str, default=PERFORMANCE_RTS_HOST
     )

redisbench_admin/watchdog/watchdog.py

@@ -19,7 +19,6 @@
 )
 from redisbench_admin.utils.utils import EC2_REGION, EC2_SECRET_KEY, EC2_ACCESS_KEY
 
-terminate_after_secs = 45 * 60
 dry_run = True
 ci_machines_prefix = "/tmp/"
 
@@ -96,6 +95,7 @@ def watchdog_command_logic(args):
     cloud = "aws"
     prefix = "ci.benchmarks.redislabs/{}/{}".format(cloud, EC2_REGION)
     tsname_overall_running = "{}/state-running".format(prefix)
+    terminate_after_secs = args.terminate_after_secs
     check_ec2_env()
     boto3.setup_default_session(
         region_name=EC2_REGION,