Added watchdog feature to keep track of current aws bench VMs. Enabled tracking rdb load time (#131)

* [add] Included track of benchmark duration. Small refactor/reuse of code within break/by timeseries

* Bumping version from 0.2.1 to 0.2.2

* [add] added watchdog feature to keep track of current aws bench VMs. Enabled tracking rdb load time

Author: filipecosta90

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.2.2"
+version = "0.2.3"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>"]
 readme = "README.md"
@@ -27,6 +27,7 @@ GitPython = "^3.1.12"
 PyYAML = "^5.4.0"
 wget = "^3.2"
 pytablewriter = "^0.60.0"
+sshtunnel = "^0.4.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^4.6"

pyproject.toml

@@ -19,6 +19,8 @@
 from redisbench_admin.run_local.run_local import run_local_command_logic
 from redisbench_admin.run_remote.args import create_run_remote_arguments, LOG_LEVEL
 from redisbench_admin.run_remote.run_remote import run_remote_command_logic
+from redisbench_admin.watchdog.args import create_watchdog_arguments
+from redisbench_admin.watchdog.watchdog import watchdog_command_logic
 
 
 def populate_with_poetry_data():
@@ -73,14 +75,22 @@ def main():
         parser = create_extract_arguments(parser)
     elif requested_tool == "export":
         parser = create_export_arguments(parser)
+    elif requested_tool == "watchdog":
+        parser = create_watchdog_arguments(parser)
     elif requested_tool == "--version":
         print_version(project_name, project_version)
         sys.exit(0)
     elif requested_tool == "--help":
         print_help(project_name, project_version)
         sys.exit(0)
     else:
-        valid_tool_options = ["run-local", "run-remote", "export", "extract"]
+        valid_tool_options = [
+            "run-local",
+            "run-remote",
+            "export",
+            "extract",
+            "watchdog",
+        ]
         print_invalid_tool_option(requested_tool, valid_tool_options)
         sys.exit(1)
 
@@ -94,6 +104,8 @@ def main():
         export_command_logic(args)
     if requested_tool == "extract":
         extract_command_logic(args)
+    if requested_tool == "watchdog":
+        watchdog_command_logic(args)
 
 
 def print_invalid_tool_option(requested_tool, valid_tool_options):

redisbench_admin/cli.py

@@ -0,0 +1,5 @@
+#  Apache License Version 2.0
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#

redisbench_admin/compare/__init__.py

@@ -72,10 +72,16 @@ def split_key_metrics_by_step(key_metrics_specs):
     return key_metrics_by_step
 
 
-def get_or_none(dictionary, prop: str):
+def get_or_none(dictionary, prop: str, inner_prop=None):
     result = None
     if prop in dictionary:
         result = dictionary[prop]
+        if inner_prop is not None:
+            if inner_prop in result:
+                result = result[inner_prop]
+            else:
+                None
+
     return result
 
 

redisbench_admin/export/common/common.py

@@ -10,6 +10,7 @@
 import subprocess
 import time
 
+
 from redisbench_admin.profilers.pprof import (
     PPROF_FORMAT_TEXT,
     run_pprof,
@@ -19,6 +20,18 @@
 from redisbench_admin.utils.utils import whereis
 
 
+PERF_CALLGRAPH_MODE_DEFAULT = "lbr"
+LINUX_PERF_SETTINGS_MESSAGE = (
+    "If running in non-root user please confirm that you have:\n"
+    + " - access to Kernel address maps."
+    + " Check if `0` ( disabled ) appears from the output of `cat /proc/sys/kernel/kptr_restrict`\n"
+    + '          If not then fix via: `sudo sh -c " echo 0 > /proc/sys/kernel/kptr_restrict"`\n'
+    + " - permission to collect stats."
+    + " Check if `-1` appears from the output of `cat /proc/sys/kernel/perf_event_paranoid`\n"
+    + "          If not then fix via: `sudo sh -c 'echo -1 > /proc/sys/kernel/perf_event_paranoid'`\n"
+)
+
+
 class Perf:
     def __init__(self):
         """
@@ -29,13 +42,11 @@ def __init__(self):
         if not self.perf:
             self.perf = "perf"
 
-        self.stack_collapser = os.getenv("STACKCOLLAPSE_PATH")
-        if not self.stack_collapser:
-            self.stack_collapser = STACKCOLLAPSE_PATH
-
-        self.flamegraph_utity = os.getenv("FLAMEGRAPH_PATH")
-        if not self.flamegraph_utity:
-            self.flamegraph_utity = FLAMEGRAPH_PATH
+        self.stack_collapser = os.getenv("STACKCOLLAPSE_PATH", STACKCOLLAPSE_PATH)
+        self.flamegraph_utity = os.getenv("FLAMEGRAPH_PATH", FLAMEGRAPH_PATH)
+        self.callgraph_mode = os.getenv(
+            "PERF_CALLGRAPH_MODE", PERF_CALLGRAPH_MODE_DEFAULT
+        )
 
         self.output = None
         self.profiler_process = None
@@ -44,6 +55,7 @@ def __init__(self):
         self.profiler_process_exit_code = None
         self.trace_file = None
         self.collapsed_stack_file = None
+        self.stack_collapse_file = None
         self.collapsed_stacks = []
         self.pid = None
         self.started_profile = False
@@ -71,7 +83,7 @@ def retrieve_perf_version(self):
             self.version_minor = m.group(2)
         return m, self.version_major, self.version_minor
 
-    def generate_record_command(self, pid, output, frequency=None, call_graph="lbr"):
+    def generate_record_command(self, pid, output, frequency=None):
         self.output = output
         self.pid = pid
         cmd = [
@@ -85,7 +97,7 @@ def generate_record_command(self, pid, output, frequency=None, call_graph="lbr")
             "--output",
             output,
             "--call-graph",
-            call_graph,
+            self.callgraph_mode,
         ]
         if frequency:
             cmd += ["--freq", "{}".format(frequency)]
@@ -172,13 +184,7 @@ def stop_profile(self, **kwargs):
                     "Profiler process exit with error. Exit code: {}\n\n".format(
                         self.profiler_process_exit_code
                     )
-                    + "If running in non-root user please confirm that you have:\n"
-                    + " - access to Kernel address maps."
-                    + " Check if `0` ( disabled ) appears from the output of `cat /proc/sys/kernel/kptr_restrict`\n"
-                    + '          If not then fix via: `sudo sh -c " echo 0 > /proc/sys/kernel/kptr_restrict"`\n'
-                    + " - permission to collect stats."
-                    + " Check if `-1` appears from the output of `cat /proc/sys/kernel/perf_event_paranoid`\n"
-                    + "          If not then fix via: `sudo sh -c 'echo -1 > /proc/sys/kernel/perf_event_paranoid'`\n"
+                    + LINUX_PERF_SETTINGS_MESSAGE
                 )
 
         except OSError as e:

redisbench_admin/profilers/perf.py

@@ -6,6 +6,7 @@
 
 import datetime as dt
 import logging
+import os
 
 from redisbench_admin.run.redis_benchmark.redis_benchmark import (
     prepare_redis_benchmark_command,
@@ -28,6 +29,8 @@
     extract_perbranch_timeseries_from_results,
 )
 
+BENCHMARK_REPETITIONS = int(os.getenv("BENCHMARK_REPETITIONS", 1))
+
 
 def prepare_benchmark_parameters(
     benchmark_config,
@@ -221,3 +224,16 @@ def get_start_time_vars(start_time=None):
     start_time_ms = int((start_time - dt.datetime(1970, 1, 1)).total_seconds() * 1000)
     start_time_str = start_time.strftime("%Y-%m-%d-%H-%M-%S")
     return start_time, start_time_ms, start_time_str
+
+
+def execute_init_commands(benchmark_config, r, dbconfig_keyname="dbconfig"):
+    cmds = None
+    if dbconfig_keyname in benchmark_config:
+        for k in benchmark_config[dbconfig_keyname]:
+            if "init_commands" in k:
+                cmds = k["init_commands"]
+    if cmds is not None:
+        for cmd in cmds:
+            cmd_split = cmd.split(None, 2)
+            stdout = r.execute_command(*cmd_split)
+            print(stdout)

redisbench_admin/run/common.py

@@ -23,6 +23,7 @@
 from redisbench_admin.run.common import (
     prepare_benchmark_parameters,
     get_start_time_vars,
+    execute_init_commands,
 )
 from redisbench_admin.run_local.args import PROFILE_FREQ
 from redisbench_admin.utils.benchmark_config import (
@@ -149,7 +150,6 @@ def run_local_command_logic(args):
 
             r = redis.StrictRedis(port=args.port)
             stdout = r.execute_command("info modules")
-            print(stdout)
             (
                 module_names,
                 _,
@@ -525,16 +525,3 @@ def which_local(benchmark_tool, executable, full_path, which_benchmark_tool):
                 which_benchmark_tool = full_path_filename
                 break
     return which_benchmark_tool
-
-
-def execute_init_commands(benchmark_config, r, dbconfig_keyname="dbconfig"):
-    cmds = None
-    if dbconfig_keyname in benchmark_config:
-        for k in benchmark_config[dbconfig_keyname]:
-            if "init_commands" in k:
-                cmds = k["init_commands"]
-    if cmds is not None:
-        for cmd in cmds:
-            cmd_split = cmd.split(None, 2)
-            stdout = r.execute_command(*cmd_split)
-            print(stdout)

redisbench_admin/run_local/run_local.py

@@ -8,16 +8,21 @@
 import socket
 
 # environment variables
-PERFORMANCE_RTS_PUSH = bool(os.getenv("PUSH_RTS", False))
-PERFORMANCE_RTS_AUTH = os.getenv("PERFORMANCE_RTS_AUTH", None)
-PERFORMANCE_RTS_HOST = os.getenv("PERFORMANCE_RTS_HOST", 6379)
-PERFORMANCE_RTS_PORT = os.getenv("PERFORMANCE_RTS_PORT", None)
-TERRAFORM_BIN_PATH = os.getenv("TERRAFORM_BIN_PATH", "terraform")
+from redisbench_admin.utils.remote import (
+    TERRAFORM_BIN_PATH,
+    PERFORMANCE_RTS_HOST,
+    PERFORMANCE_RTS_PORT,
+    PERFORMANCE_RTS_AUTH,
+    PERFORMANCE_RTS_PUSH,
+)
 
 LOG_LEVEL = logging.INFO
 if os.getenv("VERBOSE", "1") == "0":
     LOG_LEVEL = logging.WARN
 
+DEFAULT_TRIGGERING_ENV = socket.gethostname()
+TRIGGERING_ENV = os.getenv("TRIGGERING_ENV", DEFAULT_TRIGGERING_ENV)
+
 
 def create_run_remote_arguments(parser):
     parser.add_argument("--module_path", type=str, required=True)
@@ -45,7 +50,7 @@ def create_run_remote_arguments(parser):
         "You can use `--required-module` more than once",
     )
     parser.add_argument("--github_branch", type=str, default=None, nargs="?", const="")
-    parser.add_argument("--triggering_env", type=str, default=socket.gethostname())
+    parser.add_argument("--triggering_env", type=str, default=TRIGGERING_ENV)
     parser.add_argument("--terraform_bin_path", type=str, default=TERRAFORM_BIN_PATH)
     parser.add_argument("--setup_name_sufix", type=str, default="")
     parser.add_argument(