profiler-daemon failure improvements (#273)

* [fix] Fixed spin_up_redis_cluster_remote_redis after module config parameters change

* Fixed profiler daemon log extraction

Author: filipecosta90

README.md

@@ -34,6 +34,9 @@ Installation is done using pip, the package installer for Python, in the followi
 python3 -m pip install redisbench-admin
 ```
 
+## Profiler daemon
+
+
 ## Development
 
 1. Install [pypoetry](https://python-poetry.org/) to manage your dependencies and trigger tooling.

docs/export.md

@@ -73,7 +73,7 @@ Optional arguments:
                         branch/ref time-series are created (default: None)
   --local-dir LOCAL_DIR
                         local dir to use as storage (default: ./)
-  --logname LOGNAME     logname to write the logs to (default: None)
+  --local_file PERF_DAEMON_LOGNAME     local_file to write the logs to (default: None)
   --github_actor [GITHUB_ACTOR]
   --github_repo GITHUB_REPO
   --github_org GITHUB_ORG

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.6.10"
+version = "0.6.11"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "README.md"

redisbench_admin/cli.py

@@ -71,7 +71,7 @@ def main():
         "--local-dir", type=str, default="./", help="local dir to use as storage"
     )
     parser.add_argument(
-        "--logname", type=str, default=None, help="logname to write the logs to"
+        "--logname", type=str, default=None, help="local file to write the logs to"
     )
 
     if requested_tool == "run-remote":

redisbench_admin/profilers/daemon.py

@@ -4,27 +4,26 @@
 #  All rights reserved.
 #
 
+import argparse
+import json
+
 # !/usr/bin/env python3
 import logging
-
-import argparse
+import os
+import sys
 
 import botocore
-from flask import Flask, request
 import daemonize
-import json
-import sys
-import os
+from flask import Flask, request
 
 from redisbench_admin.cli import populate_with_poetry_data
+from redisbench_admin.profilers.perf import Perf
+from redisbench_admin.profilers.perf_daemon_caller import PERF_DAEMON_LOGNAME
+from redisbench_admin.profilers.profile_local import local_profilers_platform_checks
 from redisbench_admin.run.args import S3_BUCKET_NAME
+from redisbench_admin.run.common import get_start_time_vars
 from redisbench_admin.run.s3 import get_test_s3_bucket_path
 from redisbench_admin.utils.remote import extract_git_vars
-
-from redisbench_admin.profilers.perf import Perf
-from redisbench_admin.run.common import get_start_time_vars
-
-from redisbench_admin.run_local.profile_local import local_profilers_platform_checks
 from redisbench_admin.utils.utils import upload_artifacts_to_s3
 
 PID_FILE = "/tmp/perfdaemon.pid"
@@ -34,7 +33,6 @@
     LOG_LEVEL = logging.INFO
 LOG_FORMAT = "%(asctime)s %(levelname)-4s %(message)s"
 LOG_DATEFMT = "%Y-%m-%d %H:%M:%S"
-LOGNAME = "/tmp/perf-daemon.log"
 
 app = Flask(__name__)
 app.use_reloader = False
@@ -53,8 +51,10 @@ def main(self):
         app.run(host="0.0.0.0", debug=False, port=5000)
 
     def set_app_loggers(self, app):
-        print("Writting log to {}".format(LOGNAME))
-        handler = logging.handlers.RotatingFileHandler(LOGNAME, maxBytes=1024 * 1024)
+        print("Writting log to {}".format(PERF_DAEMON_LOGNAME))
+        handler = logging.handlers.RotatingFileHandler(
+            PERF_DAEMON_LOGNAME, maxBytes=1024 * 1024
+        )
         logging.getLogger("werkzeug").setLevel(logging.DEBUG)
         logging.getLogger("werkzeug").addHandler(handler)
         app.logger.setLevel(LOG_LEVEL)

redisbench_admin/profilers/perf.py

@@ -10,7 +10,6 @@
 import subprocess
 import time
 
-
 from redisbench_admin.profilers.pprof import (
     PPROF_FORMAT_TEXT,
     run_pprof,
@@ -19,7 +18,6 @@
 from redisbench_admin.profilers.profilers import STACKCOLLAPSE_PATH, FLAMEGRAPH_PATH
 from redisbench_admin.utils.utils import whereis
 
-
 PERF_CALLGRAPH_MODE_DEFAULT = "fp"
 LINUX_PERF_SETTINGS_MESSAGE = (
     "If running in non-root user please confirm that you have:\n"

redisbench_admin/profilers/perf_daemon_caller.py

@@ -4,10 +4,13 @@
 #  All rights reserved.
 #
 import logging
+
 import requests
+
 from redisbench_admin.utils.remote import extract_git_vars
 
 PERF_CALLGRAPH_MODE_DEFAULT = "fp"
+PERF_DAEMON_LOGNAME = "/tmp/perf-daemon.log"
 
 
 class PerfDaemonRemoteCaller:

redisbench_admin/profilers/pprof.py

@@ -6,8 +6,8 @@
 
 import logging
 import os
-import subprocess
 import re
+import subprocess
 
 PPROF_FORMAT_TEXT = "-text"
 PPROF_FORMAT_PS = "-ps"

…isbench_admin/run_local/profile_local.py …bench_admin/profilers/profilers_local.pyredisbench_admin/run_local/profile_local.py renamed to redisbench_admin/profilers/profilers_local.py redisbench_admin/run_local/profile_local.py renamed to redisbench_admin/profilers/profilers_local.py

@@ -1,110 +1,24 @@
 #  BSD 3-Clause License
 #
-#  Copyright (c) 2021., Redis Labs Modules
+#  Copyright (c) 2022., Redis Labs Modules
 #  All rights reserved.
 #
+
+#  BSD 3-Clause License
+#
+#
 import logging
 import platform
 
 from cpuinfo import get_cpu_info
-from pytablewriter import MarkdownTableWriter
 
 from redisbench_admin.profilers.perf import Perf
 from redisbench_admin.profilers.vtune import Vtune
 from redisbench_admin.run.args import PROFILE_FREQ, MAX_PROFILERS_PER_TYPE
-
 from redisbench_admin.run.s3 import get_test_s3_bucket_path
 from redisbench_admin.utils.utils import upload_artifacts_to_s3
 
 
-def local_profilers_print_artifacts_table(profilers_artifacts_matrix):
-    logging.info("Printing profiler generated artifacts")
-    test_cases = []
-    profilers = []
-    use_local_file = True
-    table_name = "Profiler artifacts"
-    for row in profilers_artifacts_matrix:
-        test_case = row[0]
-        profiler = row[1]
-        artifact = row[2]
-        local_file = row[3]
-        s3_link = row[4]
-        if s3_link is not None:
-            if len(s3_link) > 0:
-                use_local_file = False
-
-        if test_case not in test_cases:
-            test_cases.append(test_case)
-        if profiler not in profilers:
-            profilers.append(profiler)
-
-    # We only need to print the testcase if there are more than one
-    use_test_cases_row = True
-
-    if len(test_cases) == 1:
-        use_test_cases_row = False
-        test_case = test_cases[0]
-        table_name = "{} for testcase {}".format(table_name, test_case)
-
-    # We only need to print the profiler type if there are more than one
-    use_profilers_row = True
-    if len(profilers) == 1:
-        use_profilers_row = False
-        profiler = test_cases[0]
-        table_name = "{}. Used profiler {}".format(table_name, profiler)
-
-    headers = []
-    if use_test_cases_row:
-        headers.append("Test Case")
-
-    if use_profilers_row:
-        headers.append("Profiler")
-
-    headers.append("Artifact")
-    if use_local_file:
-        headers.append("Local file")
-    else:
-        headers.append("s3 link")
-
-    profilers_final_matrix = []
-    for row in profilers_artifacts_matrix:
-        test_case = row[0]
-        profiler = row[1]
-        artifact = row[2]
-        local_file = "{} ".format(row[3])
-        s3_link = "{} ".format(row[4])
-
-        final_row = []
-        if use_test_cases_row:
-            final_row.append(test_case)
-
-        if use_profilers_row:
-            final_row.append(profiler)
-
-        final_row.append(artifact)
-        if use_local_file:
-            final_row.append(local_file)
-        else:
-            final_row.append(s3_link)
-        profilers_final_matrix.append(final_row)
-
-    writer = MarkdownTableWriter(
-        table_name=table_name,
-        headers=headers,
-        value_matrix=profilers_final_matrix,
-    )
-    writer.write_table()
-
-
-def get_profilers_rts_key_prefix(triggering_env, tf_github_org, tf_github_repo):
-    zset_name = "ci.benchmarks.redis.com/{triggering_env}/{github_org}/{github_repo}:profiles".format(
-        triggering_env=triggering_env,
-        github_org=tf_github_org,
-        github_repo=tf_github_repo,
-    )
-    return zset_name
-
-
 def profilers_stop_if_required(
     args,
     benchmark_duration_seconds,

redisbench_admin/profilers/profilers_schema.py

@@ -0,0 +1,96 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2022., Redis Labs Modules
+#  All rights reserved.
+#
+import logging
+
+from pytablewriter import MarkdownTableWriter
+
+
+def get_profilers_rts_key_prefix(triggering_env, tf_github_org, tf_github_repo):
+    zset_name = "ci.benchmarks.redis.com/{triggering_env}/{github_org}/{github_repo}:profiles".format(
+        triggering_env=triggering_env,
+        github_org=tf_github_org,
+        github_repo=tf_github_repo,
+    )
+    return zset_name
+
+
+def local_profilers_print_artifacts_table(profilers_artifacts_matrix):
+    logging.info("Printing profiler generated artifacts")
+    test_cases = []
+    profilers = []
+    use_local_file = True
+    table_name = "Profiler artifacts"
+    for row in profilers_artifacts_matrix:
+        test_case = row[0]
+        profiler = row[1]
+        artifact = row[2]
+        local_file = row[3]
+        s3_link = row[4]
+        if s3_link is not None:
+            if len(s3_link) > 0:
+                use_local_file = False
+
+        if test_case not in test_cases:
+            test_cases.append(test_case)
+        if profiler not in profilers:
+            profilers.append(profiler)
+
+    # We only need to print the testcase if there are more than one
+    use_test_cases_row = True
+
+    if len(test_cases) == 1:
+        use_test_cases_row = False
+        test_case = test_cases[0]
+        table_name = "{} for testcase {}".format(table_name, test_case)
+
+    # We only need to print the profiler type if there are more than one
+    use_profilers_row = True
+    if len(profilers) == 1:
+        use_profilers_row = False
+        profiler = test_cases[0]
+        table_name = "{}. Used profiler {}".format(table_name, profiler)
+
+    headers = []
+    if use_test_cases_row:
+        headers.append("Test Case")
+
+    if use_profilers_row:
+        headers.append("Profiler")
+
+    headers.append("Artifact")
+    if use_local_file:
+        headers.append("Local file")
+    else:
+        headers.append("s3 link")
+
+    profilers_final_matrix = []
+    for row in profilers_artifacts_matrix:
+        test_case = row[0]
+        profiler = row[1]
+        artifact = row[2]
+        local_file = "{} ".format(row[3])
+        s3_link = "{} ".format(row[4])
+
+        final_row = []
+        if use_test_cases_row:
+            final_row.append(test_case)
+
+        if use_profilers_row:
+            final_row.append(profiler)
+
+        final_row.append(artifact)
+        if use_local_file:
+            final_row.append(local_file)
+        else:
+            final_row.append(s3_link)
+        profilers_final_matrix.append(final_row)
+
+    writer = MarkdownTableWriter(
+        table_name=table_name,
+        headers=headers,
+        value_matrix=profilers_final_matrix,
+    )
+    writer.write_table()