[add] Enabled Enabled specifying dataset_load_timeout_secs property. Added --logfile option to log to file instead of stdout. (#135)

filipecosta90 · web-flow · commit 4ba7663deb6e · 2021-06-11T11:23:36.000+01:00
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.2.6"
+version = "0.2.7"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>"]
 readme = "README.md"
diff --git a/redisbench_admin/cli.py b/redisbench_admin/cli.py
@@ -6,6 +6,7 @@
 
 import argparse
 import logging
+import os
 import sys
 
 import toml
@@ -17,12 +18,19 @@
 from redisbench_admin.extract.extract import extract_command_logic
 from redisbench_admin.run_local.args import create_run_local_arguments
 from redisbench_admin.run_local.run_local import run_local_command_logic
-from redisbench_admin.run_remote.args import create_run_remote_arguments, LOG_LEVEL
+from redisbench_admin.run_remote.args import create_run_remote_arguments
 from redisbench_admin.run_remote.run_remote import run_remote_command_logic
 from redisbench_admin.watchdog.args import create_watchdog_arguments
 from redisbench_admin.watchdog.watchdog import watchdog_command_logic
 
 
+LOG_LEVEL = logging.INFO
+if os.getenv("VERBOSE", "1") == "0":
+    LOG_LEVEL = logging.WARN
+LOG_FORMAT = "%(asctime)s %(levelname)-4s %(message)s"
+LOG_DATEFMT = "%Y-%m-%d %H:%M:%S"
+
+
 def populate_with_poetry_data():
     project_name = "redisbench-admin"
     project_version = __version__
@@ -38,14 +46,6 @@ def populate_with_poetry_data():
     return project_name, project_description, project_version
 
 
-# logging settings
-logging.basicConfig(
-    format="%(asctime)s %(levelname)-4s %(message)s",
-    level=LOG_LEVEL,
-    datefmt="%Y-%m-%d %H:%M:%S",
-)
-
-
 def main():
     if len(sys.argv) < 2:
         print(
@@ -66,6 +66,9 @@ def main():
     parser.add_argument(
         "--local-dir", type=str, default="./", help="local dir to use as storage"
     )
+    parser.add_argument(
+        "--logname", type=str, default=None, help="logname to write the logs to"
+    )
 
     if requested_tool == "run-remote":
         parser = create_run_remote_arguments(parser)
@@ -96,6 +99,24 @@ def main():
 
     argv = sys.argv[2:]
     args = parser.parse_args(args=argv)
+
+    if args.logname is not None:
+        print("Writting log to {}".format(args.logname))
+        logging.basicConfig(
+            filename=args.logname,
+            filemode="a",
+            format=LOG_FORMAT,
+            datefmt=LOG_DATEFMT,
+            level=LOG_LEVEL,
+        )
+    else:
+        # logging settings
+        logging.basicConfig(
+            format=LOG_FORMAT,
+            level=LOG_LEVEL,
+            datefmt=LOG_DATEFMT,
+        )
+
     if requested_tool == "run-local":
         run_local_command_logic(args)
     if requested_tool == "run-remote":
diff --git a/redisbench_admin/run_local/run_local.py b/redisbench_admin/run_local/run_local.py
@@ -31,7 +31,7 @@
     extract_benchmark_tool_settings,
     check_required_modules,
     results_dict_kpi_check,
-    extract_redis_configuration_parameters,
+    extract_redis_dbconfig_parameters,
 )
 from redisbench_admin.run.redis_benchmark.redis_benchmark import (
     redis_benchmark_ensure_min_version_local,
@@ -134,7 +134,7 @@ def run_local_command_logic(args):
             )
             check_dataset_local_requirements(benchmark_config, temporary_dir, dirname)
 
-            redis_configuration_parameters = extract_redis_configuration_parameters(
+            redis_configuration_parameters, _ = extract_redis_dbconfig_parameters(
                 benchmark_config, "dbconfig"
             )
 
@@ -158,7 +158,17 @@ def run_local_command_logic(args):
             check_required_modules(module_names, required_modules)
 
             # run initialization commands before benchmark starts
+            logging.info("Running initialization commands before benchmark starts.")
+            execute_init_commands_start_time = datetime.datetime.now()
             execute_init_commands(benchmark_config, r)
+            execute_init_commands_duration_seconds = (
+                datetime.datetime.now() - execute_init_commands_start_time
+            ).seconds
+            logging.info(
+                "Running initialization commands took {} secs.".format(
+                    execute_init_commands_duration_seconds
+                )
+            )
 
             # setup the benchmark
             start_time, start_time_ms, start_time_str = get_start_time_vars()
diff --git a/redisbench_admin/run_remote/args.py b/redisbench_admin/run_remote/args.py
@@ -3,7 +3,6 @@
 #  Copyright (c) 2021., Redis Labs Modules
 #  All rights reserved.
 #
-import logging
 import os
 import socket
 
@@ -16,9 +15,6 @@
     PERFORMANCE_RTS_PUSH,
 )
 
-LOG_LEVEL = logging.INFO
-if os.getenv("VERBOSE", "1") == "0":
-    LOG_LEVEL = logging.WARN
 
 DEFAULT_TRIGGERING_ENV = socket.gethostname()
 TRIGGERING_ENV = os.getenv("TRIGGERING_ENV", DEFAULT_TRIGGERING_ENV)
diff --git a/redisbench_admin/run_remote/run_remote.py b/redisbench_admin/run_remote/run_remote.py
@@ -34,7 +34,7 @@
     prepare_benchmark_definitions,
     check_required_modules,
     results_dict_kpi_check,
-    extract_redis_configuration_parameters,
+    extract_redis_dbconfig_parameters,
 )
 from redisbench_admin.utils.redisgraph_benchmark_go import (
     spin_up_standalone_remote_redis,
@@ -359,11 +359,10 @@ def run_remote_command_logic(args):
                 # in case of some unexpected error we fail the test
                 try:
 
-                    redis_configuration_parameters = (
-                        extract_redis_configuration_parameters(
-                            benchmark_config, "dbconfig"
-                        )
-                    )
+                    (
+                        redis_configuration_parameters,
+                        dataset_load_timeout_secs,
+                    ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig")
                     # setup Redis
                     spin_up_standalone_remote_redis(
                         benchmark_config,
@@ -383,7 +382,7 @@ def run_remote_command_logic(args):
                         server_public_ip,
                         username,
                     )
-                    result = wait_for_conn(local_redis_conn)
+                    result = wait_for_conn(local_redis_conn, dataset_load_timeout_secs)
                     dataset_load_end_time = datetime.datetime.now()
                     if result is True:
                         logging.info("Redis available")
@@ -409,7 +408,19 @@ def run_remote_command_logic(args):
                     check_required_modules(module_names, required_modules)
 
                     # run initialization commands before benchmark starts
+                    logging.info(
+                        "Running initialization commands before benchmark starts."
+                    )
+                    execute_init_commands_start_time = datetime.datetime.now()
                     execute_init_commands(benchmark_config, local_redis_conn)
+                    execute_init_commands_duration_seconds = (
+                        datetime.datetime.now() - execute_init_commands_start_time
+                    ).seconds
+                    logging.info(
+                        "Running initialization commands took {} secs.".format(
+                            execute_init_commands_duration_seconds
+                        )
+                    )
 
                     ssh_tunnel.close()  # Close the tunnel
 
diff --git a/redisbench_admin/utils/benchmark_config.py b/redisbench_admin/utils/benchmark_config.py
@@ -123,17 +123,20 @@ def merge_default_and_specific_properties_dict_type(
                 )
 
 
-def extract_redis_configuration_parameters(benchmark_config, dbconfig_keyname):
+def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname):
     redis_configuration_parameters = {}
+    dataset_load_timeout_secs = 120
     if dbconfig_keyname in benchmark_config:
         for k in benchmark_config[dbconfig_keyname]:
             if "configuration-parameters" in k:
                 cp = k["configuration-parameters"]
                 for item in cp:
                     for k, v in item.items():
                         redis_configuration_parameters[k] = v
+            if "dataset_load_timeout_secs" in k:
+                dataset_load_timeout_secs = k["dataset_load_timeout_secs"]
 
-    return redis_configuration_parameters
+    return redis_configuration_parameters, dataset_load_timeout_secs
 
 
 def process_default_yaml_properties_file(
diff --git a/redisbench_admin/utils/utils.py b/redisbench_admin/utils/utils.py
@@ -244,4 +244,8 @@ def wait_for_conn(conn, retries=120, command="PING", should_be=True):
         time.sleep(1)
         retries -= 1
         logging.debug("Waiting for Redis")
+    if retries == 0:
+        logging.debug(
+            "Redis busy loading time surpassed the timeout of {} secs".format(retries)
+        )
     return result
diff --git a/redisbench_admin/watchdog/watchdog.py b/redisbench_admin/watchdog/watchdog.py
@@ -10,6 +10,7 @@
 import time
 
 import boto3
+import redis
 from redistimeseries.client import Client
 
 from redisbench_admin.run.common import get_start_time_vars
@@ -125,14 +126,17 @@ def watchdog_command_logic(args):
         running_count, _ = get_ci_ec2_instances_by_state(
             ec2_client, ci_machines_prefix, "running"
         )
-
-        rts.add(
-            tsname_overall_running,
-            start_time_ms,
-            running_count,
-            labels={"cloud": cloud, "region": EC2_REGION},
-        )
-
+        try:
+            rts.add(
+                tsname_overall_running,
+                start_time_ms,
+                running_count,
+                labels={"cloud": cloud, "region": EC2_REGION},
+            )
+        except redis.exceptions.ConnectionError as e:
+            logging.error(
+                "Detected an error while writing data to rts: {}".format(e.__str__())
+            )
         sleep_time_secs = float(update_interval) - (
             (datetime.datetime.now() - starttime).total_seconds()
             % float(update_interval)
diff --git a/tests/test_benchmark_config.py b/tests/test_benchmark_config.py
@@ -5,7 +5,7 @@
 from redisbench_admin.utils.benchmark_config import (
     results_dict_kpi_check,
     check_required_modules,
-    extract_redis_configuration_parameters,
+    extract_redis_dbconfig_parameters,
 )
 
 
@@ -51,19 +51,44 @@ def test_extract_redis_configuration_parameters():
         "./tests/test_data/redisgraph-benchmark-go-defaults.yml", "r"
     ) as config_fd:
         benchmark_config = yaml.safe_load(config_fd)
-        redis_configuration_parameters = extract_redis_configuration_parameters(
-            benchmark_config, "dbconfig"
-        )
+        (
+            redis_configuration_parameters,
+            dataset_load_timeout_secs,
+        ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig")
         assert redis_configuration_parameters == {}
+        assert dataset_load_timeout_secs == 120
 
         with open(
             "./tests/test_data/tsbs-devops-ingestion-scale100-4days-keyspace.yml", "r"
         ) as config_fd:
             benchmark_config = yaml.safe_load(config_fd)
-            redis_configuration_parameters = extract_redis_configuration_parameters(
-                benchmark_config, "dbconfig"
-            )
+            (
+                redis_configuration_parameters,
+                dataset_load_timeout_secs,
+            ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig")
+            assert dataset_load_timeout_secs == 120
             assert redis_configuration_parameters == {
                 "notify-keyspace-events": "KEA",
                 "timeout": 0,
             }
+
+    with open(
+        "./tests/test_data/redisgraph-benchmark-go-defaults.yml", "r"
+    ) as config_fd:
+        benchmark_config = yaml.safe_load(config_fd)
+        (
+            redis_configuration_parameters,
+            dataset_load_timeout_secs,
+        ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig")
+        assert redis_configuration_parameters == {}
+        assert dataset_load_timeout_secs == 120
+
+        with open(
+            "./tests/test_data/redisgraph-benchmark-go-dataset-timeout.yml", "r"
+        ) as config_fd:
+            benchmark_config = yaml.safe_load(config_fd)
+            (
+                redis_configuration_parameters,
+                dataset_load_timeout_secs,
+            ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig")
+            assert dataset_load_timeout_secs == 1200
diff --git a/tests/test_data/redisgraph-benchmark-go-dataset-timeout.yml b/tests/test_data/redisgraph-benchmark-go-dataset-timeout.yml
@@ -0,0 +1,21 @@
+name: "UPDATE-BASELINE"
+remote:
+  - setup: redisgraph-r5
+  - type: oss-standalone
+dbconfig:
+  - dataset: "datasets/single_node.rdb"
+  - dataset_load_timeout_secs: 1200
+clientconfig:
+  - tool: redisgraph-benchmark-go
+  - parameters:
+    - graph: "g"
+    - rps: 0
+    - clients: 32
+    - threads: 4
+    - connections: 32
+    - requests: 1000000
+    - queries:
+      - { q: "MATCH (n) WHERE ID(n) = 0 SET n.v = n.v + 1", ratio: 1 }
+kpis:
+  - le: { $.OverallClientLatencies.Total.q50: 2.0 }
+  - ge: { $.OverallQueryRates.Total: 18000 }

Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,6 @@`
`3`	`3`	`# Copyright (c) 2021., Redis Labs Modules`
`4`	`4`	`# All rights reserved.`
`5`	`5`	`#`
`6`		`-import logging`
`7`	`6`	`import os`
`8`	`7`	`import socket`
`9`	`8`
`@@ -16,9 +15,6 @@`
`16`	`15`	`PERFORMANCE_RTS_PUSH,`
`17`	`16`	`)`
`18`	`17`
`19`		`-LOG_LEVEL = logging.INFO`
`20`		`-if os.getenv("VERBOSE", "1") == "0":`
`21`		`- LOG_LEVEL = logging.WARN`
`22`	`18`
`23`	`19`	`DEFAULT_TRIGGERING_ENV = socket.gethostname()`
`24`	`20`	`TRIGGERING_ENV = os.getenv("TRIGGERING_ENV", DEFAULT_TRIGGERING_ENV)`