Introduced --tests-regex flag to ease the process of running a subsetof tests (#168)

* Introduced --tests-regex flag to ease the process of running a subset of tests

* Addressed review comment

Author: filipecosta90

redis_benchmarks_specification/__cli__/stats.py

@@ -39,7 +39,7 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
     logging.info("Using test-suites folder dir {}".format(testsuites_folder))
     testsuite_spec_files = get_benchmark_specs(testsuites_folder)
     logging.info(
-        "There are a total of {} test-suites in folder {}".format(
+        "There are a total of {} test-suites being run in folder {}".format(
             len(testsuite_spec_files), testsuites_folder
         )
     )

redis_benchmarks_specification/__common__/runner.py

@@ -1,28 +1,55 @@
 import logging
 import os
 import pathlib
+import re
 
 
-def get_benchmark_specs(testsuites_folder, test=""):
+def get_benchmark_specs(testsuites_folder, test="", test_regex=".*"):
+    final_files = []
     if test == "":
         files = pathlib.Path(testsuites_folder).glob("*.yml")
-        files = [str(x) for x in files]
-        logging.info(
-            "Running all specified benchmarks: {}".format(
-                " ".join([str(x) for x in files])
+        original_files = [str(x) for x in files]
+        if test_regex == ".*":
+            logging.info(
+                "Acception all test files. If you need further filter specify a regular expression via --tests-regexp"
             )
-        )
+            "Running all specified benchmarks: {}".format(" ".join(original_files))
+            final_files = original_files
+        else:
+            logging.info(
+                "Filtering all test names via a regular expression: {}".format(
+                    test_regex
+                )
+            )
+            test_regexp_string = re.compile(test_regex)
+            for test_name in original_files:
+                match_obj = re.search(test_regexp_string, test_name)
+                if match_obj is None:
+                    logging.info(
+                        "Skipping test file: {} given it does not match regex {}".format(
+                            test_name, test_regexp_string
+                        )
+                    )
+                else:
+                    final_files.append(test_name)
+
     else:
         files = test.split(",")
-        files = ["{}/{}".format(testsuites_folder, x) for x in files]
-        logging.info("Running specific benchmark in file: {}".format(files))
-    return files
+        final_files = ["{}/{}".format(testsuites_folder, x) for x in files]
+        logging.info(
+            "Running specific benchmark in {} files: {}".format(
+                len(final_files), final_files
+            )
+        )
+    return final_files
 
 
 def extract_testsuites(args):
     testsuites_folder = os.path.abspath(args.test_suites_folder)
     logging.info("Using test-suites folder dir {}".format(testsuites_folder))
-    testsuite_spec_files = get_benchmark_specs(testsuites_folder, args.test)
+    testsuite_spec_files = get_benchmark_specs(
+        testsuites_folder, args.test, args.tests_regexp
+    )
     logging.info(
         "There are a total of {} test-suites in folder {}".format(
             len(testsuite_spec_files), testsuites_folder

redis_benchmarks_specification/__runner__/args.py

@@ -47,6 +47,12 @@ def create_client_runner_args(project_name):
         help="specify a test to run. By default will run all the tests"
         + " present in the folder specified in --test-suites-folder.",
     )
+    parser.add_argument(
+        "--tests-regexp",
+        type=str,
+        default=".*",
+        help="Interpret PATTERN as a regular expression to filter test names",
+    )
     parser.add_argument("--db_server_host", type=str, default="localhost")
     parser.add_argument("--db_server_port", type=int, default=6379)
     parser.add_argument("--cpuset_start_pos", type=int, default=0)

redis_benchmarks_specification/__self_contained_coordinator__/args.py

@@ -84,6 +84,12 @@ def create_self_contained_coordinator_args(project_name):
         help="specify a test to run. By default will run all the tests"
         + " present in the folder specified in --test-suites-folder.",
     )
+    parser.add_argument(
+        "--tests-regexp",
+        type=str,
+        default=".*",
+        help="Interpret PATTERN as a regular expression to filter test names",
+    )
     parser.add_argument(
         "--datasink_redistimeseries_host", type=str, default=DATASINK_RTS_HOST
     )

utils/tests/test_runner.py

@@ -4,6 +4,7 @@
 import yaml
 
 from redis_benchmarks_specification.__common__.package import get_version_string
+from redis_benchmarks_specification.__common__.runner import extract_testsuites
 from redis_benchmarks_specification.__common__.spec import extract_client_tool
 from redis_benchmarks_specification.__runner__.args import create_client_runner_args
 from redis_benchmarks_specification.__runner__.runner import (
@@ -208,3 +209,45 @@ def test_run_client_runner_logic():
     r = redis.Redis(host=db_host, port=db_port_int)
     total_keys = r.info("keyspace")["db0"]["keys"]
     assert total_keys == 10
+
+
+def test_extract_testsuites():
+    project_name = "tool"
+    project_version = "v0"
+    parser = argparse.ArgumentParser(
+        description="test",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser = create_client_runner_args(
+        get_version_string(project_name, project_version)
+    )
+    args = parser.parse_args(
+        args=[
+            "--test-suites-folder",
+            "./utils/tests/test_data/test-suites",
+        ]
+    )
+    tests = extract_testsuites(args)
+    assert len(tests) == 4
+
+    args = parser.parse_args(
+        args=[
+            "--test-suites-folder",
+            "./utils/tests/test_data/test-suites",
+            "--tests-regex",
+            ".*\.yml",
+        ]
+    )
+    tests = extract_testsuites(args)
+    assert len(tests) == 4
+
+    args = parser.parse_args(
+        args=[
+            "--test-suites-folder",
+            "./utils/tests/test_data/test-suites",
+            "--tests-regex",
+            ".*expire.*",
+        ]
+    )
+    tests = extract_testsuites(args)
+    assert len(tests) == 3