benchmark coordinator is accepting benchmark group filters

Author: fcostaoliveira

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.101"
+version = "0.1.205"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

redis_benchmarks_specification/__builder__/builder.py

@@ -245,10 +245,10 @@ def builder_process_stream(
             use_git_timestamp = False
             commit_datetime = "n/a"
             if b"commit_datetime" in testDetails:
-                commit_datetime = int(testDetails[b"commit_datetime"].decode())
+                commit_datetime = testDetails[b"commit_datetime"].decode()
             commit_summary = "n/a"
             if b"commit_summary" in testDetails:
-                commit_summary = int(testDetails[b"commit_summary"].decode())
+                commit_summary = testDetails[b"commit_summary"].decode()
             git_branch, git_version = get_branch_version_from_test_details(testDetails)
             if b"use_git_timestamp" in testDetails:
                 use_git_timestamp = bool(testDetails[b"use_git_timestamp"])

redis_benchmarks_specification/__cli__/args.py

@@ -18,7 +18,7 @@
 from redisbench_admin.run.common import get_start_time_vars
 
 START_TIME_NOW_UTC, _, _ = get_start_time_vars()
-START_TIME_LAST_YEAR_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=7)
+START_TIME_LAST_YEAR_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=90)
 CLI_TOOL_STATS = "stats"
 CLI_TOOL_TRIGGER = "trigger"
 PERFORMANCE_GH_TOKEN = os.getenv("PERFORMANCE_GH_TOKEN", None)
@@ -163,8 +163,8 @@ def spec_cli_args(parser):
     parser.add_argument(
         "--last_n",
         type=int,
-        default=-1,
-        help="Use the last N samples. by default will use all available values",
+        default=1,
+        help="Use the last N samples. by default will use last commit",
     )
     parser.add_argument(
         "--platform",

redis_benchmarks_specification/__cli__/cli.py

@@ -322,6 +322,11 @@ def trigger_tests_cli_command_logic(args, project_name, project_version):
             commit_dict["tests_priority_lower_limit"] = tests_priority_lower_limit
             commit_dict["tests_regexp"] = tests_regexp
             commit_dict["tests_groups_regexp"] = tests_groups_regexp
+            if pull_request is not None:
+                logging.info(
+                    f"Have a pull request info to include in build request {pull_request}"
+                )
+                commit_dict["pull_request"] = pull_request
             git_hash = cdict["git_hash"]
             git_branch = "n/a"
             if "git_branch" in cdict:

redis_benchmarks_specification/__common__/github.py

@@ -104,6 +104,7 @@ def check_github_available_and_actionable(
     regression_comment = None
     github_pr = None
     old_regression_comment_body = ""
+    pr_link = None
     if github_token is not None:
         logging.info("Detected github token")
         g = Github(github_token)
@@ -134,9 +135,10 @@ def check_github_available_and_actionable(
                     print("".join(["-" for x in range(1, 80)]))
             else:
                 logging.info("Does not contain PR comment")
-    logging.info(
-        f"contains_regression_comment: {contains_regression_comment}, is_actionable_pr: {is_actionable_pr}, pr_link: {pr_link}"
-    )
+            logging.info(
+                f"contains_regression_comment: {contains_regression_comment}, is_actionable_pr: {is_actionable_pr}, pr_link: {pr_link}"
+            )
+
     return (
         contains_regression_comment,
         github_pr,
@@ -226,7 +228,9 @@ def check_benchmark_running_comment(comments):
 
 
 def markdown_progress_bar(current, total, bar_length=40):
-    progress = current / total
+    progress = 1.0
+    if total > 0:
+        progress = current / total
     block = int(round(bar_length * progress))
     bar = "#" * block + "-" * (bar_length - block)
     percentage = round(progress * 100, 2)

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -537,6 +537,13 @@ def process_self_contained_coordinator_stream(
                     f"detected a regexp definition on the streamdata {test_regexp}"
                 )
 
+            command_groups_regexp = None
+            if b"tests_groups_regexp" in testDetails:
+                command_groups_regexp = testDetails[b"tests_groups_regexp"].decode()
+                logging.info(
+                    f"detected a command groups regexp definition on the streamdata {command_groups_regexp}"
+                )
+
             skip_test = False
             if b"platform" in testDetails:
                 platform = testDetails[b"platform"]
@@ -591,6 +598,7 @@ def process_self_contained_coordinator_stream(
                     priority_upper_limit,
                     test_regexp,
                     testsuite_spec_files,
+                    command_groups_regexp,
                 )
 
                 for test_file in filtered_test_files:
@@ -611,16 +619,16 @@ def process_self_contained_coordinator_stream(
                 pending_tests = len(filtered_test_files)
                 failed_tests = 0
                 benchmark_suite_start_datetime = datetime.datetime.utcnow()
-                comment_body = generate_benchmark_started_pr_comment(
-                    stream_id,
-                    pending_tests,
-                    len(filtered_test_files),
-                    failed_tests,
-                    benchmark_suite_start_datetime,
-                    0,
-                )
                 # update on github if needed
                 if is_actionable_pr:
+                    comment_body = generate_benchmark_started_pr_comment(
+                        stream_id,
+                        pending_tests,
+                        len(filtered_test_files),
+                        failed_tests,
+                        benchmark_suite_start_datetime,
+                        0,
+                    )
                     if contains_benchmark_run_comment:
                         update_comment_if_needed(
                             auto_approve_github,
@@ -1261,7 +1269,7 @@ def process_self_contained_coordinator_stream(
                         to_date = datetime.datetime.utcnow()
                         from_date = to_date - datetime.timedelta(days=180)
                         baseline_branch = default_baseline_branch
-                        comparison_tag = git_tag
+                        comparison_tag = git_version
                         comparison_branch = git_branch
                         to_ts_ms = None
                         from_ts_ms = None
@@ -1362,6 +1370,7 @@ def filter_test_files(
     priority_upper_limit,
     test_regexp,
     testsuite_spec_files,
+    command_groups_regexp=None,
 ):
     filtered_test_files = []
     for test_file in testsuite_spec_files:
@@ -1397,6 +1406,34 @@ def filter_test_files(
                 )
                 continue
 
+            if command_groups_regexp is not None:
+                logging.info(
+                    "Filtering all test command groups via a regular expression: {}".format(
+                        command_groups_regexp
+                    )
+                )
+                if "tested-groups" in benchmark_config:
+                    command_groups = benchmark_config["tested-groups"]
+                    logging.info(
+                        f"The file {test_file} (test name = {test_name}) contains the following groups: {command_groups}"
+                    )
+                    groups_regex_string = re.compile(command_groups_regexp)
+                    found = False
+                    for command_group in command_groups:
+                        match_obj = re.search(groups_regex_string, command_group)
+                        if match_obj is not None:
+                            found = True
+                            logging.info(f"found the command group {command_group}")
+                    if found is False:
+                        logging.info(
+                            f"Skipping {test_file} given the following groups: {command_groups} does not match command group regex {command_groups_regexp}"
+                        )
+                        continue
+                else:
+                    logging.warning(
+                        f"The file {test_file} (test name = {test_name}) does not contain the property 'tested-groups'. Cannot filter based uppon groups..."
+                    )
+
             if "priority" in benchmark_config:
                 priority = benchmark_config["priority"]