diff --git a/pyproject.toml b/pyproject.toml index 8e1b547c..2041577e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.217" +version = "0.1.218" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__cli__/cli.py b/redis_benchmarks_specification/__cli__/cli.py index 0b6eaf48..906ccbc0 100644 --- a/redis_benchmarks_specification/__cli__/cli.py +++ b/redis_benchmarks_specification/__cli__/cli.py @@ -351,16 +351,22 @@ def trigger_tests_cli_command_logic(args, project_name, project_version): hash_regexp ) ) + enable_hash_filtering = False + hash_filters = [] + if args.git_hash != "": + enable_hash_filtering = True + hash_filters = args.git_hash.split(",") + logging.info( + f"There is a total of {len(hash_filters)} commit hash fitlers: {hash_filters}" + ) hash_regexp_string = re.compile(hash_regexp) filtered_hash_commits = [] for cdict in commits: commit_hash = cdict["git_hash"] - if args.git_hash != "": - if args.git_hash != commit_hash: + if enable_hash_filtering: + if commit_hash not in hash_filters: logging.info( - "Skipping {} given it does not match commit hash {}".format( - commit_hash, args.git_hash - ) + f"Skipping {commit_hash} given it does not match any commit hash in {hash_filters}" ) continue commit_summary = cdict["commit_summary"] diff --git a/redis_benchmarks_specification/__common__/builder_schema.py b/redis_benchmarks_specification/__common__/builder_schema.py index 0f2fcd02..8c45edc1 100644 --- a/redis_benchmarks_specification/__common__/builder_schema.py +++ b/redis_benchmarks_specification/__common__/builder_schema.py @@ -174,6 +174,10 @@ def get_branch_version_from_test_details(testDetails): git_branch = git_branch.decode() if git_branch.startswith("/refs/heads/"): git_branch = git_branch.replace("/refs/heads/", "") + if git_branch.startswith("refs/heads/"): + git_branch = git_branch.replace("refs/heads/", "") + if git_branch.startswith("/"): + git_branch = git_branch[1:] if git_version is not None: if type(git_version) == bytes: git_version = git_version.decode() diff --git a/redis_benchmarks_specification/__compare__/args.py b/redis_benchmarks_specification/__compare__/args.py index 74070963..c35c6e53 100644 --- a/redis_benchmarks_specification/__compare__/args.py +++ b/redis_benchmarks_specification/__compare__/args.py @@ -92,11 +92,19 @@ def create_compare_arguments(parser): ) parser.add_argument("--baseline-branch", type=str, default=None, required=False) parser.add_argument("--baseline-tag", type=str, default=None, required=False) + parser.add_argument("--baseline-hash", type=str, default=None, required=False) parser.add_argument( "--baseline-target-version", type=str, default=None, required=False ) parser.add_argument("--comparison-branch", type=str, default=None, required=False) + parser.add_argument( + "--baseline-github-repo", type=str, default="redis", required=False + ) + parser.add_argument( + "--comparison-github-repo", type=str, default="redis", required=False + ) parser.add_argument("--comparison-tag", type=str, default=None, required=False) + parser.add_argument("--comparison-hash", type=str, default=None, required=False) parser.add_argument( "--comparison-target-version", type=str, default=None, required=False ) diff --git a/redis_benchmarks_specification/__compare__/compare.py b/redis_benchmarks_specification/__compare__/compare.py index 001c1622..af72d698 100644 --- a/redis_benchmarks_specification/__compare__/compare.py +++ b/redis_benchmarks_specification/__compare__/compare.py @@ -250,6 +250,10 @@ def compare_command_logic(args, project_name, project_version): running_platform = args.running_platform baseline_target_version = args.baseline_target_version comparison_target_version = args.comparison_target_version + baseline_github_repo = args.baseline_github_repo + comparison_github_repo = args.comparison_github_repo + baseline_hash = args.baseline_hash + comparison_hash = args.comparison_hash if running_platform is not None: logging.info( @@ -310,6 +314,10 @@ def compare_command_logic(args, project_name, project_version): running_platform, baseline_target_version, comparison_target_version, + baseline_hash, + comparison_hash, + baseline_github_repo, + comparison_github_repo, ) prepare_regression_comment( auto_approve, @@ -535,6 +543,10 @@ def compute_regression_table( running_platform=None, baseline_target_version=None, comparison_target_version=None, + comparison_hash=None, + baseline_hash=None, + baseline_github_repo="redis", + comparison_github_repo="redis", ): START_TIME_NOW_UTC, _, _ = get_start_time_vars() START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31) @@ -560,6 +572,8 @@ def compute_regression_table( comparison_tag, baseline_target_version, comparison_target_version, + comparison_hash, + baseline_hash, ) logging.info(f"Using baseline filter {by_str_baseline}={baseline_str}") logging.info(f"Using comparison filter {by_str_comparison}={comparison_str}") @@ -605,6 +619,8 @@ def compute_regression_table( total_stable, total_unstable, total_comparison_points, + regressions_list, + improvements_list, ) = from_rts_to_regression_table( baseline_deployment_name, comparison_deployment_name, @@ -629,6 +645,8 @@ def compute_regression_table( tf_triggering_env, verbose, running_platform, + baseline_github_repo, + comparison_github_repo, ) logging.info( "Printing differential analysis between {} and {}".format( @@ -661,6 +679,8 @@ def compute_regression_table( writer_regressions.dump(mystdout, False) table_output += mystdout.getvalue() table_output += "\n\n" + test_names_str = "|".join(regressions_list) + table_output += f"Regressions test regexp names: {test_names_str}\n\n" mystdout.close() sys.stdout = old_stdout @@ -682,6 +702,8 @@ def compute_regression_table( writer_regressions.dump(mystdout, False) table_output += mystdout.getvalue() table_output += "\n\n" + test_names_str = "|".join(improvements_list) + table_output += f"Improvements test regexp names: {test_names_str}\n\n" mystdout.close() sys.stdout = old_stdout @@ -724,6 +746,8 @@ def get_by_strings( comparison_tag, baseline_target_version=None, comparison_target_version=None, + baseline_hash=None, + comparison_hash=None, ): baseline_covered = False comparison_covered = False @@ -760,6 +784,16 @@ def get_by_strings( by_str_baseline = "target+version" baseline_str = baseline_target_version + if baseline_hash is not None: + if comparison_covered: + logging.error( + "--baseline-branch, --baseline-tag, --baseline-hash, and --baseline-target-version are mutually exclusive. Pick one..." + ) + exit(1) + baseline_covered = True + by_str_baseline = "hash" + baseline_str = baseline_hash + if comparison_tag is not None: # check if we had already covered comparison if comparison_covered: @@ -781,16 +815,27 @@ def get_by_strings( by_str_comparison = "target+version" comparison_str = comparison_target_version + if comparison_hash is not None: + # check if we had already covered comparison + if comparison_covered: + logging.error( + "--comparison-branch, --comparison-tag, --comparison-hash, and --comparison-target-table are mutually exclusive. Pick one..." + ) + exit(1) + comparison_covered = True + by_str_comparison = "hash" + comparison_str = comparison_hash + if baseline_covered is False: logging.error( "You need to provider either " - + "( --baseline-branch, --baseline-tag, or --baseline-target-version ) " + + "( --baseline-branch, --baseline-tag, --baseline-hash, or --baseline-target-version ) " ) exit(1) if comparison_covered is False: logging.error( "You need to provider either " - + "( --comparison-branch, --comparison-tag, or --comparison-target-version ) " + + "( --comparison-branch, --comparison-tag, --comparison-hash, or --comparison-target-version ) " ) exit(1) return baseline_str, by_str_baseline, comparison_str, by_str_comparison @@ -820,6 +865,8 @@ def from_rts_to_regression_table( tf_triggering_env, verbose, running_platform=None, + baseline_github_repo="redis", + comparison_github_repo="redis", ): print_all = print_regressions_only is False and print_improvements_only is False table_full = [] @@ -835,6 +882,8 @@ def from_rts_to_regression_table( total_comparison_points = 0 noise_waterline = 3 progress = tqdm(unit="benchmark time-series", total=len(test_names)) + regressions_list = [] + improvements_list = [] for test_name in test_names: compare_version = "v0.1.208" github_link = "https://github.com/redis/redis-benchmarks-specification/blob" @@ -848,6 +897,7 @@ def from_rts_to_regression_table( "metric={}".format(metric_name), "{}={}".format(test_filter, test_name), "deployment_name={}".format(baseline_deployment_name), + "github_repo={}".format(baseline_github_repo), "triggering_env={}".format(tf_triggering_env), ] if running_platform is not None: @@ -855,11 +905,15 @@ def from_rts_to_regression_table( filters_comparison = [ "{}={}".format(by_str_comparison, comparison_str), "metric={}".format(metric_name), - "hash==", "{}={}".format(test_filter, test_name), "deployment_name={}".format(comparison_deployment_name), + "github_repo={}".format(comparison_github_repo), "triggering_env={}".format(tf_triggering_env), ] + if "hash" not in by_str_baseline: + filters_baseline.append("hash==") + if "hash" not in by_str_comparison: + filters_comparison.append("hash==") if running_platform is not None: filters_comparison.append("running_platform={}".format(running_platform)) baseline_timeseries = rts.ts().queryindex(filters_baseline) @@ -1037,9 +1091,11 @@ def from_rts_to_regression_table( test_link, ) if detected_regression: + regressions_list.append(test_name) table_regressions.append(line) if detected_improvement: + improvements_list.append(test_name) table_improvements.append(line) if unstable: @@ -1072,6 +1128,8 @@ def from_rts_to_regression_table( total_stable, total_unstable, total_comparison_points, + regressions_list, + improvements_list, ) diff --git a/utils/tests/test_builder.py b/utils/tests/test_builder.py index 4d700355..d6bdd653 100644 --- a/utils/tests/test_builder.py +++ b/utils/tests/test_builder.py @@ -154,9 +154,7 @@ def test_get_branch_version_from_test_details(): def test_cli_build(): try: - # if should_run_builder(): - if True: - + if should_run_builder(): db_port = int(os.getenv("DATASINK_PORT", "6379")) conn = redis.StrictRedis(port=db_port) conn.ping()