|
| 1 | +import sys |
| 2 | +import os |
| 3 | +import utils |
| 4 | +import shutil |
| 5 | +import json |
| 6 | +import filecmp |
| 7 | + |
| 8 | +""" |
| 9 | +This script compares the generated CSV coverage files with the ones in the codebase. |
| 10 | +""" |
| 11 | + |
| 12 | +artifacts_workflow_name = "Check framework coverage changes" |
| 13 | +comparison_artifact_name = "comparison" |
| 14 | +comparison_artifact_file_name = "comparison.md" |
| 15 | + |
| 16 | + |
| 17 | +def get_comment_text(output_file, repo, run_id): |
| 18 | + size = os.path.getsize(output_file) |
| 19 | + if size == 0: |
| 20 | + print("No difference in the coverage reports") |
| 21 | + return |
| 22 | + |
| 23 | + comment = ":warning: The head of this PR and the base branch were compared for differences in the framework coverage reports. " + \ |
| 24 | + f"The generated reports are available in the [artifacts of this workflow run](https://github.com/{repo}/actions/runs/{run_id}). " + \ |
| 25 | + "The differences will be picked up by the nightly job after the PR gets merged. " |
| 26 | + |
| 27 | + if size < 2000: |
| 28 | + print("There's a small change in the CSV framework coverage reports") |
| 29 | + comment += "The following differences were found: \n\n" |
| 30 | + with open(output_file, 'r') as file: |
| 31 | + comment += file.read() |
| 32 | + else: |
| 33 | + print("There's a large change in the CSV framework coverage reports") |
| 34 | + comment += f"The differences can be found in the {comparison_artifact_name} artifact of this workflow run](https://github.com/{repo}/actions/runs/{run_id})." |
| 35 | + |
| 36 | + return comment |
| 37 | + |
| 38 | + |
| 39 | +def comment_pr(repo, run_id): |
| 40 | + """ |
| 41 | + Generates coverage diff produced by the changes in the current PR. If the diff is not empty, then post it as a comment. |
| 42 | + If a workflow run produces the same diff as the directly preceeding one, then don't post a comment. |
| 43 | + """ |
| 44 | + |
| 45 | + # Store diff for current run |
| 46 | + current_diff_folder = "current_diff" |
| 47 | + utils.download_artifact(repo, comparison_artifact_name, |
| 48 | + current_diff_folder, run_id) |
| 49 | + |
| 50 | + utils.download_artifact(repo, "pr", "pr", run_id) |
| 51 | + |
| 52 | + try: |
| 53 | + with open("pr/NR") as file: |
| 54 | + pr_number = int(file.read()) |
| 55 | + finally: |
| 56 | + if os.path.isdir("pr"): |
| 57 | + shutil.rmtree("pr") |
| 58 | + |
| 59 | + # Try storing diff for previous run: |
| 60 | + try: |
| 61 | + prev_run_id = get_previous_run_id(repo, run_id, pr_number) |
| 62 | + prev_diff_folder = "prev_diff" |
| 63 | + utils.download_artifact(repo, comparison_artifact_name, |
| 64 | + prev_diff_folder, prev_run_id) |
| 65 | + |
| 66 | + if filecmp.cmp(f"{current_diff_folder}/{comparison_artifact_file_name}", f"{prev_diff_folder}/{comparison_artifact_file_name}", shallow=False): |
| 67 | + print( |
| 68 | + f"Previous run {prev_run_id} resulted in the same diff, so not commenting again.") |
| 69 | + return |
| 70 | + else: |
| 71 | + print(f"Diff of previous run {prev_run_id} differs, commenting.") |
| 72 | + except Exception: |
| 73 | + # this is not mecessarily a failure, it can also mean that there was no previous run yet. |
| 74 | + print("Couldn't generate diff for previous run:", sys.exc_info()[1]) |
| 75 | + |
| 76 | + comment = get_comment_text( |
| 77 | + f"{current_diff_folder}/{comparison_artifact_file_name}", repo, run_id) |
| 78 | + post_comment(comment, repo, pr_number) |
| 79 | + |
| 80 | + |
| 81 | +def post_comment(comment, repo, pr_number): |
| 82 | + print(f"Posting comment to PR #{pr_number}") |
| 83 | + utils.subprocess_run(["gh", "pr", "comment", str(pr_number), |
| 84 | + "--repo", repo, "--body", comment]) |
| 85 | + |
| 86 | + |
| 87 | +def get_previous_run_id(repo, run_id, pr_number): |
| 88 | + """ |
| 89 | + Gets the previous run id for a given workflow run, considering that the previous workflow run needs to come from the same PR. |
| 90 | + """ |
| 91 | + |
| 92 | + # Get branch and repo from run: |
| 93 | + this_run = utils.subprocess_check_output( |
| 94 | + ["gh", "api", "-X", "GET", f"repos/{repo}/actions/runs/{run_id}", "--jq", "{ head_branch: .head_branch, head_repository: .head_repository.full_name }"]) |
| 95 | + |
| 96 | + this_run = json.loads(this_run) |
| 97 | + pr_branch = this_run["head_branch"] |
| 98 | + pr_repo = this_run["head_repository"] |
| 99 | + |
| 100 | + # Get all previous runs that match branch, repo and workflow name: |
| 101 | + ids = utils.subprocess_check_output(["gh", "api", "-X", "GET", f"repos/{repo}/actions/runs", "-f", "event=pull_request", "-f", "status=success", "-f", "name=\"" + artifacts_workflow_name + "\"", "--jq", |
| 102 | + f"[.workflow_runs.[] | select(.head_branch==\"{pr_branch}\" and .head_repository.full_name==\"{pr_repo}\") | {{ created_at: .created_at, run_id: .id}}] | sort_by(.created_at) | reverse | [.[].run_id]"]) |
| 103 | + |
| 104 | + ids = json.loads(ids) |
| 105 | + if ids[0] != int(run_id): |
| 106 | + raise Exception( |
| 107 | + f"Expected to find {run_id} in the list of matching runs.") |
| 108 | + |
| 109 | + for previous_run_id in ids[1:]: |
| 110 | + utils.download_artifact(repo, "pr", "prev_run_pr", previous_run_id) |
| 111 | + |
| 112 | + try: |
| 113 | + with open("prev_run_pr/NR") as file: |
| 114 | + prev_pr_number = int(file.read()) |
| 115 | + print(f"PR number: {prev_pr_number}") |
| 116 | + finally: |
| 117 | + if os.path.isdir("prev_run_pr"): |
| 118 | + shutil.rmtree("prev_run_pr") |
| 119 | + |
| 120 | + # the previous run needs to be coming from the same PR: |
| 121 | + if pr_number == prev_pr_number: |
| 122 | + return previous_run_id |
| 123 | + |
| 124 | + raise Exception("Couldn't find previous run.") |
| 125 | + |
| 126 | + |
| 127 | +repo = sys.argv[1] |
| 128 | +run_id = sys.argv[2] |
| 129 | + |
| 130 | +comment_pr(repo, run_id) |
0 commit comments