Skip to content

Commit 1fc8459

Browse files
aajtoddaajtodd
authored
chore: migrate codegen diff tooling to S3 CDN (#721)
1 parent 45c619c commit 1fc8459

File tree

5 files changed

+305
-104
lines changed

5 files changed

+305
-104
lines changed

‎.github/scripts/codegen-diff-revisions.py‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
#!/usr/bin/env python3
2+
#
3+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
4+
# SPDX-License-Identifier: Apache-2.0
5+
#
6+
# This script can be run and tested locally. To do so, you should check out
7+
# a second aws-sdk-kotlin repository so that you can work on the script and still
8+
# run it without it immediately bailing for an unclean working tree (and to avoid creating
9+
# temporary branches).
10+
#
11+
# Example:
12+
# `aws-sdk-kotlin/` - the main repo you're working out of
13+
# `/tmp/aws-sdk-kotlin/` - the repo you're testing against
14+
#
15+
# ```
16+
# $ cd test/aws-sdk-kotlin
17+
# $ ../../aws-sdk-kotlin/.github/scripts/codegen-diff-revisions.py . <some commit hash to diff against>
18+
# ```
19+
#
20+
# It will diff the generated code from HEAD against any commit hash you feed it. If you want to test
21+
# a specific range, change the HEAD of the test repository.
22+
#
23+
# This script requires `diff2html-cli` to be installed from NPM:
24+
# ```
25+
# $ npm install -g [email protected]
26+
# ```
27+
# Make sure the local version matches the version referenced from the GitHub Actions workflow.
28+
29+
import argparse
30+
import os
31+
import sys
32+
import subprocess
33+
import tempfile
34+
import shlex
35+
36+
HEAD_BRANCH_NAME = "__tmp-localonly-head"
37+
BASE_BRANCH_NAME = "__tmp-localonly-base"
38+
OUTPUT_PATH = "tmp-codegen-diff/"
39+
40+
COMMIT_AUTHOR_NAME = "GitHub Action (generated codegen diff)"
41+
COMMIT_AUTHOR_EMAIL = "[email protected]"
42+
43+
CDN_URL = "https://d3l30fr4k4zcv0.cloudfront.net"
44+
45+
46+
def eprint(*args, **kwargs):
47+
"""
48+
Prints to stderr
49+
"""
50+
print(*args, file=sys.stderr, **kwargs)
51+
52+
53+
def running_in_github_action():
54+
"""
55+
Test if currently running in a GitHub action or running locally
56+
:return: True if running in GH, False otherwise
57+
"""
58+
return "GITHUB_WORKFLOW" in os.environ
59+
60+
61+
def run(command, shell=False):
62+
"""
63+
Run a command
64+
:param command: command to run
65+
:param shell: Flag indicating if shell should be used by subprocess command
66+
"""
67+
if not shell:
68+
command = shlex.split(command)
69+
subprocess.run(command, stdout=sys.stderr, stderr=sys.stderr, shell=shell, check=True)
70+
71+
72+
def get_cmd_output(command):
73+
"""
74+
Returns the output from a shell command. Bails if the command failed
75+
76+
:param command: command to run
77+
:return: output from running the given command
78+
"""
79+
result = subprocess.run(shlex.split(command), capture_output=True, check=True)
80+
return result.stdout.decode("utf-8").strip()
81+
82+
83+
def get_cmd_status(command):
84+
"""
85+
Runs a shell command and returns its exit status
86+
87+
:param command: command to run
88+
:return: exit status of the command
89+
"""
90+
return subprocess.run(command, capture_output=True, shell=True).returncode
91+
92+
93+
def generate_and_commit_generated_code(sha, services_to_bootstrap):
94+
"""
95+
Generate codegen output and commit it
96+
:param sha: The commit SHA being generated
97+
:param services_to_bootstrap: list of services to pass on to codegen bootstrap
98+
:return:
99+
"""
100+
run(f'./gradlew --rerun-tasks :codegen:sdk:bootstrap -Paws.services={services_to_bootstrap}')
101+
run(f"rm -rf {OUTPUT_PATH}")
102+
run(f"mkdir {OUTPUT_PATH}")
103+
run(f'cp -r services {OUTPUT_PATH}')
104+
run(f'git add -f {OUTPUT_PATH}')
105+
run(f"git -c 'user.name={COMMIT_AUTHOR_NAME}' -c 'user.email={COMMIT_AUTHOR_EMAIL}' commit --no-verify -m 'Generated code for {sha}' --allow-empty")
106+
107+
108+
# Writes an HTML template for diff2html so that we can add contextual information
109+
def write_html_template(title, subtitle, tmp_file):
110+
tmp_file.writelines(map(lambda line: line.encode(), [
111+
"<!doctype html>",
112+
"<html>",
113+
"<head>",
114+
' <metadata charset="utf-8">',
115+
f' <title>Codegen diff for the {title}: {subtitle}</title>',
116+
' <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.6.0/styles/github.min.css" / >',
117+
' <!--diff2html-css-->',
118+
' <!--diff2html-js-ui-->',
119+
' <script>',
120+
' document.addEventListener("DOMContentLoaded", () => {',
121+
' const targetElement = document.getElementById("diff");',
122+
' const diff2htmlUi = new Diff2HtmlUI(targetElement);',
123+
' //diff2html-fileListToggle',
124+
' //diff2html-synchronisedScroll',
125+
' //diff2html-highlightCode',
126+
' });',
127+
' </script>',
128+
"</head>",
129+
'<body style="font-family: sans-serif;">',
130+
f" <h1>Codegen diff for the {title}</h1>",
131+
f" <p>{subtitle}</p>",
132+
' <div id="diff">',
133+
' <!--diff2html-diff-->',
134+
' </div>',
135+
"</body>",
136+
"</html>",
137+
]))
138+
tmp_file.flush()
139+
140+
141+
def make_diff(title, path_to_diff, base_sha, head_sha, suffix, ignore_whitespace):
142+
ws_flag = "-b" if ignore_whitespace else ""
143+
diff_exists = get_cmd_status(f"git diff --quiet {ws_flag} {BASE_BRANCH_NAME} {HEAD_BRANCH_NAME} -- {path_to_diff}")
144+
145+
if diff_exists == 0:
146+
eprint(f"No diff output for {base_sha}..{head_sha}")
147+
return None
148+
149+
run(f'mkdir -p {OUTPUT_PATH}/{base_sha}/{head_sha}')
150+
dest_path = f"{base_sha}/{head_sha}/diff-{suffix}.html"
151+
whitespace_context = "(ignoring whitespace)" if ignore_whitespace else ""
152+
with tempfile.NamedTemporaryFile() as tmp_file:
153+
write_html_template(title, f"rev. {head_sha} {whitespace_context}", tmp_file)
154+
155+
# Generate HTML diff. This uses the diff2html-cli, which defers to `git diff` under the hood.
156+
# All arguments after the first `--` go to the `git diff` command.
157+
diff_cmd = f"diff2html -s line -f html -d word -i command --hwt " \
158+
f"{tmp_file.name} -F {OUTPUT_PATH}/{dest_path} -- " \
159+
f"-U20 {ws_flag} {BASE_BRANCH_NAME} {HEAD_BRANCH_NAME} -- {path_to_diff}"
160+
eprint(f"Running diff cmd: {diff_cmd}")
161+
run(diff_cmd)
162+
return dest_path
163+
164+
165+
def diff_link(diff_text, empty_diff_text, diff_location, alternate_text, alternate_location):
166+
if diff_location is None:
167+
return empty_diff_text
168+
169+
return f"[{diff_text}]({CDN_URL}/codegen-diff/{diff_location}) ([{alternate_text}]({CDN_URL}/codegen-diff/{alternate_location}))"
170+
171+
172+
def make_diffs(base_sha, head_sha):
173+
sdk_ws = make_diff('AWS SDK', f'{OUTPUT_PATH}/services', base_sha, head_sha, 'aws-sdk', ignore_whitespace=False)
174+
sdk_no_ws = make_diff('AWS SDK', f'{OUTPUT_PATH}/services', base_sha, head_sha, 'aws-sdk-ignore-ws',
175+
ignore_whitespace=True)
176+
177+
sdk_links = diff_link('AWS SDK', 'No codegen difference in the AWS SDK', sdk_ws, 'ignoring whitespace', sdk_no_ws)
178+
179+
return f'A new generated diff is ready to view.\\n\\n- {sdk_links}\\n'
180+
181+
182+
def create_cli():
183+
parser = argparse.ArgumentParser(
184+
prog="codegen-diff-revisions",
185+
description="Generate HTML diffs of codegen output",
186+
formatter_class=argparse.ArgumentDefaultsHelpFormatter
187+
)
188+
189+
parser.add_argument("repo_root", help="repository root")
190+
parser.add_argument("base_sha", help="base commit to diff against (SHA-like)")
191+
parser.add_argument("--bootstrap", help="services to pass to bootstrap and include in diff output",
192+
default="+dynamodb,+codebuild,+sts,+ec2,+polly,+s3")
193+
parser.add_argument("--head-sha", help="head commit to use (defaults to whatever current HEAD) is")
194+
195+
return parser
196+
197+
198+
def main():
199+
cli = create_cli()
200+
opts = cli.parse_args()
201+
print(opts)
202+
203+
os.chdir(opts.repo_root)
204+
205+
if opts.head_sha is None:
206+
head_sha = get_cmd_output("git rev-parse HEAD")
207+
else:
208+
head_sha = opts.head_sha
209+
210+
print(f"using head sha is {head_sha}")
211+
212+
# Make sure the working tree is clean
213+
if get_cmd_status("git diff --quiet") != 0:
214+
eprint("working tree is not clean. aborting")
215+
sys.exit(1)
216+
217+
# Generate code for HEAD
218+
print(f"Creating temporary branch with generated code for the HEAD revision {head_sha}")
219+
run(f"git checkout {head_sha} -b {HEAD_BRANCH_NAME}")
220+
generate_and_commit_generated_code(head_sha, opts.bootstrap)
221+
222+
# Generate code for base
223+
print(f"Creating temporary branch with generated code for the base revision {opts.base_sha}")
224+
run(f"git checkout {opts.base_sha} -b {BASE_BRANCH_NAME}")
225+
generate_and_commit_generated_code(opts.base_sha, opts.bootstrap)
226+
227+
bot_message = make_diffs(opts.base_sha, head_sha)
228+
with open(f"{OUTPUT_PATH}/bot-message", 'w') as f:
229+
f.write(bot_message)
230+
231+
# cleanup
232+
if not running_in_github_action():
233+
run(f"git checkout main")
234+
run(f"git branch -D {BASE_BRANCH_NAME}")
235+
run(f"git branch -D {HEAD_BRANCH_NAME}")
236+
237+
238+
if __name__ == '__main__':
239+
main()

‎.github/scripts/mk-generated.sh‎

Lines changed: 0 additions & 51 deletions
This file was deleted.

‎.github/scripts/runCodeBuildBatchJob.sh‎ renamed to ‎.github/scripts/run-codebuild-batch-job.sh‎

File renamed without changes.

‎.github/workflows/codebuild-ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,4 @@ jobs:
3535
aws-region: us-west-2
3636
- name: Run Service Check Batch
3737
run: |
38-
.github/scripts/runCodeBuildBatchJob.sh gh-aws-sdk-kotlin-svc-check-batch ${{ github.event.pull_request.head.sha }}
38+
.github/scripts/run-codebuild-batch-job.sh gh-aws-sdk-kotlin-svc-check-batch ${{ github.event.pull_request.head.sha }}

0 commit comments

Comments
 (0)