chore(ci): fix codegen diff (#1063)

Author: aajtodd

.github/scripts/codegen-diff-revisions.py

@@ -33,8 +33,6 @@
 import tempfile
 import shlex
 
-HEAD_BRANCH_NAME = "__tmp-localonly-head"
-BASE_BRANCH_NAME = "__tmp-localonly-base"
 OUTPUT_PATH = "tmp-codegen-diff/"
 
 COMMIT_AUTHOR_NAME = "GitHub Action (generated codegen diff)"
@@ -138,9 +136,11 @@ def write_html_template(title, subtitle, tmp_file):
     tmp_file.flush()
 
 
-def make_diff(title, path_to_diff, base_sha, head_sha, suffix, ignore_whitespace):
+def make_diff(opts, title, path_to_diff, suffix, ignore_whitespace):
+    base_sha = opts.base_sha
+    head_sha = opts.head_sha
     ws_flag = "-b" if ignore_whitespace else ""
-    diff_exists = get_cmd_status(f"git diff --quiet {ws_flag} {BASE_BRANCH_NAME} {HEAD_BRANCH_NAME} -- {path_to_diff}")
+    diff_exists = get_cmd_status(f"git diff --quiet {ws_flag} {opts.base_branch} {opts.head_branch} -- {path_to_diff}")
 
     if diff_exists == 0:
         eprint(f"No diff output for {base_sha}..{head_sha}")
@@ -156,7 +156,7 @@ def make_diff(title, path_to_diff, base_sha, head_sha, suffix, ignore_whitespace
         # All arguments after the first `--` go to the `git diff` command.
         diff_cmd = f"diff2html -s line -f html -d word -i command --hwt " \
                    f"{tmp_file.name} -F {OUTPUT_PATH}/{dest_path} -- " \
-                   f"-U20 {ws_flag} {BASE_BRANCH_NAME} {HEAD_BRANCH_NAME} -- {path_to_diff}"
+                   f"-U20 {ws_flag} {opts.base_branch} {opts.head_branch} -- {path_to_diff}"
         eprint(f"Running diff cmd: {diff_cmd}")
         run(diff_cmd)
     return dest_path
@@ -169,29 +169,46 @@ def diff_link(diff_text, empty_diff_text, diff_location, alternate_text, alterna
     return f"[{diff_text}]({CDN_URL}/codegen-diff/{diff_location}) ([{alternate_text}]({CDN_URL}/codegen-diff/{alternate_location}))"
 
 
-def make_diffs(base_sha, head_sha):
-    sdk_ws = make_diff('AWS SDK', f'{OUTPUT_PATH}/services', base_sha, head_sha, 'aws-sdk', ignore_whitespace=False)
-    sdk_no_ws = make_diff('AWS SDK', f'{OUTPUT_PATH}/services', base_sha, head_sha, 'aws-sdk-ignore-ws',
-                          ignore_whitespace=True)
-
+def make_diffs(opts):
+    sdk_ws = make_diff(opts, 'AWS SDK', f'{OUTPUT_PATH}/services', 'aws-sdk', ignore_whitespace=False)
+    sdk_no_ws = make_diff(opts, 'AWS SDK', f'{OUTPUT_PATH}/services', 'aws-sdk-ignore-ws', ignore_whitespace=True)
     sdk_links = diff_link('AWS SDK', 'No codegen difference in the AWS SDK', sdk_ws, 'ignoring whitespace', sdk_no_ws)
 
     return f'A new generated diff is ready to view.\\n\\n- {sdk_links}\\n'
 
 
+def _codegen_cmd(opts):
+    generate_and_commit_generated_code(opts.head_sha, opts.bootstrap)
+
+
+def _generate_diffs_cmd(opts):
+    bot_message = make_diffs(opts)
+    with open(f"{OUTPUT_PATH}/bot-message", 'w') as f:
+        f.write(bot_message)
+
+
 def create_cli():
     parser = argparse.ArgumentParser(
         prog="codegen-diff-revisions",
         description="Generate HTML diffs of codegen output",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
-    parser.add_argument("repo_root", help="repository root")
-    parser.add_argument("base_sha", help="base commit to diff against (SHA-like)")
-    parser.add_argument("--bootstrap", help="services to pass to bootstrap and include in diff output",
-                        default="+dynamodb,+codebuild,+sts,+ec2,+polly,+s3")
     parser.add_argument("--head-sha", help="head commit to use (defaults to whatever current HEAD) is")
 
+    subparsers = parser.add_subparsers()
+    codegen = subparsers.add_parser("codegen", help="generate and commit generated code")
+    codegen.add_argument("--bootstrap", help="services to pass to bootstrap and include in diff output",
+                         default="+dynamodb,+codebuild,+sts,+ec2,+polly,+s3")
+    codegen.set_defaults(cmd=_codegen_cmd)
+
+    generate_diffs = subparsers.add_parser("generate-diffs",
+                                           help="generate diffs between two branches and output bot message")
+    generate_diffs.add_argument("--base-sha", help="base commit to diff against (SHA-like)")
+    generate_diffs.add_argument("base_branch", help="name of the base branch to diff against")
+    generate_diffs.add_argument("head_branch", help="name of the head branch to diff against")
+    generate_diffs.set_defaults(cmd=_generate_diffs_cmd)
+
     return parser
 
 
@@ -200,39 +217,12 @@ def main():
     opts = cli.parse_args()
     print(opts)
 
-    os.chdir(opts.repo_root)
-
     if opts.head_sha is None:
-        head_sha = get_cmd_output("git rev-parse HEAD")
-    else:
-        head_sha = opts.head_sha
-
-    print(f"using head sha is {head_sha}")
-
-    # Make sure the working tree is clean
-    if get_cmd_status("git diff --quiet") != 0:
-        eprint("working tree is not clean. aborting")
-        sys.exit(1)
+        opts.head_sha = get_cmd_output("git rev-parse HEAD")
 
-    # Generate code for HEAD
-    print(f"Creating temporary branch with generated code for the HEAD revision {head_sha}")
-    run(f"git checkout {head_sha} -b {HEAD_BRANCH_NAME}")
-    generate_and_commit_generated_code(head_sha, opts.bootstrap)
-
-    # Generate code for base
-    print(f"Creating temporary branch with generated code for the base revision {opts.base_sha}")
-    run(f"git checkout {opts.base_sha} -b {BASE_BRANCH_NAME}")
-    generate_and_commit_generated_code(opts.base_sha, opts.bootstrap)
-
-    bot_message = make_diffs(opts.base_sha, head_sha)
-    with open(f"{OUTPUT_PATH}/bot-message", 'w') as f:
-        f.write(bot_message)
+    print(f"using head sha: {opts.head_sha}")
 
-    # cleanup
-    if not running_in_github_action():
-        run(f"git checkout main")
-        run(f"git branch -D {BASE_BRANCH_NAME}")
-        run(f"git branch -D {HEAD_BRANCH_NAME}")
+    opts.cmd(opts)
 
 
 if __name__ == '__main__':

.github/workflows/codegen-preview.yml

@@ -20,10 +20,6 @@ concurrency:
 
 env:
   JAVA_VERSION: 11
-  BUILDER_VERSION: v0.8.22
-  BUILDER_SOURCE: releases
-  BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
-  PACKAGE_NAME: aws-sdk-kotlin
   RUN: ${{ github.run_id }}-${{ github.run_number }}
   DIFF2HTML_VERSION: 5.2.5
   # Below is the set of services that are generated for codegen preview
@@ -35,6 +31,8 @@ env:
   #  - @restJson1: polly
   #  - @restXml: s3
   PREVIEW_SERVICES: +dynamodb,+codebuild,+sts,+ec2,+polly,+s3
+  HEAD_BRANCH_NAME: __tmp-localonly-head
+  BASE_BRANCH_NAME: __tmp-localonly-base
 
 jobs:
 
@@ -44,9 +42,20 @@ jobs:
     outputs:
       bot-message: ${{ steps.generate-diff.outputs.codegen-diff-msg }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          path: 'aws-sdk-kotlin'
+      - uses: actions/checkout@v4
+        with:
+          repository: 'awslabs/aws-kotlin-repo-tools'
+          ref: 'ci-utils'
+          path: 'aws-kotlin-repo-tools'
+      - uses: actions/checkout@v4
+        with:
+          repository: 'awslabs/smithy-kotlin'
+          fetch-depth: 0
+          path: 'smithy-kotlin'
       - uses: actions/cache@v2
         name: Gradle Cache
         with:
@@ -60,47 +69,87 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: ${{ env.JAVA_VERSION }}
-      - name: Install and build deps
-        # abuse crt builder to download and build upstream dependencies like smithy-kotlin
+
+      - name: Install deps and setup environment
         run: |
           npm install -g diff2html-cli@${{ env.DIFF2HTML_VERSION }}
-          python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
-          chmod a+x builder.pyz
-          ./builder.pyz build -p ${{ env.PACKAGE_NAME }} --variant codegen-preview
-      - name: Generate diff
-        id: generate-diff
+          env | sort
+          # store off a copy of head ref of ci.py, otherwise base ref generation will use a different version of this script
+          CODEGEN_DIFF_REVISIONS=${{ runner.temp }}/codegen-diff-revisions.py
+          cp $GITHUB_WORKSPACE/aws-sdk-kotlin/.github/scripts/codegen-diff-revisions.py $CODEGEN_DIFF_REVISIONS
+          echo "CODEGEN_DIFF_REVISIONS=$CODEGEN_DIFF_REVISIONS" >> "$GITHUB_ENV"
+          echo "REPO_TOOLS=$GITHUB_WORKSPACE/aws-kotlin-repo-tools" >> "$GITHUB_ENV"
+          echo "SMITHY_KOTLIN_DIR=$GITHUB_WORKSPACE/smithy-kotlin" >> "$GITHUB_ENV"
+          echo "SDK_DIR=$GITHUB_WORKSPACE/aws-sdk-kotlin" >> "$GITHUB_ENV"
+
+      - name: Generate code for head ref
         run: |
-          # codegen-diff-revisions requires a clean index, set-upstream-versions.py can modify local repo state
-          # we don't push these branches/commits anywhere so just commit it if necessary and move on
-          if ! git diff --quiet gradle.properties
+          branch=$(python3 $REPO_TOOLS/scripts/ci.py get-branch $SDK_DIR)
+          pushd $SMITHY_KOTLIN_DIR
+          if git ls-remote --quiet --exit-code --heads origin refs/heads/$branch
           then
-            echo "gradle.properties is dirty, committing before generating diffs"
-            git add gradle.properties
-            PRE_COMMIT_ALLOW_NO_CONFIG=1 git \
-            -c "user.name=GitHub Action (generated code preview)" \
-            -c "[email protected]" \
-            commit -m "codegen diff autocommit" --allow-empty
+            echo "using smithy-kotlin branch $branch for head ref codegen"
+          else
+            # in the case of feature branches with tentacles we may or may not have a corresponding base ref in smithy-kotlin
+            # if we do let's use that, otherwise fallback to main
+            echo "$branch does not exist in smithy-kotlin, will attempt base ref $GITHUB_BASE_REF"
+            branch=$GITHUB_BASE_REF
           fi
-          .github/scripts/codegen-diff-revisions.py \
-            --bootstrap ${{ env.PREVIEW_SERVICES }} \
+          python3 $REPO_TOOLS/scripts/ci.py -v set-branch --branch $branch
+          popd
+          pushd $SDK_DIR
+          git checkout -b $HEAD_BRANCH_NAME
+          $CODEGEN_DIFF_REVISIONS codegen --bootstrap ${{ env.PREVIEW_SERVICES }}
+          popd
+
+      - name: Generate code for base ref
+        run: |
+          branch=$GITHUB_BASE_REF
+          echo "checkout smithy-kotlin at base ref: $branch"
+          pushd $SMITHY_KOTLIN_DIR
+          git switch -f main
+          python3 $REPO_TOOLS/scripts/ci.py -v set-branch --branch $branch
+          popd
+          echo "resetting aws-sdk-kotlin"
+          pushd $SDK_DIR
+          git switch -f main
+          python3 $REPO_TOOLS/scripts/ci.py -v set-branch --branch $branch 
+          git checkout -b $BASE_BRANCH_NAME
+          $CODEGEN_DIFF_REVISIONS codegen --bootstrap ${{ env.PREVIEW_SERVICES }}
+          popd
+
+      - name: Generate diffs
+        id: generate-diff
+        run: |
+          pushd $SDK_DIR
+          $CODEGEN_DIFF_REVISIONS \
             --head-sha ${{ github.event.pull_request.head.sha }} \
-            . ${{ github.event.pull_request.base.sha }}
-          echo "codegen-diff-msg<<EOF" >> $GITHUB_OUTPUT
-          cat ./tmp-codegen-diff/bot-message) >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
+            generate-diffs \
+            --base-sha ${{ github.event.pull_request.base.sha }} \
+            $BASE_BRANCH_NAME $HEAD_BRANCH_NAME
+          cat ./tmp-codegen-diff/bot-message
+          {
+            echo 'codegen-diff-msg<<DIFF_MSG_EOF'
+            echo "$(cat ./tmp-codegen-diff/bot-message)"
+            echo 'DIFF_MSG_EOF'
+          } >> "$GITHUB_OUTPUT"
+          echo $GITHUB_OUTPUT
+
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v2
         with:
           role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
           aws-region: us-west-2
+
       - name: Upload diff to S3
         run: |
+          SDK_DIR=$GITHUB_WORKSPACE/aws-sdk-kotlin
+          pushd $SDK_DIR
           if [[ -d ./tmp-codegen-diff/${{ github.event.pull_request.base.sha }} ]]; then
             aws s3 cp ./tmp-codegen-diff/${{ github.event.pull_request.base.sha }} \
                 "s3://${{ secrets.CDN_S3_BUCKET_NAME }}/codegen-diff/${{ github.event.pull_request.base.sha }}" --recursive
           fi
 
-
   # TODO - generate doc preview for N services and upload and link as well
   post-bot-comment:
     name: Post bot comment
@@ -117,4 +166,4 @@ jobs:
               owner: context.repo.owner,
               repo: context.repo.repo,
               body: '${{ needs.generate-codegen-diff.outputs.bot-message }}\n\n'
-            })
+            })

builder.json

@@ -57,11 +57,6 @@
             "!test_steps": [
                 "{gradlew} test allTests"
             ]
-        },
-        "codegen-preview": {
-            "!imports": [],
-            "!build_steps": [],
-            "!test_steps": []
         }
     }
 }