WIP failing signing request for some reason

Author: brandonskiser

build-config/signing/qchat/manifest.yaml

@@ -0,0 +1,11 @@
+type: :app # no need to change this for .pkg, .dmg and executables
+os: :osx
+name: qchat
+outputs:
+- label: macos
+  path: qchat
+app:
+  :identifier: com.amazon.codewhisperer # this must equal the Bundle Identifier value in the embedded Info.plist
+  signing_requirements:
+    certificate_type: :developerIDAppDistribution
+    app_id_prefix: 94KV3E626L # AMZN Mobile LLC

build-config/signing/qchat/manifest.yaml.template

@@ -0,0 +1,11 @@
+type: :app # no need to change this for .pkg, .dmg and executables
+os: :osx
+name: __NAME__ # This should be the file name
+outputs:
+- label: macos
+  path: __NAME__ # This should be the file name
+app:
+  :identifier: com.amazon.codewhisperer # this must equal the Bundle Identifier value in the embedded Info.plist
+  signing_requirements:
+    certificate_type: :developerIDAppDistribution
+    app_id_prefix: 94KV3E626L # AMZN Mobile LLC

build-scripts/qchatbuild.py

@@ -1,9 +1,12 @@
+import json
 import pathlib
+from functools import cache
 import os
 import shutil
-from typing import Mapping, Sequence
+import time
+from typing import Any, Mapping, Sequence
 from build import generate_sha
-from const import CHAT_BINARY_NAME, CHAT_PACKAGE_NAME, LINUX_ARCHIVE_NAME
+from const import APPLE_TEAM_ID, CHAT_BINARY_NAME, CHAT_PACKAGE_NAME, LINUX_ARCHIVE_NAME
 from signing import (
     CdSigningData,
     CdSigningType,
@@ -12,12 +15,16 @@
     cd_sign_file,
     apple_notarize_file,
 )
-from util import info, isDarwin, run_cmd
+from util import info, isDarwin, run_cmd, warn
 from rust import cargo_cmd_name, rust_env, rust_targets
+from importlib import import_module
 
 BUILD_DIR_RELATIVE = pathlib.Path(os.environ.get("BUILD_DIR") or "build")
 BUILD_DIR = BUILD_DIR_RELATIVE.absolute()
 
+REGION = "us-west-2"
+SIGNING_API_BASE_URL = "https://api.signer.builder-tools.aws.dev"
+
 
 def run_cargo_tests():
     args = [cargo_cmd_name()]
@@ -51,14 +58,23 @@ def build_chat_bin(
 
     args = [cargo_cmd_name(), "build", "--locked", "--package", package]
 
-    if release:
-        args.append("--release")
+    for target in targets:
+        args.extend(["--target", target])
 
     if release:
+        args.append("--release")
         target_subdir = "release"
     else:
         target_subdir = "debug"
 
+    run_cmd(
+        args,
+        env={
+            **os.environ,
+            **rust_env(release=release),
+        },
+    )
+
     # create "universal" binary for macos
     if isDarwin():
         out_path = BUILD_DIR / f"{output_name or package}-universal-apple-darwin"
@@ -82,17 +98,201 @@ def build_chat_bin(
         return out_path
 
 
-def sign_and_notarize(chat_path: pathlib.Path):
+@cache
+def get_creds():
+    boto3 = import_module("boto3")
+    session = boto3.Session()
+    credentials = session.get_credentials()
+    creds = credentials.get_frozen_credentials()
+    return creds
+
+
+def cd_signer_request(method: str, path: str, data: str | None = None):
+    SigV4Auth = import_module("botocore.auth").SigV4Auth
+    AWSRequest = import_module("botocore.awsrequest").AWSRequest
+    requests = import_module("requests")
+
+    url = f"{SIGNING_API_BASE_URL}{path}"
+    headers = {"Content-Type": "application/json"}
+    request = AWSRequest(method=method, url=url, data=data, headers=headers)
+    SigV4Auth(get_creds(), "signer-builder-tools", REGION).add_auth(request)
+
+    for i in range(1, 8):
+        response = requests.request(method=method, url=url, headers=dict(request.headers), data=data)
+        info(f"CDSigner Request ({url}): {response.status_code}")
+        if response.status_code == 429:
+            warn(f"Too many requests, backing off for {2**i} seconds")
+            time.sleep(2**i)
+            continue
+        return response
+
+    raise Exception(f"Failed to request {url}")
+
+
+def cd_signer_create_request(manifest: Any) -> str:
+    response = cd_signer_request(
+        method="POST",
+        path="/signing_requests",
+        data=json.dumps({"manifest": manifest}),
+    )
+    response_json = response.json()
+    info(f"Signing request create: {response_json}")
+    request_id = response_json["signingRequestId"]
+    return request_id
+
+
+def cd_signer_start_request(request_id: str, source_key: str, destination_key: str, signing_data: CdSigningData):
+    response_text = cd_signer_request(
+        method="POST",
+        path=f"/signing_requests/{request_id}/start",
+        data=json.dumps(
+            {
+                "iamRole": f"arn:aws:iam::{signing_data.aws_account_id}:role/{signing_data.signing_role_name}",
+                "s3Location": {
+                    "bucket": signing_data.bucket_name,
+                    "sourceKey": source_key,
+                    "destinationKey": destination_key,
+                },
+            }
+        ),
+    ).text
+    info(f"Signing request start: {response_text}")
+
+
+def cd_signer_status_request(request_id: str):
+    response_json = cd_signer_request(
+        method="GET",
+        path=f"/signing_requests/{request_id}",
+    ).json()
+    info(f"Signing request status: {response_json}")
+    return response_json["signingRequest"]["status"]
+
+
+def cd_build_signed_package(file_path: pathlib.Path):
+    """
+    Creates a tarball `package.tar.gz` with the following structure:
+    ```
+    package
+    ├─ manifest.yaml
+    ├─ artifact
+    | ├─ EXECUTABLES_TO_SIGN
+    | | ├─ qchat
+    ```
+    """
+    working_dir = BUILD_DIR / "package"
+    shutil.rmtree(working_dir, ignore_errors=True)
+    (BUILD_DIR / "package" / "artifact" / "EXECUTABLES_TO_SIGN").mkdir(parents=True)
+
+    name = file_path.name
+
+    # Write the manifest.yaml
+    manifest_template_path = pathlib.Path.cwd() / "build-config" / "signing" / "qchat" / "manifest.yaml.template"
+    (working_dir / "manifest.yaml").write_text(manifest_template_path.read_text().replace("__NAME__", name))
+
+    shutil.copy2(file_path, working_dir / "artifact" / "EXECUTABLES_TO_SIGN" / file_path.name)
+    file_path.unlink()
+
+    run_cmd(
+        ["gtar", "-czf", BUILD_DIR / "package.tar.gz", "manifest.yaml", "artifact"],
+        cwd=working_dir,
+    )
+
+    return BUILD_DIR / "package.tar.gz"
+
+
+def manifest(
+    name: str,
+    identifier: str,
+):
+    """
+    Creates the required manifest argument when submitting the signing task. This has the same
+    structure as the manifest.yaml.template under `build-config/signing/qchat/manifest.yaml.template`
+    """
+    return {
+        "type": "app",
+        "os": "osx",
+        "name": name,
+        "outputs": [{"label": "macos", "path": name}],
+        "app": {
+            "identifier": identifier,
+            "signing_requirements": {
+                "certificate_type": "developerIDAppDistribution",
+                "app_id_prefix": APPLE_TEAM_ID,
+            },
+        },
+    }
+
+
+def sign_executable(signing_data: CdSigningData, chat_path: pathlib.Path):
+    name = chat_path.name
+    info(f"Signing {name}")
+
+    info("Packaging...")
+    package_path = cd_build_signed_package(chat_path)
+
+    info("Uploading...")
+    run_cmd(["aws", "s3", "rm", "--recursive", f"s3://{signing_data.bucket_name}/signed"])
+    run_cmd(["aws", "s3", "rm", "--recursive", f"s3://{signing_data.bucket_name}/pre-signed"])
+    run_cmd(["aws", "s3", "cp", package_path, f"s3://{signing_data.bucket_name}/pre-signed/package.tar.gz"])
+
+    info("Sending request...")
+    request_id = cd_signer_create_request(manifest(name, "com.amazon.codewhisperer"))
+    cd_signer_start_request(
+        request_id=request_id,
+        source_key="pre-signed/package.tar.gz",
+        destination_key="signed/signed.zip",
+        signing_data=signing_data,
+    )
+
+    max_duration = 180
+    end_time = time.time() + max_duration
+    i = 1
+    while True:
+        info(f"Checking for signed package attempt #{i}")
+        status = cd_signer_status_request(request_id)
+        info(f"Package has status: {status}")
+
+        match status:
+            case "success":
+                break
+            case "created" | "processing" | "inProgress":
+                pass
+            case "failure":
+                raise RuntimeError("Signing request failed")
+            case _:
+                warn(f"Unexpected status, ignoring: {status}")
+
+        if time.time() >= end_time:
+            raise RuntimeError("Signed package did not appear, check signer logs")
+        time.sleep(2)
+        i += 1
+
+    info("Signed!")
+
+    info("Downloading...")
+    run_cmd(["aws", "s3", "cp", f"s3://{signing_data.bucket_name}/signed/signed.zip", "signed.zip"])
+    run_cmd(["unzip", "signed.zip"])
+
+
+def sign_and_notarize(signing_data: CdSigningData, chat_path: pathlib.Path):
     # First, sign the application
+    sign_executable(signing_data, chat_path)
 
     # Next, notarize the application
 
     # Last, staple the notarization to the application
     pass
 
 
-def build_macos(chat_path: pathlib.Path):
-    sign_and_notarize(chat_path)
+def build_macos(chat_path: pathlib.Path, signing_data: CdSigningData | None):
+    chat_dst = BUILD_DIR / "qchat"
+    chat_dst.unlink(missing_ok=True)
+    shutil.copy2(chat_path, chat_dst)
+
+    if signing_data:
+        sign_and_notarize(signing_data, chat_dst)
+
+    return chat_dst
 
 
 def build_linux(chat_path: pathlib.Path):
@@ -194,4 +394,24 @@ def build(
         targets=targets,
     )
 
-    pass
+    if isDarwin():
+        if signing_bucket and aws_account_id and apple_id_secret and signing_role_name:
+            signing_data = CdSigningData(
+                bucket_name=signing_bucket,
+                aws_account_id=aws_account_id,
+                notarizing_secret_id=apple_id_secret,
+                signing_role_name=signing_role_name,
+            )
+        else:
+            signing_data = None
+
+        chat_path = build_macos(chat_path, signing_data)
+        sha_path = generate_sha(chat_path)
+
+        if output_bucket:
+            staging_location = f"s3://{output_bucket}/staging/"
+            info(f"Build complete, sending to {staging_location}")
+            run_cmd(["aws", "s3", "cp", chat_path, staging_location])
+            run_cmd(["aws", "s3", "cp", sha_path, staging_location])
+    else:
+        build_linux(chat_path)

crates/chat-cli/.gitignore

@@ -1,2 +1,5 @@
 build/
-spec.ts
+spec.ts
+
+# Info.plist is generated by the build script for macOS
+src/Info.plist