Benchmark mock contract sizes

.github/workflows/benches/show-diff.yml

@@ -0,0 +1,72 @@
+name: Benchmark Diff on PR
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+    branches:
+      - main
+      - 'release-v*'
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  comment-benchmark-diff:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Extract scarb version
+        run: |
+          SCARB_VERSION=$(grep 'scarb-version = ' Scarb.toml | sed 's/scarb-version = "\(.*\)"/\1/')
+          echo "SCARB_VERSION=$SCARB_VERSION" >> "$GITHUB_ENV"
+
+      - name: Setup scarb
+        uses: software-mansion/setup-scarb@v1
+        id: setup_scarb
+        with:
+          scarb-version: ${{ env.SCARB_VERSION }}
+
+      - name: Build mocks
+        run: scarb --release build
+
+      - name: Run benchmark and capture diff
+        id: benchmark_diff
+        run: |
+          python3 scripts/benchmark_diff.py scripts/benchmark.py benches/contract_sizes.json --dir target/release --markdown > diff_output.txt
+
+      - name: Prepare benchmark comment
+        run: |
+          {
+            echo "<!-- comment-id:benchmark-diff -->"
+            echo "### 🧪 Cairo Contract Size Benchmark Diff"
+            echo
+            echo '```diff'
+            cat diff_output.txt
+            echo '```'
+            echo
+            echo "_This comment was generated automatically from benchmark diffs._"
+          } > comment.md
+
+      - name: Find comment to update
+        uses: peter-evans/find-comment@v3
+        id: get_comment
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: benchmark-diff
+
+      - name: Echo a string
+        run: echo "secret: ${{ secrets.GITHUB_TOKEN }}"
+
+      - name: Post benchmark diff comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-id: ${{ steps.get_comment.outputs.comment-id }}
+          edit-mode: replace
+          body-file: comment.md

.github/workflows/benches/update-benches.yml

@@ -0,0 +1,61 @@
+name: Update contract sizes benchmark
+
+on:
+  pull_request:
+    types:
+      - closed   # Trigger when a PR is closed (merged or just closed)
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  run-on-merge:
+    if: github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Extract scarb version
+        run: |
+          SCARB_VERSION=$(grep 'scarb-version = ' Scarb.toml | sed 's/scarb-version = "\(.*\)"/\1/')
+          echo "SCARB_VERSION=$SCARB_VERSION" >> "$GITHUB_ENV"
+
+      - name: Setup scarb
+        uses: software-mansion/setup-scarb@v1
+        id: setup_scarb
+        with:
+          scarb-version: ${{ env.SCARB_VERSION }}
+
+      - name: Build mocks
+        run: scarb --release build
+
+      - name: Update benchmark
+        run: |
+          python3 ./scripts/benchmark.py --json --dir target/release > benches/contract_sizes.json
+
+      - name: Check if file changed
+        id: check_diff
+        run: |
+          if git diff --quiet origin/main -- benches/contract_sizes.json; then
+            echo "changed=false" >> $GITHUB_OUTPUT
+          else
+            echo "changed=true" >> $GITHUB_OUTPUT
+          fi
+
+      # Only create a PR if the file changed
+      - name: Create Pull Request with benchmark update
+        if: steps.check_diff.outputs.changed == 'true'
+        uses: peter-evans/create-pull-request@v6
+        with:
+          commit-message: Update contract sizes benchmark
+          title: Update contract sizes benchmark
+          body: |
+            This PR updates the contract size benchmarks after a recent merge to `main`.
+          branch: update/contract-sizes-${{ github.run_id }}
+          base: main
+          token: ${{ secrets.GITHUB_TOKEN }}

benches/contract_sizes.json

@@ -0,0 +1,4 @@
+{
+  "bytecode": {},
+  "contract_class": {}
+}

packages/test_common/Scarb.toml

@@ -36,4 +36,4 @@ openzeppelin_utils = { path = "../utils" }
 [[target.starknet-contract]]
 allowed-libfuncs-list.name = "experimental"
 sierra = true
-casm = false
+casm = true # Required for benchmarking

scripts/benchmark.py

@@ -0,0 +1,101 @@
+import os
+import json
+import sys
+import argparse
+
+# ANSI color codes (no external dependencies)
+RESET   = "\033[0m"
+BOLD    = "\033[1m"
+YELLOW  = "\033[33m"
+GREEN   = "\033[32m"
+RED     = "\033[31m"
+CYAN    = "\033[36m"
+
+# Set the path to your Scarb release output, e.g., "target/release"
+TARGET_DIR = "target/release"
+
+
+def try_get_name(filename):
+    """
+    Extracts the contract name from the filename:
+    - Starts at the first uppercase letter.
+    - Ends at the next '.' or end of string.
+    Returns the filename if no uppercase letter is found.
+    """
+    for i, c in enumerate(filename):
+        if c.isupper():
+            start = i
+            end = filename.find('.', start)
+            if end == -1:
+                return filename[start:]
+            else:
+                return filename[start:end]
+    return filename
+
+
+def get_bytecode_size(json_path):
+    with open(json_path, "r") as f:
+        data = json.load(f)
+    bytecode = data.get("bytecode", [])
+    num_felts = len(bytecode)
+    return num_felts
+
+
+def get_sierra_contract_class_size(json_path):
+    num_bytes = os.path.getsize(json_path)
+    return num_bytes
+
+
+def benchmark_contracts(target_dir):
+    results = {"bytecode": {}, "contract_class": {}}
+    for file in os.listdir(target_dir):
+        if file.endswith(".compiled_contract_class.json"):
+            path = os.path.join(target_dir, file)
+            try:
+                num_felts = get_bytecode_size(path)
+                results["bytecode"][file] = {"felts": num_felts}
+            except Exception as e:
+                results["bytecode"][file] = {"error": str(e)}
+        elif file.endswith(".contract_class.json"):
+            path = os.path.join(target_dir, file)
+            try:
+                num_bytes = get_sierra_contract_class_size(path)
+                results["contract_class"][file] = {"bytes": num_bytes}
+            except Exception as e:
+                results["contract_class"][file] = {"error": str(e)}
+    return results
+
+
+def print_benchmark_results(results):
+    print(f"{BOLD}{CYAN}CASM bytecode sizes:{RESET}")
+    for file, info in results["bytecode"].items():
+        name = f"{BOLD}{YELLOW}{try_get_name(file)}{RESET}"
+        if "felts" in info:
+            value = f"{BOLD}{GREEN}{info['felts']} felts{RESET}"
+            print(f"{name}: {value}")
+        else:
+            print(f"{RED}Error processing {file}: {info['error']}{RESET}")
+
+    print(f"\n{BOLD}{CYAN}Sierra contract class sizes:{RESET}")
+    for file, info in results["contract_class"].items():
+        name = f"{BOLD}{YELLOW}{try_get_name(file)}{RESET}"
+        if "bytes" in info:
+            num_bytes = info["bytes"]
+            value = f"{BOLD}{GREEN}{num_bytes} bytes{RESET} ({num_bytes/1024:.2f} KB)"
+            print(f"{name}: {value}")
+        else:
+            print(f"{RED}Error processing {file}: {info['error']}{RESET}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Benchmark Cairo contract artifact sizes.")
+    parser.add_argument("--json", action="store_true", help="Output results as JSON.")
+    parser.add_argument("--dir", type=str, default=TARGET_DIR, help="Target directory (default: target/release)")
+    args = parser.parse_args()
+
+    results = benchmark_contracts(args.dir)
+    if args.json:
+        print(json.dumps(results, indent=2))
+    else:
+        print(f"{BOLD}Benchmarking CASM and Sierra contract class sizes in: {args.dir}\n{RESET}")
+        print_benchmark_results(results)