diff --git a/.gitignore b/.gitignore index a2da8d85e5d0..0e870663ed12 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ docs/docs/protocol-specs/public-vm/gen/ # for those who use Claude Code /.claude + +__pycache__ diff --git a/bootstrap.sh b/bootstrap.sh index da77d1dc1ba8..e9e5aa7d6a11 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -256,10 +256,10 @@ function build_and_test { echo_header "build and test" # Start the test engine. - # setsid will put it in it's own process group we can terminate on cleanup. rm -f $test_cmds_file touch $test_cmds_file - setsid color_prefix "test-engine" "denoise test_engine_start" & + # put it in it's own process group via background subshell, we can terminate on cleanup. + (color_prefix "test-engine" "denoise test_engine_start") & test_engine_pid=$! test_engine_pgid=$(ps -o pgid= -p $test_engine_pid) diff --git a/ci.sh b/ci.sh index 28960855d57c..63bad60c041f 100755 --- a/ci.sh +++ b/ci.sh @@ -12,12 +12,13 @@ ci3_workflow_id=128853861 function echo_cmd { local name=$1 shift - printf "${blue}${bold}%12s${reset}: %s\n" $name "$(echo $@ | sed 's/\.\\n/.\n /g')" + printf "${blue}${bold}%21s${reset}: %s\n" $name "$(echo $@ | sed 's/\.\\n/.\n /g')" } function print_usage { echo "usage: $(basename $0) " echo + echo_cmd "dash" "Display a dashboard showing CI runs for the current user." echo_cmd "fast" "Spin up an EC2 instance and run bootstrap ci-fast." echo_cmd "full" "Spin up an EC2 instance and run bootstrap ci-full." echo_cmd "full-no-test-cache" "Spin up an EC2 instance and run bootstrap ci-full-no-test-cache." @@ -31,25 +32,16 @@ function print_usage { echo_cmd "network-teardown" "Spin up an EC2 instance to teardown a network deployment." echo_cmd "release" "Spin up an EC2 instance and run bootstrap release." echo_cmd "shell-new" "Spin up an EC2 instance, clone the repo, and drop into a shell." - echo_cmd "shell-container" "Drop into a shell in the current running build instance container." + echo_cmd "shell" "Drop into a shell in the current running build instance container." echo_cmd "shell-host" "Drop into a shell in the current running build host." - echo_cmd "run" "Trigger a GA workflow for the current branch PR and tail logs." - echo_cmd "trigger" "Trigger the GA workflow on the PR associated with the current branch." - echo_cmd "rlog" "Tail the logs of the latest GA run or the given GA run ID." - echo_cmd "ilog" "Tail the logs of the current running build instance." - echo_cmd "dlog" "Display the log of the given denoise log ID." - echo_cmd "kill" "Terminate the EC2 instance for the current branch." + echo_cmd "log" "Display the log of the given log ID." + echo_cmd "kill" "Terminate running EC2 instance with instance_name." echo_cmd "draft" "Mark the current PR as draft (no automatic CI runs when pushing)." echo_cmd "ready" "Mark the current PR as ready (enable automatic CI runs when pushing)." echo_cmd "pr-url" "Print the URL of the current PR associated with the branch." - echo_cmd "last-run-url" "Print the URL of the last GA run for the current branch PR." - echo_cmd "gh-bench" "Download CI-uploaded benchmarks for the current commit." - echo_cmd "gh-deploy-bench" "Download CI-uploaded deployment benchmarks for the current commit." - echo_cmd "gh-spartan-bench" "Download CI-uploaded spartan benchmarks for the current commit." - echo_cmd "avm-inputs-collection" "Nightly: run e2e tests, dump AVM circuit inputs, upload to cache." - echo_cmd "avm-check-circuit" "Nightly: download cached AVM inputs, run check-circuit on each." + echo_cmd "avm-inputs-collection" "Run e2e tests, dump AVM circuit inputs, upload to cache." + echo_cmd "avm-check-circuit" "Download cached AVM inputs, run check-circuit on each." echo_cmd "help" "Display this help message." - } [ -n "$cmd" ] && shift @@ -69,49 +61,45 @@ function get_latest_run_id { gh run list --workflow $ci3_workflow_id -b $BRANCH --limit 1 --json databaseId -q .[0].databaseId } -function tail_live_instance { - get_ip_for_instance - [ -z "$ip" ] && return 1; - ssh -F $ci3/aws/build_instance_ssh_config -q -t -o ConnectTimeout=5 ubuntu@$ip " - trap 'exit 0' SIGINT - docker ps -a --filter name=aztec_build --format '{{.Names}}' | grep -q '^aztec_build$' || exit 1 - docker logs -f aztec_build - " -} - -# Used in merge-queue, nightly, and release flows. -function prep_vars { - export RUN_ID=${RUN_ID:-$(date +%s%3N)} - export PARENT_LOG_URL=http://ci.aztec-labs.com/$RUN_ID - export DENOISE=1 - export DENOISE_WIDTH=32 -} +# Jobs in the ci dashboards are grouped on a single line by RUN_ID. +export RUN_ID=${RUN_ID:-$(date +%s%3N)} +export PARENT_LOG_URL=http://ci.aztec-labs.com/$RUN_ID case "$cmd" in - "help"|"") - print_usage + dash) + watch_ci -s next,prs --user --watch ;; - fast|full|full-no-test-cache|full-no-test-cache-makefile|docs|barretenberg|barretenberg-full|avm-inputs-collection|avm-check-circuit) - export JOB_ID="x1-$cmd" + fast|full|full-no-test-cache|full-no-test-cache-makefile|docs|barretenberg|barretenberg-full) + export CI_DASHBOARD="prs" + export JOB_ID="x-$cmd" bootstrap_ec2 "./bootstrap.sh ci-$cmd" ;; - "grind") - prep_vars - # Spin up ec2 instance and run the merge-queue flow. + avm-inputs-collection|avm-check-circuit) + export CI_DASHBOARD="nightly" + export JOB_ID="x-$cmd" + bootstrap_ec2 "./bootstrap.sh ci-$cmd" + ;; + grind) + # Grind a default of 5 times. + export CI_DASHBOARD="local" + export DENOISE=1 + export DENOISE_WIDTH=32 run() { JOB_ID=$1 INSTANCE_POSTFIX=$1 ARCH=$2 exec denoise "bootstrap_ec2 './bootstrap.sh $3'" } export -f run - seq 1 ${1:-5} | parallel --termseq 'TERM,10000' --tagstring '{= $_=~s/run (\w+).*/$1/; =}' --line-buffered 'run $USER-x{}-full amd64 ci-full-no-test-cache' + seq 1 ${1:-5} | parallel --termseq 'TERM,10000' --tagstring '{= $_=~s/run (\w+).*/$1/; =}' --line-buffered \ + 'run $USER-x{}-full amd64 ci-full-no-test-cache' ;; - "merge-queue") - prep_vars - # Spin up ec2 instance and run the merge-queue flow. + merge-queue) + # We perform full runs of all tests on multiple x86, and a single fast run on arm64. + export CI_DASHBOARD=${TARGET_BRANCH:-local} + export DENOISE=1 + export DENOISE_WIDTH=32 run() { JOB_ID=$1 INSTANCE_POSTFIX=$1 ARCH=$2 exec denoise "bootstrap_ec2 './bootstrap.sh $3'" } export -f run - # We perform two full runs of all tests on x86, and a single fast run on arm64 (allowing use of test cache). parallel --jobs 10 --termseq 'TERM,10000' --tagstring '{= $_=~s/run (\w+).*/$1/; =}' --line-buffered --halt now,fail=1 ::: \ 'run x1-full amd64 ci-full-no-test-cache' \ 'run x2-full amd64 ci-full-no-test-cache' \ @@ -123,29 +111,33 @@ case "$cmd" in ########################################## # NETWORK DEPLOYMENTS WITH BENCHES/TESTS # ########################################## - "network-deploy") + network-deploy) # Args: [docker_image] # If docker_image is not provided, ci-network-deploy will build and push to aztecdev. + export CI_DASHBOARD="network" export JOB_ID="x-${2:?namespace is required}-network-deploy" export INSTANCE_POSTFIX="n-deploy" bootstrap_ec2 "./bootstrap.sh ci-network-deploy $*" ;; - "network-tests") + network-tests) # Args: + export CI_DASHBOARD="network" export JOB_ID="x-${2:?namespace is required}-network-tests" export AWS_SHUTDOWN_TIME=360 # 6 hours for network tests export INSTANCE_POSTFIX="n-tests" bootstrap_ec2 "./bootstrap.sh ci-network-tests $*" ;; - "network-bench") + network-bench) # Args: [docker_image] # If docker_image is not provided, ci-network-bench will build and push to aztecdev. + export CI_DASHBOARD="network" export JOB_ID="x-${2:?namespace is required}-network-bench" CPUS=16 export INSTANCE_POSTFIX="n-bench" bootstrap_ec2 "./bootstrap.sh ci-network-bench $*" ;; - "network-teardown") + network-teardown) # Args: + export CI_DASHBOARD="network" export JOB_ID="x-${2:?namespace is required}-network-teardown" export CPUS=4 export INSTANCE_POSTFIX="n-teardown" @@ -155,9 +147,11 @@ case "$cmd" in ############ # RELEASES # ############ - "release") - prep_vars + release) # Spin up ec2 instance and run the release flow. + export CI_DASHBOARD="releases" + export DENOISE=1 + export DENOISE_WIDTH=32 run() { JOB_ID=$1 INSTANCE_POSTFIX=$1 ARCH=$2 exec denoise "bootstrap_ec2 './bootstrap.sh ci-release'" } @@ -172,16 +166,16 @@ case "$cmd" in fi ;; - ####################################### - # VARIANTS ON INTERACTIVE CI SESSIONS # - ####################################### - "shell-new") + ################## + # SHELL SESSIONS # + ################## + shell-new) # Spin up ec2 instance, clone, and drop into shell. # False triggers the shell on fail. cmd="${1:-false}" exec bootstrap_ec2 "$cmd" ;; - "shell-container") + shell-container) # Drop into a shell in the current running build instance container. get_ip_for_instance [ -z "$ip" ] && echo "No instance found: $instance_name" && exit 1 @@ -189,54 +183,27 @@ case "$cmd" in ssh -tq -F $ci3/aws/build_instance_ssh_config ubuntu@$ip \ "docker start aztec_build &>/dev/null || true && docker exec -it --user aztec-dev aztec_build $@" ;; - "shell-host") + shell-host) # Drop into a shell in the current running build host. get_ip_for_instance [ -z "$ip" ] && echo "No instance found: $instance_name" && exit 1 ssh -t -F $ci3/aws/build_instance_ssh_config ubuntu@$ip ;; + kill) + existing_instance=$(aws ec2 describe-instances \ + --region us-east-2 \ + --filters "Name=tag:Name,Values=$instance_name" \ + --query "Reservations[].Instances[?State.Name!='terminated'].InstanceId[]" \ + --output text) + if [ -n "$existing_instance" ]; then + aws_terminate_instance $existing_instance + fi + ;; ################### - # TRIGGER ci3.yml # + # DISPLAYING LOGS # ################### - "run") - # Trigger a GA workflow for current branch PR and tail logs. - $0 trigger - $0 rlog - ;; - "trigger") - # Trigger workflow. - # We use this label trick because triggering the workflow direct doesn't associate with the PR. - pr_number=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number') - if [ -z "$pr_number" ]; then - echo "No pull request found for branch $BRANCH." - exit 1 - fi - gh pr edit "$pr_number" --remove-label "trigger-workflow" &> /dev/null - gh pr edit "$pr_number" --add-label "trigger-workflow" &> /dev/null - sleep 1 - gh pr edit "$pr_number" --remove-label "trigger-workflow" &> /dev/null - run_id=$(get_latest_run_id) - echo "In progress..." | redis_setexz $run_id 3600 - echo -e "Triggered CI for PR: $pr_number (ci rlog ${yellow}$run_id${reset})" - ;; - "rlog") - [ -z "${1:-}" ] && run_id=$(get_latest_run_id) || run_id=$1 - output=$(redis_getz $run_id) - if [ -z "$output" ] || [ "$output" == "In progress..." ]; then - # If we're in progress, tail live logs from launched instance. - exec $0 ilog - else - echo "$output" | $PAGER - fi - ;; - "ilog") - while ! tail_live_instance; do - echo "Waiting on instance with name: $instance_name" - sleep 10 - done - ;; - "dlog") + log|dlog) if [ "$CI_REDIS_AVAILABLE" -ne 1 ]; then echo "No redis available for log query." exit 1 @@ -245,17 +212,11 @@ case "$cmd" in [ ! -t 0 ] && pager=cat redis_getz $1 | $pager ;; - "kill") - existing_instance=$(aws ec2 describe-instances \ - --region us-east-2 \ - --filters "Name=tag:Name,Values=$instance_name" \ - --query "Reservations[].Instances[?State.Name!='terminated'].InstanceId[]" \ - --output text) - if [ -n "$existing_instance" ]; then - aws_terminate_instance $existing_instance - fi - ;; - "draft") + + ################# + # PR MANAGEMENT # + ################# + draft) pr_number=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number') if [ -n "$pr_number" ]; then gh pr ready "$pr_number" --undo @@ -264,7 +225,7 @@ case "$cmd" in echo "No pull request found for branch $BRANCH." fi ;; - "ready") + ready) pr_number=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number') if [ -n "$pr_number" ]; then gh pr ready "$pr_number" @@ -273,7 +234,7 @@ case "$cmd" in echo "No pull request found for branch $BRANCH." fi ;; - "pr-url") + pr-url) # Print the current PR associated with the branch. pr_url=$(gh pr list --head "$BRANCH" --limit 1 --json url -q '.[0].url') if [ -z "$pr_url" ]; then @@ -282,28 +243,16 @@ case "$cmd" in fi echo "$pr_url" ;; - "last-run-url") - # Print the URL of the last GA run for the current branch PR. - run_id=$(get_latest_run_id) - if [ -z "$run_id" ] || [ "$run_id" == "null" ]; then - echo "No recent GitHub Actions run found for branch '$BRANCH'." - exit 1 - fi - repo=$(gh repo view --json nameWithOwner -q .nameWithOwner) - echo "https://github.com/$repo/actions/runs/$run_id" - ;; - ################################### - # DOWNLOAD CI-UPLOADED BENCHMARKS # - ################################### - "gh-bench") - cache_download bench-$(git rev-parse HEAD^{tree}).tar.gz - ;; - "gh-deploy-bench") - cache_download deploy-bench-$(git rev-parse HEAD^{tree}).tar.gz + ######################## + # BENCHMARK PROCESSING # + ######################## + gh-bench|gh-deploy-bench|gh-spartan-bench) + cache_download ${cmd#gh-}-$(git rev-parse HEAD^{tree}).tar.gz ;; - "gh-spartan-bench") - cache_download spartan-bench-$(git rev-parse HEAD^{tree}).tar.gz + + help|"") + print_usage ;; *) echo "Unknown command: $cmd, see ./ci.sh help" diff --git a/ci3/bootstrap_ec2 b/ci3/bootstrap_ec2 index b96678a01912..e8c193ad31f0 100755 --- a/ci3/bootstrap_ec2 +++ b/ci3/bootstrap_ec2 @@ -134,7 +134,7 @@ container_script=$( source ci3/source_redis source ci3/source_cache ci_log_id=\$(log_ci_run) - export PARENT_LOG_URL=${PARENT_LOG_URL:-http://ci.aztec-labs.com/\$ci_log_id} + export PARENT_LOG_URL=${PARENT_LOG_URL:-} if [ -n "\$DOCKERHUB_PASSWORD" ]; then echo \$DOCKERHUB_PASSWORD | docker login -u \$DOCKERHUB_USERNAME --password-stdin @@ -276,6 +276,7 @@ function run { -e JOB_ID=${JOB_ID:-} \ -e REF_NAME=${REF_NAME:-} \ -e TARGET_BRANCH=${TARGET_BRANCH:-} \ + -e CI_DASHBOARD=${CI_DASHBOARD:-} \ -e PARENT_LOG_URL=${PARENT_LOG_URL:-} \ -e NO_CACHE=${NO_CACHE:-} \ -e NO_FAIL_FAST=${NO_FAIL_FAST:-} \ diff --git a/ci3/dashboard/chonk-breakdowns/README.md b/ci3/dashboard/chonk-breakdowns/README.md new file mode 100644 index 000000000000..a6ff27c1ffb3 --- /dev/null +++ b/ci3/dashboard/chonk-breakdowns/README.md @@ -0,0 +1,60 @@ +# Chonk Breakdowns - Barretenberg Benchmark Viewer + +This viewer displays hierarchical timing breakdowns for Barretenberg's Chonk (IVC) proving system. It shows where time is spent during proof generation, with drill-down capabilities to explore component performance. + +## What This Shows + +The breakdown viewer visualizes: +- **Hierarchical component timings** from Barretenberg's `--bench_out_hierarchical` flag +- **Time spent in each proving phase**: ChonkAccumulate, ChonkProve, OinkProver, etc. +- **Nested operations**: Click on components to drill down into child operations +- **Percentage distribution**: See which operations dominate proving time + +Data comes from CI benchmark runs and is stored in `/logs-disk/bench/bb-breakdown/`. + +## Local Testing + +### Quick Start + +1. **Install dependencies (first time only):** + ```bash + cd rkapp + python3 -m venv venv + venv/bin/pip install -r requirements.txt + ``` + +2. **Fetch sample test data:** + ```bash + mkdir -p /tmp/rkapp-test-data/bench/bb-breakdown + # Fetch all breakdowns for a specific SHA (e.g., f4decd6) + scp -i ~/.ssh/build_instance_key \ + "ubuntu@ci.aztec-labs.com:/logs-disk/bench/bb-breakdown/*-f4decd6*.log.gz" \ + /tmp/rkapp-test-data/bench/bb-breakdown/ + ``` + +3. **Start the server:** + ```bash + cd rkapp + ./chonk-breakdowns/run-local.sh + ``` + +4. **Test in browser:** + - Open: http://localhost:8080/chonk-breakdowns + - Enter credentials when prompted + - The SHA field will auto-populate with the latest commit from `aztec-packages` `next` branch + - Select Runtime: `native` or `wasm` + - The Flow dropdown will dynamically show flows available for the selected runtime and SHA + - Click "Load Breakdown" + +## What the Script Does + +- Sets `LOGS_DISK_PATH=/tmp/rkapp-test-data` to use local test data +- Sets `REDIS_TIMEOUT=0.1` for fast failover to disk +- Runs rk.py directly (no mocking needed) + +## Environment Variables + +- `LOGS_DISK_PATH` - Path to logs directory (default: `/logs-disk`) +- `REDIS_HOST` - Redis hostname (default for local: `localhost`) +- `REDIS_PORT` - Redis port (default for local: `9999` - non-existent) +- `REDIS_TIMEOUT` - Connection timeout in seconds (default: `0.1`) diff --git a/ci3/dashboard/chonk-breakdowns/breakdown-viewer.html b/ci3/dashboard/chonk-breakdowns/breakdown-viewer.html new file mode 100644 index 000000000000..31d88dce684d --- /dev/null +++ b/ci3/dashboard/chonk-breakdowns/breakdown-viewer.html @@ -0,0 +1,855 @@ + + + + + + Barretenberg Benchmark Breakdown Viewer + + + +
+

🔍 Barretenberg Benchmark Breakdown Viewer

+

Visualize hierarchical benchmark data showing time spent in each component

+ + +
+

Step 1: Enter Commit SHA

+
+ + +
+
+ +
+
+ + + + +
+ + + +
+
+
+ + + + + diff --git a/ci3/dashboard/chonk-breakdowns/run-local.sh b/ci3/dashboard/chonk-breakdowns/run-local.sh new file mode 100755 index 000000000000..ca5d3578512c --- /dev/null +++ b/ci3/dashboard/chonk-breakdowns/run-local.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Run rk.py locally for testing +# This script sets up the environment to use local test data + +set -e + +echo "============================================================" +echo "Starting rkapp locally for testing" +echo "============================================================" +echo "" + +# Check if venv exists +if [ ! -d "venv" ]; then + echo "Creating virtual environment..." + python3 -m venv venv + echo "Installing dependencies..." + venv/bin/pip install -q -r requirements.txt + echo "" +fi + +# Set environment variables for local testing +# Use a dummy Redis host to skip Redis and go straight to disk +export REDIS_HOST="${REDIS_HOST:-localhost}" +export REDIS_PORT="${REDIS_PORT:-9999}" +export REDIS_TIMEOUT="0.1" # Very short timeout to fail fast +# Point to /tmp for test data (not committed to git) +export LOGS_DISK_PATH="/tmp/rkapp-test-data" + +echo "Test data location: $LOGS_DISK_PATH" +echo "" + +# Check if test data exists +if [ ! -d "$LOGS_DISK_PATH/bench/bb-breakdown" ]; then + echo "⚠️ No test data found at: $LOGS_DISK_PATH/bench/bb-breakdown/" + echo " (Will fall back to Redis if available)" +else + echo "Available test data:" + for f in $LOGS_DISK_PATH/bench/bb-breakdown/*.log.gz; do + if [ -f "$f" ]; then + basename "$f" | sed -E 's/^([^-]+)-(.+)-([^-]+)\.log\.gz$/ Runtime: \1, Flow: \2, SHA: \3/' + fi + done +fi +echo "" + +echo "Starting Flask server..." +echo "Server will be available at: http://localhost:8080" +echo "Chonk breakdowns viewer: http://localhost:8080/chonk-breakdowns" +echo "" +echo "Note: You'll need Redis access or use test data that exists locally" +echo " Test data is in test-data/bench/bb-breakdown/" +echo "" +echo "Press Ctrl+C to stop" +echo "============================================================" +echo "" + +# Run with modified LOGS_DISK_PATH if we want to use test-data +venv/bin/python3 rk.py diff --git a/ci3/dashboard/requirements.txt b/ci3/dashboard/requirements.txt new file mode 100644 index 000000000000..9c1526f5b7a8 --- /dev/null +++ b/ci3/dashboard/requirements.txt @@ -0,0 +1,7 @@ +flask +gunicorn +redis +ansi2html +Flask-Compress +requests +Flask-HTTPAuth diff --git a/ci3/dashboard/rk.py b/ci3/dashboard/rk.py new file mode 100644 index 000000000000..11e6d84d1655 --- /dev/null +++ b/ci3/dashboard/rk.py @@ -0,0 +1,422 @@ +from flask import Flask, render_template_string, request, Response +from flask_compress import Compress +from flask_httpauth import HTTPBasicAuth +import gzip +import os +import re +import requests +import threading +from ansi2html import Ansi2HTMLConverter + +# Import core rendering logic +from rk_core import ( + YELLOW, BLUE, GREEN, RED, PURPLE, BOLD, RESET, + hyperlink, r, get_section_data, get_list_as_string +) + +LOGS_DISK_PATH = os.getenv('LOGS_DISK_PATH', '/logs-disk') +app = Flask(__name__) +Compress(app) +auth = HTTPBasicAuth() + +def read_from_disk(key): + """Read log from disk as fallback when Redis key not found.""" + try: + # Use first 4 chars as subdirectory + prefix = key[:4] + log_file = f"/logs-disk/{prefix}/{key}.log.gz" + log_file = f"{LOGS_DISK_PATH}/{prefix}/{key}.log.gz" + if os.path.exists(log_file): + with gzip.open(log_file, 'rb') as f: + return f.read().decode('utf-8', errors='replace') + except Exception as e: + print(f"Error reading from disk: {e}") + return None + +def read_breakdown_from_disk(runtime, flow_name, sha): + """Read benchmark breakdown JSON from disk.""" + try: + # Breakdown files are stored in {LOGS_DISK_PATH}/bench/bb-breakdown/ + # Format: --.log.gz + # SHA can be 7-40 chars (prefix or full) + breakdown_dir = f"{LOGS_DISK_PATH}/bench/bb-breakdown" + + # First try exact match + breakdown_file = f"{breakdown_dir}/{runtime}-{flow_name}-{sha}.log.gz" + if os.path.exists(breakdown_file): + with gzip.open(breakdown_file, 'rb') as f: + return f.read().decode('utf-8', errors='replace') + + # If not found, search for files starting with the SHA prefix + if os.path.exists(breakdown_dir): + prefix = f"{runtime}-{flow_name}-{sha}" + for filename in os.listdir(breakdown_dir): + if filename.startswith(prefix) and filename.endswith('.log.gz'): + breakdown_file = os.path.join(breakdown_dir, filename) + with gzip.open(breakdown_file, 'rb') as f: + return f.read().decode('utf-8', errors='replace') + except Exception as e: + print(f"Error reading breakdown from disk: {e}") + return None + +@auth.verify_password +def verify_password(username, password): + if username == "aztec" and password == "letmeseethoselogs": + return username + return None + +_github_status_cache = {"status": None, "ts": 0} +_github_status_lock = threading.Lock() + +def convert_to_ocs8(text): + # Replace URLs not already part of an OCS8 link using negative lookbehind. + pattern = r'(? None: + ga_status = get_github_actions_status() + return ( + f"{BOLD}{BLUE}{hyperlink('/', 'AZTEC LABS CI SYSTEM')}{RESET}: " + f"(offset: {offset}) (filter: {filter_str or 'unset'}) (filter_prop: {filter_prop or 'unset'} [status,name,author,msg]) ({ga_status})\n" + ) + +def get_github_actions_status(): + # Cache for 60 seconds + import time + now = time.time() + ga_url='https://githubstatus.com' + with _github_status_lock: + if _github_status_cache["status"] and now - _github_status_cache["ts"] < 60: + return _github_status_cache["status"] + try: + resp = requests.get("https://www.githubstatus.com/api/v2/components.json", timeout=2) + resp.raise_for_status() + data = resp.json() + for comp in data.get("components", []): + if comp.get("name", "").lower() == "actions": + status = comp.get("status", "") + if status in ("operational", "none"): + result = f"Github Actions: {GREEN}{hyperlink(ga_url, 'NOMINAL')}{RESET}" + else: + result = f"Github Actions: {RED}{hyperlink(ga_url, 'DEGRADED')}{RESET}" + _github_status_cache["status"] = result + _github_status_cache["ts"] = now + return result + except Exception: + result = f"Github Actions: {YELLOW}UNKNOWN{RESET}" + _github_status_cache["status"] = result + _github_status_cache["ts"] = now + return result + +def root() -> str: + # Show the default (no section) view with updated links + return ( + update_status(0, '', '') + + f"\n" + f"Select a filter:\n" + f"\n{YELLOW}" + f"{hyperlink('/section/master?fail_list=failed_tests_master', 'master queue')}\n" + f"{hyperlink('/section/staging?fail_list=failed_tests_staging', 'staging queue')}\n" + f"{hyperlink('/section/next?fail_list=failed_tests_next&limit=200', 'next queue')}\n" + f"{hyperlink('/section/prs', 'prs')}\n" + f"{hyperlink('/section/releases', 'releases')}\n" + f"{hyperlink('/section/nightly', 'nightly')}\n" + f"{hyperlink('/section/network', 'network')}\n" + f"{RESET}" + f"\n" + f"Benchmarks:\n" + f"\n{YELLOW}" + f"{hyperlink('https://aztecprotocol.github.io/aztec-packages/bench?branch=master', 'master')}\n" + f"{hyperlink('https://aztecprotocol.github.io/aztec-packages/bench?branch=staging', 'staging')}\n" + f"{hyperlink('https://aztecprotocol.github.io/aztec-packages/bench?branch=next', 'next')}\n" + f"{hyperlink('/chonk-breakdowns', 'crypto team chonk breakdowns')}\n" + f"{RESET}" + ) + +def section_view(section: str) -> str: + offset = int(request.args.get('offset', 0)) + limit = int(request.args.get('limit', 100)) + filter_str = request.args.get('filter', default='', type=str) + filter_prop = request.args.get('filter_prop', default='', type=str) + fail_list = request.args.get('fail_list', default='', type=str) + + lines = update_status(offset, filter_str, filter_prop) + lines += "\n" + lines += f"Last {limit} ci runs on {section}:\n\n" + lines += get_section_data(section, offset, limit, filter_str, filter_prop, fail_list) + return lines + +TEMPLATE = """ + + + + ACI · {{ filter_str|default('') }} + + + + +
{{ value|safe }}
+ + +""" + +@app.route('/') +@auth.login_required +def show_root(): + return render_template_string( + TEMPLATE, + value=ansi_to_html(root()), + filter_str='', + filter_prop='' + ) + +@app.route('/section/
') +@auth.login_required +def show_section(section): + return render_template_string( + TEMPLATE, + value=ansi_to_html(section_view(section)), + filter_str=request.args.get('filter', default='', type=str), + filter_prop=request.args.get('filter_prop', default='', type=str), + follow='top' + ) + +@app.route('/list/') +@auth.login_required +def get_list(key): + value = get_list_as_string(key) + follow = request.args.get('follow', 'top') + return render_template_string(TEMPLATE, value=ansi_to_html(value), follow=follow, filter_str='', filter_prop='') + +@app.route('/chonk-breakdowns') +@auth.login_required +def chonk_breakdowns(): + """Serve the chonk breakdowns viewer page.""" + breakdown_html_path = Path('chonk-breakdowns/breakdown-viewer.html') + if breakdown_html_path.exists(): + with breakdown_html_path.open('r') as f: + return f.read() + else: + return "Breakdown viewer not found", 404 + +@app.route('/api/breakdown/flows') +@auth.login_required +def list_available_flows(): + """API endpoint to list available breakdown flows from disk, filtered by runtime and SHA.""" + runtime = request.args.get('runtime') + sha = request.args.get('sha') + flows = set() + breakdown_dir = f"{LOGS_DISK_PATH}/bench/bb-breakdown" + + try: + if os.path.exists(breakdown_dir): + for filename in os.listdir(breakdown_dir): + if filename.endswith('.log.gz'): + # Parse: runtime-flow_name-sha.log.gz + parts = filename.replace('.log.gz', '').split('-', 1) + if len(parts) == 2: + file_runtime = parts[0] + rest = parts[1] + # Split from end to get SHA (7-40 chars) + last_dash = rest.rfind('-') + if last_dash != -1: + flow_name = rest[:last_dash] + file_sha = rest[last_dash + 1:] + + # Filter by runtime and SHA if provided + if runtime and file_runtime != runtime: + continue + if sha and not file_sha.startswith(sha): + continue + + flows.add(flow_name) + except Exception as e: + print(f"Error listing flows: {e}") + + return Response(json.dumps(sorted(list(flows))), mimetype='application/json') + +@app.route('/api/breakdown///') +@auth.login_required +def get_breakdown(runtime, flow_name, sha): + """API endpoint to fetch breakdown JSON from disk.""" + breakdown_data = read_breakdown_from_disk(runtime, flow_name, sha) + if breakdown_data: + return Response(breakdown_data, mimetype='application/json') + + return Response('{"error": "Breakdown not found"}', mimetype='application/json', status=404) + + +@app.route('/') +@auth.login_required +def get_value(key): + # Check if raw text format is requested + raw_text = key.endswith('.txt') + if raw_text: + key = key[:-4] # Remove .txt extension + + value = r.get(key) + if value is None: + # Try disk fallback + value = read_from_disk(key) + if value is None: + value = "Key not found" + else: + try: + if value.startswith(b"\x1f\x8b"): + value = gzip.decompress(value).decode() + else: + value = value.decode() + except Exception: + value = "Failed to decompress" + + # Return raw text if .txt extension was used + if raw_text: + return Response(value, mimetype='text/plain') + + return render_template_string(TEMPLATE, value=ansi_to_html(value), filter_str='', filter_prop='') + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=8080) diff --git a/ci3/dashboard/rk_cli.py b/ci3/dashboard/rk_cli.py new file mode 100644 index 000000000000..87e73a6bbefd --- /dev/null +++ b/ci3/dashboard/rk_cli.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Lightweight CLI for CI run viewer - minimal dependencies for fast startup.""" +import argparse +from rk_core import get_section_data, set_base_url + +def main(): + parser = argparse.ArgumentParser(description='CI Run Viewer (CLI)') + parser.add_argument('--section', '-s', type=str, required=True, + help='Section to display (master, staging, next, prs, releases)') + parser.add_argument('--offset', type=int, default=0, help='Offset for pagination') + parser.add_argument('--limit', '-l', type=int, default=200, help='Number of results to fetch') + parser.add_argument('--filter', '-f', dest='filter_str', type=str, default='', + help='Filter pattern (comma-separated)') + parser.add_argument('--filter-prop', '-p', type=str, default='', + help='Property to filter on (status,name,author,msg)') + parser.add_argument('--fail-list', type=str, default='', help='Redis key for failed tests list') + + args = parser.parse_args() + + set_base_url("http://ci.aztec-labs.com") + output = get_section_data(args.section, args.offset, args.limit, + args.filter_str, args.filter_prop, args.fail_list) + print(output, end='') + +if __name__ == '__main__': + main() diff --git a/ci3/dashboard/rk_core.py b/ci3/dashboard/rk_core.py new file mode 100644 index 000000000000..1f4043187c7e --- /dev/null +++ b/ci3/dashboard/rk_core.py @@ -0,0 +1,120 @@ +"""Core CI rendering logic - minimal dependencies for fast CLI startup.""" +import redis +import re +import os +import json +import time +from datetime import datetime +from pathlib import Path + +# Color definitions as constants. +YELLOW = "\033[38;2;250;217;121m" +BLUE = "\033[38;2;95;167;241m" +GREEN = "\033[38;2;97;214;104m" +RED = "\033[38;2;230;69;83m" +PURPLE = "\033[38;2;188;109;208m" +BOLD = "\033[1m" +RESET = "\033[0m" +LINK_OPEN = '\033]8;;' +LINK_CLOSE = '\x07' +CLEAR_EOL = "\033[K" +BASE_URL = "" # Set to full URL in CLI mode + +def set_base_url(url: str): + global BASE_URL + BASE_URL = url + +def hyperlink(link, text): + return f"{LINK_OPEN}{link}{LINK_CLOSE}{text}{LINK_OPEN}{LINK_CLOSE}" + +REDIS_HOST = os.getenv('REDIS_HOST', 'localhost') +REDIS_PORT = int(os.getenv('REDIS_PORT', '6379')) +r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=False) + +def get_list_as_string(key, limit=None): + try: + if limit is None: + values = r.lrange(key, 0, -1) + else: + values = r.lrange(key, 0, limit - 1) + if not values: + value = "List is empty or key not found" + else: + concatenated = [] + for item in values: + concatenated.append(item.decode()) + value = "\n".join(concatenated) + except Exception as e: + value = f"Error retrieving list: {str(e)}" + return value + +def render(group: list) -> str: + # Use the first JSON object for common properties. + data = group[0] + ts = data['timestamp'] + msg = data['msg'] + name, author, complete = data['name'], data['author'], data.get('complete') + + # Generate comma-delimited job links with color reflecting each job's status. + valid_jobs = [job for job in group if 'job_id' in job] + job_links = [] + for job in sorted(valid_jobs, key=lambda j: j['job_id']): + job_status = job.get('status', '') + if job_status == 'RUNNING': + link_color = BLUE + elif job_status == 'FAILED': + link_color = RED + elif job_status == 'PASSED': + link_color = GREEN + else: + link_color = RESET + job_links.append(f"{link_color}{hyperlink(BASE_URL + '/' + str(job['timestamp']), job['job_id'])}{RESET}") + links_str = ",".join(job_links) + date_time = datetime.fromtimestamp(ts // 1000).strftime("%m-%d %H:%M:%S") + + statuses = [job.get('status') for job in group if 'status' in job] + if statuses and all(s == 'INACTIVE' for s in statuses): + return f"{date_time}: {links_str} {BOLD}{name}{RESET} {author}: {msg}{CLEAR_EOL}\n" + + current_time = int(time.time() * 1000) + durations = [] + for job in group: + # ignore inactive jobs in duration calculation + if job.get('status') == 'INACTIVE': + continue + if 'timestamp' in job: + job_start = job['timestamp'] + job_complete = job.get('complete') + job_end = job_complete if job_complete is not None else current_time + durations.append(job_end - job_start) + max_duration = max(durations) if durations else 0 + duration = max_duration // 1000 + duration_str = f"({duration // 60}m{duration % 60}s)" + return f"{date_time}: {links_str} {BOLD}{name}{RESET} {PURPLE}{author}{RESET}: {msg} {duration_str}{CLEAR_EOL}\n" + +def get_section_data(section: str, offset: int = 0, limit: int = 100, + filter_str: str = '', filter_prop: str = '', + fail_list: str = '') -> str: + """Core logic for fetching and rendering section data.""" + lua_script_path = Path(__file__).parent / 'set-filter.lua' + with lua_script_path.open('r') as f: + result = r.eval(f.read(), 1, 'ci-run-' + section, offset, limit, filter_prop, filter_str) + + lines = "" + if not result: + lines += "Nothing to see here...\n" + else: + groups = {} + for line in result: + item = json.loads(line) + run = item.get('run_id', 'unknown') + groups.setdefault(run, []).append(item) + for group in groups.values(): + group_sorted = sorted(group, key=lambda x: x.get('ts', x.get('timestamp', 0))) + lines += render(group_sorted) + + if fail_list: + lines += "\n" + lines += f"Last 100 failed or flaked tests:\n\n" + lines += get_list_as_string(fail_list, 100) + return lines diff --git a/ci3/dashboard/set-filter.lua b/ci3/dashboard/set-filter.lua new file mode 100644 index 000000000000..a7aec5d329d4 --- /dev/null +++ b/ci3/dashboard/set-filter.lua @@ -0,0 +1,54 @@ +local key = KEYS[1] +local offset = tonumber(ARGV[1]) +local limit = tonumber(ARGV[2]) +local property = ARGV[3] +local pattern = ARGV[4] +local result = {} +local skip = 0 +local seen_runs = {} +local run_count = 0 + +local patterns = {} +for pat in string.gmatch(pattern, '([^,]+)') do + table.insert(patterns, pat) +end + +local function matches_any(str, patterns) + for _, pat in ipairs(patterns) do + if string.find(str, pat) then return true end + end + return false +end + +local allMembers = redis.call('ZREVRANGE', key, 0, -1) +for _, member in ipairs(allMembers) do + local ok, parsed = pcall(cjson.decode, member) + if ok then + if parsed["status"] == "RUNNING" then + local ts = parsed["timestamp"] + local hb_key = "hb-" .. tostring(ts) + if redis.call("EXISTS", hb_key) == 0 then + parsed["status"] = "INACTIVE" + end + end + local include=true + if property and pattern and parsed[property] and not matches_any(parsed[property], patterns) then + include=false + end + if include then + if skip >= offset then + local run_id = parsed["run_id"] or "unknown" + if not seen_runs[run_id] then + seen_runs[run_id] = true + run_count = run_count + 1 + if run_count > limit then break end + end + table.insert(result, cjson.encode(parsed)) + else + skip = skip + 1 + end + end + end +end + +return result diff --git a/ci3/log_ci_run b/ci3/log_ci_run index 32121149249a..5c9567ae91dd 100755 --- a/ci3/log_ci_run +++ b/ci3/log_ci_run @@ -7,16 +7,26 @@ source $ci3/source_color status=${1:-RUNNING} key=${2:-} -# CI runs are grouped by: -# - The 'release' group if tagged with a semver. -# - Or by the target branch of the merge queue they enter. -# - Or in the 'prs' group. -if semver check "$REF_NAME"; then - range_key="ci-run-releases" -elif [[ "$REF_NAME" =~ ^gh-readonly-queue/ ]]; then - range_key="ci-run-$TARGET_BRANCH" +if [ -z "${RUN_ID:-}" ]; then + echo_stderr "No RUN_ID set in log_ci_run. This ci run won't be seen on the dashboards." + exit +fi + +if [ -z "$CI_DASHBOARD:-}" ]; then + # This path is deprecated. Set CI_DASHBOARD explicitly in ci.sh. + # CI runs are grouped by: + # - The 'release' group if tagged with a semver. + # - Or by the target branch of the merge queue they enter. + # - Or in the 'prs' group. + if semver check "$REF_NAME"; then + range_key="ci-run-releases" + elif [[ "$REF_NAME" =~ ^gh-readonly-queue/ ]]; then + range_key="ci-run-$TARGET_BRANCH" + else + range_key="ci-run-prs" + fi else - range_key="ci-run-prs" + range_key="ci-run-$CI_DASHBOARD" fi if [ -z "$key" ]; then diff --git a/ci3/watch_ci b/ci3/watch_ci index 9bfdc7b06425..418ebbd43411 100755 --- a/ci3/watch_ci +++ b/ci3/watch_ci @@ -1,128 +1,99 @@ #!/usr/bin/env bash source $(git rev-parse --show-toplevel)/ci3/source -source $ci3/source_redis +source $ci3/source_color +source $HOME/py-env/bin/activate -filter_property=${1:-} -filter_string=${2:-} +# Parse arguments +sections="next" +filter_property="" +filter_string="" +use_pr=false +use_user=false +watch_mode=false -function get_tag { - case "$1" in - RUNNING) echo -e " ${blue}$1${reset}" ;; - PASSED) echo -e " ${green}$1${reset}" ;; - FAILED) echo -e " ${red}$1${reset}" ;; - INACTIVE) echo -e "${bold}$1${reset}" ;; +while [[ $# -gt 0 ]]; do + case $1 in + --pr) use_pr=true; shift ;; + --user) use_user=true; shift ;; + --watch|-w) watch_mode=true; shift ;; + --section|-s) sections="$2"; shift 2 ;; + --filter-prop|-p) filter_property="$2"; shift 2 ;; + --filter|-f) filter_string="$2"; shift 2 ;; + *) sections="$1"; shift ;; # Backwards compat: first positional arg is sections esac -} +done -function render { - while IFS= read -r json_line; do - readarray -t arr < <(jq -r '[.timestamp, .status, .msg, .name, .author, .complete // "", .arch // ""][]' <<< "$json_line") - ts="${arr[0]}" - status="${arr[1]}" - msg="${arr[2]}" - name="${arr[3]}" - author="${arr[4]}" - complete="${arr[5]}" - arch="${arr[6]}" - local date_time=$(date -d @${ts:0:10} "+%m-%d %H:%M:%S") - local link="${link_open}http://ci.aztec-labs.com/$ts${link_close}$ts${link_open}${link_close}" - if [ -z "$complete" ]; then - local from=$(date +%s%3N) - else - local from=complete - fi +# Split sections into array +IFS=',' read -ra section_array <<< "$sections" +num_sections=${#section_array[@]} - if [[ $msg =~ \(#([0-9]+)\) ]]; then - pr_number="${BASH_REMATCH[1]}" - msg="${link_open}https://github.com/aztecprotocol/aztec-packages/pull/$pr_number${link_close}$msg${link_open}${link_close}" - fi +DASHBOARD_PATH="$ci3/dashboard" - if [ "$status" != "INACTIVE" ]; then - local duration=$(( (from - ts) / 1000 )) - local duration_str="($((duration / 60))m$((duration % 60))s)" - echo -en "\n$date_time $(get_tag $status) (${yellow}$link${reset}): $arch ${bold}$name${reset} ${purple}$author${reset}: $msg ${duration_str} \e[K" - else - echo -en "\n$date_time $(get_tag $status) ($link): $arch ${bold}$name${reset} $author: $msg \e[K" +# Handle --pr flag: get PR URL for current branch +if [ "$use_pr" = true ]; then + branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null) + if [ -n "$branch" ]; then + pr_url=$(gh pr list --head "$branch" --limit 1 --json url -q '.[0].url' 2>/dev/null) + if [ -n "$pr_url" ]; then + filter_string="${filter_string:+$filter_string,}$pr_url" + filter_property="${filter_property:-msg}" fi - done -} + fi +fi -SECONDS=5 +# Handle --user flag: get git user name +if [ "$use_user" = true ]; then + user=$(git config user.name 2>/dev/null) + if [ -n "$user" ]; then + filter_string="${filter_string:+$filter_string,}$user" + filter_property="${filter_property:-author}" + fi +fi function update_status { - # Jump to top left and clear. - echo -en "\e[0;0H\e[K" - echo -en "${bold}${blue}AZTEC LABS TERMINAL CI SYSTEM${reset}:" \ - "(offset: $offset) (filter: ${filter_string:-unset}) (filter_prop: ${filter_property:-unset} [status,name,author,msg])" + echo -en "\e[0;0H\e[K\n\e[K\e[0;0H" + echo -e "${bold}${blue}AZTEC LABS TERMINAL CI SYSTEM${reset}: " \ + "(sections: $sections) (filter: ${filter_string:-unset}) (filter_prop: ${filter_property:-unset} [status,name,author,msg])" + echo } function refresh { height=$(tput lines) - width=$(tput cols) - if [ "$SECONDS" -ge 5 ]; then - result=$(redis_cli --eval $ci3/lua/set-filter.lua ci-run , \ - $offset $((offset + height - 1)) $filter_property $filter_string) - SECONDS=0 - fi - update_status - if [ -z "$result" ]; then - echo -n "\nNothing to see here..." - else - echo -e "$result" | head -n $((height - 1)) | render - fi - # Clear remainder. - echo -e -n "\e[J" -} -function cleanup { - tput rmcup - tput cnorm - echo -en '\e[?7h' - stty echo - exit -} + cmd="python3 $DASHBOARD_PATH/rk_cli.py --section {}" + [ -n "$filter_property" ] && cmd+=" --filter-prop $filter_property" + [ -n "$filter_string" ] && cmd+=" --filter '$filter_string'" -tput smcup -# Hide cursor -tput civis -# No wrap -echo -en '\e[?7l' -# No echo keypress -stty -echo -trap cleanup SIGINT EXIT + parallel -j${#section_array[@]} -k $cmd ::: "${section_array[@]}" | sort -r | head -n $((height - 2)) + echo -e -n "\e[J" +} -offset=0 +if [ "$watch_mode" = true ]; then + # Interactive watch mode with auto-refresh + tput smcup # Save screen + tput civis # Hide cursor + echo -en '\e[?7l' # No wrap + stty -echo # No echo keypress + trap 'tput rmcup; tput cnorm; echo -en "\e[?7h"; stty echo; exit' SIGINT EXIT + trap 'SECONDS=5' WINCH # Refresh on terminal resize -function offset_up { - offset=$(( offset + 1 )) SECONDS=5 - update_status -} -function offset_down { - offset=$(( offset > 0 ? offset - 1 : 0 )) - SECONDS=5 - update_status -} + while true; do + if [ "$SECONDS" -ge 5 ]; then + update_status + refresh + SECONDS=0 + fi -refresh -while true; do - # Read with a timeout. - if read -rsn1 -t 1 key; then - # Check for escape sequence start. - case "$key" in - $'\e') - read -rsn2 -t 1 key2 - case "$key2" in - "[A") offset_up ;; - "[B") offset_down ;; - esac - ;; - j) offset_up ;; - k) offset_down ;; - q) break ;; - esac - continue - fi + # Read with timeout, q to quit + if read -rsn1 -t 0.2 key; then + case "$key" in + q) break ;; + esac + fi + done +else + # One-shot mode refresh -done +fi diff --git a/noir/precommit.sh b/noir/precommit.sh index b9b3715fdf1b..ededb9b9b631 100755 --- a/noir/precommit.sh +++ b/noir/precommit.sh @@ -7,16 +7,21 @@ if git diff --cached --submodule=short noir/noir-repo | grep -q "^-Subproject co echo "⚠️ WARNING: You are about to change the noir/noir-repo submodule hash" echo "" - # Ask for confirmation - read -p "Do you really want to commit this submodule change? (y/N): " -n 1 -r - echo "" - - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "Commit aborted." - echo "" - echo "Maybe you want to re-pull submodules with:" - echo " git submodule update --init --recursive" + if [ -t 0 ]; then + # Ask for confirmation + read -p "Do you really want to commit this submodule change? (y/N): " -n 1 -r echo "" + + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Commit aborted." + echo "" + echo "Maybe you want to re-pull submodules with:" + echo " git submodule update --init --recursive" + echo "" + exit 1 + fi + else + echo "You can git commit --no-verify if you're sure." exit 1 fi fi