Skip to content

fix: CI Updates #18987

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AztecBot merged 1 commit into from
Jan 5, 2026
Merged

fix: CI Updates #18987

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,5 @@ docs/docs/protocol-specs/public-vm/gen/

# for those who use Claude Code
/.claude

__pycache__
4 changes: 2 additions & 2 deletions bootstrap.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,10 +256,10 @@ function build_and_test {
echo_header "build and test"

# Start the test engine.
# setsid will put it in it's own process group we can terminate on cleanup.
rm -f $test_cmds_file
touch $test_cmds_file
setsid color_prefix "test-engine" "denoise test_engine_start" &
# put it in it's own process group via background subshell, we can terminate on cleanup.
(color_prefix "test-engine" "denoise test_engine_start") &
test_engine_pid=$!
test_engine_pgid=$(ps -o pgid= -p $test_engine_pid)

Expand Down
205 changes: 77 additions & 128 deletions ci.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,13 @@ ci3_workflow_id=128853861
function echo_cmd {
local name=$1
shift
printf "${blue}${bold}%12s${reset}: %s\n" $name "$(echo $@ | sed 's/\.\\n/.\n /g')"
printf "${blue}${bold}%21s${reset}: %s\n" $name "$(echo $@ | sed 's/\.\\n/.\n /g')"
}

function print_usage {
echo "usage: $(basename $0) <cmd>"
echo
echo_cmd "dash" "Display a dashboard showing CI runs for the current user."
echo_cmd "fast" "Spin up an EC2 instance and run bootstrap ci-fast."
echo_cmd "full" "Spin up an EC2 instance and run bootstrap ci-full."
echo_cmd "full-no-test-cache" "Spin up an EC2 instance and run bootstrap ci-full-no-test-cache."
Expand All @@ -31,25 +32,16 @@ function print_usage {
echo_cmd "network-teardown" "Spin up an EC2 instance to teardown a network deployment."
echo_cmd "release" "Spin up an EC2 instance and run bootstrap release."
echo_cmd "shell-new" "Spin up an EC2 instance, clone the repo, and drop into a shell."
echo_cmd "shell-container" "Drop into a shell in the current running build instance container."
echo_cmd "shell" "Drop into a shell in the current running build instance container."
echo_cmd "shell-host" "Drop into a shell in the current running build host."
echo_cmd "run" "Trigger a GA workflow for the current branch PR and tail logs."
echo_cmd "trigger" "Trigger the GA workflow on the PR associated with the current branch."
echo_cmd "rlog" "Tail the logs of the latest GA run or the given GA run ID."
echo_cmd "ilog" "Tail the logs of the current running build instance."
echo_cmd "dlog" "Display the log of the given denoise log ID."
echo_cmd "kill" "Terminate the EC2 instance for the current branch."
echo_cmd "log" "Display the log of the given log ID."
echo_cmd "kill" "Terminate running EC2 instance with instance_name."
echo_cmd "draft" "Mark the current PR as draft (no automatic CI runs when pushing)."
echo_cmd "ready" "Mark the current PR as ready (enable automatic CI runs when pushing)."
echo_cmd "pr-url" "Print the URL of the current PR associated with the branch."
echo_cmd "last-run-url" "Print the URL of the last GA run for the current branch PR."
echo_cmd "gh-bench" "Download CI-uploaded benchmarks for the current commit."
echo_cmd "gh-deploy-bench" "Download CI-uploaded deployment benchmarks for the current commit."
echo_cmd "gh-spartan-bench" "Download CI-uploaded spartan benchmarks for the current commit."
echo_cmd "avm-inputs-collection" "Nightly: run e2e tests, dump AVM circuit inputs, upload to cache."
echo_cmd "avm-check-circuit" "Nightly: download cached AVM inputs, run check-circuit on each."
echo_cmd "avm-inputs-collection" "Run e2e tests, dump AVM circuit inputs, upload to cache."
echo_cmd "avm-check-circuit" "Download cached AVM inputs, run check-circuit on each."
echo_cmd "help" "Display this help message."

}

[ -n "$cmd" ] && shift
Expand All @@ -69,49 +61,45 @@ function get_latest_run_id {
gh run list --workflow $ci3_workflow_id -b $BRANCH --limit 1 --json databaseId -q .[0].databaseId
}

function tail_live_instance {
get_ip_for_instance
[ -z "$ip" ] && return 1;
ssh -F $ci3/aws/build_instance_ssh_config -q -t -o ConnectTimeout=5 ubuntu@$ip "
trap 'exit 0' SIGINT
docker ps -a --filter name=aztec_build --format '{{.Names}}' | grep -q '^aztec_build$' || exit 1
docker logs -f aztec_build
"
}

# Used in merge-queue, nightly, and release flows.
function prep_vars {
export RUN_ID=${RUN_ID:-$(date +%s%3N)}
export PARENT_LOG_URL=http://ci.aztec-labs.com/$RUN_ID
export DENOISE=1
export DENOISE_WIDTH=32
}
# Jobs in the ci dashboards are grouped on a single line by RUN_ID.
export RUN_ID=${RUN_ID:-$(date +%s%3N)}
export PARENT_LOG_URL=http://ci.aztec-labs.com/$RUN_ID

case "$cmd" in
"help"|"")
print_usage
dash)
watch_ci -s next,prs --user --watch
;;
fast|full|full-no-test-cache|full-no-test-cache-makefile|docs|barretenberg|barretenberg-full|avm-inputs-collection|avm-check-circuit)
export JOB_ID="x1-$cmd"
fast|full|full-no-test-cache|full-no-test-cache-makefile|docs|barretenberg|barretenberg-full)
export CI_DASHBOARD="prs"
export JOB_ID="x-$cmd"
bootstrap_ec2 "./bootstrap.sh ci-$cmd"
;;
"grind")
prep_vars
# Spin up ec2 instance and run the merge-queue flow.
avm-inputs-collection|avm-check-circuit)
export CI_DASHBOARD="nightly"
export JOB_ID="x-$cmd"
bootstrap_ec2 "./bootstrap.sh ci-$cmd"
;;
grind)
# Grind a default of 5 times.
export CI_DASHBOARD="local"
export DENOISE=1
export DENOISE_WIDTH=32
run() {
JOB_ID=$1 INSTANCE_POSTFIX=$1 ARCH=$2 exec denoise "bootstrap_ec2 './bootstrap.sh $3'"
}
export -f run
seq 1 ${1:-5} | parallel --termseq 'TERM,10000' --tagstring '{= $_=~s/run (\w+).*/$1/; =}' --line-buffered 'run $USER-x{}-full amd64 ci-full-no-test-cache'
seq 1 ${1:-5} | parallel --termseq 'TERM,10000' --tagstring '{= $_=~s/run (\w+).*/$1/; =}' --line-buffered \
'run $USER-x{}-full amd64 ci-full-no-test-cache'
;;
"merge-queue")
prep_vars
# Spin up ec2 instance and run the merge-queue flow.
merge-queue)
# We perform full runs of all tests on multiple x86, and a single fast run on arm64.
export CI_DASHBOARD=${TARGET_BRANCH:-local}
export DENOISE=1
export DENOISE_WIDTH=32
run() {
JOB_ID=$1 INSTANCE_POSTFIX=$1 ARCH=$2 exec denoise "bootstrap_ec2 './bootstrap.sh $3'"
}
export -f run
# We perform two full runs of all tests on x86, and a single fast run on arm64 (allowing use of test cache).
parallel --jobs 10 --termseq 'TERM,10000' --tagstring '{= $_=~s/run (\w+).*/$1/; =}' --line-buffered --halt now,fail=1 ::: \
'run x1-full amd64 ci-full-no-test-cache' \
'run x2-full amd64 ci-full-no-test-cache' \
Expand All @@ -123,29 +111,33 @@ case "$cmd" in
##########################################
# NETWORK DEPLOYMENTS WITH BENCHES/TESTS #
##########################################
"network-deploy")
network-deploy)
# Args: <scenario> <namespace> [docker_image]
# If docker_image is not provided, ci-network-deploy will build and push to aztecdev.
export CI_DASHBOARD="network"
export JOB_ID="x-${2:?namespace is required}-network-deploy"
export INSTANCE_POSTFIX="n-deploy"
bootstrap_ec2 "./bootstrap.sh ci-network-deploy $*"
;;
"network-tests")
network-tests)
# Args: <scenario> <namespace>
export CI_DASHBOARD="network"
export JOB_ID="x-${2:?namespace is required}-network-tests"
export AWS_SHUTDOWN_TIME=360 # 6 hours for network tests
export INSTANCE_POSTFIX="n-tests"
bootstrap_ec2 "./bootstrap.sh ci-network-tests $*"
;;
"network-bench")
network-bench)
# Args: <scenario> <namespace> [docker_image]
# If docker_image is not provided, ci-network-bench will build and push to aztecdev.
export CI_DASHBOARD="network"
export JOB_ID="x-${2:?namespace is required}-network-bench" CPUS=16
export INSTANCE_POSTFIX="n-bench"
bootstrap_ec2 "./bootstrap.sh ci-network-bench $*"
;;
"network-teardown")
network-teardown)
# Args: <scenario> <namespace>
export CI_DASHBOARD="network"
export JOB_ID="x-${2:?namespace is required}-network-teardown"
export CPUS=4
export INSTANCE_POSTFIX="n-teardown"
Expand All @@ -155,9 +147,11 @@ case "$cmd" in
############
# RELEASES #
############
"release")
prep_vars
release)
# Spin up ec2 instance and run the release flow.
export CI_DASHBOARD="releases"
export DENOISE=1
export DENOISE_WIDTH=32
run() {
JOB_ID=$1 INSTANCE_POSTFIX=$1 ARCH=$2 exec denoise "bootstrap_ec2 './bootstrap.sh ci-release'"
}
Expand All @@ -172,71 +166,44 @@ case "$cmd" in
fi
;;

#######################################
# VARIANTS ON INTERACTIVE CI SESSIONS #
#######################################
"shell-new")
##################
# SHELL SESSIONS #
##################
shell-new)
# Spin up ec2 instance, clone, and drop into shell.
# False triggers the shell on fail.
cmd="${1:-false}"
exec bootstrap_ec2 "$cmd"
;;
"shell-container")
shell-container)
# Drop into a shell in the current running build instance container.
get_ip_for_instance
[ -z "$ip" ] && echo "No instance found: $instance_name" && exit 1
[ "$#" -eq 0 ] && set -- "zsh" || true
ssh -tq -F $ci3/aws/build_instance_ssh_config ubuntu@$ip \
"docker start aztec_build &>/dev/null || true && docker exec -it --user aztec-dev aztec_build $@"
;;
"shell-host")
shell-host)
# Drop into a shell in the current running build host.
get_ip_for_instance
[ -z "$ip" ] && echo "No instance found: $instance_name" && exit 1
ssh -t -F $ci3/aws/build_instance_ssh_config ubuntu@$ip
;;
kill)
existing_instance=$(aws ec2 describe-instances \
--region us-east-2 \
--filters "Name=tag:Name,Values=$instance_name" \
--query "Reservations[].Instances[?State.Name!='terminated'].InstanceId[]" \
--output text)
if [ -n "$existing_instance" ]; then
aws_terminate_instance $existing_instance
fi
;;

###################
# TRIGGER ci3.yml #
# DISPLAYING LOGS #
###################
"run")
# Trigger a GA workflow for current branch PR and tail logs.
$0 trigger
$0 rlog
;;
"trigger")
# Trigger workflow.
# We use this label trick because triggering the workflow direct doesn't associate with the PR.
pr_number=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number')
if [ -z "$pr_number" ]; then
echo "No pull request found for branch $BRANCH."
exit 1
fi
gh pr edit "$pr_number" --remove-label "trigger-workflow" &> /dev/null
gh pr edit "$pr_number" --add-label "trigger-workflow" &> /dev/null
sleep 1
gh pr edit "$pr_number" --remove-label "trigger-workflow" &> /dev/null
run_id=$(get_latest_run_id)
echo "In progress..." | redis_setexz $run_id 3600
echo -e "Triggered CI for PR: $pr_number (ci rlog ${yellow}$run_id${reset})"
;;
"rlog")
[ -z "${1:-}" ] && run_id=$(get_latest_run_id) || run_id=$1
output=$(redis_getz $run_id)
if [ -z "$output" ] || [ "$output" == "In progress..." ]; then
# If we're in progress, tail live logs from launched instance.
exec $0 ilog
else
echo "$output" | $PAGER
fi
;;
"ilog")
while ! tail_live_instance; do
echo "Waiting on instance with name: $instance_name"
sleep 10
done
;;
"dlog")
log|dlog)
if [ "$CI_REDIS_AVAILABLE" -ne 1 ]; then
echo "No redis available for log query."
exit 1
Expand All @@ -245,17 +212,11 @@ case "$cmd" in
[ ! -t 0 ] && pager=cat
redis_getz $1 | $pager
;;
"kill")
existing_instance=$(aws ec2 describe-instances \
--region us-east-2 \
--filters "Name=tag:Name,Values=$instance_name" \
--query "Reservations[].Instances[?State.Name!='terminated'].InstanceId[]" \
--output text)
if [ -n "$existing_instance" ]; then
aws_terminate_instance $existing_instance
fi
;;
"draft")

#################
# PR MANAGEMENT #
#################
draft)
pr_number=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number')
if [ -n "$pr_number" ]; then
gh pr ready "$pr_number" --undo
Expand All @@ -264,7 +225,7 @@ case "$cmd" in
echo "No pull request found for branch $BRANCH."
fi
;;
"ready")
ready)
pr_number=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number')
if [ -n "$pr_number" ]; then
gh pr ready "$pr_number"
Expand All @@ -273,7 +234,7 @@ case "$cmd" in
echo "No pull request found for branch $BRANCH."
fi
;;
"pr-url")
pr-url)
# Print the current PR associated with the branch.
pr_url=$(gh pr list --head "$BRANCH" --limit 1 --json url -q '.[0].url')
if [ -z "$pr_url" ]; then
Expand All @@ -282,28 +243,16 @@ case "$cmd" in
fi
echo "$pr_url"
;;
"last-run-url")
# Print the URL of the last GA run for the current branch PR.
run_id=$(get_latest_run_id)
if [ -z "$run_id" ] || [ "$run_id" == "null" ]; then
echo "No recent GitHub Actions run found for branch '$BRANCH'."
exit 1
fi
repo=$(gh repo view --json nameWithOwner -q .nameWithOwner)
echo "https://github.com/$repo/actions/runs/$run_id"
;;

###################################
# DOWNLOAD CI-UPLOADED BENCHMARKS #
###################################
"gh-bench")
cache_download bench-$(git rev-parse HEAD^{tree}).tar.gz
;;
"gh-deploy-bench")
cache_download deploy-bench-$(git rev-parse HEAD^{tree}).tar.gz
########################
# BENCHMARK PROCESSING #
########################
gh-bench|gh-deploy-bench|gh-spartan-bench)
cache_download ${cmd#gh-}-$(git rev-parse HEAD^{tree}).tar.gz
;;
"gh-spartan-bench")
cache_download spartan-bench-$(git rev-parse HEAD^{tree}).tar.gz

help|"")
print_usage
;;
*)
echo "Unknown command: $cmd, see ./ci.sh help"
Expand Down
3 changes: 2 additions & 1 deletion ci3/bootstrap_ec2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ container_script=$(
source ci3/source_redis
source ci3/source_cache
ci_log_id=\$(log_ci_run)
export PARENT_LOG_URL=${PARENT_LOG_URL:-http://ci.aztec-labs.com/\$ci_log_id}
export PARENT_LOG_URL=${PARENT_LOG_URL:-}

if [ -n "\$DOCKERHUB_PASSWORD" ]; then
echo \$DOCKERHUB_PASSWORD | docker login -u \$DOCKERHUB_USERNAME --password-stdin
Expand Down Expand Up @@ -276,6 +276,7 @@ function run {
-e JOB_ID=${JOB_ID:-} \
-e REF_NAME=${REF_NAME:-} \
-e TARGET_BRANCH=${TARGET_BRANCH:-} \
-e CI_DASHBOARD=${CI_DASHBOARD:-} \
-e PARENT_LOG_URL=${PARENT_LOG_URL:-} \
-e NO_CACHE=${NO_CACHE:-} \
-e NO_FAIL_FAST=${NO_FAIL_FAST:-} \
Expand Down
Loading
Loading