Merge pull request #4144 from ProvableHQ/release-mainnet-4.5.0

Release mainnet 4.5.0

Author: vicsn

.cargo/audit.toml

@@ -7,4 +7,10 @@ ignore = [
   "RUSTSEC-2025-0057",
   # TODO remove once newer versions of tracing-sbuscriber support colored logs again.
   "RUSTSEC-2025-0055",
+  # TODO remove once `self_update` updates its dependency for `indicatif`.
+  "RUSTSEC-2025-0119",
+  # TODO remove once no longer lru 0.12.x is unsound via ratatui; upgrade ratatui when available
+  "RUSTSEC-2026-0002",
+  # TODO remove once we migrate away from bincode, or it becomes maintained again.
+  "RUSTSEC-2025-0141",
 ]

.ci/bench_bft_sync.sh

@@ -22,59 +22,62 @@ log_filter="info,snarkos_node_sync=trace,snarkos_node_bft::sync=trace,snarkos_no
 max_wait=1800 # Wait for up to 30 minutes
 poll_interval=1 # Check block heights every second
 
-. ./.ci/utils.sh
+#shellcheck source=SCRIPTDIR/utils.sh
+. .ci/utils.sh
+
+# Create log directory
+init_log_dir
 
 branch_name=$(git rev-parse --abbrev-ref HEAD)
-echo "On branch: ${branch_name}"
+log "On branch: ${branch_name}"
 
 network_name=$(get_network_name $network_id)
-echo "Using network: $network_name (ID: $network_id)"
+log "Using network: $network_name (ID: $network_id)"
 
 snapshot_info=$(<info.txt)
-echo "Snapshot_info: ${snapshot_info}"
-
-# Create log directory
-log_dir=".logs-$(date +"%Y%m%d%H%M%S")"
-mkdir -p "$log_dir"
+log "Snapshot_info: ${snapshot_info}"
 
 # Define a trap handler that cleans up all processes on exit.
+# shellcheck disable=SC2329
 function exit_handler() {
   stop_nodes
 }
 trap exit_handler EXIT
-trap child_exit_handler CHLD
 
 # Define a trap handler that prints a message when an error occurs 
-trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR
+trap 'log "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR
 
 # Shared flags betwen all nodes
 common_flags=(
   --nobanner --noupdater --nodisplay \
   "--network=$network_id"
   --nocdn
+  "--dev-num-clients=0"
   "--dev-num-validators=$num_validators"
   --no-dev-txs
   "--log-filter=$log_filter"
 )
 
 # The validator that has the ledger to by synced from.
-$TASKSET1 snarkos start --dev 0 --validator "${common_flags[@]}" \
-  --logfile="$log_dir/validator-0.log" &
+# shellcheck disable=SC2086
+run_with_prefix "validator-0" $TASKSET1 snarkos start --dev 0 --validator "${common_flags[@]}" \
+  --logfile="$log_dir/validator-0.log"
 PIDS[0]=$!
 
 # Stores the list of all validators.
 validators="127.0.0.1:5000"
 
-# Spawn the clients that will sync the ledger
+# Spawn the validators that will sync the ledger
 for node_index in $(seq 1 "$num_nodes"); do
   name="validator-$node_index"
 
   # Ensure there are no old ledger files and the node syncs from scratch
   snarkos clean "--dev=$node_index" "--network=$network_id" || true
 
-  $TASKSET2 snarkos start "--dev=$node_index" --validator \
+  # shellcheck disable=SC2086
+  run_with_prefix "$name" $TASKSET2 snarkos start "--dev=$node_index" --validator \
     "${common_flags[@]}" "--validators=$validators" \
-    "--logfile=$log_dir/$name.log" &
+    "--logfile=$log_dir/$name.log"
   PIDS[node_index]=$!
 
   # Add the validators BFT address to the validators list.
@@ -86,19 +89,19 @@ for node_index in $(seq 1 "$num_nodes"); do
 done
 
 # Block until nodes are running and connected to each other.
-wait_for_nodes $((num_nodes+1)) 0
+wait_for_nodes $((num_nodes+1)) 0 "$network_name"
 
 SECONDS=0
 
-# TODO add API call for number of connected validators.
-#for ((node_index = 0; node_index < num_nodes+1; node_index++)); do
-#  if ! (wait_for_peers "$node_index" $num_nodes); then
-#    exit 1
-#  fi
-#done
+# Wait for all validators to be connected to each other via the gateway.
+for ((node_index = 0; node_index < num_nodes+1; node_index++)); do
+  if ! (wait_for_bft_connections "$node_index" $num_nodes "$network_name"); then
+    exit 1
+  fi
+done
 
 connect_time=$SECONDS
-echo "ℹ️ Nodes are fully connected (took $connect_time secs). Starting block sync measurement."
+log "ℹ️ Nodes are fully connected (took $connect_time secs). Starting block sync measurement."
 
 # Check heights periodically with a timeout
 SECONDS=0
@@ -111,7 +114,7 @@ while (( SECONDS < max_wait )); do
     total_wait=$SECONDS
     throughput=$(compute_throughput "$min_height" "$total_wait")
 
-    echo "🎉 BFT sync benchmark done! Waited $total_wait seconds for $min_height blocks. Throughput was $throughput blocks/s."
+    log "🎉 BFT sync benchmark done! Waited $total_wait seconds for $min_height blocks. Throughput was $throughput blocks/s."
 
     # Append data to results file.
     printf "{ \"name\": \"bft-sync\", \"unit\": \"blocks/s\", \"value\": %.3f, \"extra\": \"total_wait=%is, target_height=%i, connect_time=%i, branch=%s, %s\" },\n" \
@@ -123,13 +126,7 @@ while (( SECONDS < max_wait )); do
   sleep $poll_interval
 done
 
-echo "❌ Benchmark failed! Validators did not sync within 30 minutes."
-
-# Print logs for debugging
-echo "Last 20 lines of validators logs:"
-for ((node_index = 0; node_index <= num_nodes; node_index++)); do
-  echo "=== Node $node_index logs ==="
-  tail -n 20 "$log_dir/validator-$node_index.log"
-done
+log "❌ Benchmark failed! Validators did not sync within 30 minutes."
+print_validator_logs "$log_dir" "$num_validators" "$num_nodes"
 
 exit 1

.ci/bench_cdn_sync.sh

@@ -15,24 +15,24 @@ log_filter="info,snarkos_node_rest=warn,snarkos_node_cdn=debug"
 max_wait=1800 # Wait for up to 30 minutes
 poll_interval=1 # Check block heights every second
 
+# shellcheck source=SCRIPTDIR/utils.sh
 . ./.ci/utils.sh
 
-network_name=$(get_network_name $network_id)
-echo "Using network: $network_name (ID: $network_id)"
-
 # Create log directory
-log_dir=".logs-$(date +"%Y%m%d%H%M%S")"
-mkdir -p "$log_dir"
+init_log_dir
+
+network_name=$(get_network_name $network_id)
+log "Using network: $network_name (ID: $network_id)"
 
 # Define a trap handler that cleans up all processes on exit.
+# shellcheck disable=SC2329
 function exit_handler() {
   stop_nodes
 }
 trap exit_handler EXIT
-trap child_exit_handler CHLD
 
 # Define a trap handler that prints a message when an error occurs.
-trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR
+trap 'log "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR
 
 # Ensure there are no old ledger files and the node syncs from scratch
 snarkos clean "--network=$network_id" || true
@@ -42,15 +42,16 @@ args=(
   "--network=$network_id"
   --nobanner --noupdater --nodisplay # reduce clutter in the output and hide TUI
   --rest-rps=1000000 # ensure benchmarks don't fail due to rate limiting
-  --log-filter=$log_filter # only show the logs we care about
+  "--log-filter=$log_filter" # only show the logs we care about
 )
 
 # Spawn the client that will sync the ledger.
 # Use the same CPU cores as in the other benchmarks, so the numbers are comparable.
-$TASKSET2 snarkos start --client "${args[@]}" &
+# shellcheck disable=SC2086
+run_with_prefix "client-0" $TASKSET2 snarkos start --client "${args[@]}"
 PIDS[0]=$!
 
-wait_for_nodes 0 1
+wait_for_nodes 0 1 "$network_name"
 
 # Check heights periodically with a timeout
 SECONDS=0
@@ -59,7 +60,7 @@ while (( SECONDS < max_wait )); do
     total_wait=$SECONDS
     throughput=$(compute_throughput "$min_height" "$total_wait")
 
-    echo "🎉 Benchmark done! Waited ${total_wait}s for $min_height blocks. Throughput was $throughput blocks/s."
+    log "🎉 Benchmark done! Waited ${total_wait}s for $min_height blocks. Throughput was $throughput blocks/s."
 
     # Append data to results file.
     printf "{ \"name\": \"cdn-sync\", \"unit\": \"blocks/s\", \"value\": %.3f, \"extra\": \"total_wait=%is, target_height=${min_height}\" }\n" \
@@ -71,6 +72,6 @@ while (( SECONDS < max_wait )); do
   sleep $poll_interval
 done
 
-echo "❌ Benchmark failed! Client did not sync within 30 minutes."
+log "❌ Benchmark failed! Client did not sync within 30 minutes."
 
 exit 1

.ci/bench_p2p_sync.sh

@@ -14,6 +14,8 @@ min_height=250
 num_validators=40
 
 # The number of clients that are syncing
+# Note: Because the first indexes 0-39 are resevered for validators, the first client will have index 40.
+# The script works around this by manually setting the storage, ports, and log files for the clients. 
 num_clients=1
 
 # Adjust this to show more/less log messages
@@ -22,8 +24,12 @@ log_filter="info,snarkos_node::client=trace,snarkos_node_sync=trace,snarkos_node
 max_wait=2400 # Wait for up to 40 minutes
 poll_interval=1 # Check block heights every second
 
+# shellcheck source=SCRIPTDIR/utils.sh
 . ./.ci/utils.sh
 
+# Create log directory
+init_log_dir
+
 # Running sums for variance: use sum and sumsq for unbiased sample variance
 sum_speed=0
 sumsq_speed=0
@@ -45,13 +51,13 @@ function sample_sync_speeds() {
 
     # Skip null or empty
     if [[ -z "$speed" ]] || [[ "$speed" == "null" ]]; then
-      echo "Invalid speed value $speed"
+      log "Invalid speed value $speed"
       continue
     fi
 
     # Validate numeric (allow exponent)
     if ! (is_float "$speed"); then
-        echo "Invalid speed value $speed"
+        log "Invalid speed value $speed"
        continue
     fi
 
@@ -73,24 +79,19 @@ function sample_sync_speeds() {
 }
 
 branch_name=$(git rev-parse --abbrev-ref HEAD)
-echo "On branch: ${branch_name}"
+log "On branch: ${branch_name}"
 
 network_name=$(get_network_name $network_id)
-echo "Using network: $network_name (ID: $network_id)"
+log "Using network: $network_name (ID: $network_id)"
 
 snapshot_info=$(<info.txt)
-echo "Snapshot_info: ${snapshot_info}"
-
-# Create log directory
-log_dir=".logs-$(date +"%Y%m%d%H%M%S")"
-mkdir -p "$log_dir"
+log "Snapshot_info: ${snapshot_info}"
 
 # Define a trap handler that cleans up all processes on exit.
 trap stop_nodes EXIT
-trap child_exit_handler CHLD
 
 # Define a trap handler that prints a message when an error occurs.
-trap 'echo "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR
+trap 'log "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR
 
 # Shared flags between all nodes
 common_flags=(
@@ -104,41 +105,47 @@ common_flags=(
 
 # The client that has the ledger
 # (runs on the first two cores)
-$TASKSET1 snarkos start --dev 0 --client "${common_flags[@]}" \
-  --logfile="$log_dir/client-0.log" &
+# shellcheck disable=SC2086
+run_with_prefix "client-0" $TASKSET1 snarkos start "--dev=$num_validators" --client "${common_flags[@]}" \
+  "--logfile=$log_dir/client-0.log" "--storage=.ledger-$network_id-0" \
+  "--node=127.0.0.1:4130" "--rest=127.0.0.1:3030"
 PIDS[0]=$!
 
 # Spawn the clients that will sync the ledger
 # (running on the other two cores)
 for client_index in $(seq 1 "$num_clients"); do
+  node_index=$((num_validators + client_index))
   prev_port=$((4130+client_index-1))
+  node_addr="127.0.0.1:$((4130+client_index))"
   name="client-$client_index"
 
   # Ensure there are no old ledger files and the node syncs from scratch
-  snarkos clean "--dev=$client_index" "--network=$network_id" || true
+  snarkos clean "--dev=$node_index" "--network=$network_id" "--path=.ledger-$network_id-$client_index" || true
 
-  $TASKSET2 snarkos start "--dev=$client_index" --client \
-    "${common_flags[@]}" "--peers=127.0.0.1:$prev_port" \
-    "--logfile=$log_dir/$name.log" &
+  # shellcheck disable=SC2086
+  run_with_prefix "$name" $TASKSET2 snarkos start "--dev=$node_index" --client \
+    "${common_flags[@]}" "--peers=127.0.0.1:$prev_port" "--node=$node_addr" \
+    "--rest=127.0.0.1:$((3030+client_index))" \
+    "--logfile=$log_dir/$name.log" "--storage=.ledger-$network_id-$client_index"
   PIDS[client_index]=$!
 
   # Add 1-second delay between starting nodes to avoid hitting rate limits
   sleep 1
 done
 
 # Block until nodes are running and connected to each other.
-wait_for_nodes 0 $((num_clients+1))
+wait_for_nodes $((num_clients+1)) 0 "$network_name"
 
 # It takes about 30s for nodes to connect. Do not measure this time.
 SECONDS=0
 for node_index in $(seq 0 "$num_clients"); do
-  if ! (wait_for_peers "$node_index" $num_clients); then
+  if ! (wait_for_peers "$node_index" $num_clients "$network_name"); then
     exit 1
   fi
 done
 
 connect_time=$SECONDS
-echo "ℹ️ Nodes are fully connected (took $connect_time secs). Starting block sync measurement."
+log "ℹ️ Nodes are fully connected (took $connect_time secs). Starting block sync measurement."
 
 # Ensure the first node actually has the ledger snapshot.
 # This should succeed instantly in most cases
@@ -149,10 +156,12 @@ while (( SECONDS < 30 )); do
     has_blocks=true
     break
   fi
+
+  sleep $poll_interval
 done
 
 if ! $has_blocks; then
-  echo "Node #0 has not reached the expected height. Maybe the ledger snapshot is corrupted or outdated?"
+  log "Node #0 has not reached the expected height. Maybe the ledger snapshot is corrupted or outdated?"
   exit 1
 fi
 
@@ -178,7 +187,7 @@ while (( SECONDS < max_wait )); do
       variance=$(echo "scale=8; 0" | bc -l)
     fi
 
-    echo "🎉 P2P sync benchmark done! Waited $total_wait seconds for $min_height blocks. Throughput was $throughput blocks/s."
+    log "🎉 P2P sync benchmark done! Waited $total_wait seconds for $min_height blocks. Throughput was $throughput blocks/s."
 
     # Append data to results file.
     printf "{ \"name\": \"p2p-sync\", \"unit\": \"blocks/s\", \"value\": %.3f, \"extra\": \"total_wait=%is, target_height=%i, connect_time=%is, %s\" },\n" \
@@ -193,13 +202,9 @@ while (( SECONDS < max_wait )); do
   sleep $poll_interval
 done
 
-echo "❌ Benchmark failed! Clients did not sync within 40 minutes."
+log "❌ Benchmark failed! Clients did not sync within 40 minutes."
 
 # Print logs for debugging
-echo "Last 20 lines of client logs:"
-for ((client_index = 0; client_index < num_clients; client_index++)); do
-  echo "=== Client $client_index logs ==="
-  tail -n 20 "$log_dir/client-$client_index.log"
-done
+print_client_logs "$log_dir" "$num_validators" "$num_clients"
 
 exit 1