[devnet] pass --metrics to all validators; also support 1 or 2 core clients per validator

Author: bendyarm

devnet-topo.sh

@@ -4,21 +4,23 @@
 
 # Like devnet.sh, but with these changes that stabilize the network topology:
 #
-# * Only asks for the number N of validators, not the number of clients.
-#   Makes the same number of clients as validators, to reflect
-#   the concept that each validator has has a dedicated core client.
-#   (Currently we are waiting for more information about how mainnet validators
-#   set up their trusted peers; this script can then be updated to match.)
+# * Asks for the number N of validators, not the number of clients.
+#   Asks how many core clients each validator should have.
+#   Makes the appropriate number of clients, to reflect
+#   the concept that each validator has 1 or 2 dedicated core clients.
 # * For the Gateway (communication for BFT consensus)
 #   sets --validators starting up each validator to all the other validators
 #   (devnet.sh ends up setting it only to the first two validators).
 # * For the Router (p2p communication), sets --peers (the trusted peers)
 #   directly, rather than allowing them to be set by parse_development(),
-#   * Each validator has one trusted peer: its core client.
+#   * Each validator has 1 or 2 trusted peers: its core clients.
 #     No longer allow validators to accept external peer connections
 #     (removed the "--allow-external-peers" flag).
-#   * Each client has N trusted peers: its one trusted validator
-#     and the N-1 other clients.
+#   * Each client has either 
+#     N trusted peers (when there is 1 core client per validator)
+#     or 2N trusted peers (when there are 2 core clients per validator).
+#     In the first case, the client's trusted peers are its validator and N-1 other clients.
+#     In the second case, the client's trusted peers are its validator and 2N-1 other clients.
 
 if [[ -n "$TMUX" ]]; then
    echo "Detected nested tmux session. Try again after unsetting \$TMUX, e.g., using \`unset TMUX\` in bash."
@@ -29,13 +31,35 @@ fi
 read -p "Enter the total number of validators (default: 4): " total_validators
 total_validators=${total_validators:-4}
 
-# Each validator trusts one core client.
-total_clients=${total_validators}
+# Validate that total_validators is a positive integer
+if ! [[ "$total_validators" =~ ^[1-9][0-9]*$ ]]; then
+    echo "Error: Number of validators must be a positive integer"
+    exit 1
+fi
 
 # Read the network ID from user or use a default value of 1
 read -p "Enter the network ID (mainnet = 0, testnet = 1, canary = 2) (default: 1): " network_id
 network_id=${network_id:-1}
 
+# Validate that network_id is 0, 1, or 2
+if ! [[ "$network_id" =~ ^[012]$ ]]; then
+    echo "Error: Network ID must be 0 (mainnet), 1 (testnet), or 2 (canary)"
+    exit 1
+fi
+
+# Read the number of core clients per validator from user or use a default value of 2
+read -p "Enter the number of core clients for each validator (1 or 2; default: 2): " core_clients_per_validator
+core_clients_per_validator=${core_clients_per_validator:-2}
+
+# Validate that core_clients_per_validator is either 1 or 2
+if [[ "$core_clients_per_validator" != "1" && "$core_clients_per_validator" != "2" ]]; then
+    echo "Error: Number of core clients per validator must be 1 or 2"
+    exit 1
+fi
+
+# The total number of clients is the number of validators times the number of core clients per validator.
+total_clients=$((total_validators * core_clients_per_validator))
+
 # Ask the user if they want to run 'cargo install --locked --path .' or use a pre-installed binary
 read -p "Do you want to run 'cargo install --locked --path .' to build the binary? (y/n, default: y): " build_binary
 build_binary=${build_binary:-y}
@@ -111,14 +135,9 @@ for validator_index in "${validator_indices[@]}"; do
   name="validator-$validator_index"
   log_file="$log_dir/$name.log"
   window_index=$((validator_index + index_offset))
-  metrics_flag=""
+  metrics_port=$((validator_index + 9000))
 
-  if [ "$validator_index" -eq 0 ]; then
-    # Enable metrics only for the first validator.
-    metrics_flag="--metrics"
-    # We don't need to create a window for the first validator because the tmux
-    # session already starts with one window.
-  else
+  if [ "$validator_index" -ne 0 ]; then
     # Create a new window with a unique name
     tmux new-window -t "devnet:$window_index" -n $name
   fi
@@ -135,31 +154,43 @@ for validator_index in "${validator_indices[@]}"; do
       fi
   done
 
-  # Create the list of peers (i.e., trusted peers) for the validator at the_validator_index
-  # It is the corresponding client now.
-  # I.e., validators 0,1,2,3 each has one trusted peer which is the client at ports 4130 + num_validators + 0,1,2,3, respectively
-  validator_trusted_peers="127.0.0.1:$((4130 + total_validators + validator_index))";
+  # Create the list of peers (i.e., trusted peers) for the validator at validator_index.
+  # Each validator has 1 or 2 trusted peers: its core clients
+  if [ "$core_clients_per_validator" -eq 1 ]; then
+      # Each validator has 1 trusted peer: its core client
+      client_offset=$((validator_index * core_clients_per_validator))
+      validator_trusted_peers="127.0.0.1:$((4130 + total_validators + client_offset))"
+  else
+      # Each validator has 2 trusted peers: its two core clients
+      client_offset_1=$((validator_index * core_clients_per_validator))
+      client_offset_2=$((validator_index * core_clients_per_validator + 1))
+      validator_trusted_peers="127.0.0.1:$((4130 + total_validators + client_offset_1)),127.0.0.1:$((4130 + total_validators + client_offset_2))"
+  fi
 
   # Send the command to start the validator to the new window and capture output to the log file
-  tmux send-keys -t "devnet:$window_index" "snarkos start --nodisplay --network $network_id --dev $validator_index --dev-num-validators $total_validators --validator --validators $trusted_validators --peers $validator_trusted_peers --logfile $log_file --verbosity $verbosity $metrics_flag" C-m
+  tmux send-keys -t "devnet:$window_index" "snarkos start --nodisplay --network $network_id --dev $validator_index --dev-num-validators $total_validators --validator --validators $trusted_validators --peers $validator_trusted_peers --logfile $log_file --verbosity $verbosity  --metrics --metrics-ip=0.0.0.0:$metrics_port" C-m
 done
 
 # Define a function that generates the string to be passed as --peers when starting a client node.
 # The trusted peers consist of the one paired validator plus all the other clients.
-# The string looks like "vvv${i},ccc0,ccc1,...,ccc${n}"
+# The string looks like "vvv${v},ccc0,ccc1,...,ccc${n}"
 # excluding ccc${i} from the list,
-# where vvv${i} is "127.0.0.1:$((4130 + i))"
+# where vvv${v} is "127.0.0.1:$((4130 + (i / core_clients_per_validator)))"
 # and ccc${i} is "127.0.0.1:$((4130 + total_validators + i))"
 client_trusted_peers() {
-    # n is the number of validators (= number of clients)
+    # n is the number of validators
     # i is the index of the client for which we want a trusted peers list
     local n=$1
     local i=$2
 
-    local result="127.0.0.1:$((4130 + i))"
+    # Calculate which validator this client belongs to
+    local validator_index=$((i / core_clients_per_validator))
+    
+    # Start with the client's validator as the first trusted peer.
+    local result="127.0.0.1:$((4130 + validator_index))"
 
-    # Appends all ccc{number} from 0 to n-1, except ccc${i}
-    for ((j=0; j<n; j++)); do
+    # Add all other clients as trusted peers
+    for ((j=0; j<total_clients; j++)); do
         if [ $j -ne $i ]; then
             result="${result},127.0.0.1:$((4130 + total_validators + j))"
         fi
@@ -168,25 +199,23 @@ client_trusted_peers() {
     echo "$result"
 }
 
-if [ "$total_clients" -ne 0 ]; then
-  # Generate client indices from 0 to (total_clients - 1)
-  client_indices=($(seq 0 $((total_clients - 1))))
+# Generate client indices from 0 to (total_clients - 1)
+client_indices=($(seq 0 $((total_clients - 1))))
 
-  # Loop through the list of client indices and create a new window for each
-  for client_index in "${client_indices[@]}"; do
-    # Generate a unique and incrementing log file name based on the client index
-    name="client-$client_index"
-    log_file="$log_dir/$name.log"
+# Loop through the list of client indices and create a new window for each
+for client_index in "${client_indices[@]}"; do
+  # Generate a unique and incrementing log file name based on the client index
+  name="client-$client_index"
+  log_file="$log_dir/$name.log"
 
-    window_index=$((client_index + total_validators + index_offset))
+  window_index=$((client_index + total_validators + index_offset))
 
-    # Create a new window with a unique name
-    tmux new-window -t "devnet:$window_index" -n $name
+  # Create a new window with a unique name
+  tmux new-window -t "devnet:$window_index" -n $name
 
-    # Send the command to start the client to the new window and capture output to the log file
-    tmux send-keys -t "devnet:$window_index" "snarkos start --nodisplay --network $network_id --dev $((total_validators + client_index)) --dev-num-validators $total_validators --client --peers $(client_trusted_peers $total_validators $client_index) --logfile $log_file  --verbosity $verbosity" C-m
-  done
-fi
+  # Send the command to start the client to the new window and capture output to the log file
+  tmux send-keys -t "devnet:$window_index" "snarkos start --nodisplay --network $network_id --dev $((total_validators + client_index)) --dev-num-validators $total_validators --client --peers $(client_trusted_peers $total_validators $client_index) --logfile $log_file  --verbosity $verbosity" C-m
+done
 
 # Attach to the tmux session to view and interact with the windows
 tmux attach-session -t "devnet"