Merge branch 'feat/deprecate-batcher-url' into fix/partially-verified-batch-should-display-batches

Author: uri-99

Makefile

@@ -7,7 +7,7 @@ CONFIG_FILE?=config-files/config.yaml
 export OPERATOR_ADDRESS ?= $(shell yq -r '.operator.address' $(CONFIG_FILE))
 AGG_CONFIG_FILE?=config-files/config-aggregator.yaml
 
-OPERATOR_VERSION=v0.12.2
+OPERATOR_VERSION=v0.13.0
 
 ifeq ($(OS),Linux)
 	BUILD_ALL_FFI = $(MAKE) build_all_ffi_linux
@@ -290,6 +290,24 @@ verifier_disable:
 	@echo "Disabling verifier with ID: $(VERIFIER_ID)"
 	@. contracts/scripts/.env && . contracts/scripts/disable_verifier.sh $(VERIFIER_ID)
 
+strategies_get_weight:
+	@echo "Getting weight of strategy: $(STRATEGY_INDEX)"
+	@. contracts/scripts/.env.$(NETWORK) && . contracts/scripts/get_strategy_weight.sh $(STRATEGY_INDEX)
+
+strategies_update_weight:
+	@echo "Updating strategy weights: "
+	@echo "STRATEGY_INDICES: $(STRATEGY_INDICES)"
+	@echo "NEW_MULTIPLIERS: $(NEW_MULTIPLIERS)"
+	@. contracts/scripts/.env.$(NETWORK) && . contracts/scripts/update_strategy_weight.sh $(STRATEGY_INDICES) $(NEW_MULTIPLIERS)
+
+strategies_remove:
+	@echo "Removing strategies: $(INDICES_TO_REMOVE)"
+	@. contracts/scripts/.env.$(NETWORK) && . contracts/scripts/remove_strategy.sh $(INDICES_TO_REMOVE)
+
+strategies_get_addresses:
+	@echo "Getting strategy addresses"
+	@. contracts/scripts/.env.$(NETWORK) && . contracts/scripts/get_restakeable_strategies.sh
+
 __BATCHER__:
 
 BURST_SIZE ?= 5
@@ -486,16 +504,15 @@ task_sender_send_infinite_proofs_devnet:
 	cargo run --release -- send-infinite-proofs \
 	--burst-size $(BURST_SIZE) --burst-time-secs $(BURST_TIME_SECS) \
 	--eth-rpc-url http://localhost:8545 \
-	--batcher-url ws://localhost:8080 \
 	--network devnet \
 	--proofs-dirpath $(CURDIR)/scripts/test_files/task_sender/proofs \
 	--private-keys-filepath $(CURDIR)/batcher/aligned-task-sender/wallets/devnet
 
 task_sender_test_connections_devnet:
 	@cd batcher/aligned-task-sender && \
 	cargo run --release -- test-connections \
-	--batcher-url ws://localhost:8080 \
-	--num-senders $(NUM_SENDERS)
+	--num-senders $(NUM_SENDERS) \
+	--network devnet
 
 # ===== HOLESKY-STAGE =====
 task_sender_generate_and_fund_wallets_holesky_stage:
@@ -514,21 +531,21 @@ task_sender_send_infinite_proofs_holesky_stage:
 	cargo run --release -- send-infinite-proofs \
 	--burst-size $(BURST_SIZE) --burst-time-secs $(BURST_TIME_SECS) \
 	--eth-rpc-url https://ethereum-holesky-rpc.publicnode.com \
-	--batcher-url wss://stage.batcher.alignedlayer.com  \
 	--network holesky-stage \
 	--proofs-dirpath $(CURDIR)/scripts/test_files/task_sender/proofs \
 	--private-keys-filepath $(CURDIR)/batcher/aligned-task-sender/wallets/holesky-stage
 
 task_sender_test_connections_holesky_stage:
 	@cd batcher/aligned-task-sender && \
 	cargo run --release -- test-connections \
-	--batcher-url wss://stage.batcher.alignedlayer.com \
-	--num-senders $(NUM_SENDERS)
+	--num-senders $(NUM_SENDERS) \
+	--network holesky-stage
 
 __UTILS__:
 aligned_get_user_balance_devnet:
 	@cd batcher/aligned/ && cargo run --release -- get-user-balance \
-		--user_addr $(USER_ADDR)
+		--user_addr $(USER_ADDR) \
+		--network devnet
 
 aligned_get_user_balance_holesky:
 	@cd batcher/aligned/ && cargo run --release -- get-user-balance \
@@ -557,7 +574,13 @@ generate_groth16_ineq_proof: ## Run the gnark_plonk_bn254_script
 	@go run scripts/test_files/gnark_groth16_bn254_infinite_script/cmd/main.go 1
 
 __METRICS__:
-# Prometheus and graphana
+# Prometheus and Grafana
+metrics_remove_containers:
+	@docker stop prometheus grafana
+	@docker rm prometheus grafana
+metrics_clean_db: metrics_remove_containers
+	@docker volume rm aligned_layer_grafana_data aligned_layer_prometheus_data
+
 run_metrics: ## Run metrics using metrics-docker-compose.yaml
 	@echo "Running metrics..."
 	@docker compose -f metrics-docker-compose.yaml up
@@ -1040,11 +1063,18 @@ docker_verify_proof_submission_success:
 				verification=$$(aligned verify-proof-onchain \
 									--aligned-verification-data $${proof} \
 									--rpc_url $$(echo $(DOCKER_RPC_URL)) 2>&1); \
+				cat $${proof%.cbor}.json; \
+				echo "$$verification"; \
 				if echo "$$verification" | grep -q not; then \
 					echo "ERROR: Proof verification failed for $${proof}"; \
 					exit 1; \
 				elif echo "$$verification" | grep -q verified; then \
 					echo "Proof verification succeeded for $${proof}"; \
+				else \
+					echo "WARNING: Unexpected verification result for $${proof}"; \
+					echo "Output:"; \
+					echo "$$verification"; \
+					exit 1; \
 				fi; \
 				echo "---------------------------------------------------------------------------------------------------"; \
 			done; \
@@ -1202,3 +1232,62 @@ ansible_operator_deploy: ## Deploy the Operator. Parameters: INVENTORY
 		-i $(INVENTORY) \
 		-e "ecdsa_keystore_path=$(ECDSA_KEYSTORE)" \
 		-e "bls_keystore_path=$(BLS_KEYSTORE)"
+
+__ETHEREUM_PACKAGE__:  ## ____
+
+ethereum_package_start: ## Starts the ethereum_package environment
+	kurtosis run --enclave aligned github.com/ethpandaops/ethereum-package --args-file network_params.yaml
+
+ethereum_package_inspect: ## Prints detailed information about the net
+	kurtosis enclave inspect aligned
+
+ethereum_package_rm: ## Stops and removes the ethereum_package environment and used resources
+	kurtosis enclave rm aligned -f
+
+batcher_start_ethereum_package: user_fund_payment_service
+	@echo "Starting Batcher..."
+	@$(MAKE) run_storage &
+	@cargo run --manifest-path ./batcher/aligned-batcher/Cargo.toml --release -- --config ./config-files/config-batcher-ethereum-package.yaml --env-file ./batcher/aligned-batcher/.env.dev
+
+aggregator_start_ethereum_package:
+	$(MAKE) aggregator_start AGG_CONFIG_FILE=config-files/config-aggregator-ethereum-package.yaml
+
+operator_start_ethereum_package:
+	$(MAKE) operator_start OPERATOR_ADDRESS=0x70997970C51812dc3A010C7d01b50e0d17dc79C8 CONFIG_FILE=config-files/config-operator-1-ethereum-package.yaml
+
+operator_register_start_ethereum_package:
+	$(MAKE) operator_full_registration OPERATOR_ADDRESS=0x70997970C51812dc3A010C7d01b50e0d17dc79C8 CONFIG_FILE=config-files/config-operator-1-ethereum-package.yaml \
+	$(MAKE) operator_start OPERATOR_ADDRESS=0x70997970C51812dc3A010C7d01b50e0d17dc79C8 CONFIG_FILE=config-files/config-operator-1-ethereum-package.yaml
+
+
+install_spamoor: ## Instal spamoor to spam transactions
+	@echo "Installing spamoor..."
+	@git clone https://github.com/ethpandaops/spamoor.git
+	@cd spamoor && make
+	@mv spamoor/bin/spamoor $(HOME)/.local/bin
+	@rm -rf spamoor
+	@echo "======================================================================="
+	@echo "Installation complete! Run 'spamoor --help' to verify the installation."
+	@echo "If 'spamoor' is not recognized, make sure it's in your PATH by adding the following line to your shell configuration:"
+	@echo "export PATH=\$$PATH:\$$HOME/.local/bin"
+	@echo "======================================================================="
+
+# Spamoor funding wallet
+SPAMOOR_PRIVATE_KEY?=dbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97
+NUM_WALLETS?=1000
+TX_PER_BLOCK?=250
+# Similar to a swap
+TX_CONSUMES_GAS?=150000
+
+spamoor_send_transactions: ## Sends normal transactions and also replacement transactions
+	spamoor gasburnertx -p $(SPAMOOR_PRIVATE_KEY) -c $(COUNT) \
+		--gas-units-to-burn $(TX_CONSUMES_GAS) \
+		--max-wallets $(NUM_WALLETS) --max-pending $(TX_PER_BLOCK) \
+		-t $(TX_PER_BLOCK) -h http://127.0.0.1:8545/ -h http://127.0.0.1:8550/ -h http://127.0.0.1:8555/ -h http://127.0.0.1:8565/ \
+		--refill-amount 5 --refill-balance 2 --tipfee $(TIP_FEE) --basefee 100  \
+		2>&1 | grep -v 'checked child wallets (no funding needed)'
+
+__NODE_EXPORTER_: ##__
+
+install_node_exporter:
+	@./scripts/install_node_exporter.sh

alerts/.env.devnet

@@ -24,7 +24,6 @@
 # Variables for sender_with_alert.sh
 REPETITIONS=8
 SENDER_ADDRESS=0x14dC79964da2C08b23698B3D3cc7Ca32193d9955
-BATCHER_URL=ws://localhost:8080
 RPC_URL=http://localhost:8545
 EXPLORER_URL=http://localhost:3000
 NETWORK=devnet

alerts/.env.example

@@ -19,12 +19,13 @@ EXPRESSION=<GREP_EXPRESSION>
 RPC_URL=<YOUR_RPC_URL>
 PAYMENT_CONTRACT_ADDRESS=<YOUR_PAYMENT_CONTRACT_ADDRESS>
 BALANCE_THRESHOLD=<YOUR_BALANCE_THRESHOLD_IN_ETH>
+WALLET_NAME=<YOUR_WALLET_NAME> # Example: "Task sender"
 WALLET_ADDRESS=<YOUR_WALLET_ADDRESS>
+NETWORK=<MAINNET|HOLESKY|STAGE>
 
 # Variables for sender_with_alert.sh
 REPETITIONS=<REPETITIONS>
 SENDER_ADDRESS=<YOUR_SENDER_ADDRESS>
-BATCHER_URL=<BATCHER_URL>
 RPC_URL=<RPC_BASE_URL>
 EXPLORER_URL=<EXPLORER_BASE_URL>
 NETWORK=<NETWORK>

alerts/balance_alerts.sh

@@ -25,12 +25,12 @@ balance_alert=false
 
 while :
 do
-  balance_wei=$(cast call --rpc-url $RPC_URL $PAYMENT_CONTRACT_ADDRESS "UserBalances(address)(uint256)" $WALLET_ADDRESS | cut -d' ' -f1)
+  balance_wei=$(cast call --rpc-url $RPC_URL $PAYMENT_CONTRACT_ADDRESS "user_balances(address)(uint256)" $WALLET_ADDRESS | cut -d' ' -f1)
 
   balance_eth=$(cast from-wei $balance_wei)
 
   if [ 1 -eq "$(echo "$balance_eth < $BALANCE_THRESHOLD" | bc)" ]; then
-    message="⚠️ WARNING: Wallet $WALLET_ADDRESS balance ($balance_eth ETH) is below $BALANCE_THRESHOLD ETH"
+    message="⚠️ WARNING: $WALLET_NAME ($NETWORK) Wallet ($WALLET_ADDRESS) balance ($balance_eth ETH) is below $BALANCE_THRESHOLD ETH"
     printf "$message\n"
     if [ "$balance_alert" = false ]; then
       send_slack_message "$message"

alerts/sender_with_alert.sh

@@ -4,7 +4,6 @@
 # - REPETITIONS
 # - EXPLORER_URL
 # - SENDER_ADDRESS
-# - BATCHER_URL
 # - RPC_URL
 # - EXPLORER_URL
 # - NETWORK
@@ -53,6 +52,22 @@ function fetch_tx_cost() {
   fi
 }
 
+# Function to get the tx cost from the tx hash
+# @param tx_hash
+function get_number_proofs_in_batch_from_create_task_tx() {
+  if [[ -z "$1" ]]; then
+    echo 0
+  else
+    # Get the tx receipt from the blockchain
+    calldata=$(cast tx $1 --rpc-url $RPC_URL input)
+    decoded_calldata=$(cast --calldata-decode --json "createNewTask(bytes32 batchMerkleRoot, string batchDataPointer, address[] proofSubmitters, uint256 feeForAggregator, uint256 feePerProof, uint256 respondToTaskFeeLimit)" $calldata)
+    # We count the number of proofSubmitters within the tx which corresponds to the number of proofs sent in the last batch
+    number_proofs_in_batch=$(echo $decoded_calldata | jq '.[2] | [ match(","; "g")] | length + 1')
+
+    echo $number_proofs_in_batch
+  fi
+}
+
 # Function to send PagerDuty alert
 # @param message
 function send_pagerduty_alert() {
@@ -76,7 +91,7 @@ do
   mkdir -p ./scripts/test_files/gnark_groth16_bn254_infinite_script/infinite_proofs
 
   ## Generate Proof
-  nonce=$(aligned get-user-nonce --batcher_url $BATCHER_URL --user_addr $SENDER_ADDRESS 2>&1 | awk '{print $9}')
+  nonce=$(aligned get-user-nonce --network $NETWORK --user_addr $SENDER_ADDRESS 2>&1 | awk '{print $9}')
   x=$((nonce + 1)) # So we don't have any issues with nonce = 0
   echo "Generating proof $x != 0"
   go run ./scripts/test_files/gnark_groth16_bn254_infinite_script/cmd/main.go $x
@@ -92,9 +107,8 @@ do
     --proof_generator_addr $SENDER_ADDRESS \
     --private_key $PRIVATE_KEY \
     --rpc_url $RPC_URL \
-    --batcher_url $BATCHER_URL \
     --network $NETWORK \
-    --max_fee 4000000000000000 \
+    --max_fee 0.004ether \
     2>&1)
 
   echo "$submit"
@@ -113,6 +127,7 @@ do
   fi
 
   total_fee_in_wei=0
+  total_number_proofs=0
   batch_explorer_urls=()
   for batch_merkle_root in $batch_merkle_roots 
   do
@@ -127,12 +142,16 @@ do
     response_tx_hash=$(echo "$log" | grep -oE "transactionHash: 0x[[:alnum:]]{64}" | awk '{ print $2 }')
 
     # Calculate fees for transactions
+    number_proofs_in_batch=$(get_number_proofs_in_batch_from_create_task_tx $submission_tx_hash)
     submission_fee_in_wei=$(fetch_tx_cost $submission_tx_hash)
     response_fee_in_wei=$(fetch_tx_cost $response_tx_hash)
     batch_fee_in_wei=$((submission_fee_in_wei + response_fee_in_wei))
 
     # Accumulate the fee
     total_fee_in_wei=$(($total_fee_in_wei + $batch_fee_in_wei))
+
+    # Accumulate proofs in batch
+    total_number_proofs=$(($total_number_proofs + $number_proofs_in_batch))
   done
 
   # Calculate the spent amount by converting the fee to ETH
@@ -168,9 +187,9 @@ do
   done
 
   if [ $verified -eq 1 ]; then
-    slack_message="$REPETITIONS Proofs submitted and verified. Spent amount: $spent_amount ETH ($ $spent_amount_usd) [ ${batch_explorer_urls[@]} ]"
+    slack_message="$total_number_proofs proofs submitted and verified. We sent $REPETITIONS proofs. Spent amount: $spent_amount ETH ($ $spent_amount_usd) [ ${batch_explorer_urls[@]} ]"
   else
-    slack_message="$REPETITIONS Proofs submitted but not verified. Spent amount: $spent_amount ETH ($ $spent_amount_usd) [ ${batch_explorer_urls[@]} ]"
+    slack_message="$total_number_proofs proofs submitted but not verified. We sent $REPETITIONS proofs. Spent amount: $spent_amount ETH ($ $spent_amount_usd) [ ${batch_explorer_urls[@]} ]"
   fi
 
   ## Send Update to Slack

audits/erc20_contracts/open_zeppelin.pdf

@@ -37,6 +37,10 @@ impl NonPayingConfig {
 
 #[derive(Debug, Deserialize)]
 pub struct BatcherConfigFromYaml {
+    #[serde(default = "default_aggregator_fee_percentage_multiplier")]
+    pub aggregator_fee_percentage_multiplier: u128,
+    #[serde(default = "default_aggregator_gas_cost")]
+    pub aggregator_gas_cost: u128,
     pub block_interval: u64,
     pub transaction_wait_timeout: u64,
     pub max_proof_size: usize,
@@ -86,3 +90,11 @@ impl ContractDeploymentOutput {
         serde_json::from_str(&deployment_output).expect("Failed to parse deployment output file")
     }
 }
+
+fn default_aggregator_fee_percentage_multiplier() -> u128 {
+    aligned_sdk::core::constants::DEFAULT_AGGREGATOR_FEE_PERCENTAGE_MULTIPLIER
+}
+
+fn default_aggregator_gas_cost() -> u128 {
+    aligned_sdk::core::constants::DEFAULT_AGGREGATOR_GAS_COST
+}

batcher/Cargo.lock

@@ -21,7 +21,9 @@ pub(crate) async fn send_batch_inclusion_data_responses(
     finalized_batch: Vec<BatchQueueEntry>,
     batch_merkle_tree: &MerkleTree<VerificationCommitmentBatch>,
 ) -> Result<(), BatcherError> {
-    for (vd_batch_idx, entry) in finalized_batch.iter().enumerate() {
+    // Finalized_batch is ordered as the PriorityQueue, ordered by: ascending max_fee && if max_fee is equal, by descending nonce.
+    // We iter it in reverse because each sender wants to receive responses in ascending nonce order
+    for (vd_batch_idx, entry) in finalized_batch.iter().enumerate().rev() {
         let batch_inclusion_data = BatchInclusionData::new(
             vd_batch_idx,
             batch_merkle_tree,