Bug, Full node crash with block execution error: Unexpected error from known Move function, 'block_prologue' (#1257)

Author: musitdev

Cargo.lock

@@ -7,6 +7,8 @@ services:
       MAYBE_RUN_LOCAL: "false"
       RUST_LOG: info,aws_sdk_s3=debug
       MAPTOS_DA_SEQUENCER_CONNECTION_URL: ${MAPTOS_DA_SEQUENCER_CONNECTION_URL}
+      MAPTOS_CHAIN_ID: ${MAPTOS_CHAIN_ID}
+      MAPTOS_PRIVATE_KEY: ${MAPTOS_PRIVATE_KEY}
     volumes:
       - ${DOT_MOVEMENT_PATH}:/.movement
       # mount if exists

docker/compose/movement-full-node/docker-compose.fullnode_setup.yml

@@ -49,28 +49,40 @@ impl BasicScenario {
 impl Scenario for BasicScenario {
 	#[tracing::instrument(skip(self), fields(scenario = self.id))]
 	async fn prepare(&mut self) -> Result<(), anyhow::Error> {
+		info!(
+			"Scenario:{} prepare start. NODE_URL:{} FAUCET_URL:{}",
+			self.id,
+			&NODE_URL.to_string(),
+			&FAUCET_URL.to_string()
+		);
+
 		let rest_client = Client::new(NODE_URL.clone());
 		let faucet_client = FaucetClient::new(FAUCET_URL.clone(), NODE_URL.clone());
-		let faucet_client = faucet_client.with_auth_token("notreal".to_string());
+		//let faucet_client = faucet_client.with_auth_token("notreal".to_string());
 
 		if std::env::var_os("TEST_ALICE_BOB_ACCOUNT_READ_FROM_FILE").is_some() {
 			//bob always 0. Only use to receive transfer.
 			let (priv_key, address) = get_account_from_list(0)?;
 			let sequence_number = match rest_client.get_account(address).await {
 				Ok(response) => response.into_inner().sequence_number,
 				Err(_) => {
-					tracing::warn!("File private_key:{priv_key} not created, create it.");
+					tracing::warn!("Bod account:{address} not created, create it.");
 					faucet_client.fund(address, 100_000_000_000).await.unwrap();
+					tracing::warn!("Bod account:{address} funded.");
 					0
 				}
 			};
+			info!(
+				"Scenario:{} prepare. Bob Account:{address} sequence_number:{sequence_number}",
+				self.id,
+			);
 			self.bob = Some(LocalAccount::new(address, priv_key, sequence_number));
 			// Alice account is the scenario id   + 1 (because bob is 0)
 			let (priv_key, address) = get_account_from_list(self.id + 1)?;
 			let sequence_number = match rest_client.get_account(address).await {
 				Ok(response) => response.into_inner().sequence_number,
 				Err(_) => {
-					tracing::warn!("File private_key:{priv_key} not created, create it.");
+					tracing::warn!("Alice account:{address} not created, create it.");
 					faucet_client.fund(address, 100_000_000_000).await.unwrap();
 					0
 				}
@@ -127,6 +139,8 @@ impl Scenario for BasicScenario {
 
 		let mut sequence_number = alice.sequence_number();
 
+		info!("Scenario:{} Start sending transactions.", self.id,);
+
 		for index in 0..2 {
 			//			let _ = tokio::time::sleep(tokio::time::Duration::from_millis(10)).await;
 			// Have Alice send Bob some more coins.
@@ -147,7 +161,7 @@ impl Scenario for BasicScenario {
 
 			sequence_number += 1;
 
-			//			info!(scenario = %self.id, tx_hash = ?tx_hash, index = %index, "waiting for Alice -> Bod transfer to complete");
+			info!(scenario = %self.id, tx_hash = ?tx_hash, index = %index, "waiting for Alice -> Bod transfer to complete");
 
 			rest_client
 				.wait_for_transaction(&tx_hash)
@@ -156,6 +170,7 @@ impl Scenario for BasicScenario {
 					anyhow::anyhow!("Alice Tx failed:{pending_tx:?} index:{index} because {err}")
 				})
 				.unwrap();
+			info!("Scenario:{} Transaction executed.", self.id,);
 		}
 
 		// Print final balances.

networks/movement/da-sequencer-node/Cargo.toml

@@ -231,7 +231,7 @@ impl RestoreParam {
 		]);
 
 		match push_pipe.pull(Some(syncador::Package::null())).await {
-			Ok(package) => {
+			Ok(_) => {
 				tracing::info!("Files restored");
 			}
 			Err(err) => {

networks/movement/da-sequencer-node/bin/mock-da.rs

@@ -20,7 +20,7 @@ impl Manager {
 	}
 
 	pub async fn try_run(&self) -> Result<(), anyhow::Error> {
-		let (stop_tx, mut stop_rx) = tokio::sync::watch::channel(());
+		let (stop_tx, stop_rx) = tokio::sync::watch::channel(());
 		tokio::spawn({
 			let mut sigterm =
 				signal(SignalKind::terminate()).context("can't register to SIGTERM.")?;
@@ -54,14 +54,6 @@ impl Manager {
 
 		let join_handle = tokio::spawn(node.run(mempool_commit_tx_receiver, stop_rx));
 		join_handle.await??;
-		// // Use tokio::select! to wait for either the handle or a cancellation signal
-		// tokio::select! {
-		// 	_ = stop_rx.changed() =>(),
-		// 	// manage Movement node execution return.
-		// 	res = join_handle => {
-		// 		res??;
-		// 	},
-		// };
 
 		Ok(())
 	}

networks/movement/da-sequencer-node/src/main.rs

@@ -245,16 +245,20 @@ where
 					block_timestamp,
 					block_retry_count,
 					block_retry_increment_microseconds,
-				)?;
+				);
+
+				// Block must be added to da_db after execution otherwise if an error occurs,
+				// the block will be reexecuted. Generating the error:
+				// `Unexpected error from known Move function, 'block_prologue'. Error: ABORTED
+				da_db
+					.set_synced_height(da_block.height)
+					.and_then(|_| da_db.add_executed_block(da_block.block_id.clone()))?;
+				// Forward execution error after.
+				let exec_result = exec_result?;
 
 				// decrement the number of transactions in flight on the executor
 				executor.decrement_transactions_in_flight(transactions_count as u64);
 
-				da_db.set_synced_height(da_block.height)?;
-
-				// set the block as executed
-				da_db.add_executed_block(da_block.block_id.clone())?;
-
 				Ok((Some(exec_result), executor))
 			}
 		})

networks/movement/movement-client/src/bin/e2e/load_alice_bob.rs

@@ -64,7 +64,12 @@ impl Local {
 		config.execution_config.maptos_config.chain.maptos_db_path.replace(db_path);
 
 		// Set as main node that send state.
-		config.execution_config.maptos_config.da_sequencer.propagate_execution_state = true;
+		let local = std::env::var_os("MAYBE_RUN_LOCAL").unwrap_or("false".into());
+		if local == "false" {
+			config.execution_config.maptos_config.da_sequencer.propagate_execution_state = false;
+		} else {
+			config.execution_config.maptos_config.da_sequencer.propagate_execution_state = true;
+		}
 
 		// write the maptos signer address to the default signer address whitelist
 		let default_signer_address_whitelist_path =
@@ -84,7 +89,12 @@ impl Local {
 		mut config: movement_config::Config,
 	) -> Result<movement_config::Config, anyhow::Error> {
 		// Allow Da sync from Height zero
-		config.da_db.allow_sync_from_zero = true;
+		let local = std::env::var_os("MAYBE_RUN_LOCAL").unwrap_or("false".into());
+		if local == "false" {
+			config.da_db.allow_sync_from_zero = false;
+		} else {
+			config.da_db.allow_sync_from_zero = true;
+		}
 		// update the db path
 		let db_path = dot_movement.get_path().join(config.da_db.da_db_path.clone());
 		config.da_db.da_db_path = db_path