Dev mode for integration tests

crates/client/src/common/mod.rs

@@ -725,6 +725,7 @@ pub fn initialize_not_light_node_modules(
             consensus.clone(),
             sync.clone(),
             txpool.clone(),
+            blockgen.clone(),
             notifications.clone(),
             task_executor.clone(),
             conf,

crates/client/src/rpc/mod.rs

@@ -2,6 +2,7 @@
 // Conflux is free software and distributed under GNU General Public License.
 // See http://www.gnu.org/licenses/
 
+use blockgen::BlockGenerator;
 use cfx_rpc_builder::{
     RpcModuleBuilder, RpcServerConfig, RpcServerHandle,
     TransportRpcModuleConfig,
@@ -406,8 +407,9 @@ where
 // start espace rpc server v2(async)
 pub async fn launch_async_rpc_servers(
     consensus: SharedConsensusGraph, sync: SharedSynchronizationService,
-    tx_pool: SharedTransactionPool, notifications: Arc<Notifications>,
-    executor: TaskExecutor, conf: &Configuration,
+    tx_pool: SharedTransactionPool, block_gen: Arc<BlockGenerator>,
+    notifications: Arc<Notifications>, executor: TaskExecutor,
+    conf: &Configuration,
 ) -> Result<Option<RpcServerHandle>, String> {
     let http_config = conf.eth_http_config();
     let ws_config = conf.eth_ws_config();
@@ -455,6 +457,7 @@ pub async fn launch_async_rpc_servers(
         consensus,
         sync,
         tx_pool,
+        block_gen,
         executor,
         notifications,
     );

crates/rpc/rpc-builder/Cargo.toml

@@ -31,5 +31,6 @@ log = { workspace = true }
 cfx-rpc-middlewares = { workspace = true }
 cfx-tasks = { workspace = true }
 tokio = { workspace = true }
+blockgen = { workspace = true }
 
 [dev-dependencies]

crates/rpc/rpc-builder/src/lib.rs

@@ -37,6 +37,7 @@ pub use id_provider::EthSubscriptionIdProvider;
 use log::debug;
 pub use module::{EthRpcModule, RpcModuleSelection};
 
+use blockgen::BlockGenerator;
 use cfx_rpc::{helpers::ChainInfo, *};
 use cfx_rpc_cfx_types::RpcImplConfiguration;
 use cfx_rpc_eth_api::*;
@@ -75,6 +76,7 @@ pub struct RpcModuleBuilder {
     consensus: SharedConsensusGraph,
     sync: SharedSynchronizationService,
     tx_pool: SharedTransactionPool,
+    block_gen: Arc<BlockGenerator>,
     executor: TaskExecutor,
     notifications: Arc<Notifications>,
 }
@@ -83,13 +85,15 @@ impl RpcModuleBuilder {
     pub fn new(
         config: RpcImplConfiguration, consensus: SharedConsensusGraph,
         sync: SharedSynchronizationService, tx_pool: SharedTransactionPool,
-        executor: TaskExecutor, notifications: Arc<Notifications>,
+        block_gen: Arc<BlockGenerator>, executor: TaskExecutor,
+        notifications: Arc<Notifications>,
     ) -> Self {
         Self {
             config,
             consensus,
             sync,
             tx_pool,
+            block_gen,
             executor,
             notifications,
         }
@@ -111,6 +115,7 @@ impl RpcModuleBuilder {
                 consensus,
                 sync,
                 tx_pool,
+                block_gen,
                 executor,
                 notifications,
             } = self;
@@ -120,6 +125,7 @@ impl RpcModuleBuilder {
                 consensus,
                 sync,
                 tx_pool,
+                block_gen,
                 executor,
                 notifications,
             );
@@ -140,6 +146,7 @@ pub struct RpcRegistryInner {
     config: RpcImplConfiguration,
     sync: SharedSynchronizationService,
     tx_pool: SharedTransactionPool,
+    block_gen: Arc<BlockGenerator>,
     modules: HashMap<EthRpcModule, Methods>,
     executor: TaskExecutor,
     notifications: Arc<Notifications>,
@@ -149,13 +156,15 @@ impl RpcRegistryInner {
     pub fn new(
         config: RpcImplConfiguration, consensus: SharedConsensusGraph,
         sync: SharedSynchronizationService, tx_pool: SharedTransactionPool,
-        executor: TaskExecutor, notifications: Arc<Notifications>,
+        block_gen: Arc<BlockGenerator>, executor: TaskExecutor,
+        notifications: Arc<Notifications>,
     ) -> Self {
         Self {
             consensus,
             config,
             sync,
             tx_pool,
+            block_gen,
             modules: Default::default(),
             executor,
             notifications,
@@ -248,6 +257,7 @@ impl RpcRegistryInner {
                             self.consensus.clone(),
                             self.sync.clone(),
                             self.tx_pool.clone(),
+                            self.block_gen.clone(),
                             self.executor.clone(),
                         )
                         .into_rpc();
@@ -286,6 +296,7 @@ impl RpcRegistryInner {
                             self.consensus.clone(),
                             self.sync.clone(),
                             self.tx_pool.clone(),
+                            self.block_gen.clone(),
                             self.executor.clone(),
                         );
                         ParityApi::new(eth_api).into_rpc().into()

crates/rpc/rpc-eth-impl/Cargo.toml

@@ -19,6 +19,7 @@ cfx-types = { workspace = true }
 cfx-rpc-eth-types = { workspace = true }
 cfx-rpc-primitives = { workspace = true }
 cfx-rpc-cfx-impl = { workspace = true }
+blockgen = { workspace = true }
 async-trait = { workspace = true }
 keccak-hash = { workspace = true }
 alloy-rpc-types-trace = { workspace = true }
@@ -49,4 +50,4 @@ solidity-abi = { workspace = true }
 cfx-rpc-common-impl = { workspace = true }
 cfx-tasks = { workspace = true }
 cfx-parity-trace-types = { workspace = true }
-cfxcore-errors = { workspace = true }
+cfxcore-errors = { workspace = true }

crates/rpc/rpc-eth-impl/src/eth.rs

@@ -1,5 +1,6 @@
 use crate::helpers::{FeeHistoryCache, MAX_FEE_HISTORY_CACHE_BLOCK_COUNT};
 use async_trait::async_trait;
+use blockgen::BlockGenerator;
 use cfx_execute_helper::estimation::EstimateRequest;
 use cfx_executor::executive::{
     Executed, ExecutionError, ExecutionOutcome, TxDropError,
@@ -32,6 +33,7 @@ use cfx_types::{
 use cfx_util_macros::bail;
 use cfx_vm_types::Error as VmError;
 use cfxcore::{
+    consensus_parameters::DEFERRED_STATE_EPOCH_COUNT,
     errors::{Error as CoreError, Result as CoreResult},
     ConsensusGraph, SharedConsensusGraph, SharedSynchronizationService,
     SharedTransactionPool,
@@ -45,7 +47,9 @@ use primitives::{
 };
 use rustc_hex::ToHex;
 use solidity_abi::string_revert_reason_decode;
-use std::{collections::HashMap, future::Future};
+use std::{
+    collections::HashMap, future::Future, sync::Arc, thread, time::Duration,
+};
 
 type BlockNumber = BlockId;
 type BlockNumberOrTag = BlockId;
@@ -59,6 +63,7 @@ pub struct EthApi {
     consensus: SharedConsensusGraph,
     sync: SharedSynchronizationService,
     tx_pool: SharedTransactionPool,
+    block_gen: Arc<BlockGenerator>,
     fee_history_cache: FeeHistoryCache,
     executor: TaskExecutor,
 }
@@ -67,13 +72,14 @@ impl EthApi {
     pub fn new(
         config: RpcImplConfiguration, consensus: SharedConsensusGraph,
         sync: SharedSynchronizationService, tx_pool: SharedTransactionPool,
-        executor: TaskExecutor,
+        block_gen: Arc<BlockGenerator>, executor: TaskExecutor,
     ) -> Self {
         EthApi {
             config,
             consensus,
             sync,
             tx_pool,
+            block_gen,
             fee_history_cache: FeeHistoryCache::new(),
             executor,
         }
@@ -83,6 +89,17 @@ impl EthApi {
 
     pub fn tx_pool(&self) -> &SharedTransactionPool { &self.tx_pool }
 
+    fn generate_one_block(
+        &self, num_txs: usize, block_size_limit: usize,
+    ) -> CoreResult<H256> {
+        let hash = self.block_gen.test_api().generate_block(
+            num_txs,
+            block_size_limit,
+            vec![],
+        );
+        Ok(hash)
+    }
+
     pub fn fetch_block_by_height(
         &self, height: u64,
     ) -> Result<PhantomBlock, ProviderBlockError> {
@@ -1491,6 +1508,31 @@ impl EthApiServer for EthApi {
         }
 
         let r = self.send_transaction_with_signature(tx)?;
+        // evm txs are packed every ratio blocks
+        // always generate another ratio blocks to ensure the tx is packed
+        let ratio = self.tx_pool.machine().params().evm_transaction_block_ratio;
+        if self.config.dev_pack_tx_immediately {
+            // Try to pack and execute this new tx.
+            for _ in 0..(DEFERRED_STATE_EPOCH_COUNT + ratio - 1) {
+                let generated = self.generate_one_block(
+                    1, /* num_txs */
+                    self.sync
+                        .get_synchronization_graph()
+                        .verification_config
+                        .max_block_size_in_bytes,
+                )?;
+                loop {
+                    // Wait for the new block to be fully processed, so all
+                    // generated blocks form a chain for
+                    // `tx` to be executed.
+                    if self.consensus.best_block_hash() == generated {
+                        break;
+                    } else {
+                        thread::sleep(Duration::from_millis(10));
+                    }
+                }
+            }
+        }
         Ok(r)
     }
 

deny.toml

@@ -121,6 +121,7 @@ allow = [
 
     "GPL-3.0",
     "MPL-2.0",
+    "CDLA-Permissive-2.0",
     "Unicode-DFS-2016",
 
     # https://github.com/briansmith/ring/issues/902

integration_tests/readme.md

@@ -158,6 +158,8 @@ def network(framework_class: Type[ConfluxTestFramework], port_min: int, request:
 
 In the tests, you can start by directly using the `network` fixture for the test framework setup.
 
+You can check available pre-defined framework classes in [test_framework/framework_templates.py](./test_framework/framework_templates.py). It is recommended to directly use `DefaultDevFramework` or inherit your `framework_class` fixture from `DefaultDevFramework` if you don't care about block generation or p2p network setup. This template will setup a dev node, resulting in txs being packed and executed immediately.
+
 You can overwrite the `framework_class` fixture in the test file to customize the test framework setup. Check [cip137_test.py](./tests/cip137_test.py) for an example.
 
 #### Common Fixtures

integration_tests/test_framework/framework_templates.py

@@ -14,6 +14,22 @@ def setup_network(self):
         self.setup_nodes()
         self.rpc = RpcClient(self.nodes[0])
 
+
+# dev framework runs a single node in dev mode
+# when recieveing a tx, it will pack and execute it(generate 5 blocks) immediately
+class DefaultDevFramework(ConfluxTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+        self.conf_parameters["mode"] = "'dev'"
+
+    def setup_network(self):
+        self.setup_nodes()
+        self.rpc = RpcClient(self.nodes[0])
+
+    # do nothing because in dev mode
+    def _setup_w3_block_control_middleware(self):
+        pass
+
 class DefaultPoSFramework(ConfluxTestFramework):
     def set_test_params(self):
         self.num_nodes = 4        

integration_tests/test_framework/test_framework.py

@@ -375,8 +375,8 @@ def setup_w3(self):
 
         self.ew3.middleware_onion.add(SignAndSendRawMiddlewareBuilder.build(self.evm_secrets)) # type: ignore
         self.eth.default_account = self.evm_accounts[0].address
-        
-        class TestNodeMiddleware(ConfluxWeb3Middleware):
+
+        class MaxFeePerGasMiddleware(ConfluxWeb3Middleware):
             def request_processor(self, method: RPCEndpoint, params: Any) -> Any:
                 if method == RPC.cfx_sendRawTransaction or method == RPC.cfx_sendTransaction or method == "eth_sendRawTransaction" or method == "eth_sendTransaction":
                     client.node.wait_for_phase(["NormalSyncPhase"])
@@ -388,21 +388,31 @@ def request_processor(self, method: RPCEndpoint, params: Any) -> Any:
                         client.generate_blocks_to_state(num_txs=1)
                 return super().request_processor(method, params)
 
+        self.cw3.middleware_onion.add(MaxFeePerGasMiddleware)
+        self.ew3.middleware_onion.add(MaxFeePerGasMiddleware)
+
+        if self.options.trace_tx:
+            self.cw3.middleware_onion.add(AutoTraceMiddleware)
+            self.ew3.middleware_onion.add(AutoTraceMiddleware)
+
+        self._setup_w3_block_control_middleware()
+
+    # setup middleware to control the block generation to wait for tx receipt
+    def _setup_w3_block_control_middleware(self):
+        client = RpcClient(self.nodes[0])
+        log = self.log
+
+        class BlockControlMiddleware(ConfluxWeb3Middleware):
+
             def response_processor(self, method: RPCEndpoint, response: Any):
                 if method == RPC.cfx_getTransactionReceipt or method == "eth_getTransactionReceipt":
                     if "result" in response and response["result"] is None:
                         log.debug("Auto generate 5 blocks because did not get tx receipt")
                         client.generate_blocks_to_state(num_txs=1)  # why num_txs=1?
                 return response
-
-
-        self.cw3.middleware_onion.add(TestNodeMiddleware)
-        self.ew3.middleware_onion.add(TestNodeMiddleware)
-
-        if self.options.trace_tx:
-            self.cw3.middleware_onion.add(AutoTraceMiddleware)
-            self.ew3.middleware_onion.add(AutoTraceMiddleware)
 
+        self.cw3.middleware_onion.add(BlockControlMiddleware)
+        self.ew3.middleware_onion.add(BlockControlMiddleware)
 
     def add_nodes(self, num_nodes, genesis_nodes=None, rpchost=None, binary=None, auto_recovery=False,
                   recovery_timeout=30, is_consortium=True):

integration_tests/tests/cross_space/conftest.py

@@ -3,13 +3,14 @@
 import pytest
 
 from integration_tests.test_framework.test_framework import ConfluxTestFramework
+from integration_tests.test_framework.framework_templates import DefaultDevFramework
 
 
 @pytest.fixture(scope="module")
 def framework_class() -> Type[ConfluxTestFramework]:
-    class DefaultFramework(ConfluxTestFramework):
+    class DefaultFramework(DefaultDevFramework):
         def set_test_params(self):
-            self.num_nodes = 1
+            super().set_test_params()
             # self.conf_parameters["min_native_base_price"] = 10000
             self.conf_parameters["next_hardfork_transition_height"] = 1
             self.conf_parameters["next_hardfork_transition_number"] = 1

integration_tests/tests/cross_space/phantom_transaction_test.py

@@ -2,7 +2,7 @@
 from conflux_web3 import Web3
 import pytest
 from integration_tests.conflux.rpc import RpcClient
-from integration_tests.test_framework.test_framework import ConfluxTestFramework
+from integration_tests.test_framework.framework_templates import DefaultDevFramework
 from integration_tests.test_framework.util import assert_equal, assert_is_hex_string, assert_ne, load_contract_metadata
 from eth_utils import decode_hex, encode_hex as encode_hex_0x
 
@@ -16,9 +16,9 @@
 
 @pytest.fixture(scope="module")
 def framework_class():
-    class PhantomTransactionTestEnv(ConfluxTestFramework):
+    class PhantomTransactionTestEnv(DefaultDevFramework):
         def set_test_params(self):
-            self.num_nodes = 1
+            super().set_test_params()
             self.conf_parameters["chain_id"] = str(10)
             self.conf_parameters["evm_chain_id"] = str(11)
             self.conf_parameters["evm_transaction_block_ratio"] = str(1)