fix(ci): solve gas meter race condition in integration tests (#732)

* create new context

* add teardown

* add multistore to test setup so that cachecontext can be used

* add explicit subpool resets

* add longer timeout

* add mutex control

* lint

* update changelog

* update comments

* add separated builds

* rename attr

---------

Co-authored-by: Alex | Cosmos Labs <[email protected]>

Author: almk-dev

CHANGELOG.md

@@ -47,6 +47,7 @@
 - [\#730](https://github.com/cosmos/evm/pull/730) Fix panic if evm mempool not used.
 - [\#733](https://github.com/cosmos/evm/pull/733) Avoid rejecting tx with unsupported extension option for ExtensionOptionDynamicFeeTx.
 - [\#736](https://github.com/cosmos/evm/pull/736) Add InitEvmCoinInfo upgrade to avoid panic when denom is not registered.
+- [\#732](https://github.com/cosmos/evm/pull/732) Fix gas meter race condition in integration tests
 
 ### IMPROVEMENTS
 

Makefile

@@ -156,7 +156,7 @@ test-race: run-tests
 
 test-evmd: ARGS=-timeout=15m
 test-evmd:
-	@cd evmd && go test -race -tags=test -mod=readonly $(ARGS) $(EXTRA_ARGS) $(PACKAGES_EVMD)
+	@cd evmd && go test -count=1 -race -tags=test -mod=readonly $(ARGS) $(EXTRA_ARGS) $(PACKAGES_EVMD)
 
 test-unit-cover: ARGS=-timeout=15m -coverprofile=coverage.txt -covermode=atomic
 test-unit-cover: TEST_PACKAGES=$(PACKAGES_UNIT)
@@ -182,9 +182,9 @@ test-all:
 
 run-tests:
 ifneq (,$(shell which tparse 2>/dev/null))
-	go test -race -tags=test -mod=readonly -json $(ARGS) $(EXTRA_ARGS) $(TEST_PACKAGES) | tparse
+	go test -count=1 -race -tags=test -mod=readonly -json $(ARGS) $(EXTRA_ARGS) $(TEST_PACKAGES) | tparse
 else
-	go test -race -tags=test -mod=readonly $(ARGS) $(EXTRA_ARGS) $(TEST_PACKAGES)
+	go test -count=1 -race -tags=test -mod=readonly $(ARGS) $(EXTRA_ARGS) $(TEST_PACKAGES)
 endif
 
 # Use the old Apple linker to workaround broken xcode - https://github.com/golang/go/issues/65169

mempool/blockchain.go

@@ -44,6 +44,8 @@ type Blockchain struct {
 	previousHeaderHash common.Hash
 	latestCtx          sdk.Context
 	mu                 sync.RWMutex
+
+	testingCommitMu sync.RWMutex
 }
 
 // NewBlockchain creates a new Blockchain instance that bridges Cosmos SDK state with Ethereum mempools.
@@ -211,13 +213,34 @@ func (b *Blockchain) StateAt(hash common.Hash) (vm.StateDB, error) {
 		return nil, fmt.Errorf("failed to get latest context for StateAt: %w", err)
 	}
 
-	appHash := ctx.BlockHeader().AppHash
-	stateDB := statedb.New(ctx, b.vmKeeper, statedb.NewEmptyTxConfig())
+	// Create a cache context to isolate the StateDB from concurrent commits.
+	// This prevents race conditions when the background txpool reorg goroutine
+	// reads state while the main thread is committing new blocks.
+	cacheCtx, _ := ctx.CacheContext()
+	// Use an infinite gas meter to avoid tracking gas for read-only state queries
+	cacheCtx = cacheCtx.WithGasMeter(sdktypes.NewInfiniteGasMeter())
+
+	appHash := cacheCtx.BlockHeader().AppHash
+	stateDB := statedb.New(cacheCtx, b.vmKeeper, statedb.NewEmptyTxConfig())
 
 	b.logger.Debug("StateDB created successfully", "app_hash", common.Hash(appHash).Hex())
 	return stateDB, nil
 }
 
+// BeginCommit acquires an exclusive lock to prevent mempool state reads during Commit.
+// This avoids data races in the underlying storage (e.g., IAVL) when tests run with -race.
+func (b *Blockchain) BeginCommit() { b.testingCommitMu.Lock() }
+
+// EndCommit releases the exclusive lock acquired by BeginCommit.
+func (b *Blockchain) EndCommit() { b.testingCommitMu.Unlock() }
+
+// BeginRead acquires a shared lock for background readers (e.g., txpool reorg).
+// This enables optional coordination with Commit without importing the type.
+func (b *Blockchain) BeginRead() { b.testingCommitMu.RLock() }
+
+// EndRead releases the shared read lock acquired by BeginRead.
+func (b *Blockchain) EndRead() { b.testingCommitMu.RUnlock() }
+
 func (b *Blockchain) getPreviousHeaderHash() common.Hash {
 	b.mu.RLock()
 	defer b.mu.RUnlock()

mempool/blockchain_test.go

@@ -21,12 +21,18 @@ import (
 	"cosmossdk.io/log"
 	storetypes "cosmossdk.io/store/types"
 
+	"github.com/cosmos/cosmos-sdk/testutil"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 )
 
-// createMockContext creates a basic mock context for testing
+// createMockContext creates a basic mock context for testing with multistore
 func createMockContext() sdk.Context {
-	return sdk.Context{}.
+	// crate ctx with multistore so that CacheContext() can be used
+	storeKey := storetypes.NewKVStoreKey("test")
+	transientKey := storetypes.NewTransientStoreKey("transient_test")
+	ctx := testutil.DefaultContext(storeKey, transientKey)
+
+	return ctx.
 		WithBlockTime(time.Now()).
 		WithBlockHeader(cmtproto.Header{AppHash: []byte("00000000000000000000000000000000")}).
 		WithBlockHeight(1)

mempool/txpool/legacypool/commit_lock_default.go

@@ -0,0 +1,6 @@
+//go:build !race
+
+package legacypool
+
+// beginCommitRead is a no-op in non-race builds to avoid unnecessary overhead.
+func beginCommitRead(_ any) func() { return func() {} }

mempool/txpool/legacypool/commit_lock_race.go

@@ -0,0 +1,21 @@
+//go:build race
+
+package legacypool
+
+// commitAwareChain is implemented by chains that expose optional commit RW locks.
+// These are used to coordinate background readers with Commit in tests/race builds.
+type commitAwareChain interface {
+	BeginRead()
+	EndRead()
+}
+
+// beginCommitRead acquires a shared read lock when available and returns a
+// function that releases it. In race/test builds this coordinates with Commit
+// to avoid read/write races in storage. In non-race builds this is a no-op.
+func beginCommitRead(chain any) func() {
+	if ca, ok := chain.(commitAwareChain); ok {
+		ca.BeginRead()
+		return func() { ca.EndRead() }
+	}
+	return func() {}
+}

mempool/txpool/legacypool/legacypool.go

@@ -1273,6 +1273,9 @@ func (pool *LegacyPool) runReorg(done chan struct{}, reset *txpoolResetRequest,
 	defer close(done)
 
 	var promoteAddrs []common.Address
+	// Optionally acquire a shared read lock to coordinate with Commit in tests.
+	unlock := beginCommitRead(any(pool.chain))
+	defer unlock()
 	if dirtyAccounts != nil && reset == nil {
 		// Only dirty accounts need to be promoted, unless we're resetting.
 		// For resets, all addresses in the tx queue will be promoted and

tests/integration/mempool/test_helpers.go

@@ -4,7 +4,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"math/big"
-	"time"
 
 	abci "github.com/cometbft/cometbft/abci/types"
 	"github.com/cometbft/cometbft/crypto/tmhash"
@@ -205,17 +204,20 @@ func (s *IntegrationTestSuite) notifyNewBlockToMempool() {
 	// Access the underlying blockchain interface from the EVM mempool
 	if evmMempoolCast, ok := evmMempool.(*evmmempool.ExperimentalEVMMempool); ok {
 		blockchain := evmMempoolCast.GetBlockchain()
+		txPool := evmMempoolCast.GetTxPool()
 
-		// Trigger a new block notification
-		// This sends a ChainHeadEvent that the mempool subscribes to.
-		// The TxPool's event loop receives this and calls Reset() for each subpool,
-		// which naturally removes committed transactions via demoteUnexecutables().
+		// Get the current and new block headers for reset
+		oldHead := blockchain.CurrentBlock()
+
+		// Trigger a new block notification to update the blockchain state
 		blockchain.NotifyNewBlock()
+		newHead := blockchain.CurrentBlock()
 
-		// The ChainHeadEvent is processed asynchronously, so we need to wait a bit
-		// for the event to be processed and the reset to complete.
-		// In integration tests, this might need a small delay to ensure the event
-		// is processed before we check the mempool state.
-		time.Sleep(100 * time.Millisecond)
+		// Directly call Reset on each subpool to ensure synchronous completion
+		// This prevents race conditions by waiting for the reset to complete
+		// before continuing with test assertions
+		for _, subpool := range txPool.Subpools {
+			subpool.Reset(oldHead, newHead)
+		}
 	}
 }

tests/integration/mempool/test_mempool_integration.go

@@ -134,6 +134,8 @@ func (s *IntegrationTestSuite) TestMempoolInsert() {
 
 	for _, tc := range testCases {
 		s.Run(tc.name, func() {
+			// Clean up previous test's resources before resetting
+			s.TearDownTest()
 			// Reset test setup to ensure clean state
 			s.SetupTest()
 
@@ -220,6 +222,8 @@ func (s *IntegrationTestSuite) TestMempoolRemove() {
 
 	for _, tc := range testCases {
 		s.Run(tc.name, func() {
+			// Clean up previous test's resources before resetting
+			s.TearDownTest()
 			// Reset test setup to ensure clean state
 			s.SetupTest()
 
@@ -304,6 +308,8 @@ func (s *IntegrationTestSuite) TestMempoolSelect() {
 
 	for _, tc := range testCases {
 		s.Run(tc.name, func() {
+			// Clean up previous test's resources before resetting
+			s.TearDownTest()
 			// Reset test setup to ensure clean state
 			s.SetupTest()
 
@@ -419,6 +425,8 @@ func (s *IntegrationTestSuite) TestMempoolIterator() {
 
 	for _, tc := range testCases {
 		s.Run(tc.name, func() {
+			// Clean up previous test's resources before resetting
+			s.TearDownTest()
 			// Reset test setup to ensure clean state
 			s.SetupTest()
 
@@ -805,6 +813,8 @@ func (s *IntegrationTestSuite) TestTransactionOrdering() {
 
 	for _, tc := range testCases {
 		s.Run(tc.name, func() {
+			// Clean up previous test's resources before resetting
+			s.TearDownTest()
 			// Reset test setup to ensure clean state
 			s.SetupTest()
 

tests/integration/mempool/test_mempool_integration_abci.go

@@ -166,6 +166,8 @@ func (s *IntegrationTestSuite) TestTransactionOrderingWithABCIMethodCalls() {
 
 	for _, tc := range testCases {
 		s.Run(tc.name, func() {
+			// Clean up previous test's resources before resetting
+			s.TearDownTest()
 			// Reset test setup to ensure clean state
 			s.SetupTest()
 
@@ -372,6 +374,8 @@ func (s *IntegrationTestSuite) TestNonceGappedEVMTransactionsWithABCIMethodCalls
 
 	for _, tc := range testCases {
 		s.Run(tc.name, func() {
+			// Clean up previous test's resources before resetting
+			s.TearDownTest()
 			s.SetupTest()
 
 			txs, expTxHashes := tc.setupTxs()
@@ -479,6 +483,8 @@ func (s *IntegrationTestSuite) TestCheckTxHandlerForCommittedAndLowerNonceTxs()
 
 	for _, tc := range testCases {
 		s.Run(tc.name, func() {
+			// Clean up previous test's resources before resetting
+			s.TearDownTest()
 			// Reset test setup to ensure clean state
 			s.SetupTest()