Fixes synchronized state

common/common.go

@@ -454,6 +454,7 @@ func CreateLastExecutionResultFromPrevHeader(prevHeader data.HeaderHandler, prev
 				HeaderHash:  prevHeaderHash,
 				HeaderNonce: prevHeader.GetNonce(),
 				HeaderRound: prevHeader.GetRound(),
+				HeaderEpoch: prevHeader.GetEpoch(),
 				RootHash:    prevHeader.GetRootHash(),
 				GasUsed:     0, // we don't have this information in previous header
 			},
@@ -472,6 +473,7 @@ func CreateLastExecutionResultFromPrevHeader(prevHeader data.HeaderHandler, prev
 				HeaderHash:  prevHeaderHash,
 				HeaderNonce: prevMetaHeader.GetNonce(),
 				HeaderRound: prevMetaHeader.GetRound(),
+				HeaderEpoch: prevMetaHeader.GetEpoch(),
 				RootHash:    prevMetaHeader.GetRootHash(),
 				GasUsed:     0, // we don't have this information in previous header
 			},

integrationTests/chainSimulator/staking/common.go

@@ -6,12 +6,13 @@ import (
 	"testing"
 
 	"github.com/multiversx/mx-chain-core-go/core"
+	"github.com/stretchr/testify/require"
+
 	chainSimulatorIntegrationTests "github.com/multiversx/mx-chain-go/integrationTests/chainSimulator"
 	"github.com/multiversx/mx-chain-go/node/chainSimulator/dtos"
 	chainSimulatorProcess "github.com/multiversx/mx-chain-go/node/chainSimulator/process"
 	"github.com/multiversx/mx-chain-go/process"
 	"github.com/multiversx/mx-chain-go/vm"
-	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -22,7 +23,7 @@ const (
 	// GasLimitForUnBond the const for the gas limit value for the unBond operation
 	GasLimitForUnBond = 12_000_000
 	// MaxNumOfBlockToGenerateWhenExecutingTx the const for the maximum number of block to generate when execute a transaction
-	MaxNumOfBlockToGenerateWhenExecutingTx = 7
+	MaxNumOfBlockToGenerateWhenExecutingTx = 10
 
 	// QueuedStatus the const for the queued status of a validators
 	QueuedStatus = "queued"
@@ -37,7 +38,7 @@ const (
 )
 
 var (
-	//InitialDelegationValue the variable for the initial delegation value
+	// InitialDelegationValue the variable for the initial delegation value
 	InitialDelegationValue = big.NewInt(0).Mul(chainSimulatorIntegrationTests.OneEGLD, big.NewInt(1250))
 )
 

process/common.go

@@ -1155,6 +1155,7 @@ func CreateLastExecutionResultInfoFromExecutionResult(notarizedInRound uint64, l
 				HeaderHash:  lastExecResult.GetHeaderHash(),
 				HeaderNonce: lastExecResult.GetHeaderNonce(),
 				HeaderRound: lastExecResult.GetHeaderRound(),
+				HeaderEpoch: lastExecResult.GetHeaderEpoch(),
 				RootHash:    lastExecResult.GetRootHash(),
 				GasUsed:     lastExecResult.GetGasUsed(),
 			},
@@ -1173,6 +1174,7 @@ func CreateLastExecutionResultInfoFromExecutionResult(notarizedInRound uint64, l
 				HeaderHash:  lastMetaExecResult.GetHeaderHash(),
 				HeaderNonce: lastMetaExecResult.GetHeaderNonce(),
 				HeaderRound: lastMetaExecResult.GetHeaderRound(),
+				HeaderEpoch: lastMetaExecResult.GetHeaderEpoch(),
 				RootHash:    lastMetaExecResult.GetRootHash(),
 				GasUsed:     lastMetaExecResult.GetGasUsed(),
 			},

process/missingData/missingDataResolver.go

@@ -8,13 +8,16 @@ import (
 	"github.com/multiversx/mx-chain-core-go/core/check"
 	"github.com/multiversx/mx-chain-core-go/data"
 	"github.com/multiversx/mx-chain-core-go/data/block"
+	logger "github.com/multiversx/mx-chain-logger-go"
 
 	"github.com/multiversx/mx-chain-go/dataRetriever"
 	"github.com/multiversx/mx-chain-go/process"
 )
 
 const checkMissingDataStep = 10 * time.Millisecond
 
+var log = logger.GetOrCreate("missingDataResolver")
+
 // ResolverArgs holds the arguments needed to create a Resolver
 type ResolverArgs struct {
 	HeadersPool        dataRetriever.HeadersPool
@@ -219,15 +222,35 @@ func (r *Resolver) WaitForMissingData(timeout time.Duration) error {
 		return haveTime
 	}
 
+	err := r.blockDataRequester.IsDataPreparedForProcessing(stepHaveTime(checkMissingDataStep))
 	for {
-		err := r.blockDataRequester.IsDataPreparedForProcessing(stepHaveTime(checkMissingDataStep))
 		if r.allDataReceived() && err == nil {
+			log.Debug("missingDataResolver.WaitForMissingData: all missing data received")
 			return nil
 		}
 
 		if time.Now().After(waitDeadline) {
+			r.mutHeaders.RLock()
+			numMissingHeaders := len(r.missingHeaders)
+			r.mutHeaders.RUnlock()
+
+			r.mutProofs.RLock()
+			numMissingProofs := len(r.missingProofs)
+			r.mutProofs.RUnlock()
+
+			log.Debug("missingDataResolver.WaitForMissingData: timeout reached while waiting for missing data",
+				"missingHeaders", numMissingHeaders,
+				"missingProofs", numMissingProofs,
+				"IsDataPreparedError", err)
+
 			return process.ErrTimeIsOut
 		}
+
+		if err != nil {
+			err = r.blockDataRequester.IsDataPreparedForProcessing(stepHaveTime(checkMissingDataStep))
+		}
+
+		time.Sleep(checkMissingDataStep)
 	}
 }
 

process/missingData/missingDataResolver_test.go

@@ -567,7 +567,11 @@ func TestResolver_RequestMissingMetaHeadersBlocking(t *testing.T) {
 		require.Equal(t, process.ErrTimeIsOut, err)
 		require.False(t, mdr.allHeadersReceived())
 		require.False(t, mdr.allProofsReceived())
+
+		mutRequestedData.Lock()
+		defer mutRequestedData.Unlock()
 		require.Equal(t, len(expectedMetaHeadersRequested), len(requestedMetaHeaders))
+
 		for i := 0; i < len(expectedMetaHeadersRequested); i++ {
 			require.Contains(t, requestedMetaHeaders, expectedMetaHeadersRequested[i])
 		}
@@ -729,8 +733,14 @@ func TestResolver_RequestMissingShardHeadersBlocking(t *testing.T) {
 		require.Equal(t, process.ErrTimeIsOut, err)
 		require.False(t, mdr.allHeadersReceived())
 		require.False(t, mdr.allProofsReceived())
+
+		mutRequestedData.Lock()
 		require.ElementsMatch(t, expectedShardHeadersRequested, requestedShardHeaders)
+		mutRequestedData.Unlock()
+
+		mutRequestedProofs.Lock()
 		require.ElementsMatch(t, expectedShardHeadersRequested, requestedShardProofs)
+		mutRequestedProofs.Unlock()
 	})
 
 	t.Run("requesting missing shard headers with nonce gaps", func(t *testing.T) {
@@ -864,16 +874,26 @@ func TestResolver_RequestMissingShardHeadersBlocking(t *testing.T) {
 		require.Equal(t, process.ErrTimeIsOut, err)
 		require.False(t, mdr.allHeadersReceived())
 		require.False(t, mdr.allProofsReceived())
+
+		mutRequestedData.Lock()
 		require.ElementsMatch(t, expectedShardHeadersRequested, requestedShardHeaders)
+		mutRequestedData.Unlock()
+
+		mutRequestedProofs.Lock()
 		require.ElementsMatch(t, expectedShardHeadersRequested, requestedShardProofs)
+		mutRequestedProofs.Unlock()
 
+		mutRequestedProofs.Lock()
 		require.Len(t, requestProofNonces, 2)
 		require.ElementsMatch(t, requestProofNonces[1], []uint64{4})
 		require.ElementsMatch(t, requestProofNonces[2], []uint64{3, 4})
+		mutRequestedProofs.Unlock()
 
+		mutRequestedData.Lock()
 		require.Len(t, requestShardHeaderNonces, 2)
 		require.ElementsMatch(t, requestShardHeaderNonces[1], []uint64{4})
 		require.ElementsMatch(t, requestShardHeaderNonces[2], []uint64{3, 4})
+		mutRequestedData.Unlock()
 	})
 
 	t.Run("request missing shard headers and proofs, all received", func(t *testing.T) {

process/sync/baseSync.go

@@ -21,9 +21,10 @@ import (
 	"github.com/multiversx/mx-chain-core-go/data/typeConverters"
 	"github.com/multiversx/mx-chain-core-go/hashing"
 	"github.com/multiversx/mx-chain-core-go/marshal"
-	"github.com/multiversx/mx-chain-go/process/asyncExecution/queue"
 	logger "github.com/multiversx/mx-chain-logger-go"
 
+	"github.com/multiversx/mx-chain-go/process/asyncExecution/queue"
+
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/consensus"
 	"github.com/multiversx/mx-chain-go/dataRetriever"
@@ -1051,42 +1052,12 @@ func (boot *baseBootstrap) prepareForSyncIfNeeded(syncingNonce uint64) error {
 
 	currentHeader := boot.getCurrentBlock()
 	currentHeaderHash := boot.getCurrentBlockHash()
-	lastExecResult, err := process.GetPrevBlockLastExecutionResult(boot.chainHandler)
-	if err != nil {
-		return err
-	}
-
-	lastExecResultsHandler, err := common.ExtractBaseExecutionResultHandler(lastExecResult)
+	lastExecutionResultHeaderNonce, err := boot.getExecutionResultHeaderNonceForSyncStart(syncingNonce, currentHeader, currentHeaderHash)
 	if err != nil {
 		return err
 	}
 
-	log.Debug("prepareForSyncIfNeeded",
-		"syncingNonce", syncingNonce,
-		"currHeader nonce", currentHeader.GetNonce(),
-		"currHeader hash", currentHeaderHash,
-		"lastExecRes nonce", lastExecResultsHandler.GetHeaderNonce(),
-		"lastExecRes hash", lastExecResultsHandler.GetHeaderHash(),
-		"lastExecRes rootHash", lastExecResultsHandler.GetRootHash(),
-	)
-
-	lastExecutedHash := lastExecResultsHandler.GetHeaderHash()
-	lastExecutedHeader, err := boot.getHeader(lastExecutedHash)
-	if err != nil {
-		return err
-	}
-
-	rootHash := lastExecResultsHandler.GetRootHash()
-
-	txPool := boot.poolsHolder.Transactions()
-	err = txPool.OnExecutedBlock(lastExecutedHeader, rootHash)
-	if err != nil {
-		txPool.ResetTracker()
-		return err
-	}
-
-	lastExecutedNonce := lastExecutedHeader.GetNonce()
-	if syncingNonce == lastExecutedNonce+2 {
+	if syncingNonce == lastExecutionResultHeaderNonce+2 {
 		// the ideal/most common case:
 		// the previous block was already processed, its nonce should have been syncingNonce-1,
 		// and it notarized its previous block, which should have had nonce equal to syncingNonce-2
@@ -1120,7 +1091,7 @@ func (boot *baseBootstrap) prepareForSyncIfNeeded(syncingNonce uint64) error {
 
 	// if there are multiple headers in between the syncing header and the last one executed,
 	// add them into the queue and pool
-	for i := lastExecutedNonce + 1; i < syncingNonce; i++ {
+	for i := lastExecutionResultHeaderNonce + 1; i < syncingNonce; i++ {
 		hdr, hdrHash, errGetHdr := boot.getHeaderFromPoolWithNonce(i)
 		if errGetHdr != nil {
 			log.Debug("prepareForSyncIfNeeded: failed to get header with nonce", "nonce", i, "error", errGetHdr)
@@ -1132,7 +1103,7 @@ func (boot *baseBootstrap) prepareForSyncIfNeeded(syncingNonce uint64) error {
 			return errGetBody
 		}
 
-		err := boot.saveProposedTxsToPool(hdr, body)
+		err = boot.saveProposedTxsToPool(hdr, body)
 		if err != nil {
 			return err
 		}
@@ -1160,6 +1131,58 @@ func (boot *baseBootstrap) prepareForSyncIfNeeded(syncingNonce uint64) error {
 	return nil
 }
 
+func (boot *baseBootstrap) getExecutionResultHeaderNonceForSyncStart(
+	syncingNonce uint64,
+	currentHeader data.HeaderHandler,
+	currentHeaderHash []byte,
+) (uint64, error) {
+	lastNotarizedExecResult, err := process.GetPrevBlockLastExecutionResult(boot.chainHandler)
+	if err != nil {
+		return 0, err
+	}
+
+	lastNotarizedExecResultsHandler, err := common.ExtractBaseExecutionResultHandler(lastNotarizedExecResult)
+	if err != nil {
+		return 0, err
+	}
+
+	log.Debug("getExecutionResultHeaderNonceForSyncStart",
+		"syncingNonce", syncingNonce,
+		"currHeader nonce", currentHeader.GetNonce(),
+		"currHeader hash", currentHeaderHash,
+		"lastNotarizedExecRes nonce", lastNotarizedExecResultsHandler.GetHeaderNonce(),
+		"lastNotarizedExecRes hash", lastNotarizedExecResultsHandler.GetHeaderHash(),
+		"lastNotarizedExecRes rootHash", lastNotarizedExecResultsHandler.GetRootHash(),
+	)
+
+	lastNotarizedExecutedHash := lastNotarizedExecResultsHandler.GetHeaderHash()
+	lastNotarizedExecutedHeader, err := boot.getHeader(lastNotarizedExecutedHash)
+	if err != nil {
+		return 0, err
+	}
+
+	rootHash := lastNotarizedExecResultsHandler.GetRootHash()
+
+	txPool := boot.poolsHolder.Transactions()
+	err = txPool.OnExecutedBlock(lastNotarizedExecutedHeader, rootHash)
+	if err != nil {
+		txPool.ResetTracker()
+		return 0, err
+	}
+
+	lastExecutionResultNonce := lastNotarizedExecutedHeader.GetNonce()
+
+	// in case there is a more recent execution result available, use it
+	lastExecutionResult := boot.chainHandler.GetLastExecutionResult()
+	if !check.IfNil(lastExecutionResult) && lastExecutionResult.GetHeaderNonce() > lastExecutionResultNonce {
+		lastExecutionResultNonce = lastExecutionResult.GetHeaderNonce()
+	}
+
+	log.Debug("getExecutionResultHeaderNonceForSyncStart", "lastExecutionResultNonce", lastExecutionResultNonce)
+
+	return lastExecutionResultNonce, nil
+}
+
 func (boot *baseBootstrap) saveProposedTxsToPool(
 	header data.HeaderHandler,
 	body data.BodyHandler,