make batch gas costs part of mel consensus

Author: ganeshvanahalli

arbnode/mel/extraction/message_extraction_function.go

@@ -151,16 +151,6 @@ func extractMessagesImpl(
 		if delayed.Message.Header.Kind == arbostypes.L1MessageType_BatchPostingReport || delayed.Message.Header.Kind == arbostypes.L1MessageType_Initialize { // Let's consider the init message as a batch posting report, since it is seen as a batch as well, we can later ignore filling its batchGasCost anyway
 			batchPostingReports = append(batchPostingReports, delayed)
 		}
-		if err = state.AccumulateDelayedMessage(delayed); err != nil {
-			return nil, nil, nil, nil, err
-		}
-		state.DelayedMessagesSeen += 1
-	}
-	if len(delayedMessages) > 0 {
-		// Only need to calculate partials once, after all the delayed messages are `seen`
-		if err := state.GenerateDelayedMessagesSeenMerklePartialsAndRoot(); err != nil {
-			return nil, nil, nil, nil, err
-		}
 	}
 
 	// Batch posting reports are included in the same transaction as a batch, so there should
@@ -175,6 +165,7 @@ func extractMessagesImpl(
 
 	var batchMetas []*mel.BatchMetadata
 	var messages []*arbostypes.MessageWithMetadata
+	var serializedBatches [][]byte
 	for i, batch := range batches {
 		batchTx := batchTxs[i]
 		serialized, err := serialize(
@@ -209,7 +200,26 @@ func extractMessagesImpl(
 		} else if !(inputState.DelayedMessagesSeen == 0 && i == 0 && delayedMessages[i] == batchPostReport) {
 			return nil, nil, nil, nil, errors.New("encountered initialize message that is not the first delayed message and the first batch ")
 		}
+		serializedBatches = append(serializedBatches, serialized)
+	}
+
+	// Update the delayed message accumulator in the MEL state.
+	for _, delayed := range delayedMessages {
+		if err = state.AccumulateDelayedMessage(delayed); err != nil {
+			return nil, nil, nil, nil, err
+		}
+		state.DelayedMessagesSeen += 1
+	}
+	if len(delayedMessages) > 0 {
+		// Only need to calculate partials once, after all the delayed messages are `seen`
+		if err := state.GenerateDelayedMessagesSeenMerklePartialsAndRoot(); err != nil {
+			return nil, nil, nil, nil, err
+		}
+	}
 
+	// Extract L2 messages from batches
+	for i, batch := range batches {
+		serialized := serializedBatches[i]
 		rawSequencerMsg, err := parseSequencerMessage(
 			ctx,
 			batch.SequenceNumber,

arbnode/mel/messages.go

@@ -59,24 +59,13 @@ func (m *DelayedInboxMessage) AfterInboxAcc() common.Hash {
 }
 
 func (m *DelayedInboxMessage) Hash() common.Hash {
-	encoded, err := rlp.EncodeToBytes(m.WithMELRelevantFields())
+	encoded, err := rlp.EncodeToBytes(m)
 	if err != nil {
 		panic(err)
 	}
 	return crypto.Keccak256Hash(encoded)
 }
 
-func (m *DelayedInboxMessage) WithMELRelevantFields() *DelayedInboxMessage {
-	return &DelayedInboxMessage{
-		BlockHash: m.BlockHash,
-		Message: &arbostypes.L1IncomingMessage{
-			Header: m.Message.Header,
-			L2msg:  m.Message.L2msg,
-		},
-		ParentChainBlockNumber: m.ParentChainBlockNumber,
-	}
-}
-
 type BatchMetadata struct {
 	Accumulator         common.Hash
 	MessageCount        arbutil.MessageIndex

arbnode/mel/recording/delayed_msg_database.go

@@ -63,7 +63,7 @@ func (r *DelayedMsgDatabase) ReadDelayedMessage(ctx context.Context, state *mel.
 	if err != nil {
 		return nil, err
 	}
-	delayedMsgBytes, err := rlp.EncodeToBytes(delayed.WithMELRelevantFields())
+	delayedMsgBytes, err := rlp.EncodeToBytes(delayed)
 	if err != nil {
 		return nil, err
 	}

arbnode/mel/state.go

@@ -168,7 +168,7 @@ func (s *State) AccumulateMessage(msg *arbostypes.MessageWithMetadata) error {
 			}
 		}
 	}
-	msgBytes, err := rlp.EncodeToBytes(msg.WithMELRelevantFields())
+	msgBytes, err := rlp.EncodeToBytes(msg)
 	if err != nil {
 		return err
 	}

arbos/arbostypes/messagewithmeta.go

@@ -16,23 +16,13 @@ type MessageWithMetadata struct {
 }
 
 func (m *MessageWithMetadata) Hash() common.Hash {
-	encoded, err := rlp.EncodeToBytes(m.WithMELRelevantFields())
+	encoded, err := rlp.EncodeToBytes(m)
 	if err != nil {
 		panic(err)
 	}
 	return crypto.Keccak256Hash(encoded)
 }
 
-func (m *MessageWithMetadata) WithMELRelevantFields() *MessageWithMetadata {
-	return &MessageWithMetadata{
-		Message: &L1IncomingMessage{
-			Header: m.Message.Header,
-			L2msg:  m.Message.L2msg,
-		},
-		DelayedMessagesRead: m.DelayedMessagesRead,
-	}
-}
-
 // lint:require-exhaustive-initialization
 type MessageWithMetadataAndBlockInfo struct {
 	MessageWithMeta MessageWithMetadata

staker/block_validation_entry_creator.go

@@ -17,19 +17,16 @@ import (
 type MELEnabledValidationEntryCreator struct {
 	melValidator MELValidatorInterface
 	txStreamer   TransactionStreamerInterface
-	melRunner    MELRunnerInterface
 }
 
 // NewMELEnabledValidationEntryCreator creates a new instance of MELEnabledValidationEntryCreator.
 func NewMELEnabledValidationEntryCreator(
 	melValidator MELValidatorInterface,
 	txStreamer TransactionStreamerInterface,
-	melRunner MELRunnerInterface,
 ) *MELEnabledValidationEntryCreator {
 	return &MELEnabledValidationEntryCreator{
 		melValidator: melValidator,
 		txStreamer:   txStreamer,
-		melRunner:    melRunner,
 	}
 }
 
@@ -113,7 +110,7 @@ func (m *MELEnabledValidationEntryCreator) CreateBlockValidationEntry(
 	created = true
 	return &validationEntry{
 		Stage:       ReadyForRecord,
-		Pos:         arbutil.MessageIndex(position),
+		Pos:         position,
 		Start:       startGs,
 		End:         endGlobalState,
 		msg:         msg,

staker/block_validator.go

@@ -383,7 +383,6 @@ func NewBlockValidator(
 		validationEntryCreator := NewMELEnabledValidationEntryCreator(
 			melValidator,
 			streamer,
-			melRunner,
 		)
 		ret.melEnabledEntryCreator = validationEntryCreator
 	}

system_tests/message_extraction_layer_test.go

@@ -289,19 +289,15 @@ func TestMessageExtractionLayer_SequencerBatchMessageEquivalence_Blobs(t *testin
 			numDelayedMessages,
 		)
 	}
+
 	// Start from 1 to ignore the init message.
-	readHelperState := &mel.State{DelayedMessagesSeen: 1}
-	readHelperState.SetDelayedMessageBacklog(&mel.DelayedMessageBacklog{})
-	readHelperState.SetReadCountFromBacklog(numDelayedMessages) // skip checking against accumulator- not the purpose of this test
 	for i := uint64(1); i < numDelayedMessages; i++ {
 		fromInboxTracker, err := builder.L2.ConsensusNode.InboxTracker.GetDelayedMessage(ctx, i)
 		Require(t, err)
-		Require(t, readHelperState.AccumulateDelayedMessage(&mel.DelayedInboxMessage{Message: fromInboxTracker}))
-		readHelperState.DelayedMessagesSeen++
-		fromMelDB, err := melDB.ReadDelayedMessage(ctx, readHelperState, i)
+		delayedMsg, err := extractor.GetDelayedMessage(i)
 		Require(t, err)
 		// Check the messages we extracted from MEL and the inbox tracker are the same.
-		if !fromInboxTracker.Equals(fromMelDB.Message) {
+		if !fromInboxTracker.Equals(delayedMsg.Message) {
 			t.Fatal("Messages from MEL and inbox tracker do not match")
 		}
 	}
@@ -386,18 +382,13 @@ func TestMessageExtractionLayer_DelayedMessageEquivalence_Simple(t *testing.T) {
 	}
 
 	// Start from 1 to ignore the init message.
-	readHelperState := &mel.State{DelayedMessagesSeen: 1}
-	readHelperState.SetDelayedMessageBacklog(&mel.DelayedMessageBacklog{})
-	readHelperState.SetReadCountFromBacklog(numDelayedMessages) // skip checking against accumulator- not the purpose of this test
 	for i := uint64(1); i < numDelayedMessages; i++ {
 		fromInboxTracker, err := builder.L2.ConsensusNode.InboxTracker.GetDelayedMessage(ctx, i)
 		Require(t, err)
-		Require(t, readHelperState.AccumulateDelayedMessage(&mel.DelayedInboxMessage{Message: fromInboxTracker}))
-		readHelperState.DelayedMessagesSeen++
-		fromMelDB, err := melDB.ReadDelayedMessage(ctx, readHelperState, i)
+		delayedMsg, err := extractor.GetDelayedMessage(i)
 		Require(t, err)
 		// Check the messages we extracted from MEL and the inbox tracker are the same.
-		if !fromInboxTracker.Equals(fromMelDB.Message) {
+		if !fromInboxTracker.Equals(delayedMsg.Message) {
 			t.Fatal("Messages from MEL and inbox tracker do not match")
 		}
 	}

system_tests/message_extraction_layer_validation_test.go

@@ -2,6 +2,7 @@ package arbtest
 
 import (
 	"context"
+	"fmt"
 	"math/big"
 	"testing"
 	"time"
@@ -83,7 +84,7 @@ func TestUnifiedReplayBinary_ValidationOfMELAndBlockExecution(t *testing.T) {
 		latestValidatedState: endMELState,
 	}
 	entryCreator := staker.NewMELEnabledValidationEntryCreator(
-		mockMElV, builder.L2.ConsensusNode.TxStreamer, builder.L2.ConsensusNode.MessageExtractor,
+		mockMElV, builder.L2.ConsensusNode.TxStreamer,
 	)
 	Require(t, err)
 
@@ -144,6 +145,7 @@ func TestUnifiedReplayBinary_ValidationOfMELAndBlockExecution(t *testing.T) {
 			t.Fatalf("Expected to compute %s block hash but computed %s", blockValidatorEntry.End.BlockHash, lastStep.GlobalState.BlockHash)
 		}
 		t.Logf("Validated block execution of message index %+v\n", lastStep.GlobalState)
+		fmt.Println("Validated block execution of message index ", lastStep.GlobalState.PosInBatch)
 
 		// Update the computed global state to the one just computed by Arbitrator.
 		computedGlobalState = lastStep.GlobalState