[MEL] - Implement L2 messages accumulation and introduce MessageReader to extract messages from preimages

arbnode/mel/extraction/batch_lookup.go

@@ -60,6 +60,12 @@ func ParseBatchesFromBlock(
 			return nil, nil, fmt.Errorf("error fetching tx by hash: %v in ParseBatchesFromBlock: %w ", log.TxHash, err)
 		}
 
+		// Record this log for MEL validation. This is a very cheap operation in native mode
+		// and is optimized for recording mode as well.
+		if _, err := logsFetcher.LogsForTxIndex(ctx, parentChainHeader.Hash(), log.TxIndex); err != nil {
+			return nil, nil, fmt.Errorf("error recording relevant logs: %w", err)
+		}
+
 		batch := &mel.SequencerInboxBatch{
 			BlockHash:              log.BlockHash,
 			ParentChainBlockNumber: log.BlockNumber,

arbnode/mel/extraction/batch_lookup_test.go

@@ -246,6 +246,7 @@ func Test_parseBatchesFromBlock(t *testing.T) {
 			receipts,
 			trie.NewStackTrie(nil),
 		)
+		receipt.BlockHash = block.Hash()
 		blockLogsFetcher := newMockBlockLogsFetcher(receipts)
 		eventUnpacker := &mockEventUnpacker{
 			events: []*bridgegen.SequencerInboxSequencerBatchDelivered{event},
@@ -324,6 +325,7 @@ func Test_parseBatchesFromBlock_outOfOrderBatches(t *testing.T) {
 		receipts,
 		trie.NewStackTrie(nil),
 	)
+	receipt.BlockHash = block.Hash()
 	blockLogsFetcher := newMockBlockLogsFetcher(receipts)
 	eventUnpacker := &mockEventUnpacker{
 		events: []*bridgegen.SequencerInboxSequencerBatchDelivered{

arbnode/mel/extraction/delayed_message_lookup.go

@@ -39,6 +39,11 @@ func parseDelayedMessagesFromBlock(
 		// On Arbitrum One, this is the bridge contract which emits a MessageDelivered event.
 		if log.Address == melState.DelayedMessagePostingTargetAddress {
 			relevantLogs = append(relevantLogs, log)
+			// Record this log for MEL validation. This is a very cheap operation in native mode
+			// and is optimized for recording mode as well.
+			if _, err := logsFetcher.LogsForTxIndex(ctx, parentChainHeader.Hash(), log.TxIndex); err != nil {
+				return nil, fmt.Errorf("error recording relevant logs: %w", err)
+			}
 		}
 	}
 	if len(relevantLogs) > 0 {
@@ -78,6 +83,11 @@ func parseDelayedMessagesFromBlock(
 			return nil, err
 		}
 		messageData[common.BigToHash(msgNum)] = msg
+		// Record this log for MEL validation. This is a very cheap operation in native mode
+		// and is optimized for recording mode as well.
+		if _, err := logsFetcher.LogsForTxIndex(ctx, parentChainHeader.Hash(), inboxMsgLog.TxIndex); err != nil {
+			return nil, fmt.Errorf("error recording relevant logs: %w", err)
+		}
 	}
 	for i, parsedLog := range messageDeliveredEvents {
 		msgKey := common.BigToHash(parsedLog.MessageIndex)

arbnode/mel/extraction/delayed_message_lookup_test.go

@@ -124,6 +124,7 @@ func Test_parseDelayedMessagesFromBlock(t *testing.T) {
 			receipts,
 			trie.NewStackTrie(nil),
 		)
+		receipt.BlockHash = block.Hash()
 		blockLogsFetcher = newMockBlockLogsFetcher(receipts)
 		_, err := parseDelayedMessagesFromBlock(
 			ctx,
@@ -202,6 +203,8 @@ func Test_parseDelayedMessagesFromBlock(t *testing.T) {
 			receipts,
 			trie.NewStackTrie(nil),
 		)
+		receipt1.BlockHash = block.Hash()
+		receipt2.BlockHash = block.Hash()
 		blockLogsFetcher = newMockBlockLogsFetcher(receipts)
 		_, err = parseDelayedMessagesFromBlock(
 			ctx,
@@ -300,6 +303,8 @@ func Test_parseDelayedMessagesFromBlock(t *testing.T) {
 			receipts,
 			trie.NewStackTrie(nil),
 		)
+		receipt1.BlockHash = block.Hash()
+		receipt2.BlockHash = block.Hash()
 		blockLogsFetcher = newMockBlockLogsFetcher(receipts)
 		delayedMessages, err := parseDelayedMessagesFromBlock(
 			ctx,
@@ -394,6 +399,8 @@ func Test_parseDelayedMessagesFromBlock(t *testing.T) {
 			receipts,
 			trie.NewStackTrie(nil),
 		)
+		receipt1.BlockHash = block.Hash()
+		receipt2.BlockHash = block.Hash()
 		blockLogsFetcher = newMockBlockLogsFetcher(receipts)
 		_, err = parseDelayedMessagesFromBlock(
 			ctx,
@@ -492,6 +499,8 @@ func Test_parseDelayedMessagesFromBlock(t *testing.T) {
 			receipts,
 			trie.NewStackTrie(nil),
 		)
+		receipt1.BlockHash = block.Hash()
+		receipt2.BlockHash = block.Hash()
 		blockLogsFetcher = newMockBlockLogsFetcher(receipts)
 		delayedMessages, err := parseDelayedMessagesFromBlock(
 			ctx,

arbnode/mel/extraction/message_extraction_function.go

@@ -245,5 +245,11 @@ func extractMessagesImpl(
 			ParentChainBlock:    state.ParentChainBlockNumber,
 		})
 	}
+	if len(messages) > 0 {
+		// Only need to calculate partials once, after all the messages are extracted
+		if err := state.GenerateMessageMerklePartialsAndRoot(); err != nil {
+			return nil, nil, nil, nil, err
+		}
+	}
 	return state, messages, delayedMessages, batchMetas, nil
 }

arbnode/mel/messages.go

@@ -6,6 +6,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
 
 	"github.com/offchainlabs/nitro/arbos/arbostypes"
 	"github.com/offchainlabs/nitro/arbutil"
@@ -59,16 +60,11 @@ func (m *DelayedInboxMessage) AfterInboxAcc() common.Hash {
 
 // Hash will replace AfterInboxAcc
 func (m *DelayedInboxMessage) Hash() common.Hash {
-	hash := crypto.Keccak256(
-		[]byte{m.Message.Header.Kind},
-		m.Message.Header.Poster.Bytes(),
-		arbmath.UintToBytes(m.Message.Header.BlockNumber),
-		arbmath.UintToBytes(m.Message.Header.Timestamp),
-		m.Message.Header.RequestId.Bytes(),
-		arbmath.U256Bytes(m.Message.Header.L1BaseFee),
-		crypto.Keccak256(m.Message.L2msg),
-	)
-	return crypto.Keccak256Hash(hash)
+	encoded, err := rlp.EncodeToBytes(m)
+	if err != nil {
+		panic(err)
+	}
+	return crypto.Keccak256Hash(encoded)
 }
 
 type BatchMetadata struct {

arbnode/mel/recording/dap_reader_source.go

@@ -14,9 +14,9 @@ import (
 	"github.com/offchainlabs/nitro/validator"
 )
 
-// DAPReader implements recording of preimages when melextraction.ExtractMessages function is called by MEL validator for creation
-// of validation entry. Since ExtractMessages function would use daprovider.Reader interface to fetch the sequencer batch via RecoverPayload
-// we implement collecting of preimages as well in the same method and record it
+// DAPReader implements recording of data avaialability preimages when melextraction.ExtractMessages function is called by
+// MEL validator for creation of validation entry. Since ExtractMessages function would use daprovider.Reader interface to
+// fetch the sequencer batch via RecoverPayload we implement collecting of preimages as well in the same method and record it
 type DAPReader struct {
 	validatorCtx context.Context
 	reader       daprovider.Reader

arbnode/mel/recording/msg_recording.go

@@ -0,0 +1,30 @@
+package melrecording
+
+import (
+	"errors"
+
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/offchainlabs/nitro/arbnode/mel"
+	"github.com/offchainlabs/nitro/arbos/merkleAccumulator"
+	"github.com/offchainlabs/nitro/arbutil"
+	"github.com/offchainlabs/nitro/daprovider"
+)
+
+// InitializeRecordingMsgPreimages initializes the given state's msgAcc to record preimages related
+// to the extracted messages needed for MEL validation into the given preimages map
+func InitializeRecordingMsgPreimages(state *mel.State, preimages daprovider.PreimagesMap) error {
+	if preimages == nil {
+		return errors.New("preimages recording destination cannot be nil")
+	}
+	if _, ok := preimages[arbutil.Keccak256PreimageType]; !ok {
+		preimages[arbutil.Keccak256PreimageType] = make(map[common.Hash][]byte)
+	}
+	acc, err := merkleAccumulator.NewNonpersistentMerkleAccumulatorFromPartials(mel.ToPtrSlice(state.MessageMerklePartials))
+	if err != nil {
+		return err
+	}
+	acc.RecordPreimagesTo(preimages[arbutil.Keccak256PreimageType])
+	state.SetMsgsAcc(acc)
+	return nil
+}

arbnode/mel/recording/receipt_recorder.go

@@ -1,6 +1,7 @@
 package melrecording
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -16,19 +17,22 @@ import (
 
 	"github.com/offchainlabs/nitro/arbutil"
 	"github.com/offchainlabs/nitro/daprovider"
+	"github.com/offchainlabs/nitro/mel-replay"
 )
 
 // ReceiptRecorder records preimages corresponding to the receipts of a parent chain block
 // needed during the message extraction. These preimages are needed for MEL validation and
 // are used in creation of the validation entries by the MEL validator
 type ReceiptRecorder struct {
-	parentChainReader    BlockReader
-	parentChainBlockHash common.Hash
-	recordPreimages      daprovider.PreimageRecorder
-	receipts             []*types.Receipt
-	logs                 []*types.Log
-	trieDB               *triedb.Database
-	blockReceiptHash     common.Hash
+	parentChainReader      BlockReader
+	parentChainBlockHash   common.Hash
+	parentChainBlockNumber uint64
+	recordPreimages        daprovider.PreimageRecorder
+	receipts               []*types.Receipt
+	logs                   []*types.Log
+	relevantLogsTxIndexes  map[uint]struct{}
+	trieDB                 *triedb.Database
+	blockReceiptHash       common.Hash
 }
 
 // NewReceiptRecorder returns ReceiptRecorder that records
@@ -42,9 +46,10 @@ func NewReceiptRecorder(
 		return nil, errors.New("preimages recording destination cannot be nil")
 	}
 	return &ReceiptRecorder{
-		parentChainReader:    parentChainReader,
-		parentChainBlockHash: parentChainBlockHash,
-		recordPreimages:      daprovider.RecordPreimagesTo(preimages),
+		parentChainReader:     parentChainReader,
+		parentChainBlockHash:  parentChainBlockHash,
+		recordPreimages:       daprovider.RecordPreimagesTo(preimages),
+		relevantLogsTxIndexes: make(map[uint]struct{}),
 	}, nil
 }
 
@@ -98,6 +103,7 @@ func (rr *ReceiptRecorder) Initialize(ctx context.Context) error {
 	rr.receipts = receipts
 	rr.trieDB = tdb
 	rr.blockReceiptHash = root
+	rr.parentChainBlockNumber = block.NumberU64()
 	return nil
 }
 
@@ -108,6 +114,9 @@ func (rr *ReceiptRecorder) LogsForTxIndex(ctx context.Context, parentChainBlockH
 	if rr.parentChainBlockHash != parentChainBlockHash {
 		return nil, fmt.Errorf("parentChainBlockHash mismatch. expected: %v got: %v", rr.parentChainBlockHash, parentChainBlockHash)
 	}
+	if _, recorded := rr.relevantLogsTxIndexes[txIndex]; recorded {
+		return rr.receipts[txIndex].Logs, nil
+	}
 	// #nosec G115
 	if int(txIndex) >= len(rr.receipts) {
 		return nil, fmt.Errorf("index out of range: %d", txIndex)
@@ -141,7 +150,10 @@ func (rr *ReceiptRecorder) LogsForTxIndex(ctx context.Context, parentChainBlockH
 	// We use this same trick in validation as well in order to link a tx with its logs
 	for _, log := range receipt.Logs {
 		log.TxIndex = txIndex
+		log.BlockHash = parentChainBlockHash
+		log.BlockNumber = rr.parentChainBlockNumber
 	}
+	rr.relevantLogsTxIndexes[txIndex] = struct{}{}
 	return receipt.Logs, nil
 }
 
@@ -154,3 +166,19 @@ func (rr *ReceiptRecorder) LogsForBlockHash(ctx context.Context, parentChainBloc
 	}
 	return rr.logs, nil
 }
+
+// CollectTxIndicesPreimage adds the array of relevant tx indexes to the preimages map as a value
+// to the key represented by parentChainBlockHash.
+func (rr *ReceiptRecorder) CollectTxIndicesPreimage() error {
+	var relevantLogsTxIndexes []uint
+	for k := range rr.relevantLogsTxIndexes {
+		relevantLogsTxIndexes = append(relevantLogsTxIndexes, k)
+	}
+	var buf bytes.Buffer
+	if err := rlp.Encode(&buf, relevantLogsTxIndexes); err != nil {
+		return err
+	}
+	relevantTxIndicesKey := melreplay.RelevantTxIndexesKey(rr.parentChainBlockHash)
+	rr.recordPreimages(relevantTxIndicesKey, buf.Bytes(), arbutil.Keccak256PreimageType)
+	return nil
+}