Separate receipts req and process, add tests

martinmsb · martinmsb · commit 4fb6dd1da1ea · 2025-12-16T14:13:51.000+01:00
diff --git a/db/db_test.go b/db/db_test.go
@@ -16,23 +16,27 @@ func Test_GetMissingEpochs(t *testing.T) {
 	db.CreateTables()
 
 	db.StoreValidatorPerformance(schemas.ValidatorPerformanceMetrics{
-		Time:          time.Now(),
-		Epoch:         100,
-		EarnedBalance: big.NewInt(100),
-		LosedBalance:  big.NewInt(100),
-		MEVRewards:    big.NewInt(100),
+		Time:             time.Now(),
+		Epoch:            100,
+		EarnedBalance:    big.NewInt(100),
+		LosedBalance:     big.NewInt(100),
+		EffectiveBalance: big.NewInt(100),
+		MEVRewards:       big.NewInt(100),
+		ProposerTips:     big.NewInt(100),
 	})
 
 	epochs, err := db.GetMissingEpochs(200, 4)
 	require.NoError(t, err)
 	require.Equal(t, []uint64{197, 198, 199, 200}, epochs)
 
 	db.StoreValidatorPerformance(schemas.ValidatorPerformanceMetrics{
-		Time:          time.Now(),
-		Epoch:         197,
-		EarnedBalance: big.NewInt(100),
-		LosedBalance:  big.NewInt(100),
-		MEVRewards:    big.NewInt(100),
+		Time:             time.Now(),
+		Epoch:            197,
+		EarnedBalance:    big.NewInt(100),
+		LosedBalance:     big.NewInt(100),
+		EffectiveBalance: big.NewInt(100),
+		MEVRewards:       big.NewInt(100),
+		ProposerTips:     big.NewInt(100),
 	})
 
 	epochs, err = db.GetMissingEpochs(200, 4)
diff --git a/metrics/blockdata.go b/metrics/blockdata.go
@@ -78,10 +78,27 @@ func (b *BlockData) GetEpochBlockData(epoch uint64) (*EpochBlockData, error) {
 
 		block := beaconBlock.Data
 
-		b.extractWithdrawals(block, data.Withdrawals)
+		b.ExtractWithdrawals(block, data.Withdrawals)
 
 		// Extract transaction fees
-		proposerTip, err := b.GetProposerTip(block)
+		blockNumber := b.GetBlockNumber(block)
+
+		retryOpts := []retry.Option{
+			retry.Attempts(5),
+			retry.Delay(5 * time.Second),
+		}
+
+		header, err := b.getBlockHeader(blockNumber, retryOpts)
+		if err != nil {
+			return nil, errors.Wrap(err, "error getting block header and receipts")
+		}
+		rawTxs := b.GetBlockTransactions(block)
+		receipts, err := b.getBlockReceipts(rawTxs, retryOpts)
+		if err != nil {
+			return nil, errors.Wrap(err, "error getting block receipts")
+		}
+
+		proposerTip, err := b.GetProposerTip(block, header, receipts)
 		if err != nil {
 			return nil, errors.Wrap(err, "error getting proposer tip")
 		}
@@ -95,7 +112,7 @@ func (b *BlockData) GetEpochBlockData(epoch uint64) (*EpochBlockData, error) {
 	return data, nil
 }
 
-func (b *BlockData) extractWithdrawals(beaconBlock *spec.VersionedSignedBeaconBlock, withdrawals map[uint64]*big.Int) {
+func (b *BlockData) ExtractWithdrawals(beaconBlock *spec.VersionedSignedBeaconBlock, withdrawals map[uint64]*big.Int) {
 	blockWithdrawals := b.GetBlockWithdrawals(beaconBlock)
 	for _, withdrawal := range blockWithdrawals {
 		idx := uint64(withdrawal.ValidatorIndex)
@@ -106,74 +123,53 @@ func (b *BlockData) extractWithdrawals(beaconBlock *spec.VersionedSignedBeaconBl
 	}
 }
 
-func (b *BlockData) GetProposerTip(beaconBlock *spec.VersionedSignedBeaconBlock) (*big.Int, error) {
-	blockNumber := b.GetBlockNumber(beaconBlock)
+func (b *BlockData) GetProposerTip(
+	beaconBlock *spec.VersionedSignedBeaconBlock,
+	header *types.Header,
+	receipts []*types.Receipt,
+) (*big.Int, error) {
 	rawTxs := b.GetBlockTransactions(beaconBlock)
-	retryOpts := []retry.Option{
-		retry.Attempts(5),
-		retry.Delay(5 * time.Second),
-	}
-	header, err := b.getBlockHeader(blockNumber, retryOpts)
-	if err != nil {
-		return nil, errors.Wrap(err, "error getting block header and receipts")
-	}
+
 	baseFeePerGasBytes := b.GetBaseFeePerGas(beaconBlock)
 	baseFeePerGas := new(big.Int).SetBytes(baseFeePerGasBytes[:])
 
 	tips := big.NewInt(0)
 
-	var g errgroup.Group
-	var mu sync.Mutex
-	// Limit concurrent requests
-	g.SetLimit(10)
+	for i, rawTx := range rawTxs {
+		var tx types.Transaction
+		err := tx.UnmarshalBinary(rawTx)
+		if err != nil {
+			return nil, errors.Wrap(err, "error unmarshalling transaction")
+		}
+		txReceipt := receipts[i]
 
-	for _, rawTx := range rawTxs {
-		g.Go(func() error {
-			var tx types.Transaction
-			err = tx.UnmarshalBinary(rawTx)
-			if err != nil {
-				return errors.Wrap(err, "error unmarshalling transaction")
-			}
-			txReceipt, err := b.getTransactionReceipt(&tx, retryOpts)
-			if err != nil {
-				return errors.Wrap(err, "error getting block receipt")
-			}
-			if err != nil {
-				return errors.Wrap(err, "error unmarshalling transaction")
-			}
-			if tx.Hash() != txReceipt.TxHash {
-				return errors.New("transaction hash mismatch")
-			}
+		if tx.Hash() != txReceipt.TxHash {
+			return nil, errors.New("transaction hash mismatch")
+		}
 
-			tipFee := new(big.Int)
-			gasPrice := tx.GasPrice()
-			gasUsed := big.NewInt(int64(txReceipt.GasUsed))
-
-			switch tx.Type() {
-			case 0, 1:
-				tipFee.Mul(gasPrice, gasUsed)
-			case 2, 3, 4:
-				tip := new(big.Int).Add(tx.GasTipCap(), header.BaseFee)
-				gasFeeCap := tx.GasFeeCap()
-				var usedGasPrice *big.Int
-				if gasFeeCap.Cmp(tip) < 0 {
-					usedGasPrice = gasFeeCap
-				} else {
-					usedGasPrice = tip
-				}
-				tipFee = new(big.Int).Mul(usedGasPrice, gasUsed)
-			default:
-				return errors.Errorf("unknown transaction type: %d, hash: %s", tx.Type(), tx.Hash().String())
+		tipFee := new(big.Int)
+		gasPrice := tx.GasPrice()
+		gasUsed := big.NewInt(int64(txReceipt.GasUsed))
+
+		switch tx.Type() {
+		case 0, 1:
+			tipFee.Mul(gasPrice, gasUsed)
+		case 2, 3, 4:
+			tip := new(big.Int).Add(tx.GasTipCap(), header.BaseFee)
+			gasFeeCap := tx.GasFeeCap()
+			var usedGasPrice *big.Int
+			if gasFeeCap.Cmp(tip) < 0 {
+				usedGasPrice = gasFeeCap
+			} else {
+				usedGasPrice = tip
 			}
-			mu.Lock()
-			tips.Add(tips, tipFee)
-			mu.Unlock()
-			return nil
-		})
-	}
-	if err := g.Wait(); err != nil {
-		return nil, errors.Wrap(err, "error getting proposer tip")
+			tipFee = new(big.Int).Mul(usedGasPrice, gasUsed)
+		default:
+			return nil, errors.Errorf("unknown transaction type: %d, hash: %s", tx.Type(), tx.Hash().String())
+		}
+		tips.Add(tips, tipFee)
 	}
+
 	burnt := new(big.Int).Mul(big.NewInt(int64(b.GetGasUsed(beaconBlock))), baseFeePerGas)
 	proposerReward := new(big.Int).Sub(tips, burnt)
 	return proposerReward, nil
@@ -203,6 +199,38 @@ func (b *BlockData) getBlockHeader(
 	return header, nil
 }
 
+func (b *BlockData) getBlockReceipts(rawTxs []bellatrix.Transaction, retryOpts []retry.Option) ([]*types.Receipt, error) {
+	receipts := make([]*types.Receipt, len(rawTxs))
+	var err error
+
+	var g errgroup.Group
+	var mu sync.Mutex
+	// Limit concurrent requests
+	g.SetLimit(10)
+
+	for i, rawTx := range rawTxs {
+		g.Go(func() error {
+			var tx types.Transaction
+			err = tx.UnmarshalBinary(rawTx)
+			if err != nil {
+				return errors.Wrap(err, "error unmarshalling transaction")
+			}
+			receipt, err := b.getTransactionReceipt(&tx, retryOpts)
+			if err != nil {
+				return errors.Wrap(err, "error getting transaction receipt")
+			}
+			mu.Lock()
+			receipts[i] = receipt
+			mu.Unlock()
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return nil, errors.Wrap(err, "error getting block receipts")
+	}
+	return receipts, nil
+}
+
 func (b *BlockData) getTransactionReceipt(tx *types.Transaction, retryOpts []retry.Option) (*types.Receipt, error) {
 	var receipt *types.Receipt
 	var err error
diff --git a/metrics/blockdata_test.go b/metrics/blockdata_test.go
@@ -0,0 +1,99 @@
+package metrics
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"math/big"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/attestantio/go-eth2-client/spec"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_GetProposerTip(t *testing.T) {
+	bd := &BlockData{
+		networkParameters: &NetworkParameters{
+			slotsInEpoch: 32,
+		},
+	}
+
+	blockData, err := LoadBlockData(5214302)
+	if err != nil {
+		t.Fatalf("error loading block data: %s", err)
+	}
+
+	proposerTip, err := bd.GetProposerTip(blockData.BeaconBlock, blockData.Header, blockData.Receipts)
+	if err != nil {
+		t.Fatalf("error getting proposer tip: %s", err)
+	}
+	tip := big.NewInt(38657065851824731)
+	assert.Equal(t, proposerTip, tip)
+}
+
+func Test_ExtractWithdrawals(t *testing.T) {
+	bd := &BlockData{
+		networkParameters: &NetworkParameters{
+			slotsInEpoch: 32,
+		},
+	}
+
+	blockData, err := LoadBlockData(5214302)
+	if err != nil {
+		t.Fatalf("error loading block data: %s", err)
+	}
+
+	withdrawals := make(map[uint64]*big.Int)
+	bd.ExtractWithdrawals(blockData.BeaconBlock, withdrawals)
+	assert.Equal(t, withdrawals, map[uint64]*big.Int{
+		416729: big.NewInt(1701196),
+		416730: big.NewInt(1731482),
+		416731: big.NewInt(1683530),
+		416732: big.NewInt(1765666),
+		416733: big.NewInt(1753893),
+		416734: big.NewInt(45764133),
+		416735: big.NewInt(1740038),
+		416736: big.NewInt(1736192),
+		416737: big.NewInt(1732742),
+		416738: big.NewInt(1776043),
+		416739: big.NewInt(1746233),
+		416740: big.NewInt(1713045),
+		416741: big.NewInt(1761575),
+		416742: big.NewInt(1719014),
+		416743: big.NewInt(1735415),
+		416744: big.NewInt(1714423),
+	})
+}
+
+type MockBlockData struct {
+	BeaconBlock *spec.VersionedSignedBeaconBlock `json:"consensus_block"`
+	Header      *types.Header                    `json:"execution_header"`
+	Receipts    []*types.Receipt                 `json:"execution_receipts"`
+}
+
+func LoadBlockData(slot uint64) (*MockBlockData, error) {
+	path := filepath.Join("../mock", fmt.Sprintf("fullblock_slot_%d.json", slot))
+	jsonFile, err := os.Open(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not open json file")
+	}
+	defer jsonFile.Close()
+
+	byteValue, err := io.ReadAll(jsonFile)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not read json file")
+	}
+
+	var blockData MockBlockData
+
+	err = json.Unmarshal(byteValue, &blockData)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not unmarshal json file")
+	}
+
+	return &blockData, nil
+}
diff --git a/mock/fullblock_slot_5214302.json b/mock/fullblock_slot_5214302.json
