Add proposer tips

Author: martinmsb

db/db.go

@@ -32,6 +32,7 @@ CREATE TABLE IF NOT EXISTS t_pools_metrics_summary (
 	 f_epoch_lost_balace_gwei BIGINT,
 	 f_epoch_effective_balance_gwei BIGINT,
 	 f_mev_rewards_wei BIGINT,
+	 f_proposer_tips_wei BIGINT,
 
 	 f_n_scheduled_blocks BIGINT,
 	 f_n_proposed_blocks BIGINT,
@@ -98,8 +99,9 @@ INSERT INTO t_pools_metrics_summary(
 	f_epoch_effective_balance_gwei,
 	f_epoch_earned_balance_gwei,
 	f_epoch_lost_balace_gwei,
-	f_mev_rewards_wei)
-VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+	f_mev_rewards_wei,
+	f_proposer_tips_wei)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 ON CONFLICT (f_epoch, f_pool)
 DO UPDATE SET
    f_timestamp=EXCLUDED.f_timestamp,
@@ -115,7 +117,8 @@ DO UPDATE SET
 	 f_epoch_effective_balance_gwei=EXCLUDED.f_epoch_effective_balance_gwei,
 	 f_epoch_earned_balance_gwei=EXCLUDED.f_epoch_earned_balance_gwei,
 	 f_epoch_lost_balace_gwei=EXCLUDED.f_epoch_lost_balace_gwei,
-	 f_mev_rewards_wei=EXCLUDED.f_mev_rewards_wei
+	 f_mev_rewards_wei=EXCLUDED.f_mev_rewards_wei,
+	 f_proposer_tips_wei=EXCLUDED.f_proposer_tips_wei
 `
 
 // TODO: Add f_epoch_timestamp
@@ -236,6 +239,7 @@ func (a *Database) StoreValidatorPerformance(validatorPerformance schemas.Valida
 		validatorPerformance.EarnedBalance.Int64(),
 		validatorPerformance.LosedBalance.Int64(),
 		validatorPerformance.MEVRewards.Int64(),
+		validatorPerformance.ProposerTips.Int64(),
 	)
 
 	if err != nil {

metrics/beaconstate.go

@@ -54,7 +54,8 @@ func (p *BeaconState) Run(
 	prevBeaconState *spec.VersionedBeaconState,
 	valKeyToIndex map[string]uint64,
 	relayRewards *big.Int,
-	validatorIndexToWithdrawalAmount map[uint64]*big.Int) error {
+	validatorIndexToWithdrawalAmount map[uint64]*big.Int,
+	proposerTips map[uint64]*big.Int) error {
 
 	if currentBeaconState == nil || prevBeaconState == nil {
 		return errors.New("current or previous beacon state is nil")
@@ -97,6 +98,12 @@ func (p *BeaconState) Run(
 	metrics.NOfActiveValidators = uint64(len(activeValidatorIndexes))
 	metrics.MEVRewards = relayRewards
 
+	aggregatedProposerTips := big.NewInt(0)
+	for _, activeValidatorIndex := range activeValidatorIndexes {
+		aggregatedProposerTips.Add(aggregatedProposerTips, proposerTips[activeValidatorIndex])
+	}
+	metrics.ProposerTips = aggregatedProposerTips
+
 	syncCommitteeKeys := BLSPubKeyToByte(GetCurrentSyncCommittee(currentBeaconState))
 	syncCommitteeIndexes := GetIndexesFromKeys(syncCommitteeKeys, valKeyToIndex)
 	poolSyncIndexes := GetValidatorsIn(syncCommitteeIndexes, activeValidatorIndexes)

metrics/blockdata.go

@@ -16,12 +16,14 @@ import (
 	"github.com/bilinearlabs/eth-metrics/config"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 )
 
 type EpochBlockData struct {
-	Withdrawals map[uint64]*big.Int
+	Withdrawals  map[uint64]*big.Int
+	ProposerTips map[uint64]*big.Int
 }
 
 type BlockData struct {
@@ -49,7 +51,8 @@ func (b *BlockData) GetEpochBlockData(epoch uint64) (*EpochBlockData, error) {
 	log.Info("Fetching block data for epoch: ", epoch)
 
 	data := &EpochBlockData{
-		Withdrawals: make(map[uint64]*big.Int),
+		Withdrawals:  make(map[uint64]*big.Int),
+		ProposerTips: make(map[uint64]*big.Int),
 	}
 
 	firstSlot := epoch * b.networkParameters.slotsInEpoch
@@ -76,12 +79,16 @@ func (b *BlockData) GetEpochBlockData(epoch uint64) (*EpochBlockData, error) {
 
 		b.extractWithdrawals(block, data.Withdrawals)
 
-		// Extract transaction fees (add when needed)
+		// Extract transaction fees
 		proposerTip, err := b.GetProposerTip(block)
 		if err != nil {
 			return nil, errors.Wrap(err, "error getting proposer tip")
 		}
-		// TODO get proposer index of this block and add to data
+		proposerIndex := b.GetProposerIndex(block)
+		if _, ok := data.ProposerTips[proposerIndex]; !ok {
+			data.ProposerTips[proposerIndex] = big.NewInt(0)
+		}
+		data.ProposerTips[proposerIndex].Add(data.ProposerTips[proposerIndex], proposerTip)
 	}
 
 	return data, nil
@@ -101,27 +108,36 @@ func (b *BlockData) extractWithdrawals(beaconBlock *spec.VersionedSignedBeaconBl
 func (b *BlockData) GetProposerTip(beaconBlock *spec.VersionedSignedBeaconBlock) (*big.Int, error) {
 	blockNumber := b.GetBlockNumber(beaconBlock)
 	rawTxs := b.GetBlockTransactions(beaconBlock)
-	header, receipts, err := b.getBlockHeaderAndReceipts(blockNumber, rawTxs)
+	retryOpts := []retry.Option{
+		retry.Attempts(5),
+		retry.Delay(5 * time.Second),
+	}
+	header, err := b.getBlockHeader(blockNumber, retryOpts)
 	if err != nil {
 		return nil, errors.Wrap(err, "error getting block header and receipts")
 	}
 	baseFeePerGasBytes := b.GetBaseFeePerGas(beaconBlock)
 	baseFeePerGas := new(big.Int).SetBytes(baseFeePerGasBytes[:])
 
 	tips := big.NewInt(0)
-	for i, rawTx := range rawTxs {
-		var tx *types.Transaction
-		err = tx.UnmarshalBinary(rawTx)
+	for _, rawTx := range rawTxs {
+		println(rawTx)
+		txReceipt, err := b.getTransactionReceipt(rawTx, retryOpts)
+		if err != nil {
+			return nil, errors.Wrap(err, "error getting block receipt")
+		}
+		var tx types.Transaction
+		err = rlp.DecodeBytes(rawTx, &tx)
 		if err != nil {
 			return nil, errors.Wrap(err, "error unmarshalling transaction")
 		}
-		if tx.Hash() != receipts[i].TxHash {
+		if tx.Hash() != txReceipt.TxHash {
 			return nil, errors.New("transaction hash mismatch")
 		}
 
 		tipFee := new(big.Int)
 		gasPrice := tx.GasPrice()
-		gasUsed := big.NewInt(int64(receipts[i].GasUsed))
+		gasUsed := big.NewInt(int64(txReceipt.GasUsed))
 
 		switch tx.Type() {
 		case 0, 1:
@@ -139,25 +155,21 @@ func (b *BlockData) GetProposerTip(beaconBlock *spec.VersionedSignedBeaconBlock)
 		default:
 			return nil, errors.Errorf("unknown transaction type: %d, hash: %s", tx.Type(), tx.Hash().String())
 		}
+		tips.Add(tips, tipFee)
 	}
 	burnt := new(big.Int).Mul(big.NewInt(int64(b.GetGasUsed(beaconBlock))), baseFeePerGas)
 	proposerReward := new(big.Int).Sub(tips, burnt)
 	return proposerReward, nil
 }
 
-func (b *BlockData) getBlockHeaderAndReceipts(
+func (b *BlockData) getBlockHeader(
 	blockNumber uint64,
-	rawTxs []bellatrix.Transaction,
-) (*types.Header, []*types.Receipt, error) {
-
+	retryOpts []retry.Option,
+) (*types.Header, error) {
 	var header *types.Header
 	var err error
 
 	blockNumberBig := new(big.Int).SetUint64(blockNumber)
-	retryOpts := []retry.Option{
-		retry.Attempts(5),
-		retry.Delay(5 * time.Second),
-	}
 
 	err = retry.Do(func() error {
 		header, err = b.executionClient.HeaderByNumber(context.Background(), blockNumberBig)
@@ -168,32 +180,33 @@ func (b *BlockData) getBlockHeaderAndReceipts(
 		return nil
 	}, retryOpts...)
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "error getting header for block "+blockNumberBig.String())
+		return nil, errors.Wrap(err, "error getting header for block "+blockNumberBig.String())
 	}
 
-	var receipts []*types.Receipt = make([]*types.Receipt, 0)
-	for _, rawTx := range rawTxs {
-		var tx *types.Transaction
-		err = tx.UnmarshalBinary(rawTx)
-		if err != nil {
-			return nil, nil, errors.Wrap(err, "error unmarshalling transaction")
-		}
-		var receipt *types.Receipt
-		err = retry.Do(func() error {
-			receipt, err = b.executionClient.TransactionReceipt(context.Background(), tx.Hash())
-			if err != nil {
-				log.Warnf("error getting transaction receipt for tx %s: %s. Retrying...", tx.Hash().String(), err)
-				return errors.Wrap(err, "error getting transaction receipt for block")
-			}
-			return nil
-		}, retryOpts...)
+	return header, nil
+}
+
+func (b *BlockData) getTransactionReceipt(rawTx []byte, retryOpts []retry.Option) (*types.Receipt, error) {
+	var tx types.Transaction
+	err := tx.UnmarshalBinary(rawTx)
+	if err != nil {
+		return nil, errors.Wrap(err, "error unmarshalling transaction")
+	}
+	var receipt *types.Receipt
+	err = retry.Do(func() error {
+		receipt, err = b.executionClient.TransactionReceipt(context.Background(), tx.Hash())
 		if err != nil {
-			return nil, nil, errors.Wrap(err, "error getting transaction receipt for block "+blockNumberBig.String())
+			log.Warnf("error getting transaction receipt for tx %s: %s. Retrying...", tx.Hash().String(), err)
+			return errors.Wrap(err, "error getting transaction receipt")
 		}
-		receipts = append(receipts, receipt)
+		return nil
+	}, retryOpts...)
+
+	if err != nil {
+		return nil, errors.Wrap(err, "error getting transaction receipt for tx "+tx.Hash().String())
 	}
 
-	return header, receipts, nil
+	return receipt, nil
 }
 
 func (b *BlockData) GetBlockWithdrawals(beaconBlock *spec.VersionedSignedBeaconBlock) []*capella.Withdrawal {
@@ -304,3 +317,23 @@ func (b *BlockData) GetGasUsed(beaconBlock *spec.VersionedSignedBeaconBlock) uin
 	}
 	return gasUsed
 }
+
+func (b *BlockData) GetProposerIndex(beaconBlock *spec.VersionedSignedBeaconBlock) uint64 {
+	var proposerIndex uint64
+	if beaconBlock.Altair != nil {
+		proposerIndex = uint64(beaconBlock.Altair.Message.ProposerIndex)
+	} else if beaconBlock.Bellatrix != nil {
+		proposerIndex = uint64(beaconBlock.Bellatrix.Message.ProposerIndex)
+	} else if beaconBlock.Capella != nil {
+		proposerIndex = uint64(beaconBlock.Capella.Message.ProposerIndex)
+	} else if beaconBlock.Deneb != nil {
+		proposerIndex = uint64(beaconBlock.Deneb.Message.ProposerIndex)
+	} else if beaconBlock.Electra != nil {
+		proposerIndex = uint64(beaconBlock.Electra.Message.ProposerIndex)
+	} else if beaconBlock.Fulu != nil {
+		proposerIndex = uint64(beaconBlock.Fulu.Message.ProposerIndex)
+	} else {
+		log.Fatal("Beacon block was empty")
+	}
+	return proposerIndex
+}

metrics/metrics.go

@@ -10,11 +10,14 @@ import (
 	"strings"
 	"time"
 
+	nethttp "net/http"
+
 	"github.com/attestantio/go-eth2-client/api"
 	"github.com/attestantio/go-eth2-client/http"
 	"github.com/attestantio/go-eth2-client/spec"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/rs/zerolog"
 
 	"github.com/bilinearlabs/eth-metrics/config"
@@ -138,11 +141,21 @@ func NewMetrics(
 	log.Info("Slots per epoch: ", slotsPerEpoch)
 	log.Info("Seconds per slot: ", secondsPerSlot)
 
-	executionClient, err := ethclient.Dial(config.Eth1Address)
+	rcpClient, err := rpc.DialOptions(
+		context.Background(),
+		config.Eth1Address,
+		rpc.WithHTTPAuth(func(h nethttp.Header) error {
+			h.Set("Authorization", "Basic "+encodedCredentials)
+			return nil
+		}),
+		rpc.WithHTTPClient(&nethttp.Client{Timeout: 60 * time.Second}),
+	)
 	if err != nil {
 		return nil, errors.Wrap(err, "error dialing execution client")
 	}
 
+	executionClient := ethclient.NewClient(rcpClient)
+
 	networkParameters := &NetworkParameters{
 		genesisSeconds: uint64(genesis.Data.GenesisTime.Unix()),
 		slotsInEpoch:   slotsPerEpoch,
@@ -351,11 +364,12 @@ func (a *Metrics) ProcessEpoch(
 	}
 
 	// Get withdrawals from all blocks of the epoch
-	blockData, err := a.blockData.GetEpochBlockData(currentEpoch)
+	epochBlockData, err := a.blockData.GetEpochBlockData(currentEpoch)
 	if err != nil {
-		return nil, errors.Wrap(err, "error getting epoch withdrawals")
+		return nil, errors.Wrap(err, "error getting epoch block data")
 	}
-	validatorIndexToWithdrawalAmount := blockData.Withdrawals
+	validatorIndexToWithdrawalAmount := epochBlockData.Withdrawals
+	proposerTips := epochBlockData.ProposerTips
 
 	err = a.networkStats.Run(currentEpoch, currentBeaconState)
 	if err != nil {
@@ -370,7 +384,7 @@ func (a *Metrics) ProcessEpoch(
 		if reward, ok := relayRewardsPerPool[poolName]; ok {
 			relayRewards.Add(relayRewards, reward)
 		}
-		err = a.beaconState.Run(pubKeys, poolName, currentBeaconState, prevBeaconState, valKeyToIndex, relayRewards, validatorIndexToWithdrawalAmount)
+		err = a.beaconState.Run(pubKeys, poolName, currentBeaconState, prevBeaconState, valKeyToIndex, relayRewards, validatorIndexToWithdrawalAmount, proposerTips)
 		if err != nil {
 			return nil, errors.Wrap(err, "error running beacon state")
 		}

schemas/schemas.go

@@ -27,6 +27,7 @@ type ValidatorPerformanceMetrics struct {
 	TotalRewards           *big.Int
 	DeltaEpochBalance      *big.Int
 	MEVRewards             *big.Int
+	ProposerTips           *big.Int
 }
 
 type ValidatorStatusMetrics struct {