diff --git a/cmd/p2p/crawl/crawl.go b/cmd/p2p/crawl/crawl.go index e8e8a057e..6bd2ffba2 100644 --- a/cmd/p2p/crawl/crawl.go +++ b/cmd/p2p/crawl/crawl.go @@ -1,6 +1,7 @@ package crawl import ( + _ "embed" "fmt" "time" @@ -32,6 +33,8 @@ type ( ) var ( + //go:embed usage.md + crawlUsage string inputCrawlParams crawlParams ) @@ -40,7 +43,7 @@ var ( var CrawlCmd = &cobra.Command{ Use: "crawl [nodes file]", Short: "Crawl a network on the devp2p layer and generate a nodes JSON file.", - Long: "If no nodes.json file exists, it will be created.", + Long: crawlUsage, Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, args []string) (err error) { inputCrawlParams.NodesFile = args[0] diff --git a/cmd/p2p/crawl/usage.md b/cmd/p2p/crawl/usage.md new file mode 100644 index 000000000..1d167f606 --- /dev/null +++ b/cmd/p2p/crawl/usage.md @@ -0,0 +1,11 @@ +To crawl the network for nodes and write the output json to a file. This will +not engage in block or transaction propagation, but it can give a good indicator +of network size, and the output json can be used to quick start other nodes. + +## Example + +```bash +polycli p2p crawl nodes.json \ + --bootnodes "enode://0cb82b395094ee4a2915e9714894627de9ed8498fb881cec6db7c65e8b9a5bd7f2f25cc84e71e89d0947e51c76e85d0847de848c7782b13c0255247a6758178c@44.232.55.71:30303,enode://88116f4295f5a31538ae409e4d44ad40d22e44ee9342869e7d68bdec55b0f83c1530355ce8b41fbec0928a7d75a5745d528450d30aec92066ab6ba1ee351d710@159.203.9.164:30303,enode://4be7248c3a12c5f95d4ef5fff37f7c44ad1072fdb59701b2e5987c5f3846ef448ce7eabc941c5575b13db0fb016552c1fa5cca0dda1a8008cf6d63874c0f3eb7@3.93.224.197:30303,enode://32dd20eaf75513cf84ffc9940972ab17a62e88ea753b0780ea5eca9f40f9254064dacb99508337043d944c2a41b561a17deaad45c53ea0be02663e55e6a302b2@3.212.183.151:30303" \ + --network-id 137 +``` diff --git a/cmd/p2p/p2p.go b/cmd/p2p/p2p.go index 159827d44..982dba935 100644 --- a/cmd/p2p/p2p.go +++ b/cmd/p2p/p2p.go @@ -3,8 +3,6 @@ package p2p import ( "github.com/spf13/cobra" - _ "embed" - "github.com/0xPolygon/polygon-cli/cmd/p2p/crawl" "github.com/0xPolygon/polygon-cli/cmd/p2p/nodelist" "github.com/0xPolygon/polygon-cli/cmd/p2p/ping" @@ -12,13 +10,9 @@ import ( "github.com/0xPolygon/polygon-cli/cmd/p2p/sensor" ) -//go:embed usage.md -var usage string - var P2pCmd = &cobra.Command{ Use: "p2p", Short: "Set of commands related to devp2p.", - Long: usage, } func init() { diff --git a/cmd/p2p/ping/ping.go b/cmd/p2p/ping/ping.go index 1ec7d9849..2989b76d9 100644 --- a/cmd/p2p/ping/ping.go +++ b/cmd/p2p/ping/ping.go @@ -2,6 +2,7 @@ package ping import ( "crypto/ecdsa" + _ "embed" "net" "sync" "time" @@ -30,19 +31,16 @@ type ( ) var ( + //go:embed usage.md + pingUsage string inputPingParams pingParams ) var PingCmd = &cobra.Command{ Use: "ping [enode/enr or nodes file]", Short: "Ping node(s) and return the output.", - Long: `Ping nodes by either giving a single enode/enr or an entire nodes file. - -This command will establish a handshake and status exchange to get the Hello and -Status messages and output JSON. If providing a enode/enr rather than a nodes -file, then the connection will remain open by default (--listen=true), and you -can see other messages the peer sends (e.g. blocks, transactions, etc.).`, - Args: cobra.MinimumNArgs(1), + Long: pingUsage, + Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, args []string) (err error) { inputPingParams.privateKey, err = p2p.ParsePrivateKey(inputPingParams.KeyFile, inputPingParams.PrivateKey) if err != nil { diff --git a/cmd/p2p/ping/usage.md b/cmd/p2p/ping/usage.md new file mode 100644 index 000000000..c0e7e437b --- /dev/null +++ b/cmd/p2p/ping/usage.md @@ -0,0 +1,10 @@ +Pinging a peer is useful to determine information about the peer and retrieving +the `Hello` and `Status` messages. By default, it will listen to the peer after +the status exchange for blocks and transactions. To disable this behavior, set +the `--listen` flag. + +## Example + +```bash +polycli p2p ping +``` diff --git a/cmd/p2p/sensor/api.go b/cmd/p2p/sensor/api.go index a7744d0ab..fb86cfb02 100644 --- a/cmd/p2p/sensor/api.go +++ b/cmd/p2p/sensor/api.go @@ -8,6 +8,7 @@ import ( "time" "github.com/0xPolygon/polygon-cli/p2p" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" ethp2p "github.com/ethereum/go-ethereum/p2p" "github.com/prometheus/client_golang/prometheus" @@ -27,12 +28,33 @@ type peerData struct { DurationSeconds float64 `json:"duration_seconds"` } +// blockInfo represents basic block information. +type blockInfo struct { + Hash string `json:"hash"` + Number uint64 `json:"number"` +} + +// newBlockInfo creates a blockInfo from a types.Header. +// Returns nil if the header is nil. +func newBlockInfo(header *types.Header) *blockInfo { + if header == nil { + return nil + } + + return &blockInfo{ + Hash: header.Hash().Hex(), + Number: header.Number.Uint64(), + } +} + // apiData represents all sensor information including node info and peer data. type apiData struct { ENR string `json:"enr"` URL string `json:"enode"` PeerCount int `json:"peer_count"` Peers map[string]peerData `json:"peers"` + Head *blockInfo `json:"head_block"` + Oldest *blockInfo `json:"oldest_block"` } // handleAPI sets up the API for interacting with the sensor. All endpoints @@ -54,7 +76,7 @@ func handleAPI(server *ethp2p.Server, counter *prometheus.CounterVec, conns *p2p url := peer.Node().URLv4() peerID := peer.Node().ID().String() name := peer.Fullname() - connectedAt := conns.GetPeerConnectedAt(peerID) + connectedAt := conns.PeerConnectedAt(peerID) if connectedAt.IsZero() { continue } @@ -71,11 +93,21 @@ func handleAPI(server *ethp2p.Server, counter *prometheus.CounterVec, conns *p2p peers[url] = msgs } + head := conns.HeadBlock() + oldest := conns.OldestBlock() + + var headHeader *types.Header + if head.Block != nil { + headHeader = head.Block.Header() + } + data := apiData{ ENR: server.NodeInfo().ENR, URL: server.Self().URLv4(), PeerCount: len(peers), Peers: peers, + Head: newBlockInfo(headHeader), + Oldest: newBlockInfo(oldest), } if err := json.NewEncoder(w).Encode(data); err != nil { diff --git a/cmd/p2p/sensor/sensor.go b/cmd/p2p/sensor/sensor.go index 9127fbc28..414692039 100644 --- a/cmd/p2p/sensor/sensor.go +++ b/cmd/p2p/sensor/sensor.go @@ -3,11 +3,11 @@ package sensor import ( "context" "crypto/ecdsa" + _ "embed" "errors" "fmt" "os" "os/signal" - "sync" "syscall" "time" @@ -17,6 +17,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/protocols/eth" ethp2p "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/dnsdisc" "github.com/ethereum/go-ethereum/p2p/enode" @@ -82,6 +83,8 @@ type ( ) var ( + //go:embed usage.md + sensorUsage string inputSensorParams sensorParams ) @@ -90,7 +93,7 @@ var ( var SensorCmd = &cobra.Command{ Use: "sensor [nodes file]", Short: "Start a devp2p sensor that discovers other peers and will receive blocks and transactions.", - Long: "If no nodes.json file exists, it will be created.", + Long: sensorUsage, Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, args []string) (err error) { inputSensorParams.NodesFile = args[0] @@ -170,14 +173,14 @@ var SensorCmd = &cobra.Command{ // Fetch the latest block which will be used later when crafting the status // message. This call will only be made once and stored in the head field // until the sensor receives a new block it can overwrite it with. - block, err := getLatestBlock(inputSensorParams.RPC) + rpcBlock, err := getLatestBlock(inputSensorParams.RPC) if err != nil { return err } - head := p2p.HeadBlock{ - Hash: block.Hash.ToHash(), - TotalDifficulty: block.TotalDifficulty.ToBigInt(), - Number: block.Number.ToUint64(), + + head := eth.NewBlockPacket{ + Block: rpcBlock.ToBlock(), + TD: rpcBlock.TotalDifficulty.ToBigInt(), } peersGauge := promauto.NewGauge(prometheus.GaugeOpts{ @@ -192,10 +195,13 @@ var SensorCmd = &cobra.Command{ Help: "The number and type of messages the sensor has sent and received", }, []string{"message", "url", "name", "direction"}) + metrics := p2p.NewBlockMetrics(head.Block) + // Create peer connection manager for broadcasting transactions // and managing the global blocks cache conns := p2p.NewConns(p2p.ConnsOptions{ BlocksCache: inputSensorParams.BlocksCache, + Head: head, }) opts := p2p.EthProtocolOptions{ @@ -206,8 +212,6 @@ var SensorCmd = &cobra.Command{ SensorID: inputSensorParams.SensorID, NetworkID: inputSensorParams.NetworkID, Conns: conns, - Head: &head, - HeadMutex: &sync.RWMutex{}, ForkID: forkid.ID{Hash: [4]byte(inputSensorParams.ForkID)}, MsgCounter: msgCounter, RequestsCache: inputSensorParams.RequestsCache, @@ -280,6 +284,8 @@ var SensorCmd = &cobra.Command{ peersGauge.Set(float64(server.PeerCount())) db.WritePeers(cmd.Context(), server.Peers(), time.Now()) + metrics.Update(conns.HeadBlock().Block, conns.OldestBlock()) + urls := []string{} for _, peer := range server.Peers() { urls = append(urls, peer.Node().URLv4()) diff --git a/cmd/p2p/sensor/usage.md b/cmd/p2p/sensor/usage.md new file mode 100644 index 000000000..c45a77c54 --- /dev/null +++ b/cmd/p2p/sensor/usage.md @@ -0,0 +1,68 @@ +Running the sensor will do peer discovery and continue to watch for blocks and +transactions from those peers. This is useful for observing the network for +forks and reorgs without the need to run the entire full node infrastructure. + +The sensor can persist data to various backends including Google Cloud Datastore +or JSON output. If no nodes.json file exists at the specified path, it will be +created automatically. + +The bootnodes may change, so refer to the [Polygon Knowledge Layer][bootnodes] +if the sensor is not discovering peers. + +## Metrics + +The sensor exposes Prometheus metrics at `http://localhost:2112/metrics` +(configurable via `--prom-port`). For a complete list of available metrics, see +[polycli_p2p_sensor_metrics.md](polycli_p2p_sensor_metrics.md). + +## Examples + +### Mainnet + +To run a Polygon Mainnet sensor, copy the `genesis.json` from [here][mainnet-genesis]. + +```bash +polycli p2p sensor nodes.json \ + --bootnodes "enode://b8f1cc9c5d4403703fbf377116469667d2b1823c0daf16b7250aa576bacf399e42c3930ccfcb02c5df6879565a2b8931335565f0e8d3f8e72385ecf4a4bf160a@3.36.224.80:30303,enode://8729e0c825f3d9cad382555f3e46dcff21af323e89025a0e6312df541f4a9e73abfa562d64906f5e59c51fe6f0501b3e61b07979606c56329c020ed739910759@54.194.245.5:30303" \ + --network-id 137 \ + --sensor-id "sensor" \ + --write-blocks=true \ + --write-block-events=true \ + --write-txs=true \ + --write-tx-events=true \ + --genesis-hash "0xa9c28ce2141b56c474f1dc504bee9b01eb1bd7d1a507580d5519d4437a97de1b" \ + --fork-id "22d523b2" \ + --rpc "https://polygon-rpc.com" \ + --discovery-dns "enrtree://AKUEZKN7PSKVNR65FZDHECMKOJQSGPARGTPPBI7WS2VUL4EGR6XPC@pos.polygon-peers.io" \ + --pprof \ + --verbosity 700 \ + --pretty-logs=true \ + --database "json" +``` + +### Amoy + +To run a Polygon Amoy sensor, copy the `genesis.json` from [here][amoy-genesis]. + +```bash +polycli p2p sensor amoy-nodes.json \ + --bootnodes "enode://b8f1cc9c5d4403703fbf377116469667d2b1823c0daf16b7250aa576bacf399e42c3930ccfcb02c5df6879565a2b8931335565f0e8d3f8e72385ecf4a4bf160a@3.36.224.80:30303,enode://8729e0c825f3d9cad382555f3e46dcff21af323e89025a0e6312df541f4a9e73abfa562d64906f5e59c51fe6f0501b3e61b07979606c56329c020ed739910759@54.194.245.5:30303" \ + --network-id 80002 \ + --sensor-id "sensor-amoy" \ + --write-blocks=true \ + --write-block-events=true \ + --write-txs=true \ + --write-tx-events=true \ + --genesis-hash "0x7202b2b53c5a0836e773e319d18922cc756dd67432f9a1f65352b61f4406c697" \ + --fork-id "8b7e4175" \ + --rpc "https://rpc-amoy.polygon.technology" \ + --discovery-dns "enrtree://AKUEZKN7PSKVNR65FZDHECMKOJQSGPARGTPPBI7WS2VUL4EGR6XPC@amoy.polygon-peers.io" \ + --pprof \ + --verbosity 700 \ + --pretty-logs=true \ + --database "json" +``` + +[mainnet-genesis]: https://github.com/0xPolygon/bor/blob/master/builder/files/genesis-mainnet-v1.json +[amoy-genesis]: https://github.com/0xPolygon/bor/blob/master/builder/files/genesis-amoy.json +[bootnodes]: https://docs.polygon.technology/pos/reference/seed-and-bootnodes/ diff --git a/cmd/p2p/usage.md b/cmd/p2p/usage.md deleted file mode 100644 index 94b2dc202..000000000 --- a/cmd/p2p/usage.md +++ /dev/null @@ -1,62 +0,0 @@ -### Ping - -Pinging a peer is useful to determine information about the peer and retrieving -the `Hello` and `Status` messages. By default, it will listen to the peer after -the status exchange for blocks and transactions. To disable this behavior, set -the `--listen` flag. - -```bash -polycli p2p ping -``` - -### Sensor - -Running the sensor will do peer discovery and continue to watch for blocks and -transactions from those peers. This is useful for observing the network for -forks and reorgs without the need to run the entire full node infrastructure. - -The bootnodes may change, so refer to the [Wiki][bootnodes] if the sensor is not -discovering peers. - -#### Mainnet - -To run a Polygon Mainnet sensor, copy the `genesis.json` from -[here][mainnet-genesis]. - -```bash -polycli p2p sensor nodes.json \ - --bootnodes "enode://b8f1cc9c5d4403703fbf377116469667d2b1823c0daf16b7250aa576bacf399e42c3930ccfcb02c5df6879565a2b8931335565f0e8d3f8e72385ecf4a4bf160a@3.36.224.80:30303,enode://8729e0c825f3d9cad382555f3e46dcff21af323e89025a0e6312df541f4a9e73abfa562d64906f5e59c51fe6f0501b3e61b07979606c56329c020ed739910759@54.194.245.5:30303" \ - --network-id 137 \ - --sensor-id sensor \ - --rpc "https://polygon-rpc.com" -``` - -#### Amoy - -To run a Polygon Amoy sensor, copy the `genesis.json` from -[here][amoy-genesis]. - -```bash -polycli p2p sensor nodes.json \ - --bootnodes "enode://0ef8758cafc0063405f3f31fe22f2a3b566aa871bd7cd405e35954ec8aa7237c21e1ccc1f65f1b6099ab36db029362bc2fecf001a771b3d9803bbf1968508cef@35.197.249.21:30303,enode://c9c8c18cde48b41d46ced0c564496aef721a9b58f8724025a0b1f3f26f1b826f31786f890f8f8781e18b16dbb3c7bff805c7304d1273ac11630ed25a3f0dc41c@34.89.39.114:30303" \ - --network-id 80002 \ - --sensor-id "sensor" \ - --genesis-hash "0x7202b2b53c5a0836e773e319d18922cc756dd67432f9a1f65352b61f4406c697" \ - --fork-id "8b7e4175" -``` - -### Crawl - -To crawl the network for nodes and write the output json to a file. This will -not engage in block or transaction propagation, but it can give a good indicator -of network size, and the output json can be used to quick start other nodes. - -```bash -polycli p2p crawl nodes.json \ - --bootnodes "enode://0cb82b395094ee4a2915e9714894627de9ed8498fb881cec6db7c65e8b9a5bd7f2f25cc84e71e89d0947e51c76e85d0847de848c7782b13c0255247a6758178c@44.232.55.71:30303,enode://88116f4295f5a31538ae409e4d44ad40d22e44ee9342869e7d68bdec55b0f83c1530355ce8b41fbec0928a7d75a5745d528450d30aec92066ab6ba1ee351d710@159.203.9.164:30303,enode://4be7248c3a12c5f95d4ef5fff37f7c44ad1072fdb59701b2e5987c5f3846ef448ce7eabc941c5575b13db0fb016552c1fa5cca0dda1a8008cf6d63874c0f3eb7@3.93.224.197:30303,enode://32dd20eaf75513cf84ffc9940972ab17a62e88ea753b0780ea5eca9f40f9254064dacb99508337043d944c2a41b561a17deaad45c53ea0be02663e55e6a302b2@3.212.183.151:30303" \ - --network-id 137 -``` - -[mainnet-genesis]: https://github.com/0xPolygon/bor/blob/master/builder/files/genesis-mainnet-v1.json -[amoy-genesis]: https://github.com/0xPolygon/bor/blob/master/builder/files/genesis-amoy.json -[bootnodes]: https://wiki.polygon.technology/docs/pos/operate/node/full-node-binaries/#configure-bor-seeds-mainnet diff --git a/doc/polycli_p2p.md b/doc/polycli_p2p.md index 87d302ab6..a0d18aeef 100644 --- a/doc/polycli_p2p.md +++ b/doc/polycli_p2p.md @@ -13,71 +13,6 @@ Set of commands related to devp2p. -## Usage - -### Ping - -Pinging a peer is useful to determine information about the peer and retrieving -the `Hello` and `Status` messages. By default, it will listen to the peer after -the status exchange for blocks and transactions. To disable this behavior, set -the `--listen` flag. - -```bash -polycli p2p ping -``` - -### Sensor - -Running the sensor will do peer discovery and continue to watch for blocks and -transactions from those peers. This is useful for observing the network for -forks and reorgs without the need to run the entire full node infrastructure. - -The bootnodes may change, so refer to the [Wiki][bootnodes] if the sensor is not -discovering peers. - -#### Mainnet - -To run a Polygon Mainnet sensor, copy the `genesis.json` from -[here][mainnet-genesis]. - -```bash -polycli p2p sensor nodes.json \ - --bootnodes "enode://b8f1cc9c5d4403703fbf377116469667d2b1823c0daf16b7250aa576bacf399e42c3930ccfcb02c5df6879565a2b8931335565f0e8d3f8e72385ecf4a4bf160a@3.36.224.80:30303,enode://8729e0c825f3d9cad382555f3e46dcff21af323e89025a0e6312df541f4a9e73abfa562d64906f5e59c51fe6f0501b3e61b07979606c56329c020ed739910759@54.194.245.5:30303" \ - --network-id 137 \ - --sensor-id sensor \ - --rpc "https://polygon-rpc.com" -``` - -#### Amoy - -To run a Polygon Amoy sensor, copy the `genesis.json` from -[here][amoy-genesis]. - -```bash -polycli p2p sensor nodes.json \ - --bootnodes "enode://0ef8758cafc0063405f3f31fe22f2a3b566aa871bd7cd405e35954ec8aa7237c21e1ccc1f65f1b6099ab36db029362bc2fecf001a771b3d9803bbf1968508cef@35.197.249.21:30303,enode://c9c8c18cde48b41d46ced0c564496aef721a9b58f8724025a0b1f3f26f1b826f31786f890f8f8781e18b16dbb3c7bff805c7304d1273ac11630ed25a3f0dc41c@34.89.39.114:30303" \ - --network-id 80002 \ - --sensor-id "sensor" \ - --genesis-hash "0x7202b2b53c5a0836e773e319d18922cc756dd67432f9a1f65352b61f4406c697" \ - --fork-id "8b7e4175" -``` - -### Crawl - -To crawl the network for nodes and write the output json to a file. This will -not engage in block or transaction propagation, but it can give a good indicator -of network size, and the output json can be used to quick start other nodes. - -```bash -polycli p2p crawl nodes.json \ - --bootnodes "enode://0cb82b395094ee4a2915e9714894627de9ed8498fb881cec6db7c65e8b9a5bd7f2f25cc84e71e89d0947e51c76e85d0847de848c7782b13c0255247a6758178c@44.232.55.71:30303,enode://88116f4295f5a31538ae409e4d44ad40d22e44ee9342869e7d68bdec55b0f83c1530355ce8b41fbec0928a7d75a5745d528450d30aec92066ab6ba1ee351d710@159.203.9.164:30303,enode://4be7248c3a12c5f95d4ef5fff37f7c44ad1072fdb59701b2e5987c5f3846ef448ce7eabc941c5575b13db0fb016552c1fa5cca0dda1a8008cf6d63874c0f3eb7@3.93.224.197:30303,enode://32dd20eaf75513cf84ffc9940972ab17a62e88ea753b0780ea5eca9f40f9254064dacb99508337043d944c2a41b561a17deaad45c53ea0be02663e55e6a302b2@3.212.183.151:30303" \ - --network-id 137 -``` - -[mainnet-genesis]: https://github.com/0xPolygon/bor/blob/master/builder/files/genesis-mainnet-v1.json -[amoy-genesis]: https://github.com/0xPolygon/bor/blob/master/builder/files/genesis-amoy.json -[bootnodes]: https://wiki.polygon.technology/docs/pos/operate/node/full-node-binaries/#configure-bor-seeds-mainnet - ## Flags ```bash diff --git a/doc/polycli_p2p_crawl.md b/doc/polycli_p2p_crawl.md index 6cb006fad..e9b118036 100644 --- a/doc/polycli_p2p_crawl.md +++ b/doc/polycli_p2p_crawl.md @@ -19,7 +19,18 @@ polycli p2p crawl [nodes file] [flags] ## Usage -If no nodes.json file exists, it will be created. +To crawl the network for nodes and write the output json to a file. This will +not engage in block or transaction propagation, but it can give a good indicator +of network size, and the output json can be used to quick start other nodes. + +## Example + +```bash +polycli p2p crawl nodes.json \ + --bootnodes "enode://0cb82b395094ee4a2915e9714894627de9ed8498fb881cec6db7c65e8b9a5bd7f2f25cc84e71e89d0947e51c76e85d0847de848c7782b13c0255247a6758178c@44.232.55.71:30303,enode://88116f4295f5a31538ae409e4d44ad40d22e44ee9342869e7d68bdec55b0f83c1530355ce8b41fbec0928a7d75a5745d528450d30aec92066ab6ba1ee351d710@159.203.9.164:30303,enode://4be7248c3a12c5f95d4ef5fff37f7c44ad1072fdb59701b2e5987c5f3846ef448ce7eabc941c5575b13db0fb016552c1fa5cca0dda1a8008cf6d63874c0f3eb7@3.93.224.197:30303,enode://32dd20eaf75513cf84ffc9940972ab17a62e88ea753b0780ea5eca9f40f9254064dacb99508337043d944c2a41b561a17deaad45c53ea0be02663e55e6a302b2@3.212.183.151:30303" \ + --network-id 137 +``` + ## Flags ```bash diff --git a/doc/polycli_p2p_ping.md b/doc/polycli_p2p_ping.md index aa25990d0..6da41266a 100644 --- a/doc/polycli_p2p_ping.md +++ b/doc/polycli_p2p_ping.md @@ -19,12 +19,17 @@ polycli p2p ping [enode/enr or nodes file] [flags] ## Usage -Ping nodes by either giving a single enode/enr or an entire nodes file. +Pinging a peer is useful to determine information about the peer and retrieving +the `Hello` and `Status` messages. By default, it will listen to the peer after +the status exchange for blocks and transactions. To disable this behavior, set +the `--listen` flag. + +## Example + +```bash +polycli p2p ping +``` -This command will establish a handshake and status exchange to get the Hello and -Status messages and output JSON. If providing a enode/enr rather than a nodes -file, then the connection will remain open by default (--listen=true), and you -can see other messages the peer sends (e.g. blocks, transactions, etc.). ## Flags ```bash diff --git a/doc/polycli_p2p_sensor.md b/doc/polycli_p2p_sensor.md index 7e77acb7f..bed39f197 100644 --- a/doc/polycli_p2p_sensor.md +++ b/doc/polycli_p2p_sensor.md @@ -19,7 +19,75 @@ polycli p2p sensor [nodes file] [flags] ## Usage -If no nodes.json file exists, it will be created. +Running the sensor will do peer discovery and continue to watch for blocks and +transactions from those peers. This is useful for observing the network for +forks and reorgs without the need to run the entire full node infrastructure. + +The sensor can persist data to various backends including Google Cloud Datastore +or JSON output. If no nodes.json file exists at the specified path, it will be +created automatically. + +The bootnodes may change, so refer to the [Polygon Knowledge Layer][bootnodes] +if the sensor is not discovering peers. + +## Metrics + +The sensor exposes Prometheus metrics at `http://localhost:2112/metrics` +(configurable via `--prom-port`). For a complete list of available metrics, see +[polycli_p2p_sensor_metrics.md](polycli_p2p_sensor_metrics.md). + +## Examples + +### Mainnet + +To run a Polygon Mainnet sensor, copy the `genesis.json` from [here][mainnet-genesis]. + +```bash +polycli p2p sensor nodes.json \ + --bootnodes "enode://b8f1cc9c5d4403703fbf377116469667d2b1823c0daf16b7250aa576bacf399e42c3930ccfcb02c5df6879565a2b8931335565f0e8d3f8e72385ecf4a4bf160a@3.36.224.80:30303,enode://8729e0c825f3d9cad382555f3e46dcff21af323e89025a0e6312df541f4a9e73abfa562d64906f5e59c51fe6f0501b3e61b07979606c56329c020ed739910759@54.194.245.5:30303" \ + --network-id 137 \ + --sensor-id "sensor" \ + --write-blocks=true \ + --write-block-events=true \ + --write-txs=true \ + --write-tx-events=true \ + --genesis-hash "0xa9c28ce2141b56c474f1dc504bee9b01eb1bd7d1a507580d5519d4437a97de1b" \ + --fork-id "22d523b2" \ + --rpc "https://polygon-rpc.com" \ + --discovery-dns "enrtree://AKUEZKN7PSKVNR65FZDHECMKOJQSGPARGTPPBI7WS2VUL4EGR6XPC@pos.polygon-peers.io" \ + --pprof \ + --verbosity 700 \ + --pretty-logs=true \ + --database "json" +``` + +### Amoy + +To run a Polygon Amoy sensor, copy the `genesis.json` from [here][amoy-genesis]. + +```bash +polycli p2p sensor amoy-nodes.json \ + --bootnodes "enode://b8f1cc9c5d4403703fbf377116469667d2b1823c0daf16b7250aa576bacf399e42c3930ccfcb02c5df6879565a2b8931335565f0e8d3f8e72385ecf4a4bf160a@3.36.224.80:30303,enode://8729e0c825f3d9cad382555f3e46dcff21af323e89025a0e6312df541f4a9e73abfa562d64906f5e59c51fe6f0501b3e61b07979606c56329c020ed739910759@54.194.245.5:30303" \ + --network-id 80002 \ + --sensor-id "sensor-amoy" \ + --write-blocks=true \ + --write-block-events=true \ + --write-txs=true \ + --write-tx-events=true \ + --genesis-hash "0x7202b2b53c5a0836e773e319d18922cc756dd67432f9a1f65352b61f4406c697" \ + --fork-id "8b7e4175" \ + --rpc "https://rpc-amoy.polygon.technology" \ + --discovery-dns "enrtree://AKUEZKN7PSKVNR65FZDHECMKOJQSGPARGTPPBI7WS2VUL4EGR6XPC@amoy.polygon-peers.io" \ + --pprof \ + --verbosity 700 \ + --pretty-logs=true \ + --database "json" +``` + +[mainnet-genesis]: https://github.com/0xPolygon/bor/blob/master/builder/files/genesis-mainnet-v1.json +[amoy-genesis]: https://github.com/0xPolygon/bor/blob/master/builder/files/genesis-amoy.json +[bootnodes]: https://docs.polygon.technology/pos/reference/seed-and-bootnodes/ + ## Flags ```bash diff --git a/doc/polycli_p2p_sensor_metrics.md b/doc/polycli_p2p_sensor_metrics.md new file mode 100644 index 000000000..ca50d7935 --- /dev/null +++ b/doc/polycli_p2p_sensor_metrics.md @@ -0,0 +1,51 @@ + +## Sensor Metrics + + +### sensor_block_range +Difference between head and oldest block numbers + +Metric Type: Gauge + + +### sensor_head_block_age +Time since head block was received (in seconds) + +Metric Type: Gauge + + +### sensor_head_block_number +Current head block number + +Metric Type: Gauge + + +### sensor_head_block_timestamp +Head block timestamp in Unix epoch seconds + +Metric Type: Gauge + + +### sensor_messages +The number and type of messages the sensor has sent and received + +Metric Type: CounterVec + +Variable Labels: +- message +- url +- name +- direction + + +### sensor_oldest_block_number +Oldest block number (floor for parent fetching) + +Metric Type: Gauge + + +### sensor_peers +The number of peers the sensor is connected to + +Metric Type: Gauge + diff --git a/docutil/doc.go b/docutil/doc.go index d557ee5c7..3684b3b7a 100644 --- a/docutil/doc.go +++ b/docutil/doc.go @@ -63,7 +63,7 @@ func genMarkdownPage(cmd *cobra.Command, w io.Writer) error { buf.WriteString("## Description\n\n") buf.WriteString(short + "\n\n") if cmd.Runnable() { - buf.WriteString(fmt.Sprintf("```bash\n%s\n```\n\n", cmd.UseLine())) + fmt.Fprintf(buf, "```bash\n%s\n```\n\n", cmd.UseLine()) } if len(cmd.Long) != 0 { @@ -77,7 +77,7 @@ func genMarkdownPage(cmd *cobra.Command, w io.Writer) error { if len(cmd.Example) > 0 { buf.WriteString("## Examples\n\n") - buf.WriteString(fmt.Sprintf("```bash\n%s\n```\n\n", cmd.Example)) + fmt.Fprintf(buf, "```bash\n%s\n```\n\n", cmd.Example) } if hasSeeAlso(cmd) { @@ -106,7 +106,7 @@ func printToC(buf *bytes.Buffer, cmd *cobra.Command) { } // Print the command flags. This is a modified fork of Cobra's `printOptions` function. -func printFlags(buf *bytes.Buffer, cmd *cobra.Command, name string) error { +func printFlags(buf *bytes.Buffer, cmd *cobra.Command, _ string) error { flags := cmd.NonInheritedFlags() parentFlags := cmd.InheritedFlags() if flags.HasAvailableFlags() || parentFlags.HasAvailableFlags() { @@ -156,9 +156,9 @@ func printSeeAlso(buf *bytes.Buffer, cmd *cobra.Command, name string, linkHandle pname = cleanDuppedSeparator(pname, " ") pname = strings.TrimSuffix(pname, " ") link := pname + ".md" - link = strings.Replace(link, " ", "_", -1) + link = strings.ReplaceAll(link, " ", "_") link = cleanDuppedSeparator(link, "_") - buf.WriteString(fmt.Sprintf("\n- [%s](%s) - %s", pname, linkHandler(link), parent.Short)) + fmt.Fprintf(buf, "\n- [%s](%s) - %s", pname, linkHandler(link), parent.Short) cmd.VisitParents(func(c *cobra.Command) { if c.DisableAutoGenTag { cmd.DisableAutoGenTag = c.DisableAutoGenTag @@ -177,9 +177,9 @@ func printSeeAlso(buf *bytes.Buffer, cmd *cobra.Command, name string, linkHandle cname = cleanDuppedSeparator(cname, " ") cname = strings.TrimSuffix(cname, " ") link := cname + ".md" - link = strings.Replace(link, " ", "_", -1) + link = strings.ReplaceAll(link, " ", "_") link = cleanDuppedSeparator(link, "_") - buf.WriteString(fmt.Sprintf("\n- [%s](%s) - %s\n", cname, linkHandler(link), child.Short)) + fmt.Fprintf(buf, "\n- [%s](%s) - %s\n", cname, linkHandler(link), child.Short) } buf.WriteString("\n") } diff --git a/docutil/main.go b/docutil/main.go index 0edbf18a2..dfc0ee45c 100644 --- a/docutil/main.go +++ b/docutil/main.go @@ -31,4 +31,12 @@ func main() { log.Fatal(err) } fmt.Println("`README.md` updated!") + + // Generate Prometheus metrics documentation + metricsPath := fmt.Sprintf("%s/polycli_p2p_sensor_metrics.md", docDir) + if err := genMetricsDoc(metricsPath); err != nil { + fmt.Println("Unable to generate metrics documentation.") + log.Fatal(err) + } + fmt.Println("Metrics documentation generated!") } diff --git a/docutil/metrics.go b/docutil/metrics.go new file mode 100644 index 000000000..f10bdd1ee --- /dev/null +++ b/docutil/metrics.go @@ -0,0 +1,180 @@ +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "sort" + "strings" +) + +// metricInfo represents a Prometheus metric definition. +type metricInfo struct { + Name string + Namespace string + Help string + Type string // Gauge, Counter, Histogram, Summary, GaugeVec, CounterVec, etc. + Labels []string + File string +} + +// fullName returns the complete metric name with namespace. +func (m metricInfo) fullName() string { + if m.Namespace != "" { + return m.Namespace + "_" + m.Name + } + return m.Name +} + +// genMetricsDoc generates a METRICS.md file documenting all Prometheus metrics. +func genMetricsDoc(outputPath string) error { + metrics := []metricInfo{} + + // Parse p2p package metrics + p2pMetrics, err := parseMetricsFile("p2p/metrics.go") + if err != nil { + return fmt.Errorf("failed to parse p2p/metrics.go: %w", err) + } + metrics = append(metrics, p2pMetrics...) + + // Parse sensor command metrics + sensorMetrics, err := parseMetricsFile("cmd/p2p/sensor/sensor.go") + if err != nil { + return fmt.Errorf("failed to parse cmd/p2p/sensor/sensor.go: %w", err) + } + metrics = append(metrics, sensorMetrics...) + + // Sort metrics by full name + sort.Slice(metrics, func(i, j int) bool { + return metrics[i].fullName() < metrics[j].fullName() + }) + + // Generate markdown + return writeMetricsMarkdown(metrics, outputPath) +} + +// parseMetricsFile extracts Prometheus metrics from a Go file. +func parseMetricsFile(filePath string) ([]metricInfo, error) { + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) + if err != nil { + return nil, err + } + + metrics := []metricInfo{} + relPath, _ := filepath.Rel(".", filePath) + + ast.Inspect(node, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + + // Look for promauto.New* calls + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + + pkg, ok := sel.X.(*ast.Ident) + if !ok || pkg.Name != "promauto" { + return true + } + + metricType := strings.TrimPrefix(sel.Sel.Name, "New") + if metricType == "" { + return true + } + + // Extract the options (first argument should be *Opts struct) + if len(call.Args) == 0 { + return true + } + + metric := metricInfo{ + Type: metricType, + File: relPath, + } + + // Parse the options struct + if comp, ok := call.Args[0].(*ast.CompositeLit); ok { + for _, elt := range comp.Elts { + if kv, ok := elt.(*ast.KeyValueExpr); ok { + key := kv.Key.(*ast.Ident).Name + + switch key { + case "Name": + if lit, ok := kv.Value.(*ast.BasicLit); ok { + metric.Name = strings.Trim(lit.Value, `"`) + } + case "Namespace": + if lit, ok := kv.Value.(*ast.BasicLit); ok { + metric.Namespace = strings.Trim(lit.Value, `"`) + } + case "Help": + if lit, ok := kv.Value.(*ast.BasicLit); ok { + metric.Help = strings.Trim(lit.Value, `"`) + } + } + } + } + } + + // For Vec types, extract labels from second argument + if strings.HasSuffix(metricType, "Vec") && len(call.Args) > 1 { + if comp, ok := call.Args[1].(*ast.CompositeLit); ok { + for _, elt := range comp.Elts { + if lit, ok := elt.(*ast.BasicLit); ok { + label := strings.Trim(lit.Value, `"`) + metric.Labels = append(metric.Labels, label) + } + } + } + } + + if metric.Name != "" { + metrics = append(metrics, metric) + } + + return true + }) + + return metrics, nil +} + +// writeMetricsMarkdown generates the METRICS.md file in panoptichain format. +func writeMetricsMarkdown(metrics []metricInfo, outputPath string) error { + f, err := os.Create(outputPath) + if err != nil { + return err + } + defer f.Close() + + fmt.Fprintln(f) + fmt.Fprintln(f, "## Sensor Metrics") + fmt.Fprintln(f) + + // Group metrics by category (BlockMetrics vs sensor.go metrics) + // For now, just output them in order with their descriptions + for _, m := range metrics { + fmt.Fprintf(f, "\n### %s\n", m.fullName()) + fmt.Fprintln(f, m.Help) + fmt.Fprintln(f) + fmt.Fprintf(f, "Metric Type: %s\n", m.Type) + + if len(m.Labels) > 0 { + fmt.Fprintln(f) + fmt.Fprintln(f, "Variable Labels:") + for _, label := range m.Labels { + fmt.Fprintf(f, "- %s\n", label) + } + } + + fmt.Fprintln(f) + } + + return nil +} diff --git a/p2p/conns.go b/p2p/conns.go index ed0e881d8..631f203cf 100644 --- a/p2p/conns.go +++ b/p2p/conns.go @@ -20,18 +20,34 @@ type Conns struct { // blocks tracks blocks written to the database across all peers // to avoid duplicate writes and requests. blocks *Cache[common.Hash, BlockCache] + + // oldest stores the first block the sensor has seen so when fetching + // parent blocks, it does not request blocks older than this. + oldest *Locked[*types.Header] + + // head keeps track of the current head block of the chain. + head *Locked[eth.NewBlockPacket] } // ConnsOptions contains configuration options for creating a new Conns manager. type ConnsOptions struct { BlocksCache CacheOptions + Head eth.NewBlockPacket } // NewConns creates a new connection manager with a blocks cache. func NewConns(opts ConnsOptions) *Conns { + head := &Locked[eth.NewBlockPacket]{} + head.Set(opts.Head) + + oldest := &Locked[*types.Header]{} + oldest.Set(opts.Head.Block.Header()) + return &Conns{ conns: make(map[string]*conn), blocks: NewCache[common.Hash, BlockCache](opts.BlocksCache), + oldest: oldest, + head: head, } } @@ -91,9 +107,9 @@ func (c *Conns) Nodes() []*enode.Node { return nodes } -// GetPeerConnectedAt returns the connection time for a peer by their ID. +// PeerConnectedAt returns the connection time for a peer by their ID. // Returns zero time if the peer is not found. -func (c *Conns) GetPeerConnectedAt(peerID string) time.Time { +func (c *Conns) PeerConnectedAt(peerID string) time.Time { c.mu.RLock() defer c.mu.RUnlock() @@ -108,3 +124,26 @@ func (c *Conns) GetPeerConnectedAt(peerID string) time.Time { func (c *Conns) Blocks() *Cache[common.Hash, BlockCache] { return c.blocks } + +// OldestBlock returns the oldest block the sensor will fetch parents for. +// This is set once at initialization to the head block and acts as a floor +// to prevent the sensor from crawling backwards indefinitely. +func (c *Conns) OldestBlock() *types.Header { + return c.oldest.Get() +} + +// HeadBlock returns the current head block packet. +func (c *Conns) HeadBlock() eth.NewBlockPacket { + return c.head.Get() +} + +// UpdateHeadBlock updates the head block if the provided block is newer. +// Returns true if the head block was updated, false otherwise. +func (c *Conns) UpdateHeadBlock(packet eth.NewBlockPacket) bool { + return c.head.Update(func(current eth.NewBlockPacket) (eth.NewBlockPacket, bool) { + if current.Block == nil || (packet.Block.NumberU64() > current.Block.NumberU64() && packet.TD.Cmp(current.TD) == 1) { + return packet, true + } + return current, false + }) +} diff --git a/p2p/locked.go b/p2p/locked.go new file mode 100644 index 000000000..d2b8593e6 --- /dev/null +++ b/p2p/locked.go @@ -0,0 +1,34 @@ +package p2p + +import "sync" + +// Locked wraps a value with a RWMutex for thread-safe access. +type Locked[T any] struct { + value T + mu sync.RWMutex +} + +// Get returns the current value (thread-safe read). +func (l *Locked[T]) Get() T { + l.mu.RLock() + defer l.mu.RUnlock() + return l.value +} + +// Set updates the value (thread-safe write). +func (l *Locked[T]) Set(value T) { + l.mu.Lock() + defer l.mu.Unlock() + l.value = value +} + +// Update atomically updates the value using a function. +// The function receives the current value and returns the new value and a result. +// The result is returned to the caller. +func (l *Locked[T]) Update(fn func(T) (T, bool)) bool { + l.mu.Lock() + defer l.mu.Unlock() + newValue, changed := fn(l.value) + l.value = newValue + return changed +} diff --git a/p2p/metrics.go b/p2p/metrics.go new file mode 100644 index 000000000..2867c9033 --- /dev/null +++ b/p2p/metrics.go @@ -0,0 +1,74 @@ +package p2p + +import ( + "time" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// BlockMetrics contains Prometheus metrics for tracking head and oldest blocks. +type BlockMetrics struct { + HeadBlockNumber prometheus.Gauge + HeadBlockTimestamp prometheus.Gauge + HeadBlockAge prometheus.Gauge + OldestBlockNumber prometheus.Gauge + BlockRange prometheus.Gauge +} + +// NewBlockMetrics creates and registers all block-related Prometheus metrics. +func NewBlockMetrics(block *types.Block) *BlockMetrics { + m := &BlockMetrics{ + HeadBlockNumber: promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "sensor", + Name: "head_block_number", + Help: "Current head block number", + }), + HeadBlockTimestamp: promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "sensor", + Name: "head_block_timestamp", + Help: "Head block timestamp in Unix epoch seconds", + }), + HeadBlockAge: promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "sensor", + Name: "head_block_age", + Help: "Time since head block was received (in seconds)", + }), + OldestBlockNumber: promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "sensor", + Name: "oldest_block_number", + Help: "Oldest block number (floor for parent fetching)", + }), + BlockRange: promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "sensor", + Name: "block_range", + Help: "Difference between head and oldest block numbers", + }), + } + + m.HeadBlockNumber.Set(float64(block.NumberU64())) + m.HeadBlockTimestamp.Set(float64(block.Time())) + m.HeadBlockAge.Set(0) + m.OldestBlockNumber.Set(float64(block.NumberU64())) + m.BlockRange.Set(0) + + return m +} + +// Update updates all block metrics. +func (m *BlockMetrics) Update(block *types.Block, oldest *types.Header) { + if m == nil || block == nil || oldest == nil { + return + } + + hn := block.NumberU64() + on := oldest.Number.Uint64() + ht := time.Unix(int64(block.Time()), 0) + + m.HeadBlockNumber.Set(float64(hn)) + m.HeadBlockTimestamp.Set(float64(block.Time())) + m.HeadBlockAge.Set(time.Since(ht).Seconds()) + m.OldestBlockNumber.Set(float64(on)) + m.BlockRange.Set(float64(hn - on)) +} diff --git a/p2p/protocol.go b/p2p/protocol.go index 4500aabe4..c7b828034 100644 --- a/p2p/protocol.go +++ b/p2p/protocol.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "math/big" - "sync" "time" "github.com/ethereum/go-ethereum/common" @@ -31,15 +30,13 @@ type BlockCache struct { // conn represents an individual connection with a peer. type conn struct { - sensorID string - node *enode.Node - logger zerolog.Logger - rw ethp2p.MsgReadWriter - db database.Database - head *HeadBlock - headMutex *sync.RWMutex - counter *prometheus.CounterVec - peer *ethp2p.Peer + sensorID string + node *enode.Node + logger zerolog.Logger + rw ethp2p.MsgReadWriter + db database.Database + counter *prometheus.CounterVec + peer *ethp2p.Peer // requests is used to store the request ID and the block hash. This is used // when fetching block bodies because the eth protocol block bodies do not @@ -55,10 +52,6 @@ type conn struct { // the blocks cache shared across all peers. conns *Conns - // oldestBlock stores the first block the sensor has seen so when fetching - // parent blocks, it does not request blocks older than this. - oldestBlock *types.Header - // connectedAt stores when this peer connection was established. connectedAt time.Time @@ -79,24 +72,11 @@ type EthProtocolOptions struct { ForkID forkid.ID MsgCounter *prometheus.CounterVec - // Head keeps track of the current head block of the chain. This is required - // when doing the status exchange. - Head *HeadBlock - HeadMutex *sync.RWMutex - // Cache configurations RequestsCache CacheOptions ParentsCache CacheOptions } -// HeadBlock contains the necessary head block data for the status message. -type HeadBlock struct { - Hash common.Hash - TotalDifficulty *big.Int - Number uint64 - Time uint64 -} - // NewEthProtocol creates the new eth protocol. This will handle writing the // status exchange, message handling, and writing blocks/txs to the database. func NewEthProtocol(version uint, opts EthProtocolOptions) ethp2p.Protocol { @@ -115,8 +95,6 @@ func NewEthProtocol(version uint, opts EthProtocolOptions) ethp2p.Protocol { requests: NewCache[uint64, common.Hash](opts.RequestsCache), requestNum: 0, parents: NewCache[common.Hash, struct{}](opts.ParentsCache), - head: opts.Head, - headMutex: opts.HeadMutex, counter: opts.MsgCounter, peer: p, conns: opts.Conns, @@ -125,17 +103,16 @@ func NewEthProtocol(version uint, opts EthProtocolOptions) ethp2p.Protocol { peerFullname: p.Fullname(), } - c.headMutex.RLock() + head := c.conns.HeadBlock() status := eth.StatusPacket{ ProtocolVersion: uint32(version), NetworkID: opts.NetworkID, Genesis: opts.GenesisHash, ForkID: opts.ForkID, - Head: opts.Head.Hash, - TD: opts.Head.TotalDifficulty, + Head: head.Block.Hash(), + TD: head.TD, } err := c.statusExchange(&status) - c.headMutex.RUnlock() if err != nil { return err } @@ -326,15 +303,15 @@ func (c *conn) getBlockData(hash common.Hash, cache BlockCache, isParent bool) e } // getParentBlock will send a request to the peer if the parent of the header -// does not exist in the database. +// does not exist in the database. It only fetches parents back to the oldest +// block (initialized to the head block at sensor startup). func (c *conn) getParentBlock(ctx context.Context, header *types.Header) error { if !c.db.ShouldWriteBlocks() { return nil } - if c.oldestBlock == nil { - c.logger.Info().Interface("block", header).Msg("Setting oldest block") - c.oldestBlock = header + oldestBlock := c.conns.OldestBlock() + if oldestBlock == nil { return nil } @@ -344,7 +321,8 @@ func (c *conn) getParentBlock(ctx context.Context, header *types.Header) error { return nil } - if c.db.HasBlock(ctx, header.ParentHash) || header.Number.Cmp(c.oldestBlock.Number) != 1 { + // Don't fetch parents older than our starting point (oldest block) + if c.db.HasBlock(ctx, header.ParentHash) || header.Number.Cmp(oldestBlock.Number) != 1 { return nil } @@ -524,17 +502,13 @@ func (c *conn) handleNewBlock(ctx context.Context, msg ethp2p.Msg) error { c.countMsgReceived(block.Name(), 1) // Set the head block if newer. - c.headMutex.Lock() - if block.Block.Number().Uint64() > c.head.Number && block.TD.Cmp(c.head.TotalDifficulty) == 1 { - *c.head = HeadBlock{ - Hash: hash, - TotalDifficulty: block.TD, - Number: block.Block.Number().Uint64(), - Time: block.Block.Time(), - } - c.logger.Info().Interface("head", c.head).Msg("Setting head block") + if c.conns.UpdateHeadBlock(block) { + c.logger.Info(). + Str("hash", hash.Hex()). + Uint64("number", block.Block.Number().Uint64()). + Str("td", block.TD.String()). + Msg("Updated head block") } - c.headMutex.Unlock() if err := c.getParentBlock(ctx, block.Block.Header()); err != nil { return err diff --git a/rpctypes/rpctypes.go b/rpctypes/rpctypes.go index 8b4368e9c..db7e853ab 100644 --- a/rpctypes/rpctypes.go +++ b/rpctypes/rpctypes.go @@ -10,6 +10,7 @@ import ( "strings" ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/rs/zerolog/log" ) @@ -568,7 +569,7 @@ func ConvHexToBigInt(raw any) (bi *big.Int, err error) { if err != nil { return nil, err } - hexString = strings.Replace(hexString, "0x", "", -1) + hexString = strings.ReplaceAll(hexString, "0x", "") if len(hexString)%2 != 0 { hexString = "0" + hexString } @@ -613,7 +614,7 @@ func ConvHexToUint64(raw any) (uint64, error) { return 0, err } - hexString = strings.Replace(hexString, "0x", "", -1) + hexString = strings.ReplaceAll(hexString, "0x", "") if len(hexString)%2 != 0 { hexString = "0" + hexString } @@ -643,8 +644,32 @@ func NewRawBlockResponseFromAny(raw any) (*RawBlockResponse, error) { } +// ToBlock converts a RawBlockResponse to a types.Block with header only. +// The block will not contain transactions, uncles, or withdrawals. +func (r *RawBlockResponse) ToBlock() *types.Block { + header := &types.Header{ + ParentHash: r.ParentHash.ToHash(), + UncleHash: r.SHA3Uncles.ToHash(), + Coinbase: r.Miner.ToAddress(), + Root: r.StateRoot.ToHash(), + TxHash: r.TransactionsRoot.ToHash(), + ReceiptHash: r.ReceiptsRoot.ToHash(), + Bloom: types.BytesToBloom(r.LogsBloom.ToBytes()), + Difficulty: r.Difficulty.ToBigInt(), + Number: r.Number.ToBigInt(), + GasLimit: r.GasLimit.ToUint64(), + GasUsed: r.GasUsed.ToUint64(), + Time: r.Timestamp.ToUint64(), + Extra: r.ExtraData.ToBytes(), + MixDigest: r.MixHash.ToHash(), + Nonce: types.EncodeNonce(r.Nonce.ToUint64()), + BaseFee: r.BaseFeePerGas.ToBigInt(), + } + return types.NewBlockWithHeader(header) +} + func normalizeHexString(s string) string { - hexString := strings.Replace(s, "0x", "", -1) + hexString := strings.ReplaceAll(s, "0x", "") if len(hexString)%2 != 0 { hexString = "0" + hexString }