Merge branch 'minhd-vu/oldest-and-head-block' into minhd-vu/sensor-first-seen

Author: minhd-vu

cmd/p2p/crawl/crawl.go

@@ -1,6 +1,7 @@
 package crawl
 
 import (
+	_ "embed"
 	"fmt"
 	"time"
 
@@ -32,6 +33,8 @@ type (
 )
 
 var (
+	//go:embed usage.md
+	crawlUsage       string
 	inputCrawlParams crawlParams
 )
 
@@ -40,7 +43,7 @@ var (
 var CrawlCmd = &cobra.Command{
 	Use:   "crawl [nodes file]",
 	Short: "Crawl a network on the devp2p layer and generate a nodes JSON file.",
-	Long:  "If no nodes.json file exists, it will be created.",
+	Long:  crawlUsage,
 	Args:  cobra.MinimumNArgs(1),
 	PreRunE: func(cmd *cobra.Command, args []string) (err error) {
 		inputCrawlParams.NodesFile = args[0]

cmd/p2p/crawl/usage.md

@@ -0,0 +1,11 @@
+To crawl the network for nodes and write the output json to a file. This will
+not engage in block or transaction propagation, but it can give a good indicator
+of network size, and the output json can be used to quick start other nodes.
+
+## Example
+
+```bash
+polycli p2p crawl nodes.json \
+  --bootnodes "enode://0cb82b395094ee4a2915e9714894627de9ed8498fb881cec6db7c65e8b9a5bd7f2f25cc84e71e89d0947e51c76e85d0847de848c7782b13c0255247a6758178c@44.232.55.71:30303,enode://88116f4295f5a31538ae409e4d44ad40d22e44ee9342869e7d68bdec55b0f83c1530355ce8b41fbec0928a7d75a5745d528450d30aec92066ab6ba1ee351d710@159.203.9.164:30303,enode://4be7248c3a12c5f95d4ef5fff37f7c44ad1072fdb59701b2e5987c5f3846ef448ce7eabc941c5575b13db0fb016552c1fa5cca0dda1a8008cf6d63874c0f3eb7@3.93.224.197:30303,enode://32dd20eaf75513cf84ffc9940972ab17a62e88ea753b0780ea5eca9f40f9254064dacb99508337043d944c2a41b561a17deaad45c53ea0be02663e55e6a302b2@3.212.183.151:30303" \
+  --network-id 137
+```

cmd/p2p/p2p.go

@@ -3,22 +3,16 @@ package p2p
 import (
 	"github.com/spf13/cobra"
 
-	_ "embed"
-
 	"github.com/0xPolygon/polygon-cli/cmd/p2p/crawl"
 	"github.com/0xPolygon/polygon-cli/cmd/p2p/nodelist"
 	"github.com/0xPolygon/polygon-cli/cmd/p2p/ping"
 	"github.com/0xPolygon/polygon-cli/cmd/p2p/query"
 	"github.com/0xPolygon/polygon-cli/cmd/p2p/sensor"
 )
 
-//go:embed usage.md
-var usage string
-
 var P2pCmd = &cobra.Command{
 	Use:   "p2p",
 	Short: "Set of commands related to devp2p.",
-	Long:  usage,
 }
 
 func init() {

cmd/p2p/ping/ping.go

@@ -2,6 +2,7 @@ package ping
 
 import (
 	"crypto/ecdsa"
+	_ "embed"
 	"net"
 	"sync"
 	"time"
@@ -30,19 +31,16 @@ type (
 )
 
 var (
+	//go:embed usage.md
+	pingUsage       string
 	inputPingParams pingParams
 )
 
 var PingCmd = &cobra.Command{
 	Use:   "ping [enode/enr or nodes file]",
 	Short: "Ping node(s) and return the output.",
-	Long: `Ping nodes by either giving a single enode/enr or an entire nodes file.
-
-This command will establish a handshake and status exchange to get the Hello and
-Status messages and output JSON. If providing a enode/enr rather than a nodes
-file, then the connection will remain open by default (--listen=true), and you
-can see other messages the peer sends (e.g. blocks, transactions, etc.).`,
-	Args: cobra.MinimumNArgs(1),
+	Long:  pingUsage,
+	Args:  cobra.MinimumNArgs(1),
 	PreRunE: func(cmd *cobra.Command, args []string) (err error) {
 		inputPingParams.privateKey, err = p2p.ParsePrivateKey(inputPingParams.KeyFile, inputPingParams.PrivateKey)
 		if err != nil {

cmd/p2p/ping/usage.md

@@ -0,0 +1,10 @@
+Pinging a peer is useful to determine information about the peer and retrieving
+the `Hello` and `Status` messages. By default, it will listen to the peer after
+the status exchange for blocks and transactions. To disable this behavior, set
+the `--listen` flag.
+
+## Example
+
+```bash
+polycli p2p ping <enode/enr or nodes.json file>
+```

cmd/p2p/sensor/api.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/0xPolygon/polygon-cli/p2p"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/eth/protocols/eth"
 	ethp2p "github.com/ethereum/go-ethereum/p2p"
 	"github.com/prometheus/client_golang/prometheus"
@@ -27,12 +28,33 @@ type peerData struct {
 	DurationSeconds float64          `json:"duration_seconds"`
 }
 
+// blockInfo represents basic block information.
+type blockInfo struct {
+	Hash   string `json:"hash"`
+	Number uint64 `json:"number"`
+}
+
+// newBlockInfo creates a blockInfo from a types.Header.
+// Returns nil if the header is nil.
+func newBlockInfo(header *types.Header) *blockInfo {
+	if header == nil {
+		return nil
+	}
+
+	return &blockInfo{
+		Hash:   header.Hash().Hex(),
+		Number: header.Number.Uint64(),
+	}
+}
+
 // apiData represents all sensor information including node info and peer data.
 type apiData struct {
 	ENR       string              `json:"enr"`
 	URL       string              `json:"enode"`
 	PeerCount int                 `json:"peer_count"`
 	Peers     map[string]peerData `json:"peers"`
+	Head      *blockInfo          `json:"head_block"`
+	Oldest    *blockInfo          `json:"oldest_block"`
 }
 
 // handleAPI sets up the API for interacting with the sensor. All endpoints
@@ -54,7 +76,7 @@ func handleAPI(server *ethp2p.Server, counter *prometheus.CounterVec, conns *p2p
 			url := peer.Node().URLv4()
 			peerID := peer.Node().ID().String()
 			name := peer.Fullname()
-			connectedAt := conns.GetPeerConnectedAt(peerID)
+			connectedAt := conns.PeerConnectedAt(peerID)
 			if connectedAt.IsZero() {
 				continue
 			}
@@ -71,11 +93,21 @@ func handleAPI(server *ethp2p.Server, counter *prometheus.CounterVec, conns *p2p
 			peers[url] = msgs
 		}
 
+		head := conns.HeadBlock()
+		oldest := conns.OldestBlock()
+
+		var headHeader *types.Header
+		if head.Block != nil {
+			headHeader = head.Block.Header()
+		}
+
 		data := apiData{
 			ENR:       server.NodeInfo().ENR,
 			URL:       server.Self().URLv4(),
 			PeerCount: len(peers),
 			Peers:     peers,
+			Head:      newBlockInfo(headHeader),
+			Oldest:    newBlockInfo(oldest),
 		}
 
 		if err := json.NewEncoder(w).Encode(data); err != nil {

cmd/p2p/sensor/sensor.go

@@ -3,11 +3,11 @@ package sensor
 import (
 	"context"
 	"crypto/ecdsa"
+	_ "embed"
 	"errors"
 	"fmt"
 	"os"
 	"os/signal"
-	"sync"
 	"syscall"
 	"time"
 
@@ -17,6 +17,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/forkid"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/protocols/eth"
 	ethp2p "github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/dnsdisc"
 	"github.com/ethereum/go-ethereum/p2p/enode"
@@ -82,6 +83,8 @@ type (
 )
 
 var (
+	//go:embed usage.md
+	sensorUsage       string
 	inputSensorParams sensorParams
 )
 
@@ -90,7 +93,7 @@ var (
 var SensorCmd = &cobra.Command{
 	Use:   "sensor [nodes file]",
 	Short: "Start a devp2p sensor that discovers other peers and will receive blocks and transactions.",
-	Long:  "If no nodes.json file exists, it will be created.",
+	Long:  sensorUsage,
 	Args:  cobra.MinimumNArgs(1),
 	PreRunE: func(cmd *cobra.Command, args []string) (err error) {
 		inputSensorParams.NodesFile = args[0]
@@ -170,14 +173,14 @@ var SensorCmd = &cobra.Command{
 		// Fetch the latest block which will be used later when crafting the status
 		// message. This call will only be made once and stored in the head field
 		// until the sensor receives a new block it can overwrite it with.
-		block, err := getLatestBlock(inputSensorParams.RPC)
+		rpcBlock, err := getLatestBlock(inputSensorParams.RPC)
 		if err != nil {
 			return err
 		}
-		head := p2p.HeadBlock{
-			Hash:            block.Hash.ToHash(),
-			TotalDifficulty: block.TotalDifficulty.ToBigInt(),
-			Number:          block.Number.ToUint64(),
+
+		head := eth.NewBlockPacket{
+			Block: rpcBlock.ToBlock(),
+			TD:    rpcBlock.TotalDifficulty.ToBigInt(),
 		}
 
 		peersGauge := promauto.NewGauge(prometheus.GaugeOpts{
@@ -192,10 +195,13 @@ var SensorCmd = &cobra.Command{
 			Help:      "The number and type of messages the sensor has sent and received",
 		}, []string{"message", "url", "name", "direction"})
 
+		metrics := p2p.NewBlockMetrics(head.Block)
+
 		// Create peer connection manager for broadcasting transactions
 		// and managing the global blocks cache
 		conns := p2p.NewConns(p2p.ConnsOptions{
 			BlocksCache: inputSensorParams.BlocksCache,
+			Head:        head,
 		})
 
 		opts := p2p.EthProtocolOptions{
@@ -206,8 +212,6 @@ var SensorCmd = &cobra.Command{
 			SensorID:      inputSensorParams.SensorID,
 			NetworkID:     inputSensorParams.NetworkID,
 			Conns:         conns,
-			Head:          &head,
-			HeadMutex:     &sync.RWMutex{},
 			ForkID:        forkid.ID{Hash: [4]byte(inputSensorParams.ForkID)},
 			MsgCounter:    msgCounter,
 			RequestsCache: inputSensorParams.RequestsCache,
@@ -280,6 +284,8 @@ var SensorCmd = &cobra.Command{
 				peersGauge.Set(float64(server.PeerCount()))
 				db.WritePeers(cmd.Context(), server.Peers(), time.Now())
 
+				metrics.Update(conns.HeadBlock().Block, conns.OldestBlock())
+
 				urls := []string{}
 				for _, peer := range server.Peers() {
 					urls = append(urls, peer.Node().URLv4())

cmd/p2p/sensor/usage.md

@@ -0,0 +1,68 @@
+Running the sensor will do peer discovery and continue to watch for blocks and
+transactions from those peers. This is useful for observing the network for
+forks and reorgs without the need to run the entire full node infrastructure.
+
+The sensor can persist data to various backends including Google Cloud Datastore
+or JSON output. If no nodes.json file exists at the specified path, it will be
+created automatically.
+
+The bootnodes may change, so refer to the [Polygon Knowledge Layer][bootnodes]
+if the sensor is not discovering peers.
+
+## Metrics
+
+The sensor exposes Prometheus metrics at `http://localhost:2112/metrics`
+(configurable via `--prom-port`). For a complete list of available metrics, see
+[polycli_p2p_sensor_metrics.md](polycli_p2p_sensor_metrics.md).
+
+## Examples
+
+### Mainnet
+
+To run a Polygon Mainnet sensor, copy the `genesis.json` from [here][mainnet-genesis].
+
+```bash
+polycli p2p sensor nodes.json \
+  --bootnodes "enode://b8f1cc9c5d4403703fbf377116469667d2b1823c0daf16b7250aa576bacf399e42c3930ccfcb02c5df6879565a2b8931335565f0e8d3f8e72385ecf4a4bf160a@3.36.224.80:30303,enode://8729e0c825f3d9cad382555f3e46dcff21af323e89025a0e6312df541f4a9e73abfa562d64906f5e59c51fe6f0501b3e61b07979606c56329c020ed739910759@54.194.245.5:30303" \
+  --network-id 137 \
+  --sensor-id "sensor" \
+  --write-blocks=true \
+  --write-block-events=true \
+  --write-txs=true \
+  --write-tx-events=true \
+  --genesis-hash "0xa9c28ce2141b56c474f1dc504bee9b01eb1bd7d1a507580d5519d4437a97de1b" \
+  --fork-id "22d523b2" \
+  --rpc "https://polygon-rpc.com" \
+  --discovery-dns "enrtree://AKUEZKN7PSKVNR65FZDHECMKOJQSGPARGTPPBI7WS2VUL4EGR6XPC@pos.polygon-peers.io" \
+  --pprof \
+  --verbosity 700 \
+  --pretty-logs=true \
+  --database "json"
+```
+
+### Amoy
+
+To run a Polygon Amoy sensor, copy the `genesis.json` from [here][amoy-genesis].
+
+```bash
+polycli p2p sensor amoy-nodes.json \
+  --bootnodes "enode://b8f1cc9c5d4403703fbf377116469667d2b1823c0daf16b7250aa576bacf399e42c3930ccfcb02c5df6879565a2b8931335565f0e8d3f8e72385ecf4a4bf160a@3.36.224.80:30303,enode://8729e0c825f3d9cad382555f3e46dcff21af323e89025a0e6312df541f4a9e73abfa562d64906f5e59c51fe6f0501b3e61b07979606c56329c020ed739910759@54.194.245.5:30303" \
+  --network-id 80002 \
+  --sensor-id "sensor-amoy" \
+  --write-blocks=true \
+  --write-block-events=true \
+  --write-txs=true \
+  --write-tx-events=true \
+  --genesis-hash "0x7202b2b53c5a0836e773e319d18922cc756dd67432f9a1f65352b61f4406c697" \
+  --fork-id "8b7e4175" \
+  --rpc "https://rpc-amoy.polygon.technology" \
+  --discovery-dns "enrtree://AKUEZKN7PSKVNR65FZDHECMKOJQSGPARGTPPBI7WS2VUL4EGR6XPC@amoy.polygon-peers.io" \
+  --pprof \
+  --verbosity 700 \
+  --pretty-logs=true \
+  --database "json"
+```
+
+[mainnet-genesis]: https://github.com/0xPolygon/bor/blob/master/builder/files/genesis-mainnet-v1.json
+[amoy-genesis]: https://github.com/0xPolygon/bor/blob/master/builder/files/genesis-amoy.json
+[bootnodes]: https://docs.polygon.technology/pos/reference/seed-and-bootnodes/