Fix eth_syncing checks, track nodes that are lagging behind

Author: wille

internal/eth/eth.go

@@ -0,0 +1,27 @@
+package eth
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+)
+
+// ParseHexNumber parses a hex string and returns decimal number
+// Note that this function does not handle large Ethereum balances in wei.
+func ParseHexNumber(hex string) (uint64, error) {
+	if !strings.HasPrefix(hex, "0x") {
+		return 0, fmt.Errorf("hex number must start with 0x")
+	}
+
+	i, err := strconv.ParseUint(hex[2:], 16, 64)
+	if err != nil {
+		return 0, err
+	}
+
+	if i == math.MaxUint64 {
+		return 0, fmt.Errorf("number is too large")
+	}
+
+	return i, nil
+}

internal/eth/eth_test.go

@@ -0,0 +1,18 @@
+package eth
+
+import (
+	"testing"
+)
+
+func TestParseHexNumber(t *testing.T) {
+	i, _ := ParseHexNumber("0x11")
+
+	if i != 17 {
+		t.Errorf("ParseHexNumber(%q) = %v, want %v", "0x11", i, 17)
+	}
+
+	_, err := ParseHexNumber("0x10000000000000000000000000000000000000000000000000000000000000000")
+	if err == nil {
+		t.Errorf("Did not fail on out of range number")
+	}
+}

internal/healthcheck.go

@@ -4,9 +4,10 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"strconv"
+	"log/slog"
 	"time"
 
+	"github.com/wille/haprovider/internal/eth"
 	"github.com/wille/haprovider/internal/rpc"
 )
 
@@ -18,6 +19,8 @@ const (
 
 	// Internal JSON-RPC error
 	EthErrorInternalError = -32603
+
+	BlockLagTolerance = 50
 )
 
 type Healthcheck func(ctx context.Context, p *Endpoint, e *Provider, read rpc.ReaderFunc) error
@@ -58,8 +61,24 @@ func EthereumHealthCheck(ctx context.Context, p *Endpoint, e *Provider, read rpc
 		return err
 	}
 
-	if _, ok := res.Result.(string); !ok {
-		return fmt.Errorf("block number is not a string")
+	if currentBlockHex, ok := res.Result.(string); ok {
+		currentBlock, err := eth.ParseHexNumber(currentBlockHex)
+		if err != nil {
+			return fmt.Errorf("failed to parse block number: %v (%s)", err, currentBlockHex)
+		}
+
+		if e.highestBlock != 0 && currentBlock < e.highestBlock-BlockLagTolerance {
+			log := slog.With("provider", e.Name, "endpoint", p.Name)
+			log.Warn("node is behind", "currentBlock", currentBlock, "highestBlock", e.highestBlock)
+
+			return fmt.Errorf("node is behind: currentBlock=%d, highestBlock=%d", currentBlock, e.highestBlock)
+		}
+
+		if currentBlock > e.highestBlock {
+			e.highestBlock = currentBlock
+		}
+	} else {
+		return fmt.Errorf("block number is not a string: %v", res.Result)
 	}
 
 	res, err = read(ctx, rpc.NewRequest("ha_syncing", "eth_syncing", nil), true)
@@ -71,8 +90,8 @@ func EthereumHealthCheck(ctx context.Context, p *Endpoint, e *Provider, read rpc
 		currentBlockHex := m["currentBlock"].(string)
 		highestBlockHex := m["highestBlock"].(string)
 
-		currentBlock, _ := strconv.ParseUint(currentBlockHex, 16, 64)
-		highestBlock, _ := strconv.ParseUint(highestBlockHex, 16, 64)
+		currentBlock, _ := eth.ParseHexNumber(currentBlockHex)
+		highestBlock, _ := eth.ParseHexNumber(highestBlockHex)
 
 		return fmt.Errorf("node is not synced: currentBlock=%d, highestBlock=%d", currentBlock, highestBlock)
 	}
@@ -102,7 +121,16 @@ func SolanaHealthcheck(ctx context.Context, endpoint *Endpoint, provider *Provid
 		return err
 	}
 
-	if _, ok := res.Result.(float64); !ok {
+	if bf, ok := res.Result.(float64); !ok {
+		blockHeight := uint64(bf)
+
+		if blockHeight > provider.highestBlock {
+			provider.highestBlock = blockHeight
+		}
+
+		if blockHeight < provider.highestBlock-BlockLagTolerance {
+			return fmt.Errorf("node is behind: currentBlock=%d, highestBlock=%d", blockHeight, provider.highestBlock)
+		}
 		return fmt.Errorf("block height is not a number: %v", res.Result)
 	}
 

internal/provider.go

@@ -32,6 +32,8 @@ type Provider struct {
 	online   bool
 	onlineAt time.Time
 
+	highestBlock uint64
+
 	clientVersion string
 
 	m sync.Mutex
@@ -124,17 +126,8 @@ func (e *Provider) IsRateLimited() bool {
 	return !e.retryAt.IsZero() && time.Since(e.retryAt) < 0
 }
 
-// func (p *Provider) DelayedHealthcheck(p *Provider, delay time.Duration) error {
-// 	t := time.NewTicker(delay)
-// 	defer t.Stop()
-
-// 	<-t.C
-
-// 	return e.Healthcheck(p)
-// }
-
 func (e *Provider) Healthcheck(p *Endpoint) error {
-	ctx := context.TODO()
+	ctx := context.Background()
 
 	fn := func(ctx context.Context, req *rpc.Request, errRpcError bool) (*rpc.Response, error) {
 		res, err := SendHTTPRPCRequest(ctx, e, req)