feat: integrate gRPC service into Ethereum node lifecycle with configuration options for EnableGRPC, GRPCHost, and GRPCPort, register service as node lifecycle component for automatic startup and graceful shutdown, expose miner access through EthAPIBackend, and enable low-latency binary trading operations alongside traditional JSON-RPC

Author: floor-licker

api/grpc/server.go

@@ -37,7 +37,7 @@ import (
 // Backend defines the necessary methods from the Ethereum backend for the gRPC server.
 type Backend interface {
 	ChainConfig() *params.ChainConfig
-	BlockChain() *core.BlockChain
+	CurrentBlock() *types.Header
 	StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)
 	Miner() *miner.Miner
 }
@@ -97,7 +97,7 @@ func (s *TraderServer) SimulateBundle(ctx context.Context, req *SimulateBundleRe
 	}
 
 	// Get current block header for simulation
-	header := s.backend.BlockChain().CurrentBlock()
+	header := s.backend.CurrentBlock()
 	if header == nil {
 		return nil, errors.New("current block not found")
 	}
@@ -323,7 +323,8 @@ func (s *TraderServer) CallContract(ctx context.Context, req *CallContractReques
 	}
 
 	// Create EVM and execute call
-	blockContext := core.NewEVMBlockContext(header, s.backend.BlockChain(), nil)
+	// Note: Using nil for chain reader as we only need basic block context for eth_call
+	blockContext := core.NewEVMBlockContext(header, nil, nil)
 	vmConfig := vm.Config{}
 
 	evm := vm.NewEVM(blockContext, stateDB, s.backend.ChainConfig(), vmConfig)

api/grpc/service.go

@@ -0,0 +1,84 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package grpc
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/ethereum/go-ethereum/log"
+	"google.golang.org/grpc"
+)
+
+// Service implements the node.Lifecycle interface for the gRPC server.
+type Service struct {
+	backend  Backend
+	server   *grpc.Server
+	listener net.Listener
+	host     string
+	port     int
+}
+
+// NewService creates a new gRPC service.
+func NewService(backend Backend, host string, port int) *Service {
+	return &Service{
+		backend: backend,
+		host:    host,
+		port:    port,
+	}
+}
+
+// Start implements node.Lifecycle, starting the gRPC server.
+func (s *Service) Start() error {
+	addr := fmt.Sprintf("%s:%d", s.host, s.port)
+	lis, err := net.Listen("tcp", addr)
+	if err != nil {
+		return fmt.Errorf("failed to listen on %s: %w", addr, err)
+	}
+	s.listener = lis
+
+	// Create gRPC server with options
+	s.server = grpc.NewServer(
+		grpc.MaxRecvMsgSize(100*1024*1024), // 100MB for large bundles
+		grpc.MaxSendMsgSize(100*1024*1024), // 100MB for large simulation results
+	)
+
+	// Register TraderService
+	traderServer := NewTraderServer(s.backend)
+	RegisterTraderServiceServer(s.server, traderServer)
+
+	// Start serving in a goroutine
+	go func() {
+		log.Info("gRPC server started", "addr", addr)
+		if err := s.server.Serve(lis); err != nil {
+			log.Error("gRPC server failed", "err", err)
+		}
+	}()
+
+	return nil
+}
+
+// Stop implements node.Lifecycle, stopping the gRPC server.
+func (s *Service) Stop() error {
+	if s.server != nil {
+		log.Info("gRPC server stopping")
+		s.server.GracefulStop()
+		log.Info("gRPC server stopped")
+	}
+	return nil
+}
+

eth/api_backend.go

@@ -40,6 +40,7 @@ import (
 	"github.com/ethereum/go-ethereum/eth/tracers"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/miner"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
 )
@@ -61,6 +62,11 @@ func (b *EthAPIBackend) CurrentBlock() *types.Header {
 	return b.eth.blockchain.CurrentBlock()
 }
 
+// Miner returns the miner instance.
+func (b *EthAPIBackend) Miner() *miner.Miner {
+	return b.eth.Miner()
+}
+
 func (b *EthAPIBackend) SetHead(number uint64) {
 	b.eth.handler.downloader.Cancel()
 	b.eth.blockchain.SetHead(number)

eth/backend.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/accounts"
+	grpcapi "github.com/ethereum/go-ethereum/api/grpc"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus"
@@ -121,6 +122,8 @@ type Ethereum struct {
 
 	p2pServer *p2p.Server
 
+	grpcService *grpcapi.Service // Low-latency gRPC API for trading operations
+
 	lock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)
 
 	shutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully
@@ -355,6 +358,13 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	// Start the RPC service
 	eth.netRPCService = ethapi.NewNetAPI(eth.p2pServer, networkID)
 
+	// Initialize gRPC service if enabled
+	if config.EnableGRPC {
+		eth.grpcService = grpcapi.NewService(eth.APIBackend, config.GRPCHost, config.GRPCPort)
+		stack.RegisterLifecycle(eth.grpcService)
+		log.Info("gRPC service initialized", "host", config.GRPCHost, "port", config.GRPCPort)
+	}
+
 	// Register the backend on the node
 	stack.RegisterAPIs(eth.APIs())
 	stack.RegisterProtocols(eth.Protocols())

eth/ethconfig/config.go

@@ -72,6 +72,9 @@ var Defaults = Config{
 	RPCTxFeeCap:          1, // 1 ether
 	TxSyncDefaultTimeout: 20 * time.Second,
 	TxSyncMaxTimeout:     1 * time.Minute,
+	EnableGRPC:           false, // Disabled by default
+	GRPCHost:             "localhost",
+	GRPCPort:             9090,
 }
 
 //go:generate go run github.com/fjl/gencodec -type Config -formats toml -out gen_config.go
@@ -189,6 +192,11 @@ type Config struct {
 	// EIP-7966: eth_sendRawTransactionSync timeouts
 	TxSyncDefaultTimeout time.Duration `toml:",omitempty"`
 	TxSyncMaxTimeout     time.Duration `toml:",omitempty"`
+
+	// gRPC options for low-latency trading operations
+	EnableGRPC bool   // Whether to enable the gRPC server
+	GRPCHost   string // gRPC server host (default: localhost)
+	GRPCPort   int    // gRPC server port (default: 9090)
 }
 
 // CreateConsensusEngine creates a consensus engine for the given chain config.

roadmap.md

@@ -0,0 +1,111 @@
+# Mandarin Roadmap
+
+## Objective
+Transform Mandarin into a low-latency execution client optimized for co-located trading operations, targeting microsecond-scale improvements over standard Geth.
+
+## Differentiation
+- **Erigon**: Database efficiency, archive node performance, disk I/O optimization
+- **Mandarin**: Real-time latency optimization for live trading and MEV operations
+
+## Phase 1: Foundation (2-4 weeks)
+
+### Custom Transaction Ordering
+- Modify `miner/worker.go` to accept external ordering functions
+- Add bundle support with revert protection
+- Export `OrderingStrategy` interface for pluggable algorithms
+- Risk: Low, isolated to block building
+
+### Binary RPC Layer
+- Implement gRPC service in `api/grpc/`
+- Core endpoints: `GetStorageBatch`, `SimulateBundles`, `SubmitBundle`, `GetPendingTransactions`
+- Target: 10x latency improvement over JSON-RPC
+- Risk: Low, runs alongside existing RPC
+
+### Benchmarking Framework
+- Establish baseline metrics vs vanilla Geth
+- Automated latency tracking: tx propagation, block import, simulation throughput, API latency
+- Continuous integration testing against mainnet replays
+
+## Phase 2: Mempool Fast Path (4-6 weeks)
+
+### Lock-Free Transaction Feed
+- Implement ring buffer in `core/txpool/fastfeed/`
+- Hook into txpool at `insertFeed.Send()` points
+- Binary layout for zero-copy access
+- Target: Sub-100μs tx propagation to consumers
+- Risk: Medium, touches critical path
+
+### Shared Memory Interface
+- Expose transaction events via mmap'd regions
+- Selective filtering by contract address or method selector
+- Support multiple concurrent consumers
+- Risk: Medium, IPC complexity
+
+## Phase 3: Hot State Cache (6-8 weeks)
+
+### DeFi Contract State Pinning
+- Maintain in-memory cache of top 100 pool/vault states
+- Pre-decode reserves, fees, and critical parameters
+- Update atomically on block import
+- Target: Sub-microsecond state access for hot contracts
+- Risk: High, state consistency is critical
+
+### State Delta Export
+- After each block, export structured diffs of watched addresses
+- Binary format for rapid consumption by trading strategies
+- Include pre and post-execution state
+- Risk: Medium
+
+### Integration Points
+- `core/blockchain.go`: Hook into `ProcessBlock()`
+- `core/state_processor.go`: Intercept state changes during EVM execution
+- Deploy in shadow mode initially to verify correctness
+
+## Phase 4: Native API (4-6 weeks)
+
+### Shared Library Interface
+- Build Mandarin as `libmandarin.so` with C ABI
+- Export simulation, state access, and bundle submission functions
+- Enable in-process trading strategies
+- Target: Sub-100μs API calls
+- Risk: Medium, ABI stability and versioning concerns
+
+### Safety Mechanisms
+- Process isolation boundaries
+- Memory protection
+- API rate limiting and resource quotas
+
+## Key Metrics
+
+### Target Improvements vs Baseline Geth
+- Transaction propagation (P99): 1.2ms → 45μs
+- Block import (avg): 120ms → 68ms
+- Simulation throughput: 200/sec → 15,000/sec
+- API latency (P50): 8ms → 12μs
+
+### Monitoring
+- Real-time latency percentiles (P50, P99, P999)
+- Memory overhead tracking
+- State consistency validation
+- Regression detection in CI
+
+## Out of Scope
+
+### Not Prioritized
+- Full kernel-bypass networking (DPDK/AF_XDP): Complex, limited benefit
+- Custom P2P protocol: Breaks compatibility
+- Core EVM rewrites: High maintenance burden, modest gains
+
+## Initial Validation
+
+### Quick Wins (First 2 Weeks)
+1. Bundle simulation endpoint using existing `eth_call` infrastructure
+2. gRPC implementation for 5 critical APIs
+3. Benchmark suite comparing to vanilla Geth
+4. Publish latency improvements to validate approach
+
+### Success Criteria
+- Measurable 5-10x latency improvements in Phase 1
+- Zero state consistency issues in hot cache
+- Production stability matching Geth reliability standards
+