Add support for VM HTTP2 handlers (#3294)

Signed-off-by: Joshua Kim <[email protected]>
Co-authored-by: Stephen Buttolph <[email protected]>

Author: joshua-kim

api/grpcclient/client.go

@@ -0,0 +1,67 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package grpcclient
+
+import (
+	"context"
+	"fmt"
+
+	"google.golang.org/grpc"
+
+	"github.com/ava-labs/avalanchego/ids"
+)
+
+// NewChainClient returns grpc.ClientConn that prefixes method calls with the
+// provided chainID prefix
+func NewChainClient(uri string, chainID ids.ID, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
+	dialOpts := []grpc.DialOption{
+		grpc.WithUnaryInterceptor(PrefixChainIDUnaryClientInterceptor(chainID)),
+		grpc.WithStreamInterceptor(PrefixChainIDStreamClientInterceptor(chainID)),
+	}
+
+	dialOpts = append(dialOpts, opts...)
+
+	conn, err := grpc.NewClient(uri, dialOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize chain grpc client: %w", err)
+	}
+
+	return conn, nil
+}
+
+// PrefixChainIDUnaryClientInterceptor prefixes unary grpc calls with the
+// provided chainID prefix
+func PrefixChainIDUnaryClientInterceptor(chainID ids.ID) grpc.UnaryClientInterceptor {
+	return func(
+		ctx context.Context,
+		method string,
+		req any,
+		reply any,
+		cc *grpc.ClientConn,
+		invoker grpc.UnaryInvoker,
+		opts ...grpc.CallOption,
+	) error {
+		return invoker(ctx, prefix(chainID, method), req, reply, cc, opts...)
+	}
+}
+
+// PrefixChainIDStreamClientInterceptor prefixes streaming grpc calls with the
+// provided chainID prefix
+func PrefixChainIDStreamClientInterceptor(chainID ids.ID) grpc.StreamClientInterceptor {
+	return func(
+		ctx context.Context,
+		desc *grpc.StreamDesc,
+		cc *grpc.ClientConn,
+		method string,
+		streamer grpc.Streamer,
+		opts ...grpc.CallOption,
+	) (grpc.ClientStream, error) {
+		return streamer(ctx, desc, cc, prefix(chainID, method), opts...)
+	}
+}
+
+// http/2 :path takes the form of /ChainID/Service/Method
+func prefix(chainID ids.ID, method string) string {
+	return "/" + chainID.String() + method
+}

api/server/http2_router.go

@@ -0,0 +1,67 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package server
+
+import (
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/ava-labs/avalanchego/ids"
+)
+
+var _ http.Handler = (*http2Router)(nil)
+
+type http2Router struct {
+	lock     sync.RWMutex
+	handlers map[string]http.Handler
+}
+
+func newHTTP2Router() *http2Router {
+	return &http2Router{
+		handlers: make(map[string]http.Handler),
+	}
+}
+
+func (h *http2Router) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// The :path pseudo-header takes the form of /Prefix/Path
+	parsed := strings.Split(r.URL.Path, "/")
+	if len(parsed) < 2 {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	chainID := parsed[1]
+
+	h.lock.RLock()
+	handler, ok := h.handlers[chainID]
+	h.lock.RUnlock()
+	if !ok {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
+	// Deep copy the request to avoid weird behavior from modifying r
+	requestDeepCopy := r.Clone(r.Context())
+	// Route this request to the http2 handler using the chain prefix
+	requestDeepCopy.URL.Path = strings.TrimPrefix(
+		requestDeepCopy.URL.Path,
+		"/"+chainID,
+	)
+
+	handler.ServeHTTP(w, requestDeepCopy)
+}
+
+func (h *http2Router) Add(chainID ids.ID, handler http.Handler) bool {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	chainIDStr := chainID.String()
+	if _, ok := h.handlers[chainIDStr]; ok {
+		return false
+	}
+
+	h.handlers[chainIDStr] = handler
+	return true
+}

api/server/http2_router_test.go

@@ -0,0 +1,72 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ava-labs/avalanchego/ids"
+)
+
+func TestHTTP2RouterAdd(t *testing.T) {
+	require := require.New(t)
+	h := newHTTP2Router()
+	handler := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
+
+	require.True(h.Add(ids.Empty, handler))
+	require.False(h.Add(ids.Empty, handler))
+}
+
+func TestHTTP2RouterServeHTTP(t *testing.T) {
+	tests := []struct {
+		name     string
+		chainIDs []ids.ID
+		path     string
+		wantCode int
+	}{
+		{
+			name:     "invalid request",
+			path:     "foo",
+			wantCode: http.StatusBadRequest,
+		},
+		{
+			name:     "invalid handler",
+			path:     "/foo/bar/method",
+			wantCode: http.StatusNotFound,
+		},
+		{
+			name:     "valid handler",
+			chainIDs: []ids.ID{{'f', 'o', 'o'}},
+			path:     fmt.Sprintf("/%s/bar/method", ids.ID{'f', 'o', 'o'}.String()),
+			wantCode: http.StatusOK,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require := require.New(t)
+
+			h := newHTTP2Router()
+			handler := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
+			writer := httptest.NewRecorder()
+			request := httptest.NewRequest(http.MethodPost, "/", nil)
+			request.URL = &url.URL{
+				Path: tt.path,
+			}
+
+			for _, chainID := range tt.chainIDs {
+				require.True(h.Add(chainID, handler))
+			}
+
+			h.ServeHTTP(writer, request)
+			require.Equal(tt.wantCode, writer.Code)
+		})
+	}
+}

api/server/server.go

@@ -17,6 +17,7 @@ import (
 	"github.com/rs/cors"
 	"go.uber.org/zap"
 	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
 
 	"github.com/ava-labs/avalanchego/api"
 	"github.com/ava-labs/avalanchego/ids"
@@ -88,7 +89,8 @@ type server struct {
 	metrics *metrics
 
 	// Maps endpoints to handlers
-	router *router
+	router      *router
+	http2Router *http2Router
 
 	srv *http.Server
 
@@ -115,33 +117,30 @@ func New(
 	}
 
 	router := newRouter()
-	allowedHostsHandler := filterInvalidHosts(router, allowedHosts)
-	corsHandler := cors.New(cors.Options{
-		AllowedOrigins:   allowedOrigins,
-		AllowCredentials: true,
-	}).Handler(allowedHostsHandler)
-	gzipHandler := gziphandler.GzipHandler(corsHandler)
-	var handler http.Handler = http.HandlerFunc(
-		func(w http.ResponseWriter, r *http.Request) {
-			// Attach this node's ID as a header
-			w.Header().Set("node-id", nodeID.String())
-			gzipHandler.ServeHTTP(w, r)
-		},
-	)
+	handler := wrapHandler(router, nodeID, allowedOrigins, allowedHosts, true)
+
+	http2Router := newHTTP2Router()
+	// Do not use gzip middleware because it breaks the grpc spec
+	http2Handler := wrapHandler(http2Router, nodeID, allowedOrigins, allowedHosts, false)
 
 	httpServer := &http.Server{
-		Handler:           handler,
+		Handler: h2c.NewHandler(
+			http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.ProtoMajor == 2 {
+					http2Handler.ServeHTTP(w, r)
+					return
+				}
+
+				handler.ServeHTTP(w, r)
+			}),
+			&http2.Server{
+				MaxConcurrentStreams: maxConcurrentStreams,
+			}),
 		ReadTimeout:       httpConfig.ReadTimeout,
 		ReadHeaderTimeout: httpConfig.ReadHeaderTimeout,
 		WriteTimeout:      httpConfig.WriteTimeout,
 		IdleTimeout:       httpConfig.IdleTimeout,
 	}
-	err = http2.ConfigureServer(httpServer, &http2.Server{
-		MaxConcurrentStreams: maxConcurrentStreams,
-	})
-	if err != nil {
-		return nil, err
-	}
 
 	log.Info("API created",
 		zap.Strings("allowedOrigins", allowedOrigins),
@@ -154,6 +153,7 @@ func New(
 		tracer:          tracer,
 		metrics:         m,
 		router:          router,
+		http2Router:     http2Router,
 		srv:             httpServer,
 		listener:        listener,
 	}, nil
@@ -199,6 +199,30 @@ func (s *server) RegisterChain(chainName string, ctx *snow.ConsensusContext, vm
 			)
 		}
 	}
+
+	ctx.Lock.Lock()
+	http2Handler, err := vm.CreateHTTP2Handler(context.TODO())
+	ctx.Lock.Unlock()
+	if err != nil {
+		s.log.Error("failed to create http2 handler",
+			zap.String("chainName", chainName),
+			zap.Error(err),
+		)
+		return
+	}
+
+	if http2Handler == nil {
+		return
+	}
+
+	http2Handler = s.wrapMiddleware(chainName, http2Handler, ctx)
+	if !s.http2Router.Add(ctx.ChainID, http2Handler) {
+		s.log.Error(
+			"failed to add route to http2 handler",
+			zap.String("chainName", chainName),
+			zap.Error(err),
+		)
+	}
 }
 
 func (s *server) addChainRoute(chainName string, handler http.Handler, ctx *snow.ConsensusContext, base, endpoint string) error {
@@ -207,13 +231,17 @@ func (s *server) addChainRoute(chainName string, handler http.Handler, ctx *snow
 		zap.String("url", url),
 		zap.String("endpoint", endpoint),
 	)
+	handler = s.wrapMiddleware(chainName, handler, ctx)
+	return s.router.AddRouter(url, endpoint, handler)
+}
+
+func (s *server) wrapMiddleware(chainName string, handler http.Handler, ctx *snow.ConsensusContext) http.Handler {
 	if s.tracingEnabled {
 		handler = api.TraceHandler(handler, chainName, s.tracer)
 	}
 	// Apply middleware to reject calls to the handler before the chain finishes bootstrapping
 	handler = rejectMiddleware(handler, ctx)
-	handler = s.metrics.wrapHandler(chainName, handler)
-	return s.router.AddRouter(url, endpoint, handler)
+	return s.metrics.wrapHandler(chainName, handler)
 }
 
 func (s *server) AddRoute(handler http.Handler, base, endpoint string) error {
@@ -299,3 +327,27 @@ func (a readPathAdder) AddRoute(handler http.Handler, base, endpoint string) err
 func (a readPathAdder) AddAliases(endpoint string, aliases ...string) error {
 	return a.pather.AddAliasesWithReadLock(endpoint, aliases...)
 }
+
+func wrapHandler(
+	handler http.Handler,
+	nodeID ids.NodeID,
+	allowedOrigins []string,
+	allowedHosts []string,
+	gzip bool,
+) http.Handler {
+	h := filterInvalidHosts(handler, allowedHosts)
+	h = cors.New(cors.Options{
+		AllowedOrigins:   allowedOrigins,
+		AllowCredentials: true,
+	}).Handler(h)
+	if gzip {
+		h = gziphandler.GzipHandler(h)
+	}
+	return http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			// Attach this node's ID as a header
+			w.Header().Set("node-id", nodeID.String())
+			h.ServeHTTP(w, r)
+		},
+	)
+}

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/StephenButtolph/canoto v0.15.0
 	github.com/antithesishq/antithesis-sdk-go v0.3.8
-	github.com/ava-labs/coreth v0.15.1-rc.0.0.20250530184801-28421010abae
+	github.com/ava-labs/coreth v0.15.2-rc.0
 	github.com/ava-labs/ledger-avalanche/go v0.0.0-20241009183145-e6f90a8a1a60
 	github.com/ava-labs/libevm v1.13.14-0.2.0.release
 	github.com/btcsuite/btcd/btcutil v1.1.3

go.sum

@@ -68,8 +68,8 @@ github.com/antithesishq/antithesis-sdk-go v0.3.8/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
-github.com/ava-labs/coreth v0.15.1-rc.0.0.20250530184801-28421010abae h1:PPcOEtY3SoXruzMPYJMxoeoExRVhf5UOiH7SvdPHQwY=
-github.com/ava-labs/coreth v0.15.1-rc.0.0.20250530184801-28421010abae/go.mod h1:Tl6TfpOwSq0bXvN7DTENXB4As3hml8ma9OQEWS9DH9I=
+github.com/ava-labs/coreth v0.15.2-rc.0 h1:RnCH59A/WMEdMyuqugaPIk1xrQQuo9vEgMVOjk9ZjWQ=
+github.com/ava-labs/coreth v0.15.2-rc.0/go.mod h1:Tl6TfpOwSq0bXvN7DTENXB4As3hml8ma9OQEWS9DH9I=
 github.com/ava-labs/ledger-avalanche/go v0.0.0-20241009183145-e6f90a8a1a60 h1:EL66gtXOAwR/4KYBjOV03LTWgkEXvLePribLlJNu4g0=
 github.com/ava-labs/ledger-avalanche/go v0.0.0-20241009183145-e6f90a8a1a60/go.mod h1:/7qKobTfbzBu7eSTVaXMTr56yTYk4j2Px6/8G+idxHo=
 github.com/ava-labs/libevm v1.13.14-0.2.0.release h1:uKGCc5/ceeBbfAPRVtBUxbQt50WzB2pEDb8Uy93ePgQ=