TUN-5138: Switch to QUIC on auto protocol based on threshold

Author: sudarshan-reddy

cmd/cloudflared/tunnel/configuration.go

@@ -238,7 +238,7 @@ func prepareTunnelConfig(
 		log.Info().Msgf("Warp-routing is enabled")
 	}
 
-	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), warpRoutingEnabled, namedTunnel, edgediscovery.HTTP2Percentage, origin.ResolveTTL, log)
+	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), warpRoutingEnabled, namedTunnel, edgediscovery.ProtocolPercentage, origin.ResolveTTL, log)
 	if err != nil {
 		return nil, ingress.Ingress{}, err
 	}

connection/protocol.go

@@ -7,6 +7,8 @@ import (
 	"time"
 
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/edgediscovery"
 )
 
 const (
@@ -24,7 +26,7 @@ const (
 
 var (
 	// ProtocolList represents a list of supported protocols for communication with the edge.
-	ProtocolList = []Protocol{H2mux, HTTP2, QUIC}
+	ProtocolList = []Protocol{H2mux, HTTP2, HTTP2Warp, QUIC, QUICWarp}
 )
 
 type Protocol int64
@@ -36,6 +38,12 @@ const (
 	HTTP2
 	// QUIC is used only with named tunnels.
 	QUIC
+	// HTTP2Warp is used only with named tunnels. It's useful for warp-routing where we don't want to fallback to
+	// H2mux on HTTP2 failure to connect.
+	HTTP2Warp
+	//QUICWarp is used only with named tunnels. It's useful for warp-routing where we want to fallback to HTTP2 but
+	// dont' want HTTP2 to fallback to H2mux
+	QUICWarp
 )
 
 // Fallback returns the fallback protocol and whether the protocol has a fallback
@@ -45,8 +53,12 @@ func (p Protocol) fallback() (Protocol, bool) {
 		return 0, false
 	case HTTP2:
 		return H2mux, true
+	case HTTP2Warp:
+		return 0, false
 	case QUIC:
 		return HTTP2, true
+	case QUICWarp:
+		return HTTP2Warp, true
 	default:
 		return 0, false
 	}
@@ -56,9 +68,9 @@ func (p Protocol) String() string {
 	switch p {
 	case H2mux:
 		return "h2mux"
-	case HTTP2:
+	case HTTP2, HTTP2Warp:
 		return "http2"
-	case QUIC:
+	case QUIC, QUICWarp:
 		return "quic"
 	default:
 		return fmt.Sprintf("unknown protocol")
@@ -71,11 +83,11 @@ func (p Protocol) TLSSettings() *TLSSettings {
 		return &TLSSettings{
 			ServerName: edgeH2muxTLSServerName,
 		}
-	case HTTP2:
+	case HTTP2, HTTP2Warp:
 		return &TLSSettings{
 			ServerName: edgeH2TLSServerName,
 		}
-	case QUIC:
+	case QUIC, QUICWarp:
 		return &TLSSettings{
 			ServerName: edgeQUICServerName,
 			NextProtos: []string{"argotunnel"},
@@ -108,29 +120,36 @@ func (s *staticProtocolSelector) Fallback() (Protocol, bool) {
 }
 
 type autoProtocolSelector struct {
-	lock           sync.RWMutex
-	current        Protocol
-	switchThrehold int32
-	fetchFunc      PercentageFetcher
-	refreshAfter   time.Time
-	ttl            time.Duration
-	log            *zerolog.Logger
+	lock sync.RWMutex
+
+	current Protocol
+
+	// protocolPool is desired protocols in the order of priority they should be picked in.
+	protocolPool []Protocol
+
+	switchThreshold int32
+	fetchFunc       PercentageFetcher
+	refreshAfter    time.Time
+	ttl             time.Duration
+	log             *zerolog.Logger
 }
 
 func newAutoProtocolSelector(
 	current Protocol,
-	switchThrehold int32,
+	protocolPool []Protocol,
+	switchThreshold int32,
 	fetchFunc PercentageFetcher,
 	ttl time.Duration,
 	log *zerolog.Logger,
 ) *autoProtocolSelector {
 	return &autoProtocolSelector{
-		current:        current,
-		switchThrehold: switchThrehold,
-		fetchFunc:      fetchFunc,
-		refreshAfter:   time.Now().Add(ttl),
-		ttl:            ttl,
-		log:            log,
+		current:         current,
+		protocolPool:    protocolPool,
+		switchThreshold: switchThreshold,
+		fetchFunc:       fetchFunc,
+		refreshAfter:    time.Now().Add(ttl),
+		ttl:             ttl,
+		log:             log,
 	}
 }
 
@@ -141,28 +160,39 @@ func (s *autoProtocolSelector) Current() Protocol {
 		return s.current
 	}
 
-	percentage, err := s.fetchFunc()
+	protocol, err := getProtocol(s.protocolPool, s.fetchFunc, s.switchThreshold)
 	if err != nil {
 		s.log.Err(err).Msg("Failed to refresh protocol")
 		return s.current
 	}
+	s.current = protocol
 
-	if s.switchThrehold < percentage {
-		s.current = HTTP2
-	} else {
-		s.current = H2mux
-	}
 	s.refreshAfter = time.Now().Add(s.ttl)
 	return s.current
 }
 
+func getProtocol(protocolPool []Protocol, fetchFunc PercentageFetcher, switchThreshold int32) (Protocol, error) {
+	protocolPercentages, err := fetchFunc()
+	if err != nil {
+		return 0, err
+	}
+	for _, protocol := range protocolPool {
+		protocolPercentage := protocolPercentages.GetPercentage(protocol.String())
+		if protocolPercentage > switchThreshold {
+			return protocol, nil
+		}
+	}
+
+	return protocolPool[len(protocolPool)-1], nil
+}
+
 func (s *autoProtocolSelector) Fallback() (Protocol, bool) {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
 	return s.current.fallback()
 }
 
-type PercentageFetcher func() (int32, error)
+type PercentageFetcher func() (edgediscovery.ProtocolPercents, error)
 
 func NewProtocolSelector(
 	protocolFlag string,
@@ -179,54 +209,76 @@ func NewProtocolSelector(
 		}, nil
 	}
 
-	// warp routing cannot be served over h2mux connections
-	if warpRoutingEnabled {
-		if protocolFlag == H2mux.String() {
-			log.Warn().Msg("Warp routing is not supported in h2mux protocol. Upgrading to http2 to allow it.")
-		}
-
-		if protocolFlag == QUIC.String() {
-			return &staticProtocolSelector{
-				current: QUIC,
-			}, nil
-		}
+	threshold := switchThreshold(namedTunnel.Credentials.AccountTag)
+	fetchedProtocol, err := getProtocol([]Protocol{QUIC, HTTP2}, fetchFunc, threshold)
+	if err != nil {
+		log.Err(err).Msg("Unable to lookup protocol. Defaulting to `http2`. If this fails, you can set `--protocol h2mux` in your cloudflared command.")
 		return &staticProtocolSelector{
 			current: HTTP2,
 		}, nil
 	}
+	if warpRoutingEnabled {
+		if protocolFlag == H2mux.String() || fetchedProtocol == H2mux {
+			log.Warn().Msg("Warp routing is not supported in h2mux protocol. Upgrading to http2 to allow it.")
+			protocolFlag = HTTP2.String()
+			fetchedProtocol = HTTP2Warp
+		}
+		return selectWarpRoutingProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol)
+	}
+
+	return selectNamedTunnelProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol)
+}
 
+func selectNamedTunnelProtocols(
+	protocolFlag string,
+	fetchFunc PercentageFetcher,
+	ttl time.Duration,
+	log *zerolog.Logger,
+	threshold int32,
+	protocol Protocol,
+) (ProtocolSelector, error) {
 	if protocolFlag == H2mux.String() {
 		return &staticProtocolSelector{
 			current: H2mux,
 		}, nil
 	}
 
 	if protocolFlag == QUIC.String() {
-		return newAutoProtocolSelector(QUIC, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
+		return newAutoProtocolSelector(QUIC, []Protocol{QUIC, HTTP2, H2mux}, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
 	}
 
-	http2Percentage, err := fetchFunc()
-	if err != nil {
-		log.Err(err).Msg("Unable to lookup protocol. Defaulting to `http2`. If this fails, you can set `--protocol h2mux` in your cloudflared command.")
-		return &staticProtocolSelector{
-			current: HTTP2,
-		}, nil
-	}
 	if protocolFlag == HTTP2.String() {
-		if http2Percentage < 0 {
-			return newAutoProtocolSelector(H2mux, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
-		}
-		return newAutoProtocolSelector(HTTP2, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
+		return newAutoProtocolSelector(HTTP2, []Protocol{HTTP2, H2mux}, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
 	}
 
 	if protocolFlag != autoSelectFlag {
 		return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
 	}
-	threshold := switchThreshold(namedTunnel.Credentials.AccountTag)
-	if threshold < http2Percentage {
-		return newAutoProtocolSelector(HTTP2, threshold, fetchFunc, ttl, log), nil
+
+	return newAutoProtocolSelector(protocol, []Protocol{QUIC, HTTP2, H2mux}, threshold, fetchFunc, ttl, log), nil
+}
+
+func selectWarpRoutingProtocols(
+	protocolFlag string,
+	fetchFunc PercentageFetcher,
+	ttl time.Duration,
+	log *zerolog.Logger,
+	threshold int32,
+	protocol Protocol,
+) (ProtocolSelector, error) {
+	if protocolFlag == QUIC.String() {
+		return newAutoProtocolSelector(QUICWarp, []Protocol{QUICWarp, HTTP2Warp}, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
 	}
-	return newAutoProtocolSelector(H2mux, threshold, fetchFunc, ttl, log), nil
+
+	if protocolFlag == HTTP2.String() {
+		return newAutoProtocolSelector(HTTP2Warp, []Protocol{HTTP2Warp}, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
+	}
+
+	if protocolFlag != autoSelectFlag {
+		return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
+	}
+
+	return newAutoProtocolSelector(protocol, []Protocol{QUICWarp, HTTP2Warp}, threshold, fetchFunc, ttl, log), nil
 }
 
 func switchThreshold(accountTag string) int32 {