feat: Support configuring maxConnectionAge in xDSServer (#7374)

Fixes: #7007

Signed-off-by: Arko Dasgupta <[email protected]>

Author: arkodg

api/v1alpha1/envoygateway_helpers.go

@@ -27,6 +27,7 @@ func DefaultEnvoyGateway() *EnvoyGateway {
 			Logging:   DefaultEnvoyGatewayLogging(),
 			Admin:     DefaultEnvoyGatewayAdmin(),
 			Telemetry: DefaultEnvoyGatewayTelemetry(),
+			XDSServer: DefaultXDSServer(),
 		},
 	}
 }
@@ -67,6 +68,9 @@ func (e *EnvoyGateway) SetEnvoyGatewayDefaults() {
 	if e.Telemetry == nil {
 		e.Telemetry = DefaultEnvoyGatewayTelemetry()
 	}
+	if e.XDSServer == nil {
+		e.XDSServer = DefaultXDSServer()
+	}
 }
 
 // GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified.
@@ -172,6 +176,11 @@ func DefaultGateway() *Gateway {
 	}
 }
 
+// DefaultXDSServer returns a new XDSServer with default configuration parameters.
+func DefaultXDSServer() *XDSServer {
+	return &XDSServer{}
+}
+
 // DefaultEnvoyGatewayLogging returns a new EnvoyGatewayLogging with default configuration parameters.
 func DefaultEnvoyGatewayLogging() *EnvoyGatewayLogging {
 	return &EnvoyGatewayLogging{

api/v1alpha1/envoygateway_types.go

@@ -74,6 +74,12 @@ type EnvoyGatewaySpec struct {
 	// +optional
 	Telemetry *EnvoyGatewayTelemetry `json:"telemetry,omitempty"`
 
+	// XDSServer defines the configuration for the Envoy Gateway xDS gRPC server.
+	// If unspecified, default connection keepalive settings will be used.
+	//
+	// +optional
+	XDSServer *XDSServer `json:"xdsServer,omitempty"`
+
 	// RateLimit defines the configuration associated with the Rate Limit service
 	// deployed by Envoy Gateway required to implement the Global Rate limiting
 	// functionality. The specific rate limit service used here is the reference
@@ -140,6 +146,21 @@ type KubernetesClientRateLimit struct {
 	Burst *int32 `json:"burst,omitempty"`
 }
 
+// XDSServer defines configuration values for the xDS gRPC server.
+type XDSServer struct {
+	// MaxConnectionAge is the maximum age of an active connection before Envoy Gateway will initiate a graceful close.
+	// If unspecified, Envoy Gateway randomly selects a value between 10h and 12h to stagger reconnects across replicas.
+	//
+	// +optional
+	MaxConnectionAge *gwapiv1.Duration `json:"maxConnectionAge,omitempty"`
+
+	// MaxConnectionAgeGrace is the grace period granted after reaching MaxConnectionAge before the connection is forcibly closed.
+	// The default grace period is 2m.
+	//
+	// +optional
+	MaxConnectionAgeGrace *gwapiv1.Duration `json:"maxConnectionAgeGrace,omitempty"`
+}
+
 // LeaderElection defines the desired leader election settings.
 type LeaderElection struct {
 	// LeaseDuration defines the time non-leader contenders will wait before attempting to claim leadership.

api/v1alpha1/validation/envoygateway_validate.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"net/url"
 	"strings"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 
@@ -62,6 +63,10 @@ func ValidateEnvoyGateway(eg *egv1a1.EnvoyGateway) error {
 		return err
 	}
 
+	if err := validateEnvoyGatewayXDSServer(eg.XDSServer); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -225,6 +230,34 @@ func validateEnvoyGatewayExtensionManager(extensionManager *egv1a1.ExtensionMana
 	return nil
 }
 
+func validateEnvoyGatewayXDSServer(xdsServer *egv1a1.XDSServer) error {
+	if xdsServer == nil {
+		return nil
+	}
+
+	if xdsServer.MaxConnectionAge != nil {
+		d, err := time.ParseDuration(string(*xdsServer.MaxConnectionAge))
+		if err != nil {
+			return fmt.Errorf("invalid xdsServer.maxConnectionAge: %w", err)
+		}
+		if d <= 0 {
+			return fmt.Errorf("xdsServer.maxConnectionAge must be greater than zero")
+		}
+	}
+
+	if xdsServer.MaxConnectionAgeGrace != nil {
+		d, err := time.ParseDuration(string(*xdsServer.MaxConnectionAgeGrace))
+		if err != nil {
+			return fmt.Errorf("invalid xdsServer.maxConnectionAgeGrace: %w", err)
+		}
+		if d <= 0 {
+			return fmt.Errorf("xdsServer.maxConnectionAgeGrace must be greater than zero")
+		}
+	}
+
+	return nil
+}
+
 func validateEnvoyGatewayTelemetry(telemetry *egv1a1.EnvoyGatewayTelemetry) error {
 	if telemetry == nil {
 		return nil

api/v1alpha1/validation/envoygateway_validate_test.go

@@ -903,6 +903,31 @@ func TestEnvoyGateway(t *testing.T) {
 	assert.Equal(t, egv1a1.LogLevelInfo, gatewayLogging.Level[egv1a1.LogComponentGatewayDefault])
 }
 
+func TestValidateEnvoyGatewayXDSServer(t *testing.T) {
+	t.Run("valid no overrides", func(t *testing.T) {
+		require.NoError(t, validateEnvoyGatewayXDSServer(nil))
+	})
+
+	t.Run("valid overrides", func(t *testing.T) {
+		age := gwapiv1.Duration("30m")
+		grace := gwapiv1.Duration("1m")
+		x := &egv1a1.XDSServer{MaxConnectionAge: &age, MaxConnectionAgeGrace: &grace}
+		require.NoError(t, validateEnvoyGatewayXDSServer(x))
+	})
+
+	t.Run("invalid duration", func(t *testing.T) {
+		age := gwapiv1.Duration("bad")
+		x := &egv1a1.XDSServer{MaxConnectionAge: &age}
+		require.Error(t, validateEnvoyGatewayXDSServer(x))
+	})
+
+	t.Run("non positive", func(t *testing.T) {
+		age := gwapiv1.Duration("0s")
+		x := &egv1a1.XDSServer{MaxConnectionAgeGrace: &age}
+		require.Error(t, validateEnvoyGatewayXDSServer(x))
+	})
+}
+
 func TestDefaultEnvoyGatewayLoggingLevel(t *testing.T) {
 	type args struct {
 		component string

api/v1alpha1/zz_generated.deepcopy.go

@@ -96,11 +96,41 @@ func (r *Runner) Name() string {
 	return string(egv1a1.LogComponentXdsRunner)
 }
 
-func defaultServerKeepaliveParams() keepalive.ServerParameters {
-	return keepalive.ServerParameters{
+func (r *Runner) serverKeepaliveParams() (keepalive.ServerParameters, error) {
+	params := keepalive.ServerParameters{
 		MaxConnectionAge:      getRandomMaxConnectionAge(),
 		MaxConnectionAgeGrace: defaultMaxConnectionAgeGrace,
 	}
+
+	if r.EnvoyGateway == nil || r.EnvoyGateway.XDSServer == nil {
+		return params, nil
+	}
+
+	cfg := r.EnvoyGateway.XDSServer
+
+	if cfg.MaxConnectionAge != nil {
+		d, err := time.ParseDuration(string(*cfg.MaxConnectionAge))
+		if err != nil {
+			return keepalive.ServerParameters{}, fmt.Errorf("invalid xdsServer.maxConnectionAge: %w", err)
+		}
+		if d <= 0 {
+			return keepalive.ServerParameters{}, fmt.Errorf("xdsServer.maxConnectionAge must be greater than zero")
+		}
+		params.MaxConnectionAge = d
+	}
+
+	if cfg.MaxConnectionAgeGrace != nil {
+		d, err := time.ParseDuration(string(*cfg.MaxConnectionAgeGrace))
+		if err != nil {
+			return keepalive.ServerParameters{}, fmt.Errorf("invalid xdsServer.maxConnectionAgeGrace: %w", err)
+		}
+		if d <= 0 {
+			return keepalive.ServerParameters{}, fmt.Errorf("xdsServer.maxConnectionAgeGrace must be greater than zero")
+		}
+		params.MaxConnectionAgeGrace = d
+	}
+
+	return params, nil
 }
 
 // getRandomMaxConnectionAge picks a random maxConnectionAge value
@@ -127,8 +157,11 @@ func (r *Runner) Start(ctx context.Context) (err error) {
 	}
 	r.Logger.Info("loaded TLS certificate and key")
 
-	keepaliveParams := defaultServerKeepaliveParams()
-	r.Logger.Info("configured gRPC keepalive defaults", "maxConnectionAge", keepaliveParams.MaxConnectionAge, "maxConnectionAgeGrace", keepaliveParams.MaxConnectionAgeGrace)
+	keepaliveParams, err := r.serverKeepaliveParams()
+	if err != nil {
+		return err
+	}
+	r.Logger.Info("configured gRPC keepalive", "maxConnectionAge", keepaliveParams.MaxConnectionAge, "maxConnectionAgeGrace", keepaliveParams.MaxConnectionAgeGrace)
 
 	enforcementPolicy := keepalive.EnforcementPolicy{
 		MinTime:             15 * time.Second,

internal/xds/runner/runner.go

@@ -1279,6 +1279,7 @@ EnvoyGateway is the schema for the envoygateways API.
 | `logging` | _[EnvoyGatewayLogging](#envoygatewaylogging)_ |  false  | \{ default:info \} | Logging defines logging parameters for Envoy Gateway. |
 | `admin` | _[EnvoyGatewayAdmin](#envoygatewayadmin)_ |  false  |  | Admin defines the desired admin related abilities.<br />If unspecified, the Admin is used with default configuration<br />parameters. |
 | `telemetry` | _[EnvoyGatewayTelemetry](#envoygatewaytelemetry)_ |  false  |  | Telemetry defines the desired control plane telemetry related abilities.<br />If unspecified, the telemetry is used with default configuration. |
+| `xdsServer` | _[XDSServer](#xdsserver)_ |  false  |  | XDSServer defines the configuration for the Envoy Gateway xDS gRPC server.<br />If unspecified, default connection keepalive settings will be used. |
 | `rateLimit` | _[RateLimit](#ratelimit)_ |  false  |  | RateLimit defines the configuration associated with the Rate Limit service<br />deployed by Envoy Gateway required to implement the Global Rate limiting<br />functionality. The specific rate limit service used here is the reference<br />implementation in Envoy. For more details visit https://github.com/envoyproxy/ratelimit.<br />This configuration is unneeded for "Local" rate limiting. |
 | `extensionManager` | _[ExtensionManager](#extensionmanager)_ |  false  |  | ExtensionManager defines an extension manager to register for the Envoy Gateway Control Plane. |
 | `extensionApis` | _[ExtensionAPISettings](#extensionapisettings)_ |  false  |  | ExtensionAPIs defines the settings related to specific Gateway API Extensions<br />implemented by Envoy Gateway |
@@ -1547,6 +1548,7 @@ _Appears in:_
 | `logging` | _[EnvoyGatewayLogging](#envoygatewaylogging)_ |  false  | \{ default:info \} | Logging defines logging parameters for Envoy Gateway. |
 | `admin` | _[EnvoyGatewayAdmin](#envoygatewayadmin)_ |  false  |  | Admin defines the desired admin related abilities.<br />If unspecified, the Admin is used with default configuration<br />parameters. |
 | `telemetry` | _[EnvoyGatewayTelemetry](#envoygatewaytelemetry)_ |  false  |  | Telemetry defines the desired control plane telemetry related abilities.<br />If unspecified, the telemetry is used with default configuration. |
+| `xdsServer` | _[XDSServer](#xdsserver)_ |  false  |  | XDSServer defines the configuration for the Envoy Gateway xDS gRPC server.<br />If unspecified, default connection keepalive settings will be used. |
 | `rateLimit` | _[RateLimit](#ratelimit)_ |  false  |  | RateLimit defines the configuration associated with the Rate Limit service<br />deployed by Envoy Gateway required to implement the Global Rate limiting<br />functionality. The specific rate limit service used here is the reference<br />implementation in Envoy. For more details visit https://github.com/envoyproxy/ratelimit.<br />This configuration is unneeded for "Local" rate limiting. |
 | `extensionManager` | _[ExtensionManager](#extensionmanager)_ |  false  |  | ExtensionManager defines an extension manager to register for the Envoy Gateway Control Plane. |
 | `extensionApis` | _[ExtensionAPISettings](#extensionapisettings)_ |  false  |  | ExtensionAPIs defines the settings related to specific Gateway API Extensions<br />implemented by Envoy Gateway |
@@ -5356,6 +5358,22 @@ _Appears in:_
 | `DropHeader` | WithUnderscoresActionDropHeader drops the client header with name containing underscores. The header<br />is dropped before the filter chain is invoked and as such filters will not see<br />dropped headers.<br /> | 
 
 
+#### XDSServer
+
+
+
+XDSServer defines configuration values for the xDS gRPC server.
+
+_Appears in:_
+- [EnvoyGateway](#envoygateway)
+- [EnvoyGatewaySpec](#envoygatewayspec)
+
+| Field | Type | Required | Default | Description |
+| ---   | ---  | ---      | ---     | ---         |
+| `maxConnectionAge` | _[Duration](https://gateway-api.sigs.k8s.io/reference/spec/#duration)_ |  false  |  | MaxConnectionAge is the maximum age of an active connection before Envoy Gateway will initiate a graceful close.<br />If unspecified, Envoy Gateway randomly selects a value between 10h and 12h to stagger reconnects across replicas. |
+| `maxConnectionAgeGrace` | _[Duration](https://gateway-api.sigs.k8s.io/reference/spec/#duration)_ |  false  |  | MaxConnectionAgeGrace is the grace period granted after reaching MaxConnectionAge before the connection is forcibly closed.<br />The default grace period is 2m. |
+
+
 #### XDSTranslatorHook
 
 _Underlying type:_ _string_