HTTP/3 change use-http3 config to enable-quic flag to not make braking changes (for deployment where 443 UDP port is not available)

Author: marcel2012

build/dev-env.sh

@@ -89,7 +89,8 @@ controller:
     digest:
   config:
     worker-processes: "1"
-    use-http3: "true"
+  extraArgs:
+    enable-quic: "true"
   podLabels:
     deploy-date: "$(date +%s)"
   updateStrategy:

docs/user-guide/cli-arguments.md

@@ -27,6 +27,7 @@ They are set in the container spec of the `ingress-nginx-controller` Deployment
 | `--enable-metrics`                 | Enables the collection of NGINX metrics. (Default: false) |
 | `--enable-ssl-chain-completion`    | Autocomplete SSL certificate chains with missing intermediate CA certificates. Certificates uploaded to Kubernetes must have the "Authority Information Access" X.509 v3 extension for this to succeed. (default false)|
 | `--enable-ssl-passthrough`         | Enable SSL Passthrough. (default false) |
+| `--enable-quic`                    | Enable QUIC. (default false) |
 | `--disable-leader-election`        | Disable Leader Election on Nginx Controller. (default false) |
 | `--enable-topology-aware-routing`  | Enable topology aware routing feature, needs service object annotation service.kubernetes.io/topology-mode sets to auto. (default false) |
 | `--exclude-socket-metrics`         | Set of socket request metrics to exclude which won't be exported nor being calculated. The possible socket request metrics to exclude are documented in the monitoring guide e.g. 'nginx_ingress_controller_request_duration_seconds,nginx_ingress_controller_response_size'|

docs/user-guide/nginx-configuration/configmap.md

@@ -108,7 +108,6 @@ The following table shows a configuration option's name, type, and the default v
 | [brotli-min-length](#brotli-min-length)                                         | int          | 20                                                                                                                                                                                                                                                                                                                                                           |                                                                                     |
 | [brotli-types](#brotli-types)                                                   | string       | "application/xml+rss application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/javascript text/plain text/x-component" |                                                                                     |
 | [use-http2](#use-http2)                                                         | bool         | "true"                                                                                                                                                                                                                                                                                                                                                       |                                                                                     |
-| [use-http3](#use-http3)                                                         | bool         | "false"                                                                                                                                                                                                                                                                                                                                                      |                                                                                     |
 | [gzip-disable](#gzip-disable)                                                   | string       | ""                                                                                                                                                                                                                                                                                                                                                           |                                                                                     |
 | [gzip-level](#gzip-level)                                                       | int          | 1                                                                                                                                                                                                                                                                                                                                                            |                                                                                     |
 | [gzip-min-length](#gzip-min-length)                                             | int          | 256                                                                                                                                                                                                                                                                                                                                                          |                                                                                     |
@@ -770,10 +769,6 @@ _**default:**_ `application/xml+rss application/atom+xml application/javascript
 
 Enables or disables [HTTP/2](https://nginx.org/en/docs/http/ngx_http_v2_module.html) support in secure connections.
 
-## use-http3
-
-Enables or disables [HTTP/3](https://nginx.org/en/docs/http/ngx_http_v3_module.html) support in secure connections.
-
 ## gzip-disable
 
 Disables [gzipping](http://nginx.org/en/docs/http/ngx_http_gzip_module.html#gzip_disable) of responses for requests with "User-Agent" header fields matching any of the specified regular expressions.

internal/ingress/controller/config/config.go

@@ -458,11 +458,6 @@ type Configuration struct {
 	// Default: true
 	UseHTTP2 bool `json:"use-http2,omitempty"`
 
-	// Enables or disables the HTTP/3 support in secure connections
-	// https://nginx.org/en/docs/http/ngx_http_v3_module.html
-	// Default: false
-	UseHTTP3 bool `json:"use-http3,omitempty"`
-
 	// Disables gzipping of responses for requests with "User-Agent" header fields matching any of
 	// the specified regular expressions.
 	// http://nginx.org/en/docs/http/ngx_http_gzip_module.html#gzip_disable
@@ -862,7 +857,6 @@ func NewDefault() Configuration {
 		VariablesHashBucketSize:          256,
 		VariablesHashMaxSize:             2048,
 		UseHTTP2:                         true,
-		UseHTTP3:                         false,
 		DisableProxyInterceptErrors:      false,
 		RelativeRedirects:                false,
 		ProxyStreamTimeout:               "600s",
@@ -952,6 +946,7 @@ type TemplateConfig struct {
 	Cfg                      Configuration                    `json:"Cfg"`
 	IsIPV6Enabled            bool                             `json:"IsIPV6Enabled"`
 	IsSSLPassthroughEnabled  bool                             `json:"IsSSLPassthroughEnabled"`
+	IsQUICEnabled            bool                             `json:"IsQUICEnabled"`
 	NginxStatusIpv4Whitelist []string                         `json:"NginxStatusIpv4Whitelist"`
 	NginxStatusIpv6Whitelist []string                         `json:"NginxStatusIpv6Whitelist"`
 	RedirectServers          interface{}                      `json:"RedirectServers"`

internal/ingress/controller/controller.go

@@ -101,6 +101,8 @@ type Configuration struct {
 
 	EnableSSLPassthrough bool
 
+	EnableQUIC bool
+
 	DisableLeaderElection bool
 
 	EnableProfiling bool

internal/ingress/controller/nginx.go

@@ -620,6 +620,7 @@ func (n *NGINXController) generateTemplate(cfg ngx_config.Configuration, ingress
 		NginxStatusIpv6Whitelist: cfg.NginxStatusIpv6Whitelist,
 		RedirectServers:          utilingress.BuildRedirects(ingressCfg.Servers),
 		IsSSLPassthroughEnabled:  n.cfg.EnableSSLPassthrough,
+		IsQUICEnabled:            n.cfg.EnableQUIC,
 		ListenPorts:              n.cfg.ListenPorts,
 		EnableMetrics:            n.cfg.EnableMetrics,
 		MaxmindEditionFiles:      n.cfg.MaxmindEditionFiles,

internal/ingress/controller/template/template.go

@@ -1509,17 +1509,13 @@ func httpsListener(addresses []string, co string, tc *config.TemplateConfig) []s
 
 		out = append(out, strings.Join(lo, " "))
 	}
-	if !tc.Cfg.UseHTTP3 {
+	if !tc.IsQUICEnabled {
 		return out
 	}
 	if strings.Contains(co, "backlog=") {
 		klog.V(3).InfoS("Skipping HTTP/3 because of incompatible backlog parameter")
 		return out
 	}
-	if tc.IsSSLPassthroughEnabled {
-		klog.V(3).InfoS("Skipping HTTP/3 in SSL Passthrough mode")
-		return out
-	}
 	for _, address := range addresses {
 		lo := []string{"listen"}
 

pkg/flags/flags.go

@@ -150,6 +150,9 @@ Requires the update-status parameter.`)
 		enableSSLPassthrough = flags.Bool("enable-ssl-passthrough", false,
 			`Enable SSL Passthrough.`)
 
+		enableQUIC = flags.Bool("enable-quic", false,
+			`Enable QUIC.`)
+
 		disableLeaderElection = flags.Bool("disable-leader-election", false,
 			`Disable Leader Election on NGINX Controller.`)
 
@@ -275,10 +278,6 @@ https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geol
 		return false, nil, fmt.Errorf("port %v is already in use. Please check the flag --https-port", *httpsPort)
 	}
 
-	if !ing_net.IsUDPPortAvailable(*quicPort) {
-		return false, nil, fmt.Errorf("port %v is already in use. Please check the flag --quic-port", *quicPort)
-	}
-
 	if !ing_net.IsPortAvailable(*defServerPort) {
 		return false, nil, fmt.Errorf("port %v is already in use. Please check the flag --default-server-port", *defServerPort)
 	}
@@ -304,10 +303,18 @@ https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geol
 		return false, nil, fmt.Errorf("port %v is already in use. Please check the flag --ssl-passthrough-proxy-port", *sslProxyPort)
 	}
 
+	if *enableQUIC && !ing_net.IsUDPPortAvailable(*quicPort) {
+		return false, nil, fmt.Errorf("port %v is already in use. Please check the flag --quic-port", *quicPort)
+	}
+
 	if *publishSvc != "" && *publishStatusAddress != "" {
 		return false, nil, fmt.Errorf("flags --publish-service and --publish-status-address are mutually exclusive")
 	}
 
+	if *enableSSLPassthrough && *enableQUIC {
+		return false, nil, fmt.Errorf("flags --enable-ssl-passthrough and --enable-quic are mutually exclusive")
+	}
+
 	nginx.HealthPath = *defHealthzURL
 
 	if *defHealthCheckTimeout > 0 {
@@ -361,6 +368,7 @@ https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geol
 		MonitorMaxBatchSize:         *monitorMaxBatchSize,
 		DisableServiceExternalName:  *disableServiceExternalName,
 		EnableSSLPassthrough:        *enableSSLPassthrough,
+		EnableQUIC:                  *enableQUIC,
 		DisableLeaderElection:       *disableLeaderElection,
 		ResyncPeriod:                *resyncPeriod,
 		DefaultService:              *defaultSvc,

test/data/config.json

@@ -48,7 +48,6 @@
     "sslSessionTimeout": "10m",
     "useGzip": true,
     "useHttp2": true,
-    "useHttp3": false,
     "proxyStreamTimeout": "600s",
     "workerProcesses": 1,
     "limitConnZoneVariable": "$remote_addr"

test/e2e/settings/http3.go

@@ -17,41 +17,60 @@ limitations under the License.
 package settings
 
 import (
+	"context"
 	"strings"
 
 	"github.com/onsi/ginkgo/v2"
+	"github.com/stretchr/testify/assert"
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"k8s.io/ingress-nginx/test/e2e/framework"
 )
 
 var _ = framework.DescribeSetting("http3", func() {
 	f := framework.NewDefaultFramework("http3")
-
-	ginkgo.It("should disable HTTP/3", func() {
-		host := "http3.com"
+	host := "http3.com"
+
+	ginkgo.BeforeEach(func() {
+		err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error {
+			args := deployment.Spec.Template.Spec.Containers[0].Args
+			args = append(args, "--enable-quic")
+			deployment.Spec.Template.Spec.Containers[0].Args = args
+			_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
+			return err
+		})
+		assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags")
 		annotations := map[string]string{}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations)
 		f.EnsureIngress(ing)
 
-		f.UpdateNginxConfigMapData("use-http3", "false")
-
-		f.WaitForNginxConfiguration(func(cfg string) bool {
-			return !strings.Contains(cfg, "quic;")
-		})
+		f.WaitForNginxServer(host,
+			func(server string) bool {
+				return strings.Contains(server, "listen 443 quic;")
+			})
 	})
 
-	ginkgo.It("should enable HTTP/3", func() {
-		host := "http3.com"
-		annotations := map[string]string{}
-
-		ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations)
-		f.EnsureIngress(ing)
+	ginkgo.It("should enable HTTP/3 on a custom port", func() {
+		err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error {
+			args := deployment.Spec.Template.Spec.Containers[0].Args
+			args = append(args, "--quic-port=4321")
+			deployment.Spec.Template.Spec.Containers[0].Args = args
+			_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
+			return err
+		})
+		assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags")
 
-		f.UpdateNginxConfigMapData("use-http3", "true")
+		f.WaitForNginxServer(host,
+			func(server string) bool {
+				return strings.Contains(server, "listen 4321 quic;")
+			})
+	})
 
+	ginkgo.It("should have default http3_max_concurrent_streams value", func() {
 		f.WaitForNginxConfiguration(func(cfg string) bool {
-			return strings.Contains(cfg, "quic;")
+			return strings.Contains(cfg, "http3_max_concurrent_streams 128;")
 		})
 	})
 
@@ -63,6 +82,12 @@ var _ = framework.DescribeSetting("http3", func() {
 		})
 	})
 
+	ginkgo.It("should have default http3_stream_buffer_size value", func() {
+		f.WaitForNginxConfiguration(func(cfg string) bool {
+			return strings.Contains(cfg, "http3_stream_buffer_size 64k;")
+		})
+	})
+
 	ginkgo.It("should set http3_stream_buffer_size value", func() {
 		f.UpdateNginxConfigMapData("http3-stream-buffer-size", "128k")